luxferre/tii
1
0
Fork 0
Code Actions Activity

implemented compression for HTTP-requested data

Browse Source
This commit is contained in:
Luxferre
2024-11-23 08:11:48 +02:00
parent badc92ea99
commit 203f6a30ca
2 changed files with 93 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tiid.tcl
+73 -42
View File
@@ -25,6 +25,7 @@ set nodename "tiid"
proc createdb {fname} { proc createdb {fname} {
sqlite3 fdb $fname sqlite3 fdb $fname
fdb eval { fdb eval {
PRAGMA journal_mode=WAL;
CREATE TABLE `msg` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, CREATE TABLE `msg` (`id` INTEGER PRIMARY KEY AUTOINCREMENT,
`msgid` VARCHAR(20) NOT NULL UNIQUE, `msgid` VARCHAR(20) NOT NULL UNIQUE,
`timestamp` INT NOT NULL, `timestamp` INT NOT NULL,
@@ -256,12 +257,25 @@ proc indexpage {} {
# TCP logic here # TCP logic here
# compression function
proc repcompress {compfunc data} {
if {$compfunc eq {gzip}} {
return [zlib gzip $data -level 9]
} else {
return [zlib $compfunc $data 9]
}
}
# error report/reply # error report/reply
proc reperror {sock ishttp errmsg} { proc reperror {sock ishttp errmsg compfunc} {
set errmsg "error: $errmsg\n" set errmsg "error: $errmsg\n"
if {$ishttp eq 1} { if {$ishttp eq 1} {
set msglen [string length $errmsg] set msglen [string length $errmsg]
set hdrs "Content-Type: text/plain;charset=utf-8\r\nContent-Length: $msglen\r\nConnection: close\r\n" set hdrs "Content-Type: text/plain;charset=utf-8\r\nContent-Length: $msglen\r\nConnection: close\r\n"
if {$compfunc ne {none}} {
set hdrs [string cat $hdrs "Content-Encoding: $compfunc\r\n"]
set errmsg [repcompress $compfunc $errmsg]
}
puts -nonewline $sock "HTTP/1.0 400 Bad Request\r\n$hdrs\r\n$errmsg" puts -nonewline $sock "HTTP/1.0 400 Bad Request\r\n$hdrs\r\n$errmsg"
} else { } else {
puts -nonewline $sock "$errmsg" puts -nonewline $sock "$errmsg"
@@ -270,10 +284,14 @@ proc reperror {sock ishttp errmsg} {
} }
# successful reply with data # successful reply with data
proc repdata {sock ishttp data} { proc repdata {sock ishttp data compfunc} {
if {$ishttp eq 1} { if {$ishttp eq 1} {
set msglen [string length $data] set msglen [string length $data]
set hdrs "Content-Type: text/plain;charset=utf-8\r\nContent-Length: $msglen\r\nConnection: close\r\n" set hdrs "Content-Type: text/plain;charset=utf-8\r\nContent-Length: $msglen\r\nConnection: close\r\n"
if {$compfunc ne {none}} {
set hdrs [string cat $hdrs "Content-Encoding: $compfunc\r\n"]
set data [repcompress $compfunc $data]
}
puts -nonewline $sock "HTTP/1.0 200 OK\r\n$hdrs\r\n$data" puts -nonewline $sock "HTTP/1.0 200 OK\r\n$hdrs\r\n$data"
} else { } else {
puts -nonewline $sock $data puts -nonewline $sock $data
@@ -284,7 +302,7 @@ proc repdata {sock ishttp data} {
# path router # path router
# it only must write to the socket, not read from it or close it # it only must write to the socket, not read from it or close it
# supported paths: /e, /m, /u/e, /u/m, /u/point, /list.txt, /blacklist.txt # supported paths: /e, /m, /u/e, /u/m, /u/point, /list.txt, /blacklist.txt
proc routepath {dbfile sock ishttp path body} { proc routepath {dbfile sock ishttp path body compfunc} {
fconfigure $sock -translation binary fconfigure $sock -translation binary
set pathparts [split [string trim $path] /] set pathparts [split [string trim $path] /]
if {[llength $pathparts] > 1} { if {[llength $pathparts] > 1} {
@@ -309,12 +327,12 @@ proc routepath {dbfile sock ishttp path body} {
[validecho $ename] ? $ename : [continue] [validecho $ename] ? $ename : [continue]
}}] }}]
if {[llength $erange] > 0} { # recheck length after validation if {[llength $erange] > 0} { # recheck length after validation
repdata $sock $ishttp [indexechos $dbfile $erange 1 $offset $limit] repdata $sock $ishttp [indexechos $dbfile $erange 1 $offset $limit] $compfunc
} else { } else {
reperror $sock $ishttp "invalid request" reperror $sock $ishttp "invalid request" $compfunc
} }
} else { } else {
reperror $sock $ishttp "invalid request" reperror $sock $ishttp "invalid request" $compfunc
} }
} }
m { # validate and shape the message ID list m { # validate and shape the message ID list
@@ -322,9 +340,9 @@ proc routepath {dbfile sock ishttp path body} {
[validmsgid $mid] ? $mid : [continue] [validmsgid $mid] ? $mid : [continue]
}}] }}]
if {[llength $mrange] > 0} { # we have some valid messages if {[llength $mrange] > 0} { # we have some valid messages
repdata $sock $ishttp [multimsg $dbfile $mrange] repdata $sock $ishttp [multimsg $dbfile $mrange] $compfunc
} else { } else {
reperror $sock $ishttp "invalid request" reperror $sock $ishttp "invalid request" $compfunc
} }
} }
point { point {
@@ -349,12 +367,12 @@ proc routepath {dbfile sock ishttp path body} {
if {$authstr ne "" && $msgbody ne ""} { if {$authstr ne "" && $msgbody ne ""} {
set postres [postmsg $dbfile $authstr $msgbody] set postres [postmsg $dbfile $authstr $msgbody]
if [string match "msg ok*" $postres] { if [string match "msg ok*" $postres] {
repdata $sock $ishttp $postres repdata $sock $ishttp $postres $compfunc
} else { } else {
reperror $sock $ishttp $postres reperror $sock $ishttp $postres $compfunc
} }
} else { } else {
reperror $sock $ishttp "invalid request" reperror $sock $ishttp "invalid request" $compfunc
} }
} }
push { push {
@@ -369,68 +387,68 @@ proc routepath {dbfile sock ishttp path body} {
set msgbody [decurl [dict get $params upush]] set msgbody [decurl [dict get $params upush]]
} }
} else { } else {
reperror $sock $ishttp "/u/push is only available over HTTP POST" reperror $sock $ishttp "/u/push is only available over HTTP POST" $compfunc
return return
} }
if {$authstr ne "" && $msgbody ne ""} { if {$authstr ne "" && $msgbody ne ""} {
set postres [postbundle $dbfile $authstr $msgbody] set postres [postbundle $dbfile $authstr $msgbody]
if [string match "message saved: ok*" $postres] { if [string match "message saved: ok*" $postres] {
repdata $sock $ishttp $postres repdata $sock $ishttp $postres $compfunc
} else { } else {
reperror $sock $ishttp $postres reperror $sock $ishttp $postres $compfunc
} }
} else { } else {
reperror $sock $ishttp "invalid request" reperror $sock $ishttp "invalid request" $compfunc
} }
} }
default { default {
reperror $sock $ishttp "invalid request" reperror $sock $ishttp "invalid request" $compfunc
} }
} }
} else { } else {
reperror $sock $ishttp "invalid request" reperror $sock $ishttp "invalid request" $compfunc
} }
} }
e { e {
if {[llength $pathparts] > 2} { if {[llength $pathparts] > 2} {
set echoname [string trim [lindex $pathparts 2]] set echoname [string trim [lindex $pathparts 2]]
if {[validecho $echoname]} { if {[validecho $echoname]} {
repdata $sock $ishttp [indexechos $dbfile [list $echoname] 0 0 0] repdata $sock $ishttp [indexechos $dbfile [list $echoname] 0 0 0] $compfunc
} else { } else {
reperror $sock $ishttp "invalid request" reperror $sock $ishttp "invalid request" $compfunc
} }
} else { } else {
reperror $sock $ishttp "invalid request" reperror $sock $ishttp "invalid request" $compfunc
} }
} }
m { m {
if {[llength $pathparts] > 2} { if {[llength $pathparts] > 2} {
set mid [string trim [lindex $pathparts 2]] set mid [string trim [lindex $pathparts 2]]
if {[validmsgid $mid]} { if {[validmsgid $mid]} {
repdata $sock $ishttp [singlemsg $dbfile $mid] repdata $sock $ishttp [singlemsg $dbfile $mid] $compfunc
} else { } else {
reperror $sock $ishttp "invalid request" reperror $sock $ishttp "invalid request" $compfunc
} }
} else { } else {
reperror $sock $ishttp "invalid request" reperror $sock $ishttp "invalid request" $compfunc
} }
} }
list.txt { list.txt {
repdata $sock $ishttp [listechos $dbfile] repdata $sock $ishttp [listechos $dbfile] $compfunc
} }
blacklist.txt { blacklist.txt {
repdata $sock $ishttp [blacklisted $dbfile] repdata $sock $ishttp [blacklisted $dbfile] $compfunc
} }
{} { {} {
repdata $sock $ishttp [indexpage] repdata $sock $ishttp [indexpage] $compfunc
} }
default { default {
reperror $sock $ishttp "invalid request" reperror $sock $ishttp "invalid request" $compfunc
} }
} }
} else { } else {
reperror $sock $ishttp "invalid request" reperror $sock $ishttp "invalid request" $compfunc
} }
} }
@@ -440,27 +458,20 @@ proc reqhandler {dbfile sock} {
# ignore all invalid requests by closing the connection # ignore all invalid requests by closing the connection
if {[gets $sock line] >= 0} { if {[gets $sock line] >= 0} {
if {[string match /* $line]} { # bare TCP request (Gopher/Nex) if {[string match /* $line]} { # bare TCP request (Gopher/Nex)
routepath $dbfile $sock 0 [string trim $line] {} routepath $dbfile $sock 0 [string trim $line] {} none
} elseif {[string match -nocase {GET /*} $line]} { # HTTP GET request } else { # HTTP request, read headers
set rparts [split [string trimleft $line]]
if {[llength $rparts] > 1} { # valid GET request
routepath $dbfile $sock 1 [lindex $rparts 1] {}
}
} elseif {[string match -nocase {POST /*} $line]} { # HTTP POST request
set rparts [split [string trimleft $line]]
if {[llength $rparts] > 1} { # valid POST request, read POST headers and data
set hdrread 0 set hdrread 0
set hdata {} set hdata {}
set pdata {} set docomp none
while {$hdrread < 1} { while {$hdrread < 1} {
if {[eof $sock] || [catch {gets $sock line}]} { if {[eof $sock] || [catch {gets $sock hline}]} {
break break
} else { } else {
if {$line eq ""} { if {$hline eq ""} {
incr hdrread incr hdrread
continue continue
} }
set hparts [split [string trimleft $line] :] set hparts [split [string trimleft $hline] :]
if {[llength $hparts] > 1} { if {[llength $hparts] > 1} {
set hname [string tolower [string trimright [lindex $hparts 0]]] set hname [string tolower [string trimright [lindex $hparts 0]]]
set hval [string trim [lindex $hparts 1]] set hval [string trim [lindex $hparts 1]]
@@ -468,6 +479,25 @@ proc reqhandler {dbfile sock} {
} }
} }
} }
if {[dict exists $hdata accept-encoding]} { # detect if we can gzip the output
set enclist [dict get $hdata accept-encoding]
if {[string match -nocase {*gzip*} $enclist]} {
set docomp gzip
} elseif {[string match -nocase {*deflate*} $enclist]} {
set docomp deflate
} elseif {[string match -nocase {*compress*} $enclist]} {
set docomp compress
}
}
if {[string match -nocase {GET /*} $line]} { # GET request
set rparts [split [string trimleft $line]]
if {[llength $rparts] > 1} { # valid GET request
routepath $dbfile $sock 1 [lindex $rparts 1] {} $docomp
}
} elseif {[string match -nocase {POST /*} $line]} { # POST request
set rparts [split [string trimleft $line]]
if {[llength $rparts] > 1} { # valid POST request, read POST headers and data
set pdata {}
set readlen 0 set readlen 0
if {[dict exists $hdata content-length]} { if {[dict exists $hdata content-length]} {
set readlen [dict get $hdata content-length] set readlen [dict get $hdata content-length]
@@ -476,7 +506,8 @@ proc reqhandler {dbfile sock} {
fconfigure $sock -translation {binary lf} -buffering none fconfigure $sock -translation {binary lf} -buffering none
set pdata [read $sock $readlen] set pdata [read $sock $readlen]
} }
routepath $dbfile $sock 1 [lindex $rparts 1] [string trimleft $pdata] routepath $dbfile $sock 1 [lindex $rparts 1] [string trimleft $pdata] $docomp
}
} }
} }
} }
tiifetch.tcl
+1
View File
@@ -203,6 +203,7 @@ proc n2p_id {binmsg} {
proc createdb {fname} { proc createdb {fname} {
sqlite3 fdb $fname sqlite3 fdb $fname
fdb eval { fdb eval {
PRAGMA journal_mode=WAL;
CREATE TABLE `msg` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, CREATE TABLE `msg` (`id` INTEGER PRIMARY KEY AUTOINCREMENT,
`msgid` VARCHAR(20) NOT NULL UNIQUE, `msgid` VARCHAR(20) NOT NULL UNIQUE,
`timestamp` INT NOT NULL, `timestamp` INT NOT NULL,