luxferre/tii
1
0
Fork 0
Code Actions Activity

reverted fetching to a simpler and more robust algo

Browse Source
This commit is contained in:
Luxferre
2024-10-27 09:44:54 +02:00
parent d8b658a92b
commit 6bc8a2072c
2 changed files with 72 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ii-doc.txt
-13
View File
@@ -32,8 +32,6 @@ and depend on the message contents hash (see the exact algorithm below).
Station (node) HTTP API (as implemented in tii) Station (node) HTTP API (as implemented in tii)
----------------------------------------------- -----------------------------------------------
Every station must implement the following HTTP API calls. Every station must implement the following HTTP API calls.
The tii codebase also implements some IDEC extenstions but they are entirely
optional.
In case of multi-line responses, the newline separator must be "\n" character In case of multi-line responses, the newline separator must be "\n" character
(line feed, ASCII 10). (line feed, ASCII 10).
@@ -49,8 +47,6 @@ reported by nodes still should not be the basis for any client-side logic.
- Listing messages in the echo(s) - - Listing messages in the echo(s) -
Request: GET /u/e/echo.1.name/echo.2.name/... Request: GET /u/e/echo.1.name/echo.2.name/...
IDEC extended syntax: GET /u/e/echo.1.name/echo.2.name/.../offset:count
(where offset can be negative)
Response: newline-separated list of echo names and message IDs in the format: Response: newline-separated list of echo names and message IDs in the format:
echo.1.name echo.1.name
@@ -82,15 +78,6 @@ Response: in case of success, must start with "msg ok"
where base64_msgtext is the Base64-encoded Point-to-Node Message (see below). where base64_msgtext is the Base64-encoded Point-to-Node Message (see below).
The maximum length of the tmsg field must be 87382 bytes. The maximum length of the tmsg field must be 87382 bytes.
- Listing IDEC features (IDEC extension) -
Request: GET /x/features
Response: newline-separated list of supported non-standard URL paths
(u/e, /list.txt etc)
Note: if the /x/features path is unavailable, the client must assume that no
IDEC extensions except /list.txt are supported by the server.
Node-to-Point Message format Node-to-Point Message format
---------------------------- ----------------------------
The encoding must be UTF-8 and the newline separator must be "\n" (ASCII 10). The encoding must be UTF-8 and the newline separator must be "\n" (ASCII 10).
tiifetch.tcl
+71 -91
View File
@@ -181,14 +181,11 @@ proc writefileln {fname data} {
} }
# list comparison helper (listcomp $new $old) # list comparison helper (listcomp $new $old)
proc listcomp {a b} { proc listcomp {new old} {
set diff {} foreach i $old {
foreach i $a { set new [lsearch -all -inline -not -exact $new $i]
if {[lsearch -exact $b $i]==-1} {
lappend diff $i
}
} }
return $diff return $new
} }
# generate ID from the Node-to-Point msg contents # generate ID from the Node-to-Point msg contents
@@ -199,6 +196,25 @@ proc n2p_id {binmsg} {
return [string map {+ A - A / z _ z} $trimbased] return [string map {+ A - A / z _ z} $trimbased]
} }
# ensure database file is created
proc createdb {fname} {
sqlite3 fdb $fname
fdb eval {
CREATE TABLE `msg` (`id` INTEGER PRIMARY KEY AUTOINCREMENT,
`msgid` VARCHAR(20) NOT NULL UNIQUE,
`timestamp` INT NOT NULL,
`echoname` VARCHAR(120) NOT NULL,
`repto` VARCHAR(120) NOT NULL,
`msgfrom` VARCHAR(120) NOT NULL,
`msgfromaddr` VARCHAR(120) NOT NULL,
`msgto` VARCHAR(120) NOT NULL,
`subj` VARCHAR(120) NOT NULL,
`body` TEXT NOT NULL,
`content_id` VARCHAR(20) NOT NULL);
}
fdb close
}
# main logic proc # main logic proc
proc fetchiidb {url echos dbfile dolog maxids} { proc fetchiidb {url echos dbfile dolog maxids} {
if {$maxids < 12} {set maxids 12} if {$maxids < 12} {set maxids 12}
@@ -206,14 +222,8 @@ proc fetchiidb {url echos dbfile dolog maxids} {
set url [string trim $url] set url [string trim $url]
set echos [string trim $echos] set echos [string trim $echos]
set dbfile [file normalize [string trim $dbfile]] set dbfile [file normalize [string trim $dbfile]]
# prepare starting script if {![file exists $dbfile]} {createdb $dbfile}
sqlite3 msgdb $dbfile sqlite3 msgdb $dbfile
msgdb eval {
CREATE TABLE IF NOT EXISTS `msg` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `msgid` VARCHAR(20) UNIQUE,
`timestamp` INT, `echoname` VARCHAR(120), `repto` TEXT, `msgfrom` TEXT, `msgfromaddr` TEXT,
`msgto` TEXT, `subj` TEXT, `body` TEXT, `content_id` VARCHAR(20));
}
# attempt to fetch the echolist if echos are empty # attempt to fetch the echolist if echos are empty
if {$echos eq {}} { if {$echos eq {}} {
if {$dolog eq 1} {puts "Fetching echolist..."} if {$dolog eq 1} {puts "Fetching echolist..."}
@@ -222,55 +232,17 @@ proc fetchiidb {url echos dbfile dolog maxids} {
} else { } else {
set echos [split $echos "/,;"] set echos [split $echos "/,;"]
} }
set echos [lmap s $echos {string trim $s}] set echos [string trim [lmap s $echos {string trim $s " \t\r\n"}] " \t\r\n"]
if {$dolog eq 1} {puts "Echos to fetch: $echos"} if {$dolog eq 1} {puts "Echos to fetch: $echos"}
if {$dolog eq 1} {puts "Building message indexes..."} if {$dolog eq 1} {puts "Building message indexes..."}
# get the IDEC extended feature list set echodata [getfile [string cat $url "/u/e/" [join $echos "/"]]]
set featurelist "" set datalines [split $echodata \n]
catch {set featurelist [getfile [string cat $url "/x/features"]]}
set featurelist [lmap s [split $featurelist \n] {string trim $s}]
set datalines ""
if {[lsearch $featurelist u/e] > -1} {
# echoname => localcount map
set localcounts [msgdb eval {SELECT `echoname`, COUNT(`id`) FROM `msg` GROUP BY `echoname`;}]
foreach ename $echos {
if {$ename ne ""} {
set localdata ""
set localcount 0
if {[dict exists $localcounts $ename]} {
set localcount [dict get $localcounts $ename]
}
if {$localcount > 12} {
set diff 6
set needsync 1
while {$needsync eq 1} {
incr diff $diff
set localdata [getfile [string cat $url "/u/e/" $ename "/-$diff:1"]]
# control message id
set cmsg [string trim [lindex [split $localdata \n] 1]]
msgdb eval {SELECT `msgid` FROM `msg` WHERE `msgid` = $cmsg;} row {
if {[string trim $row(msgid)] eq $cmsg} {
set needsync 0
}
}
}
set localdata [getfile [string cat $url "/u/e/" $ename "/-$diff:$diff"]]
} else {
set localdata [getfile [string cat $url "/u/e/" $ename]]
}
append datalines [string trim $localdata] \n
}
}
} else { # no extended feature support, pass the echo list and fetch the message IDs
set echodata [getfile [string cat $url "/u/e/" [join $echos "/"]]]
set datalines [split $echodata \n]
}
# iterate over the fetched data and fetch corresponding messages # iterate over the fetched data and fetch corresponding messages
set curecho "" set curecho ""
set echomap "" set echomap ""
# build the map of lists of message IDs # build the map of lists of message IDs
foreach line $datalines { foreach line $datalines {
set line [string trim $line] set line [string trim $line " \t\r\n"]
if {$line ne ""} { if {$line ne ""} {
# detect if the line is related to echo name or message ID # detect if the line is related to echo name or message ID
if {[string first "." $line] eq -1} { # message ID if {[string first "." $line] eq -1} { # message ID
@@ -290,53 +262,61 @@ proc fetchiidb {url echos dbfile dolog maxids} {
# now, process the map we've built # now, process the map we've built
dict for {echoname msgids} $echomap { dict for {echoname msgids} $echomap {
if {![string match *.* $echoname]} {continue} if {![string match *.* $echoname]} {continue}
if {[llength msgids] eq 0} {continue} if {[llength $msgids] eq 0} {continue}
# get the existing message IDs in the echo # get the existing message IDs in the echo
set oldmsgids [msgdb eval {SELECT `msgid` FROM `msg` WHERE `echoname` = $echoname ORDER BY `id` ASC;}] set oldmsgids [msgdb eval {SELECT `msgid` FROM `msg` WHERE `echoname` = $echoname ORDER BY `id` ASC;}]
# pre-filter the new message IDs to fetch # pre-filter the new message IDs to fetch
set newmsgids [listcomp $msgids $oldmsgids] set newmsgids [listcomp $msgids $oldmsgids]
if {$dolog eq 1} {puts "Fetching [llength $newmsgids] new messages from $echoname..."}
set idgroups "" set idgroups ""
set grcount 0 set grcount 0
set localcount 0 set localcount 0
set globalcount 0
foreach nmid $newmsgids { # iterate over new messages to group them foreach nmid $newmsgids { # iterate over new messages to group them
if {$nmid ne ""} { if {$nmid ne ""} {
# insert new message ID to the echo mapping set cid [string trim [msgdb eval {SELECT `msgid` FROM `msg` WHERE `msgid` = $nmid;}]]
dict lappend idgroups $grcount $nmid if {$nmid ne $cid} {
incr localcount incr globalcount
if {$localcount > $maxids} { # insert new message ID to the echo mapping
incr grcount dict lappend idgroups $grcount $nmid
set localcount 0 incr localcount
if {$localcount > $maxids} {
incr grcount
set localcount 0
}
} }
} }
} }
dict for {mgrpind mgrp} $idgroups { # iterate over groups to fetch the messages if {$globalcount > 0} {
# get the message data in the bundle format if {$dolog eq 1} {puts "Fetching $globalcount new messages from $echoname..."}
set msgbundle [getfile [string cat $url "/u/m/" [join $mgrp "/"]]] dict for {mgrpind mgrp} $idgroups { # iterate over groups to fetch the messages
set bdata [split $msgbundle "\n"] # get the message data in the bundle format
foreach bline $bdata { set msgbundle [getfile [string cat $url "/u/m/" [join $mgrp "/"]]]
set parts [split $bline ":"] set bdata [split $msgbundle "\n"]
if {[llength $parts] > 1} { # valid message foreach bline $bdata {
set mid [lindex $parts 0] set parts [split $bline ":"]
set bdata [binary decode base64 [lindex $parts 1]] if {[llength $parts] > 1} { # valid message
# calculate ii Node-to-Point ID to verify the message integrity set mid [string trim [lindex $parts 0]]
set content_id [n2p_id $bdata] set bdata [binary decode base64 [lindex $parts 1]]
set mdata [encoding convertfrom utf-8 $bdata] # calculate ii Node-to-Point ID to verify the message integrity
set msglines [split $mdata "\n"] set content_id [n2p_id $bdata]
set replyto "" set mdata [encoding convertfrom utf-8 $bdata]
set tags [split [lindex $msglines 0] "/"] set msglines [split $mdata "\n"]
if {[dict exists $tags repto]} { set replyto ""
set replyto [dict get $tags repto] set tags [split [lindex $msglines 0] "/"]
} else {set replyto ""} if {[dict exists $tags repto]} {
set echoarea [string trim [lindex $msglines 1]] set replyto [dict get $tags repto]
set timestamp [string trim [lindex $msglines 2]] } else {set replyto ""}
set msgfrom [string trim [lindex $msglines 3]] set echoarea [string trim [lindex $msglines 1]]
set msgfromaddr [string trim [lindex $msglines 4]] set timestamp [string trim [lindex $msglines 2]]
set msgto [string trim [lindex $msglines 5]] set msgfrom [string trim [lindex $msglines 3]]
set subj [string trim [lindex $msglines 6]] set msgfromaddr [string trim [lindex $msglines 4]]
set msgbody [string trimright [lrange $msglines 8 end]] set msgto [string trim [lindex $msglines 5]]
msgdb eval {INSERT OR IGNORE INTO `msg` (`msgid`, `timestamp`, `echoname`, `repto`, `msgfrom`, `msgfromaddr`, `msgto`, `subj`, `body`, `content_id`) set subj [string trim [lindex $msglines 6]]
VALUES ($mid, $timestamp, $echoarea, $replyto, $msgfrom, $msgfromaddr, $msgto, $subj, $msgbody, $content_id);} set msgbody [string trimright [lrange $msglines 8 end]]
msgdb eval {INSERT OR IGNORE INTO `msg` (`msgid`, `timestamp`, `echoname`, `repto`, `msgfrom`,
`msgfromaddr`, `msgto`, `subj`, `body`, `content_id`)
VALUES ($mid, $timestamp, $echoarea, $replyto, $msgfrom, $msgfromaddr, $msgto, $subj, $msgbody, $content_id);}
}
} }
} }
} }