reverted fetching to a simpler and more robust algo

ii-doc.txt

@@ -32,8 +32,6 @@ and depend on the message contents hash (see the exact algorithm below).
 Station (node) HTTP API (as implemented in tii)
 -----------------------------------------------
 Every station must implement the following HTTP API calls.
-The tii codebase also implements some IDEC extenstions but they are entirely
-optional.
 In case of multi-line responses, the newline separator must be "\n" character
 (line feed, ASCII 10).
 
@@ -49,8 +47,6 @@ reported by nodes still should not be the basis for any client-side logic.
 - Listing messages in the echo(s) -
 
 Request: GET /u/e/echo.1.name/echo.2.name/...
-IDEC extended syntax: GET /u/e/echo.1.name/echo.2.name/.../offset:count
-(where offset can be negative)
 Response: newline-separated list of echo names and message IDs in the format:
 
 echo.1.name
@@ -82,15 +78,6 @@ Response: in case of success, must start with "msg ok"
 where base64_msgtext is the Base64-encoded Point-to-Node Message (see below).
 The maximum length of the tmsg field must be 87382 bytes.
 
-- Listing IDEC features (IDEC extension) -
-
-Request: GET /x/features
-Response: newline-separated list of supported non-standard URL paths
-(u/e, /list.txt etc)
-
-Note: if the /x/features path is unavailable, the client must assume that no
-IDEC extensions except /list.txt are supported by the server.
-
 Node-to-Point Message format
 ----------------------------
 The encoding must be UTF-8 and the newline separator must be "\n" (ASCII 10).

tiifetch.tcl

@@ -181,14 +181,11 @@ proc writefileln {fname data} {
 }
 
 # list comparison helper (listcomp $new $old)
-proc listcomp {a b} {
+proc listcomp {new old} {
-  set diff {}
+  foreach i $old {
-  foreach i $a {
+    set new [lsearch -all -inline -not -exact $new $i]
-    if {[lsearch -exact $b $i]==-1} {
-      lappend diff $i
   }
-  }
+  return $new
-  return $diff
 }
 
 # generate ID from the Node-to-Point msg contents
@@ -199,6 +196,25 @@ proc n2p_id {binmsg} {
   return [string map {+ A - A / z _ z} $trimbased]
 }
 
+# ensure database file is created
+proc createdb {fname} {
+  sqlite3 fdb $fname
+  fdb eval {
+    CREATE TABLE `msg` (`id` INTEGER PRIMARY KEY AUTOINCREMENT,
+      `msgid` VARCHAR(20) NOT NULL UNIQUE,
+      `timestamp` INT NOT NULL,
+      `echoname` VARCHAR(120) NOT NULL,
+      `repto` VARCHAR(120) NOT NULL,
+      `msgfrom` VARCHAR(120) NOT NULL,
+      `msgfromaddr` VARCHAR(120) NOT NULL,
+      `msgto` VARCHAR(120) NOT NULL,
+      `subj` VARCHAR(120) NOT NULL,
+      `body` TEXT NOT NULL,
+      `content_id` VARCHAR(20) NOT NULL);
+  }
+  fdb close
+}
+
 # main logic proc
 proc fetchiidb {url echos dbfile dolog maxids} {
   if {$maxids < 12} {set maxids 12}
@@ -206,14 +222,8 @@ proc fetchiidb {url echos dbfile dolog maxids} {
   set url [string trim $url]
   set echos [string trim $echos]
   set dbfile [file normalize [string trim $dbfile]]
-  # prepare starting script
+  if {![file exists $dbfile]} {createdb $dbfile}
   sqlite3 msgdb $dbfile
-  msgdb eval {
-    CREATE TABLE IF NOT EXISTS `msg` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `msgid` VARCHAR(20) UNIQUE,
-      `timestamp` INT, `echoname` VARCHAR(120), `repto` TEXT, `msgfrom` TEXT, `msgfromaddr` TEXT,
-      `msgto` TEXT, `subj` TEXT, `body` TEXT, `content_id` VARCHAR(20));
-  }
-
   # attempt to fetch the echolist if echos are empty
   if {$echos eq {}} {
     if {$dolog eq 1} {puts "Fetching echolist..."}
@@ -222,55 +232,17 @@ proc fetchiidb {url echos dbfile dolog maxids} {
   } else {
     set echos [split $echos "/,;"]
   }
-  set echos [lmap s $echos {string trim $s}]
+  set echos [string trim [lmap s $echos {string trim $s " \t\r\n"}] " \t\r\n"]
   if {$dolog eq 1} {puts "Echos to fetch: $echos"}
   if {$dolog eq 1} {puts "Building message indexes..."}
-  # get the IDEC extended feature list
-  set featurelist ""
-  catch {set featurelist [getfile [string cat $url "/x/features"]]}
-  set featurelist [lmap s [split $featurelist \n] {string trim $s}]
-  set datalines ""
-  if {[lsearch $featurelist u/e] > -1} {
-    # echoname => localcount map
-    set localcounts [msgdb eval {SELECT `echoname`, COUNT(`id`) FROM `msg` GROUP BY `echoname`;}]
-    foreach ename $echos {
-      if {$ename ne ""} {
-        set localdata ""
-        set localcount 0
-        if {[dict exists $localcounts $ename]} {
-          set localcount [dict get $localcounts $ename]
-        }
-        if {$localcount > 12} {
-          set diff 6
-          set needsync 1
-          while {$needsync eq 1} {
-            incr diff $diff
-            set localdata [getfile [string cat $url "/u/e/" $ename "/-$diff:1"]]
-            # control message id
-            set cmsg [string trim [lindex [split $localdata \n] 1]]
-            msgdb eval {SELECT `msgid` FROM `msg` WHERE `msgid` = $cmsg;} row {
-              if {[string trim $row(msgid)] eq $cmsg} {
-                set needsync 0
-              }
-            }
-          }
-          set localdata [getfile [string cat $url "/u/e/" $ename "/-$diff:$diff"]]
-        } else {
-          set localdata [getfile [string cat $url "/u/e/" $ename]]
-        }
-        append datalines [string trim $localdata] \n
-      }
-    }
-  } else { # no extended feature support, pass the echo list and fetch the message IDs
   set echodata [getfile [string cat $url "/u/e/" [join $echos "/"]]]
   set datalines [split $echodata \n]
-  }
   # iterate over the fetched data and fetch corresponding messages
   set curecho ""
   set echomap ""
   # build the map of lists of message IDs
   foreach line $datalines {
-    set line [string trim $line]
+    set line [string trim $line " \t\r\n"]
     if {$line ne ""} {
       # detect if the line is related to echo name or message ID 
       if {[string first "." $line] eq -1} { # message ID
@@ -290,17 +262,20 @@ proc fetchiidb {url echos dbfile dolog maxids} {
   # now, process the map we've built
   dict for {echoname msgids} $echomap {
     if {![string match *.* $echoname]} {continue} 
-    if {[llength msgids] eq 0} {continue}
+    if {[llength $msgids] eq 0} {continue}
     # get the existing message IDs in the echo
     set oldmsgids [msgdb eval {SELECT `msgid` FROM `msg` WHERE `echoname` = $echoname ORDER BY `id` ASC;}]
     # pre-filter the new message IDs to fetch
     set newmsgids [listcomp $msgids $oldmsgids] 
-    if {$dolog eq 1} {puts "Fetching [llength $newmsgids] new messages from $echoname..."}
     set idgroups ""
     set grcount 0
     set localcount 0
+    set globalcount 0
     foreach nmid $newmsgids { # iterate over new messages to group them
       if {$nmid ne ""} {
+        set cid [string trim [msgdb eval {SELECT `msgid` FROM `msg` WHERE `msgid` = $nmid;}]]
+        if {$nmid ne $cid} {
+          incr globalcount
           # insert new message ID to the echo mapping
           dict lappend idgroups $grcount $nmid 
           incr localcount
@@ -310,6 +285,9 @@ proc fetchiidb {url echos dbfile dolog maxids} {
           } 
         }
       }
+    }
+    if {$globalcount > 0} {
+      if {$dolog eq 1} {puts "Fetching $globalcount new messages from $echoname..."}
       dict for {mgrpind mgrp} $idgroups { # iterate over groups to fetch the messages
         # get the message data in the bundle format
         set msgbundle [getfile [string cat $url "/u/m/" [join $mgrp "/"]]]
@@ -317,7 +295,7 @@ proc fetchiidb {url echos dbfile dolog maxids} {
         foreach bline $bdata {
           set parts [split $bline ":"]
           if {[llength $parts] > 1} { # valid message
-          set mid [lindex $parts 0]
+            set mid [string trim [lindex $parts 0]]
             set bdata [binary decode base64 [lindex $parts 1]]
             # calculate ii Node-to-Point ID to verify the message integrity
             set content_id [n2p_id $bdata]
@@ -335,12 +313,14 @@ proc fetchiidb {url echos dbfile dolog maxids} {
             set msgto [string trim [lindex $msglines 5]]
             set subj [string trim [lindex $msglines 6]]
             set msgbody [string trimright [lrange $msglines 8 end]]
-          msgdb eval {INSERT OR IGNORE INTO `msg` (`msgid`, `timestamp`, `echoname`, `repto`, `msgfrom`, `msgfromaddr`, `msgto`, `subj`, `body`, `content_id`) 
+            msgdb eval {INSERT OR IGNORE INTO `msg` (`msgid`, `timestamp`, `echoname`, `repto`, `msgfrom`, 
+              `msgfromaddr`, `msgto`, `subj`, `body`, `content_id`) 
               VALUES ($mid, $timestamp, $echoarea, $replyto, $msgfrom, $msgfromaddr, $msgto, $subj, $msgbody, $content_id);}
           }
         }
       }
     }
+  }
   msgdb close
 }