Implemented max message ID per request customization, also added ii-doc.txt

README

@@ -62,9 +62,17 @@ Fetching is supported for the following station URL schemes and protocols:
 
 If the station_url parameter is empty or no parameters are passed at all,
 tiifetch.tcl will look for a file called stations.txt that lists (each on a
-new line) all the station URLs to sync from. Messages from all listed stations
+new line) all the station URLs to sync from and the amount of IDs that can
-will be merged into the same echo conference database. You can comment out a
+be fetched from a particular station with a single GET request (this amount
-station to temporarily stop fetching from it by prepending the # sign.
+can be 12 or more):
+
+https://url1 389
+gemini://url2 24
+http://url3 12
+
+Messages from all listed stations will be merged into the same echo database.
+You also can comment out a station to temporarily stop fetching from it by
+prepending the # sign.
 
 ### Viewing the messages from CLI (tiiview.tcl): ###
 

ii-doc.txt

@@ -0,0 +1,135 @@
+ii protocol documentation
+=========================
+This document describes the basic ii protocol, as implemented in tii.
+It aims to be as clear and concise as possible.
+
+Network structure
+-----------------
+Clients aka points can:
+* post messages
+* fetch echos (conferences) and their message ID lists
+* fetch messages by their IDs
+
+Nodes aka stations can:
+* accept (or not accept) posted messages from points
+* serve echo lists
+* serve echos and their message ID lists
+* serve message bundles
+* fetch echos and messages from other stations
+
+The main transport protocol is currently HTTP/HTTPS, although the spec doesn't
+theoretically limit the ways of message transfer. E.g. fetching can be easily
+implemented over Gopher/Finger/Nex/Spartan/Gemini etc.
+
+The API spec below is given for the HTTP(S) protocol.
+
+Echo and message naming convention
+----------------------------------
+Within the network, echo names must be from 3 to 120 characters long and have
+at least one dot (.) character. Message IDs must be exactly 20 characters long
+and depend on the message contents hash (see the exact algorithm below).
+
+Station (node) HTTP API (as implemented in tii)
+-----------------------------------------------
+Every station must implement the following HTTP API calls.
+In case of multi-line responses, the newline separator must be "\n" character
+(line feed, ASCII 10).
+
+- Fetching the public echo list -
+
+Request: GET /list.txt
+Response: newline-separated list of echo_name:msg_count:echo_description
+
+- Listing messages in the echo (s) -
+
+Request: GET /u/e/echo.1.name/echo.2.name/...
+Response: newline-separated list of echo names and message IDs in the format:
+
+echo.1.name
+msgid1fromecho1
+msgid2fromecho1
+...
+echo.2.name
+msgid1fromecho2
+msgid2fromecho2
+...
+
+When a new message is posted to the echo, it gets appended to the end of the
+corresponding message ID list for this echo.
+
+- Fetching the message bundles -
+
+Request: GET /u/m/msgid1/msgid2/...
+Response: newline-separated list of msgid:base64_msgtext
+
+where base64_msgtext is a Base64-encoded Node-to-Point Message (see below).
+
+- Posting a message -
+
+Request: POST /u/point
+Content-Type: application/x-www-form-urlencoded 
+Data: pauth=auth_string&tmsg=base64_msgtext
+Response: in case of success, must start with "msg ok"
+
+where base64_msgtext is the Base64-encoded Point-to-Node Message (see below).
+The maximum length of the tmsg field must be 87382 bytes.
+
+Node-to-Point Message format
+----------------------------
+The encoding must be UTF-8 and the newline separator must be "\n" (ASCII 10).
+Every Node-to-Point message contains the following fields in this particular
+order, all of them are mandatory and start on a new line each:
+
+* Line 1: message tags. Must start with "ii/ok". If "ii/ok/repto/id" form is
+  encountered, then the id refers to the message this message replies to.
+* Line 2: echo name where the message was posted.
+* Line 3: message Unix timestamp (integer, in seconds, UTC)
+* Line 4: message sender name
+* Line 5: message sender address (autofilled by the originating station)
+* Line 6: message recipient name (or All if there's no particular recipient)
+* Line 7: message subject
+* Line 8: must be empty
+* Line 9 and further: message body
+
+Point-to-Node Message format
+----------------------------
+The encoding must be UTF-8 and the newline separator must be "\n" (ASCII 10).
+Every Point-to-Node message contains the following fields in this particular
+order, all of them are mandatory and start on a new line each:
+
+* Line 1: echo name where the message is being posted.
+* Line 2: message recipient name (or All if there's no particular recipient)
+* Line 3: message subject
+* Line 4: must be empty
+* Line 5 and further: message body
+
+If you are replying to a message [msgid], then message body must begin with:
+
+@repto:msgid
+
+and the message text itself must start on the next line.
+
+Message ID generation algorithm
+-------------------------------
+This algorithm must be implemented by every station to generate message IDs:
+
+1. Calculate SHA256 of the message in the Node-to-Point format as binary data.
+2. Calculate Base64 of the resulting binary hash sum.
+3. Truncate to the first 20 characters.
+4. Replace all occurrences of + or - with A, and / or _ with Z.
+5. The result of these operations is your ii message ID.
+
+Implementation notes
+--------------------
+* Most HTTP servers are configured to reject long GET lines, so tii passes a
+  limited amount of message IDs to the /u/m endpoints. This behaviour can be
+  configured in the stations.txt file.
+* Some of the crucial validations is for message IDs to be 20 lines and for
+  all messages (Node-to-Point and Point-to-Node) to strictly have LF line
+  endings, not CRLF.
+* The message order in an echo does not always match the timestamp ordering;
+  it is fully up to the client on how to sort the messages internally. The
+  messages are only guaranteed to be saved by the server in the order they
+  arrive onto the server.
+
+--- Luxferre ---

tiifetch.tcl

@@ -189,7 +189,8 @@ proc listcomp {a b} {
 }
 
 # main logic proc
-proc fetchiidb {url echos dbfile dolog} {
+proc fetchiidb {url echos dbfile dolog maxids} {
+  if {$maxids < 12} {set maxids 12}
   # trim the parameters 
   set url [string trim $url]
   set echos [string trim $echos]
@@ -233,10 +234,6 @@ proc fetchiidb {url echos dbfile dolog} {
     }
   }
   if {$dolog eq 1} {puts "Echomap built"}
-  # set how many message IDs we can pass in a single query
-  # (assuming the maximum GET length is 256 chars)
-  # and we assume 21 character per message ID
-  set maxids 12
   # pass the echo list and fetch the message IDs
   # now, process the map we've built
   dict for {echoname msgids} $echomap {
@@ -297,12 +294,12 @@ proc massfetch {echos db dolog} {
   if {$dolog eq 1} {puts "No ii/idec station URL specified, using stations.txt"}
   set stfile [file join $appdir "stations.txt"]
   if {[file exists $stfile]} {
-    set stlist [split [readfile $stfile] "\n"]
+    set stlist [readfile $stfile]
-    foreach station $stlist {
+    dict for {station stmaxids} $stlist {
       set station [string trim $station]
       if {$station ne "" && ![string match "#*" $station]} {
         if {$dolog eq 1} {puts "Fetching from $station"}
-        fetchiidb $station $echos $db $dolog
+        fetchiidb $station $echos $db $dolog $stmaxids
       }
     }
   } else {
@@ -345,7 +342,7 @@ if {$argc > 0} {
   if {$sturl eq ""} {
     massfetch [lindex $argv 1] $localdb 1
   } else {
-    fetchiidb $sturl [lindex $argv 1] $localdb 1
+    fetchiidb $sturl [lindex $argv 1] $localdb 1 12
   }
   puts "Messages fetched"
 } else {