Specify and implement fragmented control messages to allow for (among other things) long GETINFO replies. Otherwise we could hit the 64K barrier on questions like "please dump your client-side DNS cache."

svn:r3726

doc/TODO

@@ -62,7 +62,9 @@ N  . Implement pending controller features.
     o MAPADDRESS
       o Map A->B.
       o Map DontCare->B.
-    - Way to handle overlong messages?
+    o Way to handle overlong messages
+      o Specify fragmented format
+      o Implement fragmented format
     - Event for "new descriptors"
     o Better stream IDs
     - EXTENDCIRCUIT <depends on revised circ selection stuff.>

doc/control-spec.txt

@@ -51,6 +51,11 @@ the message.
 
 3. Message types
 
+  Message types are drawn from the following ranges:
+
+  0x0000-0xEFFF   : Reserved for use by official versions of this spec.
+  0xF000-0xFFFF   : Unallocated; usable by unofficial extensions.
+
 3.1. ERROR (Type 0x0000)
 
   Sent in response to a message that could not be processed as requested.
@@ -232,8 +237,6 @@ the message.
 
 3.11. MAPADDRESS (Type 0x000A)
 
-  [Proposal; not finalized]
-
   Sent from the client to the server.  The body contains a sequence of
   address mappings, each consisting of the address to be mapped, a single
   space, the replacement address, and a NL character.
@@ -279,8 +282,6 @@ the message.
 
 3.12 GETINFO (Type 0x000B)
 
-  [Proposal; not finalized]
-
   Sent from the client to the server.  The message body is as for GETCONF:
   one or more NL-terminated strings.  The server replies with an INFOVALUE
   message.
@@ -304,8 +305,6 @@ the message.
 
 3.13 INFOVALUE (Type 0x000C)
 
-  [Proposal; not finalized]
-
   Sent from the server to the client in response to a GETINFO message.
   Contains one or more items of the format:
 
@@ -360,6 +359,32 @@ the message.
   the descriptor for any reason, the server must send an appropriate error
   message.
 
+3.17 FRAGMENTHEADER (Type 0x0010)
+
+  [Proposal; not finalized]
+
+  Sent in either direction.  Used to encapsulate messages longer than 65535
+  bytes long.
+
+      Underlying type [2 bytes]
+      Total Length    [4 bytes]
+      Data            [Rest of message]
+
+  A FRAGMENTHEADER message MUST be followed immediately by a number of
+  FRAGMENT messages, such that lengths of the "Data" fields of the
+  FRAGMENTHEADER and FRAGMENT messages add to the "Total Length" field of the
+  FRAGMENTHEADER message.
+
+  Implementations MUST NOT fragment messages of length less than 65536 bytes.
+  Implementations MUST be able to process fragmented messages that not
+  optimally packed.
+
+3.18 FRAGMENT (Type 0x0011)
+
+  [Proposal; not finalized]
+
+      Data           [Entire message]
+
 4. Implementation notes
 
 4.1. There are four ways we could authenticate, for now:

src/or/buffers.c

@@ -645,20 +645,24 @@ int fetch_from_buf_socks(buf_t *buf, socks_request_t *req) {
   }
 }
 
+
+#define CONTROL_CMD_FRAGMENTHEADER 0x0010
+#define CONTROL_CMD_FRAGMENT       0x0011
 /** If there is a complete control message waiting on buf, then store
  * its contents into *<b>type_out</b>, store its body's length into
  * *<b>len_out</b>, allocate and store a string for its body into
- * *<b>body_out</b>, and return -1.  (body_out will always be NUL-terminated,
+ * *<b>body_out</b>, and return 1.  (body_out will always be NUL-terminated,
  * even if the control message body doesn't end with NUL.)
  *
  * If there is not a complete control message waiting, return 0.
  *
  * Return -1 on error.
  */
-int fetch_from_buf_control(buf_t *buf, uint16_t *len_out, uint16_t *type_out,
+int fetch_from_buf_control(buf_t *buf, uint32_t *len_out, uint16_t *type_out,
                            char **body_out)
 {
-  uint16_t len;
+  uint32_t msglen;
+  uint16_t type;
 
   tor_assert(buf);
   tor_assert(len_out);
@@ -668,23 +672,82 @@ int fetch_from_buf_control(buf_t *buf, uint16_t *len_out, uint16_t *type_out,
   if (buf->datalen < 4)
     return 0;
 
-  len = ntohs(get_uint16(buf->mem));
-  if (buf->datalen < 4 + (unsigned)len)
+  msglen = ntohs(get_uint16(buf->mem));
+  if (buf->datalen < 4 + (unsigned)msglen)
     return 0;
 
-  *len_out = len;
-  *type_out = ntohs(get_uint16(buf->mem+2));
-  if (len) {
-    *body_out = tor_malloc(len+1);
-    memcpy(*body_out, buf->mem+4, len);
-    (*body_out)[len] = '\0';
+  type = ntohs(get_uint16(buf->mem+2));
+  if (type != CONTROL_CMD_FRAGMENTHEADER) {
+    *len_out = msglen;
+    *type_out = type;
+    if (msglen) {
+      *body_out = tor_malloc(msglen+1);
+      memcpy(*body_out, buf->mem+4, msglen);
+      (*body_out)[msglen] = '\0';
+    } else {
+      *body_out = NULL;
+    }
+    buf_remove_from_front(buf, 4+msglen);
+
+    return 1;
   } else {
-    *body_out = NULL;
-  }
+    uint32_t totallen, sofar;
+    char *cp, *endp, *outp;
+
+    /* Okay, we have a fragmented message.  Is it all here? */
+    if (msglen < 6)
+      return -1;
+    type = htons(get_uint16(buf->mem+4));
+    totallen = htonl(get_uint32(buf->mem+6));
+    if (totallen < 65536)
+      return -1;
+
+    if (buf->datalen<4+6+totallen)
+      /* The data can't possibly be here yet, no matter how well it's packed.*/
+      return 0;
+
+    /* Count how much data is really here. */
+    sofar = msglen-6;
+    cp = buf->mem+4+msglen;
+    endp = buf->mem+buf->datalen;
+    while (sofar < totallen) {
+      if ((endp-cp)<4)
+        return 0; /* Fragment header not all here. */
+      msglen = ntohs(get_uint16(cp));
+      if (ntohs(get_uint16(cp+2) != CONTROL_CMD_FRAGMENT))
+        return -1; /* Missing fragment message; error. */
+      if ((endp-cp) < 4+msglen)
+        return 0; /* Fragment not all here. */
+      sofar += msglen;
+      cp += (4+msglen);
+    }
+    if (sofar > totallen)
+      return -1; /* Fragments add to more than expected; error. */
+
+    /* Okay, everything is here. */
+    *len_out = totallen;
+    *type_out = type;
+    *body_out = tor_malloc(totallen+1);
+
+    /* copy FRAGMENTED packet contents. */
+    msglen = ntohs(get_uint16(buf->mem));
+    if (msglen>6)
+      memcpy(*body_out,buf->mem+4+6,msglen-6);
+    sofar = msglen-6;
+    outp = *body_out+sofar;
+    cp = buf->mem+4+msglen;
+    while (sofar < totallen) {
+      msglen = ntohs(get_uint16(cp));
+      memcpy(outp,cp+4,msglen);
+      outp += msglen;
+      cp += 4+msglen;
+      sofar -= msglen;
+    }
+    (*body_out)[totallen]='\0';
 
-  buf_remove_from_front(buf, 4+len);
+    return 1;
+  }
 
-  return 1;
 }
 
 /** Log an error and exit if <b>buf</b> is corrupted.

src/or/control.c

@@ -46,7 +46,9 @@ const char control_c_id[] = "$Id$";
 #define CONTROL_CMD_EXTENDCIRCUIT  0x000D
 #define CONTROL_CMD_ATTACHSTREAM   0x000E
 #define CONTROL_CMD_POSTDESCRIPTOR 0x000F
-#define _CONTROL_CMD_MAX_RECOGNIZED 0x000F
+#define CONTROL_CMD_FRAGMENTHEADER 0x0010
+#define CONTROL_CMD_FRAGMENT       0x0011
+#define _CONTROL_CMD_MAX_RECOGNIZED 0x0011
 
 /* Recognized error codes. */
 #define ERR_UNSPECIFIED             0x0000
@@ -89,7 +91,9 @@ static const char * CONTROL_COMMANDS[_CONTROL_CMD_MAX_RECOGNIZED+1] = {
   "infovalue",
   "extendcircuit",
   "attachstream",
-  "postdescriptor"
+  "postdescriptor",
+  "fragmentheader",
+  "fragment",
 };
 
 /** Bitfield: The bit 1&lt;&lt;e is set if <b>any</b> open control
@@ -115,33 +119,33 @@ static char authentication_cookie[AUTHENTICATION_COOKIE_LEN];
 
 static void update_global_event_mask(void);
 static void send_control_message(connection_t *conn, uint16_t type,
-                                 uint16_t len, const char *body);
+                                 uint32_t len, const char *body);
 static void send_control_done(connection_t *conn);
 static void send_control_done2(connection_t *conn, const char *msg, size_t len);
 static void send_control_error(connection_t *conn, uint16_t error,
                                const char *message);
-static void send_control_event(uint16_t event, uint16_t len, const char *body);
-static int handle_control_setconf(connection_t *conn, uint16_t len,
+static void send_control_event(uint16_t event, uint32_t len, const char *body);
+static int handle_control_setconf(connection_t *conn, uint32_t len,
                                   char *body);
-static int handle_control_getconf(connection_t *conn, uint16_t len,
+static int handle_control_getconf(connection_t *conn, uint32_t len,
                                   const char *body);
-static int handle_control_setevents(connection_t *conn, uint16_t len,
+static int handle_control_setevents(connection_t *conn, uint32_t len,
                                     const char *body);
-static int handle_control_authenticate(connection_t *conn, uint16_t len,
+static int handle_control_authenticate(connection_t *conn, uint32_t len,
                                        const char *body);
-static int handle_control_saveconf(connection_t *conn, uint16_t len,
+static int handle_control_saveconf(connection_t *conn, uint32_t len,
                                    const char *body);
-static int handle_control_signal(connection_t *conn, uint16_t len,
+static int handle_control_signal(connection_t *conn, uint32_t len,
                                  const char *body);
-static int handle_control_mapaddress(connection_t *conn, uint16_t len,
+static int handle_control_mapaddress(connection_t *conn, uint32_t len,
                                      const char *body);
-static int handle_control_getinfo(connection_t *conn, uint16_t len,
+static int handle_control_getinfo(connection_t *conn, uint32_t len,
                                   const char *body);
-static int handle_control_extendcircuit(connection_t *conn, uint16_t len,
+static int handle_control_extendcircuit(connection_t *conn, uint32_t len,
                                         const char *body);
-static int handle_control_attachstream(connection_t *conn, uint16_t len,
+static int handle_control_attachstream(connection_t *conn, uint32_t len,
                                         const char *body);
-static int handle_control_postdescriptor(connection_t *conn, uint16_t len,
+static int handle_control_postdescriptor(connection_t *conn, uint32_t len,
                                          const char *body);
 
 /** Given a possibly invalid message type code <b>cmd</b>, return a
@@ -172,18 +176,38 @@ static void update_global_event_mask(void)
 /** Send a message of type <b>type</b> containing <b>len</b> bytes
  * from <b>body</b> along the control connection <b>conn</b> */
 static void
-send_control_message(connection_t *conn, uint16_t type, uint16_t len,
+send_control_message(connection_t *conn, uint16_t type, uint32_t len,
                      const char *body)
 {
-  char buf[4];
+  char buf[10];
   tor_assert(conn);
   tor_assert(len || !body);
   tor_assert(type <= _CONTROL_CMD_MAX_RECOGNIZED);
-  set_uint16(buf, htons(len));
-  set_uint16(buf+2, htons(type));
-  connection_write_to_buf(buf, 4, conn);
-  if (len)
-    connection_write_to_buf(body, len, conn);
+  if (len < 65536) {
+    set_uint16(buf, htons(len));
+    set_uint16(buf+2, htons(type));
+    connection_write_to_buf(buf, 4, conn);
+    if (len)
+      connection_write_to_buf(body, len, conn);
+  } else {
+    set_uint16(buf, htons(65535));
+    set_uint16(buf+2, htons(CONTROL_CMD_FRAGMENTHEADER));
+    set_uint16(buf+4, htons(type));
+    set_uint32(buf+6, htonl(len));
+    connection_write_to_buf(buf, 10, conn);
+    connection_write_to_buf(body, 65535-6, conn);
+    len -= (65535-6);
+    body += (65535-6);
+    while (len) {
+      size_t chunklen = (len<65535)?len:65535;
+      set_uint16(buf, htons((uint16_t)chunklen));
+      set_uint16(buf+2, htons(CONTROL_CMD_FRAGMENT));
+      connection_write_to_buf(buf, 4, conn);
+      connection_write_to_buf(body, chunklen, conn);
+      len -= chunklen;
+      body += chunklen;
+    }
+  }
 }
 
 /** Send a "DONE" message down the control connection <b>conn</b> */
@@ -216,7 +240,7 @@ send_control_error(connection_t *conn, uint16_t error, const char *message)
  * <b>len</b> bytes in <b>body</b> to every control connection that
  * is interested in it. */
 static void
-send_control_event(uint16_t event, uint16_t len, const char *body)
+send_control_event(uint16_t event, uint32_t len, const char *body)
 {
   connection_t **conns;
   int n_conns, i;
@@ -233,7 +257,7 @@ send_control_event(uint16_t event, uint16_t len, const char *body)
     if (conns[i]->type == CONN_TYPE_CONTROL &&
         conns[i]->state == CONTROL_CONN_STATE_OPEN &&
         conns[i]->event_mask & (1<<event)) {
-      send_control_message(conns[i], CONTROL_CMD_EVENT, (uint16_t)(buflen), buf);
+      send_control_message(conns[i], CONTROL_CMD_EVENT, buflen, buf);
     }
   }
 
@@ -243,7 +267,7 @@ send_control_event(uint16_t event, uint16_t len, const char *body)
 /** Called when we receive a SETCONF message: parse the body and try
  * to update our configuration.  Reply with a DONE or ERROR message. */
 static int
-handle_control_setconf(connection_t *conn, uint16_t len, char *body)
+handle_control_setconf(connection_t *conn, uint32_t len, char *body)
 {
   int r;
   struct config_line_t *lines=NULL;
@@ -278,7 +302,7 @@ handle_control_setconf(connection_t *conn, uint16_t len, char *body)
 /** Called when we receive a GETCONF message.  Parse the request, and
  * reply with a CONFVALUE or an ERROR message */
 static int
-handle_control_getconf(connection_t *conn, uint16_t body_len, const char *body)
+handle_control_getconf(connection_t *conn, uint32_t body_len, const char *body)
 {
   smartlist_t *questions = NULL;
   smartlist_t *answers = NULL;
@@ -332,7 +356,7 @@ handle_control_getconf(connection_t *conn, uint16_t body_len, const char *body)
 /** Called when we get a SETEVENTS message: update conn->event_mask,
  * and reply with DONE or ERROR. */
 static int
-handle_control_setevents(connection_t *conn, uint16_t len, const char *body)
+handle_control_setevents(connection_t *conn, uint32_t len, const char *body)
 {
   uint16_t event_code;
   uint32_t event_mask = 0;
@@ -382,7 +406,7 @@ decode_hashed_password(char *buf, const char *hashed)
  * OPEN.  Reply with DONE or ERROR.
  */
 static int
-handle_control_authenticate(connection_t *conn, uint16_t len, const char *body)
+handle_control_authenticate(connection_t *conn, uint32_t len, const char *body)
 {
   or_options_t *options = get_options();
   if (options->CookieAuthentication) {
@@ -421,7 +445,7 @@ handle_control_authenticate(connection_t *conn, uint16_t len, const char *body)
 }
 
 static int
-handle_control_saveconf(connection_t *conn, uint16_t len,
+handle_control_saveconf(connection_t *conn, uint32_t len,
                         const char *body)
 {
   if (save_current_config()<0) {
@@ -434,7 +458,7 @@ handle_control_saveconf(connection_t *conn, uint16_t len,
 }
 
 static int
-handle_control_signal(connection_t *conn, uint16_t len,
+handle_control_signal(connection_t *conn, uint32_t len,
                       const char *body)
 {
   if (len != 1) {
@@ -449,7 +473,7 @@ handle_control_signal(connection_t *conn, uint16_t len,
 }
 
 static int
-handle_control_mapaddress(connection_t *conn, uint16_t len, const char *body)
+handle_control_mapaddress(connection_t *conn, uint32_t len, const char *body)
 {
   smartlist_t *elts;
   smartlist_t *lines;
@@ -478,8 +502,9 @@ handle_control_mapaddress(connection_t *conn, uint16_t len, const char *body)
           log_fn(LOG_WARN,
                  "Unable to allocate address for '%s' in AdressMap msg", line);
         } else {
-          char *ans = tor_malloc(strlen(addr)+strlen(to)+2);
-          tor_snprintf(ans, "%s %s", addr, to);
+          size_t anslen = strlen(addr)+strlen(to)+2;
+          char *ans = tor_malloc(anslen);
+          tor_snprintf(ans, anslen, "%s %s", addr, to);
           addressmap_register(addr, tor_strdup(to), 0);
           smartlist_add(reply, ans);
         }
@@ -542,7 +567,7 @@ handle_getinfo_helper(const char *question)
 }
 
 static int
-handle_control_getinfo(connection_t *conn, uint16_t len, const char *body)
+handle_control_getinfo(connection_t *conn, uint32_t len, const char *body)
 {
   smartlist_t *questions = NULL;
   smartlist_t *answers = NULL;
@@ -574,7 +599,7 @@ handle_control_getinfo(connection_t *conn, uint16_t len, const char *body)
 
   msg = smartlist_join_strings2(answers, "\0", 1, 1, &msg_len);
   send_control_message(conn, CONTROL_CMD_INFOVALUE,
-                       (uint16_t)msg_len, msg_len?msg:NULL);
+                       msg_len, msg_len?msg:NULL);
 
  done:
   if (answers) SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp));
@@ -586,20 +611,20 @@ handle_control_getinfo(connection_t *conn, uint16_t len, const char *body)
   return 0;
 }
 static int
-handle_control_extendcircuit(connection_t *conn, uint16_t len,
+handle_control_extendcircuit(connection_t *conn, uint32_t len,
                              const char *body)
 {
   send_control_error(conn,ERR_UNRECOGNIZED_TYPE,"not yet implemented");
   return 0;
 }
-static int handle_control_attachstream(connection_t *conn, uint16_t len,
+static int handle_control_attachstream(connection_t *conn, uint32_t len,
                                         const char *body)
 {
   send_control_error(conn,ERR_UNRECOGNIZED_TYPE,"not yet implemented");
   return 0;
 }
 static int
-handle_control_postdescriptor(connection_t *conn, uint16_t len,
+handle_control_postdescriptor(connection_t *conn, uint32_t len,
                               const char *body)
 {
   if (router_load_single_router(body)<0) {
@@ -634,7 +659,8 @@ int connection_control_reached_eof(connection_t *conn) {
  */
 int
 connection_control_process_inbuf(connection_t *conn) {
-  uint16_t body_len, command_type;
+  uint32_t body_len;
+  uint16_t command_type;
   char *body;
 
   tor_assert(conn);
@@ -726,6 +752,10 @@ connection_control_process_inbuf(connection_t *conn) {
       send_control_error(conn, ERR_UNRECOGNIZED_TYPE,
                          "Command type only valid from server to tor client");
       break;
+    case CONTROL_CMD_FRAGMENTHEADER:
+    case CONTROL_CMD_FRAGMENT:
+      log_fn(LOG_WARN, "Recieved command fragment out of order; ignoring.");
+      send_control_error(conn, ERR_SYNTAX, "Bad fragmentation on command.");
     default:
       log_fn(LOG_WARN, "Received unrecognized command type %d; ignoring.",
              (int)command_type);
@@ -756,7 +786,7 @@ control_event_circuit_status(circuit_t *circ, circuit_status_event_t tp)
   set_uint32(msg+1, htonl(circ->global_identifier));
   strlcpy(msg+5,path,path_len+1);
 
-  send_control_event(EVENT_CIRCUIT_STATUS, (uint16_t)(path_len+6), msg);
+  send_control_event(EVENT_CIRCUIT_STATUS, (uint32_t)(path_len+6), msg);
   tor_free(path);
   tor_free(msg);
   return 0;
@@ -784,7 +814,7 @@ control_event_stream_status(connection_t *conn, stream_status_event_t tp)
   set_uint32(msg+1, htonl(conn->global_identifier));
   strlcpy(msg+5, buf, len+1);
 
-  send_control_event(EVENT_STREAM_STATUS, (uint16_t)(5+len+1), msg);
+  send_control_event(EVENT_STREAM_STATUS, (uint32_t)(5+len+1), msg);
   tor_free(msg);
   return 0;
 }
@@ -805,7 +835,7 @@ control_event_or_conn_status(connection_t *conn,or_conn_status_event_t tp)
   buf[0] = (uint8_t)tp;
   strlcpy(buf+1,conn->nickname,sizeof(buf)-1);
   len = strlen(buf+1);
-  send_control_event(EVENT_OR_CONN_STATUS, (uint16_t)(len+1), buf);
+  send_control_event(EVENT_OR_CONN_STATUS, (uint32_t)(len+1), buf);
   return 0;
 }
 
@@ -837,7 +867,7 @@ control_event_logmsg(int severity, const char *msg)
     return;
 
   len = strlen(msg);
-  send_control_event(EVENT_WARNING, (uint16_t)(len+1), msg);
+  send_control_event(EVENT_WARNING, (uint32_t)(len+1), msg);
 }
 
 /** Choose a random authentication cookie and write it to disk.

src/or/or.h

@@ -1100,7 +1100,7 @@ int fetch_from_buf_http(buf_t *buf,
                         char **headers_out, size_t max_headerlen,
                         char **body_out, size_t *body_used, size_t max_bodylen);
 int fetch_from_buf_socks(buf_t *buf, socks_request_t *req);
-int fetch_from_buf_control(buf_t *buf, uint16_t *len_out, uint16_t *type_out,
+int fetch_from_buf_control(buf_t *buf, uint32_t *len_out, uint16_t *type_out,
                            char **body_out);
 
 void assert_buf_ok(buf_t *buf);