Move protocol-specific functions out of buffers.c

This commit does not change the implementation of any function: it
only moves code and adds new includes as necessary.  Part of #23149.

changes/refactor-buffer

@@ -0,0 +1,3 @@
+  o Code simplifications and refactoring:
+    - Split the portions of the buffer.c module that handle particular
+      protocols into separate modules. Part of ticket 23149.

src/or/buffers.h

@@ -39,23 +39,13 @@ int write_to_buf(const char *string, size_t string_len, buf_t *buf);
 int write_to_buf_compress(buf_t *buf, tor_compress_state_t *state,
                           const char *data, size_t data_len, int done);
 int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen);
+void peek_from_buf(char *string, size_t string_len, const buf_t *buf);
+void buf_remove_from_front(buf_t *buf, size_t n);
 int fetch_from_buf(char *string, size_t string_len, buf_t *buf);
-int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto);
-int fetch_from_buf_http(buf_t *buf,
-                        char **headers_out, size_t max_headerlen,
-                        char **body_out, size_t *body_used, size_t max_bodylen,
-                        int force_complete);
-socks_request_t *socks_request_new(void);
-void socks_request_free(socks_request_t *req);
-int fetch_from_buf_socks(buf_t *buf, socks_request_t *req,
-                         int log_sockstype, int safe_socks);
-int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason);
 int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len);
 
-int peek_buf_has_control0_command(buf_t *buf);
 #define PEEK_BUF_STARTSWITH_MAX 16
 int peek_buf_startswith(const buf_t *buf, const char *cmd);
-int peek_buf_has_http_command(const buf_t *buf);
 
 int fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out);
 
@@ -64,14 +54,15 @@ int buf_set_to_copy(buf_t **output,
 
 void assert_buf_ok(buf_t *buf);
 
+int buf_find_string_offset(const buf_t *buf, const char *s, size_t n);
+void buf_pullup(buf_t *buf, size_t bytes);
+
 #ifdef BUFFERS_PRIVATE
-STATIC int buf_find_string_offset(const buf_t *buf, const char *s, size_t n);
-STATIC void buf_pullup(buf_t *buf, size_t bytes);
 #ifdef TOR_UNIT_TESTS
 void buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz);
 buf_t *buf_new_with_data(const char *cp, size_t sz);
 #endif
-STATIC size_t preferred_chunk_size(size_t target);
+ATTR_UNUSED STATIC size_t preferred_chunk_size(size_t target);
 
 #define DEBUG_CHUNK_ALLOC
 /** A single chunk on a buffer. */
@@ -103,10 +94,5 @@ struct buf_t {
 };
 #endif
 
-#ifdef BUFFERS_PRIVATE
-STATIC int buf_http_find_content_length(const char *headers, size_t headerlen,
-                                        size_t *result_out);
-#endif
-
 #endif
 

src/or/connection.c

@@ -86,6 +86,8 @@
 #include "hs_common.h"
 #include "hs_ident.h"
 #include "nodelist.h"
+#include "proto_http.h"
+#include "proto_socks.h"
 #include "policies.h"
 #include "reasons.h"
 #include "relay.h"

src/or/connection_edge.c

@@ -82,6 +82,7 @@
 #include "main.h"
 #include "nodelist.h"
 #include "policies.h"
+#include "proto_socks.h"
 #include "reasons.h"
 #include "relay.h"
 #include "rendclient.h"

src/or/connection_or.c

@@ -46,6 +46,7 @@
 #include "microdesc.h"
 #include "networkstatus.h"
 #include "nodelist.h"
+#include "proto_cell.h"
 #include "reasons.h"
 #include "relay.h"
 #include "rephist.h"

src/or/control.c

@@ -64,6 +64,8 @@
 #include "networkstatus.h"
 #include "nodelist.h"
 #include "policies.h"
+#include "proto_control0.h"
+#include "proto_http.h"
 #include "reasons.h"
 #include "rendclient.h"
 #include "rendcommon.h"

src/or/include.am

@@ -79,6 +79,10 @@ LIBTOR_A_SOURCES = \
 	src/or/parsecommon.c			\
 	src/or/periodic.c				\
 	src/or/protover.c				\
+	src/or/proto_cell.c				\
+	src/or/proto_control0.c				\
+	src/or/proto_http.c				\
+	src/or/proto_socks.c				\
 	src/or/policies.c				\
 	src/or/reasons.c				\
 	src/or/relay.c					\
@@ -215,6 +219,10 @@ ORHEADERS = \
 	src/or/periodic.h				\
 	src/or/policies.h				\
 	src/or/protover.h				\
+	src/or/proto_cell.h				\
+	src/or/proto_control0.h				\
+	src/or/proto_http.h				\
+	src/or/proto_socks.h				\
 	src/or/reasons.h				\
 	src/or/relay.h					\
 	src/or/rendcache.h				\

src/or/proto_cell.c

@@ -0,0 +1,84 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define BUFFERS_PRIVATE // XXXX remove.
+#include "or.h"
+#include "buffers.h"
+#include "proto_cell.h"
+
+#include "connection_or.h"
+
+/** True iff the cell command <b>command</b> is one that implies a
+ * variable-length cell in Tor link protocol <b>linkproto</b>. */
+static inline int
+cell_command_is_var_length(uint8_t command, int linkproto)
+{
+  /* If linkproto is v2 (2), CELL_VERSIONS is the only variable-length cells
+   * work as implemented here. If it's 1, there are no variable-length cells.
+   * Tor does not support other versions right now, and so can't negotiate
+   * them.
+   */
+  switch (linkproto) {
+  case 1:
+    /* Link protocol version 1 has no variable-length cells. */
+    return 0;
+  case 2:
+    /* In link protocol version 2, VERSIONS is the only variable-length cell */
+    return command == CELL_VERSIONS;
+  case 0:
+  case 3:
+  default:
+    /* In link protocol version 3 and later, and in version "unknown",
+     * commands 128 and higher indicate variable-length. VERSIONS is
+     * grandfathered in. */
+    return command == CELL_VERSIONS || command >= 128;
+  }
+}
+
+/** Check <b>buf</b> for a variable-length cell according to the rules of link
+ * protocol version <b>linkproto</b>.  If one is found, pull it off the buffer
+ * and assign a newly allocated var_cell_t to *<b>out</b>, and return 1.
+ * Return 0 if whatever is on the start of buf_t is not a variable-length
+ * cell.  Return 1 and set *<b>out</b> to NULL if there seems to be the start
+ * of a variable-length cell on <b>buf</b>, but the whole thing isn't there
+ * yet. */
+int
+fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto)
+{
+  char hdr[VAR_CELL_MAX_HEADER_SIZE];
+  var_cell_t *result;
+  uint8_t command;
+  uint16_t length;
+  const int wide_circ_ids = linkproto >= MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS;
+  const int circ_id_len = get_circ_id_size(wide_circ_ids);
+  const unsigned header_len = get_var_cell_header_size(wide_circ_ids);
+  *out = NULL;
+  if (buf->datalen < header_len)
+    return 0;
+  peek_from_buf(hdr, header_len, buf);
+
+  command = get_uint8(hdr + circ_id_len);
+  if (!(cell_command_is_var_length(command, linkproto)))
+    return 0;
+
+  length = ntohs(get_uint16(hdr + circ_id_len + 1));
+  if (buf->datalen < (size_t)(header_len+length))
+    return 1;
+  result = var_cell_new(length);
+  result->command = command;
+  if (wide_circ_ids)
+    result->circ_id = ntohl(get_uint32(hdr));
+  else
+    result->circ_id = ntohs(get_uint16(hdr));
+
+  buf_remove_from_front(buf, header_len);
+  peek_from_buf((char*) result->payload, length, buf);
+  buf_remove_from_front(buf, length);
+
+  *out = result;
+  return 1;
+}
+

src/or/proto_cell.h

@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_CELL_H
+#define TOR_PROTO_CELL_H
+
+struct buf_t;
+struct var_cell_t;
+
+int fetch_var_cell_from_buf(struct buf_t *buf, struct var_cell_t **out,
+                            int linkproto);
+
+#endif
+

src/or/proto_control0.c

@@ -0,0 +1,27 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define BUFFERS_PRIVATE // XXXX remove.
+#include "or.h"
+#include "buffers.h"
+#include "proto_control0.h"
+
+/** Return 1 iff buf looks more like it has an (obsolete) v0 controller
+ * command on it than any valid v1 controller command. */
+int
+peek_buf_has_control0_command(buf_t *buf)
+{
+  if (buf->datalen >= 4) {
+    char header[4];
+    uint16_t cmd;
+    peek_from_buf(header, sizeof(header), buf);
+    cmd = ntohs(get_uint16(header+2));
+    if (cmd <= 0x14)
+      return 1; /* This is definitely not a v1 control command. */
+  }
+  return 0;
+}
+

src/or/proto_control0.h

@@ -0,0 +1,14 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_CONTROL0_H
+#define TOR_PROTO_CONTROL0_H
+
+struct buf_t;
+int peek_buf_has_control0_command(struct buf_t *buf);
+
+#endif
+

src/or/proto_http.c

@@ -0,0 +1,173 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define BUFFERS_PRIVATE // XXXX remove.
+#define PROTO_HTTP_PRIVATE
+#include "or.h"
+#include "buffers.h"
+#include "proto_http.h"
+
+/** Return true if <b>cmd</b> looks like a HTTP (proxy) request. */
+int
+peek_buf_has_http_command(const buf_t *buf)
+{
+  if (peek_buf_startswith(buf, "CONNECT ") ||
+      peek_buf_startswith(buf, "DELETE ") ||
+      peek_buf_startswith(buf, "GET ") ||
+      peek_buf_startswith(buf, "POST ") ||
+      peek_buf_startswith(buf, "PUT " ))
+    return 1;
+  return 0;
+}
+
+/** There is a (possibly incomplete) http statement on <b>buf</b>, of the
+ * form "\%s\\r\\n\\r\\n\%s", headers, body. (body may contain NULs.)
+ * If a) the headers include a Content-Length field and all bytes in
+ * the body are present, or b) there's no Content-Length field and
+ * all headers are present, then:
+ *
+ *  - strdup headers into <b>*headers_out</b>, and NUL-terminate it.
+ *  - memdup body into <b>*body_out</b>, and NUL-terminate it.
+ *  - Then remove them from <b>buf</b>, and return 1.
+ *
+ *  - If headers or body is NULL, discard that part of the buf.
+ *  - If a headers or body doesn't fit in the arg, return -1.
+ *  (We ensure that the headers or body don't exceed max len,
+ *   _even if_ we're planning to discard them.)
+ *  - If force_complete is true, then succeed even if not all of the
+ *    content has arrived.
+ *
+ * Else, change nothing and return 0.
+ */
+int
+fetch_from_buf_http(buf_t *buf,
+                    char **headers_out, size_t max_headerlen,
+                    char **body_out, size_t *body_used, size_t max_bodylen,
+                    int force_complete)
+{
+  char *headers;
+  size_t headerlen, bodylen, contentlen=0;
+  int crlf_offset;
+  int r;
+
+  if (!buf->head)
+    return 0;
+
+  crlf_offset = buf_find_string_offset(buf, "\r\n\r\n", 4);
+  if (crlf_offset > (int)max_headerlen ||
+      (crlf_offset < 0 && buf->datalen > max_headerlen)) {
+    log_debug(LD_HTTP,"headers too long.");
+    return -1;
+  } else if (crlf_offset < 0) {
+    log_debug(LD_HTTP,"headers not all here yet.");
+    return 0;
+  }
+  /* Okay, we have a full header.  Make sure it all appears in the first
+   * chunk. */
+  if ((int)buf->head->datalen < crlf_offset + 4)
+    buf_pullup(buf, crlf_offset+4);
+  headerlen = crlf_offset + 4;
+
+  headers = buf->head->data;
+  bodylen = buf->datalen - headerlen;
+  log_debug(LD_HTTP,"headerlen %d, bodylen %d.", (int)headerlen, (int)bodylen);
+
+  if (max_headerlen <= headerlen) {
+    log_warn(LD_HTTP,"headerlen %d larger than %d. Failing.",
+             (int)headerlen, (int)max_headerlen-1);
+    return -1;
+  }
+  if (max_bodylen <= bodylen) {
+    log_warn(LD_HTTP,"bodylen %d larger than %d. Failing.",
+             (int)bodylen, (int)max_bodylen-1);
+    return -1;
+  }
+
+  r = buf_http_find_content_length(headers, headerlen, &contentlen);
+  if (r == -1) {
+    log_warn(LD_PROTOCOL, "Content-Length is bogus; maybe "
+             "someone is trying to crash us.");
+    return -1;
+  } else if (r == 1) {
+    /* if content-length is malformed, then our body length is 0. fine. */
+    log_debug(LD_HTTP,"Got a contentlen of %d.",(int)contentlen);
+    if (bodylen < contentlen) {
+      if (!force_complete) {
+        log_debug(LD_HTTP,"body not all here yet.");
+        return 0; /* not all there yet */
+      }
+    }
+    if (bodylen > contentlen) {
+      bodylen = contentlen;
+      log_debug(LD_HTTP,"bodylen reduced to %d.",(int)bodylen);
+    }
+  } else {
+    tor_assert(r == 0);
+    /* Leave bodylen alone */
+  }
+
+  /* all happy. copy into the appropriate places, and return 1 */
+  if (headers_out) {
+    *headers_out = tor_malloc(headerlen+1);
+    fetch_from_buf(*headers_out, headerlen, buf);
+    (*headers_out)[headerlen] = 0; /* NUL terminate it */
+  }
+  if (body_out) {
+    tor_assert(body_used);
+    *body_used = bodylen;
+    *body_out = tor_malloc(bodylen+1);
+    fetch_from_buf(*body_out, bodylen, buf);
+    (*body_out)[bodylen] = 0; /* NUL terminate it */
+  }
+  return 1;
+}
+
+/**
+ * Scan the HTTP headers in the <b>headerlen</b>-byte memory range at
+ * <b>headers</b>, looking for a "Content-Length" header.  Try to set
+ * *<b>result_out</b> to the numeric value of that header if possible.
+ * Return -1 if the header was malformed, 0 if it was missing, and 1 if
+ * it was present and well-formed.
+ */
+STATIC int
+buf_http_find_content_length(const char *headers, size_t headerlen,
+                             size_t *result_out)
+{
+  const char *p, *newline;
+  char *len_str, *eos=NULL;
+  size_t remaining, result;
+  int ok;
+  *result_out = 0; /* The caller shouldn't look at this unless the
+                    * return value is 1, but let's prevent confusion */
+
+#define CONTENT_LENGTH "\r\nContent-Length: "
+  p = (char*) tor_memstr(headers, headerlen, CONTENT_LENGTH);
+  if (p == NULL)
+    return 0;
+
+  tor_assert(p >= headers && p < headers+headerlen);
+  remaining = (headers+headerlen)-p;
+  p += strlen(CONTENT_LENGTH);
+  remaining -= strlen(CONTENT_LENGTH);
+
+  newline = memchr(p, '\n', remaining);
+  if (newline == NULL)
+    return -1;
+
+  len_str = tor_memdup_nulterm(p, newline-p);
+  /* We limit the size to INT_MAX because other parts of the buffer.c
+   * code don't like buffers to be any bigger than that. */
+  result = (size_t) tor_parse_uint64(len_str, 10, 0, INT_MAX, &ok, &eos);
+  if (eos && !tor_strisspace(eos)) {
+    ok = 0;
+  } else {
+    *result_out = result;
+  }
+  tor_free(len_str);
+
+  return ok ? 1 : -1;
+}
+

src/or/proto_http.h

@@ -0,0 +1,24 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_HTTP_H
+#define TOR_PROTO_HTTP_H
+
+struct buf_t;
+
+int fetch_from_buf_http(struct buf_t *buf,
+                        char **headers_out, size_t max_headerlen,
+                        char **body_out, size_t *body_used, size_t max_bodylen,
+                        int force_complete);
+int peek_buf_has_http_command(const struct buf_t *buf);
+
+#ifdef PROTO_HTTP_PRIVATE
+STATIC int buf_http_find_content_length(const char *headers, size_t headerlen,
+                                        size_t *result_out);
+#endif
+
+#endif
+

src/or/proto_socks.c

@@ -0,0 +1,723 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define BUFFERS_PRIVATE // XXXX remove.
+#include "or.h"
+#include "addressmap.h"
+#include "buffers.h"
+#include "control.h"
+#include "config.h"
+#include "ext_orport.h"
+#include "proto_socks.h"
+#include "reasons.h"
+
+static void socks_request_set_socks5_error(socks_request_t *req,
+                              socks5_reply_status_t reason);
+
+static int parse_socks(const char *data, size_t datalen, socks_request_t *req,
+                       int log_sockstype, int safe_socks, ssize_t *drain_out,
+                       size_t *want_length_out);
+static int parse_socks_client(const uint8_t *data, size_t datalen,
+                              int state, char **reason,
+                              ssize_t *drain_out);
+/**
+ * Wait this many seconds before warning the user about using SOCKS unsafely
+ * again. */
+#define SOCKS_WARN_INTERVAL 5
+
+/** Warn that the user application has made an unsafe socks request using
+ * protocol <b>socks_protocol</b> on port <b>port</b>.  Don't warn more than
+ * once per SOCKS_WARN_INTERVAL, unless <b>safe_socks</b> is set. */
+static void
+log_unsafe_socks_warning(int socks_protocol, const char *address,
+                         uint16_t port, int safe_socks)
+{
+  static ratelim_t socks_ratelim = RATELIM_INIT(SOCKS_WARN_INTERVAL);
+
+  if (safe_socks) {
+    log_fn_ratelim(&socks_ratelim, LOG_WARN, LD_APP,
+             "Your application (using socks%d to port %d) is giving "
+             "Tor only an IP address. Applications that do DNS resolves "
+             "themselves may leak information. Consider using Socks4A "
+             "(e.g. via privoxy or socat) instead. For more information, "
+             "please see https://wiki.torproject.org/TheOnionRouter/"
+             "TorFAQ#SOCKSAndDNS.%s",
+             socks_protocol,
+             (int)port,
+             safe_socks ? " Rejecting." : "");
+  }
+  control_event_client_status(LOG_WARN,
+                              "DANGEROUS_SOCKS PROTOCOL=SOCKS%d ADDRESS=%s:%d",
+                              socks_protocol, address, (int)port);
+}
+
+/** Do not attempt to parse socks messages longer than this.  This value is
+ * actually significantly higher than the longest possible socks message. */
+#define MAX_SOCKS_MESSAGE_LEN 512
+
+/** Return a new socks_request_t. */
+socks_request_t *
+socks_request_new(void)
+{
+  return tor_malloc_zero(sizeof(socks_request_t));
+}
+
+/** Free all storage held in the socks_request_t <b>req</b>. */
+void
+socks_request_free(socks_request_t *req)
+{
+  if (!req)
+    return;
+  if (req->username) {
+    memwipe(req->username, 0x10, req->usernamelen);
+    tor_free(req->username);
+  }
+  if (req->password) {
+    memwipe(req->password, 0x04, req->passwordlen);
+    tor_free(req->password);
+  }
+  memwipe(req, 0xCC, sizeof(socks_request_t));
+  tor_free(req);
+}
+
+/** There is a (possibly incomplete) socks handshake on <b>buf</b>, of one
+ * of the forms
+ *  - socks4: "socksheader username\\0"
+ *  - socks4a: "socksheader username\\0 destaddr\\0"
+ *  - socks5 phase one: "version #methods methods"
+ *  - socks5 phase two: "version command 0 addresstype..."
+ * If it's a complete and valid handshake, and destaddr fits in
+ *   MAX_SOCKS_ADDR_LEN bytes, then pull the handshake off the buf,
+ *   assign to <b>req</b>, and return 1.
+ *
+ * If it's invalid or too big, return -1.
+ *
+ * Else it's not all there yet, leave buf alone and return 0.
+ *
+ * If you want to specify the socks reply, write it into <b>req->reply</b>
+ *   and set <b>req->replylen</b>, else leave <b>req->replylen</b> alone.
+ *
+ * If <b>log_sockstype</b> is non-zero, then do a notice-level log of whether
+ * the connection is possibly leaking DNS requests locally or not.
+ *
+ * If <b>safe_socks</b> is true, then reject unsafe socks protocols.
+ *
+ * If returning 0 or -1, <b>req->address</b> and <b>req->port</b> are
+ * undefined.
+ */
+int
+fetch_from_buf_socks(buf_t *buf, socks_request_t *req,
+                     int log_sockstype, int safe_socks)
+{
+  int res;
+  ssize_t n_drain;
+  size_t want_length = 128;
+
+  if (buf->datalen < 2) /* version and another byte */
+    return 0;
+
+  do {
+    n_drain = 0;
+    buf_pullup(buf, want_length);
+    tor_assert(buf->head && buf->head->datalen >= 2);
+    want_length = 0;
+
+    res = parse_socks(buf->head->data, buf->head->datalen, req, log_sockstype,
+                      safe_socks, &n_drain, &want_length);
+
+    if (n_drain < 0)
+      buf_clear(buf);
+    else if (n_drain > 0)
+      buf_remove_from_front(buf, n_drain);
+
+  } while (res == 0 && buf->head && want_length < buf->datalen &&
+           buf->datalen >= 2);
+
+  return res;
+}
+
+/** The size of the header of an Extended ORPort message: 2 bytes for
+ *  COMMAND, 2 bytes for BODYLEN */
+#define EXT_OR_CMD_HEADER_SIZE 4
+
+/** Read <b>buf</b>, which should contain an Extended ORPort message
+ *  from a transport proxy. If well-formed, create and populate
+ *  <b>out</b> with the Extended ORport message. Return 0 if the
+ *  buffer was incomplete, 1 if it was well-formed and -1 if we
+ *  encountered an error while parsing it.  */
+int
+fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out)
+{
+  char hdr[EXT_OR_CMD_HEADER_SIZE];
+  uint16_t len;
+
+  if (buf->datalen < EXT_OR_CMD_HEADER_SIZE)
+    return 0;
+  peek_from_buf(hdr, sizeof(hdr), buf);
+  len = ntohs(get_uint16(hdr+2));
+  if (buf->datalen < (unsigned)len + EXT_OR_CMD_HEADER_SIZE)
+    return 0;
+  *out = ext_or_cmd_new(len);
+  (*out)->cmd = ntohs(get_uint16(hdr));
+  (*out)->len = len;
+  buf_remove_from_front(buf, EXT_OR_CMD_HEADER_SIZE);
+  fetch_from_buf((*out)->body, len, buf);
+  return 1;
+}
+
+/** Create a SOCKS5 reply message with <b>reason</b> in its REP field and
+ * have Tor send it as error response to <b>req</b>.
+ */
+static void
+socks_request_set_socks5_error(socks_request_t *req,
+                  socks5_reply_status_t reason)
+{
+   req->replylen = 10;
+   memset(req->reply,0,10);
+
+   req->reply[0] = 0x05;   // VER field.
+   req->reply[1] = reason; // REP field.
+   req->reply[3] = 0x01;   // ATYP field.
+}
+
+static const char SOCKS_PROXY_IS_NOT_AN_HTTP_PROXY_MSG[] =
+  "HTTP/1.0 501 Tor is not an HTTP Proxy\r\n"
+  "Content-Type: text/html; charset=iso-8859-1\r\n\r\n"
+  "<html>\n"
+  "<head>\n"
+  "<title>Tor is not an HTTP Proxy</title>\n"
+  "</head>\n"
+  "<body>\n"
+  "<h1>Tor is not an HTTP Proxy</h1>\n"
+  "<p>\n"
+  "It appears you have configured your web browser to use Tor as "
+  "an HTTP proxy.\n\n"
+  "This is not correct: Tor is a SOCKS proxy, not an HTTP proxy.\n"
+  "Please configure your client accordingly.\n"
+  "</p>\n"
+  "<p>\n"
+  "See <a href=\"https://www.torproject.org/documentation.html\">"
+  "https://www.torproject.org/documentation.html</a> for more "
+  "information.\n"
+  "<!-- Plus this comment, to make the body response more than 512 bytes, so "
+  "     IE will be willing to display it. Comment comment comment comment "
+  "     comment comment comment comment comment comment comment comment.-->\n"
+  "</p>\n"
+  "</body>\n"
+  "</html>\n";
+
+/** Implementation helper to implement fetch_from_*_socks.  Instead of looking
+ * at a buffer's contents, we look at the <b>datalen</b> bytes of data in
+ * <b>data</b>. Instead of removing data from the buffer, we set
+ * <b>drain_out</b> to the amount of data that should be removed (or -1 if the
+ * buffer should be cleared).  Instead of pulling more data into the first
+ * chunk of the buffer, we set *<b>want_length_out</b> to the number of bytes
+ * we'd like to see in the input buffer, if they're available. */
+static int
+parse_socks(const char *data, size_t datalen, socks_request_t *req,
+            int log_sockstype, int safe_socks, ssize_t *drain_out,
+            size_t *want_length_out)
+{
+  unsigned int len;
+  char tmpbuf[TOR_ADDR_BUF_LEN+1];
+  tor_addr_t destaddr;
+  uint32_t destip;
+  uint8_t socksver;
+  char *next, *startaddr;
+  unsigned char usernamelen, passlen;
+  struct in_addr in;
+
+  if (datalen < 2) {
+    /* We always need at least 2 bytes. */
+    *want_length_out = 2;
+    return 0;
+  }
+
+  if (req->socks_version == 5 && !req->got_auth) {
+    /* See if we have received authentication.  Strictly speaking, we should
+       also check whether we actually negotiated username/password
+       authentication.  But some broken clients will send us authentication
+       even if we negotiated SOCKS_NO_AUTH. */
+    if (*data == 1) { /* username/pass version 1 */
+      /* Format is: authversion [1 byte] == 1
+                    usernamelen [1 byte]
+                    username    [usernamelen bytes]
+                    passlen     [1 byte]
+                    password    [passlen bytes] */
+      usernamelen = (unsigned char)*(data + 1);
+      if (datalen < 2u + usernamelen + 1u) {
+        *want_length_out = 2u + usernamelen + 1u;
+        return 0;
+      }
+      passlen = (unsigned char)*(data + 2u + usernamelen);
+      if (datalen < 2u + usernamelen + 1u + passlen) {
+        *want_length_out = 2u + usernamelen + 1u + passlen;
+        return 0;
+      }
+      req->replylen = 2; /* 2 bytes of response */
+      req->reply[0] = 1; /* authversion == 1 */
+      req->reply[1] = 0; /* authentication successful */
+      log_debug(LD_APP,
+               "socks5: Accepted username/password without checking.");
+      if (usernamelen) {
+        req->username = tor_memdup(data+2u, usernamelen);
+        req->usernamelen = usernamelen;
+      }
+      if (passlen) {
+        req->password = tor_memdup(data+3u+usernamelen, passlen);
+        req->passwordlen = passlen;
+      }
+      *drain_out = 2u + usernamelen + 1u + passlen;
+      req->got_auth = 1;
+      *want_length_out = 7; /* Minimal socks5 command. */
+      return 0;
+    } else if (req->auth_type == SOCKS_USER_PASS) {
+      /* unknown version byte */
+      log_warn(LD_APP, "Socks5 username/password version %d not recognized; "
+               "rejecting.", (int)*data);
+      return -1;
+    }
+  }
+
+  socksver = *data;
+
+  switch (socksver) { /* which version of socks? */
+    case 5: /* socks5 */
+
+      if (req->socks_version != 5) { /* we need to negotiate a method */
+        unsigned char nummethods = (unsigned char)*(data+1);
+        int have_user_pass, have_no_auth;
+        int r=0;
+        tor_assert(!req->socks_version);
+        if (datalen < 2u+nummethods) {
+          *want_length_out = 2u+nummethods;
+          return 0;
+        }
+        if (!nummethods)
+          return -1;
+        req->replylen = 2; /* 2 bytes of response */
+        req->reply[0] = 5; /* socks5 reply */
+        have_user_pass = (memchr(data+2, SOCKS_USER_PASS, nummethods) !=NULL);
+        have_no_auth   = (memchr(data+2, SOCKS_NO_AUTH,   nummethods) !=NULL);
+        if (have_user_pass && !(have_no_auth && req->socks_prefer_no_auth)) {
+          req->auth_type = SOCKS_USER_PASS;
+          req->reply[1] = SOCKS_USER_PASS; /* tell client to use "user/pass"
+                                              auth method */
+          req->socks_version = 5; /* remember we've already negotiated auth */
+          log_debug(LD_APP,"socks5: accepted method 2 (username/password)");
+          r=0;
+        } else if (have_no_auth) {
+          req->reply[1] = SOCKS_NO_AUTH; /* tell client to use "none" auth
+                                            method */
+          req->socks_version = 5; /* remember we've already negotiated auth */
+          log_debug(LD_APP,"socks5: accepted method 0 (no authentication)");
+          r=0;
+        } else {
+          log_warn(LD_APP,
+                    "socks5: offered methods don't include 'no auth' or "
+                    "username/password. Rejecting.");
+          req->reply[1] = '\xFF'; /* reject all methods */
+          r=-1;
+        }
+        /* Remove packet from buf. Some SOCKS clients will have sent extra
+         * junk at this point; let's hope it's an authentication message. */
+        *drain_out = 2u + nummethods;
+
+        return r;
+      }
+      if (req->auth_type != SOCKS_NO_AUTH && !req->got_auth) {
+        log_warn(LD_APP,
+                 "socks5: negotiated authentication, but none provided");
+        return -1;
+      }
+      /* we know the method; read in the request */
+      log_debug(LD_APP,"socks5: checking request");
+      if (datalen < 7) {/* basic info plus >=1 for addr plus 2 for port */
+        *want_length_out = 7;
+        return 0; /* not yet */
+      }
+      req->command = (unsigned char) *(data+1);
+      if (req->command != SOCKS_COMMAND_CONNECT &&
+          req->command != SOCKS_COMMAND_RESOLVE &&
+          req->command != SOCKS_COMMAND_RESOLVE_PTR) {
+        /* not a connect or resolve or a resolve_ptr? we don't support it. */
+        socks_request_set_socks5_error(req,SOCKS5_COMMAND_NOT_SUPPORTED);
+
+        log_warn(LD_APP,"socks5: command %d not recognized. Rejecting.",
+                 req->command);
+        return -1;
+      }
+      switch (*(data+3)) { /* address type */
+        case 1: /* IPv4 address */
+        case 4: /* IPv6 address */ {
+          const int is_v6 = *(data+3) == 4;
+          const unsigned addrlen = is_v6 ? 16 : 4;
+          log_debug(LD_APP,"socks5: ipv4 address type");
+          if (datalen < 6+addrlen) {/* ip/port there? */
+            *want_length_out = 6+addrlen;
+            return 0; /* not yet */
+          }
+
+          if (is_v6)
+            tor_addr_from_ipv6_bytes(&destaddr, data+4);
+          else
+            tor_addr_from_ipv4n(&destaddr, get_uint32(data+4));
+
+          tor_addr_to_str(tmpbuf, &destaddr, sizeof(tmpbuf), 1);
+
+          if (strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN) {
+            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
+            log_warn(LD_APP,
+                     "socks5 IP takes %d bytes, which doesn't fit in %d. "
+                     "Rejecting.",
+                     (int)strlen(tmpbuf)+1,(int)MAX_SOCKS_ADDR_LEN);
+            return -1;
+          }
+          strlcpy(req->address,tmpbuf,sizeof(req->address));
+          req->port = ntohs(get_uint16(data+4+addrlen));
+          *drain_out = 6+addrlen;
+          if (req->command != SOCKS_COMMAND_RESOLVE_PTR &&
+              !addressmap_have_mapping(req->address,0)) {
+            log_unsafe_socks_warning(5, req->address, req->port, safe_socks);
+            if (safe_socks) {
+              socks_request_set_socks5_error(req, SOCKS5_NOT_ALLOWED);
+              return -1;
+            }
+          }
+          return 1;
+        }
+        case 3: /* fqdn */
+          log_debug(LD_APP,"socks5: fqdn address type");
+          if (req->command == SOCKS_COMMAND_RESOLVE_PTR) {
+            socks_request_set_socks5_error(req,
+                                           SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED);
+            log_warn(LD_APP, "socks5 received RESOLVE_PTR command with "
+                     "hostname type. Rejecting.");
+            return -1;
+          }
+          len = (unsigned char)*(data+4);
+          if (datalen < 7+len) { /* addr/port there? */
+            *want_length_out = 7+len;
+            return 0; /* not yet */
+          }
+          if (len+1 > MAX_SOCKS_ADDR_LEN) {
+            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
+            log_warn(LD_APP,
+                     "socks5 hostname is %d bytes, which doesn't fit in "
+                     "%d. Rejecting.", len+1,MAX_SOCKS_ADDR_LEN);
+            return -1;
+          }
+          memcpy(req->address,data+5,len);
+          req->address[len] = 0;
+          req->port = ntohs(get_uint16(data+5+len));
+          *drain_out = 5+len+2;
+
+          if (!string_is_valid_hostname(req->address)) {
+            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
+
+            log_warn(LD_PROTOCOL,
+                     "Your application (using socks5 to port %d) gave Tor "
+                     "a malformed hostname: %s. Rejecting the connection.",
+                     req->port, escaped_safe_str_client(req->address));
+            return -1;
+          }
+          if (log_sockstype)
+            log_notice(LD_APP,
+                  "Your application (using socks5 to port %d) instructed "
+                  "Tor to take care of the DNS resolution itself if "
+                  "necessary. This is good.", req->port);
+          return 1;
+        default: /* unsupported */
+          socks_request_set_socks5_error(req,
+                                         SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED);
+          log_warn(LD_APP,"socks5: unsupported address type %d. Rejecting.",
+                   (int) *(data+3));
+          return -1;
+      }
+      tor_assert(0);
+      break;
+    case 4: { /* socks4 */
+      enum {socks4, socks4a} socks4_prot = socks4a;
+      const char *authstart, *authend;
+      /* http://ss5.sourceforge.net/socks4.protocol.txt */
+      /* http://ss5.sourceforge.net/socks4A.protocol.txt */
+
+      req->socks_version = 4;
+      if (datalen < SOCKS4_NETWORK_LEN) {/* basic info available? */
+        *want_length_out = SOCKS4_NETWORK_LEN;
+        return 0; /* not yet */
+      }
+      // buf_pullup(buf, 1280);
+      req->command = (unsigned char) *(data+1);
+      if (req->command != SOCKS_COMMAND_CONNECT &&
+          req->command != SOCKS_COMMAND_RESOLVE) {
+        /* not a connect or resolve? we don't support it. (No resolve_ptr with
+         * socks4.) */
+        log_warn(LD_APP,"socks4: command %d not recognized. Rejecting.",
+                 req->command);
+        return -1;
+      }
+
+      req->port = ntohs(get_uint16(data+2));
+      destip = ntohl(get_uint32(data+4));
+      if ((!req->port && req->command!=SOCKS_COMMAND_RESOLVE) || !destip) {
+        log_warn(LD_APP,"socks4: Port or DestIP is zero. Rejecting.");
+        return -1;
+      }
+      if (destip >> 8) {
+        log_debug(LD_APP,"socks4: destip not in form 0.0.0.x.");
+        in.s_addr = htonl(destip);
+        tor_inet_ntoa(&in,tmpbuf,sizeof(tmpbuf));
+        if (strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN) {
+          log_debug(LD_APP,"socks4 addr (%d bytes) too long. Rejecting.",
+                    (int)strlen(tmpbuf));
+          return -1;
+        }
+        log_debug(LD_APP,
+                  "socks4: successfully read destip (%s)",
+                  safe_str_client(tmpbuf));
+        socks4_prot = socks4;
+      }
+
+      authstart = data + SOCKS4_NETWORK_LEN;
+      next = memchr(authstart, 0,
+                    datalen-SOCKS4_NETWORK_LEN);
+      if (!next) {
+        if (datalen >= 1024) {
+          log_debug(LD_APP, "Socks4 user name too long; rejecting.");
+          return -1;
+        }
+        log_debug(LD_APP,"socks4: Username not here yet.");
+        *want_length_out = datalen+1024; /* More than we need, but safe */
+        return 0;
+      }
+      authend = next;
+      tor_assert(next < data+datalen);
+
+      startaddr = NULL;
+      if (socks4_prot != socks4a &&
+          !addressmap_have_mapping(tmpbuf,0)) {
+        log_unsafe_socks_warning(4, tmpbuf, req->port, safe_socks);
+
+        if (safe_socks)
+          return -1;
+      }
+      if (socks4_prot == socks4a) {
+        if (next+1 == data+datalen) {
+          log_debug(LD_APP,"socks4: No part of destaddr here yet.");
+          *want_length_out = datalen + 1024; /* More than we need, but safe */
+          return 0;
+        }
+        startaddr = next+1;
+        next = memchr(startaddr, 0, data + datalen - startaddr);
+        if (!next) {
+          if (datalen >= 1024) {
+            log_debug(LD_APP,"socks4: Destaddr too long.");
+            return -1;
+          }
+          log_debug(LD_APP,"socks4: Destaddr not all here yet.");
+          *want_length_out = datalen + 1024; /* More than we need, but safe */
+          return 0;
+        }
+        if (MAX_SOCKS_ADDR_LEN <= next-startaddr) {
+          log_warn(LD_APP,"socks4: Destaddr too long. Rejecting.");
+          return -1;
+        }
+        // tor_assert(next < buf->cur+buf->datalen);
+
+        if (log_sockstype)
+          log_notice(LD_APP,
+                     "Your application (using socks4a to port %d) instructed "
+                     "Tor to take care of the DNS resolution itself if "
+                     "necessary. This is good.", req->port);
+      }
+      log_debug(LD_APP,"socks4: Everything is here. Success.");
+      strlcpy(req->address, startaddr ? startaddr : tmpbuf,
+              sizeof(req->address));
+      if (!string_is_valid_hostname(req->address)) {
+        log_warn(LD_PROTOCOL,
+                 "Your application (using socks4 to port %d) gave Tor "
+                 "a malformed hostname: %s. Rejecting the connection.",
+                 req->port, escaped_safe_str_client(req->address));
+        return -1;
+      }
+      if (authend != authstart) {
+        req->got_auth = 1;
+        req->usernamelen = authend - authstart;
+        req->username = tor_memdup(authstart, authend - authstart);
+      }
+      /* next points to the final \0 on inbuf */
+      *drain_out = next - data + 1;
+      return 1;
+    }
+    case 'G': /* get */
+    case 'H': /* head */
+    case 'P': /* put/post */
+    case 'C': /* connect */
+      strlcpy((char*)req->reply, SOCKS_PROXY_IS_NOT_AN_HTTP_PROXY_MSG,
+              MAX_SOCKS_REPLY_LEN);
+      req->replylen = strlen((char*)req->reply)+1;
+      /* fall through */
+    default: /* version is not socks4 or socks5 */
+      log_warn(LD_APP,
+               "Socks version %d not recognized. (Tor is not an http proxy.)",
+               *(data));
+      {
+        /* Tell the controller the first 8 bytes. */
+        char *tmp = tor_strndup(data, datalen < 8 ? datalen : 8);
+        control_event_client_status(LOG_WARN,
+                                    "SOCKS_UNKNOWN_PROTOCOL DATA=\"%s\"",
+                                    escaped(tmp));
+        tor_free(tmp);
+      }
+      return -1;
+  }
+}
+
+/** Inspect a reply from SOCKS server stored in <b>buf</b> according
+ * to <b>state</b>, removing the protocol data upon success. Return 0 on
+ * incomplete response, 1 on success and -1 on error, in which case
+ * <b>reason</b> is set to a descriptive message (free() when finished
+ * with it).
+ *
+ * As a special case, 2 is returned when user/pass is required
+ * during SOCKS5 handshake and user/pass is configured.
+ */
+int
+fetch_from_buf_socks_client(buf_t *buf, int state, char **reason)
+{
+  ssize_t drain = 0;
+  int r;
+  if (buf->datalen < 2)
+    return 0;
+
+  buf_pullup(buf, MAX_SOCKS_MESSAGE_LEN);
+  tor_assert(buf->head && buf->head->datalen >= 2);
+
+  r = parse_socks_client((uint8_t*)buf->head->data, buf->head->datalen,
+                         state, reason, &drain);
+  if (drain > 0)
+    buf_remove_from_front(buf, drain);
+  else if (drain < 0)
+    buf_clear(buf);
+
+  return r;
+}
+
+/** Implementation logic for fetch_from_*_socks_client. */
+static int
+parse_socks_client(const uint8_t *data, size_t datalen,
+                   int state, char **reason,
+                   ssize_t *drain_out)
+{
+  unsigned int addrlen;
+  *drain_out = 0;
+  if (datalen < 2)
+    return 0;
+
+  switch (state) {
+    case PROXY_SOCKS4_WANT_CONNECT_OK:
+      /* Wait for the complete response */
+      if (datalen < 8)
+        return 0;
+
+      if (data[1] != 0x5a) {
+        *reason = tor_strdup(socks4_response_code_to_string(data[1]));
+        return -1;
+      }
+
+      /* Success */
+      *drain_out = 8;
+      return 1;
+
+    case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE:
+      /* we don't have any credentials */
+      if (data[1] != 0x00) {
+        *reason = tor_strdup("server doesn't support any of our "
+                             "available authentication methods");
+        return -1;
+      }
+
+      log_info(LD_NET, "SOCKS 5 client: continuing without authentication");
+      *drain_out = -1;
+      return 1;
+
+    case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929:
+      /* we have a username and password. return 1 if we can proceed without
+       * providing authentication, or 2 otherwise. */
+      switch (data[1]) {
+        case 0x00:
+          log_info(LD_NET, "SOCKS 5 client: we have auth details but server "
+                            "doesn't require authentication.");
+          *drain_out = -1;
+          return 1;
+        case 0x02:
+          log_info(LD_NET, "SOCKS 5 client: need authentication.");
+          *drain_out = -1;
+          return 2;
+        /* fall through */
+      }
+
+      *reason = tor_strdup("server doesn't support any of our available "
+                           "authentication methods");
+      return -1;
+
+    case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK:
+      /* handle server reply to rfc1929 authentication */
+      if (data[1] != 0x00) {
+        *reason = tor_strdup("authentication failed");
+        return -1;
+      }
+
+      log_info(LD_NET, "SOCKS 5 client: authentication successful.");
+      *drain_out = -1;
+      return 1;
+
+    case PROXY_SOCKS5_WANT_CONNECT_OK:
+      /* response is variable length. BND.ADDR, etc, isn't needed
+       * (don't bother with buf_pullup()), but make sure to eat all
+       * the data used */
+
+      /* wait for address type field to arrive */
+      if (datalen < 4)
+        return 0;
+
+      switch (data[3]) {
+        case 0x01: /* ip4 */
+          addrlen = 4;
+          break;
+        case 0x04: /* ip6 */
+          addrlen = 16;
+          break;
+        case 0x03: /* fqdn (can this happen here?) */
+          if (datalen < 5)
+            return 0;
+          addrlen = 1 + data[4];
+          break;
+        default:
+          *reason = tor_strdup("invalid response to connect request");
+          return -1;
+      }
+
+      /* wait for address and port */
+      if (datalen < 6 + addrlen)
+        return 0;
+
+      if (data[1] != 0x00) {
+        *reason = tor_strdup(socks5_response_code_to_string(data[1]));
+        return -1;
+      }
+
+      *drain_out = 6 + addrlen;
+      return 1;
+  }
+
+  /* shouldn't get here... */
+  tor_assert(0);
+
+  return -1;
+}
+

src/or/proto_socks.h

@@ -0,0 +1,20 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_SOCKS_H
+#define TOR_PROTO_SOCKS_H
+
+struct socks_request_t;
+struct buf_t;
+
+struct socks_request_t *socks_request_new(void);
+void socks_request_free(struct socks_request_t *req);
+int fetch_from_buf_socks(struct buf_t *buf, socks_request_t *req,
+                         int log_sockstype, int safe_socks);
+int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason);
+
+#endif
+

src/test/test_buffers.c

@@ -4,9 +4,14 @@
 /* See LICENSE for licensing information */
 
 #define BUFFERS_PRIVATE
+#define PROTO_HTTP_PRIVATE
 #include "or.h"
 #include "buffers.h"
 #include "ext_orport.h"
+#include "proto_cell.h"
+#include "proto_http.h"
+#include "proto_control0.h"
+#include "proto_socks.h"
 #include "test.h"
 
 /** Run unit tests for buffers.c */

src/test/test_dir_handle_get.c

@@ -28,6 +28,7 @@
 #include "entrynodes.h"
 #include "routerparse.h"
 #include "networkstatus.h"
+#include "proto_http.h"
 #include "geoip.h"
 #include "dirserv.h"
 #include "dirvote.h"

src/test/test_hs_cache.c

@@ -15,6 +15,7 @@
 #include "rendcache.h"
 #include "directory.h"
 #include "connection.h"
+#include "proto_http.h"
 
 #include "hs_test_helpers.h"
 #include "test_helpers.h"

src/test/test_socks.c

@@ -6,6 +6,7 @@
 #include "or.h"
 #include "buffers.h"
 #include "config.h"
+#include "proto_socks.h"
 #include "test.h"
 
 typedef struct socks_test_data_t {