Merge branch 'refactor_buffers_api_3'

changes/refactor-buffer

@@ -0,0 +1,3 @@
+  o Code simplifications and refactoring:
+    - Split the portions of the buffer.c module that handle particular
+      protocols into separate modules. Part of ticket 23149.

src/common/buffers.c

@@ -0,0 +1,1069 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file buffers.c
+ * \brief Implements a generic buffer interface.
+ *
+ * A buf_t is a (fairly) opaque byte-oriented FIFO that can read to or flush
+ * from memory, sockets, file descriptors, TLS connections, or another buf_t.
+ * Buffers are implemented as linked lists of memory chunks.
+ *
+ * All socket-backed and TLS-based connection_t objects have a pair of
+ * buffers: one for incoming data, and one for outcoming data.  These are fed
+ * and drained from functions in connection.c, trigged by events that are
+ * monitored in main.c.
+ **/
+
+#define BUFFERS_PRIVATE
+#include "orconfig.h"
+#include <stddef.h>
+#include "buffers.h"
+#include "compat.h"
+#include "compress.h"
+#include "util.h"
+#include "torint.h"
+#include "torlog.h"
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+//#define PARANOIA
+
+#ifdef PARANOIA
+/** Helper: If PARANOIA is defined, assert that the buffer in local variable
+ * <b>buf</b> is well-formed. */
+#define check() STMT_BEGIN buf_assert_ok(buf); STMT_END
+#else
+#define check() STMT_NIL
+#endif
+
+/* Implementation notes:
+ *
+ * After flirting with memmove, and dallying with ring-buffers, we're finally
+ * getting up to speed with the 1970s and implementing buffers as a linked
+ * list of small chunks.  Each buffer has such a list; data is removed from
+ * the head of the list, and added at the tail.  The list is singly linked,
+ * and the buffer keeps a pointer to the head and the tail.
+ *
+ * Every chunk, except the tail, contains at least one byte of data.  Data in
+ * each chunk is contiguous.
+ *
+ * When you need to treat the first N characters on a buffer as a contiguous
+ * string, use the buf_pullup function to make them so.  Don't do this more
+ * than necessary.
+ *
+ * The major free Unix kernels have handled buffers like this since, like,
+ * forever.
+ */
+
+/* Chunk manipulation functions */
+
+#define CHUNK_HEADER_LEN offsetof(chunk_t, mem[0])
+
+/* We leave this many NUL bytes at the end of the buffer. */
+#ifdef DISABLE_MEMORY_SENTINELS
+#define SENTINEL_LEN 0
+#else
+#define SENTINEL_LEN 4
+#endif
+
+/* Header size plus NUL bytes at the end */
+#define CHUNK_OVERHEAD (CHUNK_HEADER_LEN + SENTINEL_LEN)
+
+/** Return the number of bytes needed to allocate a chunk to hold
+ * <b>memlen</b> bytes. */
+#define CHUNK_ALLOC_SIZE(memlen) (CHUNK_OVERHEAD + (memlen))
+/** Return the number of usable bytes in a chunk allocated with
+ * malloc(<b>memlen</b>). */
+#define CHUNK_SIZE_WITH_ALLOC(memlen) ((memlen) - CHUNK_OVERHEAD)
+
+#define DEBUG_SENTINEL
+
+#if defined(DEBUG_SENTINEL) && !defined(DISABLE_MEMORY_SENTINELS)
+#define DBG_S(s) s
+#else
+#define DBG_S(s) (void)0
+#endif
+
+#ifdef DISABLE_MEMORY_SENTINELS
+#define CHUNK_SET_SENTINEL(chunk, alloclen) STMT_NIL
+#else
+#define CHUNK_SET_SENTINEL(chunk, alloclen) do {                        \
+    uint8_t *a = (uint8_t*) &(chunk)->mem[(chunk)->memlen];             \
+    DBG_S(uint8_t *b = &((uint8_t*)(chunk))[(alloclen)-SENTINEL_LEN]);  \
+    DBG_S(tor_assert(a == b));                                          \
+    memset(a,0,SENTINEL_LEN);                                           \
+  } while (0)
+#endif
+
+/** Move all bytes stored in <b>chunk</b> to the front of <b>chunk</b>->mem,
+ * to free up space at the end. */
+static inline void
+chunk_repack(chunk_t *chunk)
+{
+  if (chunk->datalen && chunk->data != &chunk->mem[0]) {
+    memmove(chunk->mem, chunk->data, chunk->datalen);
+  }
+  chunk->data = &chunk->mem[0];
+}
+
+/** Keep track of total size of allocated chunks for consistency asserts */
+static size_t total_bytes_allocated_in_chunks = 0;
+static void
+buf_chunk_free_unchecked(chunk_t *chunk)
+{
+  if (!chunk)
+    return;
+#ifdef DEBUG_CHUNK_ALLOC
+  tor_assert(CHUNK_ALLOC_SIZE(chunk->memlen) == chunk->DBG_alloc);
+#endif
+  tor_assert(total_bytes_allocated_in_chunks >=
+             CHUNK_ALLOC_SIZE(chunk->memlen));
+  total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen);
+  tor_free(chunk);
+}
+static inline chunk_t *
+chunk_new_with_alloc_size(size_t alloc)
+{
+  chunk_t *ch;
+  ch = tor_malloc(alloc);
+  ch->next = NULL;
+  ch->datalen = 0;
+#ifdef DEBUG_CHUNK_ALLOC
+  ch->DBG_alloc = alloc;
+#endif
+  ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc);
+  total_bytes_allocated_in_chunks += alloc;
+  ch->data = &ch->mem[0];
+  CHUNK_SET_SENTINEL(ch, alloc);
+  return ch;
+}
+
+/** Expand <b>chunk</b> until it can hold <b>sz</b> bytes, and return a
+ * new pointer to <b>chunk</b>.  Old pointers are no longer valid. */
+static inline chunk_t *
+chunk_grow(chunk_t *chunk, size_t sz)
+{
+  off_t offset;
+  const size_t memlen_orig = chunk->memlen;
+  const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig);
+  const size_t new_alloc = CHUNK_ALLOC_SIZE(sz);
+  tor_assert(sz > chunk->memlen);
+  offset = chunk->data - chunk->mem;
+  chunk = tor_realloc(chunk, new_alloc);
+  chunk->memlen = sz;
+  chunk->data = chunk->mem + offset;
+#ifdef DEBUG_CHUNK_ALLOC
+  tor_assert(chunk->DBG_alloc == orig_alloc);
+  chunk->DBG_alloc = new_alloc;
+#endif
+  total_bytes_allocated_in_chunks += new_alloc - orig_alloc;
+  CHUNK_SET_SENTINEL(chunk, new_alloc);
+  return chunk;
+}
+
+/** Every chunk should take up at least this many bytes. */
+#define MIN_CHUNK_ALLOC 256
+/** No chunk should take up more than this many bytes. */
+#define MAX_CHUNK_ALLOC 65536
+
+/** Return the allocation size we'd like to use to hold <b>target</b>
+ * bytes. */
+STATIC size_t
+preferred_chunk_size(size_t target)
+{
+  tor_assert(target <= SIZE_T_CEILING - CHUNK_OVERHEAD);
+  if (CHUNK_ALLOC_SIZE(target) >= MAX_CHUNK_ALLOC)
+    return CHUNK_ALLOC_SIZE(target);
+  size_t sz = MIN_CHUNK_ALLOC;
+  while (CHUNK_SIZE_WITH_ALLOC(sz) < target) {
+    sz <<= 1;
+  }
+  return sz;
+}
+
+/** Collapse data from the first N chunks from <b>buf</b> into buf->head,
+ * growing it as necessary, until buf->head has the first <b>bytes</b> bytes
+ * of data from the buffer, or until buf->head has all the data in <b>buf</b>.
+ *
+ * Set *<b>head_out</b> to point to the first byte of available data, and
+ * *<b>len_out</b> to the number of bytes of data available at
+ * *<b>head_out</b>. Note that *<b>len_out</b> may be more or less than
+ * <b>bytes</b>, depending on the number of bytes available.
+ */
+void
+buf_pullup(buf_t *buf, size_t bytes, const char **head_out, size_t *len_out)
+{
+  chunk_t *dest, *src;
+  size_t capacity;
+  if (!buf->head) {
+    *head_out = NULL;
+    *len_out = 0;
+    return;
+  }
+
+  check();
+  if (buf->datalen < bytes)
+    bytes = buf->datalen;
+
+  capacity = bytes;
+  if (buf->head->datalen >= bytes) {
+    *head_out = buf->head->data;
+    *len_out = buf->head->datalen;
+    return;
+  }
+
+  if (buf->head->memlen >= capacity) {
+    /* We don't need to grow the first chunk, but we might need to repack it.*/
+    size_t needed = capacity - buf->head->datalen;
+    if (CHUNK_REMAINING_CAPACITY(buf->head) < needed)
+      chunk_repack(buf->head);
+    tor_assert(CHUNK_REMAINING_CAPACITY(buf->head) >= needed);
+  } else {
+    chunk_t *newhead;
+    size_t newsize;
+    /* We need to grow the chunk. */
+    chunk_repack(buf->head);
+    newsize = CHUNK_SIZE_WITH_ALLOC(preferred_chunk_size(capacity));
+    newhead = chunk_grow(buf->head, newsize);
+    tor_assert(newhead->memlen >= capacity);
+    if (newhead != buf->head) {
+      if (buf->tail == buf->head)
+        buf->tail = newhead;
+      buf->head = newhead;
+    }
+  }
+
+  dest = buf->head;
+  while (dest->datalen < bytes) {
+    size_t n = bytes - dest->datalen;
+    src = dest->next;
+    tor_assert(src);
+    if (n >= src->datalen) {
+      memcpy(CHUNK_WRITE_PTR(dest), src->data, src->datalen);
+      dest->datalen += src->datalen;
+      dest->next = src->next;
+      if (buf->tail == src)
+        buf->tail = dest;
+      buf_chunk_free_unchecked(src);
+    } else {
+      memcpy(CHUNK_WRITE_PTR(dest), src->data, n);
+      dest->datalen += n;
+      src->data += n;
+      src->datalen -= n;
+      tor_assert(dest->datalen == bytes);
+    }
+  }
+
+  check();
+  *head_out = buf->head->data;
+  *len_out = buf->head->datalen;
+}
+
+#ifdef TOR_UNIT_TESTS
+/* Write sz bytes from cp into a newly allocated buffer buf.
+ * Returns NULL when passed a NULL cp or zero sz.
+ * Asserts on failure: only for use in unit tests.
+ * buf must be freed using buf_free(). */
+buf_t *
+buf_new_with_data(const char *cp, size_t sz)
+{
+  /* Validate arguments */
+  if (!cp || sz <= 0) {
+    return NULL;
+  }
+
+  tor_assert(sz < SSIZE_T_CEILING);
+
+  /* Allocate a buffer */
+  buf_t *buf = buf_new_with_capacity(sz);
+  tor_assert(buf);
+  buf_assert_ok(buf);
+  tor_assert(!buf->head);
+
+  /* Allocate a chunk that is sz bytes long */
+  buf->head = chunk_new_with_alloc_size(CHUNK_ALLOC_SIZE(sz));
+  buf->tail = buf->head;
+  tor_assert(buf->head);
+  buf_assert_ok(buf);
+  tor_assert(buf_allocation(buf) >= sz);
+
+  /* Copy the data and size the buffers */
+  tor_assert(sz <= buf_slack(buf));
+  tor_assert(sz <= CHUNK_REMAINING_CAPACITY(buf->head));
+  memcpy(&buf->head->mem[0], cp, sz);
+  buf->datalen = sz;
+  buf->head->datalen = sz;
+  buf->head->data = &buf->head->mem[0];
+  buf_assert_ok(buf);
+
+  /* Make sure everything is large enough */
+  tor_assert(buf_allocation(buf) >= sz);
+  tor_assert(buf_allocation(buf) >= buf_datalen(buf) + buf_slack(buf));
+  /* Does the buffer implementation allocate more than the requested size?
+   * (for example, by rounding up). If so, these checks will fail. */
+  tor_assert(buf_datalen(buf) == sz);
+  tor_assert(buf_slack(buf) == 0);
+
+  return buf;
+}
+#endif
+
+/** Remove the first <b>n</b> bytes from buf. */
+void
+buf_drain(buf_t *buf, size_t n)
+{
+  tor_assert(buf->datalen >= n);
+  while (n) {
+    tor_assert(buf->head);
+    if (buf->head->datalen > n) {
+      buf->head->datalen -= n;
+      buf->head->data += n;
+      buf->datalen -= n;
+      return;
+    } else {
+      chunk_t *victim = buf->head;
+      n -= victim->datalen;
+      buf->datalen -= victim->datalen;
+      buf->head = victim->next;
+      if (buf->tail == victim)
+        buf->tail = NULL;
+      buf_chunk_free_unchecked(victim);
+    }
+  }
+  check();
+}
+
+/** Create and return a new buf with default chunk capacity <b>size</b>.
+ */
+buf_t *
+buf_new_with_capacity(size_t size)
+{
+  buf_t *b = buf_new();
+  b->default_chunk_size = preferred_chunk_size(size);
+  return b;
+}
+
+/** Allocate and return a new buffer with default capacity. */
+buf_t *
+buf_new(void)
+{
+  buf_t *buf = tor_malloc_zero(sizeof(buf_t));
+  buf->magic = BUFFER_MAGIC;
+  buf->default_chunk_size = 4096;
+  return buf;
+}
+
+size_t
+buf_get_default_chunk_size(const buf_t *buf)
+{
+  return buf->default_chunk_size;
+}
+
+/** Remove all data from <b>buf</b>. */
+void
+buf_clear(buf_t *buf)
+{
+  chunk_t *chunk, *next;
+  buf->datalen = 0;
+  for (chunk = buf->head; chunk; chunk = next) {
+    next = chunk->next;
+    buf_chunk_free_unchecked(chunk);
+  }
+  buf->head = buf->tail = NULL;
+}
+
+/** Return the number of bytes stored in <b>buf</b> */
+MOCK_IMPL(size_t,
+buf_datalen, (const buf_t *buf))
+{
+  return buf->datalen;
+}
+
+/** Return the total length of all chunks used in <b>buf</b>. */
+size_t
+buf_allocation(const buf_t *buf)
+{
+  size_t total = 0;
+  const chunk_t *chunk;
+  for (chunk = buf->head; chunk; chunk = chunk->next) {
+    total += CHUNK_ALLOC_SIZE(chunk->memlen);
+  }
+  return total;
+}
+
+/** Return the number of bytes that can be added to <b>buf</b> without
+ * performing any additional allocation. */
+size_t
+buf_slack(const buf_t *buf)
+{
+  if (!buf->tail)
+    return 0;
+  else
+    return CHUNK_REMAINING_CAPACITY(buf->tail);
+}
+
+/** Release storage held by <b>buf</b>. */
+void
+buf_free(buf_t *buf)
+{
+  if (!buf)
+    return;
+
+  buf_clear(buf);
+  buf->magic = 0xdeadbeef;
+  tor_free(buf);
+}
+
+/** Return a new copy of <b>in_chunk</b> */
+static chunk_t *
+chunk_copy(const chunk_t *in_chunk)
+{
+  chunk_t *newch = tor_memdup(in_chunk, CHUNK_ALLOC_SIZE(in_chunk->memlen));
+  total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(in_chunk->memlen);
+#ifdef DEBUG_CHUNK_ALLOC
+  newch->DBG_alloc = CHUNK_ALLOC_SIZE(in_chunk->memlen);
+#endif
+  newch->next = NULL;
+  if (in_chunk->data) {
+    off_t offset = in_chunk->data - in_chunk->mem;
+    newch->data = newch->mem + offset;
+  }
+  return newch;
+}
+
+/** Return a new copy of <b>buf</b> */
+buf_t *
+buf_copy(const buf_t *buf)
+{
+  chunk_t *ch;
+  buf_t *out = buf_new();
+  out->default_chunk_size = buf->default_chunk_size;
+  for (ch = buf->head; ch; ch = ch->next) {
+    chunk_t *newch = chunk_copy(ch);
+    if (out->tail) {
+      out->tail->next = newch;
+      out->tail = newch;
+    } else {
+      out->head = out->tail = newch;
+    }
+  }
+  out->datalen = buf->datalen;
+  return out;
+}
+
+/** Append a new chunk with enough capacity to hold <b>capacity</b> bytes to
+ * the tail of <b>buf</b>.  If <b>capped</b>, don't allocate a chunk bigger
+ * than MAX_CHUNK_ALLOC. */
+chunk_t *
+buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped)
+{
+  chunk_t *chunk;
+
+  if (CHUNK_ALLOC_SIZE(capacity) < buf->default_chunk_size) {
+    chunk = chunk_new_with_alloc_size(buf->default_chunk_size);
+  } else if (capped && CHUNK_ALLOC_SIZE(capacity) > MAX_CHUNK_ALLOC) {
+    chunk = chunk_new_with_alloc_size(MAX_CHUNK_ALLOC);
+  } else {
+    chunk = chunk_new_with_alloc_size(preferred_chunk_size(capacity));
+  }
+
+  chunk->inserted_time = (uint32_t)monotime_coarse_absolute_msec();
+
+  if (buf->tail) {
+    tor_assert(buf->head);
+    buf->tail->next = chunk;
+    buf->tail = chunk;
+  } else {
+    tor_assert(!buf->head);
+    buf->head = buf->tail = chunk;
+  }
+  check();
+  return chunk;
+}
+
+/** Return the age of the oldest chunk in the buffer <b>buf</b>, in
+ * milliseconds.  Requires the current monotonic time, in truncated msec,
+ * as its input <b>now</b>.
+ */
+uint32_t
+buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now)
+{
+  if (buf->head) {
+    return now - buf->head->inserted_time;
+  } else {
+    return 0;
+  }
+}
+
+size_t
+buf_get_total_allocation(void)
+{
+  return total_bytes_allocated_in_chunks;
+}
+
+/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into
+ * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set
+ * *<b>reached_eof</b> to 1.  Return -1 on error, 0 on eof or blocking,
+ * and the number of bytes read otherwise. */
+static inline int
+read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most,
+              int *reached_eof, int *socket_error)
+{
+  ssize_t read_result;
+  if (at_most > CHUNK_REMAINING_CAPACITY(chunk))
+    at_most = CHUNK_REMAINING_CAPACITY(chunk);
+  read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0);
+
+  if (read_result < 0) {
+    int e = tor_socket_errno(fd);
+    if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
+#ifdef _WIN32
+      if (e == WSAENOBUFS)
+        log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?");
+#endif
+      *socket_error = e;
+      return -1;
+    }
+    return 0; /* would block. */
+  } else if (read_result == 0) {
+    log_debug(LD_NET,"Encountered eof on fd %d", (int)fd);
+    *reached_eof = 1;
+    return 0;
+  } else { /* actually got bytes. */
+    buf->datalen += read_result;
+    chunk->datalen += read_result;
+    log_debug(LD_NET,"Read %ld bytes. %d on inbuf.", (long)read_result,
+              (int)buf->datalen);
+    tor_assert(read_result < INT_MAX);
+    return (int)read_result;
+  }
+}
+
+/** Read from socket <b>s</b>, writing onto end of <b>buf</b>.  Read at most
+ * <b>at_most</b> bytes, growing the buffer as necessary.  If recv() returns 0
+ * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
+ * error; else return the number of bytes read.
+ */
+/* XXXX indicate "read blocked" somehow? */
+int
+buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
+                     int *reached_eof,
+                     int *socket_error)
+{
+  /* XXXX It's stupid to overload the return values for these functions:
+   * "error status" and "number of bytes read" are not mutually exclusive.
+   */
+  int r = 0;
+  size_t total_read = 0;
+
+  check();
+  tor_assert(reached_eof);
+  tor_assert(SOCKET_OK(s));
+
+  if (BUG(buf->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf->datalen >= INT_MAX - at_most))
+    return -1;
+
+  while (at_most > total_read) {
+    size_t readlen = at_most - total_read;
+    chunk_t *chunk;
+    if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) {
+      chunk = buf_add_chunk_with_capacity(buf, at_most, 1);
+      if (readlen > chunk->memlen)
+        readlen = chunk->memlen;
+    } else {
+      size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail);
+      chunk = buf->tail;
+      if (cap < readlen)
+        readlen = cap;
+    }
+
+    r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error);
+    check();
+    if (r < 0)
+      return r; /* Error */
+    tor_assert(total_read+r < INT_MAX);
+    total_read += r;
+    if ((size_t)r < readlen) { /* eof, block, or no more to read. */
+      break;
+    }
+  }
+  return (int)total_read;
+}
+
+/** Helper for buf_flush_to_socket(): try to write <b>sz</b> bytes from chunk
+ * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  On success, deduct
+ * the bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
+ * written on success, 0 on blocking, -1 on failure.
+ */
+static inline int
+flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz,
+            size_t *buf_flushlen)
+{
+  ssize_t write_result;
+
+  if (sz > chunk->datalen)
+    sz = chunk->datalen;
+  write_result = tor_socket_send(s, chunk->data, sz, 0);
+
+  if (write_result < 0) {
+    int e = tor_socket_errno(s);
+    if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
+#ifdef _WIN32
+      if (e == WSAENOBUFS)
+        log_warn(LD_NET,"write() failed: WSAENOBUFS. Not enough ram?");
+#endif
+      return -1;
+    }
+    log_debug(LD_NET,"write() would block, returning.");
+    return 0;
+  } else {
+    *buf_flushlen -= write_result;
+    buf_drain(buf, write_result);
+    tor_assert(write_result < INT_MAX);
+    return (int)write_result;
+  }
+}
+
+/** Write data from <b>buf</b> to the socket <b>s</b>.  Write at most
+ * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
+ * the number of bytes actually written, and remove the written bytes
+ * from the buffer.  Return the number of bytes written on success,
+ * -1 on failure.  Return 0 if write() would block.
+ */
+int
+buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
+                    size_t *buf_flushlen)
+{
+  /* XXXX It's stupid to overload the return values for these functions:
+   * "error status" and "number of bytes flushed" are not mutually exclusive.
+   */
+  int r;
+  size_t flushed = 0;
+  tor_assert(buf_flushlen);
+  tor_assert(SOCKET_OK(s));
+  tor_assert(*buf_flushlen <= buf->datalen);
+  tor_assert(sz <= *buf_flushlen);
+
+  check();
+  while (sz) {
+    size_t flushlen0;
+    tor_assert(buf->head);
+    if (buf->head->datalen >= sz)
+      flushlen0 = sz;
+    else
+      flushlen0 = buf->head->datalen;
+
+    r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen);
+    check();
+    if (r < 0)
+      return r;
+    flushed += r;
+    sz -= r;
+    if (r == 0 || (size_t)r < flushlen0) /* can't flush any more now. */
+      break;
+  }
+  tor_assert(flushed < INT_MAX);
+  return (int)flushed;
+}
+
+/** Append <b>string_len</b> bytes from <b>string</b> to the end of
+ * <b>buf</b>.
+ *
+ * Return the new length of the buffer on success, -1 on failure.
+ */
+int
+buf_add(buf_t *buf, const char *string, size_t string_len)
+{
+  if (!string_len)
+    return (int)buf->datalen;
+  check();
+
+  if (BUG(buf->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf->datalen >= INT_MAX - string_len))
+    return -1;
+
+  while (string_len) {
+    size_t copy;
+    if (!buf->tail || !CHUNK_REMAINING_CAPACITY(buf->tail))
+      buf_add_chunk_with_capacity(buf, string_len, 1);
+
+    copy = CHUNK_REMAINING_CAPACITY(buf->tail);
+    if (copy > string_len)
+      copy = string_len;
+    memcpy(CHUNK_WRITE_PTR(buf->tail), string, copy);
+    string_len -= copy;
+    string += copy;
+    buf->datalen += copy;
+    buf->tail->datalen += copy;
+  }
+
+  check();
+  tor_assert(buf->datalen < INT_MAX);
+  return (int)buf->datalen;
+}
+
+/** Helper: copy the first <b>string_len</b> bytes from <b>buf</b>
+ * onto <b>string</b>.
+ */
+void
+buf_peek(const buf_t *buf, char *string, size_t string_len)
+{
+  chunk_t *chunk;
+
+  tor_assert(string);
+  /* make sure we don't ask for too much */
+  tor_assert(string_len <= buf->datalen);
+  /* buf_assert_ok(buf); */
+
+  chunk = buf->head;
+  while (string_len) {
+    size_t copy = string_len;
+    tor_assert(chunk);
+    if (chunk->datalen < copy)
+      copy = chunk->datalen;
+    memcpy(string, chunk->data, copy);
+    string_len -= copy;
+    string += copy;
+    chunk = chunk->next;
+  }
+}
+
+/** Remove <b>string_len</b> bytes from the front of <b>buf</b>, and store
+ * them into <b>string</b>.  Return the new buffer size.  <b>string_len</b>
+ * must be \<= the number of bytes on the buffer.
+ */
+int
+buf_get_bytes(buf_t *buf, char *string, size_t string_len)
+{
+  /* There must be string_len bytes in buf; write them onto string,
+   * then memmove buf back (that is, remove them from buf).
+   *
+   * Return the number of bytes still on the buffer. */
+
+  check();
+  buf_peek(buf, string, string_len);
+  buf_drain(buf, string_len);
+  check();
+  tor_assert(buf->datalen < INT_MAX);
+  return (int)buf->datalen;
+}
+
+/** Move up to *<b>buf_flushlen</b> bytes from <b>buf_in</b> to
+ * <b>buf_out</b>, and modify *<b>buf_flushlen</b> appropriately.
+ * Return the number of bytes actually copied.
+ */
+int
+buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen)
+{
+  /* We can do way better here, but this doesn't turn up in any profiles. */
+  char b[4096];
+  size_t cp, len;
+
+  if (BUG(buf_out->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf_out->datalen >= INT_MAX - *buf_flushlen))
+    return -1;
+
+  len = *buf_flushlen;
+  if (len > buf_in->datalen)
+    len = buf_in->datalen;
+
+  cp = len; /* Remember the number of bytes we intend to copy. */
+  tor_assert(cp < INT_MAX);
+  while (len) {
+    /* This isn't the most efficient implementation one could imagine, since
+     * it does two copies instead of 1, but I kinda doubt that this will be
+     * critical path. */
+    size_t n = len > sizeof(b) ? sizeof(b) : len;
+    buf_get_bytes(buf_in, b, n);
+    buf_add(buf_out, b, n);
+    len -= n;
+  }
+  *buf_flushlen -= cp;
+  return (int)cp;
+}
+
+/** Internal structure: represents a position in a buffer. */
+typedef struct buf_pos_t {
+  const chunk_t *chunk; /**< Which chunk are we pointing to? */
+  int pos;/**< Which character inside the chunk's data are we pointing to? */
+  size_t chunk_pos; /**< Total length of all previous chunks. */
+} buf_pos_t;
+
+/** Initialize <b>out</b> to point to the first character of <b>buf</b>.*/
+static void
+buf_pos_init(const buf_t *buf, buf_pos_t *out)
+{
+  out->chunk = buf->head;
+  out->pos = 0;
+  out->chunk_pos = 0;
+}
+
+/** Advance <b>out</b> to the first appearance of <b>ch</b> at the current
+ * position of <b>out</b>, or later.  Return -1 if no instances are found;
+ * otherwise returns the absolute position of the character. */
+static off_t
+buf_find_pos_of_char(char ch, buf_pos_t *out)
+{
+  const chunk_t *chunk;
+  int pos;
+  tor_assert(out);
+  if (out->chunk) {
+    if (out->chunk->datalen) {
+      tor_assert(out->pos < (off_t)out->chunk->datalen);
+    } else {
+      tor_assert(out->pos == 0);
+    }
+  }
+  pos = out->pos;
+  for (chunk = out->chunk; chunk; chunk = chunk->next) {
+    char *cp = memchr(chunk->data+pos, ch, chunk->datalen - pos);
+    if (cp) {
+      out->chunk = chunk;
+      tor_assert(cp - chunk->data < INT_MAX);
+      out->pos = (int)(cp - chunk->data);
+      return out->chunk_pos + out->pos;
+    } else {
+      out->chunk_pos += chunk->datalen;
+      pos = 0;
+    }
+  }
+  return -1;
+}
+
+/** Advance <b>pos</b> by a single character, if there are any more characters
+ * in the buffer.  Returns 0 on success, -1 on failure. */
+static inline int
+buf_pos_inc(buf_pos_t *pos)
+{
+  ++pos->pos;
+  if (pos->pos == (off_t)pos->chunk->datalen) {
+    if (!pos->chunk->next)
+      return -1;
+    pos->chunk_pos += pos->chunk->datalen;
+    pos->chunk = pos->chunk->next;
+    pos->pos = 0;
+  }
+  return 0;
+}
+
+/** Return true iff the <b>n</b>-character string in <b>s</b> appears
+ * (verbatim) at <b>pos</b>. */
+static int
+buf_matches_at_pos(const buf_pos_t *pos, const char *s, size_t n)
+{
+  buf_pos_t p;
+  if (!n)
+    return 1;
+
+  memcpy(&p, pos, sizeof(p));
+
+  while (1) {
+    char ch = p.chunk->data[p.pos];
+    if (ch != *s)
+      return 0;
+    ++s;
+    /* If we're out of characters that don't match, we match.  Check this
+     * _before_ we test incrementing pos, in case we're at the end of the
+     * string. */
+    if (--n == 0)
+      return 1;
+    if (buf_pos_inc(&p)<0)
+      return 0;
+  }
+}
+
+/** Return the first position in <b>buf</b> at which the <b>n</b>-character
+ * string <b>s</b> occurs, or -1 if it does not occur. */
+int
+buf_find_string_offset(const buf_t *buf, const char *s, size_t n)
+{
+  buf_pos_t pos;
+  buf_pos_init(buf, &pos);
+  while (buf_find_pos_of_char(*s, &pos) >= 0) {
+    if (buf_matches_at_pos(&pos, s, n)) {
+      tor_assert(pos.chunk_pos + pos.pos < INT_MAX);
+      return (int)(pos.chunk_pos + pos.pos);
+    } else {
+      if (buf_pos_inc(&pos)<0)
+        return -1;
+    }
+  }
+  return -1;
+}
+
+/** Return 1 iff <b>buf</b> starts with <b>cmd</b>. <b>cmd</b> must be a null
+ * terminated string, of no more than PEEK_BUF_STARTSWITH_MAX bytes. */
+int
+buf_peek_startswith(const buf_t *buf, const char *cmd)
+{
+  char tmp[PEEK_BUF_STARTSWITH_MAX];
+  size_t clen = strlen(cmd);
+  if (BUG(clen > sizeof(tmp)))
+    return 0;
+  if (buf->datalen < clen)
+    return 0;
+  buf_peek(buf, tmp, clen);
+  return fast_memeq(tmp, cmd, clen);
+}
+
+/** Return the index within <b>buf</b> at which <b>ch</b> first appears,
+ * or -1 if <b>ch</b> does not appear on buf. */
+static off_t
+buf_find_offset_of_char(buf_t *buf, char ch)
+{
+  chunk_t *chunk;
+  off_t offset = 0;
+  for (chunk = buf->head; chunk; chunk = chunk->next) {
+    char *cp = memchr(chunk->data, ch, chunk->datalen);
+    if (cp)
+      return offset + (cp - chunk->data);
+    else
+      offset += chunk->datalen;
+  }
+  return -1;
+}
+
+/** Try to read a single LF-terminated line from <b>buf</b>, and write it
+ * (including the LF), NUL-terminated, into the *<b>data_len</b> byte buffer
+ * at <b>data_out</b>.  Set *<b>data_len</b> to the number of bytes in the
+ * line, not counting the terminating NUL.  Return 1 if we read a whole line,
+ * return 0 if we don't have a whole line yet, and return -1 if the line
+ * length exceeds *<b>data_len</b>.
+ */
+int
+buf_get_line(buf_t *buf, char *data_out, size_t *data_len)
+{
+  size_t sz;
+  off_t offset;
+
+  if (!buf->head)
+    return 0;
+
+  offset = buf_find_offset_of_char(buf, '\n');
+  if (offset < 0)
+    return 0;
+  sz = (size_t) offset;
+  if (sz+2 > *data_len) {
+    *data_len = sz + 2;
+    return -1;
+  }
+  buf_get_bytes(buf, data_out, sz+1);
+  data_out[sz+1] = '\0';
+  *data_len = sz+1;
+  return 1;
+}
+
+/** Compress on uncompress the <b>data_len</b> bytes in <b>data</b> using the
+ * compression state <b>state</b>, appending the result to <b>buf</b>.  If
+ * <b>done</b> is true, flush the data in the state and finish the
+ * compression/uncompression.  Return -1 on failure, 0 on success. */
+int
+buf_add_compress(buf_t *buf, tor_compress_state_t *state,
+                      const char *data, size_t data_len,
+                      const int done)
+{
+  char *next;
+  size_t old_avail, avail;
+  int over = 0;
+
+  do {
+    int need_new_chunk = 0;
+    if (!buf->tail || ! CHUNK_REMAINING_CAPACITY(buf->tail)) {
+      size_t cap = data_len / 4;
+      buf_add_chunk_with_capacity(buf, cap, 1);
+    }
+    next = CHUNK_WRITE_PTR(buf->tail);
+    avail = old_avail = CHUNK_REMAINING_CAPACITY(buf->tail);
+    switch (tor_compress_process(state, &next, &avail,
+                                 &data, &data_len, done)) {
+      case TOR_COMPRESS_DONE:
+        over = 1;
+        break;
+      case TOR_COMPRESS_ERROR:
+        return -1;
+      case TOR_COMPRESS_OK:
+        if (data_len == 0) {
+          tor_assert_nonfatal(!done);
+          over = 1;
+        }
+        break;
+      case TOR_COMPRESS_BUFFER_FULL:
+        if (avail) {
+          /* The compression module says we need more room
+           * (TOR_COMPRESS_BUFFER_FULL).  Start a new chunk automatically,
+           * whether were going to or not. */
+          need_new_chunk = 1;
+        }
+        if (data_len == 0 && !done) {
+          /* We've consumed all the input data, though, so there's no
+           * point in forging ahead right now. */
+          over = 1;
+        }
+        break;
+    }
+    buf->datalen += old_avail - avail;
+    buf->tail->datalen += old_avail - avail;
+    if (need_new_chunk) {
+      buf_add_chunk_with_capacity(buf, data_len/4, 1);
+    }
+
+  } while (!over);
+  check();
+  return 0;
+}
+
+/** Set *<b>output</b> to contain a copy of the data in *<b>input</b> */
+int
+buf_set_to_copy(buf_t **output,
+                const buf_t *input)
+{
+  if (*output)
+    buf_free(*output);
+  *output = buf_copy(input);
+  return 0;
+}
+
+/** Log an error and exit if <b>buf</b> is corrupted.
+ */
+void
+buf_assert_ok(buf_t *buf)
+{
+  tor_assert(buf);
+  tor_assert(buf->magic == BUFFER_MAGIC);
+
+  if (! buf->head) {
+    tor_assert(!buf->tail);
+    tor_assert(buf->datalen == 0);
+  } else {
+    chunk_t *ch;
+    size_t total = 0;
+    tor_assert(buf->tail);
+    for (ch = buf->head; ch; ch = ch->next) {
+      total += ch->datalen;
+      tor_assert(ch->datalen <= ch->memlen);
+      tor_assert(ch->data >= &ch->mem[0]);
+      tor_assert(ch->data <= &ch->mem[0]+ch->memlen);
+      if (ch->data == &ch->mem[0]+ch->memlen) {
+        static int warned = 0;
+        if (! warned) {
+          log_warn(LD_BUG, "Invariant violation in buf.c related to #15083");
+          warned = 1;
+        }
+      }
+      tor_assert(ch->data+ch->datalen <= &ch->mem[0] + ch->memlen);
+      if (!ch->next)
+        tor_assert(ch == buf->tail);
+    }
+    tor_assert(buf->datalen == total);
+  }
+}
+

src/or/buffers.h → src/common/buffers.h

@@ -12,8 +12,15 @@
 #ifndef TOR_BUFFERS_H
 #define TOR_BUFFERS_H
 
+#include "compat.h"
+#include "compat.h"
+#include "torint.h"
 #include "testsupport.h"
 
+typedef struct buf_t buf_t;
+
+struct tor_compress_state_t;
+
 buf_t *buf_new(void);
 buf_t *buf_new_with_capacity(size_t size);
 size_t buf_get_default_chunk_size(const buf_t *buf);
@@ -28,50 +35,39 @@ size_t buf_slack(const buf_t *buf);
 uint32_t buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now);
 size_t buf_get_total_allocation(void);
 
-int read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof,
-                int *socket_error);
-int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf);
+int buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
+                         int *reached_eof,
+                         int *socket_error);
 
-int flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen);
-int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen);
+int buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
+                        size_t *buf_flushlen);
 
-int write_to_buf(const char *string, size_t string_len, buf_t *buf);
-int write_to_buf_compress(buf_t *buf, tor_compress_state_t *state,
+int buf_add(buf_t *buf, const char *string, size_t string_len);
+int buf_add_compress(buf_t *buf, struct tor_compress_state_t *state,
                           const char *data, size_t data_len, int done);
-int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen);
-int fetch_from_buf(char *string, size_t string_len, buf_t *buf);
-int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto);
-int fetch_from_buf_http(buf_t *buf,
-                        char **headers_out, size_t max_headerlen,
-                        char **body_out, size_t *body_used, size_t max_bodylen,
-                        int force_complete);
-socks_request_t *socks_request_new(void);
-void socks_request_free(socks_request_t *req);
-int fetch_from_buf_socks(buf_t *buf, socks_request_t *req,
-                         int log_sockstype, int safe_socks);
-int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason);
-int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len);
-
-int peek_buf_has_control0_command(buf_t *buf);
-#define PEEK_BUF_STARTSWITH_MAX 16
-int peek_buf_startswith(const buf_t *buf, const char *cmd);
-int peek_buf_has_http_command(const buf_t *buf);
+int buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen);
+void buf_peek(const buf_t *buf, char *string, size_t string_len);
+void buf_drain(buf_t *buf, size_t n);
+int buf_get_bytes(buf_t *buf, char *string, size_t string_len);
+int buf_get_line(buf_t *buf, char *data_out, size_t *data_len);
 
-int fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out);
+#define PEEK_BUF_STARTSWITH_MAX 16
+int buf_peek_startswith(const buf_t *buf, const char *cmd);
 
 int buf_set_to_copy(buf_t **output,
                     const buf_t *input);
 
-void assert_buf_ok(buf_t *buf);
+void buf_assert_ok(buf_t *buf);
+
+int buf_find_string_offset(const buf_t *buf, const char *s, size_t n);
+void buf_pullup(buf_t *buf, size_t bytes,
+                const char **head_out, size_t *len_out);
 
 #ifdef BUFFERS_PRIVATE
-STATIC int buf_find_string_offset(const buf_t *buf, const char *s, size_t n);
-STATIC void buf_pullup(buf_t *buf, size_t bytes);
 #ifdef TOR_UNIT_TESTS
-void buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz);
 buf_t *buf_new_with_data(const char *cp, size_t sz);
 #endif
-STATIC size_t preferred_chunk_size(size_t target);
+ATTR_UNUSED STATIC size_t preferred_chunk_size(size_t target);
 
 #define DEBUG_CHUNK_ALLOC
 /** A single chunk on a buffer. */
@@ -101,11 +97,28 @@ struct buf_t {
   chunk_t *head; /**< First chunk in the list, or NULL for none. */
   chunk_t *tail; /**< Last chunk in the list, or NULL for none. */
 };
-#endif
 
-#ifdef BUFFERS_PRIVATE
-STATIC int buf_http_find_content_length(const char *headers, size_t headerlen,
-                                        size_t *result_out);
+chunk_t *buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped);
+/** If a read onto the end of a chunk would be smaller than this number, then
+ * just start a new chunk. */
+#define MIN_READ_LEN 8
+
+/** Return the number of bytes that can be written onto <b>chunk</b> without
+ * running out of space. */
+static inline size_t
+CHUNK_REMAINING_CAPACITY(const chunk_t *chunk)
+{
+  return (chunk->mem + chunk->memlen) - (chunk->data + chunk->datalen);
+}
+
+/** Return the next character in <b>chunk</b> onto which data can be appended.
+ * If the chunk is full, this might be off the end of chunk->mem. */
+static inline char *
+CHUNK_WRITE_PTR(chunk_t *chunk)
+{
+  return chunk->data + chunk->datalen;
+}
+
 #endif
 
 #endif

src/common/buffers_tls.c

@@ -0,0 +1,175 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define BUFFERS_PRIVATE
+#include "orconfig.h"
+#include <stddef.h>
+#include "buffers.h"
+#include "buffers_tls.h"
+#include "compat.h"
+#include "compress.h"
+#include "util.h"
+#include "torint.h"
+#include "torlog.h"
+#include "tortls.h"
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/** As read_to_chunk(), but return (negative) error code on error, blocking,
+ * or TLS, and the number of bytes read otherwise. */
+static inline int
+read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls,
+                  size_t at_most)
+{
+  int read_result;
+
+  tor_assert(CHUNK_REMAINING_CAPACITY(chunk) >= at_most);
+  read_result = tor_tls_read(tls, CHUNK_WRITE_PTR(chunk), at_most);
+  if (read_result < 0)
+    return read_result;
+  buf->datalen += read_result;
+  chunk->datalen += read_result;
+  return read_result;
+}
+
+/** As read_to_buf, but reads from a TLS connection, and returns a TLS
+ * status value rather than the number of bytes read.
+ *
+ * Using TLS on OR connections complicates matters in two ways.
+ *
+ * First, a TLS stream has its own read buffer independent of the
+ * connection's read buffer.  (TLS needs to read an entire frame from
+ * the network before it can decrypt any data.  Thus, trying to read 1
+ * byte from TLS can require that several KB be read from the network
+ * and decrypted.  The extra data is stored in TLS's decrypt buffer.)
+ * Because the data hasn't been read by Tor (it's still inside the TLS),
+ * this means that sometimes a connection "has stuff to read" even when
+ * poll() didn't return POLLIN. The tor_tls_get_pending_bytes function is
+ * used in connection.c to detect TLS objects with non-empty internal
+ * buffers and read from them again.
+ *
+ * Second, the TLS stream's events do not correspond directly to network
+ * events: sometimes, before a TLS stream can read, the network must be
+ * ready to write -- or vice versa.
+ */
+int
+buf_read_from_tls(buf_t *buf, tor_tls_t *tls, size_t at_most)
+{
+  int r = 0;
+  size_t total_read = 0;
+
+  check_no_tls_errors();
+
+  if (BUG(buf->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf->datalen >= INT_MAX - at_most))
+    return -1;
+
+  while (at_most > total_read) {
+    size_t readlen = at_most - total_read;
+    chunk_t *chunk;
+    if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) {
+      chunk = buf_add_chunk_with_capacity(buf, at_most, 1);
+      if (readlen > chunk->memlen)
+        readlen = chunk->memlen;
+    } else {
+      size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail);
+      chunk = buf->tail;
+      if (cap < readlen)
+        readlen = cap;
+    }
+
+    r = read_to_chunk_tls(buf, chunk, tls, readlen);
+    if (r < 0)
+      return r; /* Error */
+    tor_assert(total_read+r < INT_MAX);
+    total_read += r;
+    if ((size_t)r < readlen) /* eof, block, or no more to read. */
+      break;
+  }
+  return (int)total_read;
+}
+
+/** Helper for buf_flush_to_tls(): try to write <b>sz</b> bytes from chunk
+ * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  (Tries to write
+ * more if there is a forced pending write size.)  On success, deduct the
+ * bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
+ * written on success, and a TOR_TLS error code on failure or blocking.
+ */
+static inline int
+flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk,
+                size_t sz, size_t *buf_flushlen)
+{
+  int r;
+  size_t forced;
+  char *data;
+
+  forced = tor_tls_get_forced_write_size(tls);
+  if (forced > sz)
+    sz = forced;
+  if (chunk) {
+    data = chunk->data;
+    tor_assert(sz <= chunk->datalen);
+  } else {
+    data = NULL;
+    tor_assert(sz == 0);
+  }
+  r = tor_tls_write(tls, data, sz);
+  if (r < 0)
+    return r;
+  if (*buf_flushlen > (size_t)r)
+    *buf_flushlen -= r;
+  else
+    *buf_flushlen = 0;
+  buf_drain(buf, r);
+  log_debug(LD_NET,"flushed %d bytes, %d ready to flush, %d remain.",
+            r,(int)*buf_flushlen,(int)buf->datalen);
+  return r;
+}
+
+/** As buf_flush_to_socket(), but writes data to a TLS connection.  Can write
+ * more than <b>flushlen</b> bytes.
+ */
+int
+buf_flush_to_tls(buf_t *buf, tor_tls_t *tls, size_t flushlen,
+              size_t *buf_flushlen)
+{
+  int r;
+  size_t flushed = 0;
+  ssize_t sz;
+  tor_assert(buf_flushlen);
+  tor_assert(*buf_flushlen <= buf->datalen);
+  tor_assert(flushlen <= *buf_flushlen);
+  sz = (ssize_t) flushlen;
+
+  /* we want to let tls write even if flushlen is zero, because it might
+   * have a partial record pending */
+  check_no_tls_errors();
+
+  do {
+    size_t flushlen0;
+    if (buf->head) {
+      if ((ssize_t)buf->head->datalen >= sz)
+        flushlen0 = sz;
+      else
+        flushlen0 = buf->head->datalen;
+    } else {
+      flushlen0 = 0;
+    }
+
+    r = flush_chunk_tls(tls, buf, buf->head, flushlen0, buf_flushlen);
+    if (r < 0)
+      return r;
+    flushed += r;
+    sz -= r;
+    if (r == 0) /* Can't flush any more now. */
+      break;
+  } while (sz > 0);
+  tor_assert(flushed < INT_MAX);
+  return (int)flushed;
+}
+

src/common/buffers_tls.h

@@ -0,0 +1,19 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_BUFFERS_TLS_H
+#define TOR_BUFFERS_TLS_H
+
+struct buf_t;
+struct tor_tls_t;
+
+int buf_read_from_tls(struct buf_t *buf,
+                      struct tor_tls_t *tls, size_t at_most);
+int buf_flush_to_tls(struct buf_t *buf, struct tor_tls_t *tls,
+                     size_t sz, size_t *buf_flushlen);
+
+#endif
+

src/common/include.am

@@ -81,6 +81,7 @@ src_common_libor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS)
 LIBOR_A_SRC = \
   src/common/address.c					\
   src/common/backtrace.c				\
+  src/common/buffers.c					\
   src/common/compat.c					\
   src/common/compat_threads.c				\
   src/common/compat_time.c				\
@@ -110,6 +111,7 @@ src/common/src_common_libor_testing_a-log.$(OBJEXT) \
 
 LIBOR_CRYPTO_A_SRC = \
   src/common/aes.c		\
+  src/common/buffers_tls.c	\
   src/common/compress.c	\
   src/common/compress_lzma.c	\
   src/common/compress_none.c	\
@@ -147,6 +149,8 @@ src_common_libor_event_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 COMMONHEADERS = \
   src/common/address.h				\
   src/common/backtrace.h			\
+  src/common/buffers.h				\
+  src/common/buffers_tls.h			\
   src/common/aes.h				\
   src/common/ciphers.inc			\
   src/common/compat.h				\

src/or/buffers.c

@@ -1,2214 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2017, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file buffers.c
- * \brief Implements a generic buffer interface.
- *
- * A buf_t is a (fairly) opaque byte-oriented FIFO that can read to or flush
- * from memory, sockets, file descriptors, TLS connections, or another buf_t.
- * Buffers are implemented as linked lists of memory chunks.
- *
- * All socket-backed and TLS-based connection_t objects have a pair of
- * buffers: one for incoming data, and one for outcoming data.  These are fed
- * and drained from functions in connection.c, trigged by events that are
- * monitored in main.c.
- *
- * This module has basic support for reading and writing on buf_t objects. It
- * also contains specialized functions for handling particular protocols
- * on a buf_t backend, including SOCKS (used in connection_edge.c), Tor cells
- * (used in connection_or.c and channeltls.c), HTTP (used in directory.c), and
- * line-oriented communication (used in control.c).
- **/
-#define BUFFERS_PRIVATE
-#include "or.h"
-#include "addressmap.h"
-#include "buffers.h"
-#include "config.h"
-#include "connection_edge.h"
-#include "connection_or.h"
-#include "control.h"
-#include "reasons.h"
-#include "ext_orport.h"
-#include "util.h"
-#include "torlog.h"
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-//#define PARANOIA
-
-#ifdef PARANOIA
-/** Helper: If PARANOIA is defined, assert that the buffer in local variable
- * <b>buf</b> is well-formed. */
-#define check() STMT_BEGIN assert_buf_ok(buf); STMT_END
-#else
-#define check() STMT_NIL
-#endif
-
-/* Implementation notes:
- *
- * After flirting with memmove, and dallying with ring-buffers, we're finally
- * getting up to speed with the 1970s and implementing buffers as a linked
- * list of small chunks.  Each buffer has such a list; data is removed from
- * the head of the list, and added at the tail.  The list is singly linked,
- * and the buffer keeps a pointer to the head and the tail.
- *
- * Every chunk, except the tail, contains at least one byte of data.  Data in
- * each chunk is contiguous.
- *
- * When you need to treat the first N characters on a buffer as a contiguous
- * string, use the buf_pullup function to make them so.  Don't do this more
- * than necessary.
- *
- * The major free Unix kernels have handled buffers like this since, like,
- * forever.
- */
-
-static void socks_request_set_socks5_error(socks_request_t *req,
-                              socks5_reply_status_t reason);
-
-static int parse_socks(const char *data, size_t datalen, socks_request_t *req,
-                       int log_sockstype, int safe_socks, ssize_t *drain_out,
-                       size_t *want_length_out);
-static int parse_socks_client(const uint8_t *data, size_t datalen,
-                              int state, char **reason,
-                              ssize_t *drain_out);
-
-/* Chunk manipulation functions */
-
-#define CHUNK_HEADER_LEN offsetof(chunk_t, mem[0])
-
-/* We leave this many NUL bytes at the end of the buffer. */
-#ifdef DISABLE_MEMORY_SENTINELS
-#define SENTINEL_LEN 0
-#else
-#define SENTINEL_LEN 4
-#endif
-
-/* Header size plus NUL bytes at the end */
-#define CHUNK_OVERHEAD (CHUNK_HEADER_LEN + SENTINEL_LEN)
-
-/** Return the number of bytes needed to allocate a chunk to hold
- * <b>memlen</b> bytes. */
-#define CHUNK_ALLOC_SIZE(memlen) (CHUNK_OVERHEAD + (memlen))
-/** Return the number of usable bytes in a chunk allocated with
- * malloc(<b>memlen</b>). */
-#define CHUNK_SIZE_WITH_ALLOC(memlen) ((memlen) - CHUNK_OVERHEAD)
-
-#define DEBUG_SENTINEL
-
-#if defined(DEBUG_SENTINEL) && !defined(DISABLE_MEMORY_SENTINELS)
-#define DBG_S(s) s
-#else
-#define DBG_S(s) (void)0
-#endif
-
-#ifdef DISABLE_MEMORY_SENTINELS
-#define CHUNK_SET_SENTINEL(chunk, alloclen) STMT_NIL
-#else
-#define CHUNK_SET_SENTINEL(chunk, alloclen) do {                        \
-    uint8_t *a = (uint8_t*) &(chunk)->mem[(chunk)->memlen];             \
-    DBG_S(uint8_t *b = &((uint8_t*)(chunk))[(alloclen)-SENTINEL_LEN]);  \
-    DBG_S(tor_assert(a == b));                                          \
-    memset(a,0,SENTINEL_LEN);                                           \
-  } while (0)
-#endif
-
-/** Return the next character in <b>chunk</b> onto which data can be appended.
- * If the chunk is full, this might be off the end of chunk->mem. */
-static inline char *
-CHUNK_WRITE_PTR(chunk_t *chunk)
-{
-  return chunk->data + chunk->datalen;
-}
-
-/** Return the number of bytes that can be written onto <b>chunk</b> without
- * running out of space. */
-static inline size_t
-CHUNK_REMAINING_CAPACITY(const chunk_t *chunk)
-{
-  return (chunk->mem + chunk->memlen) - (chunk->data + chunk->datalen);
-}
-
-/** Move all bytes stored in <b>chunk</b> to the front of <b>chunk</b>->mem,
- * to free up space at the end. */
-static inline void
-chunk_repack(chunk_t *chunk)
-{
-  if (chunk->datalen && chunk->data != &chunk->mem[0]) {
-    memmove(chunk->mem, chunk->data, chunk->datalen);
-  }
-  chunk->data = &chunk->mem[0];
-}
-
-/** Keep track of total size of allocated chunks for consistency asserts */
-static size_t total_bytes_allocated_in_chunks = 0;
-static void
-buf_chunk_free_unchecked(chunk_t *chunk)
-{
-  if (!chunk)
-    return;
-#ifdef DEBUG_CHUNK_ALLOC
-  tor_assert(CHUNK_ALLOC_SIZE(chunk->memlen) == chunk->DBG_alloc);
-#endif
-  tor_assert(total_bytes_allocated_in_chunks >=
-             CHUNK_ALLOC_SIZE(chunk->memlen));
-  total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen);
-  tor_free(chunk);
-}
-static inline chunk_t *
-chunk_new_with_alloc_size(size_t alloc)
-{
-  chunk_t *ch;
-  ch = tor_malloc(alloc);
-  ch->next = NULL;
-  ch->datalen = 0;
-#ifdef DEBUG_CHUNK_ALLOC
-  ch->DBG_alloc = alloc;
-#endif
-  ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc);
-  total_bytes_allocated_in_chunks += alloc;
-  ch->data = &ch->mem[0];
-  CHUNK_SET_SENTINEL(ch, alloc);
-  return ch;
-}
-
-/** Expand <b>chunk</b> until it can hold <b>sz</b> bytes, and return a
- * new pointer to <b>chunk</b>.  Old pointers are no longer valid. */
-static inline chunk_t *
-chunk_grow(chunk_t *chunk, size_t sz)
-{
-  off_t offset;
-  const size_t memlen_orig = chunk->memlen;
-  const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig);
-  const size_t new_alloc = CHUNK_ALLOC_SIZE(sz);
-  tor_assert(sz > chunk->memlen);
-  offset = chunk->data - chunk->mem;
-  chunk = tor_realloc(chunk, new_alloc);
-  chunk->memlen = sz;
-  chunk->data = chunk->mem + offset;
-#ifdef DEBUG_CHUNK_ALLOC
-  tor_assert(chunk->DBG_alloc == orig_alloc);
-  chunk->DBG_alloc = new_alloc;
-#endif
-  total_bytes_allocated_in_chunks += new_alloc - orig_alloc;
-  CHUNK_SET_SENTINEL(chunk, new_alloc);
-  return chunk;
-}
-
-/** If a read onto the end of a chunk would be smaller than this number, then
- * just start a new chunk. */
-#define MIN_READ_LEN 8
-/** Every chunk should take up at least this many bytes. */
-#define MIN_CHUNK_ALLOC 256
-/** No chunk should take up more than this many bytes. */
-#define MAX_CHUNK_ALLOC 65536
-
-/** Return the allocation size we'd like to use to hold <b>target</b>
- * bytes. */
-STATIC size_t
-preferred_chunk_size(size_t target)
-{
-  tor_assert(target <= SIZE_T_CEILING - CHUNK_OVERHEAD);
-  if (CHUNK_ALLOC_SIZE(target) >= MAX_CHUNK_ALLOC)
-    return CHUNK_ALLOC_SIZE(target);
-  size_t sz = MIN_CHUNK_ALLOC;
-  while (CHUNK_SIZE_WITH_ALLOC(sz) < target) {
-    sz <<= 1;
-  }
-  return sz;
-}
-
-/** Collapse data from the first N chunks from <b>buf</b> into buf->head,
- * growing it as necessary, until buf->head has the first <b>bytes</b> bytes
- * of data from the buffer, or until buf->head has all the data in <b>buf</b>.
- */
-STATIC void
-buf_pullup(buf_t *buf, size_t bytes)
-{
-  chunk_t *dest, *src;
-  size_t capacity;
-  if (!buf->head)
-    return;
-
-  check();
-  if (buf->datalen < bytes)
-    bytes = buf->datalen;
-
-  capacity = bytes;
-  if (buf->head->datalen >= bytes)
-    return;
-
-  if (buf->head->memlen >= capacity) {
-    /* We don't need to grow the first chunk, but we might need to repack it.*/
-    size_t needed = capacity - buf->head->datalen;
-    if (CHUNK_REMAINING_CAPACITY(buf->head) < needed)
-      chunk_repack(buf->head);
-    tor_assert(CHUNK_REMAINING_CAPACITY(buf->head) >= needed);
-  } else {
-    chunk_t *newhead;
-    size_t newsize;
-    /* We need to grow the chunk. */
-    chunk_repack(buf->head);
-    newsize = CHUNK_SIZE_WITH_ALLOC(preferred_chunk_size(capacity));
-    newhead = chunk_grow(buf->head, newsize);
-    tor_assert(newhead->memlen >= capacity);
-    if (newhead != buf->head) {
-      if (buf->tail == buf->head)
-        buf->tail = newhead;
-      buf->head = newhead;
-    }
-  }
-
-  dest = buf->head;
-  while (dest->datalen < bytes) {
-    size_t n = bytes - dest->datalen;
-    src = dest->next;
-    tor_assert(src);
-    if (n >= src->datalen) {
-      memcpy(CHUNK_WRITE_PTR(dest), src->data, src->datalen);
-      dest->datalen += src->datalen;
-      dest->next = src->next;
-      if (buf->tail == src)
-        buf->tail = dest;
-      buf_chunk_free_unchecked(src);
-    } else {
-      memcpy(CHUNK_WRITE_PTR(dest), src->data, n);
-      dest->datalen += n;
-      src->data += n;
-      src->datalen -= n;
-      tor_assert(dest->datalen == bytes);
-    }
-  }
-
-  check();
-}
-
-#ifdef TOR_UNIT_TESTS
-/* Return the data from the first chunk of buf in cp, and its length in sz. */
-void
-buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz)
-{
-  if (!buf || !buf->head) {
-    *cp = NULL;
-    *sz = 0;
-  } else {
-    *cp = buf->head->data;
-    *sz = buf->head->datalen;
-  }
-}
-
-/* Write sz bytes from cp into a newly allocated buffer buf.
- * Returns NULL when passed a NULL cp or zero sz.
- * Asserts on failure: only for use in unit tests.
- * buf must be freed using buf_free(). */
-buf_t *
-buf_new_with_data(const char *cp, size_t sz)
-{
-  /* Validate arguments */
-  if (!cp || sz <= 0) {
-    return NULL;
-  }
-
-  tor_assert(sz < SSIZE_T_CEILING);
-
-  /* Allocate a buffer */
-  buf_t *buf = buf_new_with_capacity(sz);
-  tor_assert(buf);
-  assert_buf_ok(buf);
-  tor_assert(!buf->head);
-
-  /* Allocate a chunk that is sz bytes long */
-  buf->head = chunk_new_with_alloc_size(CHUNK_ALLOC_SIZE(sz));
-  buf->tail = buf->head;
-  tor_assert(buf->head);
-  assert_buf_ok(buf);
-  tor_assert(buf_allocation(buf) >= sz);
-
-  /* Copy the data and size the buffers */
-  tor_assert(sz <= buf_slack(buf));
-  tor_assert(sz <= CHUNK_REMAINING_CAPACITY(buf->head));
-  memcpy(&buf->head->mem[0], cp, sz);
-  buf->datalen = sz;
-  buf->head->datalen = sz;
-  buf->head->data = &buf->head->mem[0];
-  assert_buf_ok(buf);
-
-  /* Make sure everything is large enough */
-  tor_assert(buf_allocation(buf) >= sz);
-  tor_assert(buf_allocation(buf) >= buf_datalen(buf) + buf_slack(buf));
-  /* Does the buffer implementation allocate more than the requested size?
-   * (for example, by rounding up). If so, these checks will fail. */
-  tor_assert(buf_datalen(buf) == sz);
-  tor_assert(buf_slack(buf) == 0);
-
-  return buf;
-}
-#endif
-
-/** Remove the first <b>n</b> bytes from buf. */
-static inline void
-buf_remove_from_front(buf_t *buf, size_t n)
-{
-  tor_assert(buf->datalen >= n);
-  while (n) {
-    tor_assert(buf->head);
-    if (buf->head->datalen > n) {
-      buf->head->datalen -= n;
-      buf->head->data += n;
-      buf->datalen -= n;
-      return;
-    } else {
-      chunk_t *victim = buf->head;
-      n -= victim->datalen;
-      buf->datalen -= victim->datalen;
-      buf->head = victim->next;
-      if (buf->tail == victim)
-        buf->tail = NULL;
-      buf_chunk_free_unchecked(victim);
-    }
-  }
-  check();
-}
-
-/** Create and return a new buf with default chunk capacity <b>size</b>.
- */
-buf_t *
-buf_new_with_capacity(size_t size)
-{
-  buf_t *b = buf_new();
-  b->default_chunk_size = preferred_chunk_size(size);
-  return b;
-}
-
-/** Allocate and return a new buffer with default capacity. */
-buf_t *
-buf_new(void)
-{
-  buf_t *buf = tor_malloc_zero(sizeof(buf_t));
-  buf->magic = BUFFER_MAGIC;
-  buf->default_chunk_size = 4096;
-  return buf;
-}
-
-size_t
-buf_get_default_chunk_size(const buf_t *buf)
-{
-  return buf->default_chunk_size;
-}
-
-/** Remove all data from <b>buf</b>. */
-void
-buf_clear(buf_t *buf)
-{
-  chunk_t *chunk, *next;
-  buf->datalen = 0;
-  for (chunk = buf->head; chunk; chunk = next) {
-    next = chunk->next;
-    buf_chunk_free_unchecked(chunk);
-  }
-  buf->head = buf->tail = NULL;
-}
-
-/** Return the number of bytes stored in <b>buf</b> */
-MOCK_IMPL(size_t,
-buf_datalen, (const buf_t *buf))
-{
-  return buf->datalen;
-}
-
-/** Return the total length of all chunks used in <b>buf</b>. */
-size_t
-buf_allocation(const buf_t *buf)
-{
-  size_t total = 0;
-  const chunk_t *chunk;
-  for (chunk = buf->head; chunk; chunk = chunk->next) {
-    total += CHUNK_ALLOC_SIZE(chunk->memlen);
-  }
-  return total;
-}
-
-/** Return the number of bytes that can be added to <b>buf</b> without
- * performing any additional allocation. */
-size_t
-buf_slack(const buf_t *buf)
-{
-  if (!buf->tail)
-    return 0;
-  else
-    return CHUNK_REMAINING_CAPACITY(buf->tail);
-}
-
-/** Release storage held by <b>buf</b>. */
-void
-buf_free(buf_t *buf)
-{
-  if (!buf)
-    return;
-
-  buf_clear(buf);
-  buf->magic = 0xdeadbeef;
-  tor_free(buf);
-}
-
-/** Return a new copy of <b>in_chunk</b> */
-static chunk_t *
-chunk_copy(const chunk_t *in_chunk)
-{
-  chunk_t *newch = tor_memdup(in_chunk, CHUNK_ALLOC_SIZE(in_chunk->memlen));
-  total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(in_chunk->memlen);
-#ifdef DEBUG_CHUNK_ALLOC
-  newch->DBG_alloc = CHUNK_ALLOC_SIZE(in_chunk->memlen);
-#endif
-  newch->next = NULL;
-  if (in_chunk->data) {
-    off_t offset = in_chunk->data - in_chunk->mem;
-    newch->data = newch->mem + offset;
-  }
-  return newch;
-}
-
-/** Return a new copy of <b>buf</b> */
-buf_t *
-buf_copy(const buf_t *buf)
-{
-  chunk_t *ch;
-  buf_t *out = buf_new();
-  out->default_chunk_size = buf->default_chunk_size;
-  for (ch = buf->head; ch; ch = ch->next) {
-    chunk_t *newch = chunk_copy(ch);
-    if (out->tail) {
-      out->tail->next = newch;
-      out->tail = newch;
-    } else {
-      out->head = out->tail = newch;
-    }
-  }
-  out->datalen = buf->datalen;
-  return out;
-}
-
-/** Append a new chunk with enough capacity to hold <b>capacity</b> bytes to
- * the tail of <b>buf</b>.  If <b>capped</b>, don't allocate a chunk bigger
- * than MAX_CHUNK_ALLOC. */
-static chunk_t *
-buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped)
-{
-  chunk_t *chunk;
-
-  if (CHUNK_ALLOC_SIZE(capacity) < buf->default_chunk_size) {
-    chunk = chunk_new_with_alloc_size(buf->default_chunk_size);
-  } else if (capped && CHUNK_ALLOC_SIZE(capacity) > MAX_CHUNK_ALLOC) {
-    chunk = chunk_new_with_alloc_size(MAX_CHUNK_ALLOC);
-  } else {
-    chunk = chunk_new_with_alloc_size(preferred_chunk_size(capacity));
-  }
-
-  chunk->inserted_time = (uint32_t)monotime_coarse_absolute_msec();
-
-  if (buf->tail) {
-    tor_assert(buf->head);
-    buf->tail->next = chunk;
-    buf->tail = chunk;
-  } else {
-    tor_assert(!buf->head);
-    buf->head = buf->tail = chunk;
-  }
-  check();
-  return chunk;
-}
-
-/** Return the age of the oldest chunk in the buffer <b>buf</b>, in
- * milliseconds.  Requires the current monotonic time, in truncated msec,
- * as its input <b>now</b>.
- */
-uint32_t
-buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now)
-{
-  if (buf->head) {
-    return now - buf->head->inserted_time;
-  } else {
-    return 0;
-  }
-}
-
-size_t
-buf_get_total_allocation(void)
-{
-  return total_bytes_allocated_in_chunks;
-}
-
-/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into
- * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set
- * *<b>reached_eof</b> to 1.  Return -1 on error, 0 on eof or blocking,
- * and the number of bytes read otherwise. */
-static inline int
-read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most,
-              int *reached_eof, int *socket_error)
-{
-  ssize_t read_result;
-  if (at_most > CHUNK_REMAINING_CAPACITY(chunk))
-    at_most = CHUNK_REMAINING_CAPACITY(chunk);
-  read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0);
-
-  if (read_result < 0) {
-    int e = tor_socket_errno(fd);
-    if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
-#ifdef _WIN32
-      if (e == WSAENOBUFS)
-        log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?");
-#endif
-      *socket_error = e;
-      return -1;
-    }
-    return 0; /* would block. */
-  } else if (read_result == 0) {
-    log_debug(LD_NET,"Encountered eof on fd %d", (int)fd);
-    *reached_eof = 1;
-    return 0;
-  } else { /* actually got bytes. */
-    buf->datalen += read_result;
-    chunk->datalen += read_result;
-    log_debug(LD_NET,"Read %ld bytes. %d on inbuf.", (long)read_result,
-              (int)buf->datalen);
-    tor_assert(read_result < INT_MAX);
-    return (int)read_result;
-  }
-}
-
-/** As read_to_chunk(), but return (negative) error code on error, blocking,
- * or TLS, and the number of bytes read otherwise. */
-static inline int
-read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls,
-                  size_t at_most)
-{
-  int read_result;
-
-  tor_assert(CHUNK_REMAINING_CAPACITY(chunk) >= at_most);
-  read_result = tor_tls_read(tls, CHUNK_WRITE_PTR(chunk), at_most);
-  if (read_result < 0)
-    return read_result;
-  buf->datalen += read_result;
-  chunk->datalen += read_result;
-  return read_result;
-}
-
-/** Read from socket <b>s</b>, writing onto end of <b>buf</b>.  Read at most
- * <b>at_most</b> bytes, growing the buffer as necessary.  If recv() returns 0
- * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
- * error; else return the number of bytes read.
- */
-/* XXXX indicate "read blocked" somehow? */
-int
-read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof,
-            int *socket_error)
-{
-  /* XXXX It's stupid to overload the return values for these functions:
-   * "error status" and "number of bytes read" are not mutually exclusive.
-   */
-  int r = 0;
-  size_t total_read = 0;
-
-  check();
-  tor_assert(reached_eof);
-  tor_assert(SOCKET_OK(s));
-
-  if (BUG(buf->datalen >= INT_MAX))
-    return -1;
-  if (BUG(buf->datalen >= INT_MAX - at_most))
-    return -1;
-
-  while (at_most > total_read) {
-    size_t readlen = at_most - total_read;
-    chunk_t *chunk;
-    if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) {
-      chunk = buf_add_chunk_with_capacity(buf, at_most, 1);
-      if (readlen > chunk->memlen)
-        readlen = chunk->memlen;
-    } else {
-      size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail);
-      chunk = buf->tail;
-      if (cap < readlen)
-        readlen = cap;
-    }
-
-    r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error);
-    check();
-    if (r < 0)
-      return r; /* Error */
-    tor_assert(total_read+r < INT_MAX);
-    total_read += r;
-    if ((size_t)r < readlen) { /* eof, block, or no more to read. */
-      break;
-    }
-  }
-  return (int)total_read;
-}
-
-/** As read_to_buf, but reads from a TLS connection, and returns a TLS
- * status value rather than the number of bytes read.
- *
- * Using TLS on OR connections complicates matters in two ways.
- *
- * First, a TLS stream has its own read buffer independent of the
- * connection's read buffer.  (TLS needs to read an entire frame from
- * the network before it can decrypt any data.  Thus, trying to read 1
- * byte from TLS can require that several KB be read from the network
- * and decrypted.  The extra data is stored in TLS's decrypt buffer.)
- * Because the data hasn't been read by Tor (it's still inside the TLS),
- * this means that sometimes a connection "has stuff to read" even when
- * poll() didn't return POLLIN. The tor_tls_get_pending_bytes function is
- * used in connection.c to detect TLS objects with non-empty internal
- * buffers and read from them again.
- *
- * Second, the TLS stream's events do not correspond directly to network
- * events: sometimes, before a TLS stream can read, the network must be
- * ready to write -- or vice versa.
- */
-int
-read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf)
-{
-  int r = 0;
-  size_t total_read = 0;
-
-  check_no_tls_errors();
-
-  check();
-
-  if (BUG(buf->datalen >= INT_MAX))
-    return -1;
-  if (BUG(buf->datalen >= INT_MAX - at_most))
-    return -1;
-
-  while (at_most > total_read) {
-    size_t readlen = at_most - total_read;
-    chunk_t *chunk;
-    if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) {
-      chunk = buf_add_chunk_with_capacity(buf, at_most, 1);
-      if (readlen > chunk->memlen)
-        readlen = chunk->memlen;
-    } else {
-      size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail);
-      chunk = buf->tail;
-      if (cap < readlen)
-        readlen = cap;
-    }
-
-    r = read_to_chunk_tls(buf, chunk, tls, readlen);
-    check();
-    if (r < 0)
-      return r; /* Error */
-    tor_assert(total_read+r < INT_MAX);
-    total_read += r;
-    if ((size_t)r < readlen) /* eof, block, or no more to read. */
-      break;
-  }
-  return (int)total_read;
-}
-
-/** Helper for flush_buf(): try to write <b>sz</b> bytes from chunk
- * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  On success, deduct
- * the bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
- * written on success, 0 on blocking, -1 on failure.
- */
-static inline int
-flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz,
-            size_t *buf_flushlen)
-{
-  ssize_t write_result;
-
-  if (sz > chunk->datalen)
-    sz = chunk->datalen;
-  write_result = tor_socket_send(s, chunk->data, sz, 0);
-
-  if (write_result < 0) {
-    int e = tor_socket_errno(s);
-    if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
-#ifdef _WIN32
-      if (e == WSAENOBUFS)
-        log_warn(LD_NET,"write() failed: WSAENOBUFS. Not enough ram?");
-#endif
-      return -1;
-    }
-    log_debug(LD_NET,"write() would block, returning.");
-    return 0;
-  } else {
-    *buf_flushlen -= write_result;
-    buf_remove_from_front(buf, write_result);
-    tor_assert(write_result < INT_MAX);
-    return (int)write_result;
-  }
-}
-
-/** Helper for flush_buf_tls(): try to write <b>sz</b> bytes from chunk
- * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  (Tries to write
- * more if there is a forced pending write size.)  On success, deduct the
- * bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
- * written on success, and a TOR_TLS error code on failure or blocking.
- */
-static inline int
-flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk,
-                size_t sz, size_t *buf_flushlen)
-{
-  int r;
-  size_t forced;
-  char *data;
-
-  forced = tor_tls_get_forced_write_size(tls);
-  if (forced > sz)
-    sz = forced;
-  if (chunk) {
-    data = chunk->data;
-    tor_assert(sz <= chunk->datalen);
-  } else {
-    data = NULL;
-    tor_assert(sz == 0);
-  }
-  r = tor_tls_write(tls, data, sz);
-  if (r < 0)
-    return r;
-  if (*buf_flushlen > (size_t)r)
-    *buf_flushlen -= r;
-  else
-    *buf_flushlen = 0;
-  buf_remove_from_front(buf, r);
-  log_debug(LD_NET,"flushed %d bytes, %d ready to flush, %d remain.",
-            r,(int)*buf_flushlen,(int)buf->datalen);
-  return r;
-}
-
-/** Write data from <b>buf</b> to the socket <b>s</b>.  Write at most
- * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
- * the number of bytes actually written, and remove the written bytes
- * from the buffer.  Return the number of bytes written on success,
- * -1 on failure.  Return 0 if write() would block.
- */
-int
-flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen)
-{
-  /* XXXX It's stupid to overload the return values for these functions:
-   * "error status" and "number of bytes flushed" are not mutually exclusive.
-   */
-  int r;
-  size_t flushed = 0;
-  tor_assert(buf_flushlen);
-  tor_assert(SOCKET_OK(s));
-  tor_assert(*buf_flushlen <= buf->datalen);
-  tor_assert(sz <= *buf_flushlen);
-
-  check();
-  while (sz) {
-    size_t flushlen0;
-    tor_assert(buf->head);
-    if (buf->head->datalen >= sz)
-      flushlen0 = sz;
-    else
-      flushlen0 = buf->head->datalen;
-
-    r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen);
-    check();
-    if (r < 0)
-      return r;
-    flushed += r;
-    sz -= r;
-    if (r == 0 || (size_t)r < flushlen0) /* can't flush any more now. */
-      break;
-  }
-  tor_assert(flushed < INT_MAX);
-  return (int)flushed;
-}
-
-/** As flush_buf(), but writes data to a TLS connection.  Can write more than
- * <b>flushlen</b> bytes.
- */
-int
-flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t flushlen,
-              size_t *buf_flushlen)
-{
-  int r;
-  size_t flushed = 0;
-  ssize_t sz;
-  tor_assert(buf_flushlen);
-  tor_assert(*buf_flushlen <= buf->datalen);
-  tor_assert(flushlen <= *buf_flushlen);
-  sz = (ssize_t) flushlen;
-
-  /* we want to let tls write even if flushlen is zero, because it might
-   * have a partial record pending */
-  check_no_tls_errors();
-
-  check();
-  do {
-    size_t flushlen0;
-    if (buf->head) {
-      if ((ssize_t)buf->head->datalen >= sz)
-        flushlen0 = sz;
-      else
-        flushlen0 = buf->head->datalen;
-    } else {
-      flushlen0 = 0;
-    }
-
-    r = flush_chunk_tls(tls, buf, buf->head, flushlen0, buf_flushlen);
-    check();
-    if (r < 0)
-      return r;
-    flushed += r;
-    sz -= r;
-    if (r == 0) /* Can't flush any more now. */
-      break;
-  } while (sz > 0);
-  tor_assert(flushed < INT_MAX);
-  return (int)flushed;
-}
-
-/** Append <b>string_len</b> bytes from <b>string</b> to the end of
- * <b>buf</b>.
- *
- * Return the new length of the buffer on success, -1 on failure.
- */
-int
-write_to_buf(const char *string, size_t string_len, buf_t *buf)
-{
-  if (!string_len)
-    return (int)buf->datalen;
-  check();
-
-  if (BUG(buf->datalen >= INT_MAX))
-    return -1;
-  if (BUG(buf->datalen >= INT_MAX - string_len))
-    return -1;
-
-  while (string_len) {
-    size_t copy;
-    if (!buf->tail || !CHUNK_REMAINING_CAPACITY(buf->tail))
-      buf_add_chunk_with_capacity(buf, string_len, 1);
-
-    copy = CHUNK_REMAINING_CAPACITY(buf->tail);
-    if (copy > string_len)
-      copy = string_len;
-    memcpy(CHUNK_WRITE_PTR(buf->tail), string, copy);
-    string_len -= copy;
-    string += copy;
-    buf->datalen += copy;
-    buf->tail->datalen += copy;
-  }
-
-  check();
-  tor_assert(buf->datalen < INT_MAX);
-  return (int)buf->datalen;
-}
-
-/** Helper: copy the first <b>string_len</b> bytes from <b>buf</b>
- * onto <b>string</b>.
- */
-static inline void
-peek_from_buf(char *string, size_t string_len, const buf_t *buf)
-{
-  chunk_t *chunk;
-
-  tor_assert(string);
-  /* make sure we don't ask for too much */
-  tor_assert(string_len <= buf->datalen);
-  /* assert_buf_ok(buf); */
-
-  chunk = buf->head;
-  while (string_len) {
-    size_t copy = string_len;
-    tor_assert(chunk);
-    if (chunk->datalen < copy)
-      copy = chunk->datalen;
-    memcpy(string, chunk->data, copy);
-    string_len -= copy;
-    string += copy;
-    chunk = chunk->next;
-  }
-}
-
-/** Remove <b>string_len</b> bytes from the front of <b>buf</b>, and store
- * them into <b>string</b>.  Return the new buffer size.  <b>string_len</b>
- * must be \<= the number of bytes on the buffer.
- */
-int
-fetch_from_buf(char *string, size_t string_len, buf_t *buf)
-{
-  /* There must be string_len bytes in buf; write them onto string,
-   * then memmove buf back (that is, remove them from buf).
-   *
-   * Return the number of bytes still on the buffer. */
-
-  check();
-  peek_from_buf(string, string_len, buf);
-  buf_remove_from_front(buf, string_len);
-  check();
-  tor_assert(buf->datalen < INT_MAX);
-  return (int)buf->datalen;
-}
-
-/** True iff the cell command <b>command</b> is one that implies a
- * variable-length cell in Tor link protocol <b>linkproto</b>. */
-static inline int
-cell_command_is_var_length(uint8_t command, int linkproto)
-{
-  /* If linkproto is v2 (2), CELL_VERSIONS is the only variable-length cells
-   * work as implemented here. If it's 1, there are no variable-length cells.
-   * Tor does not support other versions right now, and so can't negotiate
-   * them.
-   */
-  switch (linkproto) {
-  case 1:
-    /* Link protocol version 1 has no variable-length cells. */
-    return 0;
-  case 2:
-    /* In link protocol version 2, VERSIONS is the only variable-length cell */
-    return command == CELL_VERSIONS;
-  case 0:
-  case 3:
-  default:
-    /* In link protocol version 3 and later, and in version "unknown",
-     * commands 128 and higher indicate variable-length. VERSIONS is
-     * grandfathered in. */
-    return command == CELL_VERSIONS || command >= 128;
-  }
-}
-
-/** Check <b>buf</b> for a variable-length cell according to the rules of link
- * protocol version <b>linkproto</b>.  If one is found, pull it off the buffer
- * and assign a newly allocated var_cell_t to *<b>out</b>, and return 1.
- * Return 0 if whatever is on the start of buf_t is not a variable-length
- * cell.  Return 1 and set *<b>out</b> to NULL if there seems to be the start
- * of a variable-length cell on <b>buf</b>, but the whole thing isn't there
- * yet. */
-int
-fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto)
-{
-  char hdr[VAR_CELL_MAX_HEADER_SIZE];
-  var_cell_t *result;
-  uint8_t command;
-  uint16_t length;
-  const int wide_circ_ids = linkproto >= MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS;
-  const int circ_id_len = get_circ_id_size(wide_circ_ids);
-  const unsigned header_len = get_var_cell_header_size(wide_circ_ids);
-  check();
-  *out = NULL;
-  if (buf->datalen < header_len)
-    return 0;
-  peek_from_buf(hdr, header_len, buf);
-
-  command = get_uint8(hdr + circ_id_len);
-  if (!(cell_command_is_var_length(command, linkproto)))
-    return 0;
-
-  length = ntohs(get_uint16(hdr + circ_id_len + 1));
-  if (buf->datalen < (size_t)(header_len+length))
-    return 1;
-  result = var_cell_new(length);
-  result->command = command;
-  if (wide_circ_ids)
-    result->circ_id = ntohl(get_uint32(hdr));
-  else
-    result->circ_id = ntohs(get_uint16(hdr));
-
-  buf_remove_from_front(buf, header_len);
-  peek_from_buf((char*) result->payload, length, buf);
-  buf_remove_from_front(buf, length);
-  check();
-
-  *out = result;
-  return 1;
-}
-
-/** Move up to *<b>buf_flushlen</b> bytes from <b>buf_in</b> to
- * <b>buf_out</b>, and modify *<b>buf_flushlen</b> appropriately.
- * Return the number of bytes actually copied.
- */
-int
-move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen)
-{
-  /* We can do way better here, but this doesn't turn up in any profiles. */
-  char b[4096];
-  size_t cp, len;
-
-  if (BUG(buf_out->datalen >= INT_MAX))
-    return -1;
-  if (BUG(buf_out->datalen >= INT_MAX - *buf_flushlen))
-    return -1;
-
-  len = *buf_flushlen;
-  if (len > buf_in->datalen)
-    len = buf_in->datalen;
-
-  cp = len; /* Remember the number of bytes we intend to copy. */
-  tor_assert(cp < INT_MAX);
-  while (len) {
-    /* This isn't the most efficient implementation one could imagine, since
-     * it does two copies instead of 1, but I kinda doubt that this will be
-     * critical path. */
-    size_t n = len > sizeof(b) ? sizeof(b) : len;
-    fetch_from_buf(b, n, buf_in);
-    write_to_buf(b, n, buf_out);
-    len -= n;
-  }
-  *buf_flushlen -= cp;
-  return (int)cp;
-}
-
-/** Internal structure: represents a position in a buffer. */
-typedef struct buf_pos_t {
-  const chunk_t *chunk; /**< Which chunk are we pointing to? */
-  int pos;/**< Which character inside the chunk's data are we pointing to? */
-  size_t chunk_pos; /**< Total length of all previous chunks. */
-} buf_pos_t;
-
-/** Initialize <b>out</b> to point to the first character of <b>buf</b>.*/
-static void
-buf_pos_init(const buf_t *buf, buf_pos_t *out)
-{
-  out->chunk = buf->head;
-  out->pos = 0;
-  out->chunk_pos = 0;
-}
-
-/** Advance <b>out</b> to the first appearance of <b>ch</b> at the current
- * position of <b>out</b>, or later.  Return -1 if no instances are found;
- * otherwise returns the absolute position of the character. */
-static off_t
-buf_find_pos_of_char(char ch, buf_pos_t *out)
-{
-  const chunk_t *chunk;
-  int pos;
-  tor_assert(out);
-  if (out->chunk) {
-    if (out->chunk->datalen) {
-      tor_assert(out->pos < (off_t)out->chunk->datalen);
-    } else {
-      tor_assert(out->pos == 0);
-    }
-  }
-  pos = out->pos;
-  for (chunk = out->chunk; chunk; chunk = chunk->next) {
-    char *cp = memchr(chunk->data+pos, ch, chunk->datalen - pos);
-    if (cp) {
-      out->chunk = chunk;
-      tor_assert(cp - chunk->data < INT_MAX);
-      out->pos = (int)(cp - chunk->data);
-      return out->chunk_pos + out->pos;
-    } else {
-      out->chunk_pos += chunk->datalen;
-      pos = 0;
-    }
-  }
-  return -1;
-}
-
-/** Advance <b>pos</b> by a single character, if there are any more characters
- * in the buffer.  Returns 0 on success, -1 on failure. */
-static inline int
-buf_pos_inc(buf_pos_t *pos)
-{
-  ++pos->pos;
-  if (pos->pos == (off_t)pos->chunk->datalen) {
-    if (!pos->chunk->next)
-      return -1;
-    pos->chunk_pos += pos->chunk->datalen;
-    pos->chunk = pos->chunk->next;
-    pos->pos = 0;
-  }
-  return 0;
-}
-
-/** Return true iff the <b>n</b>-character string in <b>s</b> appears
- * (verbatim) at <b>pos</b>. */
-static int
-buf_matches_at_pos(const buf_pos_t *pos, const char *s, size_t n)
-{
-  buf_pos_t p;
-  if (!n)
-    return 1;
-
-  memcpy(&p, pos, sizeof(p));
-
-  while (1) {
-    char ch = p.chunk->data[p.pos];
-    if (ch != *s)
-      return 0;
-    ++s;
-    /* If we're out of characters that don't match, we match.  Check this
-     * _before_ we test incrementing pos, in case we're at the end of the
-     * string. */
-    if (--n == 0)
-      return 1;
-    if (buf_pos_inc(&p)<0)
-      return 0;
-  }
-}
-
-/** Return the first position in <b>buf</b> at which the <b>n</b>-character
- * string <b>s</b> occurs, or -1 if it does not occur. */
-STATIC int
-buf_find_string_offset(const buf_t *buf, const char *s, size_t n)
-{
-  buf_pos_t pos;
-  buf_pos_init(buf, &pos);
-  while (buf_find_pos_of_char(*s, &pos) >= 0) {
-    if (buf_matches_at_pos(&pos, s, n)) {
-      tor_assert(pos.chunk_pos + pos.pos < INT_MAX);
-      return (int)(pos.chunk_pos + pos.pos);
-    } else {
-      if (buf_pos_inc(&pos)<0)
-        return -1;
-    }
-  }
-  return -1;
-}
-
-/**
- * Scan the HTTP headers in the <b>headerlen</b>-byte memory range at
- * <b>headers</b>, looking for a "Content-Length" header.  Try to set
- * *<b>result_out</b> to the numeric value of that header if possible.
- * Return -1 if the header was malformed, 0 if it was missing, and 1 if
- * it was present and well-formed.
- */
-STATIC int
-buf_http_find_content_length(const char *headers, size_t headerlen,
-                             size_t *result_out)
-{
-  const char *p, *newline;
-  char *len_str, *eos=NULL;
-  size_t remaining, result;
-  int ok;
-  *result_out = 0; /* The caller shouldn't look at this unless the
-                    * return value is 1, but let's prevent confusion */
-
-#define CONTENT_LENGTH "\r\nContent-Length: "
-  p = (char*) tor_memstr(headers, headerlen, CONTENT_LENGTH);
-  if (p == NULL)
-    return 0;
-
-  tor_assert(p >= headers && p < headers+headerlen);
-  remaining = (headers+headerlen)-p;
-  p += strlen(CONTENT_LENGTH);
-  remaining -= strlen(CONTENT_LENGTH);
-
-  newline = memchr(p, '\n', remaining);
-  if (newline == NULL)
-    return -1;
-
-  len_str = tor_memdup_nulterm(p, newline-p);
-  /* We limit the size to INT_MAX because other parts of the buffer.c
-   * code don't like buffers to be any bigger than that. */
-  result = (size_t) tor_parse_uint64(len_str, 10, 0, INT_MAX, &ok, &eos);
-  if (eos && !tor_strisspace(eos)) {
-    ok = 0;
-  } else {
-    *result_out = result;
-  }
-  tor_free(len_str);
-
-  return ok ? 1 : -1;
-}
-
-/** There is a (possibly incomplete) http statement on <b>buf</b>, of the
- * form "\%s\\r\\n\\r\\n\%s", headers, body. (body may contain NULs.)
- * If a) the headers include a Content-Length field and all bytes in
- * the body are present, or b) there's no Content-Length field and
- * all headers are present, then:
- *
- *  - strdup headers into <b>*headers_out</b>, and NUL-terminate it.
- *  - memdup body into <b>*body_out</b>, and NUL-terminate it.
- *  - Then remove them from <b>buf</b>, and return 1.
- *
- *  - If headers or body is NULL, discard that part of the buf.
- *  - If a headers or body doesn't fit in the arg, return -1.
- *  (We ensure that the headers or body don't exceed max len,
- *   _even if_ we're planning to discard them.)
- *  - If force_complete is true, then succeed even if not all of the
- *    content has arrived.
- *
- * Else, change nothing and return 0.
- */
-int
-fetch_from_buf_http(buf_t *buf,
-                    char **headers_out, size_t max_headerlen,
-                    char **body_out, size_t *body_used, size_t max_bodylen,
-                    int force_complete)
-{
-  char *headers;
-  size_t headerlen, bodylen, contentlen=0;
-  int crlf_offset;
-  int r;
-
-  check();
-  if (!buf->head)
-    return 0;
-
-  crlf_offset = buf_find_string_offset(buf, "\r\n\r\n", 4);
-  if (crlf_offset > (int)max_headerlen ||
-      (crlf_offset < 0 && buf->datalen > max_headerlen)) {
-    log_debug(LD_HTTP,"headers too long.");
-    return -1;
-  } else if (crlf_offset < 0) {
-    log_debug(LD_HTTP,"headers not all here yet.");
-    return 0;
-  }
-  /* Okay, we have a full header.  Make sure it all appears in the first
-   * chunk. */
-  if ((int)buf->head->datalen < crlf_offset + 4)
-    buf_pullup(buf, crlf_offset+4);
-  headerlen = crlf_offset + 4;
-
-  headers = buf->head->data;
-  bodylen = buf->datalen - headerlen;
-  log_debug(LD_HTTP,"headerlen %d, bodylen %d.", (int)headerlen, (int)bodylen);
-
-  if (max_headerlen <= headerlen) {
-    log_warn(LD_HTTP,"headerlen %d larger than %d. Failing.",
-             (int)headerlen, (int)max_headerlen-1);
-    return -1;
-  }
-  if (max_bodylen <= bodylen) {
-    log_warn(LD_HTTP,"bodylen %d larger than %d. Failing.",
-             (int)bodylen, (int)max_bodylen-1);
-    return -1;
-  }
-
-  r = buf_http_find_content_length(headers, headerlen, &contentlen);
-  if (r == -1) {
-    log_warn(LD_PROTOCOL, "Content-Length is bogus; maybe "
-             "someone is trying to crash us.");
-    return -1;
-  } else if (r == 1) {
-    /* if content-length is malformed, then our body length is 0. fine. */
-    log_debug(LD_HTTP,"Got a contentlen of %d.",(int)contentlen);
-    if (bodylen < contentlen) {
-      if (!force_complete) {
-        log_debug(LD_HTTP,"body not all here yet.");
-        return 0; /* not all there yet */
-      }
-    }
-    if (bodylen > contentlen) {
-      bodylen = contentlen;
-      log_debug(LD_HTTP,"bodylen reduced to %d.",(int)bodylen);
-    }
-  } else {
-    tor_assert(r == 0);
-    /* Leave bodylen alone */
-  }
-
-  /* all happy. copy into the appropriate places, and return 1 */
-  if (headers_out) {
-    *headers_out = tor_malloc(headerlen+1);
-    fetch_from_buf(*headers_out, headerlen, buf);
-    (*headers_out)[headerlen] = 0; /* NUL terminate it */
-  }
-  if (body_out) {
-    tor_assert(body_used);
-    *body_used = bodylen;
-    *body_out = tor_malloc(bodylen+1);
-    fetch_from_buf(*body_out, bodylen, buf);
-    (*body_out)[bodylen] = 0; /* NUL terminate it */
-  }
-  check();
-  return 1;
-}
-
-/**
- * Wait this many seconds before warning the user about using SOCKS unsafely
- * again. */
-#define SOCKS_WARN_INTERVAL 5
-
-/** Warn that the user application has made an unsafe socks request using
- * protocol <b>socks_protocol</b> on port <b>port</b>.  Don't warn more than
- * once per SOCKS_WARN_INTERVAL, unless <b>safe_socks</b> is set. */
-static void
-log_unsafe_socks_warning(int socks_protocol, const char *address,
-                         uint16_t port, int safe_socks)
-{
-  static ratelim_t socks_ratelim = RATELIM_INIT(SOCKS_WARN_INTERVAL);
-
-  if (safe_socks) {
-    log_fn_ratelim(&socks_ratelim, LOG_WARN, LD_APP,
-             "Your application (using socks%d to port %d) is giving "
-             "Tor only an IP address. Applications that do DNS resolves "
-             "themselves may leak information. Consider using Socks4A "
-             "(e.g. via privoxy or socat) instead. For more information, "
-             "please see https://wiki.torproject.org/TheOnionRouter/"
-             "TorFAQ#SOCKSAndDNS.%s",
-             socks_protocol,
-             (int)port,
-             safe_socks ? " Rejecting." : "");
-  }
-  control_event_client_status(LOG_WARN,
-                              "DANGEROUS_SOCKS PROTOCOL=SOCKS%d ADDRESS=%s:%d",
-                              socks_protocol, address, (int)port);
-}
-
-/** Do not attempt to parse socks messages longer than this.  This value is
- * actually significantly higher than the longest possible socks message. */
-#define MAX_SOCKS_MESSAGE_LEN 512
-
-/** Return a new socks_request_t. */
-socks_request_t *
-socks_request_new(void)
-{
-  return tor_malloc_zero(sizeof(socks_request_t));
-}
-
-/** Free all storage held in the socks_request_t <b>req</b>. */
-void
-socks_request_free(socks_request_t *req)
-{
-  if (!req)
-    return;
-  if (req->username) {
-    memwipe(req->username, 0x10, req->usernamelen);
-    tor_free(req->username);
-  }
-  if (req->password) {
-    memwipe(req->password, 0x04, req->passwordlen);
-    tor_free(req->password);
-  }
-  memwipe(req, 0xCC, sizeof(socks_request_t));
-  tor_free(req);
-}
-
-/** There is a (possibly incomplete) socks handshake on <b>buf</b>, of one
- * of the forms
- *  - socks4: "socksheader username\\0"
- *  - socks4a: "socksheader username\\0 destaddr\\0"
- *  - socks5 phase one: "version #methods methods"
- *  - socks5 phase two: "version command 0 addresstype..."
- * If it's a complete and valid handshake, and destaddr fits in
- *   MAX_SOCKS_ADDR_LEN bytes, then pull the handshake off the buf,
- *   assign to <b>req</b>, and return 1.
- *
- * If it's invalid or too big, return -1.
- *
- * Else it's not all there yet, leave buf alone and return 0.
- *
- * If you want to specify the socks reply, write it into <b>req->reply</b>
- *   and set <b>req->replylen</b>, else leave <b>req->replylen</b> alone.
- *
- * If <b>log_sockstype</b> is non-zero, then do a notice-level log of whether
- * the connection is possibly leaking DNS requests locally or not.
- *
- * If <b>safe_socks</b> is true, then reject unsafe socks protocols.
- *
- * If returning 0 or -1, <b>req->address</b> and <b>req->port</b> are
- * undefined.
- */
-int
-fetch_from_buf_socks(buf_t *buf, socks_request_t *req,
-                     int log_sockstype, int safe_socks)
-{
-  int res;
-  ssize_t n_drain;
-  size_t want_length = 128;
-
-  if (buf->datalen < 2) /* version and another byte */
-    return 0;
-
-  do {
-    n_drain = 0;
-    buf_pullup(buf, want_length);
-    tor_assert(buf->head && buf->head->datalen >= 2);
-    want_length = 0;
-
-    res = parse_socks(buf->head->data, buf->head->datalen, req, log_sockstype,
-                      safe_socks, &n_drain, &want_length);
-
-    if (n_drain < 0)
-      buf_clear(buf);
-    else if (n_drain > 0)
-      buf_remove_from_front(buf, n_drain);
-
-  } while (res == 0 && buf->head && want_length < buf->datalen &&
-           buf->datalen >= 2);
-
-  return res;
-}
-
-/** The size of the header of an Extended ORPort message: 2 bytes for
- *  COMMAND, 2 bytes for BODYLEN */
-#define EXT_OR_CMD_HEADER_SIZE 4
-
-/** Read <b>buf</b>, which should contain an Extended ORPort message
- *  from a transport proxy. If well-formed, create and populate
- *  <b>out</b> with the Extended ORport message. Return 0 if the
- *  buffer was incomplete, 1 if it was well-formed and -1 if we
- *  encountered an error while parsing it.  */
-int
-fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out)
-{
-  char hdr[EXT_OR_CMD_HEADER_SIZE];
-  uint16_t len;
-
-  check();
-  if (buf->datalen < EXT_OR_CMD_HEADER_SIZE)
-    return 0;
-  peek_from_buf(hdr, sizeof(hdr), buf);
-  len = ntohs(get_uint16(hdr+2));
-  if (buf->datalen < (unsigned)len + EXT_OR_CMD_HEADER_SIZE)
-    return 0;
-  *out = ext_or_cmd_new(len);
-  (*out)->cmd = ntohs(get_uint16(hdr));
-  (*out)->len = len;
-  buf_remove_from_front(buf, EXT_OR_CMD_HEADER_SIZE);
-  fetch_from_buf((*out)->body, len, buf);
-  return 1;
-}
-
-/** Create a SOCKS5 reply message with <b>reason</b> in its REP field and
- * have Tor send it as error response to <b>req</b>.
- */
-static void
-socks_request_set_socks5_error(socks_request_t *req,
-                  socks5_reply_status_t reason)
-{
-   req->replylen = 10;
-   memset(req->reply,0,10);
-
-   req->reply[0] = 0x05;   // VER field.
-   req->reply[1] = reason; // REP field.
-   req->reply[3] = 0x01;   // ATYP field.
-}
-
-static const char SOCKS_PROXY_IS_NOT_AN_HTTP_PROXY_MSG[] =
-  "HTTP/1.0 501 Tor is not an HTTP Proxy\r\n"
-  "Content-Type: text/html; charset=iso-8859-1\r\n\r\n"
-  "<html>\n"
-  "<head>\n"
-  "<title>Tor is not an HTTP Proxy</title>\n"
-  "</head>\n"
-  "<body>\n"
-  "<h1>Tor is not an HTTP Proxy</h1>\n"
-  "<p>\n"
-  "It appears you have configured your web browser to use Tor as "
-  "an HTTP proxy.\n\n"
-  "This is not correct: Tor is a SOCKS proxy, not an HTTP proxy.\n"
-  "Please configure your client accordingly.\n"
-  "</p>\n"
-  "<p>\n"
-  "See <a href=\"https://www.torproject.org/documentation.html\">"
-  "https://www.torproject.org/documentation.html</a> for more "
-  "information.\n"
-  "<!-- Plus this comment, to make the body response more than 512 bytes, so "
-  "     IE will be willing to display it. Comment comment comment comment "
-  "     comment comment comment comment comment comment comment comment.-->\n"
-  "</p>\n"
-  "</body>\n"
-  "</html>\n";
-
-/** Implementation helper to implement fetch_from_*_socks.  Instead of looking
- * at a buffer's contents, we look at the <b>datalen</b> bytes of data in
- * <b>data</b>. Instead of removing data from the buffer, we set
- * <b>drain_out</b> to the amount of data that should be removed (or -1 if the
- * buffer should be cleared).  Instead of pulling more data into the first
- * chunk of the buffer, we set *<b>want_length_out</b> to the number of bytes
- * we'd like to see in the input buffer, if they're available. */
-static int
-parse_socks(const char *data, size_t datalen, socks_request_t *req,
-            int log_sockstype, int safe_socks, ssize_t *drain_out,
-            size_t *want_length_out)
-{
-  unsigned int len;
-  char tmpbuf[TOR_ADDR_BUF_LEN+1];
-  tor_addr_t destaddr;
-  uint32_t destip;
-  uint8_t socksver;
-  char *next, *startaddr;
-  unsigned char usernamelen, passlen;
-  struct in_addr in;
-
-  if (datalen < 2) {
-    /* We always need at least 2 bytes. */
-    *want_length_out = 2;
-    return 0;
-  }
-
-  if (req->socks_version == 5 && !req->got_auth) {
-    /* See if we have received authentication.  Strictly speaking, we should
-       also check whether we actually negotiated username/password
-       authentication.  But some broken clients will send us authentication
-       even if we negotiated SOCKS_NO_AUTH. */
-    if (*data == 1) { /* username/pass version 1 */
-      /* Format is: authversion [1 byte] == 1
-                    usernamelen [1 byte]
-                    username    [usernamelen bytes]
-                    passlen     [1 byte]
-                    password    [passlen bytes] */
-      usernamelen = (unsigned char)*(data + 1);
-      if (datalen < 2u + usernamelen + 1u) {
-        *want_length_out = 2u + usernamelen + 1u;
-        return 0;
-      }
-      passlen = (unsigned char)*(data + 2u + usernamelen);
-      if (datalen < 2u + usernamelen + 1u + passlen) {
-        *want_length_out = 2u + usernamelen + 1u + passlen;
-        return 0;
-      }
-      req->replylen = 2; /* 2 bytes of response */
-      req->reply[0] = 1; /* authversion == 1 */
-      req->reply[1] = 0; /* authentication successful */
-      log_debug(LD_APP,
-               "socks5: Accepted username/password without checking.");
-      if (usernamelen) {
-        req->username = tor_memdup(data+2u, usernamelen);
-        req->usernamelen = usernamelen;
-      }
-      if (passlen) {
-        req->password = tor_memdup(data+3u+usernamelen, passlen);
-        req->passwordlen = passlen;
-      }
-      *drain_out = 2u + usernamelen + 1u + passlen;
-      req->got_auth = 1;
-      *want_length_out = 7; /* Minimal socks5 command. */
-      return 0;
-    } else if (req->auth_type == SOCKS_USER_PASS) {
-      /* unknown version byte */
-      log_warn(LD_APP, "Socks5 username/password version %d not recognized; "
-               "rejecting.", (int)*data);
-      return -1;
-    }
-  }
-
-  socksver = *data;
-
-  switch (socksver) { /* which version of socks? */
-    case 5: /* socks5 */
-
-      if (req->socks_version != 5) { /* we need to negotiate a method */
-        unsigned char nummethods = (unsigned char)*(data+1);
-        int have_user_pass, have_no_auth;
-        int r=0;
-        tor_assert(!req->socks_version);
-        if (datalen < 2u+nummethods) {
-          *want_length_out = 2u+nummethods;
-          return 0;
-        }
-        if (!nummethods)
-          return -1;
-        req->replylen = 2; /* 2 bytes of response */
-        req->reply[0] = 5; /* socks5 reply */
-        have_user_pass = (memchr(data+2, SOCKS_USER_PASS, nummethods) !=NULL);
-        have_no_auth   = (memchr(data+2, SOCKS_NO_AUTH,   nummethods) !=NULL);
-        if (have_user_pass && !(have_no_auth && req->socks_prefer_no_auth)) {
-          req->auth_type = SOCKS_USER_PASS;
-          req->reply[1] = SOCKS_USER_PASS; /* tell client to use "user/pass"
-                                              auth method */
-          req->socks_version = 5; /* remember we've already negotiated auth */
-          log_debug(LD_APP,"socks5: accepted method 2 (username/password)");
-          r=0;
-        } else if (have_no_auth) {
-          req->reply[1] = SOCKS_NO_AUTH; /* tell client to use "none" auth
-                                            method */
-          req->socks_version = 5; /* remember we've already negotiated auth */
-          log_debug(LD_APP,"socks5: accepted method 0 (no authentication)");
-          r=0;
-        } else {
-          log_warn(LD_APP,
-                    "socks5: offered methods don't include 'no auth' or "
-                    "username/password. Rejecting.");
-          req->reply[1] = '\xFF'; /* reject all methods */
-          r=-1;
-        }
-        /* Remove packet from buf. Some SOCKS clients will have sent extra
-         * junk at this point; let's hope it's an authentication message. */
-        *drain_out = 2u + nummethods;
-
-        return r;
-      }
-      if (req->auth_type != SOCKS_NO_AUTH && !req->got_auth) {
-        log_warn(LD_APP,
-                 "socks5: negotiated authentication, but none provided");
-        return -1;
-      }
-      /* we know the method; read in the request */
-      log_debug(LD_APP,"socks5: checking request");
-      if (datalen < 7) {/* basic info plus >=1 for addr plus 2 for port */
-        *want_length_out = 7;
-        return 0; /* not yet */
-      }
-      req->command = (unsigned char) *(data+1);
-      if (req->command != SOCKS_COMMAND_CONNECT &&
-          req->command != SOCKS_COMMAND_RESOLVE &&
-          req->command != SOCKS_COMMAND_RESOLVE_PTR) {
-        /* not a connect or resolve or a resolve_ptr? we don't support it. */
-        socks_request_set_socks5_error(req,SOCKS5_COMMAND_NOT_SUPPORTED);
-
-        log_warn(LD_APP,"socks5: command %d not recognized. Rejecting.",
-                 req->command);
-        return -1;
-      }
-      switch (*(data+3)) { /* address type */
-        case 1: /* IPv4 address */
-        case 4: /* IPv6 address */ {
-          const int is_v6 = *(data+3) == 4;
-          const unsigned addrlen = is_v6 ? 16 : 4;
-          log_debug(LD_APP,"socks5: ipv4 address type");
-          if (datalen < 6+addrlen) {/* ip/port there? */
-            *want_length_out = 6+addrlen;
-            return 0; /* not yet */
-          }
-
-          if (is_v6)
-            tor_addr_from_ipv6_bytes(&destaddr, data+4);
-          else
-            tor_addr_from_ipv4n(&destaddr, get_uint32(data+4));
-
-          tor_addr_to_str(tmpbuf, &destaddr, sizeof(tmpbuf), 1);
-
-          if (strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN) {
-            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
-            log_warn(LD_APP,
-                     "socks5 IP takes %d bytes, which doesn't fit in %d. "
-                     "Rejecting.",
-                     (int)strlen(tmpbuf)+1,(int)MAX_SOCKS_ADDR_LEN);
-            return -1;
-          }
-          strlcpy(req->address,tmpbuf,sizeof(req->address));
-          req->port = ntohs(get_uint16(data+4+addrlen));
-          *drain_out = 6+addrlen;
-          if (req->command != SOCKS_COMMAND_RESOLVE_PTR &&
-              !addressmap_have_mapping(req->address,0)) {
-            log_unsafe_socks_warning(5, req->address, req->port, safe_socks);
-            if (safe_socks) {
-              socks_request_set_socks5_error(req, SOCKS5_NOT_ALLOWED);
-              return -1;
-            }
-          }
-          return 1;
-        }
-        case 3: /* fqdn */
-          log_debug(LD_APP,"socks5: fqdn address type");
-          if (req->command == SOCKS_COMMAND_RESOLVE_PTR) {
-            socks_request_set_socks5_error(req,
-                                           SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED);
-            log_warn(LD_APP, "socks5 received RESOLVE_PTR command with "
-                     "hostname type. Rejecting.");
-            return -1;
-          }
-          len = (unsigned char)*(data+4);
-          if (datalen < 7+len) { /* addr/port there? */
-            *want_length_out = 7+len;
-            return 0; /* not yet */
-          }
-          if (len+1 > MAX_SOCKS_ADDR_LEN) {
-            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
-            log_warn(LD_APP,
-                     "socks5 hostname is %d bytes, which doesn't fit in "
-                     "%d. Rejecting.", len+1,MAX_SOCKS_ADDR_LEN);
-            return -1;
-          }
-          memcpy(req->address,data+5,len);
-          req->address[len] = 0;
-          req->port = ntohs(get_uint16(data+5+len));
-          *drain_out = 5+len+2;
-
-          if (!string_is_valid_hostname(req->address)) {
-            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
-
-            log_warn(LD_PROTOCOL,
-                     "Your application (using socks5 to port %d) gave Tor "
-                     "a malformed hostname: %s. Rejecting the connection.",
-                     req->port, escaped_safe_str_client(req->address));
-            return -1;
-          }
-          if (log_sockstype)
-            log_notice(LD_APP,
-                  "Your application (using socks5 to port %d) instructed "
-                  "Tor to take care of the DNS resolution itself if "
-                  "necessary. This is good.", req->port);
-          return 1;
-        default: /* unsupported */
-          socks_request_set_socks5_error(req,
-                                         SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED);
-          log_warn(LD_APP,"socks5: unsupported address type %d. Rejecting.",
-                   (int) *(data+3));
-          return -1;
-      }
-      tor_assert(0);
-      break;
-    case 4: { /* socks4 */
-      enum {socks4, socks4a} socks4_prot = socks4a;
-      const char *authstart, *authend;
-      /* http://ss5.sourceforge.net/socks4.protocol.txt */
-      /* http://ss5.sourceforge.net/socks4A.protocol.txt */
-
-      req->socks_version = 4;
-      if (datalen < SOCKS4_NETWORK_LEN) {/* basic info available? */
-        *want_length_out = SOCKS4_NETWORK_LEN;
-        return 0; /* not yet */
-      }
-      // buf_pullup(buf, 1280);
-      req->command = (unsigned char) *(data+1);
-      if (req->command != SOCKS_COMMAND_CONNECT &&
-          req->command != SOCKS_COMMAND_RESOLVE) {
-        /* not a connect or resolve? we don't support it. (No resolve_ptr with
-         * socks4.) */
-        log_warn(LD_APP,"socks4: command %d not recognized. Rejecting.",
-                 req->command);
-        return -1;
-      }
-
-      req->port = ntohs(get_uint16(data+2));
-      destip = ntohl(get_uint32(data+4));
-      if ((!req->port && req->command!=SOCKS_COMMAND_RESOLVE) || !destip) {
-        log_warn(LD_APP,"socks4: Port or DestIP is zero. Rejecting.");
-        return -1;
-      }
-      if (destip >> 8) {
-        log_debug(LD_APP,"socks4: destip not in form 0.0.0.x.");
-        in.s_addr = htonl(destip);
-        tor_inet_ntoa(&in,tmpbuf,sizeof(tmpbuf));
-        if (strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN) {
-          log_debug(LD_APP,"socks4 addr (%d bytes) too long. Rejecting.",
-                    (int)strlen(tmpbuf));
-          return -1;
-        }
-        log_debug(LD_APP,
-                  "socks4: successfully read destip (%s)",
-                  safe_str_client(tmpbuf));
-        socks4_prot = socks4;
-      }
-
-      authstart = data + SOCKS4_NETWORK_LEN;
-      next = memchr(authstart, 0,
-                    datalen-SOCKS4_NETWORK_LEN);
-      if (!next) {
-        if (datalen >= 1024) {
-          log_debug(LD_APP, "Socks4 user name too long; rejecting.");
-          return -1;
-        }
-        log_debug(LD_APP,"socks4: Username not here yet.");
-        *want_length_out = datalen+1024; /* More than we need, but safe */
-        return 0;
-      }
-      authend = next;
-      tor_assert(next < data+datalen);
-
-      startaddr = NULL;
-      if (socks4_prot != socks4a &&
-          !addressmap_have_mapping(tmpbuf,0)) {
-        log_unsafe_socks_warning(4, tmpbuf, req->port, safe_socks);
-
-        if (safe_socks)
-          return -1;
-      }
-      if (socks4_prot == socks4a) {
-        if (next+1 == data+datalen) {
-          log_debug(LD_APP,"socks4: No part of destaddr here yet.");
-          *want_length_out = datalen + 1024; /* More than we need, but safe */
-          return 0;
-        }
-        startaddr = next+1;
-        next = memchr(startaddr, 0, data + datalen - startaddr);
-        if (!next) {
-          if (datalen >= 1024) {
-            log_debug(LD_APP,"socks4: Destaddr too long.");
-            return -1;
-          }
-          log_debug(LD_APP,"socks4: Destaddr not all here yet.");
-          *want_length_out = datalen + 1024; /* More than we need, but safe */
-          return 0;
-        }
-        if (MAX_SOCKS_ADDR_LEN <= next-startaddr) {
-          log_warn(LD_APP,"socks4: Destaddr too long. Rejecting.");
-          return -1;
-        }
-        // tor_assert(next < buf->cur+buf->datalen);
-
-        if (log_sockstype)
-          log_notice(LD_APP,
-                     "Your application (using socks4a to port %d) instructed "
-                     "Tor to take care of the DNS resolution itself if "
-                     "necessary. This is good.", req->port);
-      }
-      log_debug(LD_APP,"socks4: Everything is here. Success.");
-      strlcpy(req->address, startaddr ? startaddr : tmpbuf,
-              sizeof(req->address));
-      if (!string_is_valid_hostname(req->address)) {
-        log_warn(LD_PROTOCOL,
-                 "Your application (using socks4 to port %d) gave Tor "
-                 "a malformed hostname: %s. Rejecting the connection.",
-                 req->port, escaped_safe_str_client(req->address));
-        return -1;
-      }
-      if (authend != authstart) {
-        req->got_auth = 1;
-        req->usernamelen = authend - authstart;
-        req->username = tor_memdup(authstart, authend - authstart);
-      }
-      /* next points to the final \0 on inbuf */
-      *drain_out = next - data + 1;
-      return 1;
-    }
-    case 'G': /* get */
-    case 'H': /* head */
-    case 'P': /* put/post */
-    case 'C': /* connect */
-      strlcpy((char*)req->reply, SOCKS_PROXY_IS_NOT_AN_HTTP_PROXY_MSG,
-              MAX_SOCKS_REPLY_LEN);
-      req->replylen = strlen((char*)req->reply)+1;
-      /* fall through */
-    default: /* version is not socks4 or socks5 */
-      log_warn(LD_APP,
-               "Socks version %d not recognized. (Tor is not an http proxy.)",
-               *(data));
-      {
-        /* Tell the controller the first 8 bytes. */
-        char *tmp = tor_strndup(data, datalen < 8 ? datalen : 8);
-        control_event_client_status(LOG_WARN,
-                                    "SOCKS_UNKNOWN_PROTOCOL DATA=\"%s\"",
-                                    escaped(tmp));
-        tor_free(tmp);
-      }
-      return -1;
-  }
-}
-
-/** Inspect a reply from SOCKS server stored in <b>buf</b> according
- * to <b>state</b>, removing the protocol data upon success. Return 0 on
- * incomplete response, 1 on success and -1 on error, in which case
- * <b>reason</b> is set to a descriptive message (free() when finished
- * with it).
- *
- * As a special case, 2 is returned when user/pass is required
- * during SOCKS5 handshake and user/pass is configured.
- */
-int
-fetch_from_buf_socks_client(buf_t *buf, int state, char **reason)
-{
-  ssize_t drain = 0;
-  int r;
-  if (buf->datalen < 2)
-    return 0;
-
-  buf_pullup(buf, MAX_SOCKS_MESSAGE_LEN);
-  tor_assert(buf->head && buf->head->datalen >= 2);
-
-  r = parse_socks_client((uint8_t*)buf->head->data, buf->head->datalen,
-                         state, reason, &drain);
-  if (drain > 0)
-    buf_remove_from_front(buf, drain);
-  else if (drain < 0)
-    buf_clear(buf);
-
-  return r;
-}
-
-/** Implementation logic for fetch_from_*_socks_client. */
-static int
-parse_socks_client(const uint8_t *data, size_t datalen,
-                   int state, char **reason,
-                   ssize_t *drain_out)
-{
-  unsigned int addrlen;
-  *drain_out = 0;
-  if (datalen < 2)
-    return 0;
-
-  switch (state) {
-    case PROXY_SOCKS4_WANT_CONNECT_OK:
-      /* Wait for the complete response */
-      if (datalen < 8)
-        return 0;
-
-      if (data[1] != 0x5a) {
-        *reason = tor_strdup(socks4_response_code_to_string(data[1]));
-        return -1;
-      }
-
-      /* Success */
-      *drain_out = 8;
-      return 1;
-
-    case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE:
-      /* we don't have any credentials */
-      if (data[1] != 0x00) {
-        *reason = tor_strdup("server doesn't support any of our "
-                             "available authentication methods");
-        return -1;
-      }
-
-      log_info(LD_NET, "SOCKS 5 client: continuing without authentication");
-      *drain_out = -1;
-      return 1;
-
-    case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929:
-      /* we have a username and password. return 1 if we can proceed without
-       * providing authentication, or 2 otherwise. */
-      switch (data[1]) {
-        case 0x00:
-          log_info(LD_NET, "SOCKS 5 client: we have auth details but server "
-                            "doesn't require authentication.");
-          *drain_out = -1;
-          return 1;
-        case 0x02:
-          log_info(LD_NET, "SOCKS 5 client: need authentication.");
-          *drain_out = -1;
-          return 2;
-        /* fall through */
-      }
-
-      *reason = tor_strdup("server doesn't support any of our available "
-                           "authentication methods");
-      return -1;
-
-    case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK:
-      /* handle server reply to rfc1929 authentication */
-      if (data[1] != 0x00) {
-        *reason = tor_strdup("authentication failed");
-        return -1;
-      }
-
-      log_info(LD_NET, "SOCKS 5 client: authentication successful.");
-      *drain_out = -1;
-      return 1;
-
-    case PROXY_SOCKS5_WANT_CONNECT_OK:
-      /* response is variable length. BND.ADDR, etc, isn't needed
-       * (don't bother with buf_pullup()), but make sure to eat all
-       * the data used */
-
-      /* wait for address type field to arrive */
-      if (datalen < 4)
-        return 0;
-
-      switch (data[3]) {
-        case 0x01: /* ip4 */
-          addrlen = 4;
-          break;
-        case 0x04: /* ip6 */
-          addrlen = 16;
-          break;
-        case 0x03: /* fqdn (can this happen here?) */
-          if (datalen < 5)
-            return 0;
-          addrlen = 1 + data[4];
-          break;
-        default:
-          *reason = tor_strdup("invalid response to connect request");
-          return -1;
-      }
-
-      /* wait for address and port */
-      if (datalen < 6 + addrlen)
-        return 0;
-
-      if (data[1] != 0x00) {
-        *reason = tor_strdup(socks5_response_code_to_string(data[1]));
-        return -1;
-      }
-
-      *drain_out = 6 + addrlen;
-      return 1;
-  }
-
-  /* shouldn't get here... */
-  tor_assert(0);
-
-  return -1;
-}
-
-/** Return true if <b>cmd</b> looks like a HTTP (proxy) request. */
-int
-peek_buf_has_http_command(const buf_t *buf)
-{
-  if (peek_buf_startswith(buf, "CONNECT ") ||
-      peek_buf_startswith(buf, "DELETE ") ||
-      peek_buf_startswith(buf, "GET ") ||
-      peek_buf_startswith(buf, "POST ") ||
-      peek_buf_startswith(buf, "PUT " ))
-    return 1;
-  return 0;
-}
-
-/** Return 1 iff <b>buf</b> starts with <b>cmd</b>. <b>cmd</b> must be a null
- * terminated string, of no more than PEEK_BUF_STARTSWITH_MAX bytes. */
-int
-peek_buf_startswith(const buf_t *buf, const char *cmd)
-{
-  char tmp[PEEK_BUF_STARTSWITH_MAX];
-  size_t clen = strlen(cmd);
-  if (BUG(clen > sizeof(tmp)))
-    return 0;
-  if (buf->datalen < clen)
-    return 0;
-  peek_from_buf(tmp, clen, buf);
-  return fast_memeq(tmp, cmd, clen);
-}
-
-/** Return 1 iff buf looks more like it has an (obsolete) v0 controller
- * command on it than any valid v1 controller command. */
-int
-peek_buf_has_control0_command(buf_t *buf)
-{
-  if (buf->datalen >= 4) {
-    char header[4];
-    uint16_t cmd;
-    peek_from_buf(header, sizeof(header), buf);
-    cmd = ntohs(get_uint16(header+2));
-    if (cmd <= 0x14)
-      return 1; /* This is definitely not a v1 control command. */
-  }
-  return 0;
-}
-
-/** Return the index within <b>buf</b> at which <b>ch</b> first appears,
- * or -1 if <b>ch</b> does not appear on buf. */
-static off_t
-buf_find_offset_of_char(buf_t *buf, char ch)
-{
-  chunk_t *chunk;
-  off_t offset = 0;
-  for (chunk = buf->head; chunk; chunk = chunk->next) {
-    char *cp = memchr(chunk->data, ch, chunk->datalen);
-    if (cp)
-      return offset + (cp - chunk->data);
-    else
-      offset += chunk->datalen;
-  }
-  return -1;
-}
-
-/** Try to read a single LF-terminated line from <b>buf</b>, and write it
- * (including the LF), NUL-terminated, into the *<b>data_len</b> byte buffer
- * at <b>data_out</b>.  Set *<b>data_len</b> to the number of bytes in the
- * line, not counting the terminating NUL.  Return 1 if we read a whole line,
- * return 0 if we don't have a whole line yet, and return -1 if the line
- * length exceeds *<b>data_len</b>.
- */
-int
-fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len)
-{
-  size_t sz;
-  off_t offset;
-
-  if (!buf->head)
-    return 0;
-
-  offset = buf_find_offset_of_char(buf, '\n');
-  if (offset < 0)
-    return 0;
-  sz = (size_t) offset;
-  if (sz+2 > *data_len) {
-    *data_len = sz + 2;
-    return -1;
-  }
-  fetch_from_buf(data_out, sz+1, buf);
-  data_out[sz+1] = '\0';
-  *data_len = sz+1;
-  return 1;
-}
-
-/** Compress on uncompress the <b>data_len</b> bytes in <b>data</b> using the
- * compression state <b>state</b>, appending the result to <b>buf</b>.  If
- * <b>done</b> is true, flush the data in the state and finish the
- * compression/uncompression.  Return -1 on failure, 0 on success. */
-int
-write_to_buf_compress(buf_t *buf, tor_compress_state_t *state,
-                      const char *data, size_t data_len,
-                      const int done)
-{
-  char *next;
-  size_t old_avail, avail;
-  int over = 0;
-
-  do {
-    int need_new_chunk = 0;
-    if (!buf->tail || ! CHUNK_REMAINING_CAPACITY(buf->tail)) {
-      size_t cap = data_len / 4;
-      buf_add_chunk_with_capacity(buf, cap, 1);
-    }
-    next = CHUNK_WRITE_PTR(buf->tail);
-    avail = old_avail = CHUNK_REMAINING_CAPACITY(buf->tail);
-    switch (tor_compress_process(state, &next, &avail,
-                                 &data, &data_len, done)) {
-      case TOR_COMPRESS_DONE:
-        over = 1;
-        break;
-      case TOR_COMPRESS_ERROR:
-        return -1;
-      case TOR_COMPRESS_OK:
-        if (data_len == 0) {
-          tor_assert_nonfatal(!done);
-          over = 1;
-        }
-        break;
-      case TOR_COMPRESS_BUFFER_FULL:
-        if (avail) {
-          /* The compression module says we need more room
-           * (TOR_COMPRESS_BUFFER_FULL).  Start a new chunk automatically,
-           * whether were going to or not. */
-          need_new_chunk = 1;
-        }
-        if (data_len == 0 && !done) {
-          /* We've consumed all the input data, though, so there's no
-           * point in forging ahead right now. */
-          over = 1;
-        }
-        break;
-    }
-    buf->datalen += old_avail - avail;
-    buf->tail->datalen += old_avail - avail;
-    if (need_new_chunk) {
-      buf_add_chunk_with_capacity(buf, data_len/4, 1);
-    }
-
-  } while (!over);
-  check();
-  return 0;
-}
-
-/** Set *<b>output</b> to contain a copy of the data in *<b>input</b> */
-int
-buf_set_to_copy(buf_t **output,
-                const buf_t *input)
-{
-  if (*output)
-    buf_free(*output);
-  *output = buf_copy(input);
-  return 0;
-}
-
-/** Log an error and exit if <b>buf</b> is corrupted.
- */
-void
-assert_buf_ok(buf_t *buf)
-{
-  tor_assert(buf);
-  tor_assert(buf->magic == BUFFER_MAGIC);
-
-  if (! buf->head) {
-    tor_assert(!buf->tail);
-    tor_assert(buf->datalen == 0);
-  } else {
-    chunk_t *ch;
-    size_t total = 0;
-    tor_assert(buf->tail);
-    for (ch = buf->head; ch; ch = ch->next) {
-      total += ch->datalen;
-      tor_assert(ch->datalen <= ch->memlen);
-      tor_assert(ch->data >= &ch->mem[0]);
-      tor_assert(ch->data <= &ch->mem[0]+ch->memlen);
-      if (ch->data == &ch->mem[0]+ch->memlen) {
-        static int warned = 0;
-        if (! warned) {
-          log_warn(LD_BUG, "Invariant violation in buf.c related to #15083");
-          warned = 1;
-        }
-      }
-      tor_assert(ch->data+ch->datalen <= &ch->mem[0] + ch->memlen);
-      if (!ch->next)
-        tor_assert(ch == buf->tail);
-    }
-    tor_assert(buf->datalen == total);
-  }
-}
-

src/or/channeltls.c

@@ -847,7 +847,7 @@ channel_tls_write_packed_cell_method(channel_t *chan,
   tor_assert(packed_cell);
 
   if (tlschan->conn) {
-    connection_write_to_buf(packed_cell->body, cell_network_size,
+    connection_buf_add(packed_cell->body, cell_network_size,
                             TO_CONN(tlschan->conn));
 
     /* This is where the cell is finished; used to be done from relay.c */

src/or/connection.c

@@ -37,7 +37,7 @@
  * they call connection_stop_reading() or connection_stop_writing().
  *
  * To queue data to be written on a connection, call
- * connection_write_to_buf().  When data arrives, the
+ * connection_buf_add().  When data arrives, the
  * connection_process_inbuf() callback is invoked, which dispatches to a
  * type-specific function (such as connection_edge_process_inbuf() for
  * example). Connection types that need notice of when data has been written
@@ -58,6 +58,7 @@
 #include "or.h"
 #include "bridges.h"
 #include "buffers.h"
+#include "buffers_tls.h"
 /*
  * Define this so we get channel internal functions, since we're implementing
  * part of a subclass (channel_tls_t).
@@ -86,6 +87,8 @@
 #include "hs_common.h"
 #include "hs_ident.h"
 #include "nodelist.h"
+#include "proto_http.h"
+#include "proto_socks.h"
 #include "policies.h"
 #include "reasons.h"
 #include "relay.h"
@@ -124,8 +127,9 @@ static int connection_finished_flushing(connection_t *conn);
 static int connection_flushed_some(connection_t *conn);
 static int connection_finished_connecting(connection_t *conn);
 static int connection_reached_eof(connection_t *conn);
-static int connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
-                                  int *socket_error);
+static int connection_buf_read_from_socket(connection_t *conn,
+                                           ssize_t *max_to_read,
+                                           int *socket_error);
 static int connection_process_inbuf(connection_t *conn, int package_partial);
 static void client_check_address_changed(tor_socket_t sock);
 static void set_constrained_socket_buffers(tor_socket_t sock, int size);
@@ -2140,7 +2144,7 @@ connection_proxy_connect(connection_t *conn, int type)
                      fmt_addrport(&conn->addr, conn->port));
       }
 
-      connection_write_to_buf(buf, strlen(buf), conn);
+      connection_buf_add(buf, strlen(buf), conn);
       conn->proxy_state = PROXY_HTTPS_WANT_CONNECT_OK;
       break;
     }
@@ -2206,7 +2210,7 @@ connection_proxy_connect(connection_t *conn, int type)
         buf[8] = 0; /* no userid */
       }
 
-      connection_write_to_buf((char *)buf, buf_size, conn);
+      connection_buf_add((char *)buf, buf_size, conn);
       tor_free(buf);
 
       conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK;
@@ -2237,7 +2241,7 @@ connection_proxy_connect(connection_t *conn, int type)
         conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_NONE;
       }
 
-      connection_write_to_buf((char *)buf, 2 + buf[1], conn);
+      connection_buf_add((char *)buf, 2 + buf[1], conn);
       break;
     }
 
@@ -2343,7 +2347,7 @@ connection_send_socks5_connect(connection_t *conn)
     memcpy(buf + 20, &port, 2);
   }
 
-  connection_write_to_buf((char *)buf, reqsize, conn);
+  connection_buf_add((char *)buf, reqsize, conn);
 
   conn->proxy_state = PROXY_SOCKS5_WANT_CONNECT_OK;
 }
@@ -2469,7 +2473,7 @@ connection_read_proxy_handshake(connection_t *conn)
         if (socks_args_string)
           tor_free(socks_args_string);
 
-        connection_write_to_buf((char *)buf, reqsize, conn);
+        connection_buf_add((char *)buf, reqsize, conn);
 
         conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_RFC1929_OK;
         ret = 0;
@@ -3368,7 +3372,7 @@ connection_bucket_should_increase(int bucket, or_connection_t *conn)
 
 /** Read bytes from conn-\>s and process them.
  *
- * It calls connection_read_to_buf() to bring in any new bytes,
+ * It calls connection_buf_read_from_socket() to bring in any new bytes,
  * and then calls connection_process_inbuf() to process them.
  *
  * Mark the connection and return -1 if you want to close it, else
@@ -3410,7 +3414,7 @@ connection_handle_read_impl(connection_t *conn)
   tor_assert(!conn->marked_for_close);
 
   before = buf_datalen(conn->inbuf);
-  if (connection_read_to_buf(conn, &max_to_read, &socket_error) < 0) {
+  if (connection_buf_read_from_socket(conn, &max_to_read, &socket_error) < 0) {
     /* There's a read error; kill the connection.*/
     if (conn->type == CONN_TYPE_OR) {
       connection_or_notify_error(TO_OR_CONN(conn),
@@ -3507,7 +3511,7 @@ connection_handle_read(connection_t *conn)
  * Return -1 if we want to break conn, else return 0.
  */
 static int
-connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
+connection_buf_read_from_socket(connection_t *conn, ssize_t *max_to_read,
                        int *socket_error)
 {
   int result;
@@ -3548,7 +3552,7 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
 
     initial_size = buf_datalen(conn->inbuf);
     /* else open, or closing */
-    result = read_to_buf_tls(or_conn->tls, at_most, conn->inbuf);
+    result = buf_read_from_tls(conn->inbuf, or_conn->tls, at_most);
     if (TOR_TLS_IS_ERROR(result) || result == TOR_TLS_CLOSE)
       or_conn->tls_error = result;
     else
@@ -3597,7 +3601,7 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
       /* If we have any pending bytes, we read them now.  This *can*
        * take us over our read allotment, but really we shouldn't be
        * believing that SSL bytes are the same as TCP bytes anyway. */
-      int r2 = read_to_buf_tls(or_conn->tls, pending, conn->inbuf);
+      int r2 = buf_read_from_tls(conn->inbuf, or_conn->tls, pending);
       if (BUG(r2<0)) {
         log_warn(LD_BUG, "apparently, reading pending bytes can fail.");
         return -1;
@@ -3609,7 +3613,7 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
               result, (long)n_read, (long)n_written);
   } else if (conn->linked) {
     if (conn->linked_conn) {
-      result = move_buf_to_buf(conn->inbuf, conn->linked_conn->outbuf,
+      result = buf_move_to_buf(conn->inbuf, conn->linked_conn->outbuf,
                                &conn->linked_conn->outbuf_flushlen);
     } else {
       result = 0;
@@ -3627,8 +3631,10 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
     /* !connection_speaks_cells, !conn->linked_conn. */
     int reached_eof = 0;
     CONN_LOG_PROTECT(conn,
-        result = read_to_buf(conn->s, at_most, conn->inbuf, &reached_eof,
-                             socket_error));
+                     result = buf_read_from_socket(conn->inbuf, conn->s,
+                                                   at_most,
+                                                   &reached_eof,
+                                                   socket_error));
     if (reached_eof)
       conn->inbuf_reached_eof = 1;
 
@@ -3697,17 +3703,17 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
 
 /** A pass-through to fetch_from_buf. */
 int
-connection_fetch_from_buf(char *string, size_t len, connection_t *conn)
+connection_buf_get_bytes(char *string, size_t len, connection_t *conn)
 {
-  return fetch_from_buf(string, len, conn->inbuf);
+  return buf_get_bytes(conn->inbuf, string, len);
 }
 
-/** As fetch_from_buf_line(), but read from a connection's input buffer. */
+/** As buf_get_line(), but read from a connection's input buffer. */
 int
-connection_fetch_from_buf_line(connection_t *conn, char *data,
+connection_buf_get_line(connection_t *conn, char *data,
                                size_t *data_len)
 {
-  return fetch_from_buf_line(conn->inbuf, data, data_len);
+  return buf_get_line(conn->inbuf, data, data_len);
 }
 
 /** As fetch_from_buf_http, but fetches from a connection's input buffer_t as
@@ -3744,7 +3750,7 @@ connection_outbuf_too_full(connection_t *conn)
  *
  * This function gets called either from conn_write_callback() in main.c
  * when libevent tells us that conn wants to write, or below
- * from connection_write_to_buf() when an entire TLS record is ready.
+ * from connection_buf_add() when an entire TLS record is ready.
  *
  * Update <b>conn</b>-\>timestamp_lastwritten to now, and call flush_buf
  * or flush_buf_tls appropriately. If it succeeds and there are no more
@@ -3855,7 +3861,7 @@ connection_handle_write_impl(connection_t *conn, int force)
 
     /* else open, or closing */
     initial_size = buf_datalen(conn->outbuf);
-    result = flush_buf_tls(or_conn->tls, conn->outbuf,
+    result = buf_flush_to_tls(conn->outbuf, or_conn->tls,
                            max_to_write, &conn->outbuf_flushlen);
 
     /* If we just flushed the last bytes, tell the channel on the
@@ -3918,8 +3924,8 @@ connection_handle_write_impl(connection_t *conn, int force)
     result = (int)(initial_size-buf_datalen(conn->outbuf));
   } else {
     CONN_LOG_PROTECT(conn,
-             result = flush_buf(conn->s, conn->outbuf,
-                                max_to_write, &conn->outbuf_flushlen));
+                     result = buf_flush_to_socket(conn->outbuf, conn->s,
+                                        max_to_write, &conn->outbuf_flushlen));
     if (result < 0) {
       if (CONN_IS_EDGE(conn))
         connection_edge_end_errno(TO_EDGE_CONN(conn));
@@ -4059,11 +4065,11 @@ connection_write_to_buf_impl_,(const char *string, size_t len,
   if (zlib) {
     dir_connection_t *dir_conn = TO_DIR_CONN(conn);
     int done = zlib < 0;
-    CONN_LOG_PROTECT(conn, r = write_to_buf_compress(conn->outbuf,
+    CONN_LOG_PROTECT(conn, r = buf_add_compress(conn->outbuf,
                                                      dir_conn->compress_state,
                                                      string, len, done));
   } else {
-    CONN_LOG_PROTECT(conn, r = write_to_buf(string, len, conn->outbuf));
+    CONN_LOG_PROTECT(conn, r = buf_add(conn->outbuf, string, len));
   }
   if (r < 0) {
     if (CONN_IS_EDGE(conn)) {
@@ -4974,9 +4980,9 @@ assert_connection_ok(connection_t *conn, time_t now)
 
   /* buffers */
   if (conn->inbuf)
-    assert_buf_ok(conn->inbuf);
+    buf_assert_ok(conn->inbuf);
   if (conn->outbuf)
-    assert_buf_ok(conn->outbuf);
+    buf_assert_ok(conn->outbuf);
 
   if (conn->type == CONN_TYPE_OR) {
     or_connection_t *or_conn = TO_OR_CONN(conn);

src/or/connection.h

@@ -123,8 +123,8 @@ void connection_bucket_refill(int seconds_elapsed, time_t now);
 
 int connection_handle_read(connection_t *conn);
 
-int connection_fetch_from_buf(char *string, size_t len, connection_t *conn);
-int connection_fetch_from_buf_line(connection_t *conn, char *data,
+int connection_buf_get_bytes(char *string, size_t len, connection_t *conn);
+int connection_buf_get_line(connection_t *conn, char *data,
                                    size_t *data_len);
 int connection_fetch_from_buf_http(connection_t *conn,
                                char **headers_out, size_t max_headerlen,
@@ -139,18 +139,18 @@ int connection_flush(connection_t *conn);
 MOCK_DECL(void, connection_write_to_buf_impl_,
           (const char *string, size_t len, connection_t *conn, int zlib));
 /* DOCDOC connection_write_to_buf */
-static void connection_write_to_buf(const char *string, size_t len,
+static void connection_buf_add(const char *string, size_t len,
                                     connection_t *conn);
 /* DOCDOC connection_write_to_buf_compress */
-static void connection_write_to_buf_compress(const char *string, size_t len,
+static void connection_buf_add_compress(const char *string, size_t len,
                                              dir_connection_t *conn, int done);
 static inline void
-connection_write_to_buf(const char *string, size_t len, connection_t *conn)
+connection_buf_add(const char *string, size_t len, connection_t *conn)
 {
   connection_write_to_buf_impl_(string, len, conn, 0);
 }
 static inline void
-connection_write_to_buf_compress(const char *string, size_t len,
+connection_buf_add_compress(const char *string, size_t len,
                                  dir_connection_t *conn, int done)
 {
   connection_write_to_buf_impl_(string, len, TO_CONN(conn), done ? -1 : 1);

src/or/connection_edge.c

@@ -82,6 +82,7 @@
 #include "main.h"
 #include "nodelist.h"
 #include "policies.h"
+#include "proto_socks.h"
 #include "reasons.h"
 #include "relay.h"
 #include "rendclient.h"
@@ -2275,7 +2276,7 @@ connection_ap_handshake_process_socks(entry_connection_t *conn)
 
   if (socks->replylen) {
     had_reply = 1;
-    connection_write_to_buf((const char*)socks->reply, socks->replylen,
+    connection_buf_add((const char*)socks->reply, socks->replylen,
                             base_conn);
     socks->replylen = 0;
     if (sockshere == -1) {
@@ -2372,7 +2373,7 @@ connection_ap_process_natd(entry_connection_t *conn)
 
   /* look for LF-terminated "[DEST ip_addr port]"
    * where ip_addr is a dotted-quad and port is in string form */
-  err = connection_fetch_from_buf_line(ENTRY_TO_CONN(conn), tmp_buf, &tlen);
+  err = connection_buf_get_line(ENTRY_TO_CONN(conn), tmp_buf, &tlen);
   if (err == 0)
     return 0;
   if (err < 0) {
@@ -3040,7 +3041,7 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply,
     return;
   }
   if (replylen) { /* we already have a reply in mind */
-    connection_write_to_buf(reply, replylen, ENTRY_TO_CONN(conn));
+    connection_buf_add(reply, replylen, ENTRY_TO_CONN(conn));
     conn->socks_request->has_finished = 1;
     return;
   }
@@ -3048,7 +3049,7 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply,
     memset(buf,0,SOCKS4_NETWORK_LEN);
     buf[1] = (status==SOCKS5_SUCCEEDED ? SOCKS4_GRANTED : SOCKS4_REJECT);
     /* leave version, destport, destip zero */
-    connection_write_to_buf(buf, SOCKS4_NETWORK_LEN, ENTRY_TO_CONN(conn));
+    connection_buf_add(buf, SOCKS4_NETWORK_LEN, ENTRY_TO_CONN(conn));
   } else if (conn->socks_request->socks_version == 5) {
     size_t buf_len;
     memset(buf,0,sizeof(buf));
@@ -3067,7 +3068,7 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply,
       /* 4 bytes for the header, 2 bytes for the port, 16 for the address. */
       buf_len = 22;
     }
-    connection_write_to_buf(buf,buf_len,ENTRY_TO_CONN(conn));
+    connection_buf_add(buf,buf_len,ENTRY_TO_CONN(conn));
   }
   /* If socks_version isn't 4 or 5, don't send anything.
    * This can happen in the case of AP bridges. */

src/or/connection_or.c

@@ -46,6 +46,7 @@
 #include "microdesc.h"
 #include "networkstatus.h"
 #include "nodelist.h"
+#include "proto_cell.h"
 #include "reasons.h"
 #include "relay.h"
 #include "rephist.h"
@@ -1978,7 +1979,7 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn)
   if (cell->command == CELL_PADDING)
     rep_hist_padding_count_write(PADDING_TYPE_CELL);
 
-  connection_write_to_buf(networkcell.body, cell_network_size, TO_CONN(conn));
+  connection_buf_add(networkcell.body, cell_network_size, TO_CONN(conn));
 
   /* Touch the channel's active timestamp if there is one */
   if (conn->chan) {
@@ -2008,8 +2009,8 @@ connection_or_write_var_cell_to_buf,(const var_cell_t *cell,
   tor_assert(cell);
   tor_assert(conn);
   n = var_cell_pack_header(cell, hdr, conn->wide_circ_ids);
-  connection_write_to_buf(hdr, n, TO_CONN(conn));
-  connection_write_to_buf((char*)cell->payload,
+  connection_buf_add(hdr, n, TO_CONN(conn));
+  connection_buf_add((char*)cell->payload,
                           cell->payload_len, TO_CONN(conn));
   if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3)
     or_handshake_state_record_var_cell(conn, conn->handshake_state, cell, 0);
@@ -2084,7 +2085,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn)
         channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
 
       circuit_build_times_network_is_live(get_circuit_build_times_mutable());
-      connection_fetch_from_buf(buf, cell_network_size, TO_CONN(conn));
+      connection_buf_get_bytes(buf, cell_network_size, TO_CONN(conn));
 
       /* retrieve cell info from buf (create the host-order struct from the
        * network-order string) */

src/or/control.c

@@ -64,6 +64,8 @@
 #include "networkstatus.h"
 #include "nodelist.h"
 #include "policies.h"
+#include "proto_control0.h"
+#include "proto_http.h"
 #include "reasons.h"
 #include "rendclient.h"
 #include "rendcommon.h"
@@ -355,7 +357,7 @@ static inline void
 connection_write_str_to_buf(const char *s, control_connection_t *conn)
 {
   size_t len = strlen(s);
-  connection_write_to_buf(s, len, TO_CONN(conn));
+  connection_buf_add(s, len, TO_CONN(conn));
 }
 
 /** Given a <b>len</b>-character string in <b>data</b>, made of lines
@@ -565,7 +567,7 @@ connection_printf_to_buf(control_connection_t *conn, const char *format, ...)
     tor_assert(0);
   }
 
-  connection_write_to_buf(buf, (size_t)len, TO_CONN(conn));
+  connection_buf_add(buf, (size_t)len, TO_CONN(conn));
 
   tor_free(buf);
 }
@@ -790,7 +792,7 @@ queued_events_flush_all(int force)
     SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
                             control_conn) {
       if (control_conn->event_mask & bit) {
-        connection_write_to_buf(ev->msg, msg_len, TO_CONN(control_conn));
+        connection_buf_add(ev->msg, msg_len, TO_CONN(control_conn));
       }
     } SMARTLIST_FOREACH_END(control_conn);
 
@@ -1072,7 +1074,7 @@ handle_control_getconf(control_connection_t *conn, uint32_t body_len,
     tor_assert(strlen(tmp)>4);
     tmp[3] = ' ';
     msg = smartlist_join_strings(answers, "", 0, &msg_len);
-    connection_write_to_buf(msg, msg_len, TO_CONN(conn));
+    connection_buf_add(msg, msg_len, TO_CONN(conn));
   } else {
     connection_write_str_to_buf("250 OK\r\n", conn);
   }
@@ -1654,12 +1656,12 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len,
   if (smartlist_len(reply)) {
     ((char*)smartlist_get(reply,smartlist_len(reply)-1))[3] = ' ';
     r = smartlist_join_strings(reply, "\r\n", 1, &sz);
-    connection_write_to_buf(r, sz, TO_CONN(conn));
+    connection_buf_add(r, sz, TO_CONN(conn));
     tor_free(r);
   } else {
     const char *response =
       "512 syntax error: not enough arguments to mapaddress.\r\n";
-    connection_write_to_buf(response, strlen(response), TO_CONN(conn));
+    connection_buf_add(response, strlen(response), TO_CONN(conn));
   }
 
   SMARTLIST_FOREACH(reply, char *, cp, tor_free(cp));
@@ -3244,7 +3246,7 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len,
       size_t esc_len;
       esc_len = write_escaped_data(v, strlen(v), &esc);
       connection_printf_to_buf(conn, "250+%s=\r\n", k);
-      connection_write_to_buf(esc, esc_len, TO_CONN(conn));
+      connection_buf_add(esc, esc_len, TO_CONN(conn));
       tor_free(esc);
     }
   }
@@ -4985,7 +4987,7 @@ connection_control_process_inbuf(control_connection_t *conn)
             sizeof(buf)-6);
     body_len = 2+strlen(buf+6)+2; /* code, msg, nul. */
     set_uint16(buf+0, htons(body_len));
-    connection_write_to_buf(buf, 4+body_len, TO_CONN(conn));
+    connection_buf_add(buf, 4+body_len, TO_CONN(conn));
 
     connection_mark_and_flush(TO_CONN(conn));
     return 0;
@@ -5007,7 +5009,7 @@ connection_control_process_inbuf(control_connection_t *conn)
     /* First, fetch a line. */
     do {
       data_len = conn->incoming_cmd_len - conn->incoming_cmd_cur_len;
-      r = connection_fetch_from_buf_line(TO_CONN(conn),
+      r = connection_buf_get_line(TO_CONN(conn),
                               conn->incoming_cmd+conn->incoming_cmd_cur_len,
                               &data_len);
       if (r == 0)

src/or/directory.c

@@ -1912,11 +1912,11 @@ directory_send_command(dir_connection_t *conn,
 
   request_len = strlen(request);
   total_request_len += request_len;
-  connection_write_to_buf(request, request_len, TO_CONN(conn));
+  connection_buf_add(request, request_len, TO_CONN(conn));
 
   url_len = strlen(url);
   total_request_len += url_len;
-  connection_write_to_buf(url, url_len, TO_CONN(conn));
+  connection_buf_add(url, url_len, TO_CONN(conn));
   tor_free(url);
 
   if (!strcmp(httpcommand, "POST") || payload) {
@@ -1933,11 +1933,11 @@ directory_send_command(dir_connection_t *conn,
 
   request_len = strlen(request);
   total_request_len += request_len;
-  connection_write_to_buf(request, request_len, TO_CONN(conn));
+  connection_buf_add(request, request_len, TO_CONN(conn));
 
   if (payload) {
     /* then send the payload afterwards too */
-    connection_write_to_buf(payload, payload_len, TO_CONN(conn));
+    connection_buf_add(payload, payload_len, TO_CONN(conn));
     total_request_len += payload_len;
   }
 
@@ -3466,7 +3466,7 @@ write_http_status_line(dir_connection_t *conn, int status,
     return;
   }
   log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase);
-  connection_write_to_buf(buf, strlen(buf), TO_CONN(conn));
+  connection_buf_add(buf, strlen(buf), TO_CONN(conn));
 }
 
 /** Write the header for an HTTP/1.0 response onto <b>conn</b>-\>outbuf,
@@ -3539,7 +3539,7 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
     memcpy(cp, "\r\n", 3);
   else
     tor_assert(0);
-  connection_write_to_buf(tmp, strlen(tmp), TO_CONN(conn));
+  connection_buf_add(tmp, strlen(tmp), TO_CONN(conn));
 }
 
 /** As write_http_response_header_impl, but sets encoding and content-typed
@@ -3902,7 +3902,7 @@ handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
      *  this page no matter what.] */
     write_http_response_header_impl(conn, dlen, "text/html", "identity",
                                     NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME);
-    connection_write_to_buf(frontpage, dlen, TO_CONN(conn));
+    connection_buf_add(frontpage, dlen, TO_CONN(conn));
   } else {
     write_http_status_line(conn, 404, "Not found");
   }
@@ -4539,15 +4539,15 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
         conn->compress_state = tor_compress_new(1, compress_method,
                            choose_compression_level(estimated_len));
         SMARTLIST_FOREACH(items, const char *, c,
-                 connection_write_to_buf_compress(c, strlen(c), conn, 0));
-        connection_write_to_buf_compress("", 0, conn, 1);
+                 connection_buf_add_compress(c, strlen(c), conn, 0));
+        connection_buf_add_compress("", 0, conn, 1);
       } else {
         SMARTLIST_FOREACH(items, const char *, c,
-                         connection_write_to_buf(c, strlen(c), TO_CONN(conn)));
+                         connection_buf_add(c, strlen(c), TO_CONN(conn)));
       }
     } else {
       SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
-          connection_write_to_buf(compress_method != NO_METHOD ?
+          connection_buf_add(compress_method != NO_METHOD ?
                                     d->dir_compressed : d->dir,
                                   compress_method != NO_METHOD ?
                                     d->dir_compressed_len : d->dir_len,
@@ -4795,14 +4795,14 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
       conn->compress_state = tor_compress_new(1, compress_method,
                                               choose_compression_level(len));
       SMARTLIST_FOREACH(certs, authority_cert_t *, c,
-            connection_write_to_buf_compress(
+            connection_buf_add_compress(
                 c->cache_info.signed_descriptor_body,
                 c->cache_info.signed_descriptor_len,
                 conn, 0));
-      connection_write_to_buf_compress("", 0, conn, 1);
+      connection_buf_add_compress("", 0, conn, 1);
     } else {
       SMARTLIST_FOREACH(certs, authority_cert_t *, c,
-            connection_write_to_buf(c->cache_info.signed_descriptor_body,
+            connection_buf_add(c->cache_info.signed_descriptor_body,
                                     c->cache_info.signed_descriptor_len,
                                     TO_CONN(conn)));
     }
@@ -4831,7 +4831,7 @@ handle_get_hs_descriptor_v2(dir_connection_t *conn,
       switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) {
         case 1: /* valid */
           write_http_response_header(conn, strlen(descp), NO_METHOD, 0);
-          connection_write_to_buf(descp, strlen(descp), TO_CONN(conn));
+          connection_buf_add(descp, strlen(descp), TO_CONN(conn));
           break;
         case 0: /* well-formed but not present */
           write_http_status_line(conn, 404, "Not found");
@@ -4883,7 +4883,7 @@ handle_get_hs_descriptor_v3(dir_connection_t *conn,
 
   /* Found requested descriptor! Pass it to this nice client. */
   write_http_response_header(conn, strlen(desc_str), NO_METHOD, 0);
-  connection_write_to_buf(desc_str, strlen(desc_str), TO_CONN(conn));
+  connection_buf_add(desc_str, strlen(desc_str), TO_CONN(conn));
 
  done:
   return 0;
@@ -4922,7 +4922,7 @@ handle_get_networkstatus_bridges(dir_connection_t *conn,
     status = networkstatus_getinfo_by_purpose("bridge", time(NULL));
     size_t dlen = strlen(status);
     write_http_response_header(conn, dlen, NO_METHOD, 0);
-    connection_write_to_buf(status, dlen, TO_CONN(conn));
+    connection_buf_add(status, dlen, TO_CONN(conn));
     tor_free(status);
     goto done;
   }
@@ -4939,7 +4939,7 @@ handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args)
     const char robots[] = "User-agent: *\r\nDisallow: /\r\n";
     size_t len = strlen(robots);
     write_http_response_header(conn, len, NO_METHOD, ROBOTS_CACHE_LIFETIME);
-    connection_write_to_buf(robots, len, TO_CONN(conn));
+    connection_buf_add(robots, len, TO_CONN(conn));
   }
   return 0;
 }

src/or/dirserv.c

@@ -3617,9 +3617,9 @@ spooled_resource_flush_some(spooled_resource_t *spooled,
       return SRFS_DONE;
     }
     if (conn->compress_state) {
-      connection_write_to_buf_compress((const char*)body, bodylen, conn, 0);
+      connection_buf_add_compress((const char*)body, bodylen, conn, 0);
     } else {
-      connection_write_to_buf((const char*)body, bodylen, TO_CONN(conn));
+      connection_buf_add((const char*)body, bodylen, TO_CONN(conn));
     }
     return SRFS_DONE;
   } else {
@@ -3656,11 +3656,11 @@ spooled_resource_flush_some(spooled_resource_t *spooled,
       return SRFS_ERR;
     ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining);
     if (conn->compress_state) {
-      connection_write_to_buf_compress(
+      connection_buf_add_compress(
               ptr + spooled->cached_dir_offset,
               bytes, conn, 0);
     } else {
-      connection_write_to_buf(ptr + spooled->cached_dir_offset,
+      connection_buf_add(ptr + spooled->cached_dir_offset,
                               bytes, TO_CONN(conn));
     }
     spooled->cached_dir_offset += bytes;
@@ -3925,7 +3925,7 @@ connection_dirserv_flushed_some(dir_connection_t *conn)
   if (conn->compress_state) {
     /* Flush the compression state: there could be more bytes pending in there,
      * and we don't want to omit bytes. */
-    connection_write_to_buf_compress("", 0, conn, 1);
+    connection_buf_add_compress("", 0, conn, 1);
     tor_compress_free(conn->compress_state);
     conn->compress_state = NULL;
   }

src/or/ext_orport.c

@@ -23,8 +23,9 @@
 #include "ext_orport.h"
 #include "control.h"
 #include "config.h"
-#include "util.h"
 #include "main.h"
+#include "proto_ext_or.h"
+#include "util.h"
 
 /** Allocate and return a structure capable of holding an Extended
  *  ORPort message of body length <b>len</b>. */
@@ -69,10 +70,10 @@ connection_write_ext_or_command(connection_t *conn,
     return -1;
   set_uint16(header, htons(command));
   set_uint16(header+2, htons(bodylen));
-  connection_write_to_buf(header, 4, conn);
+  connection_buf_add(header, 4, conn);
   if (bodylen) {
     tor_assert(body);
-    connection_write_to_buf(body, bodylen, conn);
+    connection_buf_add(body, bodylen, conn);
   }
   return 0;
 }
@@ -170,7 +171,7 @@ connection_ext_or_auth_neg_auth_type(connection_t *conn)
   if (connection_get_inbuf_len(conn) < 1)
     return 0;
 
-  if (connection_fetch_from_buf(authtype, 1, conn) < 0)
+  if (connection_buf_get_bytes(authtype, 1, conn) < 0)
     return -1;
 
   log_debug(LD_GENERAL, "Client wants us to use %d auth type", authtype[0]);
@@ -310,7 +311,7 @@ connection_ext_or_auth_handle_client_nonce(connection_t *conn)
   if (connection_get_inbuf_len(conn) < EXT_OR_PORT_AUTH_NONCE_LEN)
     return 0;
 
-  if (connection_fetch_from_buf(client_nonce,
+  if (connection_buf_get_bytes(client_nonce,
                                 EXT_OR_PORT_AUTH_NONCE_LEN, conn) < 0)
     return -1;
 
@@ -325,7 +326,7 @@ connection_ext_or_auth_handle_client_nonce(connection_t *conn)
                             &reply, &reply_len) < 0)
     return -1;
 
-  connection_write_to_buf(reply, reply_len, conn);
+  connection_buf_add(reply, reply_len, conn);
 
   memwipe(reply, 0, reply_len);
   tor_free(reply);
@@ -347,9 +348,9 @@ static void
 connection_ext_or_auth_send_result(connection_t *conn, int success)
 {
   if (success)
-    connection_write_to_buf("\x01", 1, conn);
+    connection_buf_add("\x01", 1, conn);
   else
-    connection_write_to_buf("\x00", 1, conn);
+    connection_buf_add("\x00", 1, conn);
 }
 
 /** Receive the client's hash from <b>conn</b>, validate that it's
@@ -367,7 +368,7 @@ connection_ext_or_auth_handle_client_hash(connection_t *conn)
   if (connection_get_inbuf_len(conn) < EXT_OR_PORT_AUTH_HASH_LEN)
     return 0;
 
-  if (connection_fetch_from_buf(provided_client_hash,
+  if (connection_buf_get_bytes(provided_client_hash,
                                 EXT_OR_PORT_AUTH_HASH_LEN, conn) < 0)
     return -1;
 
@@ -637,7 +638,7 @@ connection_ext_or_start_auth(or_connection_t *or_conn)
   log_debug(LD_GENERAL,
            "ExtORPort authentication: Sending supported authentication types");
 
-  connection_write_to_buf((const char *)authtypes, sizeof(authtypes), conn);
+  connection_buf_add((const char *)authtypes, sizeof(authtypes), conn);
   conn->state = EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE;
 
   return 0;

src/or/include.am

@@ -20,7 +20,6 @@ EXTRA_DIST+= src/or/ntmain.c src/or/Makefile.nmake
 LIBTOR_A_SOURCES = \
 	src/or/addressmap.c				\
 	src/or/bridges.c				\
-	src/or/buffers.c				\
 	src/or/channel.c				\
 	src/or/channelpadding.c				\
 	src/or/channeltls.c				\
@@ -79,6 +78,11 @@ LIBTOR_A_SOURCES = \
 	src/or/parsecommon.c			\
 	src/or/periodic.c				\
 	src/or/protover.c				\
+	src/or/proto_cell.c				\
+	src/or/proto_control0.c				\
+	src/or/proto_ext_or.c				\
+	src/or/proto_http.c				\
+	src/or/proto_socks.c				\
 	src/or/policies.c				\
 	src/or/reasons.c				\
 	src/or/relay.c					\
@@ -150,7 +154,6 @@ endif
 ORHEADERS = \
 	src/or/addressmap.h				\
 	src/or/bridges.h				\
-	src/or/buffers.h				\
 	src/or/channel.h				\
 	src/or/channelpadding.h				\
 	src/or/channeltls.h				\
@@ -215,6 +218,11 @@ ORHEADERS = \
 	src/or/periodic.h				\
 	src/or/policies.h				\
 	src/or/protover.h				\
+	src/or/proto_cell.h				\
+	src/or/proto_control0.h				\
+	src/or/proto_ext_or.h				\
+	src/or/proto_http.h				\
+	src/or/proto_socks.h				\
 	src/or/reasons.h				\
 	src/or/relay.h					\
 	src/or/rendcache.h				\

src/or/main.c

@@ -52,6 +52,7 @@
 #include "backtrace.h"
 #include "bridges.h"
 #include "buffers.h"
+#include "buffers_tls.h"
 #include "channel.h"
 #include "channeltls.h"
 #include "channelpadding.h"
@@ -835,7 +836,7 @@ conn_close_if_marked(int i)
                (int)conn->outbuf_flushlen,
                 conn->marked_for_close_file, conn->marked_for_close);
     if (conn->linked_conn) {
-      retval = move_buf_to_buf(conn->linked_conn->inbuf, conn->outbuf,
+      retval = buf_move_to_buf(conn->linked_conn->inbuf, conn->outbuf,
                                &conn->outbuf_flushlen);
       if (retval >= 0) {
         /* The linked conn will notice that it has data when it notices that
@@ -849,12 +850,13 @@ conn_close_if_marked(int i)
                 connection_wants_to_flush(conn));
     } else if (connection_speaks_cells(conn)) {
       if (conn->state == OR_CONN_STATE_OPEN) {
-        retval = flush_buf_tls(TO_OR_CONN(conn)->tls, conn->outbuf, sz,
+        retval = buf_flush_to_tls(conn->outbuf, TO_OR_CONN(conn)->tls, sz,
                                &conn->outbuf_flushlen);
       } else
         retval = -1; /* never flush non-open broken tls connections */
     } else {
-      retval = flush_buf(conn->s, conn->outbuf, sz, &conn->outbuf_flushlen);
+      retval = buf_flush_to_socket(conn->outbuf, conn->s, sz,
+                                   &conn->outbuf_flushlen);
     }
     if (retval >= 0 && /* Technically, we could survive things like
                           TLS_WANT_WRITE here. But don't bother for now. */

src/or/or.h

@@ -1179,11 +1179,8 @@ typedef struct {
   uint16_t length; /**< How long is the payload body? */
 } relay_header_t;
 
-typedef struct buf_t buf_t;
 typedef struct socks_request_t socks_request_t;
 
-#define buf_t buf_t
-
 typedef struct entry_port_cfg_t {
   /* Client port types (socks, dns, trans, natd) only: */
   uint8_t isolation_flags; /**< Zero or more isolation flags */
@@ -1243,6 +1240,8 @@ typedef struct server_port_cfg_t {
 #define CONTROL_CONNECTION_MAGIC 0x8abc765du
 #define LISTENER_CONNECTION_MAGIC 0x1a1ac741u
 
+struct buf_t;
+
 /** Description of a connection to another host or process, and associated
  * data.
  *
@@ -1314,8 +1313,9 @@ typedef struct connection_t {
 
   struct event *read_event; /**< Libevent event structure. */
   struct event *write_event; /**< Libevent event structure. */
-  buf_t *inbuf; /**< Buffer holding data read over this connection. */
-  buf_t *outbuf; /**< Buffer holding data to write over this connection. */
+  struct buf_t *inbuf; /**< Buffer holding data read over this connection. */
+  struct buf_t *outbuf; /**< Buffer holding data to write over this
+                         * connection. */
   size_t outbuf_flushlen; /**< How much data should we try to flush from the
                            * outbuf? */
   time_t timestamp_lastread; /**< When was the last time libevent said we could
@@ -1722,11 +1722,11 @@ typedef struct entry_connection_t {
   /** For AP connections only: buffer for data that we have sent
    * optimistically, which we might need to re-send if we have to
    * retry this connection. */
-  buf_t *pending_optimistic_data;
+  struct buf_t *pending_optimistic_data;
   /* For AP connections only: buffer for data that we previously sent
   * optimistically which we are currently re-sending as we retry this
   * connection. */
-  buf_t *sending_optimistic_data;
+  struct buf_t *sending_optimistic_data;
 
   /** If this is a DNSPort connection, this field holds the pending DNS
    * request that we're going to try to answer.  */

src/or/proto_cell.c

@@ -0,0 +1,83 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "buffers.h"
+#include "proto_cell.h"
+
+#include "connection_or.h"
+
+/** True iff the cell command <b>command</b> is one that implies a
+ * variable-length cell in Tor link protocol <b>linkproto</b>. */
+static inline int
+cell_command_is_var_length(uint8_t command, int linkproto)
+{
+  /* If linkproto is v2 (2), CELL_VERSIONS is the only variable-length cells
+   * work as implemented here. If it's 1, there are no variable-length cells.
+   * Tor does not support other versions right now, and so can't negotiate
+   * them.
+   */
+  switch (linkproto) {
+  case 1:
+    /* Link protocol version 1 has no variable-length cells. */
+    return 0;
+  case 2:
+    /* In link protocol version 2, VERSIONS is the only variable-length cell */
+    return command == CELL_VERSIONS;
+  case 0:
+  case 3:
+  default:
+    /* In link protocol version 3 and later, and in version "unknown",
+     * commands 128 and higher indicate variable-length. VERSIONS is
+     * grandfathered in. */
+    return command == CELL_VERSIONS || command >= 128;
+  }
+}
+
+/** Check <b>buf</b> for a variable-length cell according to the rules of link
+ * protocol version <b>linkproto</b>.  If one is found, pull it off the buffer
+ * and assign a newly allocated var_cell_t to *<b>out</b>, and return 1.
+ * Return 0 if whatever is on the start of buf_t is not a variable-length
+ * cell.  Return 1 and set *<b>out</b> to NULL if there seems to be the start
+ * of a variable-length cell on <b>buf</b>, but the whole thing isn't there
+ * yet. */
+int
+fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto)
+{
+  char hdr[VAR_CELL_MAX_HEADER_SIZE];
+  var_cell_t *result;
+  uint8_t command;
+  uint16_t length;
+  const int wide_circ_ids = linkproto >= MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS;
+  const int circ_id_len = get_circ_id_size(wide_circ_ids);
+  const unsigned header_len = get_var_cell_header_size(wide_circ_ids);
+  *out = NULL;
+  if (buf_datalen(buf) < header_len)
+    return 0;
+  buf_peek(buf, hdr, header_len);
+
+  command = get_uint8(hdr + circ_id_len);
+  if (!(cell_command_is_var_length(command, linkproto)))
+    return 0;
+
+  length = ntohs(get_uint16(hdr + circ_id_len + 1));
+  if (buf_datalen(buf) < (size_t)(header_len+length))
+    return 1;
+  result = var_cell_new(length);
+  result->command = command;
+  if (wide_circ_ids)
+    result->circ_id = ntohl(get_uint32(hdr));
+  else
+    result->circ_id = ntohs(get_uint16(hdr));
+
+  buf_drain(buf, header_len);
+  buf_peek(buf, (char*) result->payload, length);
+  buf_drain(buf, length);
+
+  *out = result;
+  return 1;
+}
+

src/or/proto_cell.h

@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_CELL_H
+#define TOR_PROTO_CELL_H
+
+struct buf_t;
+struct var_cell_t;
+
+int fetch_var_cell_from_buf(struct buf_t *buf, struct var_cell_t **out,
+                            int linkproto);
+
+#endif
+

src/or/proto_control0.c

@@ -0,0 +1,26 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "buffers.h"
+#include "proto_control0.h"
+
+/** Return 1 iff buf looks more like it has an (obsolete) v0 controller
+ * command on it than any valid v1 controller command. */
+int
+peek_buf_has_control0_command(buf_t *buf)
+{
+  if (buf_datalen(buf) >= 4) {
+    char header[4];
+    uint16_t cmd;
+    buf_peek(buf, header, sizeof(header));
+    cmd = ntohs(get_uint16(header+2));
+    if (cmd <= 0x14)
+      return 1; /* This is definitely not a v1 control command. */
+  }
+  return 0;
+}
+

src/or/proto_control0.h

@@ -0,0 +1,14 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_CONTROL0_H
+#define TOR_PROTO_CONTROL0_H
+
+struct buf_t;
+int peek_buf_has_control0_command(struct buf_t *buf);
+
+#endif
+

src/or/proto_ext_or.c

@@ -0,0 +1,40 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "buffers.h"
+#include "ext_orport.h"
+#include "proto_ext_or.h"
+
+/** The size of the header of an Extended ORPort message: 2 bytes for
+ *  COMMAND, 2 bytes for BODYLEN */
+#define EXT_OR_CMD_HEADER_SIZE 4
+
+/** Read <b>buf</b>, which should contain an Extended ORPort message
+ *  from a transport proxy. If well-formed, create and populate
+ *  <b>out</b> with the Extended ORport message. Return 0 if the
+ *  buffer was incomplete, 1 if it was well-formed and -1 if we
+ *  encountered an error while parsing it.  */
+int
+fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out)
+{
+  char hdr[EXT_OR_CMD_HEADER_SIZE];
+  uint16_t len;
+
+  if (buf_datalen(buf) < EXT_OR_CMD_HEADER_SIZE)
+    return 0;
+  buf_peek(buf, hdr, sizeof(hdr));
+  len = ntohs(get_uint16(hdr+2));
+  if (buf_datalen(buf) < (unsigned)len + EXT_OR_CMD_HEADER_SIZE)
+    return 0;
+  *out = ext_or_cmd_new(len);
+  (*out)->cmd = ntohs(get_uint16(hdr));
+  (*out)->len = len;
+  buf_drain(buf, EXT_OR_CMD_HEADER_SIZE);
+  buf_get_bytes(buf, (*out)->body, len);
+  return 1;
+}
+

src/or/proto_ext_or.h

@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_EXT_OR_H
+#define TOR_PROTO_EXT_OR_H
+
+struct buf_t;
+struct ext_or_cmt_t;
+
+int fetch_ext_or_command_from_buf(struct buf_t *buf,
+                                  struct ext_or_cmd_t **out);
+
+#endif
+

src/or/proto_http.c

@@ -0,0 +1,171 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define PROTO_HTTP_PRIVATE
+#include "or.h"
+#include "buffers.h"
+#include "proto_http.h"
+
+/** Return true if <b>cmd</b> looks like a HTTP (proxy) request. */
+int
+peek_buf_has_http_command(const buf_t *buf)
+{
+  if (buf_peek_startswith(buf, "CONNECT ") ||
+      buf_peek_startswith(buf, "DELETE ") ||
+      buf_peek_startswith(buf, "GET ") ||
+      buf_peek_startswith(buf, "POST ") ||
+      buf_peek_startswith(buf, "PUT " ))
+    return 1;
+  return 0;
+}
+
+/** There is a (possibly incomplete) http statement on <b>buf</b>, of the
+ * form "\%s\\r\\n\\r\\n\%s", headers, body. (body may contain NULs.)
+ * If a) the headers include a Content-Length field and all bytes in
+ * the body are present, or b) there's no Content-Length field and
+ * all headers are present, then:
+ *
+ *  - strdup headers into <b>*headers_out</b>, and NUL-terminate it.
+ *  - memdup body into <b>*body_out</b>, and NUL-terminate it.
+ *  - Then remove them from <b>buf</b>, and return 1.
+ *
+ *  - If headers or body is NULL, discard that part of the buf.
+ *  - If a headers or body doesn't fit in the arg, return -1.
+ *  (We ensure that the headers or body don't exceed max len,
+ *   _even if_ we're planning to discard them.)
+ *  - If force_complete is true, then succeed even if not all of the
+ *    content has arrived.
+ *
+ * Else, change nothing and return 0.
+ */
+int
+fetch_from_buf_http(buf_t *buf,
+                    char **headers_out, size_t max_headerlen,
+                    char **body_out, size_t *body_used, size_t max_bodylen,
+                    int force_complete)
+{
+  const char *headers;
+  size_t headerlen, bodylen, contentlen=0;
+  int crlf_offset;
+  int r;
+
+  if (buf_datalen(buf) == 0)
+    return 0;
+
+  crlf_offset = buf_find_string_offset(buf, "\r\n\r\n", 4);
+  if (crlf_offset > (int)max_headerlen ||
+      (crlf_offset < 0 && buf_datalen(buf) > max_headerlen)) {
+    log_debug(LD_HTTP,"headers too long.");
+    return -1;
+  } else if (crlf_offset < 0) {
+    log_debug(LD_HTTP,"headers not all here yet.");
+    return 0;
+  }
+  /* Okay, we have a full header.  Make sure it all appears in the first
+   * chunk. */
+  headerlen = crlf_offset + 4;
+  size_t headers_in_chunk = 0;
+  buf_pullup(buf, headerlen, &headers, &headers_in_chunk);
+
+  bodylen = buf_datalen(buf) - headerlen;
+  log_debug(LD_HTTP,"headerlen %d, bodylen %d.", (int)headerlen, (int)bodylen);
+
+  if (max_headerlen <= headerlen) {
+    log_warn(LD_HTTP,"headerlen %d larger than %d. Failing.",
+             (int)headerlen, (int)max_headerlen-1);
+    return -1;
+  }
+  if (max_bodylen <= bodylen) {
+    log_warn(LD_HTTP,"bodylen %d larger than %d. Failing.",
+             (int)bodylen, (int)max_bodylen-1);
+    return -1;
+  }
+
+  r = buf_http_find_content_length(headers, headerlen, &contentlen);
+  if (r == -1) {
+    log_warn(LD_PROTOCOL, "Content-Length is bogus; maybe "
+             "someone is trying to crash us.");
+    return -1;
+  } else if (r == 1) {
+    /* if content-length is malformed, then our body length is 0. fine. */
+    log_debug(LD_HTTP,"Got a contentlen of %d.",(int)contentlen);
+    if (bodylen < contentlen) {
+      if (!force_complete) {
+        log_debug(LD_HTTP,"body not all here yet.");
+        return 0; /* not all there yet */
+      }
+    }
+    if (bodylen > contentlen) {
+      bodylen = contentlen;
+      log_debug(LD_HTTP,"bodylen reduced to %d.",(int)bodylen);
+    }
+  } else {
+    tor_assert(r == 0);
+    /* Leave bodylen alone */
+  }
+
+  /* all happy. copy into the appropriate places, and return 1 */
+  if (headers_out) {
+    *headers_out = tor_malloc(headerlen+1);
+    buf_get_bytes(buf, *headers_out, headerlen);
+    (*headers_out)[headerlen] = 0; /* NUL terminate it */
+  }
+  if (body_out) {
+    tor_assert(body_used);
+    *body_used = bodylen;
+    *body_out = tor_malloc(bodylen+1);
+    buf_get_bytes(buf, *body_out, bodylen);
+    (*body_out)[bodylen] = 0; /* NUL terminate it */
+  }
+  return 1;
+}
+
+/**
+ * Scan the HTTP headers in the <b>headerlen</b>-byte memory range at
+ * <b>headers</b>, looking for a "Content-Length" header.  Try to set
+ * *<b>result_out</b> to the numeric value of that header if possible.
+ * Return -1 if the header was malformed, 0 if it was missing, and 1 if
+ * it was present and well-formed.
+ */
+STATIC int
+buf_http_find_content_length(const char *headers, size_t headerlen,
+                             size_t *result_out)
+{
+  const char *p, *newline;
+  char *len_str, *eos=NULL;
+  size_t remaining, result;
+  int ok;
+  *result_out = 0; /* The caller shouldn't look at this unless the
+                    * return value is 1, but let's prevent confusion */
+
+#define CONTENT_LENGTH "\r\nContent-Length: "
+  p = (char*) tor_memstr(headers, headerlen, CONTENT_LENGTH);
+  if (p == NULL)
+    return 0;
+
+  tor_assert(p >= headers && p < headers+headerlen);
+  remaining = (headers+headerlen)-p;
+  p += strlen(CONTENT_LENGTH);
+  remaining -= strlen(CONTENT_LENGTH);
+
+  newline = memchr(p, '\n', remaining);
+  if (newline == NULL)
+    return -1;
+
+  len_str = tor_memdup_nulterm(p, newline-p);
+  /* We limit the size to INT_MAX because other parts of the buffer.c
+   * code don't like buffers to be any bigger than that. */
+  result = (size_t) tor_parse_uint64(len_str, 10, 0, INT_MAX, &ok, &eos);
+  if (eos && !tor_strisspace(eos)) {
+    ok = 0;
+  } else {
+    *result_out = result;
+  }
+  tor_free(len_str);
+
+  return ok ? 1 : -1;
+}
+

src/or/proto_http.h

@@ -0,0 +1,24 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_HTTP_H
+#define TOR_PROTO_HTTP_H
+
+struct buf_t;
+
+int fetch_from_buf_http(struct buf_t *buf,
+                        char **headers_out, size_t max_headerlen,
+                        char **body_out, size_t *body_used, size_t max_bodylen,
+                        int force_complete);
+int peek_buf_has_http_command(const struct buf_t *buf);
+
+#ifdef PROTO_HTTP_PRIVATE
+STATIC int buf_http_find_content_length(const char *headers, size_t headerlen,
+                                        size_t *result_out);
+#endif
+
+#endif
+

src/or/proto_socks.c

@@ -0,0 +1,698 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "addressmap.h"
+#include "buffers.h"
+#include "control.h"
+#include "config.h"
+#include "ext_orport.h"
+#include "proto_socks.h"
+#include "reasons.h"
+
+static void socks_request_set_socks5_error(socks_request_t *req,
+                              socks5_reply_status_t reason);
+
+static int parse_socks(const char *data, size_t datalen, socks_request_t *req,
+                       int log_sockstype, int safe_socks, ssize_t *drain_out,
+                       size_t *want_length_out);
+static int parse_socks_client(const uint8_t *data, size_t datalen,
+                              int state, char **reason,
+                              ssize_t *drain_out);
+/**
+ * Wait this many seconds before warning the user about using SOCKS unsafely
+ * again. */
+#define SOCKS_WARN_INTERVAL 5
+
+/** Warn that the user application has made an unsafe socks request using
+ * protocol <b>socks_protocol</b> on port <b>port</b>.  Don't warn more than
+ * once per SOCKS_WARN_INTERVAL, unless <b>safe_socks</b> is set. */
+static void
+log_unsafe_socks_warning(int socks_protocol, const char *address,
+                         uint16_t port, int safe_socks)
+{
+  static ratelim_t socks_ratelim = RATELIM_INIT(SOCKS_WARN_INTERVAL);
+
+  if (safe_socks) {
+    log_fn_ratelim(&socks_ratelim, LOG_WARN, LD_APP,
+             "Your application (using socks%d to port %d) is giving "
+             "Tor only an IP address. Applications that do DNS resolves "
+             "themselves may leak information. Consider using Socks4A "
+             "(e.g. via privoxy or socat) instead. For more information, "
+             "please see https://wiki.torproject.org/TheOnionRouter/"
+             "TorFAQ#SOCKSAndDNS.%s",
+             socks_protocol,
+             (int)port,
+             safe_socks ? " Rejecting." : "");
+  }
+  control_event_client_status(LOG_WARN,
+                              "DANGEROUS_SOCKS PROTOCOL=SOCKS%d ADDRESS=%s:%d",
+                              socks_protocol, address, (int)port);
+}
+
+/** Do not attempt to parse socks messages longer than this.  This value is
+ * actually significantly higher than the longest possible socks message. */
+#define MAX_SOCKS_MESSAGE_LEN 512
+
+/** Return a new socks_request_t. */
+socks_request_t *
+socks_request_new(void)
+{
+  return tor_malloc_zero(sizeof(socks_request_t));
+}
+
+/** Free all storage held in the socks_request_t <b>req</b>. */
+void
+socks_request_free(socks_request_t *req)
+{
+  if (!req)
+    return;
+  if (req->username) {
+    memwipe(req->username, 0x10, req->usernamelen);
+    tor_free(req->username);
+  }
+  if (req->password) {
+    memwipe(req->password, 0x04, req->passwordlen);
+    tor_free(req->password);
+  }
+  memwipe(req, 0xCC, sizeof(socks_request_t));
+  tor_free(req);
+}
+
+/** There is a (possibly incomplete) socks handshake on <b>buf</b>, of one
+ * of the forms
+ *  - socks4: "socksheader username\\0"
+ *  - socks4a: "socksheader username\\0 destaddr\\0"
+ *  - socks5 phase one: "version #methods methods"
+ *  - socks5 phase two: "version command 0 addresstype..."
+ * If it's a complete and valid handshake, and destaddr fits in
+ *   MAX_SOCKS_ADDR_LEN bytes, then pull the handshake off the buf,
+ *   assign to <b>req</b>, and return 1.
+ *
+ * If it's invalid or too big, return -1.
+ *
+ * Else it's not all there yet, leave buf alone and return 0.
+ *
+ * If you want to specify the socks reply, write it into <b>req->reply</b>
+ *   and set <b>req->replylen</b>, else leave <b>req->replylen</b> alone.
+ *
+ * If <b>log_sockstype</b> is non-zero, then do a notice-level log of whether
+ * the connection is possibly leaking DNS requests locally or not.
+ *
+ * If <b>safe_socks</b> is true, then reject unsafe socks protocols.
+ *
+ * If returning 0 or -1, <b>req->address</b> and <b>req->port</b> are
+ * undefined.
+ */
+int
+fetch_from_buf_socks(buf_t *buf, socks_request_t *req,
+                     int log_sockstype, int safe_socks)
+{
+  int res;
+  ssize_t n_drain;
+  size_t want_length = 128;
+  const char *head = NULL;
+  size_t datalen = 0;
+
+  if (buf_datalen(buf) < 2) /* version and another byte */
+    return 0;
+
+  do {
+    n_drain = 0;
+    buf_pullup(buf, want_length, &head, &datalen);
+    tor_assert(head && datalen >= 2);
+    want_length = 0;
+
+    res = parse_socks(head, datalen, req, log_sockstype,
+                      safe_socks, &n_drain, &want_length);
+
+    if (n_drain < 0)
+      buf_clear(buf);
+    else if (n_drain > 0)
+      buf_drain(buf, n_drain);
+
+  } while (res == 0 && head && want_length < buf_datalen(buf) &&
+           buf_datalen(buf) >= 2);
+
+  return res;
+}
+
+/** Create a SOCKS5 reply message with <b>reason</b> in its REP field and
+ * have Tor send it as error response to <b>req</b>.
+ */
+static void
+socks_request_set_socks5_error(socks_request_t *req,
+                  socks5_reply_status_t reason)
+{
+   req->replylen = 10;
+   memset(req->reply,0,10);
+
+   req->reply[0] = 0x05;   // VER field.
+   req->reply[1] = reason; // REP field.
+   req->reply[3] = 0x01;   // ATYP field.
+}
+
+static const char SOCKS_PROXY_IS_NOT_AN_HTTP_PROXY_MSG[] =
+  "HTTP/1.0 501 Tor is not an HTTP Proxy\r\n"
+  "Content-Type: text/html; charset=iso-8859-1\r\n\r\n"
+  "<html>\n"
+  "<head>\n"
+  "<title>Tor is not an HTTP Proxy</title>\n"
+  "</head>\n"
+  "<body>\n"
+  "<h1>Tor is not an HTTP Proxy</h1>\n"
+  "<p>\n"
+  "It appears you have configured your web browser to use Tor as "
+  "an HTTP proxy.\n\n"
+  "This is not correct: Tor is a SOCKS proxy, not an HTTP proxy.\n"
+  "Please configure your client accordingly.\n"
+  "</p>\n"
+  "<p>\n"
+  "See <a href=\"https://www.torproject.org/documentation.html\">"
+  "https://www.torproject.org/documentation.html</a> for more "
+  "information.\n"
+  "<!-- Plus this comment, to make the body response more than 512 bytes, so "
+  "     IE will be willing to display it. Comment comment comment comment "
+  "     comment comment comment comment comment comment comment comment.-->\n"
+  "</p>\n"
+  "</body>\n"
+  "</html>\n";
+
+/** Implementation helper to implement fetch_from_*_socks.  Instead of looking
+ * at a buffer's contents, we look at the <b>datalen</b> bytes of data in
+ * <b>data</b>. Instead of removing data from the buffer, we set
+ * <b>drain_out</b> to the amount of data that should be removed (or -1 if the
+ * buffer should be cleared).  Instead of pulling more data into the first
+ * chunk of the buffer, we set *<b>want_length_out</b> to the number of bytes
+ * we'd like to see in the input buffer, if they're available. */
+static int
+parse_socks(const char *data, size_t datalen, socks_request_t *req,
+            int log_sockstype, int safe_socks, ssize_t *drain_out,
+            size_t *want_length_out)
+{
+  unsigned int len;
+  char tmpbuf[TOR_ADDR_BUF_LEN+1];
+  tor_addr_t destaddr;
+  uint32_t destip;
+  uint8_t socksver;
+  char *next, *startaddr;
+  unsigned char usernamelen, passlen;
+  struct in_addr in;
+
+  if (datalen < 2) {
+    /* We always need at least 2 bytes. */
+    *want_length_out = 2;
+    return 0;
+  }
+
+  if (req->socks_version == 5 && !req->got_auth) {
+    /* See if we have received authentication.  Strictly speaking, we should
+       also check whether we actually negotiated username/password
+       authentication.  But some broken clients will send us authentication
+       even if we negotiated SOCKS_NO_AUTH. */
+    if (*data == 1) { /* username/pass version 1 */
+      /* Format is: authversion [1 byte] == 1
+                    usernamelen [1 byte]
+                    username    [usernamelen bytes]
+                    passlen     [1 byte]
+                    password    [passlen bytes] */
+      usernamelen = (unsigned char)*(data + 1);
+      if (datalen < 2u + usernamelen + 1u) {
+        *want_length_out = 2u + usernamelen + 1u;
+        return 0;
+      }
+      passlen = (unsigned char)*(data + 2u + usernamelen);
+      if (datalen < 2u + usernamelen + 1u + passlen) {
+        *want_length_out = 2u + usernamelen + 1u + passlen;
+        return 0;
+      }
+      req->replylen = 2; /* 2 bytes of response */
+      req->reply[0] = 1; /* authversion == 1 */
+      req->reply[1] = 0; /* authentication successful */
+      log_debug(LD_APP,
+               "socks5: Accepted username/password without checking.");
+      if (usernamelen) {
+        req->username = tor_memdup(data+2u, usernamelen);
+        req->usernamelen = usernamelen;
+      }
+      if (passlen) {
+        req->password = tor_memdup(data+3u+usernamelen, passlen);
+        req->passwordlen = passlen;
+      }
+      *drain_out = 2u + usernamelen + 1u + passlen;
+      req->got_auth = 1;
+      *want_length_out = 7; /* Minimal socks5 command. */
+      return 0;
+    } else if (req->auth_type == SOCKS_USER_PASS) {
+      /* unknown version byte */
+      log_warn(LD_APP, "Socks5 username/password version %d not recognized; "
+               "rejecting.", (int)*data);
+      return -1;
+    }
+  }
+
+  socksver = *data;
+
+  switch (socksver) { /* which version of socks? */
+    case 5: /* socks5 */
+
+      if (req->socks_version != 5) { /* we need to negotiate a method */
+        unsigned char nummethods = (unsigned char)*(data+1);
+        int have_user_pass, have_no_auth;
+        int r=0;
+        tor_assert(!req->socks_version);
+        if (datalen < 2u+nummethods) {
+          *want_length_out = 2u+nummethods;
+          return 0;
+        }
+        if (!nummethods)
+          return -1;
+        req->replylen = 2; /* 2 bytes of response */
+        req->reply[0] = 5; /* socks5 reply */
+        have_user_pass = (memchr(data+2, SOCKS_USER_PASS, nummethods) !=NULL);
+        have_no_auth   = (memchr(data+2, SOCKS_NO_AUTH,   nummethods) !=NULL);
+        if (have_user_pass && !(have_no_auth && req->socks_prefer_no_auth)) {
+          req->auth_type = SOCKS_USER_PASS;
+          req->reply[1] = SOCKS_USER_PASS; /* tell client to use "user/pass"
+                                              auth method */
+          req->socks_version = 5; /* remember we've already negotiated auth */
+          log_debug(LD_APP,"socks5: accepted method 2 (username/password)");
+          r=0;
+        } else if (have_no_auth) {
+          req->reply[1] = SOCKS_NO_AUTH; /* tell client to use "none" auth
+                                            method */
+          req->socks_version = 5; /* remember we've already negotiated auth */
+          log_debug(LD_APP,"socks5: accepted method 0 (no authentication)");
+          r=0;
+        } else {
+          log_warn(LD_APP,
+                    "socks5: offered methods don't include 'no auth' or "
+                    "username/password. Rejecting.");
+          req->reply[1] = '\xFF'; /* reject all methods */
+          r=-1;
+        }
+        /* Remove packet from buf. Some SOCKS clients will have sent extra
+         * junk at this point; let's hope it's an authentication message. */
+        *drain_out = 2u + nummethods;
+
+        return r;
+      }
+      if (req->auth_type != SOCKS_NO_AUTH && !req->got_auth) {
+        log_warn(LD_APP,
+                 "socks5: negotiated authentication, but none provided");
+        return -1;
+      }
+      /* we know the method; read in the request */
+      log_debug(LD_APP,"socks5: checking request");
+      if (datalen < 7) {/* basic info plus >=1 for addr plus 2 for port */
+        *want_length_out = 7;
+        return 0; /* not yet */
+      }
+      req->command = (unsigned char) *(data+1);
+      if (req->command != SOCKS_COMMAND_CONNECT &&
+          req->command != SOCKS_COMMAND_RESOLVE &&
+          req->command != SOCKS_COMMAND_RESOLVE_PTR) {
+        /* not a connect or resolve or a resolve_ptr? we don't support it. */
+        socks_request_set_socks5_error(req,SOCKS5_COMMAND_NOT_SUPPORTED);
+
+        log_warn(LD_APP,"socks5: command %d not recognized. Rejecting.",
+                 req->command);
+        return -1;
+      }
+      switch (*(data+3)) { /* address type */
+        case 1: /* IPv4 address */
+        case 4: /* IPv6 address */ {
+          const int is_v6 = *(data+3) == 4;
+          const unsigned addrlen = is_v6 ? 16 : 4;
+          log_debug(LD_APP,"socks5: ipv4 address type");
+          if (datalen < 6+addrlen) {/* ip/port there? */
+            *want_length_out = 6+addrlen;
+            return 0; /* not yet */
+          }
+
+          if (is_v6)
+            tor_addr_from_ipv6_bytes(&destaddr, data+4);
+          else
+            tor_addr_from_ipv4n(&destaddr, get_uint32(data+4));
+
+          tor_addr_to_str(tmpbuf, &destaddr, sizeof(tmpbuf), 1);
+
+          if (strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN) {
+            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
+            log_warn(LD_APP,
+                     "socks5 IP takes %d bytes, which doesn't fit in %d. "
+                     "Rejecting.",
+                     (int)strlen(tmpbuf)+1,(int)MAX_SOCKS_ADDR_LEN);
+            return -1;
+          }
+          strlcpy(req->address,tmpbuf,sizeof(req->address));
+          req->port = ntohs(get_uint16(data+4+addrlen));
+          *drain_out = 6+addrlen;
+          if (req->command != SOCKS_COMMAND_RESOLVE_PTR &&
+              !addressmap_have_mapping(req->address,0)) {
+            log_unsafe_socks_warning(5, req->address, req->port, safe_socks);
+            if (safe_socks) {
+              socks_request_set_socks5_error(req, SOCKS5_NOT_ALLOWED);
+              return -1;
+            }
+          }
+          return 1;
+        }
+        case 3: /* fqdn */
+          log_debug(LD_APP,"socks5: fqdn address type");
+          if (req->command == SOCKS_COMMAND_RESOLVE_PTR) {
+            socks_request_set_socks5_error(req,
+                                           SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED);
+            log_warn(LD_APP, "socks5 received RESOLVE_PTR command with "
+                     "hostname type. Rejecting.");
+            return -1;
+          }
+          len = (unsigned char)*(data+4);
+          if (datalen < 7+len) { /* addr/port there? */
+            *want_length_out = 7+len;
+            return 0; /* not yet */
+          }
+          if (len+1 > MAX_SOCKS_ADDR_LEN) {
+            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
+            log_warn(LD_APP,
+                     "socks5 hostname is %d bytes, which doesn't fit in "
+                     "%d. Rejecting.", len+1,MAX_SOCKS_ADDR_LEN);
+            return -1;
+          }
+          memcpy(req->address,data+5,len);
+          req->address[len] = 0;
+          req->port = ntohs(get_uint16(data+5+len));
+          *drain_out = 5+len+2;
+
+          if (!string_is_valid_hostname(req->address)) {
+            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
+
+            log_warn(LD_PROTOCOL,
+                     "Your application (using socks5 to port %d) gave Tor "
+                     "a malformed hostname: %s. Rejecting the connection.",
+                     req->port, escaped_safe_str_client(req->address));
+            return -1;
+          }
+          if (log_sockstype)
+            log_notice(LD_APP,
+                  "Your application (using socks5 to port %d) instructed "
+                  "Tor to take care of the DNS resolution itself if "
+                  "necessary. This is good.", req->port);
+          return 1;
+        default: /* unsupported */
+          socks_request_set_socks5_error(req,
+                                         SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED);
+          log_warn(LD_APP,"socks5: unsupported address type %d. Rejecting.",
+                   (int) *(data+3));
+          return -1;
+      }
+      tor_assert(0);
+      break;
+    case 4: { /* socks4 */
+      enum {socks4, socks4a} socks4_prot = socks4a;
+      const char *authstart, *authend;
+      /* http://ss5.sourceforge.net/socks4.protocol.txt */
+      /* http://ss5.sourceforge.net/socks4A.protocol.txt */
+
+      req->socks_version = 4;
+      if (datalen < SOCKS4_NETWORK_LEN) {/* basic info available? */
+        *want_length_out = SOCKS4_NETWORK_LEN;
+        return 0; /* not yet */
+      }
+      // buf_pullup(buf, 1280);
+      req->command = (unsigned char) *(data+1);
+      if (req->command != SOCKS_COMMAND_CONNECT &&
+          req->command != SOCKS_COMMAND_RESOLVE) {
+        /* not a connect or resolve? we don't support it. (No resolve_ptr with
+         * socks4.) */
+        log_warn(LD_APP,"socks4: command %d not recognized. Rejecting.",
+                 req->command);
+        return -1;
+      }
+
+      req->port = ntohs(get_uint16(data+2));
+      destip = ntohl(get_uint32(data+4));
+      if ((!req->port && req->command!=SOCKS_COMMAND_RESOLVE) || !destip) {
+        log_warn(LD_APP,"socks4: Port or DestIP is zero. Rejecting.");
+        return -1;
+      }
+      if (destip >> 8) {
+        log_debug(LD_APP,"socks4: destip not in form 0.0.0.x.");
+        in.s_addr = htonl(destip);
+        tor_inet_ntoa(&in,tmpbuf,sizeof(tmpbuf));
+        if (strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN) {
+          log_debug(LD_APP,"socks4 addr (%d bytes) too long. Rejecting.",
+                    (int)strlen(tmpbuf));
+          return -1;
+        }
+        log_debug(LD_APP,
+                  "socks4: successfully read destip (%s)",
+                  safe_str_client(tmpbuf));
+        socks4_prot = socks4;
+      }
+
+      authstart = data + SOCKS4_NETWORK_LEN;
+      next = memchr(authstart, 0,
+                    datalen-SOCKS4_NETWORK_LEN);
+      if (!next) {
+        if (datalen >= 1024) {
+          log_debug(LD_APP, "Socks4 user name too long; rejecting.");
+          return -1;
+        }
+        log_debug(LD_APP,"socks4: Username not here yet.");
+        *want_length_out = datalen+1024; /* More than we need, but safe */
+        return 0;
+      }
+      authend = next;
+      tor_assert(next < data+datalen);
+
+      startaddr = NULL;
+      if (socks4_prot != socks4a &&
+          !addressmap_have_mapping(tmpbuf,0)) {
+        log_unsafe_socks_warning(4, tmpbuf, req->port, safe_socks);
+
+        if (safe_socks)
+          return -1;
+      }
+      if (socks4_prot == socks4a) {
+        if (next+1 == data+datalen) {
+          log_debug(LD_APP,"socks4: No part of destaddr here yet.");
+          *want_length_out = datalen + 1024; /* More than we need, but safe */
+          return 0;
+        }
+        startaddr = next+1;
+        next = memchr(startaddr, 0, data + datalen - startaddr);
+        if (!next) {
+          if (datalen >= 1024) {
+            log_debug(LD_APP,"socks4: Destaddr too long.");
+            return -1;
+          }
+          log_debug(LD_APP,"socks4: Destaddr not all here yet.");
+          *want_length_out = datalen + 1024; /* More than we need, but safe */
+          return 0;
+        }
+        if (MAX_SOCKS_ADDR_LEN <= next-startaddr) {
+          log_warn(LD_APP,"socks4: Destaddr too long. Rejecting.");
+          return -1;
+        }
+        // tor_assert(next < buf->cur+buf_datalen(buf));
+
+        if (log_sockstype)
+          log_notice(LD_APP,
+                     "Your application (using socks4a to port %d) instructed "
+                     "Tor to take care of the DNS resolution itself if "
+                     "necessary. This is good.", req->port);
+      }
+      log_debug(LD_APP,"socks4: Everything is here. Success.");
+      strlcpy(req->address, startaddr ? startaddr : tmpbuf,
+              sizeof(req->address));
+      if (!string_is_valid_hostname(req->address)) {
+        log_warn(LD_PROTOCOL,
+                 "Your application (using socks4 to port %d) gave Tor "
+                 "a malformed hostname: %s. Rejecting the connection.",
+                 req->port, escaped_safe_str_client(req->address));
+        return -1;
+      }
+      if (authend != authstart) {
+        req->got_auth = 1;
+        req->usernamelen = authend - authstart;
+        req->username = tor_memdup(authstart, authend - authstart);
+      }
+      /* next points to the final \0 on inbuf */
+      *drain_out = next - data + 1;
+      return 1;
+    }
+    case 'G': /* get */
+    case 'H': /* head */
+    case 'P': /* put/post */
+    case 'C': /* connect */
+      strlcpy((char*)req->reply, SOCKS_PROXY_IS_NOT_AN_HTTP_PROXY_MSG,
+              MAX_SOCKS_REPLY_LEN);
+      req->replylen = strlen((char*)req->reply)+1;
+      /* fall through */
+    default: /* version is not socks4 or socks5 */
+      log_warn(LD_APP,
+               "Socks version %d not recognized. (Tor is not an http proxy.)",
+               *(data));
+      {
+        /* Tell the controller the first 8 bytes. */
+        char *tmp = tor_strndup(data, datalen < 8 ? datalen : 8);
+        control_event_client_status(LOG_WARN,
+                                    "SOCKS_UNKNOWN_PROTOCOL DATA=\"%s\"",
+                                    escaped(tmp));
+        tor_free(tmp);
+      }
+      return -1;
+  }
+}
+
+/** Inspect a reply from SOCKS server stored in <b>buf</b> according
+ * to <b>state</b>, removing the protocol data upon success. Return 0 on
+ * incomplete response, 1 on success and -1 on error, in which case
+ * <b>reason</b> is set to a descriptive message (free() when finished
+ * with it).
+ *
+ * As a special case, 2 is returned when user/pass is required
+ * during SOCKS5 handshake and user/pass is configured.
+ */
+int
+fetch_from_buf_socks_client(buf_t *buf, int state, char **reason)
+{
+  ssize_t drain = 0;
+  int r;
+  const char *head = NULL;
+  size_t datalen = 0;
+
+  if (buf_datalen(buf) < 2)
+    return 0;
+
+  buf_pullup(buf, MAX_SOCKS_MESSAGE_LEN, &head, &datalen);
+  tor_assert(head && datalen >= 2);
+
+  r = parse_socks_client((uint8_t*)head, datalen,
+                         state, reason, &drain);
+  if (drain > 0)
+    buf_drain(buf, drain);
+  else if (drain < 0)
+    buf_clear(buf);
+
+  return r;
+}
+
+/** Implementation logic for fetch_from_*_socks_client. */
+static int
+parse_socks_client(const uint8_t *data, size_t datalen,
+                   int state, char **reason,
+                   ssize_t *drain_out)
+{
+  unsigned int addrlen;
+  *drain_out = 0;
+  if (datalen < 2)
+    return 0;
+
+  switch (state) {
+    case PROXY_SOCKS4_WANT_CONNECT_OK:
+      /* Wait for the complete response */
+      if (datalen < 8)
+        return 0;
+
+      if (data[1] != 0x5a) {
+        *reason = tor_strdup(socks4_response_code_to_string(data[1]));
+        return -1;
+      }
+
+      /* Success */
+      *drain_out = 8;
+      return 1;
+
+    case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE:
+      /* we don't have any credentials */
+      if (data[1] != 0x00) {
+        *reason = tor_strdup("server doesn't support any of our "
+                             "available authentication methods");
+        return -1;
+      }
+
+      log_info(LD_NET, "SOCKS 5 client: continuing without authentication");
+      *drain_out = -1;
+      return 1;
+
+    case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929:
+      /* we have a username and password. return 1 if we can proceed without
+       * providing authentication, or 2 otherwise. */
+      switch (data[1]) {
+        case 0x00:
+          log_info(LD_NET, "SOCKS 5 client: we have auth details but server "
+                            "doesn't require authentication.");
+          *drain_out = -1;
+          return 1;
+        case 0x02:
+          log_info(LD_NET, "SOCKS 5 client: need authentication.");
+          *drain_out = -1;
+          return 2;
+        /* fall through */
+      }
+
+      *reason = tor_strdup("server doesn't support any of our available "
+                           "authentication methods");
+      return -1;
+
+    case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK:
+      /* handle server reply to rfc1929 authentication */
+      if (data[1] != 0x00) {
+        *reason = tor_strdup("authentication failed");
+        return -1;
+      }
+
+      log_info(LD_NET, "SOCKS 5 client: authentication successful.");
+      *drain_out = -1;
+      return 1;
+
+    case PROXY_SOCKS5_WANT_CONNECT_OK:
+      /* response is variable length. BND.ADDR, etc, isn't needed
+       * (don't bother with buf_pullup()), but make sure to eat all
+       * the data used */
+
+      /* wait for address type field to arrive */
+      if (datalen < 4)
+        return 0;
+
+      switch (data[3]) {
+        case 0x01: /* ip4 */
+          addrlen = 4;
+          break;
+        case 0x04: /* ip6 */
+          addrlen = 16;
+          break;
+        case 0x03: /* fqdn (can this happen here?) */
+          if (datalen < 5)
+            return 0;
+          addrlen = 1 + data[4];
+          break;
+        default:
+          *reason = tor_strdup("invalid response to connect request");
+          return -1;
+      }
+
+      /* wait for address and port */
+      if (datalen < 6 + addrlen)
+        return 0;
+
+      if (data[1] != 0x00) {
+        *reason = tor_strdup(socks5_response_code_to_string(data[1]));
+        return -1;
+      }
+
+      *drain_out = 6 + addrlen;
+      return 1;
+  }
+
+  /* shouldn't get here... */
+  tor_assert(0);
+
+  return -1;
+}
+

src/or/proto_socks.h

@@ -0,0 +1,20 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_SOCKS_H
+#define TOR_PROTO_SOCKS_H
+
+struct socks_request_t;
+struct buf_t;
+
+struct socks_request_t *socks_request_new(void);
+void socks_request_free(struct socks_request_t *req);
+int fetch_from_buf_socks(struct buf_t *buf, socks_request_t *req,
+                         int log_sockstype, int safe_socks);
+int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason);
+
+#endif
+

src/or/relay.c

@@ -1670,7 +1670,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       }
 
       stats_n_data_bytes_received += rh.length;
-      connection_write_to_buf((char*)(cell->payload + RELAY_HEADER_SIZE),
+      connection_buf_add((char*)(cell->payload + RELAY_HEADER_SIZE),
                               rh.length, TO_CONN(conn));
 
 #ifdef MEASUREMENTS_21206
@@ -2038,13 +2038,13 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
     /* XXXX We could be more efficient here by sometimes packing
      * previously-sent optimistic data in the same cell with data
      * from the inbuf. */
-    fetch_from_buf(payload, length, entry_conn->sending_optimistic_data);
+    buf_get_bytes(entry_conn->sending_optimistic_data, payload, length);
     if (!buf_datalen(entry_conn->sending_optimistic_data)) {
         buf_free(entry_conn->sending_optimistic_data);
         entry_conn->sending_optimistic_data = NULL;
     }
   } else {
-    connection_fetch_from_buf(payload, length, TO_CONN(conn));
+    connection_buf_get_bytes(payload, length, TO_CONN(conn));
   }
 
   log_debug(domain,TOR_SOCKET_T_FORMAT": Packaging %d bytes (%d waiting).",
@@ -2056,7 +2056,7 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
        retry */
     if (!entry_conn->pending_optimistic_data)
       entry_conn->pending_optimistic_data = buf_new();
-    write_to_buf(payload, length, entry_conn->pending_optimistic_data);
+    buf_add(entry_conn->pending_optimistic_data, payload, length);
   }
 
   if (connection_edge_send_command(conn, RELAY_COMMAND_DATA,

src/test/test_buffers.c

@@ -4,9 +4,16 @@
 /* See LICENSE for licensing information */
 
 #define BUFFERS_PRIVATE
+#define PROTO_HTTP_PRIVATE
 #include "or.h"
 #include "buffers.h"
+#include "buffers_tls.h"
 #include "ext_orport.h"
+#include "proto_cell.h"
+#include "proto_ext_or.h"
+#include "proto_http.h"
+#include "proto_control0.h"
+#include "proto_socks.h"
 #include "test.h"
 
 /** Run unit tests for buffers.c */
@@ -38,15 +45,15 @@ test_buffers_basic(void *arg)
   for (j=0;j<256;++j) {
     str[j] = (char)j;
   }
-  write_to_buf(str, 256, buf);
-  write_to_buf(str, 256, buf);
+  buf_add(buf, str, 256);
+  buf_add(buf, str, 256);
   tt_int_op(buf_datalen(buf),OP_EQ, 512);
-  fetch_from_buf(str2, 200, buf);
+  buf_get_bytes(buf, str2, 200);
   tt_mem_op(str,OP_EQ, str2, 200);
   tt_int_op(buf_datalen(buf),OP_EQ, 312);
   memset(str2, 0, sizeof(str2));
 
-  fetch_from_buf(str2, 256, buf);
+  buf_get_bytes(buf, str2, 256);
   tt_mem_op(str+200,OP_EQ, str2, 56);
   tt_mem_op(str,OP_EQ, str2+56, 200);
   tt_int_op(buf_datalen(buf),OP_EQ, 56);
@@ -54,16 +61,16 @@ test_buffers_basic(void *arg)
   /* Okay, now we should be 512 bytes into the 4096-byte buffer.  If we add
    * another 3584 bytes, we hit the end. */
   for (j=0;j<15;++j) {
-    write_to_buf(str, 256, buf);
+    buf_add(buf, str, 256);
   }
-  assert_buf_ok(buf);
+  buf_assert_ok(buf);
   tt_int_op(buf_datalen(buf),OP_EQ, 3896);
-  fetch_from_buf(str2, 56, buf);
+  buf_get_bytes(buf, str2, 56);
   tt_int_op(buf_datalen(buf),OP_EQ, 3840);
   tt_mem_op(str+200,OP_EQ, str2, 56);
   for (j=0;j<15;++j) {
     memset(str2, 0, sizeof(str2));
-    fetch_from_buf(str2, 256, buf);
+    buf_get_bytes(buf, str2, 256);
     tt_mem_op(str,OP_EQ, str2, 256);
   }
   tt_int_op(buf_datalen(buf),OP_EQ, 0);
@@ -73,38 +80,38 @@ test_buffers_basic(void *arg)
   /* Okay, now make sure growing can work. */
   buf = buf_new_with_capacity(16);
   //test_eq(buf_capacity(buf), 16);
-  write_to_buf(str+1, 255, buf);
+  buf_add(buf, str+1, 255);
   //test_eq(buf_capacity(buf), 256);
-  fetch_from_buf(str2, 254, buf);
+  buf_get_bytes(buf, str2, 254);
   tt_mem_op(str+1,OP_EQ, str2, 254);
   //test_eq(buf_capacity(buf), 256);
-  assert_buf_ok(buf);
-  write_to_buf(str, 32, buf);
+  buf_assert_ok(buf);
+  buf_add(buf, str, 32);
   //test_eq(buf_capacity(buf), 256);
-  assert_buf_ok(buf);
-  write_to_buf(str, 256, buf);
-  assert_buf_ok(buf);
+  buf_assert_ok(buf);
+  buf_add(buf, str, 256);
+  buf_assert_ok(buf);
   //test_eq(buf_capacity(buf), 512);
   tt_int_op(buf_datalen(buf),OP_EQ, 33+256);
-  fetch_from_buf(str2, 33, buf);
+  buf_get_bytes(buf, str2, 33);
   tt_int_op(*str2,OP_EQ, str[255]);
 
   tt_mem_op(str2+1,OP_EQ, str, 32);
   //test_eq(buf_capacity(buf), 512);
   tt_int_op(buf_datalen(buf),OP_EQ, 256);
-  fetch_from_buf(str2, 256, buf);
+  buf_get_bytes(buf, str2, 256);
   tt_mem_op(str,OP_EQ, str2, 256);
 
   /* now try shrinking: case 1. */
   buf_free(buf);
   buf = buf_new_with_capacity(33668);
   for (j=0;j<67;++j) {
-    write_to_buf(str,255, buf);
+    buf_add(buf, str,255);
   }
   //test_eq(buf_capacity(buf), 33668);
   tt_int_op(buf_datalen(buf),OP_EQ, 17085);
   for (j=0; j < 40; ++j) {
-    fetch_from_buf(str2, 255,buf);
+    buf_get_bytes(buf, str2, 255);
     tt_mem_op(str2,OP_EQ, str, 255);
   }
 
@@ -112,18 +119,18 @@ test_buffers_basic(void *arg)
   buf_free(buf);
   buf = buf_new_with_capacity(33668);
   for (j=0;j<67;++j) {
-    write_to_buf(str,255, buf);
+    buf_add(buf, str, 255);
   }
   for (j=0; j < 20; ++j) {
-    fetch_from_buf(str2, 255,buf);
+    buf_get_bytes(buf, str2, 255);
     tt_mem_op(str2,OP_EQ, str, 255);
   }
   for (j=0;j<80;++j) {
-    write_to_buf(str,255, buf);
+    buf_add(buf, str, 255);
   }
   //test_eq(buf_capacity(buf),33668);
   for (j=0; j < 120; ++j) {
-    fetch_from_buf(str2, 255,buf);
+    buf_get_bytes(buf, str2, 255);
     tt_mem_op(str2,OP_EQ, str, 255);
   }
 
@@ -132,27 +139,27 @@ test_buffers_basic(void *arg)
   buf = buf_new_with_capacity(4096);
   buf2 = buf_new_with_capacity(4096);
   for (j=0;j<100;++j)
-    write_to_buf(str, 255, buf);
+    buf_add(buf, str, 255);
   tt_int_op(buf_datalen(buf),OP_EQ, 25500);
   for (j=0;j<100;++j) {
     r = 10;
-    move_buf_to_buf(buf2, buf, &r);
+    buf_move_to_buf(buf2, buf, &r);
     tt_int_op(r,OP_EQ, 0);
   }
   tt_int_op(buf_datalen(buf),OP_EQ, 24500);
   tt_int_op(buf_datalen(buf2),OP_EQ, 1000);
   for (j=0;j<3;++j) {
-    fetch_from_buf(str2, 255, buf2);
+    buf_get_bytes(buf2, str2, 255);
     tt_mem_op(str2,OP_EQ, str, 255);
   }
   r = 8192; /*big move*/
-  move_buf_to_buf(buf2, buf, &r);
+  buf_move_to_buf(buf2, buf, &r);
   tt_int_op(r,OP_EQ, 0);
   r = 30000; /* incomplete move */
-  move_buf_to_buf(buf2, buf, &r);
+  buf_move_to_buf(buf2, buf, &r);
   tt_int_op(r,OP_EQ, 13692);
   for (j=0;j<97;++j) {
-    fetch_from_buf(str2, 255, buf2);
+    buf_get_bytes(buf2, str2, 255);
     tt_mem_op(str2,OP_EQ, str, 255);
   }
   buf_free(buf);
@@ -162,7 +169,7 @@ test_buffers_basic(void *arg)
   buf = buf_new_with_capacity(5);
   cp = "Testing. This is a moderately long Testing string.";
   for (j = 0; cp[j]; j++)
-    write_to_buf(cp+j, 1, buf);
+    buf_add(buf, cp+j, 1);
   tt_int_op(0,OP_EQ, buf_find_string_offset(buf, "Testing", 7));
   tt_int_op(1,OP_EQ, buf_find_string_offset(buf, "esting", 6));
   tt_int_op(1,OP_EQ, buf_find_string_offset(buf, "est", 3));
@@ -180,7 +187,7 @@ test_buffers_basic(void *arg)
   {
     char *mem = tor_malloc_zero(65536);
     buf = buf_new();
-    write_to_buf(mem, 65536, buf);
+    buf_add(buf, mem, 65536);
     tor_free(mem);
 
     tt_int_op(buf_datalen(buf), OP_EQ, 65536);
@@ -215,8 +222,7 @@ test_buffer_pullup(void *arg)
 
   /* There are a bunch of cases for pullup.  One is the trivial case. Let's
      mess around with an empty buffer. */
-  buf_pullup(buf, 16);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_pullup(buf, 16, &cp, &sz);
   tt_ptr_op(cp, OP_EQ, NULL);
   tt_uint_op(sz, OP_EQ, 0);
 
@@ -227,65 +233,62 @@ test_buffer_pullup(void *arg)
 
   /* Let's add some data. */
   crypto_rand(stuff, 16384);
-  write_to_buf(stuff, 3000, buf);
-  write_to_buf(stuff+3000, 3000, buf);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_add(buf, stuff, 3000);
+  buf_add(buf, stuff+3000, 3000);
+  buf_pullup(buf, 0, &cp, &sz);
   tt_ptr_op(cp, OP_NE, NULL);
   tt_int_op(sz, OP_LE, 4096);
 
   /* Make room for 3000 bytes in the first chunk, so that the pullup-move code
    * can get tested. */
-  tt_int_op(fetch_from_buf(tmp, 3000, buf), OP_EQ, 3000);
+  tt_int_op(buf_get_bytes(buf, tmp, 3000), OP_EQ, 3000);
   tt_mem_op(tmp,OP_EQ, stuff, 3000);
-  buf_pullup(buf, 2048);
-  assert_buf_ok(buf);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_pullup(buf, 2048, &cp, &sz);
+  buf_assert_ok(buf);
   tt_ptr_op(cp, OP_NE, NULL);
   tt_int_op(sz, OP_GE, 2048);
   tt_mem_op(cp,OP_EQ, stuff+3000, 2048);
   tt_int_op(3000, OP_EQ, buf_datalen(buf));
-  tt_int_op(fetch_from_buf(tmp, 3000, buf), OP_EQ, 0);
+  tt_int_op(buf_get_bytes(buf, tmp, 3000), OP_EQ, 0);
   tt_mem_op(tmp,OP_EQ, stuff+3000, 2048);
 
   buf_free(buf);
 
   /* Now try the large-chunk case. */
   buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */
-  write_to_buf(stuff, 4000, buf);
-  write_to_buf(stuff+4000, 4000, buf);
-  write_to_buf(stuff+8000, 4000, buf);
-  write_to_buf(stuff+12000, 4000, buf);
+  buf_add(buf, stuff, 4000);
+  buf_add(buf, stuff+4000, 4000);
+  buf_add(buf, stuff+8000, 4000);
+  buf_add(buf, stuff+12000, 4000);
   tt_int_op(buf_datalen(buf), OP_EQ, 16000);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_pullup(buf, 0, &cp, &sz);
   tt_ptr_op(cp, OP_NE, NULL);
   tt_int_op(sz, OP_LE, 4096);
 
-  buf_pullup(buf, 12500);
-  assert_buf_ok(buf);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_pullup(buf, 12500, &cp, &sz);
+  buf_assert_ok(buf);
   tt_ptr_op(cp, OP_NE, NULL);
   tt_int_op(sz, OP_GE, 12500);
   tt_mem_op(cp,OP_EQ, stuff, 12500);
   tt_int_op(buf_datalen(buf), OP_EQ, 16000);
 
-  fetch_from_buf(tmp, 12400, buf);
+  buf_get_bytes(buf, tmp, 12400);
   tt_mem_op(tmp,OP_EQ, stuff, 12400);
   tt_int_op(buf_datalen(buf), OP_EQ, 3600);
-  fetch_from_buf(tmp, 3500, buf);
+  buf_get_bytes(buf, tmp, 3500);
   tt_mem_op(tmp,OP_EQ, stuff+12400, 3500);
-  fetch_from_buf(tmp, 100, buf);
+  buf_get_bytes(buf, tmp, 100);
   tt_mem_op(tmp,OP_EQ, stuff+15900, 10);
 
   buf_free(buf);
 
   /* Make sure that the pull-up-whole-buffer case works */
   buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */
-  write_to_buf(stuff, 4000, buf);
-  write_to_buf(stuff+4000, 4000, buf);
-  fetch_from_buf(tmp, 100, buf); /* dump 100 bytes from first chunk */
-  buf_pullup(buf, 16000); /* Way too much. */
-  assert_buf_ok(buf);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_add(buf, stuff, 4000);
+  buf_add(buf, stuff+4000, 4000);
+  buf_get_bytes(buf, tmp, 100); /* dump 100 bytes from first chunk */
+  buf_pullup(buf, 16000, &cp, &sz);
+  buf_assert_ok(buf);
   tt_ptr_op(cp, OP_NE, NULL);
   tt_int_op(sz, OP_EQ, 7900);
   tt_mem_op(cp,OP_EQ, stuff+100, 7900);
@@ -321,23 +324,23 @@ test_buffer_copy(void *arg)
   /* Now try with a short buffer. */
   s = "And now comes an act of enormous enormance!";
   len = strlen(s);
-  write_to_buf(s, len, buf);
+  buf_add(buf, s, len);
   tt_int_op(len, OP_EQ, buf_datalen(buf));
   /* Add junk to buf2 so we can test replacing.*/
-  write_to_buf("BLARG", 5, buf2);
+  buf_add(buf2, "BLARG", 5);
   tt_int_op(0, OP_EQ, buf_set_to_copy(&buf2, buf));
   tt_int_op(len, OP_EQ, buf_datalen(buf2));
-  fetch_from_buf(b, len, buf2);
+  buf_get_bytes(buf2, b, len);
   tt_mem_op(b, OP_EQ, s, len);
   /* Now free buf2 and retry so we can test allocating */
   buf_free(buf2);
   buf2 = NULL;
   tt_int_op(0, OP_EQ, buf_set_to_copy(&buf2, buf));
   tt_int_op(len, OP_EQ, buf_datalen(buf2));
-  fetch_from_buf(b, len, buf2);
+  buf_get_bytes(buf2, b, len);
   tt_mem_op(b, OP_EQ, s, len);
   /* Clear buf for next test */
-  fetch_from_buf(b, len, buf);
+  buf_get_bytes(buf, b, len);
   tt_int_op(buf_datalen(buf),OP_EQ,0);
 
   /* Okay, now let's try a bigger buffer. */
@@ -347,13 +350,13 @@ test_buffer_copy(void *arg)
   len = strlen(s);
   for (i = 0; i < 256; ++i) {
     b[0]=i;
-    write_to_buf(b, 1, buf);
-    write_to_buf(s, len, buf);
+    buf_add(buf, b, 1);
+    buf_add(buf, s, len);
   }
   tt_int_op(0, OP_EQ, buf_set_to_copy(&buf2, buf));
   tt_int_op(buf_datalen(buf2), OP_EQ, buf_datalen(buf));
   for (i = 0; i < 256; ++i) {
-    fetch_from_buf(b, len+1, buf2);
+    buf_get_bytes(buf2, b, len+1);
     tt_int_op((unsigned char)b[0],OP_EQ,i);
     tt_mem_op(b+1, OP_EQ, s, len);
   }
@@ -378,13 +381,13 @@ test_buffer_ext_or_cmd(void *arg)
   tt_ptr_op(NULL, OP_EQ, cmd);
 
   /* Three bytes: shouldn't work. */
-  write_to_buf("\x00\x20\x00", 3, buf);
+  buf_add(buf, "\x00\x20\x00", 3);
   tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
   tt_ptr_op(NULL, OP_EQ, cmd);
   tt_int_op(3, OP_EQ, buf_datalen(buf));
 
   /* 0020 0000: That's a nil command. It should work. */
-  write_to_buf("\x00", 1, buf);
+  buf_add(buf, "\x00", 1);
   tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
   tt_ptr_op(NULL, OP_NE, cmd);
   tt_int_op(0x20, OP_EQ, cmd->cmd);
@@ -394,10 +397,10 @@ test_buffer_ext_or_cmd(void *arg)
   cmd = NULL;
 
   /* Now try a length-6 command with one byte missing. */
-  write_to_buf("\x10\x21\x00\x06""abcde", 9, buf);
+  buf_add(buf, "\x10\x21\x00\x06""abcde", 9);
   tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
   tt_ptr_op(NULL, OP_EQ, cmd);
-  write_to_buf("f", 1, buf);
+  buf_add(buf, "f", 1);
   tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
   tt_ptr_op(NULL, OP_NE, cmd);
   tt_int_op(0x1021, OP_EQ, cmd->cmd);
@@ -408,7 +411,7 @@ test_buffer_ext_or_cmd(void *arg)
   cmd = NULL;
 
   /* Now try a length-10 command with 4 extra bytes. */
-  write_to_buf("\xff\xff\x00\x0aloremipsum\x10\x00\xff\xff", 18, buf);
+  buf_add(buf, "\xff\xff\x00\x0aloremipsum\x10\x00\xff\xff", 18);
   tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
   tt_ptr_op(NULL, OP_NE, cmd);
   tt_int_op(0xffff, OP_EQ, cmd->cmd);
@@ -422,7 +425,7 @@ test_buffer_ext_or_cmd(void *arg)
    * waiting. */
   tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
   tmp = tor_malloc_zero(65535);
-  write_to_buf(tmp, 65535, buf);
+  buf_add(buf, tmp, 65535);
   tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
   tt_ptr_op(NULL, OP_NE, cmd);
   tt_int_op(0x1000, OP_EQ, cmd->cmd);
@@ -458,36 +461,36 @@ test_buffer_allocation_tracking(void *arg)
   tt_int_op(buf_allocation(buf1), OP_EQ, 0);
   tt_int_op(buf_get_total_allocation(), OP_EQ, 0);
 
-  write_to_buf(junk, 4000, buf1);
-  write_to_buf(junk, 4000, buf1);
-  write_to_buf(junk, 4000, buf1);
-  write_to_buf(junk, 4000, buf1);
+  buf_add(buf1, junk, 4000);
+  buf_add(buf1, junk, 4000);
+  buf_add(buf1, junk, 4000);
+  buf_add(buf1, junk, 4000);
   tt_int_op(buf_allocation(buf1), OP_EQ, 16384);
-  fetch_from_buf(junk, 100, buf1);
+  buf_get_bytes(buf1, junk, 100);
   tt_int_op(buf_allocation(buf1), OP_EQ, 16384); /* still 4 4k chunks */
 
   tt_int_op(buf_get_total_allocation(), OP_EQ, 16384);
 
-  fetch_from_buf(junk, 4096, buf1); /* drop a 1k chunk... */
+  buf_get_bytes(buf1, junk, 4096); /* drop a 1k chunk... */
   tt_int_op(buf_allocation(buf1), OP_EQ, 3*4096); /* now 3 4k chunks */
 
   tt_int_op(buf_get_total_allocation(), OP_EQ, 12288); /* that chunk was really
                                                        freed. */
 
-  write_to_buf(junk, 4000, buf2);
+  buf_add(buf2, junk, 4000);
   tt_int_op(buf_allocation(buf2), OP_EQ, 4096); /* another 4k chunk. */
   /*
    * We bounce back up to 16384 by allocating a new chunk.
    */
   tt_int_op(buf_get_total_allocation(), OP_EQ, 16384);
-  write_to_buf(junk, 4000, buf2);
+  buf_add(buf2, junk, 4000);
   tt_int_op(buf_allocation(buf2), OP_EQ, 8192); /* another 4k chunk. */
   tt_int_op(buf_get_total_allocation(),
             OP_EQ, 5*4096); /* that chunk was new. */
 
   /* Make a really huge buffer */
   for (i = 0; i < 1000; ++i) {
-    write_to_buf(junk, 4000, buf2);
+    buf_add(buf2, junk, 4000);
   }
   tt_int_op(buf_allocation(buf2), OP_GE, 4008000);
   tt_int_op(buf_get_total_allocation(), OP_GE, 4008000);
@@ -530,7 +533,7 @@ test_buffer_time_tracking(void *arg)
   tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC));
   tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+1000));
 
-  write_to_buf("ABCDEFG", 7, buf);
+  buf_add(buf, "ABCDEFG", 7);
   tt_int_op(1000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+1000));
 
   buf2 = buf_copy(buf);
@@ -541,7 +544,7 @@ test_buffer_time_tracking(void *arg)
   /* Now add more bytes; enough to overflow the first chunk. */
   monotime_coarse_set_mock_time_nsec(START_NSEC + 123 * (uint64_t)1000000);
   for (i = 0; i < 600; ++i)
-    write_to_buf("ABCDEFG", 7, buf);
+    buf_add(buf, "ABCDEFG", 7);
   tt_int_op(4207, OP_EQ, buf_datalen(buf));
 
   /* The oldest bytes are still in the front. */
@@ -549,12 +552,12 @@ test_buffer_time_tracking(void *arg)
 
   /* Once those bytes are dropped, the chunk is still on the first
    * timestamp. */
-  fetch_from_buf(tmp, 100, buf);
+  buf_get_bytes(buf, tmp, 100);
   tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2000));
 
   /* But once we discard the whole first chunk, we get the data in the second
    * chunk. */
-  fetch_from_buf(tmp, 4000, buf);
+  buf_get_bytes(buf, tmp, 4000);
   tt_int_op(107, OP_EQ, buf_datalen(buf));
   tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2123));
 
@@ -562,12 +565,12 @@ test_buffer_time_tracking(void *arg)
      its time gets updated */
   monotime_coarse_set_mock_time_nsec(START_NSEC + 5617 * (uint64_t)1000000);
   for (i = 0; i < 600; ++i)
-    write_to_buf("ABCDEFG", 7, buf);
+    buf_add(buf, "ABCDEFG", 7);
   tt_int_op(4307, OP_EQ, buf_datalen(buf));
 
   tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2123));
-  fetch_from_buf(tmp, 4000, buf);
-  fetch_from_buf(tmp, 306, buf);
+  buf_get_bytes(buf, tmp, 4000);
+  buf_get_bytes(buf, tmp, 306);
   tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+5617));
   tt_int_op(383, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+6000));
 
@@ -593,23 +596,23 @@ test_buffers_compress_fin_at_chunk_end_impl(compress_method_t method,
   sz = buf_get_default_chunk_size(buf);
   msg = tor_malloc_zero(sz);
 
-  write_to_buf(msg, 1, buf);
+  buf_add(buf, msg, 1);
   tt_assert(buf->head);
 
   /* Fill up the chunk so the compression stuff won't fit in one chunk. */
   tt_uint_op(buf->head->memlen, OP_LT, sz);
   headerjunk = buf->head->memlen - 7;
-  write_to_buf(msg, headerjunk-1, buf);
+  buf_add(buf, msg, headerjunk-1);
   tt_uint_op(buf->head->datalen, OP_EQ, headerjunk);
   tt_uint_op(buf_datalen(buf), OP_EQ, headerjunk);
   /* Write an empty string, with finalization on. */
   compress_state = tor_compress_new(1, method, level);
-  tt_int_op(write_to_buf_compress(buf, compress_state, "", 0, 1), OP_EQ, 0);
+  tt_int_op(buf_add_compress(buf, compress_state, "", 0, 1), OP_EQ, 0);
 
   in_len = buf_datalen(buf);
   contents = tor_malloc(in_len);
 
-  tt_int_op(fetch_from_buf(contents, in_len, buf), OP_EQ, 0);
+  tt_int_op(buf_get_bytes(buf, contents, in_len), OP_EQ, 0);
 
   if (method == NO_METHOD) {
     tt_uint_op(in_len, OP_EQ, headerjunk);
@@ -652,23 +655,23 @@ test_buffers_compress_impl(compress_method_t method,
 
   msg = tor_malloc(512);
   crypto_rand(msg, 512);
-  tt_int_op(write_to_buf_compress(buf, compress_state,
+  tt_int_op(buf_add_compress(buf, compress_state,
                                   msg, 128, 0), OP_EQ, 0);
-  tt_int_op(write_to_buf_compress(buf, compress_state,
+  tt_int_op(buf_add_compress(buf, compress_state,
                                   msg+128, 128, 0), OP_EQ, 0);
-  tt_int_op(write_to_buf_compress(buf, compress_state,
+  tt_int_op(buf_add_compress(buf, compress_state,
                                   msg+256, 256, 0), OP_EQ, 0);
   done = !finalize_with_nil;
-  tt_int_op(write_to_buf_compress(buf, compress_state,
+  tt_int_op(buf_add_compress(buf, compress_state,
                                   "all done", 9, done), OP_EQ, 0);
   if (finalize_with_nil) {
-    tt_int_op(write_to_buf_compress(buf, compress_state, "", 0, 1), OP_EQ, 0);
+    tt_int_op(buf_add_compress(buf, compress_state, "", 0, 1), OP_EQ, 0);
   }
 
   in_len = buf_datalen(buf);
   contents = tor_malloc(in_len);
 
-  tt_int_op(fetch_from_buf(contents, in_len, buf), OP_EQ, 0);
+  tt_int_op(buf_get_bytes(buf, contents, in_len), OP_EQ, 0);
 
   tt_int_op(0, OP_EQ, tor_uncompress(&expanded, &out_len,
                                      contents, in_len,
@@ -762,11 +765,11 @@ test_buffers_tls_read_mocked(void *arg)
   buf = buf_new();
 
   next_reply_val[0] = 1024;
-  tt_int_op(128, OP_EQ, read_to_buf_tls(NULL, 128, buf));
+  tt_int_op(128, OP_EQ, buf_read_from_tls(buf, NULL, 128));
 
   next_reply_val[0] = 5000;
   next_reply_val[1] = 5000;
-  tt_int_op(6000, OP_EQ, read_to_buf_tls(NULL, 6000, buf));
+  tt_int_op(6000, OP_EQ, buf_read_from_tls(buf, NULL, 6000));
 
  done:
   UNMOCK(tor_tls_read);
@@ -846,19 +849,19 @@ test_buffer_peek_startswith(void *arg)
   buf = buf_new();
   tt_ptr_op(buf, OP_NE, NULL);
 
-  tt_assert(peek_buf_startswith(buf, ""));
-  tt_assert(! peek_buf_startswith(buf, "X"));
+  tt_assert(buf_peek_startswith(buf, ""));
+  tt_assert(! buf_peek_startswith(buf, "X"));
 
-  write_to_buf("Tor", 3, buf);
+  buf_add(buf, "Tor", 3);
 
-  tt_assert(peek_buf_startswith(buf, ""));
-  tt_assert(peek_buf_startswith(buf, "T"));
-  tt_assert(peek_buf_startswith(buf, "To"));
-  tt_assert(peek_buf_startswith(buf, "Tor"));
-  tt_assert(! peek_buf_startswith(buf, "Top"));
-  tt_assert(! peek_buf_startswith(buf, "For"));
-  tt_assert(! peek_buf_startswith(buf, "Tork"));
-  tt_assert(! peek_buf_startswith(buf, "Torpor"));
+  tt_assert(buf_peek_startswith(buf, ""));
+  tt_assert(buf_peek_startswith(buf, "T"));
+  tt_assert(buf_peek_startswith(buf, "To"));
+  tt_assert(buf_peek_startswith(buf, "Tor"));
+  tt_assert(! buf_peek_startswith(buf, "Top"));
+  tt_assert(! buf_peek_startswith(buf, "For"));
+  tt_assert(! buf_peek_startswith(buf, "Tork"));
+  tt_assert(! buf_peek_startswith(buf, "Torpor"));
 
  done:
   buf_free(buf);

src/test/test_dir_handle_get.c

@@ -28,6 +28,7 @@
 #include "entrynodes.h"
 #include "routerparse.h"
 #include "networkstatus.h"
+#include "proto_http.h"
 #include "geoip.h"
 #include "dirserv.h"
 #include "dirvote.h"

src/test/test_extorport.c

@@ -78,7 +78,7 @@ connection_write_to_buf_impl_replacement(const char *string, size_t len,
 
   tor_assert(string);
   tor_assert(conn);
-  write_to_buf(string, len, conn->outbuf);
+  buf_add(conn->outbuf, string, len);
 }
 
 static char *
@@ -89,7 +89,7 @@ buf_get_contents(buf_t *buf, size_t *sz_out)
   if (*sz_out >= ULONG_MAX)
     return NULL; /* C'mon, really? */
   out = tor_malloc(*sz_out + 1);
-  if (fetch_from_buf(out, (unsigned long)*sz_out, buf) != 0) {
+  if (buf_get_bytes(buf, out, (unsigned long)*sz_out) != 0) {
     tor_free(out);
     return NULL;
   }
@@ -399,14 +399,14 @@ handshake_start(or_connection_t *conn, int receiving)
 
 #define WRITE(s,n)                                                      \
   do {                                                                  \
-    write_to_buf((s), (n), TO_CONN(conn)->inbuf);                       \
+    buf_add(TO_CONN(conn)->inbuf, (s), (n));                           \
   } while (0)
 #define CONTAINS(s,n)                                           \
   do {                                                          \
     tt_int_op((n), OP_LE, sizeof(b));                              \
     tt_int_op(buf_datalen(TO_CONN(conn)->outbuf), OP_EQ, (n));     \
     if ((n)) {                                                  \
-      fetch_from_buf(b, (n), TO_CONN(conn)->outbuf);            \
+      buf_get_bytes(TO_CONN(conn)->outbuf, b, (n));                \
       tt_mem_op(b, OP_EQ, (s), (n));                               \
     }                                                           \
   } while (0)

src/test/test_helpers.c

@@ -114,7 +114,7 @@ connection_write_to_buf_mock(const char *string, size_t len,
   tor_assert(string);
   tor_assert(conn);
 
-  write_to_buf(string, len, conn->outbuf);
+  buf_add(conn->outbuf, string, len);
 }
 
 /* Set up a fake origin circuit with the specified number of cells,

src/test/test_hs_cache.c

@@ -15,6 +15,7 @@
 #include "rendcache.h"
 #include "directory.h"
 #include "connection.h"
+#include "proto_http.h"
 
 #include "hs_test_helpers.h"
 #include "test_helpers.h"

src/test/test_oom.c

@@ -67,7 +67,7 @@ add_bytes_to_buf(buf_t *buf, size_t n_bytes)
   while (n_bytes) {
     size_t this_add = n_bytes > sizeof(b) ? sizeof(b) : n_bytes;
     crypto_rand(b, this_add);
-    write_to_buf(b, this_add, buf);
+    buf_add(buf, b, this_add);
     n_bytes -= this_add;
   }
 }

src/test/test_socks.c

@@ -6,6 +6,7 @@
 #include "or.h"
 #include "buffers.h"
 #include "config.h"
+#include "proto_socks.h"
 #include "test.h"
 
 typedef struct socks_test_data_t {
@@ -43,7 +44,7 @@ static const struct testcase_setup_t socks_setup = {
   buf_t *buf = testdata->buf;                   \
   socks_request_t *socks = testdata->req;
 #define ADD_DATA(buf, s)                                        \
-  write_to_buf(s, sizeof(s)-1, buf)
+  buf_add(buf, s, sizeof(s)-1)
 
 static void
 socks_request_clear(socks_request_t *socks)