avoid integer overflow in and around smartlist_ensure_capacity.

This closes bug 18162; bugfix on a45b1315909c9, which fixed a related
issue long ago.

In addition to the #18162 issues, this fixes a signed integer overflow
in smarltist_add_all(), which is probably not so great either.

changes/bug18162

@@ -0,0 +1,7 @@
+  o Major bugfixes (security, pointers):
+
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug #18162;
+      bugfix on Tor 0.1.1.11-alpha, which fixed a related bug
+      incompletely. Reported by Guido Vranken.
+

src/common/container.c

@@ -60,15 +60,17 @@ smartlist_clear(smartlist_t *sl)
 
 /** Make sure that <b>sl</b> can hold at least <b>size</b> entries. */
 static INLINE void
-smartlist_ensure_capacity(smartlist_t *sl, int size)
+smartlist_ensure_capacity(smartlist_t *sl, size_t size)
 {
 #if SIZEOF_SIZE_T > SIZEOF_INT
 #define MAX_CAPACITY (INT_MAX)
 #else
 #define MAX_CAPACITY (int)((SIZE_MAX / (sizeof(void*))))
 #endif
-  if (size > sl->capacity) {
-    int higher = sl->capacity;
+  tor_assert(size <= MAX_CAPACITY);
+
+  if (size > (size_t) sl->capacity) {
+    size_t higher = (size_t) sl->capacity;
     if (PREDICT_UNLIKELY(size > MAX_CAPACITY/2)) {
       tor_assert(size <= MAX_CAPACITY);
       higher = MAX_CAPACITY;
@@ -76,7 +78,8 @@ smartlist_ensure_capacity(smartlist_t *sl, int size)
       while (size > higher)
         higher *= 2;
     }
-    sl->capacity = higher;
+    tor_assert(higher <= INT_MAX); /* Redundant */
+    sl->capacity = (int) higher;
     sl->list = tor_realloc(sl->list, sizeof(void*)*((size_t)sl->capacity));
   }
 }
@@ -85,7 +88,7 @@ smartlist_ensure_capacity(smartlist_t *sl, int size)
 void
 smartlist_add(smartlist_t *sl, void *element)
 {
-  smartlist_ensure_capacity(sl, sl->num_used+1);
+  smartlist_ensure_capacity(sl, ((size_t) sl->num_used)+1);
   sl->list[sl->num_used++] = element;
 }
 
@@ -93,11 +96,12 @@ smartlist_add(smartlist_t *sl, void *element)
 void
 smartlist_add_all(smartlist_t *s1, const smartlist_t *s2)
 {
-  int new_size = s1->num_used + s2->num_used;
-  tor_assert(new_size >= s1->num_used); /* check for overflow. */
+  size_t new_size = (size_t)s1->num_used + (size_t)s2->num_used;
+  tor_assert(new_size >= (size_t) s1->num_used); /* check for overflow. */
   smartlist_ensure_capacity(s1, new_size);
   memcpy(s1->list + s1->num_used, s2->list, s2->num_used*sizeof(void*));
-  s1->num_used = new_size;
+  tor_assert(new_size <= INT_MAX); /* redundant. */
+  s1->num_used = (int) new_size;
 }
 
 /** Remove all elements E from sl such that E==element.  Preserve
@@ -334,7 +338,7 @@ smartlist_insert(smartlist_t *sl, int idx, void *val)
   if (idx == sl->num_used) {
     smartlist_add(sl, val);
   } else {
-    smartlist_ensure_capacity(sl, sl->num_used+1);
+    smartlist_ensure_capacity(sl, ((size_t) sl->num_used)+1);
     /* Move other elements away */
     if (idx < sl->num_used)
       memmove(sl->list + idx + 1, sl->list + idx,