compress_zlib.c 8.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304
  1. /* Copyright (c) 2004, Roger Dingledine.
  2. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
  3. * Copyright (c) 2007-2018, The Tor Project, Inc. */
  4. /* See LICENSE for licensing information */
  5. /**
  6. * \file compress_zlib.c
  7. * \brief Compression backend for gzip and zlib.
  8. *
  9. * This module should never be invoked directly. Use the compress module
  10. * instead.
  11. **/
  12. #include "orconfig.h"
  13. #include "lib/log/torlog.h"
  14. #include "lib/log/util_bug.h"
  15. #include "lib/compress/compress.h"
  16. #include "lib/compress/compress_zlib.h"
  17. #include "lib/thread/threads.h"
  18. /* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of
  19. saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory
  20. that nobody will care if the compile outputs a no-such-identifier warning.
  21. Sorry, but we like -Werror over here, so I guess we need to define these.
  22. I hope that zlib 1.2.6 doesn't break these too.
  23. */
  24. #ifndef _LARGEFILE64_SOURCE
  25. #define _LARGEFILE64_SOURCE 0
  26. #endif
  27. #ifndef _LFS64_LARGEFILE
  28. #define _LFS64_LARGEFILE 0
  29. #endif
  30. #ifndef _FILE_OFFSET_BITS
  31. #define _FILE_OFFSET_BITS 0
  32. #endif
  33. #ifndef off64_t
  34. #define off64_t int64_t
  35. #endif
  36. #include <zlib.h>
  37. #if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200
  38. #error "We require zlib version 1.2 or later."
  39. #endif
  40. static size_t tor_zlib_state_size_precalc(int inflate,
  41. int windowbits, int memlevel);
  42. /** Total number of bytes allocated for zlib state */
  43. static atomic_counter_t total_zlib_allocation;
  44. /** Given <b>level</b> return the memory level. */
  45. static int
  46. memory_level(compression_level_t level)
  47. {
  48. switch (level) {
  49. default:
  50. case BEST_COMPRESSION: return 9;
  51. case HIGH_COMPRESSION: return 8;
  52. case MEDIUM_COMPRESSION: return 7;
  53. case LOW_COMPRESSION: return 6;
  54. }
  55. }
  56. /** Return the 'bits' value to tell zlib to use <b>method</b>.*/
  57. static inline int
  58. method_bits(compress_method_t method, compression_level_t level)
  59. {
  60. /* Bits+16 means "use gzip" in zlib >= 1.2 */
  61. const int flag = method == GZIP_METHOD ? 16 : 0;
  62. switch (level) {
  63. default:
  64. case BEST_COMPRESSION:
  65. case HIGH_COMPRESSION: return flag + 15;
  66. case MEDIUM_COMPRESSION: return flag + 13;
  67. case LOW_COMPRESSION: return flag + 11;
  68. }
  69. }
  70. /** Return 1 if zlib/gzip compression is supported; otherwise 0. */
  71. int
  72. tor_zlib_method_supported(void)
  73. {
  74. /* We currently always support zlib/gzip, but we keep this function around in
  75. * case we some day decide to deprecate zlib/gzip support.
  76. */
  77. return 1;
  78. }
  79. /** Return a string representation of the version of the currently running
  80. * version of zlib. */
  81. const char *
  82. tor_zlib_get_version_str(void)
  83. {
  84. return zlibVersion();
  85. }
  86. /** Return a string representation of the version of the version of zlib
  87. * used at compilation. */
  88. const char *
  89. tor_zlib_get_header_version_str(void)
  90. {
  91. return ZLIB_VERSION;
  92. }
  93. /** Internal zlib state for an incremental compression/decompression.
  94. * The body of this struct is not exposed. */
  95. struct tor_zlib_compress_state_t {
  96. struct z_stream_s stream; /**< The zlib stream */
  97. int compress; /**< True if we are compressing; false if we are inflating */
  98. /** Number of bytes read so far. Used to detect zlib bombs. */
  99. size_t input_so_far;
  100. /** Number of bytes written so far. Used to detect zlib bombs. */
  101. size_t output_so_far;
  102. /** Approximate number of bytes allocated for this object. */
  103. size_t allocation;
  104. };
  105. /** Return an approximate number of bytes used in RAM to hold a state with
  106. * window bits <b>windowBits</b> and compression level 'memlevel' */
  107. static size_t
  108. tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel)
  109. {
  110. windowbits &= 15;
  111. #define A_FEW_KILOBYTES 2048
  112. if (inflate_) {
  113. /* From zconf.h:
  114. "The memory requirements for inflate are (in bytes) 1 << windowBits
  115. that is, 32K for windowBits=15 (default value) plus a few kilobytes
  116. for small objects."
  117. */
  118. return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) +
  119. (1 << 15) + A_FEW_KILOBYTES;
  120. } else {
  121. /* Also from zconf.h:
  122. "The memory requirements for deflate are (in bytes):
  123. (1 << (windowBits+2)) + (1 << (memLevel+9))
  124. ... plus a few kilobytes for small objects."
  125. */
  126. return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) +
  127. (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES;
  128. }
  129. #undef A_FEW_KILOBYTES
  130. }
  131. /** Construct and return a tor_zlib_compress_state_t object using
  132. * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
  133. * decompression. */
  134. tor_zlib_compress_state_t *
  135. tor_zlib_compress_new(int compress_,
  136. compress_method_t method,
  137. compression_level_t compression_level)
  138. {
  139. tor_zlib_compress_state_t *out;
  140. int bits, memlevel;
  141. if (! compress_) {
  142. /* use this setting for decompression, since we might have the
  143. * max number of window bits */
  144. compression_level = BEST_COMPRESSION;
  145. }
  146. out = tor_malloc_zero(sizeof(tor_zlib_compress_state_t));
  147. out->stream.zalloc = Z_NULL;
  148. out->stream.zfree = Z_NULL;
  149. out->stream.opaque = NULL;
  150. out->compress = compress_;
  151. bits = method_bits(method, compression_level);
  152. memlevel = memory_level(compression_level);
  153. if (compress_) {
  154. if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
  155. bits, memlevel,
  156. Z_DEFAULT_STRATEGY) != Z_OK)
  157. goto err; // LCOV_EXCL_LINE
  158. } else {
  159. if (inflateInit2(&out->stream, bits) != Z_OK)
  160. goto err; // LCOV_EXCL_LINE
  161. }
  162. out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel);
  163. atomic_counter_add(&total_zlib_allocation, out->allocation);
  164. return out;
  165. err:
  166. tor_free(out);
  167. return NULL;
  168. }
  169. /** Compress/decompress some bytes using <b>state</b>. Read up to
  170. * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
  171. * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true,
  172. * we've reached the end of the input.
  173. *
  174. * Return TOR_COMPRESS_DONE if we've finished the entire
  175. * compression/decompression.
  176. * Return TOR_COMPRESS_OK if we're processed everything from the input.
  177. * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
  178. * Return TOR_COMPRESS_ERROR if the stream is corrupt.
  179. */
  180. tor_compress_output_t
  181. tor_zlib_compress_process(tor_zlib_compress_state_t *state,
  182. char **out, size_t *out_len,
  183. const char **in, size_t *in_len,
  184. int finish)
  185. {
  186. int err;
  187. tor_assert(state != NULL);
  188. if (*in_len > UINT_MAX ||
  189. *out_len > UINT_MAX) {
  190. return TOR_COMPRESS_ERROR;
  191. }
  192. state->stream.next_in = (unsigned char*) *in;
  193. state->stream.avail_in = (unsigned int)*in_len;
  194. state->stream.next_out = (unsigned char*) *out;
  195. state->stream.avail_out = (unsigned int)*out_len;
  196. if (state->compress) {
  197. err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH);
  198. } else {
  199. err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
  200. }
  201. state->input_so_far += state->stream.next_in - ((unsigned char*)*in);
  202. state->output_so_far += state->stream.next_out - ((unsigned char*)*out);
  203. *out = (char*) state->stream.next_out;
  204. *out_len = state->stream.avail_out;
  205. *in = (const char *) state->stream.next_in;
  206. *in_len = state->stream.avail_in;
  207. if (! state->compress &&
  208. tor_compress_is_compression_bomb(state->input_so_far,
  209. state->output_so_far)) {
  210. log_warn(LD_DIR, "Possible zlib bomb; abandoning stream.");
  211. return TOR_COMPRESS_ERROR;
  212. }
  213. switch (err)
  214. {
  215. case Z_STREAM_END:
  216. return TOR_COMPRESS_DONE;
  217. case Z_BUF_ERROR:
  218. if (state->stream.avail_in == 0 && !finish)
  219. return TOR_COMPRESS_OK;
  220. return TOR_COMPRESS_BUFFER_FULL;
  221. case Z_OK:
  222. if (state->stream.avail_out == 0 || finish)
  223. return TOR_COMPRESS_BUFFER_FULL;
  224. return TOR_COMPRESS_OK;
  225. default:
  226. log_warn(LD_GENERAL, "Gzip returned an error: %s",
  227. state->stream.msg ? state->stream.msg : "<no message>");
  228. return TOR_COMPRESS_ERROR;
  229. }
  230. }
  231. /** Deallocate <b>state</b>. */
  232. void
  233. tor_zlib_compress_free_(tor_zlib_compress_state_t *state)
  234. {
  235. if (state == NULL)
  236. return;
  237. atomic_counter_sub(&total_zlib_allocation, state->allocation);
  238. if (state->compress)
  239. deflateEnd(&state->stream);
  240. else
  241. inflateEnd(&state->stream);
  242. tor_free(state);
  243. }
  244. /** Return the approximate number of bytes allocated for <b>state</b>. */
  245. size_t
  246. tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state)
  247. {
  248. tor_assert(state != NULL);
  249. return state->allocation;
  250. }
  251. /** Return the approximate number of bytes allocated for all zlib states. */
  252. size_t
  253. tor_zlib_get_total_allocation(void)
  254. {
  255. return atomic_counter_get(&total_zlib_allocation);
  256. }
  257. /** Set up global state for the zlib module */
  258. void
  259. tor_zlib_init(void)
  260. {
  261. atomic_counter_init(&total_zlib_allocation);
  262. }