123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584 |
- /* Copyright (c) 2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2017, The Tor Project, Inc. */
- /* See LICENSE for licensing information */
- /**
- * \file compress_zstd.c
- * \brief Compression backend for Zstandard.
- *
- * This module should never be invoked directly. Use the compress module
- * instead.
- **/
- #include "orconfig.h"
- #include "util.h"
- #include "torlog.h"
- #include "compress.h"
- #include "compress_zstd.h"
- #ifdef HAVE_ZSTD
- #include <zstd.h>
- #include <zstd_errors.h>
- #endif
- /** Total number of bytes allocated for Zstandard state. */
- static size_t total_zstd_allocation = 0;
- /** Return a string representation of the version of the currently running
- * version of libzstd. */
- const char *
- tor_zstd_get_version_str(void)
- {
- #ifdef HAVE_ZSTD
- static char version_str[16];
- size_t version_number;
- version_number = ZSTD_versionNumber();
- tor_snprintf(version_str, sizeof(version_str),
- "%lu.%lu.%lu",
- version_number / 10000 % 100,
- version_number / 100 % 100,
- version_number % 100);
- return version_str;
- #else
- return "N/A";
- #endif
- }
- /** Return a string representation of the version of the version of libzstd
- * used at compilation. */
- const char *
- tor_zstd_get_header_version_str(void)
- {
- #ifdef HAVE_ZSTD
- return ZSTD_VERSION_STRING;
- #else
- return "N/A";
- #endif
- }
- /** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly
- * allocated buffer, using the Zstandard method. Store the compressed string
- * in *<b>out</b>, and its length in *<b>out_len</b>. Return 0 on success, -1
- * on failure.
- */
- int
- tor_zstd_compress(char **out, size_t *out_len,
- const char *in, size_t in_len,
- compress_method_t method)
- {
- #ifdef HAVE_ZSTD
- ZSTD_CStream *stream = NULL;
- size_t out_size, old_size;
- size_t retval;
- tor_assert(out);
- tor_assert(out_len);
- tor_assert(in);
- tor_assert(in_len < UINT_MAX);
- tor_assert(method == ZSTD_METHOD);
- *out = NULL;
- stream = ZSTD_createCStream();
- if (stream == NULL) {
- // Zstandard does not give us any useful error message to why this
- // happened. See https://github.com/facebook/zstd/issues/398
- log_warn(LD_GENERAL, "Error while creating Zstandard stream");
- goto err;
- }
- retval = ZSTD_initCStream(stream,
- tor_compress_memory_level(HIGH_COMPRESSION));
- if (ZSTD_isError(retval)) {
- log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",
- ZSTD_getErrorName(retval));
- goto err;
- }
- // Assume 50% compression and update our buffer in case we need to.
- out_size = in_len / 2;
- if (out_size < 1024)
- out_size = 1024;
- *out = tor_malloc(out_size);
- *out_len = 0;
- ZSTD_inBuffer input = { in, in_len, 0 };
- ZSTD_outBuffer output = { *out, out_size, 0 };
- while (input.pos < input.size) {
- retval = ZSTD_compressStream(stream, &output, &input);
- if (ZSTD_isError(retval)) {
- log_warn(LD_GENERAL, "Zstandard stream compression error: %s",
- ZSTD_getErrorName(retval));
- goto err;
- }
- if (input.pos < input.size && output.pos == output.size) {
- old_size = out_size;
- out_size *= 2;
- if (out_size < old_size) {
- log_warn(LD_GENERAL, "Size overflow in Zstandard compression.");
- goto err;
- }
- if (out_size - output.pos > UINT_MAX) {
- log_warn(LD_BUG, "Ran over unsigned int limit of Zstandard while "
- "compressing.");
- goto err;
- }
- output.dst = *out = tor_realloc(*out, out_size);
- output.size = out_size;
- }
- }
- while (1) {
- retval = ZSTD_endStream(stream, &output);
- if (retval == 0)
- break;
- if (ZSTD_isError(retval)) {
- log_warn(LD_GENERAL, "Zstandard stream error: %s",
- ZSTD_getErrorName(retval));
- goto err;
- }
- if (output.pos == output.size) {
- old_size = out_size;
- out_size *= 2;
- if (out_size < old_size) {
- log_warn(LD_GENERAL, "Size overflow in Zstandard compression.");
- goto err;
- }
- if (out_size - output.pos > UINT_MAX) {
- log_warn(LD_BUG, "Ran over unsigned int limit of Zstandard while "
- "compressing.");
- goto err;
- }
- output.dst = *out = tor_realloc(*out, out_size);
- output.size = out_size;
- }
- }
- *out_len = output.pos;
- if (tor_compress_is_compression_bomb(*out_len, in_len)) {
- log_warn(LD_BUG, "We compressed something and got an insanely high "
- "compression factor; other Tor instances would think "
- "this is a compression bomb.");
- goto err;
- }
- if (stream != NULL) {
- ZSTD_freeCStream(stream);
- }
- return 0;
- err:
- if (stream != NULL) {
- ZSTD_freeCStream(stream);
- }
- tor_free(*out);
- return -1;
- #else // HAVE_ZSTD.
- (void)out;
- (void)out_len;
- (void)in;
- (void)in_len;
- (void)method;
- return -1;
- #endif // HAVE_ZSTD.
- }
- /** Given a Zstandard compressed string of total length <b>in_len</b> bytes at
- * <b>in</b>, uncompress them into a newly allocated buffer. Store the
- * uncompressed string in *<b>out</b>, and its length in *<b>out_len</b>.
- * Return 0 on success, -1 on failure.
- *
- * If <b>complete_only</b> is true, we consider a truncated input as a failure;
- * otherwise we decompress as much as we can. Warn about truncated or corrupt
- * inputs at <b>protocol_warn_level</b>.
- */
- int
- tor_zstd_uncompress(char **out, size_t *out_len,
- const char *in, size_t in_len,
- compress_method_t method,
- int complete_only,
- int protocol_warn_level)
- {
- #ifdef HAVE_ZSTD
- ZSTD_DStream *stream = NULL;
- size_t retval;
- size_t out_size, old_size;
- tor_assert(out);
- tor_assert(out_len);
- tor_assert(in);
- tor_assert(in_len < UINT_MAX);
- tor_assert(method == ZSTD_METHOD);
- // FIXME(ahf): Handle this?
- (void)complete_only;
- (void)protocol_warn_level;
- *out = NULL;
- stream = ZSTD_createDStream();
- if (stream == NULL) {
- // Zstandard does not give us any useful error message to why this
- // happened. See https://github.com/facebook/zstd/issues/398
- log_warn(LD_GENERAL, "Error while creating Zstandard stream");
- goto err;
- }
- retval = ZSTD_initDStream(stream);
- if (ZSTD_isError(retval)) {
- log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",
- ZSTD_getErrorName(retval));
- goto err;
- }
- out_size = in_len * 2;
- if (out_size < 1024)
- out_size = 1024;
- if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX)
- goto err;
- *out = tor_malloc(out_size);
- *out_len = 0;
- ZSTD_inBuffer input = { in, in_len, 0 };
- ZSTD_outBuffer output = { *out, out_size, 0 };
- while (input.pos < input.size) {
- retval = ZSTD_decompressStream(stream, &output, &input);
- if (ZSTD_isError(retval)) {
- log_warn(LD_GENERAL, "Zstandard stream decompression error: %s",
- ZSTD_getErrorName(retval));
- goto err;
- }
- if (input.pos < input.size && output.pos == output.size) {
- old_size = out_size;
- out_size *= 2;
- if (out_size < old_size) {
- log_warn(LD_GENERAL, "Size overflow in Zstandard compression.");
- goto err;
- }
- if (tor_compress_is_compression_bomb(in_len, out_size)) {
- log_warn(LD_GENERAL, "Input looks like a possible Zstandard "
- "compression bomb. Not proceeding.");
- goto err;
- }
- if (out_size >= SIZE_T_CEILING) {
- log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing "
- "Zstandard data.");
- goto err;
- }
- if (out_size - output.pos > UINT_MAX) {
- log_warn(LD_BUG, "Ran over unsigned int limit of Zstandard while "
- "decompressing.");
- goto err;
- }
- output.dst = *out = tor_realloc(*out, out_size);
- output.size = out_size;
- }
- }
- *out_len = output.pos;
- if (stream != NULL) {
- ZSTD_freeDStream(stream);
- }
- // NUL-terminate our output.
- if (out_size == *out_len)
- *out = tor_realloc(*out, out_size + 1);
- (*out)[*out_len] = '\0';
- return 0;
- err:
- if (stream != NULL) {
- ZSTD_freeDStream(stream);
- }
- tor_free(*out);
- return -1;
- #else // HAVE_ZSTD.
- (void)out;
- (void)out_len;
- (void)in;
- (void)in_len;
- (void)method;
- (void)complete_only;
- (void)protocol_warn_level;
- return -1;
- #endif // HAVE_ZSTD.
- }
- /** Internal Zstandard state for incremental compression/decompression.
- * The body of this struct is not exposed. */
- struct tor_zstd_compress_state_t {
- #ifdef HAVE_ZSTD
- union {
- /** Compression stream. Used when <b>compress</b> is true. */
- ZSTD_CStream *compress_stream;
- /** Decompression stream. Used when <b>compress</b> is false. */
- ZSTD_DStream *decompress_stream;
- } u; /**< Zstandard stream objects. */
- #endif // HAVE_ZSTD.
- int compress; /**< True if we are compressing; false if we are inflating */
- /** Number of bytes read so far. Used to detect compression bombs. */
- size_t input_so_far;
- /** Number of bytes written so far. Used to detect compression bombs. */
- size_t output_so_far;
- /** Approximate number of bytes allocated for this object. */
- size_t allocation;
- };
- /** Construct and return a tor_zstd_compress_state_t object using
- * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
- * decompression. */
- tor_zstd_compress_state_t *
- tor_zstd_compress_new(int compress,
- compress_method_t method,
- compression_level_t compression_level)
- {
- tor_assert(method == ZSTD_METHOD);
- #ifdef HAVE_ZSTD
- tor_zstd_compress_state_t *result;
- size_t retval;
- result = tor_malloc_zero(sizeof(tor_zstd_compress_state_t));
- result->compress = compress;
- // FIXME(ahf): We should either try to do the pre-calculation that is done
- // with the zlib backend or use a custom allocator here where we pass our
- // tor_zstd_compress_state_t as the opaque value.
- result->allocation = 0;
- if (compress) {
- result->u.compress_stream = ZSTD_createCStream();
- if (result->u.compress_stream == NULL) {
- log_warn(LD_GENERAL, "Error while creating Zstandard stream");
- goto err;
- }
- retval = ZSTD_initCStream(result->u.compress_stream,
- tor_compress_memory_level(compression_level));
- if (ZSTD_isError(retval)) {
- log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",
- ZSTD_getErrorName(retval));
- goto err;
- }
- } else {
- result->u.decompress_stream = ZSTD_createDStream();
- if (result->u.decompress_stream == NULL) {
- log_warn(LD_GENERAL, "Error while creating Zstandard stream");
- goto err;
- }
- retval = ZSTD_initDStream(result->u.decompress_stream);
- if (ZSTD_isError(retval)) {
- log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",
- ZSTD_getErrorName(retval));
- goto err;
- }
- }
- return result;
- err:
- if (compress) {
- ZSTD_freeCStream(result->u.compress_stream);
- } else {
- ZSTD_freeDStream(result->u.decompress_stream);
- }
- tor_free(result);
- return NULL;
- #else // HAVE_ZSTD.
- (void)compress;
- (void)method;
- (void)compression_level;
- return NULL;
- #endif // HAVE_ZSTD.
- }
- /** Compress/decompress some bytes using <b>state</b>. Read up to
- * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
- * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true,
- * we've reached the end of the input.
- *
- * Return TOR_COMPRESS_DONE if we've finished the entire
- * compression/decompression.
- * Return TOR_COMPRESS_OK if we're processed everything from the input.
- * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
- * Return TOR_COMPRESS_ERROR if the stream is corrupt.
- */
- tor_compress_output_t
- tor_zstd_compress_process(tor_zstd_compress_state_t *state,
- char **out, size_t *out_len,
- const char **in, size_t *in_len,
- int finish)
- {
- #ifdef HAVE_ZSTD
- size_t retval;
- tor_assert(state != NULL);
- tor_assert(*in_len <= UINT_MAX);
- tor_assert(*out_len <= UINT_MAX);
- ZSTD_inBuffer input = { *in, *in_len, 0 };
- ZSTD_outBuffer output = { *out, *out_len, 0 };
- if (state->compress) {
- retval = ZSTD_compressStream(state->u.compress_stream,
- &output, &input);
- } else {
- retval = ZSTD_decompressStream(state->u.decompress_stream,
- &output, &input);
- }
- state->input_so_far += input.pos;
- state->output_so_far += output.pos;
- *out = (char *)output.dst + output.pos;
- *out_len = output.size - output.pos;
- *in = (char *)input.src + input.pos;
- *in_len = input.size - input.pos;
- if (! state->compress &&
- tor_compress_is_compression_bomb(state->input_so_far,
- state->output_so_far)) {
- log_warn(LD_DIR, "Possible compression bomb; abandoning stream.");
- return TOR_COMPRESS_ERROR;
- }
- if (ZSTD_isError(retval)) {
- log_warn(LD_GENERAL, "Zstandard %s didn't finish: %s.",
- state->compress ? "compression" : "decompression",
- ZSTD_getErrorName(retval));
- return TOR_COMPRESS_ERROR;
- }
- if (state->compress && !finish) {
- retval = ZSTD_flushStream(state->u.compress_stream, &output);
- *out = (char *)output.dst + output.pos;
- *out_len = output.size - output.pos;
- if (ZSTD_isError(retval)) {
- log_warn(LD_GENERAL, "Zstandard compression unable to flush: %s.",
- ZSTD_getErrorName(retval));
- return TOR_COMPRESS_ERROR;
- }
- if (retval > 0)
- return TOR_COMPRESS_BUFFER_FULL;
- }
- if (state->compress && finish) {
- retval = ZSTD_endStream(state->u.compress_stream, &output);
- *out = (char *)output.dst + output.pos;
- *out_len = output.size - output.pos;
- if (ZSTD_isError(retval)) {
- log_warn(LD_GENERAL, "Zstandard compression unable to write "
- "epilogue: %s.",
- ZSTD_getErrorName(retval));
- return TOR_COMPRESS_ERROR;
- }
- // endStream returns the number of bytes that is needed to write the
- // epilogue.
- if (retval > 0)
- return TOR_COMPRESS_BUFFER_FULL;
- }
- return finish ? TOR_COMPRESS_DONE : TOR_COMPRESS_OK;
- #else // HAVE_ZSTD.
- (void)state;
- (void)out;
- (void)out_len;
- (void)in;
- (void)in_len;
- (void)finish;
- return TOR_COMPRESS_ERROR;
- #endif // HAVE_ZSTD.
- }
- /** Deallocate <b>state</b>. */
- void
- tor_zstd_compress_free(tor_zstd_compress_state_t *state)
- {
- if (state == NULL)
- return;
- total_zstd_allocation -= state->allocation;
- #ifdef HAVE_ZSTD
- if (state->compress) {
- ZSTD_freeCStream(state->u.compress_stream);
- } else {
- ZSTD_freeDStream(state->u.decompress_stream);
- }
- #endif // HAVE_ZSTD.
- tor_free(state);
- }
- /** Return the approximate number of bytes allocated for <b>state</b>. */
- size_t
- tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state)
- {
- tor_assert(state != NULL);
- return state->allocation;
- }
- /** Return the approximate number of bytes allocated for all Zstandard
- * states. */
- size_t
- tor_zstd_get_total_allocation(void)
- {
- return total_zstd_allocation;
- }
|