tor/src/feature/hibernate/hibernate.c

/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
 * Copyright (c) 2007-2018, The Tor Project, Inc. */
/* See LICENSE for licensing information */

/**
 * \file hibernate.c
 * \brief Functions to close listeners, stop allowing new circuits,
 * etc in preparation for closing down or going dormant; and to track
 * bandwidth and time intervals to know when to hibernate and when to
 * stop hibernating.
 *
 * Ordinarily a Tor relay is "Live".
 *
 * A live relay can stop accepting connections for one of two reasons: either
 * it is trying to conserve bandwidth because of bandwidth accounting rules
 * ("soft hibernation"), or it is about to shut down ("exiting").
 **/

/*
hibernating, phase 1:
  - send destroy in response to create cells
  - send end (policy failed) in response to begin cells
  - close an OR conn when it has no circuits

hibernating, phase 2:
  (entered when bandwidth hard limit reached)
  - close all OR/AP/exit conns)
*/

#define HIBERNATE_PRIVATE
#include "core/or/or.h"
#include "core/or/channel.h"
#include "core/or/channeltls.h"
#include "app/config/config.h"
#include "core/mainloop/connection.h"
#include "core/or/connection_edge.h"
#include "core/or/connection_or.h"
#include "feature/control/control.h"
#include "lib/crypt_ops/crypto_rand.h"
#include "feature/hibernate/hibernate.h"
#include "core/mainloop/mainloop.h"
#include "feature/relay/router.h"
#include "app/config/statefile.h"
#include "lib/evloop/compat_libevent.h"

#include "core/or/or_connection_st.h"
#include "app/config/or_state_st.h"

#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif

#ifdef HAVE_SYSTEMD
#  if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__)
/* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse
 * Coverity. Here's a kludge to unconfuse it.
 */
#   define __INCLUDE_LEVEL__ 2
# endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */
#include <systemd/sd-daemon.h>
#endif /* defined(HAVE_SYSTEMD) */

/** Are we currently awake, asleep, running out of bandwidth, or shutting
 * down? */
static hibernate_state_t hibernate_state = HIBERNATE_STATE_INITIAL;
/** If are hibernating, when do we plan to wake up? Set to 0 if we
 * aren't hibernating. */
static time_t hibernate_end_time = 0;
/** If we are shutting down, when do we plan finally exit? Set to 0 if
 * we aren't shutting down. */
static time_t shutdown_time = 0;

/** A timed event that we'll use when it's time to wake up from
 * hibernation. */
static mainloop_event_t *wakeup_event = NULL;

/** Possible accounting periods. */
typedef enum {
  UNIT_MONTH=1, UNIT_WEEK=2, UNIT_DAY=3,
} time_unit_t;

/*
 * @file hibernate.c
 *
 * <h4>Accounting</h4>
 * Accounting is designed to ensure that no more than N bytes are sent in
 * either direction over a given interval (currently, one month, one week, or
 * one day) We could
 * try to do this by choking our bandwidth to a trickle, but that
 * would make our streams useless.  Instead, we estimate what our
 * bandwidth usage will be, and guess how long we'll be able to
 * provide that much bandwidth before hitting our limit.  We then
 * choose a random time within the accounting interval to come up (so
 * that we don't get 50 Tors running on the 1st of the month and none
 * on the 30th).
 *
 * Each interval runs as follows:
 *
 * <ol>
 * <li>We guess our bandwidth usage, based on how much we used
 *     last time.  We choose a "wakeup time" within the interval to come up.
 * <li>Until the chosen wakeup time, we hibernate.
 * <li> We come up at the wakeup time, and provide bandwidth until we are
 *    "very close" to running out.
 * <li> Then we go into low-bandwidth mode, and stop accepting new
 *    connections, but provide bandwidth until we run out.
 * <li> Then we hibernate until the end of the interval.
 *
 * If the interval ends before we run out of bandwidth, we go back to
 * step one.
 *
 * Accounting is controlled by the AccountingMax, AccountingRule, and
 * AccountingStart options.
 */

/** How many bytes have we read in this accounting interval? */
static uint64_t n_bytes_read_in_interval = 0;
/** How many bytes have we written in this accounting interval? */
static uint64_t n_bytes_written_in_interval = 0;
/** How many seconds have we been running this interval? */
static uint32_t n_seconds_active_in_interval = 0;
/** How many seconds were we active in this interval before we hit our soft
 * limit? */
static int n_seconds_to_hit_soft_limit = 0;
/** When in this interval was the soft limit hit. */
static time_t soft_limit_hit_at = 0;
/** How many bytes had we read/written when we hit the soft limit? */
static uint64_t n_bytes_at_soft_limit = 0;
/** When did this accounting interval start? */
static time_t interval_start_time = 0;
/** When will this accounting interval end? */
static time_t interval_end_time = 0;
/** How far into the accounting interval should we hibernate? */
static time_t interval_wakeup_time = 0;
/** How much bandwidth do we 'expect' to use per minute?  (0 if we have no
 * info from the last period.) */
static uint64_t expected_bandwidth_usage = 0;
/** What unit are we using for our accounting? */
static time_unit_t cfg_unit = UNIT_MONTH;

/** How many days,hours,minutes into each unit does our accounting interval
 * start? */
/** @{ */
static int cfg_start_day = 0,
           cfg_start_hour = 0,
           cfg_start_min = 0;
/** @} */

static const char *hibernate_state_to_string(hibernate_state_t state);
static void reset_accounting(time_t now);
static int read_bandwidth_usage(void);
static time_t start_of_accounting_period_after(time_t now);
static time_t start_of_accounting_period_containing(time_t now);
static void accounting_set_wakeup_time(void);
static void on_hibernate_state_change(hibernate_state_t prev_state);
static void hibernate_schedule_wakeup_event(time_t now, time_t end_time);
static void wakeup_event_callback(mainloop_event_t *ev, void *data);

/**
 * Return the human-readable name for the hibernation state <b>state</b>
 */
static const char *
hibernate_state_to_string(hibernate_state_t state)
{
  static char buf[64];
  switch (state) {
    case HIBERNATE_STATE_EXITING: return "EXITING";
    case HIBERNATE_STATE_LOWBANDWIDTH: return "SOFT";
    case HIBERNATE_STATE_DORMANT: return "HARD";
    case HIBERNATE_STATE_INITIAL:
    case HIBERNATE_STATE_LIVE:
      return "AWAKE";
    default:
      log_warn(LD_BUG, "unknown hibernate state %d", state);
      tor_snprintf(buf, sizeof(buf), "unknown [%d]", state);
      return buf;
  }
}

/* ************
 * Functions for bandwidth accounting.
 * ************/

/** Configure accounting start/end time settings based on
 * options->AccountingStart.  Return 0 on success, -1 on failure. If
 * <b>validate_only</b> is true, do not change the current settings. */
int
accounting_parse_options(const or_options_t *options, int validate_only)
{
  time_unit_t unit;
  int ok, idx;
  long d,h,m;
  smartlist_t *items;
  const char *v = options->AccountingStart;
  const char *s;
  char *cp;

  if (!v) {
    if (!validate_only) {
      cfg_unit = UNIT_MONTH;
      cfg_start_day = 1;
      cfg_start_hour = 0;
      cfg_start_min = 0;
    }
    return 0;
  }

  items = smartlist_new();
  smartlist_split_string(items, v, NULL,
                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK,0);
  if (smartlist_len(items)<2) {
    log_warn(LD_CONFIG, "Too few arguments to AccountingStart");
    goto err;
  }
  s = smartlist_get(items,0);
  if (0==strcasecmp(s, "month")) {
    unit = UNIT_MONTH;
  } else if (0==strcasecmp(s, "week")) {
    unit = UNIT_WEEK;
  } else if (0==strcasecmp(s, "day")) {
    unit = UNIT_DAY;
  } else {
    log_warn(LD_CONFIG,
             "Unrecognized accounting unit '%s': only 'month', 'week',"
             " and 'day' are supported.", s);
    goto err;
  }

  switch (unit) {
  case UNIT_WEEK:
    d = tor_parse_long(smartlist_get(items,1), 10, 1, 7, &ok, NULL);
    if (!ok) {
      log_warn(LD_CONFIG, "Weekly accounting must begin on a day between "
               "1 (Monday) and 7 (Sunday)");
      goto err;
    }
    break;
  case UNIT_MONTH:
    d = tor_parse_long(smartlist_get(items,1), 10, 1, 28, &ok, NULL);
    if (!ok) {
      log_warn(LD_CONFIG, "Monthly accounting must begin on a day between "
               "1 and 28");
      goto err;
    }
    break;
  case UNIT_DAY:
    d = 0;
    break;
    /* Coverity dislikes unreachable default cases; some compilers warn on
     * switch statements missing a case.  Tell Coverity not to worry. */
    /* coverity[dead_error_begin] */
  default:
    tor_assert(0);
  }

  idx = unit==UNIT_DAY?1:2;
  if (smartlist_len(items) != (idx+1)) {
    log_warn(LD_CONFIG,"Accounting unit '%s' requires %d argument%s.",
             s, idx, (idx>1)?"s":"");
    goto err;
  }
  s = smartlist_get(items, idx);
  h = tor_parse_long(s, 10, 0, 23, &ok, &cp);
  if (!ok) {
    log_warn(LD_CONFIG,"Accounting start time not parseable: bad hour.");
    goto err;
  }
  if (!cp || *cp!=':') {
    log_warn(LD_CONFIG,
             "Accounting start time not parseable: not in HH:MM format");
    goto err;
  }
  m = tor_parse_long(cp+1, 10, 0, 59, &ok, &cp);
  if (!ok) {
    log_warn(LD_CONFIG, "Accounting start time not parseable: bad minute");
    goto err;
  }
  if (!cp || *cp!='\0') {
    log_warn(LD_CONFIG,
             "Accounting start time not parseable: not in HH:MM format");
    goto err;
  }

  if (!validate_only) {
    cfg_unit = unit;
    cfg_start_day = (int)d;
    cfg_start_hour = (int)h;
    cfg_start_min = (int)m;
  }
  SMARTLIST_FOREACH(items, char *, item, tor_free(item));
  smartlist_free(items);
  return 0;
 err:
  SMARTLIST_FOREACH(items, char *, item, tor_free(item));
  smartlist_free(items);
  return -1;
}

/** If we want to manage the accounting system and potentially
 * hibernate, return 1, else return 0.
 */
MOCK_IMPL(int,
accounting_is_enabled,(const or_options_t *options))
{
  if (options->AccountingMax)
    return 1;
  return 0;
}

/** If accounting is enabled, return how long (in seconds) this
 * interval lasts. */
int
accounting_get_interval_length(void)
{
  return (int)(interval_end_time - interval_start_time);
}

/** Return the time at which the current accounting interval will end. */
MOCK_IMPL(time_t,
accounting_get_end_time,(void))
{
  return interval_end_time;
}

/** Called from connection.c to tell us that <b>seconds</b> seconds have
 * passed, <b>n_read</b> bytes have been read, and <b>n_written</b>
 * bytes have been written. */
void
accounting_add_bytes(size_t n_read, size_t n_written, int seconds)
{
  n_bytes_read_in_interval += n_read;
  n_bytes_written_in_interval += n_written;
  /* If we haven't been called in 10 seconds, we're probably jumping
   * around in time. */
  n_seconds_active_in_interval += (seconds < 10) ? seconds : 0;
}

/** If get_end, return the end of the accounting period that contains
 * the time <b>now</b>.  Else, return the start of the accounting
 * period that contains the time <b>now</b> */
static time_t
edge_of_accounting_period_containing(time_t now, int get_end)
{
  int before;
  struct tm tm;
  tor_localtime_r(&now, &tm);

  /* Set 'before' to true iff the current time is before the hh:mm
   * changeover time for today. */
  before = tm.tm_hour < cfg_start_hour ||
    (tm.tm_hour == cfg_start_hour && tm.tm_min < cfg_start_min);

  /* Dispatch by unit.  First, find the start day of the given period;
   * then, if get_end is true, increment to the end day. */
  switch (cfg_unit)
    {
    case UNIT_MONTH: {
      /* If this is before the Nth, we want the Nth of last month. */
      if (tm.tm_mday < cfg_start_day ||
          (tm.tm_mday == cfg_start_day && before)) {
        --tm.tm_mon;
      }
      /* Otherwise, the month is correct. */
      tm.tm_mday = cfg_start_day;
      if (get_end)
        ++tm.tm_mon;
      break;
    }
    case UNIT_WEEK: {
      /* What is the 'target' day of the week in struct tm format? (We
         say Sunday==7; struct tm says Sunday==0.) */
      int wday = cfg_start_day % 7;
      /* How many days do we subtract from today to get to the right day? */
      int delta = (7+tm.tm_wday-wday)%7;
      /* If we are on the right day, but the changeover hasn't happened yet,
       * then subtract a whole week. */
      if (delta == 0 && before)
        delta = 7;
      tm.tm_mday -= delta;
      if (get_end)
        tm.tm_mday += 7;
      break;
    }
    case UNIT_DAY:
      if (before)
        --tm.tm_mday;
      if (get_end)
        ++tm.tm_mday;
      break;
    default:
      tor_assert(0);
  }

  tm.tm_hour = cfg_start_hour;
  tm.tm_min = cfg_start_min;
  tm.tm_sec = 0;
  tm.tm_isdst = -1; /* Autodetect DST */
  return mktime(&tm);
}

/** Return the start of the accounting period containing the time
 * <b>now</b>. */
static time_t
start_of_accounting_period_containing(time_t now)
{
  return edge_of_accounting_period_containing(now, 0);
}

/** Return the start of the accounting period that comes after the one
 * containing the time <b>now</b>. */
static time_t
start_of_accounting_period_after(time_t now)
{
  return edge_of_accounting_period_containing(now, 1);
}

/** Return the length of the accounting period containing the time
 * <b>now</b>. */
static long
length_of_accounting_period_containing(time_t now)
{
  return edge_of_accounting_period_containing(now, 1) -
    edge_of_accounting_period_containing(now, 0);
}

/** Initialize the accounting subsystem. */
void
configure_accounting(time_t now)
{
  time_t s_now;
  /* Try to remember our recorded usage. */
  if (!interval_start_time)
    read_bandwidth_usage(); /* If we fail, we'll leave values at zero, and
                             * reset below.*/

  s_now = start_of_accounting_period_containing(now);

  if (!interval_start_time) {
    /* We didn't have recorded usage; Start a new interval. */
    log_info(LD_ACCT, "Starting new accounting interval.");
    reset_accounting(now);
  } else if (s_now == interval_start_time) {
    log_info(LD_ACCT, "Continuing accounting interval.");
    /* We are in the interval we thought we were in. Do nothing.*/
    interval_end_time = start_of_accounting_period_after(interval_start_time);
  } else {
    long duration =
      length_of_accounting_period_containing(interval_start_time);
    double delta = ((double)(s_now - interval_start_time)) / duration;
    if (-0.50 <= delta && delta <= 0.50) {
      /* The start of the period is now a little later or earlier than we
       * remembered.  That's fine; we might lose some bytes we could otherwise
       * have written, but better to err on the side of obeying accounting
       * settings. */
      log_info(LD_ACCT, "Accounting interval moved by %.02f%%; "
               "that's fine.", delta*100);
      interval_end_time = start_of_accounting_period_after(now);
    } else if (delta >= 0.99) {
      /* This is the regular time-moved-forward case; don't be too noisy
       * about it or people will complain */
      log_info(LD_ACCT, "Accounting interval elapsed; starting a new one");
      reset_accounting(now);
    } else {
      log_warn(LD_ACCT,
               "Mismatched accounting interval: moved by %.02f%%. "
               "Starting a fresh one.", delta*100);
      reset_accounting(now);
    }
  }
  accounting_set_wakeup_time();
}

/** Return the relevant number of bytes sent/received this interval
 * based on the set AccountingRule */
uint64_t
get_accounting_bytes(void)
{
  if (get_options()->AccountingRule == ACCT_SUM)
    return n_bytes_read_in_interval+n_bytes_written_in_interval;
  else if (get_options()->AccountingRule == ACCT_IN)
    return n_bytes_read_in_interval;
  else if (get_options()->AccountingRule == ACCT_OUT)
    return n_bytes_written_in_interval;
  else
    return MAX(n_bytes_read_in_interval, n_bytes_written_in_interval);
}

/** Set expected_bandwidth_usage based on how much we sent/received
 * per minute last interval (if we were up for at least 30 minutes),
 * or based on our declared bandwidth otherwise. */
static void
update_expected_bandwidth(void)
{
  uint64_t expected;
  const or_options_t *options= get_options();
  uint64_t max_configured = (options->RelayBandwidthRate > 0 ?
                             options->RelayBandwidthRate :
                             options->BandwidthRate) * 60;
  /* max_configured is the larger of bytes read and bytes written
   * If we are accounting based on sum, worst case is both are
   * at max, doubling the expected sum of bandwidth */
  if (get_options()->AccountingRule == ACCT_SUM)
    max_configured *= 2;

#define MIN_TIME_FOR_MEASUREMENT (1800)

  if (soft_limit_hit_at > interval_start_time && n_bytes_at_soft_limit &&
      (soft_limit_hit_at - interval_start_time) > MIN_TIME_FOR_MEASUREMENT) {
    /* If we hit our soft limit last time, only count the bytes up to that
     * time. This is a better predictor of our actual bandwidth than
     * considering the entirety of the last interval, since we likely started
     * using bytes very slowly once we hit our soft limit. */
    expected = n_bytes_at_soft_limit /
      (soft_limit_hit_at - interval_start_time);
    expected /= 60;
  } else if (n_seconds_active_in_interval >= MIN_TIME_FOR_MEASUREMENT) {
    /* Otherwise, we either measured enough time in the last interval but
     * never hit our soft limit, or we're using a state file from a Tor that
     * doesn't know to store soft-limit info.  Just take rate at which
     * we were reading/writing in the last interval as our expected rate.
     */
    uint64_t used = get_accounting_bytes();
    expected = used / (n_seconds_active_in_interval / 60);
  } else {
    /* If we haven't gotten enough data last interval, set 'expected'
     * to 0.  This will set our wakeup to the start of the interval.
     * Next interval, we'll choose our starting time based on how much
     * we sent this interval.
     */
    expected = 0;
  }
  if (expected > max_configured)
    expected = max_configured;
  expected_bandwidth_usage = expected;
}

/** Called at the start of a new accounting interval: reset our
 * expected bandwidth usage based on what happened last time, set up
 * the start and end of the interval, and clear byte/time totals.
 */
static void
reset_accounting(time_t now)
{
  log_info(LD_ACCT, "Starting new accounting interval.");
  update_expected_bandwidth();
  interval_start_time = start_of_accounting_period_containing(now);
  interval_end_time = start_of_accounting_period_after(interval_start_time);
  n_bytes_read_in_interval = 0;
  n_bytes_written_in_interval = 0;
  n_seconds_active_in_interval = 0;
  n_bytes_at_soft_limit = 0;
  soft_limit_hit_at = 0;
  n_seconds_to_hit_soft_limit = 0;
}

/** Return true iff we should save our bandwidth usage to disk. */
static inline int
time_to_record_bandwidth_usage(time_t now)
{
  /* Note every 600 sec */
#define NOTE_INTERVAL (600)
  /* Or every 20 megabytes */
#define NOTE_BYTES 20*(1024*1024)
  static uint64_t last_read_bytes_noted = 0;
  static uint64_t last_written_bytes_noted = 0;
  static time_t last_time_noted = 0;

  if (last_time_noted + NOTE_INTERVAL <= now ||
      last_read_bytes_noted + NOTE_BYTES <= n_bytes_read_in_interval ||
      last_written_bytes_noted + NOTE_BYTES <= n_bytes_written_in_interval ||
      (interval_end_time && interval_end_time <= now)) {
    last_time_noted = now;
    last_read_bytes_noted = n_bytes_read_in_interval;
    last_written_bytes_noted = n_bytes_written_in_interval;
    return 1;
  }
  return 0;
}

/** Invoked once per second.  Checks whether it is time to hibernate,
 * record bandwidth used, etc.  */
void
accounting_run_housekeeping(time_t now)
{
  if (now >= interval_end_time) {
    configure_accounting(now);
  }
  if (time_to_record_bandwidth_usage(now)) {
    if (accounting_record_bandwidth_usage(now, get_or_state())) {
      log_warn(LD_FS, "Couldn't record bandwidth usage to disk.");
    }
  }
}

/** Based on our interval and our estimated bandwidth, choose a
 * deterministic (but random-ish) time to wake up. */
static void
accounting_set_wakeup_time(void)
{
  char digest[DIGEST_LEN];
  crypto_digest_t *d_env;
  uint64_t time_to_exhaust_bw;
  int time_to_consider;

  if (! server_identity_key_is_set()) {
    if (init_keys() < 0) {
      log_err(LD_BUG, "Error initializing keys");
      tor_assert(0);
    }
  }

  if (server_identity_key_is_set()) {
    char buf[ISO_TIME_LEN+1];
    format_iso_time(buf, interval_start_time);

    if (crypto_pk_get_digest(get_server_identity_key(), digest) < 0) {
      log_err(LD_BUG, "Error getting our key's digest.");
      tor_assert(0);
    }

    d_env = crypto_digest_new();
    crypto_digest_add_bytes(d_env, buf, ISO_TIME_LEN);
    crypto_digest_add_bytes(d_env, digest, DIGEST_LEN);
    crypto_digest_get_digest(d_env, digest, DIGEST_LEN);
    crypto_digest_free(d_env);
  } else {
    crypto_rand(digest, DIGEST_LEN);
  }

  if (!expected_bandwidth_usage) {
    char buf1[ISO_TIME_LEN+1];
    char buf2[ISO_TIME_LEN+1];
    format_local_iso_time(buf1, interval_start_time);
    format_local_iso_time(buf2, interval_end_time);
    interval_wakeup_time = interval_start_time;

    log_notice(LD_ACCT,
           "Configured hibernation. This interval begins at %s "
           "and ends at %s. We have no prior estimate for bandwidth, so "
           "we will start out awake and hibernate when we exhaust our quota.",
           buf1, buf2);
    return;
  }

  time_to_exhaust_bw =
    (get_options()->AccountingMax/expected_bandwidth_usage)*60;
  if (time_to_exhaust_bw > INT_MAX) {
    time_to_exhaust_bw = INT_MAX;
    time_to_consider = 0;
  } else {
    time_to_consider = accounting_get_interval_length() -
                       (int)time_to_exhaust_bw;
  }

  if (time_to_consider<=0) {
    interval_wakeup_time = interval_start_time;
  } else {
    /* XXX can we simplify this just by picking a random (non-deterministic)
     * time to be up? If we go down and come up, then we pick a new one. Is
     * that good enough? -RD */

    /* This is not a perfectly unbiased conversion, but it is good enough:
     * in the worst case, the first half of the day is 0.06 percent likelier
     * to be chosen than the last half. */
    interval_wakeup_time = interval_start_time +
      (get_uint32(digest) % time_to_consider);
  }

  {
    char buf1[ISO_TIME_LEN+1];
    char buf2[ISO_TIME_LEN+1];
    char buf3[ISO_TIME_LEN+1];
    char buf4[ISO_TIME_LEN+1];
    time_t down_time;
    if (interval_wakeup_time+time_to_exhaust_bw > TIME_MAX)
      down_time = TIME_MAX;
    else
      down_time = (time_t)(interval_wakeup_time+time_to_exhaust_bw);
    if (down_time>interval_end_time)
      down_time = interval_end_time;
    format_local_iso_time(buf1, interval_start_time);
    format_local_iso_time(buf2, interval_wakeup_time);
    format_local_iso_time(buf3, down_time);
    format_local_iso_time(buf4, interval_end_time);

    log_notice(LD_ACCT,
           "Configured hibernation.  This interval began at %s; "
           "the scheduled wake-up time %s %s; "
           "we expect%s to exhaust our quota for this interval around %s; "
           "the next interval begins at %s (all times local)",
           buf1,
           time(NULL)<interval_wakeup_time?"is":"was", buf2,
           time(NULL)<down_time?"":"ed", buf3,
           buf4);
  }
}

/* This rounds 0 up to 1000, but that's actually a feature. */
#define ROUND_UP(x) (((x) + 0x3ff) & ~0x3ff)
/** Save all our bandwidth tracking information to disk. Return 0 on
 * success, -1 on failure. */
int
accounting_record_bandwidth_usage(time_t now, or_state_t *state)
{
  /* Just update the state */
  state->AccountingIntervalStart = interval_start_time;
  state->AccountingBytesReadInInterval = ROUND_UP(n_bytes_read_in_interval);
  state->AccountingBytesWrittenInInterval =
    ROUND_UP(n_bytes_written_in_interval);
  state->AccountingSecondsActive = n_seconds_active_in_interval;
  state->AccountingExpectedUsage = expected_bandwidth_usage;

  state->AccountingSecondsToReachSoftLimit = n_seconds_to_hit_soft_limit;
  state->AccountingSoftLimitHitAt = soft_limit_hit_at;
  state->AccountingBytesAtSoftLimit = n_bytes_at_soft_limit;

  or_state_mark_dirty(state,
                      now+(get_options()->AvoidDiskWrites ? 7200 : 60));

  return 0;
}
#undef ROUND_UP

/** Read stored accounting information from disk. Return 0 on success;
 * return -1 and change nothing on failure. */
static int
read_bandwidth_usage(void)
{
  or_state_t *state = get_or_state();

  {
    char *fname = get_datadir_fname("bw_accounting");
    int res;

    res = unlink(fname);
    if (res != 0 && errno != ENOENT) {
      log_warn(LD_FS,
               "Failed to unlink %s: %s",
               fname, strerror(errno));
    }

    tor_free(fname);
  }

  if (!state)
    return -1;

  log_info(LD_ACCT, "Reading bandwidth accounting data from state file");
  n_bytes_read_in_interval = state->AccountingBytesReadInInterval;
  n_bytes_written_in_interval = state->AccountingBytesWrittenInInterval;
  n_seconds_active_in_interval = state->AccountingSecondsActive;
  interval_start_time = state->AccountingIntervalStart;
  expected_bandwidth_usage = state->AccountingExpectedUsage;

  /* Older versions of Tor (before 0.2.2.17-alpha or so) didn't generate these
   * fields. If you switch back and forth, you might get an
   * AccountingSoftLimitHitAt value from long before the most recent
   * interval_start_time.  If that's so, then ignore the softlimit-related
   * values. */
  if (state->AccountingSoftLimitHitAt > interval_start_time) {
    soft_limit_hit_at =  state->AccountingSoftLimitHitAt;
    n_bytes_at_soft_limit = state->AccountingBytesAtSoftLimit;
    n_seconds_to_hit_soft_limit = state->AccountingSecondsToReachSoftLimit;
  } else {
    soft_limit_hit_at = 0;
    n_bytes_at_soft_limit = 0;
    n_seconds_to_hit_soft_limit = 0;
  }

  {
    char tbuf1[ISO_TIME_LEN+1];
    char tbuf2[ISO_TIME_LEN+1];
    format_iso_time(tbuf1, state->LastWritten);
    format_iso_time(tbuf2, state->AccountingIntervalStart);

    log_info(LD_ACCT,
       "Successfully read bandwidth accounting info from state written at %s "
       "for interval starting at %s.  We have been active for %lu seconds in "
       "this interval.  At the start of the interval, we expected to use "
       "about %lu KB per second. (%"PRIu64" bytes read so far, "
       "%"PRIu64" bytes written so far)",
       tbuf1, tbuf2,
       (unsigned long)n_seconds_active_in_interval,
       (unsigned long)(expected_bandwidth_usage*1024/60),
       (n_bytes_read_in_interval),
       (n_bytes_written_in_interval));
  }

  return 0;
}

/** Return true iff we have sent/received all the bytes we are willing
 * to send/receive this interval. */
static int
hibernate_hard_limit_reached(void)
{
  uint64_t hard_limit = get_options()->AccountingMax;
  if (!hard_limit)
    return 0;
  return get_accounting_bytes() >= hard_limit;
}

/** Return true iff we have sent/received almost all the bytes we are willing
 * to send/receive this interval. */
static int
hibernate_soft_limit_reached(void)
{
  const uint64_t acct_max = get_options()->AccountingMax;
#define SOFT_LIM_PCT (.95)
#define SOFT_LIM_BYTES (500*1024*1024)
#define SOFT_LIM_MINUTES (3*60)
  /* The 'soft limit' is a fair bit more complicated now than once it was.
   * We want to stop accepting connections when ALL of the following are true:
   *   - We expect to use up the remaining bytes in under 3 hours
   *   - We have used up 95% of our bytes.
   *   - We have less than 500MB of bytes left.
   */
  uint64_t soft_limit = (uint64_t) (acct_max * SOFT_LIM_PCT);
  if (acct_max > SOFT_LIM_BYTES && acct_max - SOFT_LIM_BYTES > soft_limit) {
    soft_limit = acct_max - SOFT_LIM_BYTES;
  }
  if (expected_bandwidth_usage) {
    const uint64_t expected_usage =
      expected_bandwidth_usage * SOFT_LIM_MINUTES;
    if (acct_max > expected_usage && acct_max - expected_usage > soft_limit)
      soft_limit = acct_max - expected_usage;
  }

  if (!soft_limit)
    return 0;
  return get_accounting_bytes() >= soft_limit;
}

#define TOR_USEC_PER_SEC (1000000)

/** Called when we get a SIGINT, or when bandwidth soft limit is
 * reached. Puts us into "loose hibernation": we don't accept new
 * connections, but we continue handling old ones. */
static void
hibernate_begin(hibernate_state_t new_state, time_t now)
{
  const or_options_t *options = get_options();

  if (new_state == HIBERNATE_STATE_EXITING &&
      hibernate_state != HIBERNATE_STATE_LIVE) {
    log_notice(LD_GENERAL,"SIGINT received %s; exiting now.",
               hibernate_state == HIBERNATE_STATE_EXITING ?
               "a second time" : "while hibernating");
    tor_shutdown_event_loop_and_exit(0);
    return;
  }

  if (new_state == HIBERNATE_STATE_LOWBANDWIDTH &&
      hibernate_state == HIBERNATE_STATE_LIVE) {
    soft_limit_hit_at = now;
    n_seconds_to_hit_soft_limit = n_seconds_active_in_interval;
    n_bytes_at_soft_limit = get_accounting_bytes();
  }

  /* close listeners. leave control listener(s). */
  connection_mark_all_noncontrol_listeners();

  /* XXX kill intro point circs */
  /* XXX upload rendezvous service descriptors with no intro points */

  if (new_state == HIBERNATE_STATE_EXITING) {
    log_notice(LD_GENERAL,"Interrupt: we have stopped accepting new "
               "connections, and will shut down in %d seconds. Interrupt "
               "again to exit now.", options->ShutdownWaitLength);
    shutdown_time = time(NULL) + options->ShutdownWaitLength;
#ifdef HAVE_SYSTEMD
    /* tell systemd that we may need more than the default 90 seconds to shut
     * down so they don't kill us. add some extra time to actually finish
     * shutting down, otherwise systemd will kill us immediately after the
     * EXTEND_TIMEOUT_USEC expires. this is an *upper* limit; tor will probably
     * only take one or two more seconds, but assume that maybe we got swapped
     * out and it takes a little while longer.
     *
     * as of writing, this is a no-op with all-defaults: ShutdownWaitLength is
     * 30 seconds, so this will extend the timeout to 60 seconds.
     * default systemd DefaultTimeoutStopSec is 90 seconds, so systemd will
     * wait (up to) 90 seconds anyways.
     *
     * 2^31 usec = ~2147 sec = ~35 min. probably nobody will actually set
     * ShutdownWaitLength to more than that, but use a longer type so we don't
     * need to think about UB on overflow
     */
    sd_notifyf(0, "EXTEND_TIMEOUT_USEC=%" PRIu64,
            ((uint64_t)(options->ShutdownWaitLength) + 30) * TOR_USEC_PER_SEC);
#endif
  } else { /* soft limit reached */
    hibernate_end_time = interval_end_time;
  }

  hibernate_state = new_state;
  accounting_record_bandwidth_usage(now, get_or_state());

  or_state_mark_dirty(get_or_state(),
                      get_options()->AvoidDiskWrites ? now+600 : 0);
}

/** Called when we've been hibernating and our timeout is reached. */
static void
hibernate_end(hibernate_state_t new_state)
{
  tor_assert(hibernate_state == HIBERNATE_STATE_LOWBANDWIDTH ||
             hibernate_state == HIBERNATE_STATE_DORMANT ||
             hibernate_state == HIBERNATE_STATE_INITIAL);

  /* listeners will be relaunched in run_scheduled_events() in main.c */
  if (hibernate_state != HIBERNATE_STATE_INITIAL)
    log_notice(LD_ACCT,"Hibernation period ended. Resuming normal activity.");

  hibernate_state = new_state;
  hibernate_end_time = 0; /* no longer hibernating */
  reset_uptime(); /* reset published uptime */
}

/** A wrapper around hibernate_begin, for when we get SIGINT. */
void
hibernate_begin_shutdown(void)
{
  hibernate_begin(HIBERNATE_STATE_EXITING, time(NULL));
}

/**
 * Return true iff we are currently hibernating -- that is, if we are in
 * any non-live state.
 */
MOCK_IMPL(int,
we_are_hibernating,(void))
{
  return hibernate_state != HIBERNATE_STATE_LIVE;
}

/**
 * Return true iff we are currently _fully_ hibernating -- that is, if we are
 * in a state where we expect to handle no network activity at all.
 */
MOCK_IMPL(int,
we_are_fully_hibernating,(void))
{
  return hibernate_state == HIBERNATE_STATE_DORMANT;
}

/** If we aren't currently dormant, close all connections and become
 * dormant. */
static void
hibernate_go_dormant(time_t now)
{
  connection_t *conn;

  if (hibernate_state == HIBERNATE_STATE_DORMANT)
    return;
  else if (hibernate_state == HIBERNATE_STATE_LOWBANDWIDTH)
    hibernate_state = HIBERNATE_STATE_DORMANT;
  else
    hibernate_begin(HIBERNATE_STATE_DORMANT, now);

  log_notice(LD_ACCT,"Going dormant. Blowing away remaining connections.");

  /* Close all OR/AP/exit conns. Leave dir conns because we still want
   * to be able to upload server descriptors so clients know we're still
   * running, and download directories so we can detect if we're obsolete.
   * Leave control conns because we still want to be controllable.
   */
  while ((conn = connection_get_by_type(CONN_TYPE_OR)) ||
         (conn = connection_get_by_type(CONN_TYPE_AP)) ||
         (conn = connection_get_by_type(CONN_TYPE_EXIT))) {
    if (CONN_IS_EDGE(conn)) {
      connection_edge_end(TO_EDGE_CONN(conn), END_STREAM_REASON_HIBERNATING);
    }
    log_info(LD_NET,"Closing conn type %d", conn->type);
    if (conn->type == CONN_TYPE_AP) {
      /* send socks failure if needed */
      connection_mark_unattached_ap(TO_ENTRY_CONN(conn),
                                    END_STREAM_REASON_HIBERNATING);
    } else if (conn->type == CONN_TYPE_OR) {
      if (TO_OR_CONN(conn)->chan) {
        connection_or_close_normally(TO_OR_CONN(conn), 0);
      } else {
         connection_mark_for_close(conn);
      }
    } else {
      connection_mark_for_close(conn);
    }
  }

  if (now < interval_wakeup_time)
    hibernate_end_time = interval_wakeup_time;
  else
    hibernate_end_time = interval_end_time;

  accounting_record_bandwidth_usage(now, get_or_state());

  or_state_mark_dirty(get_or_state(),
                      get_options()->AvoidDiskWrites ? now+600 : 0);

  hibernate_schedule_wakeup_event(now, hibernate_end_time);
}

/**
 * Schedule a mainloop event at <b>end_time</b> to wake up from a dormant
 * state.  We can't rely on this happening from second_elapsed_callback,
 * since second_elapsed_callback will be shut down when we're dormant.
 *
 * (Note that We might immediately go back to sleep after we set the next
 * wakeup time.)
 */
static void
hibernate_schedule_wakeup_event(time_t now, time_t end_time)
{
  struct timeval delay = { 0, 0 };

  if (now >= end_time) {
    // In these cases we always wait at least a second, to avoid running
    // the callback in a tight loop.
    delay.tv_sec = 1;
  } else {
    delay.tv_sec = (end_time - now);
  }

  if (!wakeup_event) {
    wakeup_event = mainloop_event_postloop_new(wakeup_event_callback, NULL);
  }

  mainloop_event_schedule(wakeup_event, &delay);
}

/**
 * Called at the end of the interval, or at the wakeup time of the current
 * interval, to exit the dormant state.
 **/
static void
wakeup_event_callback(mainloop_event_t *ev, void *data)
{
  (void) ev;
  (void) data;

  const time_t now = time(NULL);
  accounting_run_housekeeping(now);
  consider_hibernation(now);
  if (hibernate_state != HIBERNATE_STATE_DORMANT) {
    /* We woke up, so everything's great here */
    return;
  }

  /* We're still dormant. */
  if (now < interval_wakeup_time)
    hibernate_end_time = interval_wakeup_time;
  else
    hibernate_end_time = interval_end_time;

  hibernate_schedule_wakeup_event(now, hibernate_end_time);
}

/** Called when hibernate_end_time has arrived. */
static void
hibernate_end_time_elapsed(time_t now)
{
  char buf[ISO_TIME_LEN+1];

  /* The interval has ended, or it is wakeup time.  Find out which. */
  accounting_run_housekeeping(now);
  if (interval_wakeup_time <= now) {
    /* The interval hasn't changed, but interval_wakeup_time has passed.
     * It's time to wake up and start being a server. */
    hibernate_end(HIBERNATE_STATE_LIVE);
    return;
  } else {
    /* The interval has changed, and it isn't time to wake up yet. */
    hibernate_end_time = interval_wakeup_time;
    format_iso_time(buf,interval_wakeup_time);
    if (hibernate_state != HIBERNATE_STATE_DORMANT) {
      /* We weren't sleeping before; we should sleep now. */
      log_notice(LD_ACCT,
                 "Accounting period ended. Commencing hibernation until "
                 "%s UTC", buf);
      hibernate_go_dormant(now);
    } else {
      log_notice(LD_ACCT,
             "Accounting period ended. This period, we will hibernate"
             " until %s UTC",buf);
    }
  }
}

/** Consider our environment and decide if it's time
 * to start/stop hibernating.
 */
void
consider_hibernation(time_t now)
{
  int accounting_enabled = get_options()->AccountingMax != 0;
  char buf[ISO_TIME_LEN+1];
  hibernate_state_t prev_state = hibernate_state;

  /* If we're in 'exiting' mode, then we just shut down after the interval
   * elapses. */
  if (hibernate_state == HIBERNATE_STATE_EXITING) {
    tor_assert(shutdown_time);
    if (shutdown_time <= now) {
      log_notice(LD_GENERAL, "Clean shutdown finished. Exiting.");
      tor_shutdown_event_loop_and_exit(0);
    }
    return; /* if exiting soon, don't worry about bandwidth limits */
  }

  if (hibernate_state == HIBERNATE_STATE_DORMANT) {
    /* We've been hibernating because of bandwidth accounting. */
    tor_assert(hibernate_end_time);
    if (hibernate_end_time > now && accounting_enabled) {
      /* If we're hibernating, don't wake up until it's time, regardless of
       * whether we're in a new interval. */
      return ;
    } else {
      hibernate_end_time_elapsed(now);
    }
  }

  /* Else, we aren't hibernating. See if it's time to start hibernating, or to
   * go dormant. */
  if (hibernate_state == HIBERNATE_STATE_LIVE ||
      hibernate_state == HIBERNATE_STATE_INITIAL) {
    if (hibernate_soft_limit_reached()) {
      log_notice(LD_ACCT,
                 "Bandwidth soft limit reached; commencing hibernation. "
                 "No new connections will be accepted");
      hibernate_begin(HIBERNATE_STATE_LOWBANDWIDTH, now);
    } else if (accounting_enabled && now < interval_wakeup_time) {
      format_local_iso_time(buf,interval_wakeup_time);
      log_notice(LD_ACCT,
                 "Commencing hibernation. We will wake up at %s local time.",
                 buf);
      hibernate_go_dormant(now);
    } else if (hibernate_state == HIBERNATE_STATE_INITIAL) {
      hibernate_end(HIBERNATE_STATE_LIVE);
    }
  }

  if (hibernate_state == HIBERNATE_STATE_LOWBANDWIDTH) {
    if (!accounting_enabled) {
      hibernate_end_time_elapsed(now);
    } else if (hibernate_hard_limit_reached()) {
      hibernate_go_dormant(now);
    } else if (hibernate_end_time <= now) {
      /* The hibernation period ended while we were still in lowbandwidth.*/
      hibernate_end_time_elapsed(now);
    }
  }

  /* Dispatch a controller event if the hibernation state changed. */
  if (hibernate_state != prev_state)
    on_hibernate_state_change(prev_state);
}

/** Helper function: called when we get a GETINFO request for an
 * accounting-related key on the control connection <b>conn</b>.  If we can
 * answer the request for <b>question</b>, then set *<b>answer</b> to a newly
 * allocated string holding the result.  Otherwise, set *<b>answer</b> to
 * NULL. */
int
getinfo_helper_accounting(control_connection_t *conn,
                          const char *question, char **answer,
                          const char **errmsg)
{
  (void) conn;
  (void) errmsg;
  if (!strcmp(question, "accounting/enabled")) {
    *answer = tor_strdup(accounting_is_enabled(get_options()) ? "1" : "0");
  } else if (!strcmp(question, "accounting/hibernating")) {
    *answer = tor_strdup(hibernate_state_to_string(hibernate_state));
    tor_strlower(*answer);
  } else if (!strcmp(question, "accounting/bytes")) {
      tor_asprintf(answer, "%"PRIu64" %"PRIu64,
                 (n_bytes_read_in_interval),
                 (n_bytes_written_in_interval));
  } else if (!strcmp(question, "accounting/bytes-left")) {
    uint64_t limit = get_options()->AccountingMax;
    if (get_options()->AccountingRule == ACCT_SUM) {
      uint64_t total_left = 0;
      uint64_t total_bytes = get_accounting_bytes();
      if (total_bytes < limit)
        total_left = limit - total_bytes;
      tor_asprintf(answer, "%"PRIu64" %"PRIu64,
                   (total_left), (total_left));
    } else if (get_options()->AccountingRule == ACCT_IN) {
      uint64_t read_left = 0;
      if (n_bytes_read_in_interval < limit)
        read_left = limit - n_bytes_read_in_interval;
      tor_asprintf(answer, "%"PRIu64" %"PRIu64,
                   (read_left), (limit));
    } else if (get_options()->AccountingRule == ACCT_OUT) {
      uint64_t write_left = 0;
      if (n_bytes_written_in_interval < limit)
        write_left = limit - n_bytes_written_in_interval;
      tor_asprintf(answer, "%"PRIu64" %"PRIu64,
                   (limit), (write_left));
    } else {
      uint64_t read_left = 0, write_left = 0;
      if (n_bytes_read_in_interval < limit)
        read_left = limit - n_bytes_read_in_interval;
      if (n_bytes_written_in_interval < limit)
        write_left = limit - n_bytes_written_in_interval;
      tor_asprintf(answer, "%"PRIu64" %"PRIu64,
                   (read_left), (write_left));
    }
  } else if (!strcmp(question, "accounting/interval-start")) {
    *answer = tor_malloc(ISO_TIME_LEN+1);
    format_iso_time(*answer, interval_start_time);
  } else if (!strcmp(question, "accounting/interval-wake")) {
    *answer = tor_malloc(ISO_TIME_LEN+1);
    format_iso_time(*answer, interval_wakeup_time);
  } else if (!strcmp(question, "accounting/interval-end")) {
    *answer = tor_malloc(ISO_TIME_LEN+1);
    format_iso_time(*answer, interval_end_time);
  } else {
    *answer = NULL;
  }
  return 0;
}

/**
 * Helper function: called when the hibernation state changes, and sends a
 * SERVER_STATUS event to notify interested controllers of the accounting
 * state change.
 */
static void
on_hibernate_state_change(hibernate_state_t prev_state)
{
  control_event_server_status(LOG_NOTICE,
                              "HIBERNATION_STATUS STATUS=%s",
                              hibernate_state_to_string(hibernate_state));

  /* We are changing hibernation state, this can affect the main loop event
   * list. Rescan it to update the events state. We do this whatever the new
   * hibernation state because they can each possibly affect an event. The
   * initial state means we are booting up so we shouldn't scan here because
   * at this point the events in the list haven't been initialized. */
  if (prev_state != HIBERNATE_STATE_INITIAL) {
    rescan_periodic_events(get_options());
  }

  reschedule_per_second_timer();
}

/** Free all resources held by the accounting module */
void
accounting_free_all(void)
{
  mainloop_event_free(wakeup_event);
  hibernate_state = HIBERNATE_STATE_INITIAL;
  hibernate_end_time = 0;
  shutdown_time = 0;
}

#ifdef TOR_UNIT_TESTS
/**
 * Manually change the hibernation state.  Private; used only by the unit
 * tests.
 */
void
hibernate_set_state_for_testing_(hibernate_state_t newstate)
{
  hibernate_state = newstate;
}
#endif /* defined(TOR_UNIT_TESTS) */