geoip.c 61 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875
  1. /* Copyright (c) 2007-2016, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. /**
  4. * \file geoip.c
  5. * \brief Functions related to maintaining an IP-to-country database;
  6. * to summarizing client connections by country to entry guards, bridges,
  7. * and directory servers; and for statistics on answering network status
  8. * requests.
  9. *
  10. * There are two main kinds of functions in this module: geoip functions,
  11. * which map groups of IPv4 and IPv6 addresses to country codes, and
  12. * statistical functions, which collect statistics about different kinds of
  13. * per-country usage.
  14. *
  15. * The geoip lookup tables are implemented as sorted lists of disjoint address
  16. * ranges, each mapping to a singleton geoip_country_t. These country objects
  17. * are also indexed by their names in a hashtable.
  18. *
  19. * The tables are populated from disk at startup by the geoip_load_file()
  20. * function. For more information on the file format they read, see that
  21. * function. See the scripts and the README file in src/config for more
  22. * information about how those files are generated.
  23. *
  24. * Tor uses GeoIP information in order to implement user requests (such as
  25. * ExcludeNodes {cc}), and to keep track of how much usage relays are getting
  26. * for each country.
  27. */
  28. #define GEOIP_PRIVATE
  29. #include "or.h"
  30. #include "ht.h"
  31. #include "config.h"
  32. #include "control.h"
  33. #include "dnsserv.h"
  34. #include "dos.h"
  35. #include "geoip.h"
  36. #include "routerlist.h"
  37. static void init_geoip_countries(void);
  38. /** An entry from the GeoIP IPv4 file: maps an IPv4 range to a country. */
  39. typedef struct geoip_ipv4_entry_t {
  40. uint32_t ip_low; /**< The lowest IP in the range, in host order */
  41. uint32_t ip_high; /**< The highest IP in the range, in host order */
  42. intptr_t country; /**< An index into geoip_countries */
  43. } geoip_ipv4_entry_t;
  44. /** An entry from the GeoIP IPv6 file: maps an IPv6 range to a country. */
  45. typedef struct geoip_ipv6_entry_t {
  46. struct in6_addr ip_low; /**< The lowest IP in the range, in host order */
  47. struct in6_addr ip_high; /**< The highest IP in the range, in host order */
  48. intptr_t country; /**< An index into geoip_countries */
  49. } geoip_ipv6_entry_t;
  50. /** A per-country record for GeoIP request history. */
  51. typedef struct geoip_country_t {
  52. char countrycode[3];
  53. uint32_t n_v3_ns_requests;
  54. } geoip_country_t;
  55. /** A list of geoip_country_t */
  56. static smartlist_t *geoip_countries = NULL;
  57. /** A map from lowercased country codes to their position in geoip_countries.
  58. * The index is encoded in the pointer, and 1 is added so that NULL can mean
  59. * not found. */
  60. static strmap_t *country_idxplus1_by_lc_code = NULL;
  61. /** Lists of all known geoip_ipv4_entry_t and geoip_ipv6_entry_t, sorted
  62. * by their respective ip_low. */
  63. static smartlist_t *geoip_ipv4_entries = NULL, *geoip_ipv6_entries = NULL;
  64. /** SHA1 digest of the GeoIP files to include in extra-info descriptors. */
  65. static char geoip_digest[DIGEST_LEN];
  66. static char geoip6_digest[DIGEST_LEN];
  67. /* Total size in bytes of the geoip client history cache. Used by the OOM
  68. * handler. */
  69. static size_t geoip_client_history_cache_size;
  70. /* Increment the geoip client history cache size counter with the given bytes.
  71. * This prevents an overflow and set it to its maximum in that case. */
  72. static inline void
  73. geoip_increment_client_history_cache_size(size_t bytes)
  74. {
  75. /* This is shockingly high, lets log it so it can be reported. */
  76. IF_BUG_ONCE(geoip_client_history_cache_size > (SIZE_MAX - bytes)) {
  77. geoip_client_history_cache_size = SIZE_MAX;
  78. return;
  79. }
  80. geoip_client_history_cache_size += bytes;
  81. }
  82. /* Decrement the geoip client history cache size counter with the given bytes.
  83. * This prevents an underflow and set it to 0 in that case. */
  84. static inline void
  85. geoip_decrement_client_history_cache_size(size_t bytes)
  86. {
  87. /* Going below 0 means that we either allocated an entry without
  88. * incrementing the counter or we have different sizes when allocating and
  89. * freeing. It shouldn't happened so log it. */
  90. IF_BUG_ONCE(geoip_client_history_cache_size < bytes) {
  91. geoip_client_history_cache_size = 0;
  92. return;
  93. }
  94. geoip_client_history_cache_size -= bytes;
  95. }
  96. /** Return the index of the <b>country</b>'s entry in the GeoIP
  97. * country list if it is a valid 2-letter country code, otherwise
  98. * return -1. */
  99. MOCK_IMPL(country_t,
  100. geoip_get_country,(const char *country))
  101. {
  102. void *idxplus1_;
  103. intptr_t idx;
  104. idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country);
  105. if (!idxplus1_)
  106. return -1;
  107. idx = ((uintptr_t)idxplus1_)-1;
  108. return (country_t)idx;
  109. }
  110. /** Add an entry to a GeoIP table, mapping all IP addresses between <b>low</b>
  111. * and <b>high</b>, inclusive, to the 2-letter country code <b>country</b>. */
  112. static void
  113. geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high,
  114. const char *country)
  115. {
  116. intptr_t idx;
  117. void *idxplus1_;
  118. IF_BUG_ONCE(tor_addr_family(low) != tor_addr_family(high))
  119. return;
  120. IF_BUG_ONCE(tor_addr_compare(high, low, CMP_EXACT) < 0)
  121. return;
  122. idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country);
  123. if (!idxplus1_) {
  124. geoip_country_t *c = tor_malloc_zero(sizeof(geoip_country_t));
  125. strlcpy(c->countrycode, country, sizeof(c->countrycode));
  126. tor_strlower(c->countrycode);
  127. smartlist_add(geoip_countries, c);
  128. idx = smartlist_len(geoip_countries) - 1;
  129. strmap_set_lc(country_idxplus1_by_lc_code, country, (void*)(idx+1));
  130. } else {
  131. idx = ((uintptr_t)idxplus1_)-1;
  132. }
  133. {
  134. geoip_country_t *c = smartlist_get(geoip_countries, idx);
  135. tor_assert(!strcasecmp(c->countrycode, country));
  136. }
  137. if (tor_addr_family(low) == AF_INET) {
  138. geoip_ipv4_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv4_entry_t));
  139. ent->ip_low = tor_addr_to_ipv4h(low);
  140. ent->ip_high = tor_addr_to_ipv4h(high);
  141. ent->country = idx;
  142. smartlist_add(geoip_ipv4_entries, ent);
  143. } else if (tor_addr_family(low) == AF_INET6) {
  144. geoip_ipv6_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv6_entry_t));
  145. ent->ip_low = *tor_addr_to_in6_assert(low);
  146. ent->ip_high = *tor_addr_to_in6_assert(high);
  147. ent->country = idx;
  148. smartlist_add(geoip_ipv6_entries, ent);
  149. }
  150. }
  151. /** Add an entry to the GeoIP table indicated by <b>family</b>,
  152. * parsing it from <b>line</b>. The format is as for geoip_load_file(). */
  153. STATIC int
  154. geoip_parse_entry(const char *line, sa_family_t family)
  155. {
  156. tor_addr_t low_addr, high_addr;
  157. char c[3];
  158. char *country = NULL;
  159. if (!geoip_countries)
  160. init_geoip_countries();
  161. if (family == AF_INET) {
  162. if (!geoip_ipv4_entries)
  163. geoip_ipv4_entries = smartlist_new();
  164. } else if (family == AF_INET6) {
  165. if (!geoip_ipv6_entries)
  166. geoip_ipv6_entries = smartlist_new();
  167. } else {
  168. log_warn(LD_GENERAL, "Unsupported family: %d", family);
  169. return -1;
  170. }
  171. while (TOR_ISSPACE(*line))
  172. ++line;
  173. if (*line == '#')
  174. return 0;
  175. char buf[512];
  176. if (family == AF_INET) {
  177. unsigned int low, high;
  178. if (tor_sscanf(line,"%u,%u,%2s", &low, &high, c) == 3 ||
  179. tor_sscanf(line,"\"%u\",\"%u\",\"%2s\",", &low, &high, c) == 3) {
  180. tor_addr_from_ipv4h(&low_addr, low);
  181. tor_addr_from_ipv4h(&high_addr, high);
  182. } else
  183. goto fail;
  184. country = c;
  185. } else { /* AF_INET6 */
  186. char *low_str, *high_str;
  187. struct in6_addr low, high;
  188. char *strtok_state;
  189. strlcpy(buf, line, sizeof(buf));
  190. low_str = tor_strtok_r(buf, ",", &strtok_state);
  191. if (!low_str)
  192. goto fail;
  193. high_str = tor_strtok_r(NULL, ",", &strtok_state);
  194. if (!high_str)
  195. goto fail;
  196. country = tor_strtok_r(NULL, "\n", &strtok_state);
  197. if (!country)
  198. goto fail;
  199. if (strlen(country) != 2)
  200. goto fail;
  201. if (tor_inet_pton(AF_INET6, low_str, &low) <= 0)
  202. goto fail;
  203. tor_addr_from_in6(&low_addr, &low);
  204. if (tor_inet_pton(AF_INET6, high_str, &high) <= 0)
  205. goto fail;
  206. tor_addr_from_in6(&high_addr, &high);
  207. }
  208. geoip_add_entry(&low_addr, &high_addr, country);
  209. return 0;
  210. fail:
  211. log_warn(LD_GENERAL, "Unable to parse line from GEOIP %s file: %s",
  212. family == AF_INET ? "IPv4" : "IPv6", escaped(line));
  213. return -1;
  214. }
  215. /** Sorting helper: return -1, 1, or 0 based on comparison of two
  216. * geoip_ipv4_entry_t */
  217. static int
  218. geoip_ipv4_compare_entries_(const void **_a, const void **_b)
  219. {
  220. const geoip_ipv4_entry_t *a = *_a, *b = *_b;
  221. if (a->ip_low < b->ip_low)
  222. return -1;
  223. else if (a->ip_low > b->ip_low)
  224. return 1;
  225. else
  226. return 0;
  227. }
  228. /** bsearch helper: return -1, 1, or 0 based on comparison of an IP (a pointer
  229. * to a uint32_t in host order) to a geoip_ipv4_entry_t */
  230. static int
  231. geoip_ipv4_compare_key_to_entry_(const void *_key, const void **_member)
  232. {
  233. /* No alignment issue here, since _key really is a pointer to uint32_t */
  234. const uint32_t addr = *(uint32_t *)_key;
  235. const geoip_ipv4_entry_t *entry = *_member;
  236. if (addr < entry->ip_low)
  237. return -1;
  238. else if (addr > entry->ip_high)
  239. return 1;
  240. else
  241. return 0;
  242. }
  243. /** Sorting helper: return -1, 1, or 0 based on comparison of two
  244. * geoip_ipv6_entry_t */
  245. static int
  246. geoip_ipv6_compare_entries_(const void **_a, const void **_b)
  247. {
  248. const geoip_ipv6_entry_t *a = *_a, *b = *_b;
  249. return fast_memcmp(a->ip_low.s6_addr, b->ip_low.s6_addr,
  250. sizeof(struct in6_addr));
  251. }
  252. /** bsearch helper: return -1, 1, or 0 based on comparison of an IPv6
  253. * (a pointer to a in6_addr) to a geoip_ipv6_entry_t */
  254. static int
  255. geoip_ipv6_compare_key_to_entry_(const void *_key, const void **_member)
  256. {
  257. const struct in6_addr *addr = (struct in6_addr *)_key;
  258. const geoip_ipv6_entry_t *entry = *_member;
  259. if (fast_memcmp(addr->s6_addr, entry->ip_low.s6_addr,
  260. sizeof(struct in6_addr)) < 0)
  261. return -1;
  262. else if (fast_memcmp(addr->s6_addr, entry->ip_high.s6_addr,
  263. sizeof(struct in6_addr)) > 0)
  264. return 1;
  265. else
  266. return 0;
  267. }
  268. /** Return 1 if we should collect geoip stats on bridge users, and
  269. * include them in our extrainfo descriptor. Else return 0. */
  270. int
  271. should_record_bridge_info(const or_options_t *options)
  272. {
  273. return options->BridgeRelay && options->BridgeRecordUsageByCountry;
  274. }
  275. /** Set up a new list of geoip countries with no countries (yet) set in it,
  276. * except for the unknown country.
  277. */
  278. static void
  279. init_geoip_countries(void)
  280. {
  281. geoip_country_t *geoip_unresolved;
  282. geoip_countries = smartlist_new();
  283. /* Add a geoip_country_t for requests that could not be resolved to a
  284. * country as first element (index 0) to geoip_countries. */
  285. geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t));
  286. strlcpy(geoip_unresolved->countrycode, "??",
  287. sizeof(geoip_unresolved->countrycode));
  288. smartlist_add(geoip_countries, geoip_unresolved);
  289. country_idxplus1_by_lc_code = strmap_new();
  290. strmap_set_lc(country_idxplus1_by_lc_code, "??", (void*)(1));
  291. }
  292. /** Clear appropriate GeoIP database, based on <b>family</b>, and
  293. * reload it from the file <b>filename</b>. Return 0 on success, -1 on
  294. * failure.
  295. *
  296. * Recognized line formats for IPv4 are:
  297. * INTIPLOW,INTIPHIGH,CC
  298. * and
  299. * "INTIPLOW","INTIPHIGH","CC","CC3","COUNTRY NAME"
  300. * where INTIPLOW and INTIPHIGH are IPv4 addresses encoded as 4-byte unsigned
  301. * integers, and CC is a country code.
  302. *
  303. * Recognized line format for IPv6 is:
  304. * IPV6LOW,IPV6HIGH,CC
  305. * where IPV6LOW and IPV6HIGH are IPv6 addresses and CC is a country code.
  306. *
  307. * It also recognizes, and skips over, blank lines and lines that start
  308. * with '#' (comments).
  309. */
  310. int
  311. geoip_load_file(sa_family_t family, const char *filename)
  312. {
  313. FILE *f;
  314. const char *msg = "";
  315. const or_options_t *options = get_options();
  316. int severity = options_need_geoip_info(options, &msg) ? LOG_WARN : LOG_INFO;
  317. crypto_digest_t *geoip_digest_env = NULL;
  318. tor_assert(family == AF_INET || family == AF_INET6);
  319. if (!(f = tor_fopen_cloexec(filename, "r"))) {
  320. log_fn(severity, LD_GENERAL, "Failed to open GEOIP file %s. %s",
  321. filename, msg);
  322. return -1;
  323. }
  324. if (!geoip_countries)
  325. init_geoip_countries();
  326. if (family == AF_INET) {
  327. if (geoip_ipv4_entries) {
  328. SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, e,
  329. tor_free(e));
  330. smartlist_free(geoip_ipv4_entries);
  331. }
  332. geoip_ipv4_entries = smartlist_new();
  333. } else { /* AF_INET6 */
  334. if (geoip_ipv6_entries) {
  335. SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, e,
  336. tor_free(e));
  337. smartlist_free(geoip_ipv6_entries);
  338. }
  339. geoip_ipv6_entries = smartlist_new();
  340. }
  341. geoip_digest_env = crypto_digest_new();
  342. log_notice(LD_GENERAL, "Parsing GEOIP %s file %s.",
  343. (family == AF_INET) ? "IPv4" : "IPv6", filename);
  344. while (!feof(f)) {
  345. char buf[512];
  346. if (fgets(buf, (int)sizeof(buf), f) == NULL)
  347. break;
  348. crypto_digest_add_bytes(geoip_digest_env, buf, strlen(buf));
  349. /* FFFF track full country name. */
  350. geoip_parse_entry(buf, family);
  351. }
  352. /*XXXX abort and return -1 if no entries/illformed?*/
  353. fclose(f);
  354. /* Sort list and remember file digests so that we can include it in
  355. * our extra-info descriptors. */
  356. if (family == AF_INET) {
  357. smartlist_sort(geoip_ipv4_entries, geoip_ipv4_compare_entries_);
  358. /* Okay, now we need to maybe change our mind about what is in
  359. * which country. We do this for IPv4 only since that's what we
  360. * store in node->country. */
  361. refresh_all_country_info();
  362. crypto_digest_get_digest(geoip_digest_env, geoip_digest, DIGEST_LEN);
  363. } else {
  364. /* AF_INET6 */
  365. smartlist_sort(geoip_ipv6_entries, geoip_ipv6_compare_entries_);
  366. crypto_digest_get_digest(geoip_digest_env, geoip6_digest, DIGEST_LEN);
  367. }
  368. crypto_digest_free(geoip_digest_env);
  369. return 0;
  370. }
  371. /** Given an IP address in host order, return a number representing the
  372. * country to which that address belongs, -1 for "No geoip information
  373. * available", or 0 for the 'unknown country'. The return value will always
  374. * be less than geoip_get_n_countries(). To decode it, call
  375. * geoip_get_country_name().
  376. */
  377. STATIC int
  378. geoip_get_country_by_ipv4(uint32_t ipaddr)
  379. {
  380. geoip_ipv4_entry_t *ent;
  381. if (!geoip_ipv4_entries)
  382. return -1;
  383. ent = smartlist_bsearch(geoip_ipv4_entries, &ipaddr,
  384. geoip_ipv4_compare_key_to_entry_);
  385. return ent ? (int)ent->country : 0;
  386. }
  387. /** Given an IPv6 address, return a number representing the country to
  388. * which that address belongs, -1 for "No geoip information available", or
  389. * 0 for the 'unknown country'. The return value will always be less than
  390. * geoip_get_n_countries(). To decode it, call geoip_get_country_name().
  391. */
  392. STATIC int
  393. geoip_get_country_by_ipv6(const struct in6_addr *addr)
  394. {
  395. geoip_ipv6_entry_t *ent;
  396. if (!geoip_ipv6_entries)
  397. return -1;
  398. ent = smartlist_bsearch(geoip_ipv6_entries, addr,
  399. geoip_ipv6_compare_key_to_entry_);
  400. return ent ? (int)ent->country : 0;
  401. }
  402. /** Given an IP address, return a number representing the country to which
  403. * that address belongs, -1 for "No geoip information available", or 0 for
  404. * the 'unknown country'. The return value will always be less than
  405. * geoip_get_n_countries(). To decode it, call geoip_get_country_name().
  406. */
  407. MOCK_IMPL(int,
  408. geoip_get_country_by_addr,(const tor_addr_t *addr))
  409. {
  410. if (tor_addr_family(addr) == AF_INET) {
  411. return geoip_get_country_by_ipv4(tor_addr_to_ipv4h(addr));
  412. } else if (tor_addr_family(addr) == AF_INET6) {
  413. return geoip_get_country_by_ipv6(tor_addr_to_in6(addr));
  414. } else {
  415. return -1;
  416. }
  417. }
  418. /** Return the number of countries recognized by the GeoIP country list. */
  419. MOCK_IMPL(int,
  420. geoip_get_n_countries,(void))
  421. {
  422. if (!geoip_countries)
  423. init_geoip_countries();
  424. return (int) smartlist_len(geoip_countries);
  425. }
  426. /** Return the two-letter country code associated with the number <b>num</b>,
  427. * or "??" for an unknown value. */
  428. const char *
  429. geoip_get_country_name(country_t num)
  430. {
  431. if (geoip_countries && num >= 0 && num < smartlist_len(geoip_countries)) {
  432. geoip_country_t *c = smartlist_get(geoip_countries, num);
  433. return c->countrycode;
  434. } else
  435. return "??";
  436. }
  437. /** Return true iff we have loaded a GeoIP database.*/
  438. MOCK_IMPL(int,
  439. geoip_is_loaded,(sa_family_t family))
  440. {
  441. tor_assert(family == AF_INET || family == AF_INET6);
  442. if (geoip_countries == NULL)
  443. return 0;
  444. if (family == AF_INET)
  445. return geoip_ipv4_entries != NULL;
  446. else /* AF_INET6 */
  447. return geoip_ipv6_entries != NULL;
  448. }
  449. /** Return the hex-encoded SHA1 digest of the loaded GeoIP file. The
  450. * result does not need to be deallocated, but will be overwritten by the
  451. * next call of hex_str(). */
  452. const char *
  453. geoip_db_digest(sa_family_t family)
  454. {
  455. tor_assert(family == AF_INET || family == AF_INET6);
  456. if (family == AF_INET)
  457. return hex_str(geoip_digest, DIGEST_LEN);
  458. else /* AF_INET6 */
  459. return hex_str(geoip6_digest, DIGEST_LEN);
  460. }
  461. /** Largest allowable value for last_seen_in_minutes. (It's a 30-bit field,
  462. * so it can hold up to (1u<<30)-1, or 0x3fffffffu.
  463. */
  464. #define MAX_LAST_SEEN_IN_MINUTES 0X3FFFFFFFu
  465. /** Map from client IP address to last time seen. */
  466. static HT_HEAD(clientmap, clientmap_entry_t) client_history =
  467. HT_INITIALIZER();
  468. /** Hashtable helper: compute a hash of a clientmap_entry_t. */
  469. static inline unsigned
  470. clientmap_entry_hash(const clientmap_entry_t *a)
  471. {
  472. unsigned h = (unsigned) tor_addr_hash(&a->addr);
  473. if (a->transport_name)
  474. h += (unsigned) siphash24g(a->transport_name, strlen(a->transport_name));
  475. return h;
  476. }
  477. /** Hashtable helper: compare two clientmap_entry_t values for equality. */
  478. static inline int
  479. clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b)
  480. {
  481. if (strcmp_opt(a->transport_name, b->transport_name))
  482. return 0;
  483. return !tor_addr_compare(&a->addr, &b->addr, CMP_EXACT) &&
  484. a->action == b->action;
  485. }
  486. HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
  487. clientmap_entries_eq)
  488. HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
  489. clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
  490. /** Return the size of a client map entry. */
  491. static inline size_t
  492. clientmap_entry_size(const clientmap_entry_t *ent)
  493. {
  494. tor_assert(ent);
  495. return (sizeof(clientmap_entry_t) +
  496. (ent->transport_name ? strlen(ent->transport_name) : 0));
  497. }
  498. /** Free all storage held by <b>ent</b>. */
  499. static void
  500. clientmap_entry_free(clientmap_entry_t *ent)
  501. {
  502. if (!ent)
  503. return;
  504. /* This entry is about to be freed so pass it to the DoS subsystem to see if
  505. * any actions can be taken about it. */
  506. dos_geoip_entry_about_to_free(ent);
  507. geoip_decrement_client_history_cache_size(clientmap_entry_size(ent));
  508. tor_free(ent->transport_name);
  509. tor_free(ent);
  510. }
  511. /* Return a newly allocated clientmap entry with the given action and address
  512. * that are mandatory. The transport_name can be optional. This can't fail. */
  513. static clientmap_entry_t *
  514. clientmap_entry_new(geoip_client_action_t action, const tor_addr_t *addr,
  515. const char *transport_name)
  516. {
  517. clientmap_entry_t *entry;
  518. tor_assert(action == GEOIP_CLIENT_CONNECT ||
  519. action == GEOIP_CLIENT_NETWORKSTATUS);
  520. tor_assert(addr);
  521. entry = tor_malloc_zero(sizeof(clientmap_entry_t));
  522. entry->action = action;
  523. tor_addr_copy(&entry->addr, addr);
  524. if (transport_name) {
  525. entry->transport_name = tor_strdup(transport_name);
  526. }
  527. /* Allocated and initialized, note down its size for the OOM handler. */
  528. geoip_increment_client_history_cache_size(clientmap_entry_size(entry));
  529. return entry;
  530. }
  531. /** Clear history of connecting clients used by entry and bridge stats. */
  532. static void
  533. client_history_clear(void)
  534. {
  535. clientmap_entry_t **ent, **next, *this;
  536. for (ent = HT_START(clientmap, &client_history); ent != NULL;
  537. ent = next) {
  538. if ((*ent)->action == GEOIP_CLIENT_CONNECT) {
  539. this = *ent;
  540. next = HT_NEXT_RMV(clientmap, &client_history, ent);
  541. clientmap_entry_free(this);
  542. } else {
  543. next = HT_NEXT(clientmap, &client_history, ent);
  544. }
  545. }
  546. }
  547. /** Note that we've seen a client connect from the IP <b>addr</b>
  548. * at time <b>now</b>. Ignored by all but bridges and directories if
  549. * configured accordingly. */
  550. void
  551. geoip_note_client_seen(geoip_client_action_t action,
  552. const tor_addr_t *addr,
  553. const char *transport_name,
  554. time_t now)
  555. {
  556. const or_options_t *options = get_options();
  557. clientmap_entry_t *ent;
  558. if (action == GEOIP_CLIENT_CONNECT) {
  559. /* Only remember statistics if the DoS mitigation subsystem is enabled. If
  560. * not, only if as entry guard or as bridge. */
  561. if (!dos_enabled()) {
  562. if (!options->EntryStatistics &&
  563. (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))) {
  564. return;
  565. }
  566. }
  567. } else {
  568. /* Only gather directory-request statistics if configured, and
  569. * forcibly disable them on bridge authorities. */
  570. if (!options->DirReqStatistics || options->BridgeAuthoritativeDir)
  571. return;
  572. }
  573. log_debug(LD_GENERAL, "Seen client from '%s' with transport '%s'.",
  574. safe_str_client(fmt_addr((addr))),
  575. transport_name ? transport_name : "<no transport>");
  576. ent = geoip_lookup_client(addr, transport_name, action);
  577. if (! ent) {
  578. ent = clientmap_entry_new(action, addr, transport_name);
  579. HT_INSERT(clientmap, &client_history, ent);
  580. }
  581. if (now / 60 <= (int)MAX_LAST_SEEN_IN_MINUTES && now >= 0)
  582. ent->last_seen_in_minutes = (unsigned)(now/60);
  583. else
  584. ent->last_seen_in_minutes = 0;
  585. if (action == GEOIP_CLIENT_NETWORKSTATUS) {
  586. int country_idx = geoip_get_country_by_addr(addr);
  587. if (country_idx < 0)
  588. country_idx = 0; /** unresolved requests are stored at index 0. */
  589. if (country_idx >= 0 && country_idx < smartlist_len(geoip_countries)) {
  590. geoip_country_t *country = smartlist_get(geoip_countries, country_idx);
  591. ++country->n_v3_ns_requests;
  592. }
  593. }
  594. }
  595. /** HT_FOREACH helper: remove a clientmap_entry_t from the hashtable if it's
  596. * older than a certain time. */
  597. static int
  598. remove_old_client_helper_(struct clientmap_entry_t *ent, void *_cutoff)
  599. {
  600. time_t cutoff = *(time_t*)_cutoff / 60;
  601. if (ent->last_seen_in_minutes < cutoff) {
  602. clientmap_entry_free(ent);
  603. return 1;
  604. } else {
  605. return 0;
  606. }
  607. }
  608. /** Forget about all clients that haven't connected since <b>cutoff</b>. */
  609. void
  610. geoip_remove_old_clients(time_t cutoff)
  611. {
  612. clientmap_HT_FOREACH_FN(&client_history,
  613. remove_old_client_helper_,
  614. &cutoff);
  615. }
  616. /* Return a client entry object matching the given address, transport name and
  617. * geoip action from the clientmap. NULL if not found. The transport_name can
  618. * be NULL. */
  619. clientmap_entry_t *
  620. geoip_lookup_client(const tor_addr_t *addr, const char *transport_name,
  621. geoip_client_action_t action)
  622. {
  623. clientmap_entry_t lookup;
  624. tor_assert(addr);
  625. /* We always look for a client connection with no transport. */
  626. tor_addr_copy(&lookup.addr, addr);
  627. lookup.action = action;
  628. lookup.transport_name = (char *) transport_name;
  629. return HT_FIND(clientmap, &client_history, &lookup);
  630. }
  631. /* Cleanup client entries older than the cutoff. Used for the OOM. Return the
  632. * number of bytes freed. If 0 is returned, nothing was freed. */
  633. static size_t
  634. oom_clean_client_entries(time_t cutoff)
  635. {
  636. size_t bytes = 0;
  637. clientmap_entry_t **ent, **ent_next;
  638. for (ent = HT_START(clientmap, &client_history); ent; ent = ent_next) {
  639. clientmap_entry_t *entry = *ent;
  640. if (entry->last_seen_in_minutes < (cutoff / 60)) {
  641. ent_next = HT_NEXT_RMV(clientmap, &client_history, ent);
  642. bytes += clientmap_entry_size(entry);
  643. clientmap_entry_free(entry);
  644. } else {
  645. ent_next = HT_NEXT(clientmap, &client_history, ent);
  646. }
  647. }
  648. return bytes;
  649. }
  650. /* Below this minimum lifetime, the OOM won't cleanup any entries. */
  651. #define GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF (4 * 60 * 60)
  652. /* The OOM moves the cutoff by that much every run. */
  653. #define GEOIP_CLIENT_CACHE_OOM_STEP (15 * 50)
  654. /* Cleanup the geoip client history cache called from the OOM handler. Return
  655. * the amount of bytes removed. This can return a value below or above
  656. * min_remove_bytes but will stop as oon as the min_remove_bytes has been
  657. * reached. */
  658. size_t
  659. geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes)
  660. {
  661. time_t k;
  662. size_t bytes_removed = 0;
  663. /* Our OOM handler called with 0 bytes to remove is a code flow error. */
  664. tor_assert(min_remove_bytes != 0);
  665. /* Set k to the initial cutoff of an entry. We then going to move it by step
  666. * to try to remove as much as we can. */
  667. k = WRITE_STATS_INTERVAL;
  668. do {
  669. time_t cutoff;
  670. /* If k has reached the minimum lifetime, we have to stop else we might
  671. * remove every single entries which would be pretty bad for the DoS
  672. * mitigation subsystem if by just filling the geoip cache, it was enough
  673. * to trigger the OOM and clean every single entries. */
  674. if (k <= GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF) {
  675. break;
  676. }
  677. cutoff = now - k;
  678. bytes_removed += oom_clean_client_entries(cutoff);
  679. k -= GEOIP_CLIENT_CACHE_OOM_STEP;
  680. } while (bytes_removed < min_remove_bytes);
  681. return bytes_removed;
  682. }
  683. /* Return the total size in bytes of the client history cache. */
  684. size_t
  685. geoip_client_cache_total_allocation(void)
  686. {
  687. return geoip_client_history_cache_size;
  688. }
  689. /** How many responses are we giving to clients requesting v3 network
  690. * statuses? */
  691. static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM];
  692. /** Note that we've rejected a client's request for a v3 network status
  693. * for reason <b>reason</b> at time <b>now</b>. */
  694. void
  695. geoip_note_ns_response(geoip_ns_response_t response)
  696. {
  697. static int arrays_initialized = 0;
  698. if (!get_options()->DirReqStatistics)
  699. return;
  700. if (!arrays_initialized) {
  701. memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
  702. arrays_initialized = 1;
  703. }
  704. tor_assert(response < GEOIP_NS_RESPONSE_NUM);
  705. ns_v3_responses[response]++;
  706. }
  707. /** Do not mention any country from which fewer than this number of IPs have
  708. * connected. This conceivably avoids reporting information that could
  709. * deanonymize users, though analysis is lacking. */
  710. #define MIN_IPS_TO_NOTE_COUNTRY 1
  711. /** Do not report any geoip data at all if we have fewer than this number of
  712. * IPs to report about. */
  713. #define MIN_IPS_TO_NOTE_ANYTHING 1
  714. /** When reporting geoip data about countries, round up to the nearest
  715. * multiple of this value. */
  716. #define IP_GRANULARITY 8
  717. /** Helper type: used to sort per-country totals by value. */
  718. typedef struct c_hist_t {
  719. char country[3]; /**< Two-letter country code. */
  720. unsigned total; /**< Total IP addresses seen in this country. */
  721. } c_hist_t;
  722. /** Sorting helper: return -1, 1, or 0 based on comparison of two
  723. * geoip_ipv4_entry_t. Sort in descending order of total, and then by country
  724. * code. */
  725. static int
  726. c_hist_compare_(const void **_a, const void **_b)
  727. {
  728. const c_hist_t *a = *_a, *b = *_b;
  729. if (a->total > b->total)
  730. return -1;
  731. else if (a->total < b->total)
  732. return 1;
  733. else
  734. return strcmp(a->country, b->country);
  735. }
  736. /** When there are incomplete directory requests at the end of a 24-hour
  737. * period, consider those requests running for longer than this timeout as
  738. * failed, the others as still running. */
  739. #define DIRREQ_TIMEOUT (10*60)
  740. /** Entry in a map from either chan->global_identifier for direct requests
  741. * or a unique circuit identifier for tunneled requests to request time,
  742. * response size, and completion time of a network status request. Used to
  743. * measure download times of requests to derive average client
  744. * bandwidths. */
  745. typedef struct dirreq_map_entry_t {
  746. HT_ENTRY(dirreq_map_entry_t) node;
  747. /** Unique identifier for this network status request; this is either the
  748. * chan->global_identifier of the dir channel (direct request) or a new
  749. * locally unique identifier of a circuit (tunneled request). This ID is
  750. * only unique among other direct or tunneled requests, respectively. */
  751. uint64_t dirreq_id;
  752. unsigned int state:3; /**< State of this directory request. */
  753. unsigned int type:1; /**< Is this a direct or a tunneled request? */
  754. unsigned int completed:1; /**< Is this request complete? */
  755. /** When did we receive the request and started sending the response? */
  756. struct timeval request_time;
  757. size_t response_size; /**< What is the size of the response in bytes? */
  758. struct timeval completion_time; /**< When did the request succeed? */
  759. } dirreq_map_entry_t;
  760. /** Map of all directory requests asking for v2 or v3 network statuses in
  761. * the current geoip-stats interval. Values are
  762. * of type *<b>dirreq_map_entry_t</b>. */
  763. static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map =
  764. HT_INITIALIZER();
  765. static int
  766. dirreq_map_ent_eq(const dirreq_map_entry_t *a,
  767. const dirreq_map_entry_t *b)
  768. {
  769. return a->dirreq_id == b->dirreq_id && a->type == b->type;
  770. }
  771. /* DOCDOC dirreq_map_ent_hash */
  772. static unsigned
  773. dirreq_map_ent_hash(const dirreq_map_entry_t *entry)
  774. {
  775. unsigned u = (unsigned) entry->dirreq_id;
  776. u += entry->type << 20;
  777. return u;
  778. }
  779. HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
  780. dirreq_map_ent_eq)
  781. HT_GENERATE2(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
  782. dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_)
  783. /** Helper: Put <b>entry</b> into map of directory requests using
  784. * <b>type</b> and <b>dirreq_id</b> as key parts. If there is
  785. * already an entry for that key, print out a BUG warning and return. */
  786. static void
  787. dirreq_map_put_(dirreq_map_entry_t *entry, dirreq_type_t type,
  788. uint64_t dirreq_id)
  789. {
  790. dirreq_map_entry_t *old_ent;
  791. tor_assert(entry->type == type);
  792. tor_assert(entry->dirreq_id == dirreq_id);
  793. /* XXXX we could switch this to HT_INSERT some time, since it seems that
  794. * this bug doesn't happen. But since this function doesn't seem to be
  795. * critical-path, it's sane to leave it alone. */
  796. old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry);
  797. if (old_ent && old_ent != entry) {
  798. log_warn(LD_BUG, "Error when putting directory request into local "
  799. "map. There was already an entry for the same identifier.");
  800. return;
  801. }
  802. }
  803. /** Helper: Look up and return an entry in the map of directory requests
  804. * using <b>type</b> and <b>dirreq_id</b> as key parts. If there
  805. * is no such entry, return NULL. */
  806. static dirreq_map_entry_t *
  807. dirreq_map_get_(dirreq_type_t type, uint64_t dirreq_id)
  808. {
  809. dirreq_map_entry_t lookup;
  810. lookup.type = type;
  811. lookup.dirreq_id = dirreq_id;
  812. return HT_FIND(dirreqmap, &dirreq_map, &lookup);
  813. }
  814. /** Note that an either direct or tunneled (see <b>type</b>) directory
  815. * request for a v3 network status with unique ID <b>dirreq_id</b> of size
  816. * <b>response_size</b> has started. */
  817. void
  818. geoip_start_dirreq(uint64_t dirreq_id, size_t response_size,
  819. dirreq_type_t type)
  820. {
  821. dirreq_map_entry_t *ent;
  822. if (!get_options()->DirReqStatistics)
  823. return;
  824. ent = tor_malloc_zero(sizeof(dirreq_map_entry_t));
  825. ent->dirreq_id = dirreq_id;
  826. tor_gettimeofday(&ent->request_time);
  827. ent->response_size = response_size;
  828. ent->type = type;
  829. dirreq_map_put_(ent, type, dirreq_id);
  830. }
  831. /** Change the state of the either direct or tunneled (see <b>type</b>)
  832. * directory request with <b>dirreq_id</b> to <b>new_state</b> and
  833. * possibly mark it as completed. If no entry can be found for the given
  834. * key parts (e.g., if this is a directory request that we are not
  835. * measuring, or one that was started in the previous measurement period),
  836. * or if the state cannot be advanced to <b>new_state</b>, do nothing. */
  837. void
  838. geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
  839. dirreq_state_t new_state)
  840. {
  841. dirreq_map_entry_t *ent;
  842. if (!get_options()->DirReqStatistics)
  843. return;
  844. ent = dirreq_map_get_(type, dirreq_id);
  845. if (!ent)
  846. return;
  847. if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS)
  848. return;
  849. if (new_state - 1 != ent->state)
  850. return;
  851. ent->state = new_state;
  852. if ((type == DIRREQ_DIRECT &&
  853. new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) ||
  854. (type == DIRREQ_TUNNELED &&
  855. new_state == DIRREQ_CHANNEL_BUFFER_FLUSHED)) {
  856. tor_gettimeofday(&ent->completion_time);
  857. ent->completed = 1;
  858. }
  859. }
  860. /** Return the bridge-ip-transports string that should be inserted in
  861. * our extra-info descriptor. Return NULL if the bridge-ip-transports
  862. * line should be empty. */
  863. char *
  864. geoip_get_transport_history(void)
  865. {
  866. unsigned granularity = IP_GRANULARITY;
  867. /** String hash table (name of transport) -> (number of users). */
  868. strmap_t *transport_counts = strmap_new();
  869. /** Smartlist that contains copies of the names of the transports
  870. that have been used. */
  871. smartlist_t *transports_used = smartlist_new();
  872. /* Special string to signify that no transport was used for this
  873. connection. Pluggable transport names can't have symbols in their
  874. names, so this string will never collide with a real transport. */
  875. static const char* no_transport_str = "<OR>";
  876. clientmap_entry_t **ent;
  877. smartlist_t *string_chunks = smartlist_new();
  878. char *the_string = NULL;
  879. /* If we haven't seen any clients yet, return NULL. */
  880. if (HT_EMPTY(&client_history))
  881. goto done;
  882. /** We do the following steps to form the transport history string:
  883. * a) Foreach client that uses a pluggable transport, we increase the
  884. * times that transport was used by one. If the client did not use
  885. * a transport, we increase the number of times someone connected
  886. * without obfuscation.
  887. * b) Foreach transport we observed, we write its transport history
  888. * string and push it to string_chunks. So, for example, if we've
  889. * seen 665 obfs2 clients, we write "obfs2=665".
  890. * c) We concatenate string_chunks to form the final string.
  891. */
  892. log_debug(LD_GENERAL,"Starting iteration for transport history. %d clients.",
  893. HT_SIZE(&client_history));
  894. /* Loop through all clients. */
  895. HT_FOREACH(ent, clientmap, &client_history) {
  896. uintptr_t val;
  897. void *ptr;
  898. const char *transport_name = (*ent)->transport_name;
  899. if (!transport_name)
  900. transport_name = no_transport_str;
  901. /* Increase the count for this transport name. */
  902. ptr = strmap_get(transport_counts, transport_name);
  903. val = (uintptr_t)ptr;
  904. val++;
  905. ptr = (void*)val;
  906. strmap_set(transport_counts, transport_name, ptr);
  907. /* If it's the first time we see this transport, note it. */
  908. if (val == 1)
  909. smartlist_add(transports_used, tor_strdup(transport_name));
  910. log_debug(LD_GENERAL, "Client from '%s' with transport '%s'. "
  911. "I've now seen %d clients.",
  912. safe_str_client(fmt_addr(&(*ent)->addr)),
  913. transport_name ? transport_name : "<no transport>",
  914. (int)val);
  915. }
  916. /* Sort the transport names (helps with unit testing). */
  917. smartlist_sort_strings(transports_used);
  918. /* Loop through all seen transports. */
  919. SMARTLIST_FOREACH_BEGIN(transports_used, const char *, transport_name) {
  920. void *transport_count_ptr = strmap_get(transport_counts, transport_name);
  921. uintptr_t transport_count = (uintptr_t) transport_count_ptr;
  922. log_debug(LD_GENERAL, "We got "U64_FORMAT" clients with transport '%s'.",
  923. U64_PRINTF_ARG((uint64_t)transport_count), transport_name);
  924. smartlist_add_asprintf(string_chunks, "%s="U64_FORMAT,
  925. transport_name,
  926. U64_PRINTF_ARG(round_uint64_to_next_multiple_of(
  927. (uint64_t)transport_count,
  928. granularity)));
  929. } SMARTLIST_FOREACH_END(transport_name);
  930. the_string = smartlist_join_strings(string_chunks, ",", 0, NULL);
  931. log_debug(LD_GENERAL, "Final bridge-ip-transports string: '%s'", the_string);
  932. done:
  933. strmap_free(transport_counts, NULL);
  934. SMARTLIST_FOREACH(transports_used, char *, s, tor_free(s));
  935. smartlist_free(transports_used);
  936. SMARTLIST_FOREACH(string_chunks, char *, s, tor_free(s));
  937. smartlist_free(string_chunks);
  938. return the_string;
  939. }
  940. /** Return a newly allocated comma-separated string containing statistics
  941. * on network status downloads. The string contains the number of completed
  942. * requests, timeouts, and still running requests as well as the download
  943. * times by deciles and quartiles. Return NULL if we have not observed
  944. * requests for long enough. */
  945. static char *
  946. geoip_get_dirreq_history(dirreq_type_t type)
  947. {
  948. char *result = NULL;
  949. smartlist_t *dirreq_completed = NULL;
  950. uint32_t complete = 0, timeouts = 0, running = 0;
  951. int bufsize = 1024, written;
  952. dirreq_map_entry_t **ptr, **next;
  953. struct timeval now;
  954. tor_gettimeofday(&now);
  955. dirreq_completed = smartlist_new();
  956. for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) {
  957. dirreq_map_entry_t *ent = *ptr;
  958. if (ent->type != type) {
  959. next = HT_NEXT(dirreqmap, &dirreq_map, ptr);
  960. continue;
  961. } else {
  962. if (ent->completed) {
  963. smartlist_add(dirreq_completed, ent);
  964. complete++;
  965. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
  966. } else {
  967. if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT)
  968. timeouts++;
  969. else
  970. running++;
  971. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
  972. tor_free(ent);
  973. }
  974. }
  975. }
  976. #define DIR_REQ_GRANULARITY 4
  977. complete = round_uint32_to_next_multiple_of(complete,
  978. DIR_REQ_GRANULARITY);
  979. timeouts = round_uint32_to_next_multiple_of(timeouts,
  980. DIR_REQ_GRANULARITY);
  981. running = round_uint32_to_next_multiple_of(running,
  982. DIR_REQ_GRANULARITY);
  983. result = tor_malloc_zero(bufsize);
  984. written = tor_snprintf(result, bufsize, "complete=%u,timeout=%u,"
  985. "running=%u", complete, timeouts, running);
  986. if (written < 0) {
  987. tor_free(result);
  988. goto done;
  989. }
  990. #define MIN_DIR_REQ_RESPONSES 16
  991. if (complete >= MIN_DIR_REQ_RESPONSES) {
  992. uint32_t *dltimes;
  993. /* We may have rounded 'completed' up. Here we want to use the
  994. * real value. */
  995. complete = smartlist_len(dirreq_completed);
  996. dltimes = tor_calloc(complete, sizeof(uint32_t));
  997. SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) {
  998. uint32_t bytes_per_second;
  999. uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time,
  1000. &ent->completion_time);
  1001. if (time_diff == 0)
  1002. time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible
  1003. * by law of nature or something, but a millisecond
  1004. * is a bit greater than "instantly" */
  1005. bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff);
  1006. dltimes[ent_sl_idx] = bytes_per_second;
  1007. } SMARTLIST_FOREACH_END(ent);
  1008. median_uint32(dltimes, complete); /* sorts as a side effect. */
  1009. written = tor_snprintf(result + written, bufsize - written,
  1010. ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u,"
  1011. "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u",
  1012. dltimes[0],
  1013. dltimes[1*complete/10-1],
  1014. dltimes[2*complete/10-1],
  1015. dltimes[1*complete/4-1],
  1016. dltimes[3*complete/10-1],
  1017. dltimes[4*complete/10-1],
  1018. dltimes[5*complete/10-1],
  1019. dltimes[6*complete/10-1],
  1020. dltimes[7*complete/10-1],
  1021. dltimes[3*complete/4-1],
  1022. dltimes[8*complete/10-1],
  1023. dltimes[9*complete/10-1],
  1024. dltimes[complete-1]);
  1025. if (written<0)
  1026. tor_free(result);
  1027. tor_free(dltimes);
  1028. }
  1029. done:
  1030. SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent,
  1031. tor_free(ent));
  1032. smartlist_free(dirreq_completed);
  1033. return result;
  1034. }
  1035. /** Store a newly allocated comma-separated string in
  1036. * *<a>country_str</a> containing entries for all the countries from
  1037. * which we've seen enough clients connect as a bridge, directory
  1038. * server, or entry guard. The entry format is cc=num where num is the
  1039. * number of IPs we've seen connecting from that country, and cc is a
  1040. * lowercased country code. *<a>country_str</a> is set to NULL if
  1041. * we're not ready to export per country data yet.
  1042. *
  1043. * Store a newly allocated comma-separated string in <a>ipver_str</a>
  1044. * containing entries for clients connecting over IPv4 and IPv6. The
  1045. * format is family=num where num is the nubmer of IPs we've seen
  1046. * connecting over that protocol family, and family is 'v4' or 'v6'.
  1047. *
  1048. * Return 0 on success and -1 if we're missing geoip data. */
  1049. int
  1050. geoip_get_client_history(geoip_client_action_t action,
  1051. char **country_str, char **ipver_str)
  1052. {
  1053. unsigned granularity = IP_GRANULARITY;
  1054. smartlist_t *entries = NULL;
  1055. int n_countries = geoip_get_n_countries();
  1056. int i;
  1057. clientmap_entry_t **cm_ent;
  1058. unsigned *counts = NULL;
  1059. unsigned total = 0;
  1060. unsigned ipv4_count = 0, ipv6_count = 0;
  1061. if (!geoip_is_loaded(AF_INET) && !geoip_is_loaded(AF_INET6))
  1062. return -1;
  1063. counts = tor_calloc(n_countries, sizeof(unsigned));
  1064. HT_FOREACH(cm_ent, clientmap, &client_history) {
  1065. int country;
  1066. if ((*cm_ent)->action != (int)action)
  1067. continue;
  1068. country = geoip_get_country_by_addr(&(*cm_ent)->addr);
  1069. if (country < 0)
  1070. country = 0; /** unresolved requests are stored at index 0. */
  1071. tor_assert(0 <= country && country < n_countries);
  1072. ++counts[country];
  1073. ++total;
  1074. switch (tor_addr_family(&(*cm_ent)->addr)) {
  1075. case AF_INET:
  1076. ipv4_count++;
  1077. break;
  1078. case AF_INET6:
  1079. ipv6_count++;
  1080. break;
  1081. }
  1082. }
  1083. if (ipver_str) {
  1084. smartlist_t *chunks = smartlist_new();
  1085. smartlist_add_asprintf(chunks, "v4=%u",
  1086. round_to_next_multiple_of(ipv4_count, granularity));
  1087. smartlist_add_asprintf(chunks, "v6=%u",
  1088. round_to_next_multiple_of(ipv6_count, granularity));
  1089. *ipver_str = smartlist_join_strings(chunks, ",", 0, NULL);
  1090. SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
  1091. smartlist_free(chunks);
  1092. }
  1093. /* Don't record per country data if we haven't seen enough IPs. */
  1094. if (total < MIN_IPS_TO_NOTE_ANYTHING) {
  1095. tor_free(counts);
  1096. if (country_str)
  1097. *country_str = NULL;
  1098. return 0;
  1099. }
  1100. /* Make a list of c_hist_t */
  1101. entries = smartlist_new();
  1102. for (i = 0; i < n_countries; ++i) {
  1103. unsigned c = counts[i];
  1104. const char *countrycode;
  1105. c_hist_t *ent;
  1106. /* Only report a country if it has a minimum number of IPs. */
  1107. if (c >= MIN_IPS_TO_NOTE_COUNTRY) {
  1108. c = round_to_next_multiple_of(c, granularity);
  1109. countrycode = geoip_get_country_name(i);
  1110. ent = tor_malloc(sizeof(c_hist_t));
  1111. strlcpy(ent->country, countrycode, sizeof(ent->country));
  1112. ent->total = c;
  1113. smartlist_add(entries, ent);
  1114. }
  1115. }
  1116. /* Sort entries. Note that we must do this _AFTER_ rounding, or else
  1117. * the sort order could leak info. */
  1118. smartlist_sort(entries, c_hist_compare_);
  1119. if (country_str) {
  1120. smartlist_t *chunks = smartlist_new();
  1121. SMARTLIST_FOREACH(entries, c_hist_t *, ch, {
  1122. smartlist_add_asprintf(chunks, "%s=%u", ch->country, ch->total);
  1123. });
  1124. *country_str = smartlist_join_strings(chunks, ",", 0, NULL);
  1125. SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
  1126. smartlist_free(chunks);
  1127. }
  1128. SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c));
  1129. smartlist_free(entries);
  1130. tor_free(counts);
  1131. return 0;
  1132. }
  1133. /** Return a newly allocated string holding the per-country request history
  1134. * for v3 network statuses in a format suitable for an extra-info document,
  1135. * or NULL on failure. */
  1136. char *
  1137. geoip_get_request_history(void)
  1138. {
  1139. smartlist_t *entries, *strings;
  1140. char *result;
  1141. unsigned granularity = IP_GRANULARITY;
  1142. if (!geoip_countries)
  1143. return NULL;
  1144. entries = smartlist_new();
  1145. SMARTLIST_FOREACH_BEGIN(geoip_countries, geoip_country_t *, c) {
  1146. uint32_t tot = 0;
  1147. c_hist_t *ent;
  1148. tot = c->n_v3_ns_requests;
  1149. if (!tot)
  1150. continue;
  1151. ent = tor_malloc_zero(sizeof(c_hist_t));
  1152. strlcpy(ent->country, c->countrycode, sizeof(ent->country));
  1153. ent->total = round_to_next_multiple_of(tot, granularity);
  1154. smartlist_add(entries, ent);
  1155. } SMARTLIST_FOREACH_END(c);
  1156. smartlist_sort(entries, c_hist_compare_);
  1157. strings = smartlist_new();
  1158. SMARTLIST_FOREACH(entries, c_hist_t *, ent, {
  1159. smartlist_add_asprintf(strings, "%s=%u", ent->country, ent->total);
  1160. });
  1161. result = smartlist_join_strings(strings, ",", 0, NULL);
  1162. SMARTLIST_FOREACH(strings, char *, cp, tor_free(cp));
  1163. SMARTLIST_FOREACH(entries, c_hist_t *, ent, tor_free(ent));
  1164. smartlist_free(strings);
  1165. smartlist_free(entries);
  1166. return result;
  1167. }
  1168. /** Start time of directory request stats or 0 if we're not collecting
  1169. * directory request statistics. */
  1170. static time_t start_of_dirreq_stats_interval;
  1171. /** Initialize directory request stats. */
  1172. void
  1173. geoip_dirreq_stats_init(time_t now)
  1174. {
  1175. start_of_dirreq_stats_interval = now;
  1176. }
  1177. /** Reset counters for dirreq stats. */
  1178. void
  1179. geoip_reset_dirreq_stats(time_t now)
  1180. {
  1181. SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, {
  1182. c->n_v3_ns_requests = 0;
  1183. });
  1184. {
  1185. clientmap_entry_t **ent, **next, *this;
  1186. for (ent = HT_START(clientmap, &client_history); ent != NULL;
  1187. ent = next) {
  1188. if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS) {
  1189. this = *ent;
  1190. next = HT_NEXT_RMV(clientmap, &client_history, ent);
  1191. clientmap_entry_free(this);
  1192. } else {
  1193. next = HT_NEXT(clientmap, &client_history, ent);
  1194. }
  1195. }
  1196. }
  1197. memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
  1198. {
  1199. dirreq_map_entry_t **ent, **next, *this;
  1200. for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
  1201. this = *ent;
  1202. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
  1203. tor_free(this);
  1204. }
  1205. }
  1206. start_of_dirreq_stats_interval = now;
  1207. }
  1208. /** Stop collecting directory request stats in a way that we can re-start
  1209. * doing so in geoip_dirreq_stats_init(). */
  1210. void
  1211. geoip_dirreq_stats_term(void)
  1212. {
  1213. geoip_reset_dirreq_stats(0);
  1214. }
  1215. /** Return a newly allocated string containing the dirreq statistics
  1216. * until <b>now</b>, or NULL if we're not collecting dirreq stats. Caller
  1217. * must ensure start_of_dirreq_stats_interval is in the past. */
  1218. char *
  1219. geoip_format_dirreq_stats(time_t now)
  1220. {
  1221. char t[ISO_TIME_LEN+1];
  1222. int i;
  1223. char *v3_ips_string = NULL, *v3_reqs_string = NULL,
  1224. *v3_direct_dl_string = NULL, *v3_tunneled_dl_string = NULL;
  1225. char *result = NULL;
  1226. if (!start_of_dirreq_stats_interval)
  1227. return NULL; /* Not initialized. */
  1228. tor_assert(now >= start_of_dirreq_stats_interval);
  1229. format_iso_time(t, now);
  1230. geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS, &v3_ips_string, NULL);
  1231. v3_reqs_string = geoip_get_request_history();
  1232. #define RESPONSE_GRANULARITY 8
  1233. for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) {
  1234. ns_v3_responses[i] = round_uint32_to_next_multiple_of(
  1235. ns_v3_responses[i], RESPONSE_GRANULARITY);
  1236. }
  1237. #undef RESPONSE_GRANULARITY
  1238. v3_direct_dl_string = geoip_get_dirreq_history(DIRREQ_DIRECT);
  1239. v3_tunneled_dl_string = geoip_get_dirreq_history(DIRREQ_TUNNELED);
  1240. /* Put everything together into a single string. */
  1241. tor_asprintf(&result, "dirreq-stats-end %s (%d s)\n"
  1242. "dirreq-v3-ips %s\n"
  1243. "dirreq-v3-reqs %s\n"
  1244. "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u,"
  1245. "not-found=%u,not-modified=%u,busy=%u\n"
  1246. "dirreq-v3-direct-dl %s\n"
  1247. "dirreq-v3-tunneled-dl %s\n",
  1248. t,
  1249. (unsigned) (now - start_of_dirreq_stats_interval),
  1250. v3_ips_string ? v3_ips_string : "",
  1251. v3_reqs_string ? v3_reqs_string : "",
  1252. ns_v3_responses[GEOIP_SUCCESS],
  1253. ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS],
  1254. ns_v3_responses[GEOIP_REJECT_UNAVAILABLE],
  1255. ns_v3_responses[GEOIP_REJECT_NOT_FOUND],
  1256. ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED],
  1257. ns_v3_responses[GEOIP_REJECT_BUSY],
  1258. v3_direct_dl_string ? v3_direct_dl_string : "",
  1259. v3_tunneled_dl_string ? v3_tunneled_dl_string : "");
  1260. /* Free partial strings. */
  1261. tor_free(v3_ips_string);
  1262. tor_free(v3_reqs_string);
  1263. tor_free(v3_direct_dl_string);
  1264. tor_free(v3_tunneled_dl_string);
  1265. return result;
  1266. }
  1267. /** If 24 hours have passed since the beginning of the current dirreq
  1268. * stats period, write dirreq stats to $DATADIR/stats/dirreq-stats
  1269. * (possibly overwriting an existing file) and reset counters. Return
  1270. * when we would next want to write dirreq stats or 0 if we never want to
  1271. * write. */
  1272. time_t
  1273. geoip_dirreq_stats_write(time_t now)
  1274. {
  1275. char *str = NULL;
  1276. if (!start_of_dirreq_stats_interval)
  1277. return 0; /* Not initialized. */
  1278. if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now)
  1279. goto done; /* Not ready to write. */
  1280. /* Discard all items in the client history that are too old. */
  1281. geoip_remove_old_clients(start_of_dirreq_stats_interval);
  1282. /* Generate history string .*/
  1283. str = geoip_format_dirreq_stats(now);
  1284. if (! str)
  1285. goto done;
  1286. /* Write dirreq-stats string to disk. */
  1287. if (!check_or_create_data_subdir("stats")) {
  1288. write_to_data_subdir("stats", "dirreq-stats", str, "dirreq statistics");
  1289. /* Reset measurement interval start. */
  1290. geoip_reset_dirreq_stats(now);
  1291. }
  1292. done:
  1293. tor_free(str);
  1294. return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL;
  1295. }
  1296. /** Start time of bridge stats or 0 if we're not collecting bridge
  1297. * statistics. */
  1298. static time_t start_of_bridge_stats_interval;
  1299. /** Initialize bridge stats. */
  1300. void
  1301. geoip_bridge_stats_init(time_t now)
  1302. {
  1303. start_of_bridge_stats_interval = now;
  1304. }
  1305. /** Stop collecting bridge stats in a way that we can re-start doing so in
  1306. * geoip_bridge_stats_init(). */
  1307. void
  1308. geoip_bridge_stats_term(void)
  1309. {
  1310. client_history_clear();
  1311. start_of_bridge_stats_interval = 0;
  1312. }
  1313. /** Validate a bridge statistics string as it would be written to a
  1314. * current extra-info descriptor. Return 1 if the string is valid and
  1315. * recent enough, or 0 otherwise. */
  1316. static int
  1317. validate_bridge_stats(const char *stats_str, time_t now)
  1318. {
  1319. char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1],
  1320. *eos;
  1321. const char *BRIDGE_STATS_END = "bridge-stats-end ";
  1322. const char *BRIDGE_IPS = "bridge-ips ";
  1323. const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n";
  1324. const char *BRIDGE_TRANSPORTS = "bridge-ip-transports ";
  1325. const char *BRIDGE_TRANSPORTS_EMPTY_LINE = "bridge-ip-transports\n";
  1326. const char *tmp;
  1327. time_t stats_end_time;
  1328. int seconds;
  1329. tor_assert(stats_str);
  1330. /* Parse timestamp and number of seconds from
  1331. "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */
  1332. tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END);
  1333. if (!tmp)
  1334. return 0;
  1335. tmp += strlen(BRIDGE_STATS_END);
  1336. if (strlen(tmp) < ISO_TIME_LEN + 6)
  1337. return 0;
  1338. strlcpy(stats_end_str, tmp, sizeof(stats_end_str));
  1339. if (parse_iso_time(stats_end_str, &stats_end_time) < 0)
  1340. return 0;
  1341. if (stats_end_time < now - (25*60*60) ||
  1342. stats_end_time > now + (1*60*60))
  1343. return 0;
  1344. seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10);
  1345. if (!eos || seconds < 23*60*60)
  1346. return 0;
  1347. format_iso_time(stats_start_str, stats_end_time - seconds);
  1348. /* Parse: "bridge-ips CC=N,CC=N,..." */
  1349. tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS);
  1350. if (!tmp) {
  1351. /* Look if there is an empty "bridge-ips" line */
  1352. tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE);
  1353. if (!tmp)
  1354. return 0;
  1355. }
  1356. /* Parse: "bridge-ip-transports PT=N,PT=N,..." */
  1357. tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS);
  1358. if (!tmp) {
  1359. /* Look if there is an empty "bridge-ip-transports" line */
  1360. tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS_EMPTY_LINE);
  1361. if (!tmp)
  1362. return 0;
  1363. }
  1364. return 1;
  1365. }
  1366. /** Most recent bridge statistics formatted to be written to extra-info
  1367. * descriptors. */
  1368. static char *bridge_stats_extrainfo = NULL;
  1369. /** Return a newly allocated string holding our bridge usage stats by country
  1370. * in a format suitable for inclusion in an extrainfo document. Return NULL on
  1371. * failure. */
  1372. char *
  1373. geoip_format_bridge_stats(time_t now)
  1374. {
  1375. char *out = NULL;
  1376. char *country_data = NULL, *ipver_data = NULL, *transport_data = NULL;
  1377. long duration = now - start_of_bridge_stats_interval;
  1378. char written[ISO_TIME_LEN+1];
  1379. if (duration < 0)
  1380. return NULL;
  1381. if (!start_of_bridge_stats_interval)
  1382. return NULL; /* Not initialized. */
  1383. format_iso_time(written, now);
  1384. geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
  1385. transport_data = geoip_get_transport_history();
  1386. tor_asprintf(&out,
  1387. "bridge-stats-end %s (%ld s)\n"
  1388. "bridge-ips %s\n"
  1389. "bridge-ip-versions %s\n"
  1390. "bridge-ip-transports %s\n",
  1391. written, duration,
  1392. country_data ? country_data : "",
  1393. ipver_data ? ipver_data : "",
  1394. transport_data ? transport_data : "");
  1395. tor_free(country_data);
  1396. tor_free(ipver_data);
  1397. tor_free(transport_data);
  1398. return out;
  1399. }
  1400. /** Return a newly allocated string holding our bridge usage stats by country
  1401. * in a format suitable for the answer to a controller request. Return NULL on
  1402. * failure. */
  1403. static char *
  1404. format_bridge_stats_controller(time_t now)
  1405. {
  1406. char *out = NULL, *country_data = NULL, *ipver_data = NULL;
  1407. char started[ISO_TIME_LEN+1];
  1408. (void) now;
  1409. format_iso_time(started, start_of_bridge_stats_interval);
  1410. geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
  1411. tor_asprintf(&out,
  1412. "TimeStarted=\"%s\" CountrySummary=%s IPVersions=%s",
  1413. started,
  1414. country_data ? country_data : "",
  1415. ipver_data ? ipver_data : "");
  1416. tor_free(country_data);
  1417. tor_free(ipver_data);
  1418. return out;
  1419. }
  1420. /** Return a newly allocated string holding our bridge usage stats by
  1421. * country in a format suitable for inclusion in our heartbeat
  1422. * message. Return NULL on failure. */
  1423. char *
  1424. format_client_stats_heartbeat(time_t now)
  1425. {
  1426. const int n_hours = 6;
  1427. char *out = NULL;
  1428. int n_clients = 0;
  1429. clientmap_entry_t **ent;
  1430. unsigned cutoff = (unsigned)( (now-n_hours*3600)/60 );
  1431. if (!start_of_bridge_stats_interval)
  1432. return NULL; /* Not initialized. */
  1433. /* count unique IPs */
  1434. HT_FOREACH(ent, clientmap, &client_history) {
  1435. /* only count directly connecting clients */
  1436. if ((*ent)->action != GEOIP_CLIENT_CONNECT)
  1437. continue;
  1438. if ((*ent)->last_seen_in_minutes < cutoff)
  1439. continue;
  1440. n_clients++;
  1441. }
  1442. tor_asprintf(&out, "Heartbeat: "
  1443. "In the last %d hours, I have seen %d unique clients.",
  1444. n_hours,
  1445. n_clients);
  1446. return out;
  1447. }
  1448. /** Write bridge statistics to $DATADIR/stats/bridge-stats and return
  1449. * when we should next try to write statistics. */
  1450. time_t
  1451. geoip_bridge_stats_write(time_t now)
  1452. {
  1453. char *val = NULL;
  1454. /* Check if 24 hours have passed since starting measurements. */
  1455. if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL)
  1456. return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
  1457. /* Discard all items in the client history that are too old. */
  1458. geoip_remove_old_clients(start_of_bridge_stats_interval);
  1459. /* Generate formatted string */
  1460. val = geoip_format_bridge_stats(now);
  1461. if (val == NULL)
  1462. goto done;
  1463. /* Update the stored value. */
  1464. tor_free(bridge_stats_extrainfo);
  1465. bridge_stats_extrainfo = val;
  1466. start_of_bridge_stats_interval = now;
  1467. /* Write it to disk. */
  1468. if (!check_or_create_data_subdir("stats")) {
  1469. write_to_data_subdir("stats", "bridge-stats",
  1470. bridge_stats_extrainfo, "bridge statistics");
  1471. /* Tell the controller, "hey, there are clients!" */
  1472. {
  1473. char *controller_str = format_bridge_stats_controller(now);
  1474. if (controller_str)
  1475. control_event_clients_seen(controller_str);
  1476. tor_free(controller_str);
  1477. }
  1478. }
  1479. done:
  1480. return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
  1481. }
  1482. /** Try to load the most recent bridge statistics from disk, unless we
  1483. * have finished a measurement interval lately, and check whether they
  1484. * are still recent enough. */
  1485. static void
  1486. load_bridge_stats(time_t now)
  1487. {
  1488. char *fname, *contents;
  1489. if (bridge_stats_extrainfo)
  1490. return;
  1491. fname = get_datadir_fname2("stats", "bridge-stats");
  1492. contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
  1493. if (contents && validate_bridge_stats(contents, now)) {
  1494. bridge_stats_extrainfo = contents;
  1495. } else {
  1496. tor_free(contents);
  1497. }
  1498. tor_free(fname);
  1499. }
  1500. /** Return most recent bridge statistics for inclusion in extra-info
  1501. * descriptors, or NULL if we don't have recent bridge statistics. */
  1502. const char *
  1503. geoip_get_bridge_stats_extrainfo(time_t now)
  1504. {
  1505. load_bridge_stats(now);
  1506. return bridge_stats_extrainfo;
  1507. }
  1508. /** Return a new string containing the recent bridge statistics to be returned
  1509. * to controller clients, or NULL if we don't have any bridge statistics. */
  1510. char *
  1511. geoip_get_bridge_stats_controller(time_t now)
  1512. {
  1513. return format_bridge_stats_controller(now);
  1514. }
  1515. /** Start time of entry stats or 0 if we're not collecting entry
  1516. * statistics. */
  1517. static time_t start_of_entry_stats_interval;
  1518. /** Initialize entry stats. */
  1519. void
  1520. geoip_entry_stats_init(time_t now)
  1521. {
  1522. start_of_entry_stats_interval = now;
  1523. }
  1524. /** Reset counters for entry stats. */
  1525. void
  1526. geoip_reset_entry_stats(time_t now)
  1527. {
  1528. client_history_clear();
  1529. start_of_entry_stats_interval = now;
  1530. }
  1531. /** Stop collecting entry stats in a way that we can re-start doing so in
  1532. * geoip_entry_stats_init(). */
  1533. void
  1534. geoip_entry_stats_term(void)
  1535. {
  1536. geoip_reset_entry_stats(0);
  1537. }
  1538. /** Return a newly allocated string containing the entry statistics
  1539. * until <b>now</b>, or NULL if we're not collecting entry stats. Caller
  1540. * must ensure start_of_entry_stats_interval lies in the past. */
  1541. char *
  1542. geoip_format_entry_stats(time_t now)
  1543. {
  1544. char t[ISO_TIME_LEN+1];
  1545. char *data = NULL;
  1546. char *result;
  1547. if (!start_of_entry_stats_interval)
  1548. return NULL; /* Not initialized. */
  1549. tor_assert(now >= start_of_entry_stats_interval);
  1550. geoip_get_client_history(GEOIP_CLIENT_CONNECT, &data, NULL);
  1551. format_iso_time(t, now);
  1552. tor_asprintf(&result,
  1553. "entry-stats-end %s (%u s)\n"
  1554. "entry-ips %s\n",
  1555. t, (unsigned) (now - start_of_entry_stats_interval),
  1556. data ? data : "");
  1557. tor_free(data);
  1558. return result;
  1559. }
  1560. /** If 24 hours have passed since the beginning of the current entry stats
  1561. * period, write entry stats to $DATADIR/stats/entry-stats (possibly
  1562. * overwriting an existing file) and reset counters. Return when we would
  1563. * next want to write entry stats or 0 if we never want to write. */
  1564. time_t
  1565. geoip_entry_stats_write(time_t now)
  1566. {
  1567. char *str = NULL;
  1568. if (!start_of_entry_stats_interval)
  1569. return 0; /* Not initialized. */
  1570. if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now)
  1571. goto done; /* Not ready to write. */
  1572. /* Discard all items in the client history that are too old. */
  1573. geoip_remove_old_clients(start_of_entry_stats_interval);
  1574. /* Generate history string .*/
  1575. str = geoip_format_entry_stats(now);
  1576. /* Write entry-stats string to disk. */
  1577. if (!check_or_create_data_subdir("stats")) {
  1578. write_to_data_subdir("stats", "entry-stats", str, "entry statistics");
  1579. /* Reset measurement interval start. */
  1580. geoip_reset_entry_stats(now);
  1581. }
  1582. done:
  1583. tor_free(str);
  1584. return start_of_entry_stats_interval + WRITE_STATS_INTERVAL;
  1585. }
  1586. /** Helper used to implement GETINFO ip-to-country/... controller command. */
  1587. int
  1588. getinfo_helper_geoip(control_connection_t *control_conn,
  1589. const char *question, char **answer,
  1590. const char **errmsg)
  1591. {
  1592. (void)control_conn;
  1593. if (!strcmpstart(question, "ip-to-country/")) {
  1594. int c;
  1595. sa_family_t family;
  1596. tor_addr_t addr;
  1597. question += strlen("ip-to-country/");
  1598. family = tor_addr_parse(&addr, question);
  1599. if (family != AF_INET && family != AF_INET6) {
  1600. *errmsg = "Invalid address family";
  1601. return -1;
  1602. }
  1603. if (!geoip_is_loaded(family)) {
  1604. *errmsg = "GeoIP data not loaded";
  1605. return -1;
  1606. }
  1607. if (family == AF_INET)
  1608. c = geoip_get_country_by_ipv4(tor_addr_to_ipv4h(&addr));
  1609. else /* AF_INET6 */
  1610. c = geoip_get_country_by_ipv6(tor_addr_to_in6(&addr));
  1611. *answer = tor_strdup(geoip_get_country_name(c));
  1612. }
  1613. return 0;
  1614. }
  1615. /** Release all storage held by the GeoIP databases and country list. */
  1616. STATIC void
  1617. clear_geoip_db(void)
  1618. {
  1619. if (geoip_countries) {
  1620. SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, tor_free(c));
  1621. smartlist_free(geoip_countries);
  1622. }
  1623. strmap_free(country_idxplus1_by_lc_code, NULL);
  1624. if (geoip_ipv4_entries) {
  1625. SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, ent,
  1626. tor_free(ent));
  1627. smartlist_free(geoip_ipv4_entries);
  1628. }
  1629. if (geoip_ipv6_entries) {
  1630. SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, ent,
  1631. tor_free(ent));
  1632. smartlist_free(geoip_ipv6_entries);
  1633. }
  1634. geoip_countries = NULL;
  1635. country_idxplus1_by_lc_code = NULL;
  1636. geoip_ipv4_entries = NULL;
  1637. geoip_ipv6_entries = NULL;
  1638. }
  1639. /** Release all storage held in this file. */
  1640. void
  1641. geoip_free_all(void)
  1642. {
  1643. {
  1644. clientmap_entry_t **ent, **next, *this;
  1645. for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) {
  1646. this = *ent;
  1647. next = HT_NEXT_RMV(clientmap, &client_history, ent);
  1648. clientmap_entry_free(this);
  1649. }
  1650. HT_CLEAR(clientmap, &client_history);
  1651. }
  1652. {
  1653. dirreq_map_entry_t **ent, **next, *this;
  1654. for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
  1655. this = *ent;
  1656. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
  1657. tor_free(this);
  1658. }
  1659. HT_CLEAR(dirreqmap, &dirreq_map);
  1660. }
  1661. clear_geoip_db();
  1662. tor_free(bridge_stats_extrainfo);
  1663. }