geoip.c 61 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885
  1. /* Copyright (c) 2007-2017, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. /**
  4. * \file geoip.c
  5. * \brief Functions related to maintaining an IP-to-country database;
  6. * to summarizing client connections by country to entry guards, bridges,
  7. * and directory servers; and for statistics on answering network status
  8. * requests.
  9. *
  10. * There are two main kinds of functions in this module: geoip functions,
  11. * which map groups of IPv4 and IPv6 addresses to country codes, and
  12. * statistical functions, which collect statistics about different kinds of
  13. * per-country usage.
  14. *
  15. * The geoip lookup tables are implemented as sorted lists of disjoint address
  16. * ranges, each mapping to a singleton geoip_country_t. These country objects
  17. * are also indexed by their names in a hashtable.
  18. *
  19. * The tables are populated from disk at startup by the geoip_load_file()
  20. * function. For more information on the file format they read, see that
  21. * function. See the scripts and the README file in src/config for more
  22. * information about how those files are generated.
  23. *
  24. * Tor uses GeoIP information in order to implement user requests (such as
  25. * ExcludeNodes {cc}), and to keep track of how much usage relays are getting
  26. * for each country.
  27. */
  28. #define GEOIP_PRIVATE
  29. #include "or.h"
  30. #include "ht.h"
  31. #include "buffers.h"
  32. #include "config.h"
  33. #include "control.h"
  34. #include "dnsserv.h"
  35. #include "dos.h"
  36. #include "geoip.h"
  37. #include "routerlist.h"
  38. static void init_geoip_countries(void);
  39. /** An entry from the GeoIP IPv4 file: maps an IPv4 range to a country. */
  40. typedef struct geoip_ipv4_entry_t {
  41. uint32_t ip_low; /**< The lowest IP in the range, in host order */
  42. uint32_t ip_high; /**< The highest IP in the range, in host order */
  43. intptr_t country; /**< An index into geoip_countries */
  44. } geoip_ipv4_entry_t;
  45. /** An entry from the GeoIP IPv6 file: maps an IPv6 range to a country. */
  46. typedef struct geoip_ipv6_entry_t {
  47. struct in6_addr ip_low; /**< The lowest IP in the range, in host order */
  48. struct in6_addr ip_high; /**< The highest IP in the range, in host order */
  49. intptr_t country; /**< An index into geoip_countries */
  50. } geoip_ipv6_entry_t;
  51. /** A per-country record for GeoIP request history. */
  52. typedef struct geoip_country_t {
  53. char countrycode[3];
  54. uint32_t n_v3_ns_requests;
  55. } geoip_country_t;
  56. /** A list of geoip_country_t */
  57. static smartlist_t *geoip_countries = NULL;
  58. /** A map from lowercased country codes to their position in geoip_countries.
  59. * The index is encoded in the pointer, and 1 is added so that NULL can mean
  60. * not found. */
  61. static strmap_t *country_idxplus1_by_lc_code = NULL;
  62. /** Lists of all known geoip_ipv4_entry_t and geoip_ipv6_entry_t, sorted
  63. * by their respective ip_low. */
  64. static smartlist_t *geoip_ipv4_entries = NULL, *geoip_ipv6_entries = NULL;
  65. /** SHA1 digest of the GeoIP files to include in extra-info descriptors. */
  66. static char geoip_digest[DIGEST_LEN];
  67. static char geoip6_digest[DIGEST_LEN];
  68. /* Total size in bytes of the geoip client history cache. Used by the OOM
  69. * handler. */
  70. static size_t geoip_client_history_cache_size;
  71. /* Increment the geoip client history cache size counter with the given bytes.
  72. * This prevents an overflow and set it to its maximum in that case. */
  73. static inline void
  74. geoip_increment_client_history_cache_size(size_t bytes)
  75. {
  76. /* This is shockingly high, lets log it so it can be reported. */
  77. IF_BUG_ONCE(geoip_client_history_cache_size > (SIZE_MAX - bytes)) {
  78. geoip_client_history_cache_size = SIZE_MAX;
  79. return;
  80. }
  81. geoip_client_history_cache_size += bytes;
  82. }
  83. /* Decrement the geoip client history cache size counter with the given bytes.
  84. * This prevents an underflow and set it to 0 in that case. */
  85. static inline void
  86. geoip_decrement_client_history_cache_size(size_t bytes)
  87. {
  88. /* Going below 0 means that we either allocated an entry without
  89. * incrementing the counter or we have different sizes when allocating and
  90. * freeing. It shouldn't happened so log it. */
  91. IF_BUG_ONCE(geoip_client_history_cache_size < bytes) {
  92. geoip_client_history_cache_size = 0;
  93. return;
  94. }
  95. geoip_client_history_cache_size -= bytes;
  96. }
  97. /** Return the index of the <b>country</b>'s entry in the GeoIP
  98. * country list if it is a valid 2-letter country code, otherwise
  99. * return -1. */
  100. MOCK_IMPL(country_t,
  101. geoip_get_country,(const char *country))
  102. {
  103. void *idxplus1_;
  104. intptr_t idx;
  105. idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country);
  106. if (!idxplus1_)
  107. return -1;
  108. idx = ((uintptr_t)idxplus1_)-1;
  109. return (country_t)idx;
  110. }
  111. /** Add an entry to a GeoIP table, mapping all IP addresses between <b>low</b>
  112. * and <b>high</b>, inclusive, to the 2-letter country code <b>country</b>. */
  113. static void
  114. geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high,
  115. const char *country)
  116. {
  117. intptr_t idx;
  118. void *idxplus1_;
  119. IF_BUG_ONCE(tor_addr_family(low) != tor_addr_family(high))
  120. return;
  121. IF_BUG_ONCE(tor_addr_compare(high, low, CMP_EXACT) < 0)
  122. return;
  123. idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country);
  124. if (!idxplus1_) {
  125. geoip_country_t *c = tor_malloc_zero(sizeof(geoip_country_t));
  126. strlcpy(c->countrycode, country, sizeof(c->countrycode));
  127. tor_strlower(c->countrycode);
  128. smartlist_add(geoip_countries, c);
  129. idx = smartlist_len(geoip_countries) - 1;
  130. strmap_set_lc(country_idxplus1_by_lc_code, country, (void*)(idx+1));
  131. } else {
  132. idx = ((uintptr_t)idxplus1_)-1;
  133. }
  134. {
  135. geoip_country_t *c = smartlist_get(geoip_countries, idx);
  136. tor_assert(!strcasecmp(c->countrycode, country));
  137. }
  138. if (tor_addr_family(low) == AF_INET) {
  139. geoip_ipv4_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv4_entry_t));
  140. ent->ip_low = tor_addr_to_ipv4h(low);
  141. ent->ip_high = tor_addr_to_ipv4h(high);
  142. ent->country = idx;
  143. smartlist_add(geoip_ipv4_entries, ent);
  144. } else if (tor_addr_family(low) == AF_INET6) {
  145. geoip_ipv6_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv6_entry_t));
  146. ent->ip_low = *tor_addr_to_in6_assert(low);
  147. ent->ip_high = *tor_addr_to_in6_assert(high);
  148. ent->country = idx;
  149. smartlist_add(geoip_ipv6_entries, ent);
  150. }
  151. }
  152. /** Add an entry to the GeoIP table indicated by <b>family</b>,
  153. * parsing it from <b>line</b>. The format is as for geoip_load_file(). */
  154. STATIC int
  155. geoip_parse_entry(const char *line, sa_family_t family)
  156. {
  157. tor_addr_t low_addr, high_addr;
  158. char c[3];
  159. char *country = NULL;
  160. if (!geoip_countries)
  161. init_geoip_countries();
  162. if (family == AF_INET) {
  163. if (!geoip_ipv4_entries)
  164. geoip_ipv4_entries = smartlist_new();
  165. } else if (family == AF_INET6) {
  166. if (!geoip_ipv6_entries)
  167. geoip_ipv6_entries = smartlist_new();
  168. } else {
  169. log_warn(LD_GENERAL, "Unsupported family: %d", family);
  170. return -1;
  171. }
  172. while (TOR_ISSPACE(*line))
  173. ++line;
  174. if (*line == '#')
  175. return 0;
  176. char buf[512];
  177. if (family == AF_INET) {
  178. unsigned int low, high;
  179. if (tor_sscanf(line,"%u,%u,%2s", &low, &high, c) == 3 ||
  180. tor_sscanf(line,"\"%u\",\"%u\",\"%2s\",", &low, &high, c) == 3) {
  181. tor_addr_from_ipv4h(&low_addr, low);
  182. tor_addr_from_ipv4h(&high_addr, high);
  183. } else
  184. goto fail;
  185. country = c;
  186. } else { /* AF_INET6 */
  187. char *low_str, *high_str;
  188. struct in6_addr low, high;
  189. char *strtok_state;
  190. strlcpy(buf, line, sizeof(buf));
  191. low_str = tor_strtok_r(buf, ",", &strtok_state);
  192. if (!low_str)
  193. goto fail;
  194. high_str = tor_strtok_r(NULL, ",", &strtok_state);
  195. if (!high_str)
  196. goto fail;
  197. country = tor_strtok_r(NULL, "\n", &strtok_state);
  198. if (!country)
  199. goto fail;
  200. if (strlen(country) != 2)
  201. goto fail;
  202. if (tor_inet_pton(AF_INET6, low_str, &low) <= 0)
  203. goto fail;
  204. tor_addr_from_in6(&low_addr, &low);
  205. if (tor_inet_pton(AF_INET6, high_str, &high) <= 0)
  206. goto fail;
  207. tor_addr_from_in6(&high_addr, &high);
  208. }
  209. geoip_add_entry(&low_addr, &high_addr, country);
  210. return 0;
  211. fail:
  212. log_warn(LD_GENERAL, "Unable to parse line from GEOIP %s file: %s",
  213. family == AF_INET ? "IPv4" : "IPv6", escaped(line));
  214. return -1;
  215. }
  216. /** Sorting helper: return -1, 1, or 0 based on comparison of two
  217. * geoip_ipv4_entry_t */
  218. static int
  219. geoip_ipv4_compare_entries_(const void **_a, const void **_b)
  220. {
  221. const geoip_ipv4_entry_t *a = *_a, *b = *_b;
  222. if (a->ip_low < b->ip_low)
  223. return -1;
  224. else if (a->ip_low > b->ip_low)
  225. return 1;
  226. else
  227. return 0;
  228. }
  229. /** bsearch helper: return -1, 1, or 0 based on comparison of an IP (a pointer
  230. * to a uint32_t in host order) to a geoip_ipv4_entry_t */
  231. static int
  232. geoip_ipv4_compare_key_to_entry_(const void *_key, const void **_member)
  233. {
  234. /* No alignment issue here, since _key really is a pointer to uint32_t */
  235. const uint32_t addr = *(uint32_t *)_key;
  236. const geoip_ipv4_entry_t *entry = *_member;
  237. if (addr < entry->ip_low)
  238. return -1;
  239. else if (addr > entry->ip_high)
  240. return 1;
  241. else
  242. return 0;
  243. }
  244. /** Sorting helper: return -1, 1, or 0 based on comparison of two
  245. * geoip_ipv6_entry_t */
  246. static int
  247. geoip_ipv6_compare_entries_(const void **_a, const void **_b)
  248. {
  249. const geoip_ipv6_entry_t *a = *_a, *b = *_b;
  250. return fast_memcmp(a->ip_low.s6_addr, b->ip_low.s6_addr,
  251. sizeof(struct in6_addr));
  252. }
  253. /** bsearch helper: return -1, 1, or 0 based on comparison of an IPv6
  254. * (a pointer to a in6_addr) to a geoip_ipv6_entry_t */
  255. static int
  256. geoip_ipv6_compare_key_to_entry_(const void *_key, const void **_member)
  257. {
  258. const struct in6_addr *addr = (struct in6_addr *)_key;
  259. const geoip_ipv6_entry_t *entry = *_member;
  260. if (fast_memcmp(addr->s6_addr, entry->ip_low.s6_addr,
  261. sizeof(struct in6_addr)) < 0)
  262. return -1;
  263. else if (fast_memcmp(addr->s6_addr, entry->ip_high.s6_addr,
  264. sizeof(struct in6_addr)) > 0)
  265. return 1;
  266. else
  267. return 0;
  268. }
  269. /** Return 1 if we should collect geoip stats on bridge users, and
  270. * include them in our extrainfo descriptor. Else return 0. */
  271. int
  272. should_record_bridge_info(const or_options_t *options)
  273. {
  274. return options->BridgeRelay && options->BridgeRecordUsageByCountry;
  275. }
  276. /** Set up a new list of geoip countries with no countries (yet) set in it,
  277. * except for the unknown country.
  278. */
  279. static void
  280. init_geoip_countries(void)
  281. {
  282. geoip_country_t *geoip_unresolved;
  283. geoip_countries = smartlist_new();
  284. /* Add a geoip_country_t for requests that could not be resolved to a
  285. * country as first element (index 0) to geoip_countries. */
  286. geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t));
  287. strlcpy(geoip_unresolved->countrycode, "??",
  288. sizeof(geoip_unresolved->countrycode));
  289. smartlist_add(geoip_countries, geoip_unresolved);
  290. country_idxplus1_by_lc_code = strmap_new();
  291. strmap_set_lc(country_idxplus1_by_lc_code, "??", (void*)(1));
  292. }
  293. /** Clear appropriate GeoIP database, based on <b>family</b>, and
  294. * reload it from the file <b>filename</b>. Return 0 on success, -1 on
  295. * failure.
  296. *
  297. * Recognized line formats for IPv4 are:
  298. * INTIPLOW,INTIPHIGH,CC
  299. * and
  300. * "INTIPLOW","INTIPHIGH","CC","CC3","COUNTRY NAME"
  301. * where INTIPLOW and INTIPHIGH are IPv4 addresses encoded as 4-byte unsigned
  302. * integers, and CC is a country code.
  303. *
  304. * Recognized line format for IPv6 is:
  305. * IPV6LOW,IPV6HIGH,CC
  306. * where IPV6LOW and IPV6HIGH are IPv6 addresses and CC is a country code.
  307. *
  308. * It also recognizes, and skips over, blank lines and lines that start
  309. * with '#' (comments).
  310. */
  311. int
  312. geoip_load_file(sa_family_t family, const char *filename)
  313. {
  314. FILE *f;
  315. const char *msg = "";
  316. const or_options_t *options = get_options();
  317. int severity = options_need_geoip_info(options, &msg) ? LOG_WARN : LOG_INFO;
  318. crypto_digest_t *geoip_digest_env = NULL;
  319. tor_assert(family == AF_INET || family == AF_INET6);
  320. if (!(f = tor_fopen_cloexec(filename, "r"))) {
  321. log_fn(severity, LD_GENERAL, "Failed to open GEOIP file %s. %s",
  322. filename, msg);
  323. return -1;
  324. }
  325. if (!geoip_countries)
  326. init_geoip_countries();
  327. if (family == AF_INET) {
  328. if (geoip_ipv4_entries) {
  329. SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, e,
  330. tor_free(e));
  331. smartlist_free(geoip_ipv4_entries);
  332. }
  333. geoip_ipv4_entries = smartlist_new();
  334. } else { /* AF_INET6 */
  335. if (geoip_ipv6_entries) {
  336. SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, e,
  337. tor_free(e));
  338. smartlist_free(geoip_ipv6_entries);
  339. }
  340. geoip_ipv6_entries = smartlist_new();
  341. }
  342. geoip_digest_env = crypto_digest_new();
  343. log_notice(LD_GENERAL, "Parsing GEOIP %s file %s.",
  344. (family == AF_INET) ? "IPv4" : "IPv6", filename);
  345. while (!feof(f)) {
  346. char buf[512];
  347. if (fgets(buf, (int)sizeof(buf), f) == NULL)
  348. break;
  349. crypto_digest_add_bytes(geoip_digest_env, buf, strlen(buf));
  350. /* FFFF track full country name. */
  351. geoip_parse_entry(buf, family);
  352. }
  353. /*XXXX abort and return -1 if no entries/illformed?*/
  354. fclose(f);
  355. /* Sort list and remember file digests so that we can include it in
  356. * our extra-info descriptors. */
  357. if (family == AF_INET) {
  358. smartlist_sort(geoip_ipv4_entries, geoip_ipv4_compare_entries_);
  359. /* Okay, now we need to maybe change our mind about what is in
  360. * which country. We do this for IPv4 only since that's what we
  361. * store in node->country. */
  362. refresh_all_country_info();
  363. crypto_digest_get_digest(geoip_digest_env, geoip_digest, DIGEST_LEN);
  364. } else {
  365. /* AF_INET6 */
  366. smartlist_sort(geoip_ipv6_entries, geoip_ipv6_compare_entries_);
  367. crypto_digest_get_digest(geoip_digest_env, geoip6_digest, DIGEST_LEN);
  368. }
  369. crypto_digest_free(geoip_digest_env);
  370. return 0;
  371. }
  372. /** Given an IP address in host order, return a number representing the
  373. * country to which that address belongs, -1 for "No geoip information
  374. * available", or 0 for the 'unknown country'. The return value will always
  375. * be less than geoip_get_n_countries(). To decode it, call
  376. * geoip_get_country_name().
  377. */
  378. STATIC int
  379. geoip_get_country_by_ipv4(uint32_t ipaddr)
  380. {
  381. geoip_ipv4_entry_t *ent;
  382. if (!geoip_ipv4_entries)
  383. return -1;
  384. ent = smartlist_bsearch(geoip_ipv4_entries, &ipaddr,
  385. geoip_ipv4_compare_key_to_entry_);
  386. return ent ? (int)ent->country : 0;
  387. }
  388. /** Given an IPv6 address, return a number representing the country to
  389. * which that address belongs, -1 for "No geoip information available", or
  390. * 0 for the 'unknown country'. The return value will always be less than
  391. * geoip_get_n_countries(). To decode it, call geoip_get_country_name().
  392. */
  393. STATIC int
  394. geoip_get_country_by_ipv6(const struct in6_addr *addr)
  395. {
  396. geoip_ipv6_entry_t *ent;
  397. if (!geoip_ipv6_entries)
  398. return -1;
  399. ent = smartlist_bsearch(geoip_ipv6_entries, addr,
  400. geoip_ipv6_compare_key_to_entry_);
  401. return ent ? (int)ent->country : 0;
  402. }
  403. /** Given an IP address, return a number representing the country to which
  404. * that address belongs, -1 for "No geoip information available", or 0 for
  405. * the 'unknown country'. The return value will always be less than
  406. * geoip_get_n_countries(). To decode it, call geoip_get_country_name().
  407. */
  408. MOCK_IMPL(int,
  409. geoip_get_country_by_addr,(const tor_addr_t *addr))
  410. {
  411. if (tor_addr_family(addr) == AF_INET) {
  412. return geoip_get_country_by_ipv4(tor_addr_to_ipv4h(addr));
  413. } else if (tor_addr_family(addr) == AF_INET6) {
  414. return geoip_get_country_by_ipv6(tor_addr_to_in6(addr));
  415. } else {
  416. return -1;
  417. }
  418. }
  419. /** Return the number of countries recognized by the GeoIP country list. */
  420. MOCK_IMPL(int,
  421. geoip_get_n_countries,(void))
  422. {
  423. if (!geoip_countries)
  424. init_geoip_countries();
  425. return (int) smartlist_len(geoip_countries);
  426. }
  427. /** Return the two-letter country code associated with the number <b>num</b>,
  428. * or "??" for an unknown value. */
  429. const char *
  430. geoip_get_country_name(country_t num)
  431. {
  432. if (geoip_countries && num >= 0 && num < smartlist_len(geoip_countries)) {
  433. geoip_country_t *c = smartlist_get(geoip_countries, num);
  434. return c->countrycode;
  435. } else
  436. return "??";
  437. }
  438. /** Return true iff we have loaded a GeoIP database.*/
  439. MOCK_IMPL(int,
  440. geoip_is_loaded,(sa_family_t family))
  441. {
  442. tor_assert(family == AF_INET || family == AF_INET6);
  443. if (geoip_countries == NULL)
  444. return 0;
  445. if (family == AF_INET)
  446. return geoip_ipv4_entries != NULL;
  447. else /* AF_INET6 */
  448. return geoip_ipv6_entries != NULL;
  449. }
  450. /** Return the hex-encoded SHA1 digest of the loaded GeoIP file. The
  451. * result does not need to be deallocated, but will be overwritten by the
  452. * next call of hex_str(). */
  453. const char *
  454. geoip_db_digest(sa_family_t family)
  455. {
  456. tor_assert(family == AF_INET || family == AF_INET6);
  457. if (family == AF_INET)
  458. return hex_str(geoip_digest, DIGEST_LEN);
  459. else /* AF_INET6 */
  460. return hex_str(geoip6_digest, DIGEST_LEN);
  461. }
  462. /** Largest allowable value for last_seen_in_minutes. (It's a 30-bit field,
  463. * so it can hold up to (1u<<30)-1, or 0x3fffffffu.
  464. */
  465. #define MAX_LAST_SEEN_IN_MINUTES 0X3FFFFFFFu
  466. /** Map from client IP address to last time seen. */
  467. static HT_HEAD(clientmap, clientmap_entry_t) client_history =
  468. HT_INITIALIZER();
  469. /** Hashtable helper: compute a hash of a clientmap_entry_t. */
  470. static inline unsigned
  471. clientmap_entry_hash(const clientmap_entry_t *a)
  472. {
  473. unsigned h = (unsigned) tor_addr_hash(&a->addr);
  474. if (a->transport_name)
  475. h += (unsigned) siphash24g(a->transport_name, strlen(a->transport_name));
  476. return h;
  477. }
  478. /** Hashtable helper: compare two clientmap_entry_t values for equality. */
  479. static inline int
  480. clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b)
  481. {
  482. if (strcmp_opt(a->transport_name, b->transport_name))
  483. return 0;
  484. return !tor_addr_compare(&a->addr, &b->addr, CMP_EXACT) &&
  485. a->action == b->action;
  486. }
  487. HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
  488. clientmap_entries_eq)
  489. HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
  490. clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
  491. #define clientmap_entry_free(ent) \
  492. FREE_AND_NULL(clientmap_entry_t, clientmap_entry_free_, ent)
  493. /** Return the size of a client map entry. */
  494. static inline size_t
  495. clientmap_entry_size(const clientmap_entry_t *ent)
  496. {
  497. tor_assert(ent);
  498. return (sizeof(clientmap_entry_t) +
  499. (ent->transport_name ? strlen(ent->transport_name) : 0));
  500. }
  501. /** Free all storage held by <b>ent</b>. */
  502. static void
  503. clientmap_entry_free_(clientmap_entry_t *ent)
  504. {
  505. if (!ent)
  506. return;
  507. /* This entry is about to be freed so pass it to the DoS subsystem to see if
  508. * any actions can be taken about it. */
  509. dos_geoip_entry_about_to_free(ent);
  510. geoip_decrement_client_history_cache_size(clientmap_entry_size(ent));
  511. tor_free(ent->transport_name);
  512. tor_free(ent);
  513. }
  514. /* Return a newly allocated clientmap entry with the given action and address
  515. * that are mandatory. The transport_name can be optional. This can't fail. */
  516. static clientmap_entry_t *
  517. clientmap_entry_new(geoip_client_action_t action, const tor_addr_t *addr,
  518. const char *transport_name)
  519. {
  520. clientmap_entry_t *entry;
  521. tor_assert(action == GEOIP_CLIENT_CONNECT ||
  522. action == GEOIP_CLIENT_NETWORKSTATUS);
  523. tor_assert(addr);
  524. entry = tor_malloc_zero(sizeof(clientmap_entry_t));
  525. entry->action = action;
  526. tor_addr_copy(&entry->addr, addr);
  527. if (transport_name) {
  528. entry->transport_name = tor_strdup(transport_name);
  529. }
  530. /* Allocated and initialized, note down its size for the OOM handler. */
  531. geoip_increment_client_history_cache_size(clientmap_entry_size(entry));
  532. return entry;
  533. }
  534. /** Clear history of connecting clients used by entry and bridge stats. */
  535. static void
  536. client_history_clear(void)
  537. {
  538. clientmap_entry_t **ent, **next, *this;
  539. for (ent = HT_START(clientmap, &client_history); ent != NULL;
  540. ent = next) {
  541. if ((*ent)->action == GEOIP_CLIENT_CONNECT) {
  542. this = *ent;
  543. next = HT_NEXT_RMV(clientmap, &client_history, ent);
  544. clientmap_entry_free(this);
  545. } else {
  546. next = HT_NEXT(clientmap, &client_history, ent);
  547. }
  548. }
  549. }
  550. /** Note that we've seen a client connect from the IP <b>addr</b>
  551. * at time <b>now</b>. Ignored by all but bridges and directories if
  552. * configured accordingly. */
  553. void
  554. geoip_note_client_seen(geoip_client_action_t action,
  555. const tor_addr_t *addr,
  556. const char *transport_name,
  557. time_t now)
  558. {
  559. const or_options_t *options = get_options();
  560. clientmap_entry_t *ent;
  561. if (action == GEOIP_CLIENT_CONNECT) {
  562. /* Only remember statistics if the DoS mitigation subsystem is enabled. If
  563. * not, only if as entry guard or as bridge. */
  564. if (!dos_enabled()) {
  565. if (!options->EntryStatistics &&
  566. (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))) {
  567. return;
  568. }
  569. }
  570. } else {
  571. /* Only gather directory-request statistics if configured, and
  572. * forcibly disable them on bridge authorities. */
  573. if (!options->DirReqStatistics || options->BridgeAuthoritativeDir)
  574. return;
  575. }
  576. log_debug(LD_GENERAL, "Seen client from '%s' with transport '%s'.",
  577. safe_str_client(fmt_addr((addr))),
  578. transport_name ? transport_name : "<no transport>");
  579. ent = geoip_lookup_client(addr, transport_name, action);
  580. if (! ent) {
  581. ent = clientmap_entry_new(action, addr, transport_name);
  582. HT_INSERT(clientmap, &client_history, ent);
  583. }
  584. if (now / 60 <= (int)MAX_LAST_SEEN_IN_MINUTES && now >= 0)
  585. ent->last_seen_in_minutes = (unsigned)(now/60);
  586. else
  587. ent->last_seen_in_minutes = 0;
  588. if (action == GEOIP_CLIENT_NETWORKSTATUS) {
  589. int country_idx = geoip_get_country_by_addr(addr);
  590. if (country_idx < 0)
  591. country_idx = 0; /** unresolved requests are stored at index 0. */
  592. if (country_idx >= 0 && country_idx < smartlist_len(geoip_countries)) {
  593. geoip_country_t *country = smartlist_get(geoip_countries, country_idx);
  594. ++country->n_v3_ns_requests;
  595. }
  596. }
  597. }
  598. /** HT_FOREACH helper: remove a clientmap_entry_t from the hashtable if it's
  599. * older than a certain time. */
  600. static int
  601. remove_old_client_helper_(struct clientmap_entry_t *ent, void *_cutoff)
  602. {
  603. time_t cutoff = *(time_t*)_cutoff / 60;
  604. if (ent->last_seen_in_minutes < cutoff) {
  605. clientmap_entry_free(ent);
  606. return 1;
  607. } else {
  608. return 0;
  609. }
  610. }
  611. /** Forget about all clients that haven't connected since <b>cutoff</b>. */
  612. void
  613. geoip_remove_old_clients(time_t cutoff)
  614. {
  615. clientmap_HT_FOREACH_FN(&client_history,
  616. remove_old_client_helper_,
  617. &cutoff);
  618. }
  619. /* Return a client entry object matching the given address, transport name and
  620. * geoip action from the clientmap. NULL if not found. The transport_name can
  621. * be NULL. */
  622. clientmap_entry_t *
  623. geoip_lookup_client(const tor_addr_t *addr, const char *transport_name,
  624. geoip_client_action_t action)
  625. {
  626. clientmap_entry_t lookup;
  627. tor_assert(addr);
  628. /* We always look for a client connection with no transport. */
  629. tor_addr_copy(&lookup.addr, addr);
  630. lookup.action = action;
  631. lookup.transport_name = (char *) transport_name;
  632. return HT_FIND(clientmap, &client_history, &lookup);
  633. }
  634. /* Cleanup client entries older than the cutoff. Used for the OOM. Return the
  635. * number of bytes freed. If 0 is returned, nothing was freed. */
  636. static size_t
  637. oom_clean_client_entries(time_t cutoff)
  638. {
  639. size_t bytes = 0;
  640. clientmap_entry_t **ent, **ent_next;
  641. for (ent = HT_START(clientmap, &client_history); ent; ent = ent_next) {
  642. clientmap_entry_t *entry = *ent;
  643. if (entry->last_seen_in_minutes < (cutoff / 60)) {
  644. ent_next = HT_NEXT_RMV(clientmap, &client_history, ent);
  645. bytes += clientmap_entry_size(entry);
  646. clientmap_entry_free(entry);
  647. } else {
  648. ent_next = HT_NEXT(clientmap, &client_history, ent);
  649. }
  650. }
  651. return bytes;
  652. }
  653. /* Below this minimum lifetime, the OOM won't cleanup any entries. */
  654. #define GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF (4 * 60 * 60)
  655. /* The OOM moves the cutoff by that much every run. */
  656. #define GEOIP_CLIENT_CACHE_OOM_STEP (15 * 50)
  657. /* Cleanup the geoip client history cache called from the OOM handler. Return
  658. * the amount of bytes removed. This can return a value below or above
  659. * min_remove_bytes but will stop as oon as the min_remove_bytes has been
  660. * reached. */
  661. size_t
  662. geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes)
  663. {
  664. time_t k;
  665. size_t bytes_removed = 0;
  666. /* Our OOM handler called with 0 bytes to remove is a code flow error. */
  667. tor_assert(min_remove_bytes != 0);
  668. /* Set k to the initial cutoff of an entry. We then going to move it by step
  669. * to try to remove as much as we can. */
  670. k = WRITE_STATS_INTERVAL;
  671. do {
  672. time_t cutoff;
  673. /* If k has reached the minimum lifetime, we have to stop else we might
  674. * remove every single entries which would be pretty bad for the DoS
  675. * mitigation subsystem if by just filling the geoip cache, it was enough
  676. * to trigger the OOM and clean every single entries. */
  677. if (k <= GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF) {
  678. break;
  679. }
  680. cutoff = now - k;
  681. bytes_removed += oom_clean_client_entries(cutoff);
  682. k -= GEOIP_CLIENT_CACHE_OOM_STEP;
  683. } while (bytes_removed < min_remove_bytes);
  684. return bytes_removed;
  685. }
  686. /* Return the total size in bytes of the client history cache. */
  687. size_t
  688. geoip_client_cache_total_allocation(void)
  689. {
  690. return geoip_client_history_cache_size;
  691. }
  692. /** How many responses are we giving to clients requesting v3 network
  693. * statuses? */
  694. static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM];
  695. /** Note that we've rejected a client's request for a v3 network status
  696. * for reason <b>reason</b> at time <b>now</b>. */
  697. void
  698. geoip_note_ns_response(geoip_ns_response_t response)
  699. {
  700. static int arrays_initialized = 0;
  701. if (!get_options()->DirReqStatistics)
  702. return;
  703. if (!arrays_initialized) {
  704. memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
  705. arrays_initialized = 1;
  706. }
  707. tor_assert(response < GEOIP_NS_RESPONSE_NUM);
  708. ns_v3_responses[response]++;
  709. }
  710. /** Do not mention any country from which fewer than this number of IPs have
  711. * connected. This conceivably avoids reporting information that could
  712. * deanonymize users, though analysis is lacking. */
  713. #define MIN_IPS_TO_NOTE_COUNTRY 1
  714. /** Do not report any geoip data at all if we have fewer than this number of
  715. * IPs to report about. */
  716. #define MIN_IPS_TO_NOTE_ANYTHING 1
  717. /** When reporting geoip data about countries, round up to the nearest
  718. * multiple of this value. */
  719. #define IP_GRANULARITY 8
  720. /** Helper type: used to sort per-country totals by value. */
  721. typedef struct c_hist_t {
  722. char country[3]; /**< Two-letter country code. */
  723. unsigned total; /**< Total IP addresses seen in this country. */
  724. } c_hist_t;
  725. /** Sorting helper: return -1, 1, or 0 based on comparison of two
  726. * geoip_ipv4_entry_t. Sort in descending order of total, and then by country
  727. * code. */
  728. static int
  729. c_hist_compare_(const void **_a, const void **_b)
  730. {
  731. const c_hist_t *a = *_a, *b = *_b;
  732. if (a->total > b->total)
  733. return -1;
  734. else if (a->total < b->total)
  735. return 1;
  736. else
  737. return strcmp(a->country, b->country);
  738. }
  739. /** When there are incomplete directory requests at the end of a 24-hour
  740. * period, consider those requests running for longer than this timeout as
  741. * failed, the others as still running. */
  742. #define DIRREQ_TIMEOUT (10*60)
  743. /** Entry in a map from either chan->global_identifier for direct requests
  744. * or a unique circuit identifier for tunneled requests to request time,
  745. * response size, and completion time of a network status request. Used to
  746. * measure download times of requests to derive average client
  747. * bandwidths. */
  748. typedef struct dirreq_map_entry_t {
  749. HT_ENTRY(dirreq_map_entry_t) node;
  750. /** Unique identifier for this network status request; this is either the
  751. * chan->global_identifier of the dir channel (direct request) or a new
  752. * locally unique identifier of a circuit (tunneled request). This ID is
  753. * only unique among other direct or tunneled requests, respectively. */
  754. uint64_t dirreq_id;
  755. unsigned int state:3; /**< State of this directory request. */
  756. unsigned int type:1; /**< Is this a direct or a tunneled request? */
  757. unsigned int completed:1; /**< Is this request complete? */
  758. /** When did we receive the request and started sending the response? */
  759. struct timeval request_time;
  760. size_t response_size; /**< What is the size of the response in bytes? */
  761. struct timeval completion_time; /**< When did the request succeed? */
  762. } dirreq_map_entry_t;
  763. /** Map of all directory requests asking for v2 or v3 network statuses in
  764. * the current geoip-stats interval. Values are
  765. * of type *<b>dirreq_map_entry_t</b>. */
  766. static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map =
  767. HT_INITIALIZER();
  768. static int
  769. dirreq_map_ent_eq(const dirreq_map_entry_t *a,
  770. const dirreq_map_entry_t *b)
  771. {
  772. return a->dirreq_id == b->dirreq_id && a->type == b->type;
  773. }
  774. /* DOCDOC dirreq_map_ent_hash */
  775. static unsigned
  776. dirreq_map_ent_hash(const dirreq_map_entry_t *entry)
  777. {
  778. unsigned u = (unsigned) entry->dirreq_id;
  779. u += entry->type << 20;
  780. return u;
  781. }
  782. HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
  783. dirreq_map_ent_eq)
  784. HT_GENERATE2(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
  785. dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_)
  786. /** Helper: Put <b>entry</b> into map of directory requests using
  787. * <b>type</b> and <b>dirreq_id</b> as key parts. If there is
  788. * already an entry for that key, print out a BUG warning and return. */
  789. static void
  790. dirreq_map_put_(dirreq_map_entry_t *entry, dirreq_type_t type,
  791. uint64_t dirreq_id)
  792. {
  793. dirreq_map_entry_t *old_ent;
  794. tor_assert(entry->type == type);
  795. tor_assert(entry->dirreq_id == dirreq_id);
  796. /* XXXX we could switch this to HT_INSERT some time, since it seems that
  797. * this bug doesn't happen. But since this function doesn't seem to be
  798. * critical-path, it's sane to leave it alone. */
  799. old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry);
  800. if (old_ent && old_ent != entry) {
  801. log_warn(LD_BUG, "Error when putting directory request into local "
  802. "map. There was already an entry for the same identifier.");
  803. return;
  804. }
  805. }
  806. /** Helper: Look up and return an entry in the map of directory requests
  807. * using <b>type</b> and <b>dirreq_id</b> as key parts. If there
  808. * is no such entry, return NULL. */
  809. static dirreq_map_entry_t *
  810. dirreq_map_get_(dirreq_type_t type, uint64_t dirreq_id)
  811. {
  812. dirreq_map_entry_t lookup;
  813. lookup.type = type;
  814. lookup.dirreq_id = dirreq_id;
  815. return HT_FIND(dirreqmap, &dirreq_map, &lookup);
  816. }
  817. /** Note that an either direct or tunneled (see <b>type</b>) directory
  818. * request for a v3 network status with unique ID <b>dirreq_id</b> of size
  819. * <b>response_size</b> has started. */
  820. void
  821. geoip_start_dirreq(uint64_t dirreq_id, size_t response_size,
  822. dirreq_type_t type)
  823. {
  824. dirreq_map_entry_t *ent;
  825. if (!get_options()->DirReqStatistics)
  826. return;
  827. ent = tor_malloc_zero(sizeof(dirreq_map_entry_t));
  828. ent->dirreq_id = dirreq_id;
  829. tor_gettimeofday(&ent->request_time);
  830. ent->response_size = response_size;
  831. ent->type = type;
  832. dirreq_map_put_(ent, type, dirreq_id);
  833. }
  834. /** Change the state of the either direct or tunneled (see <b>type</b>)
  835. * directory request with <b>dirreq_id</b> to <b>new_state</b> and
  836. * possibly mark it as completed. If no entry can be found for the given
  837. * key parts (e.g., if this is a directory request that we are not
  838. * measuring, or one that was started in the previous measurement period),
  839. * or if the state cannot be advanced to <b>new_state</b>, do nothing. */
  840. void
  841. geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
  842. dirreq_state_t new_state)
  843. {
  844. dirreq_map_entry_t *ent;
  845. if (!get_options()->DirReqStatistics)
  846. return;
  847. ent = dirreq_map_get_(type, dirreq_id);
  848. if (!ent)
  849. return;
  850. if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS)
  851. return;
  852. if (new_state - 1 != ent->state)
  853. return;
  854. ent->state = new_state;
  855. if ((type == DIRREQ_DIRECT &&
  856. new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) ||
  857. (type == DIRREQ_TUNNELED &&
  858. new_state == DIRREQ_CHANNEL_BUFFER_FLUSHED)) {
  859. tor_gettimeofday(&ent->completion_time);
  860. ent->completed = 1;
  861. }
  862. }
  863. /** Return the bridge-ip-transports string that should be inserted in
  864. * our extra-info descriptor. Return NULL if the bridge-ip-transports
  865. * line should be empty. */
  866. char *
  867. geoip_get_transport_history(void)
  868. {
  869. unsigned granularity = IP_GRANULARITY;
  870. /** String hash table (name of transport) -> (number of users). */
  871. strmap_t *transport_counts = strmap_new();
  872. /** Smartlist that contains copies of the names of the transports
  873. that have been used. */
  874. smartlist_t *transports_used = smartlist_new();
  875. /* Special string to signify that no transport was used for this
  876. connection. Pluggable transport names can't have symbols in their
  877. names, so this string will never collide with a real transport. */
  878. static const char* no_transport_str = "<OR>";
  879. clientmap_entry_t **ent;
  880. smartlist_t *string_chunks = smartlist_new();
  881. char *the_string = NULL;
  882. /* If we haven't seen any clients yet, return NULL. */
  883. if (HT_EMPTY(&client_history))
  884. goto done;
  885. /** We do the following steps to form the transport history string:
  886. * a) Foreach client that uses a pluggable transport, we increase the
  887. * times that transport was used by one. If the client did not use
  888. * a transport, we increase the number of times someone connected
  889. * without obfuscation.
  890. * b) Foreach transport we observed, we write its transport history
  891. * string and push it to string_chunks. So, for example, if we've
  892. * seen 665 obfs2 clients, we write "obfs2=665".
  893. * c) We concatenate string_chunks to form the final string.
  894. */
  895. log_debug(LD_GENERAL,"Starting iteration for transport history. %d clients.",
  896. HT_SIZE(&client_history));
  897. /* Loop through all clients. */
  898. HT_FOREACH(ent, clientmap, &client_history) {
  899. uintptr_t val;
  900. void *ptr;
  901. const char *transport_name = (*ent)->transport_name;
  902. if (!transport_name)
  903. transport_name = no_transport_str;
  904. /* Increase the count for this transport name. */
  905. ptr = strmap_get(transport_counts, transport_name);
  906. val = (uintptr_t)ptr;
  907. val++;
  908. ptr = (void*)val;
  909. strmap_set(transport_counts, transport_name, ptr);
  910. /* If it's the first time we see this transport, note it. */
  911. if (val == 1)
  912. smartlist_add_strdup(transports_used, transport_name);
  913. log_debug(LD_GENERAL, "Client from '%s' with transport '%s'. "
  914. "I've now seen %d clients.",
  915. safe_str_client(fmt_addr(&(*ent)->addr)),
  916. transport_name ? transport_name : "<no transport>",
  917. (int)val);
  918. }
  919. /* Sort the transport names (helps with unit testing). */
  920. smartlist_sort_strings(transports_used);
  921. /* Loop through all seen transports. */
  922. SMARTLIST_FOREACH_BEGIN(transports_used, const char *, transport_name) {
  923. void *transport_count_ptr = strmap_get(transport_counts, transport_name);
  924. uintptr_t transport_count = (uintptr_t) transport_count_ptr;
  925. log_debug(LD_GENERAL, "We got "U64_FORMAT" clients with transport '%s'.",
  926. U64_PRINTF_ARG((uint64_t)transport_count), transport_name);
  927. smartlist_add_asprintf(string_chunks, "%s="U64_FORMAT,
  928. transport_name,
  929. U64_PRINTF_ARG(round_uint64_to_next_multiple_of(
  930. (uint64_t)transport_count,
  931. granularity)));
  932. } SMARTLIST_FOREACH_END(transport_name);
  933. the_string = smartlist_join_strings(string_chunks, ",", 0, NULL);
  934. log_debug(LD_GENERAL, "Final bridge-ip-transports string: '%s'", the_string);
  935. done:
  936. strmap_free(transport_counts, NULL);
  937. SMARTLIST_FOREACH(transports_used, char *, s, tor_free(s));
  938. smartlist_free(transports_used);
  939. SMARTLIST_FOREACH(string_chunks, char *, s, tor_free(s));
  940. smartlist_free(string_chunks);
  941. return the_string;
  942. }
  943. /** Return a newly allocated comma-separated string containing statistics
  944. * on network status downloads. The string contains the number of completed
  945. * requests, timeouts, and still running requests as well as the download
  946. * times by deciles and quartiles. Return NULL if we have not observed
  947. * requests for long enough. */
  948. static char *
  949. geoip_get_dirreq_history(dirreq_type_t type)
  950. {
  951. char *result = NULL;
  952. buf_t *buf = NULL;
  953. smartlist_t *dirreq_completed = NULL;
  954. uint32_t complete = 0, timeouts = 0, running = 0;
  955. dirreq_map_entry_t **ptr, **next;
  956. struct timeval now;
  957. tor_gettimeofday(&now);
  958. dirreq_completed = smartlist_new();
  959. for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) {
  960. dirreq_map_entry_t *ent = *ptr;
  961. if (ent->type != type) {
  962. next = HT_NEXT(dirreqmap, &dirreq_map, ptr);
  963. continue;
  964. } else {
  965. if (ent->completed) {
  966. smartlist_add(dirreq_completed, ent);
  967. complete++;
  968. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
  969. } else {
  970. if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT)
  971. timeouts++;
  972. else
  973. running++;
  974. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
  975. tor_free(ent);
  976. }
  977. }
  978. }
  979. #define DIR_REQ_GRANULARITY 4
  980. complete = round_uint32_to_next_multiple_of(complete,
  981. DIR_REQ_GRANULARITY);
  982. timeouts = round_uint32_to_next_multiple_of(timeouts,
  983. DIR_REQ_GRANULARITY);
  984. running = round_uint32_to_next_multiple_of(running,
  985. DIR_REQ_GRANULARITY);
  986. buf = buf_new_with_capacity(1024);
  987. buf_add_printf(buf, "complete=%u,timeout=%u,"
  988. "running=%u", complete, timeouts, running);
  989. #define MIN_DIR_REQ_RESPONSES 16
  990. if (complete >= MIN_DIR_REQ_RESPONSES) {
  991. uint32_t *dltimes;
  992. /* We may have rounded 'completed' up. Here we want to use the
  993. * real value. */
  994. complete = smartlist_len(dirreq_completed);
  995. dltimes = tor_calloc(complete, sizeof(uint32_t));
  996. SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) {
  997. uint32_t bytes_per_second;
  998. uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time,
  999. &ent->completion_time);
  1000. if (time_diff == 0)
  1001. time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible
  1002. * by law of nature or something, but a millisecond
  1003. * is a bit greater than "instantly" */
  1004. bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff);
  1005. dltimes[ent_sl_idx] = bytes_per_second;
  1006. } SMARTLIST_FOREACH_END(ent);
  1007. median_uint32(dltimes, complete); /* sorts as a side effect. */
  1008. buf_add_printf(buf,
  1009. ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u,"
  1010. "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u",
  1011. dltimes[0],
  1012. dltimes[1*complete/10-1],
  1013. dltimes[2*complete/10-1],
  1014. dltimes[1*complete/4-1],
  1015. dltimes[3*complete/10-1],
  1016. dltimes[4*complete/10-1],
  1017. dltimes[5*complete/10-1],
  1018. dltimes[6*complete/10-1],
  1019. dltimes[7*complete/10-1],
  1020. dltimes[3*complete/4-1],
  1021. dltimes[8*complete/10-1],
  1022. dltimes[9*complete/10-1],
  1023. dltimes[complete-1]);
  1024. tor_free(dltimes);
  1025. }
  1026. result = buf_extract(buf, NULL);
  1027. SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent,
  1028. tor_free(ent));
  1029. smartlist_free(dirreq_completed);
  1030. buf_free(buf);
  1031. return result;
  1032. }
  1033. /** Store a newly allocated comma-separated string in
  1034. * *<a>country_str</a> containing entries for all the countries from
  1035. * which we've seen enough clients connect as a bridge, directory
  1036. * server, or entry guard. The entry format is cc=num where num is the
  1037. * number of IPs we've seen connecting from that country, and cc is a
  1038. * lowercased country code. *<a>country_str</a> is set to NULL if
  1039. * we're not ready to export per country data yet.
  1040. *
  1041. * Store a newly allocated comma-separated string in <a>ipver_str</a>
  1042. * containing entries for clients connecting over IPv4 and IPv6. The
  1043. * format is family=num where num is the nubmer of IPs we've seen
  1044. * connecting over that protocol family, and family is 'v4' or 'v6'.
  1045. *
  1046. * Return 0 on success and -1 if we're missing geoip data. */
  1047. int
  1048. geoip_get_client_history(geoip_client_action_t action,
  1049. char **country_str, char **ipver_str)
  1050. {
  1051. unsigned granularity = IP_GRANULARITY;
  1052. smartlist_t *entries = NULL;
  1053. int n_countries = geoip_get_n_countries();
  1054. int i;
  1055. clientmap_entry_t **cm_ent;
  1056. unsigned *counts = NULL;
  1057. unsigned total = 0;
  1058. unsigned ipv4_count = 0, ipv6_count = 0;
  1059. if (!geoip_is_loaded(AF_INET) && !geoip_is_loaded(AF_INET6))
  1060. return -1;
  1061. counts = tor_calloc(n_countries, sizeof(unsigned));
  1062. HT_FOREACH(cm_ent, clientmap, &client_history) {
  1063. int country;
  1064. if ((*cm_ent)->action != (int)action)
  1065. continue;
  1066. country = geoip_get_country_by_addr(&(*cm_ent)->addr);
  1067. if (country < 0)
  1068. country = 0; /** unresolved requests are stored at index 0. */
  1069. tor_assert(0 <= country && country < n_countries);
  1070. ++counts[country];
  1071. ++total;
  1072. switch (tor_addr_family(&(*cm_ent)->addr)) {
  1073. case AF_INET:
  1074. ipv4_count++;
  1075. break;
  1076. case AF_INET6:
  1077. ipv6_count++;
  1078. break;
  1079. }
  1080. }
  1081. if (ipver_str) {
  1082. smartlist_t *chunks = smartlist_new();
  1083. smartlist_add_asprintf(chunks, "v4=%u",
  1084. round_to_next_multiple_of(ipv4_count, granularity));
  1085. smartlist_add_asprintf(chunks, "v6=%u",
  1086. round_to_next_multiple_of(ipv6_count, granularity));
  1087. *ipver_str = smartlist_join_strings(chunks, ",", 0, NULL);
  1088. SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
  1089. smartlist_free(chunks);
  1090. }
  1091. /* Don't record per country data if we haven't seen enough IPs. */
  1092. if (total < MIN_IPS_TO_NOTE_ANYTHING) {
  1093. tor_free(counts);
  1094. if (country_str)
  1095. *country_str = NULL;
  1096. return 0;
  1097. }
  1098. /* Make a list of c_hist_t */
  1099. entries = smartlist_new();
  1100. for (i = 0; i < n_countries; ++i) {
  1101. unsigned c = counts[i];
  1102. const char *countrycode;
  1103. c_hist_t *ent;
  1104. /* Only report a country if it has a minimum number of IPs. */
  1105. if (c >= MIN_IPS_TO_NOTE_COUNTRY) {
  1106. c = round_to_next_multiple_of(c, granularity);
  1107. countrycode = geoip_get_country_name(i);
  1108. ent = tor_malloc(sizeof(c_hist_t));
  1109. strlcpy(ent->country, countrycode, sizeof(ent->country));
  1110. ent->total = c;
  1111. smartlist_add(entries, ent);
  1112. }
  1113. }
  1114. /* Sort entries. Note that we must do this _AFTER_ rounding, or else
  1115. * the sort order could leak info. */
  1116. smartlist_sort(entries, c_hist_compare_);
  1117. if (country_str) {
  1118. smartlist_t *chunks = smartlist_new();
  1119. SMARTLIST_FOREACH(entries, c_hist_t *, ch, {
  1120. smartlist_add_asprintf(chunks, "%s=%u", ch->country, ch->total);
  1121. });
  1122. *country_str = smartlist_join_strings(chunks, ",", 0, NULL);
  1123. SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
  1124. smartlist_free(chunks);
  1125. }
  1126. SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c));
  1127. smartlist_free(entries);
  1128. tor_free(counts);
  1129. return 0;
  1130. }
  1131. /** Return a newly allocated string holding the per-country request history
  1132. * for v3 network statuses in a format suitable for an extra-info document,
  1133. * or NULL on failure. */
  1134. char *
  1135. geoip_get_request_history(void)
  1136. {
  1137. smartlist_t *entries, *strings;
  1138. char *result;
  1139. unsigned granularity = IP_GRANULARITY;
  1140. if (!geoip_countries)
  1141. return NULL;
  1142. entries = smartlist_new();
  1143. SMARTLIST_FOREACH_BEGIN(geoip_countries, geoip_country_t *, c) {
  1144. uint32_t tot = 0;
  1145. c_hist_t *ent;
  1146. tot = c->n_v3_ns_requests;
  1147. if (!tot)
  1148. continue;
  1149. ent = tor_malloc_zero(sizeof(c_hist_t));
  1150. strlcpy(ent->country, c->countrycode, sizeof(ent->country));
  1151. ent->total = round_to_next_multiple_of(tot, granularity);
  1152. smartlist_add(entries, ent);
  1153. } SMARTLIST_FOREACH_END(c);
  1154. smartlist_sort(entries, c_hist_compare_);
  1155. strings = smartlist_new();
  1156. SMARTLIST_FOREACH(entries, c_hist_t *, ent, {
  1157. smartlist_add_asprintf(strings, "%s=%u", ent->country, ent->total);
  1158. });
  1159. result = smartlist_join_strings(strings, ",", 0, NULL);
  1160. SMARTLIST_FOREACH(strings, char *, cp, tor_free(cp));
  1161. SMARTLIST_FOREACH(entries, c_hist_t *, ent, tor_free(ent));
  1162. smartlist_free(strings);
  1163. smartlist_free(entries);
  1164. return result;
  1165. }
  1166. /** Start time of directory request stats or 0 if we're not collecting
  1167. * directory request statistics. */
  1168. static time_t start_of_dirreq_stats_interval;
  1169. /** Initialize directory request stats. */
  1170. void
  1171. geoip_dirreq_stats_init(time_t now)
  1172. {
  1173. start_of_dirreq_stats_interval = now;
  1174. }
  1175. /** Reset counters for dirreq stats. */
  1176. void
  1177. geoip_reset_dirreq_stats(time_t now)
  1178. {
  1179. SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, {
  1180. c->n_v3_ns_requests = 0;
  1181. });
  1182. {
  1183. clientmap_entry_t **ent, **next, *this;
  1184. for (ent = HT_START(clientmap, &client_history); ent != NULL;
  1185. ent = next) {
  1186. if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS) {
  1187. this = *ent;
  1188. next = HT_NEXT_RMV(clientmap, &client_history, ent);
  1189. clientmap_entry_free(this);
  1190. } else {
  1191. next = HT_NEXT(clientmap, &client_history, ent);
  1192. }
  1193. }
  1194. }
  1195. memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
  1196. {
  1197. dirreq_map_entry_t **ent, **next, *this;
  1198. for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
  1199. this = *ent;
  1200. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
  1201. tor_free(this);
  1202. }
  1203. }
  1204. start_of_dirreq_stats_interval = now;
  1205. }
  1206. /** Stop collecting directory request stats in a way that we can re-start
  1207. * doing so in geoip_dirreq_stats_init(). */
  1208. void
  1209. geoip_dirreq_stats_term(void)
  1210. {
  1211. geoip_reset_dirreq_stats(0);
  1212. }
  1213. /** Return a newly allocated string containing the dirreq statistics
  1214. * until <b>now</b>, or NULL if we're not collecting dirreq stats. Caller
  1215. * must ensure start_of_dirreq_stats_interval is in the past. */
  1216. char *
  1217. geoip_format_dirreq_stats(time_t now)
  1218. {
  1219. char t[ISO_TIME_LEN+1];
  1220. int i;
  1221. char *v3_ips_string = NULL, *v3_reqs_string = NULL,
  1222. *v3_direct_dl_string = NULL, *v3_tunneled_dl_string = NULL;
  1223. char *result = NULL;
  1224. if (!start_of_dirreq_stats_interval)
  1225. return NULL; /* Not initialized. */
  1226. tor_assert(now >= start_of_dirreq_stats_interval);
  1227. format_iso_time(t, now);
  1228. geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS, &v3_ips_string, NULL);
  1229. v3_reqs_string = geoip_get_request_history();
  1230. #define RESPONSE_GRANULARITY 8
  1231. for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) {
  1232. ns_v3_responses[i] = round_uint32_to_next_multiple_of(
  1233. ns_v3_responses[i], RESPONSE_GRANULARITY);
  1234. }
  1235. #undef RESPONSE_GRANULARITY
  1236. v3_direct_dl_string = geoip_get_dirreq_history(DIRREQ_DIRECT);
  1237. v3_tunneled_dl_string = geoip_get_dirreq_history(DIRREQ_TUNNELED);
  1238. /* Put everything together into a single string. */
  1239. tor_asprintf(&result, "dirreq-stats-end %s (%d s)\n"
  1240. "dirreq-v3-ips %s\n"
  1241. "dirreq-v3-reqs %s\n"
  1242. "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u,"
  1243. "not-found=%u,not-modified=%u,busy=%u\n"
  1244. "dirreq-v3-direct-dl %s\n"
  1245. "dirreq-v3-tunneled-dl %s\n",
  1246. t,
  1247. (unsigned) (now - start_of_dirreq_stats_interval),
  1248. v3_ips_string ? v3_ips_string : "",
  1249. v3_reqs_string ? v3_reqs_string : "",
  1250. ns_v3_responses[GEOIP_SUCCESS],
  1251. ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS],
  1252. ns_v3_responses[GEOIP_REJECT_UNAVAILABLE],
  1253. ns_v3_responses[GEOIP_REJECT_NOT_FOUND],
  1254. ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED],
  1255. ns_v3_responses[GEOIP_REJECT_BUSY],
  1256. v3_direct_dl_string ? v3_direct_dl_string : "",
  1257. v3_tunneled_dl_string ? v3_tunneled_dl_string : "");
  1258. /* Free partial strings. */
  1259. tor_free(v3_ips_string);
  1260. tor_free(v3_reqs_string);
  1261. tor_free(v3_direct_dl_string);
  1262. tor_free(v3_tunneled_dl_string);
  1263. return result;
  1264. }
  1265. /** If 24 hours have passed since the beginning of the current dirreq
  1266. * stats period, write dirreq stats to $DATADIR/stats/dirreq-stats
  1267. * (possibly overwriting an existing file) and reset counters. Return
  1268. * when we would next want to write dirreq stats or 0 if we never want to
  1269. * write. */
  1270. time_t
  1271. geoip_dirreq_stats_write(time_t now)
  1272. {
  1273. char *str = NULL;
  1274. if (!start_of_dirreq_stats_interval)
  1275. return 0; /* Not initialized. */
  1276. if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now)
  1277. goto done; /* Not ready to write. */
  1278. /* Discard all items in the client history that are too old. */
  1279. geoip_remove_old_clients(start_of_dirreq_stats_interval);
  1280. /* Generate history string .*/
  1281. str = geoip_format_dirreq_stats(now);
  1282. if (! str)
  1283. goto done;
  1284. /* Write dirreq-stats string to disk. */
  1285. if (!check_or_create_data_subdir("stats")) {
  1286. write_to_data_subdir("stats", "dirreq-stats", str, "dirreq statistics");
  1287. /* Reset measurement interval start. */
  1288. geoip_reset_dirreq_stats(now);
  1289. }
  1290. done:
  1291. tor_free(str);
  1292. return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL;
  1293. }
  1294. /** Start time of bridge stats or 0 if we're not collecting bridge
  1295. * statistics. */
  1296. static time_t start_of_bridge_stats_interval;
  1297. /** Initialize bridge stats. */
  1298. void
  1299. geoip_bridge_stats_init(time_t now)
  1300. {
  1301. start_of_bridge_stats_interval = now;
  1302. }
  1303. /** Stop collecting bridge stats in a way that we can re-start doing so in
  1304. * geoip_bridge_stats_init(). */
  1305. void
  1306. geoip_bridge_stats_term(void)
  1307. {
  1308. client_history_clear();
  1309. start_of_bridge_stats_interval = 0;
  1310. }
  1311. /** Validate a bridge statistics string as it would be written to a
  1312. * current extra-info descriptor. Return 1 if the string is valid and
  1313. * recent enough, or 0 otherwise. */
  1314. static int
  1315. validate_bridge_stats(const char *stats_str, time_t now)
  1316. {
  1317. char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1],
  1318. *eos;
  1319. const char *BRIDGE_STATS_END = "bridge-stats-end ";
  1320. const char *BRIDGE_IPS = "bridge-ips ";
  1321. const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n";
  1322. const char *BRIDGE_TRANSPORTS = "bridge-ip-transports ";
  1323. const char *BRIDGE_TRANSPORTS_EMPTY_LINE = "bridge-ip-transports\n";
  1324. const char *tmp;
  1325. time_t stats_end_time;
  1326. int seconds;
  1327. tor_assert(stats_str);
  1328. /* Parse timestamp and number of seconds from
  1329. "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */
  1330. tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END);
  1331. if (!tmp)
  1332. return 0;
  1333. tmp += strlen(BRIDGE_STATS_END);
  1334. if (strlen(tmp) < ISO_TIME_LEN + 6)
  1335. return 0;
  1336. strlcpy(stats_end_str, tmp, sizeof(stats_end_str));
  1337. if (parse_iso_time(stats_end_str, &stats_end_time) < 0)
  1338. return 0;
  1339. if (stats_end_time < now - (25*60*60) ||
  1340. stats_end_time > now + (1*60*60))
  1341. return 0;
  1342. seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10);
  1343. if (!eos || seconds < 23*60*60)
  1344. return 0;
  1345. format_iso_time(stats_start_str, stats_end_time - seconds);
  1346. /* Parse: "bridge-ips CC=N,CC=N,..." */
  1347. tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS);
  1348. if (!tmp) {
  1349. /* Look if there is an empty "bridge-ips" line */
  1350. tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE);
  1351. if (!tmp)
  1352. return 0;
  1353. }
  1354. /* Parse: "bridge-ip-transports PT=N,PT=N,..." */
  1355. tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS);
  1356. if (!tmp) {
  1357. /* Look if there is an empty "bridge-ip-transports" line */
  1358. tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS_EMPTY_LINE);
  1359. if (!tmp)
  1360. return 0;
  1361. }
  1362. return 1;
  1363. }
  1364. /** Most recent bridge statistics formatted to be written to extra-info
  1365. * descriptors. */
  1366. static char *bridge_stats_extrainfo = NULL;
  1367. /** Return a newly allocated string holding our bridge usage stats by country
  1368. * in a format suitable for inclusion in an extrainfo document. Return NULL on
  1369. * failure. */
  1370. char *
  1371. geoip_format_bridge_stats(time_t now)
  1372. {
  1373. char *out = NULL;
  1374. char *country_data = NULL, *ipver_data = NULL, *transport_data = NULL;
  1375. long duration = now - start_of_bridge_stats_interval;
  1376. char written[ISO_TIME_LEN+1];
  1377. if (duration < 0)
  1378. return NULL;
  1379. if (!start_of_bridge_stats_interval)
  1380. return NULL; /* Not initialized. */
  1381. format_iso_time(written, now);
  1382. geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
  1383. transport_data = geoip_get_transport_history();
  1384. tor_asprintf(&out,
  1385. "bridge-stats-end %s (%ld s)\n"
  1386. "bridge-ips %s\n"
  1387. "bridge-ip-versions %s\n"
  1388. "bridge-ip-transports %s\n",
  1389. written, duration,
  1390. country_data ? country_data : "",
  1391. ipver_data ? ipver_data : "",
  1392. transport_data ? transport_data : "");
  1393. tor_free(country_data);
  1394. tor_free(ipver_data);
  1395. tor_free(transport_data);
  1396. return out;
  1397. }
  1398. /** Return a newly allocated string holding our bridge usage stats by country
  1399. * in a format suitable for the answer to a controller request. Return NULL on
  1400. * failure. */
  1401. static char *
  1402. format_bridge_stats_controller(time_t now)
  1403. {
  1404. char *out = NULL, *country_data = NULL, *ipver_data = NULL;
  1405. char started[ISO_TIME_LEN+1];
  1406. (void) now;
  1407. format_iso_time(started, start_of_bridge_stats_interval);
  1408. geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
  1409. tor_asprintf(&out,
  1410. "TimeStarted=\"%s\" CountrySummary=%s IPVersions=%s",
  1411. started,
  1412. country_data ? country_data : "",
  1413. ipver_data ? ipver_data : "");
  1414. tor_free(country_data);
  1415. tor_free(ipver_data);
  1416. return out;
  1417. }
  1418. /** Return a newly allocated string holding our bridge usage stats by
  1419. * country in a format suitable for inclusion in our heartbeat
  1420. * message. Return NULL on failure. */
  1421. char *
  1422. format_client_stats_heartbeat(time_t now)
  1423. {
  1424. const int n_hours = 6;
  1425. char *out = NULL;
  1426. int n_clients = 0;
  1427. clientmap_entry_t **ent;
  1428. unsigned cutoff = (unsigned)( (now-n_hours*3600)/60 );
  1429. if (!start_of_bridge_stats_interval)
  1430. return NULL; /* Not initialized. */
  1431. /* count unique IPs */
  1432. HT_FOREACH(ent, clientmap, &client_history) {
  1433. /* only count directly connecting clients */
  1434. if ((*ent)->action != GEOIP_CLIENT_CONNECT)
  1435. continue;
  1436. if ((*ent)->last_seen_in_minutes < cutoff)
  1437. continue;
  1438. n_clients++;
  1439. }
  1440. tor_asprintf(&out, "Heartbeat: "
  1441. "In the last %d hours, I have seen %d unique clients.",
  1442. n_hours,
  1443. n_clients);
  1444. return out;
  1445. }
  1446. /** Write bridge statistics to $DATADIR/stats/bridge-stats and return
  1447. * when we should next try to write statistics. */
  1448. time_t
  1449. geoip_bridge_stats_write(time_t now)
  1450. {
  1451. char *val = NULL;
  1452. /* Check if 24 hours have passed since starting measurements. */
  1453. if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL)
  1454. return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
  1455. /* Discard all items in the client history that are too old. */
  1456. geoip_remove_old_clients(start_of_bridge_stats_interval);
  1457. /* Generate formatted string */
  1458. val = geoip_format_bridge_stats(now);
  1459. if (val == NULL)
  1460. goto done;
  1461. /* Update the stored value. */
  1462. tor_free(bridge_stats_extrainfo);
  1463. bridge_stats_extrainfo = val;
  1464. start_of_bridge_stats_interval = now;
  1465. /* Write it to disk. */
  1466. if (!check_or_create_data_subdir("stats")) {
  1467. write_to_data_subdir("stats", "bridge-stats",
  1468. bridge_stats_extrainfo, "bridge statistics");
  1469. /* Tell the controller, "hey, there are clients!" */
  1470. {
  1471. char *controller_str = format_bridge_stats_controller(now);
  1472. if (controller_str)
  1473. control_event_clients_seen(controller_str);
  1474. tor_free(controller_str);
  1475. }
  1476. }
  1477. done:
  1478. return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
  1479. }
  1480. /** Try to load the most recent bridge statistics from disk, unless we
  1481. * have finished a measurement interval lately, and check whether they
  1482. * are still recent enough. */
  1483. static void
  1484. load_bridge_stats(time_t now)
  1485. {
  1486. char *fname, *contents;
  1487. if (bridge_stats_extrainfo)
  1488. return;
  1489. fname = get_datadir_fname2("stats", "bridge-stats");
  1490. contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
  1491. if (contents && validate_bridge_stats(contents, now)) {
  1492. bridge_stats_extrainfo = contents;
  1493. } else {
  1494. tor_free(contents);
  1495. }
  1496. tor_free(fname);
  1497. }
  1498. /** Return most recent bridge statistics for inclusion in extra-info
  1499. * descriptors, or NULL if we don't have recent bridge statistics. */
  1500. const char *
  1501. geoip_get_bridge_stats_extrainfo(time_t now)
  1502. {
  1503. load_bridge_stats(now);
  1504. return bridge_stats_extrainfo;
  1505. }
  1506. /** Return a new string containing the recent bridge statistics to be returned
  1507. * to controller clients, or NULL if we don't have any bridge statistics. */
  1508. char *
  1509. geoip_get_bridge_stats_controller(time_t now)
  1510. {
  1511. return format_bridge_stats_controller(now);
  1512. }
  1513. /** Start time of entry stats or 0 if we're not collecting entry
  1514. * statistics. */
  1515. static time_t start_of_entry_stats_interval;
  1516. /** Initialize entry stats. */
  1517. void
  1518. geoip_entry_stats_init(time_t now)
  1519. {
  1520. start_of_entry_stats_interval = now;
  1521. }
  1522. /** Reset counters for entry stats. */
  1523. void
  1524. geoip_reset_entry_stats(time_t now)
  1525. {
  1526. client_history_clear();
  1527. start_of_entry_stats_interval = now;
  1528. }
  1529. /** Stop collecting entry stats in a way that we can re-start doing so in
  1530. * geoip_entry_stats_init(). */
  1531. void
  1532. geoip_entry_stats_term(void)
  1533. {
  1534. geoip_reset_entry_stats(0);
  1535. }
  1536. /** Return a newly allocated string containing the entry statistics
  1537. * until <b>now</b>, or NULL if we're not collecting entry stats. Caller
  1538. * must ensure start_of_entry_stats_interval lies in the past. */
  1539. char *
  1540. geoip_format_entry_stats(time_t now)
  1541. {
  1542. char t[ISO_TIME_LEN+1];
  1543. char *data = NULL;
  1544. char *result;
  1545. if (!start_of_entry_stats_interval)
  1546. return NULL; /* Not initialized. */
  1547. tor_assert(now >= start_of_entry_stats_interval);
  1548. geoip_get_client_history(GEOIP_CLIENT_CONNECT, &data, NULL);
  1549. format_iso_time(t, now);
  1550. tor_asprintf(&result,
  1551. "entry-stats-end %s (%u s)\n"
  1552. "entry-ips %s\n",
  1553. t, (unsigned) (now - start_of_entry_stats_interval),
  1554. data ? data : "");
  1555. tor_free(data);
  1556. return result;
  1557. }
  1558. /** If 24 hours have passed since the beginning of the current entry stats
  1559. * period, write entry stats to $DATADIR/stats/entry-stats (possibly
  1560. * overwriting an existing file) and reset counters. Return when we would
  1561. * next want to write entry stats or 0 if we never want to write. */
  1562. time_t
  1563. geoip_entry_stats_write(time_t now)
  1564. {
  1565. char *str = NULL;
  1566. if (!start_of_entry_stats_interval)
  1567. return 0; /* Not initialized. */
  1568. if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now)
  1569. goto done; /* Not ready to write. */
  1570. /* Discard all items in the client history that are too old. */
  1571. geoip_remove_old_clients(start_of_entry_stats_interval);
  1572. /* Generate history string .*/
  1573. str = geoip_format_entry_stats(now);
  1574. /* Write entry-stats string to disk. */
  1575. if (!check_or_create_data_subdir("stats")) {
  1576. write_to_data_subdir("stats", "entry-stats", str, "entry statistics");
  1577. /* Reset measurement interval start. */
  1578. geoip_reset_entry_stats(now);
  1579. }
  1580. done:
  1581. tor_free(str);
  1582. return start_of_entry_stats_interval + WRITE_STATS_INTERVAL;
  1583. }
  1584. /** Helper used to implement GETINFO ip-to-country/... controller command. */
  1585. int
  1586. getinfo_helper_geoip(control_connection_t *control_conn,
  1587. const char *question, char **answer,
  1588. const char **errmsg)
  1589. {
  1590. (void)control_conn;
  1591. if (!strcmpstart(question, "ip-to-country/")) {
  1592. int c;
  1593. sa_family_t family;
  1594. tor_addr_t addr;
  1595. question += strlen("ip-to-country/");
  1596. if (!strcmp(question, "ipv4-available") ||
  1597. !strcmp(question, "ipv6-available")) {
  1598. family = !strcmp(question, "ipv4-available") ? AF_INET : AF_INET6;
  1599. const int available = geoip_is_loaded(family);
  1600. tor_asprintf(answer, "%d", !! available);
  1601. return 0;
  1602. }
  1603. family = tor_addr_parse(&addr, question);
  1604. if (family != AF_INET && family != AF_INET6) {
  1605. *errmsg = "Invalid address family";
  1606. return -1;
  1607. }
  1608. if (!geoip_is_loaded(family)) {
  1609. *errmsg = "GeoIP data not loaded";
  1610. return -1;
  1611. }
  1612. if (family == AF_INET)
  1613. c = geoip_get_country_by_ipv4(tor_addr_to_ipv4h(&addr));
  1614. else /* AF_INET6 */
  1615. c = geoip_get_country_by_ipv6(tor_addr_to_in6(&addr));
  1616. *answer = tor_strdup(geoip_get_country_name(c));
  1617. }
  1618. return 0;
  1619. }
  1620. /** Release all storage held by the GeoIP databases and country list. */
  1621. STATIC void
  1622. clear_geoip_db(void)
  1623. {
  1624. if (geoip_countries) {
  1625. SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, tor_free(c));
  1626. smartlist_free(geoip_countries);
  1627. }
  1628. strmap_free(country_idxplus1_by_lc_code, NULL);
  1629. if (geoip_ipv4_entries) {
  1630. SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, ent,
  1631. tor_free(ent));
  1632. smartlist_free(geoip_ipv4_entries);
  1633. }
  1634. if (geoip_ipv6_entries) {
  1635. SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, ent,
  1636. tor_free(ent));
  1637. smartlist_free(geoip_ipv6_entries);
  1638. }
  1639. geoip_countries = NULL;
  1640. country_idxplus1_by_lc_code = NULL;
  1641. geoip_ipv4_entries = NULL;
  1642. geoip_ipv6_entries = NULL;
  1643. }
  1644. /** Release all storage held in this file. */
  1645. void
  1646. geoip_free_all(void)
  1647. {
  1648. {
  1649. clientmap_entry_t **ent, **next, *this;
  1650. for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) {
  1651. this = *ent;
  1652. next = HT_NEXT_RMV(clientmap, &client_history, ent);
  1653. clientmap_entry_free(this);
  1654. }
  1655. HT_CLEAR(clientmap, &client_history);
  1656. }
  1657. {
  1658. dirreq_map_entry_t **ent, **next, *this;
  1659. for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
  1660. this = *ent;
  1661. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
  1662. tor_free(this);
  1663. }
  1664. HT_CLEAR(dirreqmap, &dirreq_map);
  1665. }
  1666. clear_geoip_db();
  1667. tor_free(bridge_stats_extrainfo);
  1668. }