geoip.c 62 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889
  1. /* Copyright (c) 2007-2018, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. /**
  4. * \file geoip.c
  5. * \brief Functions related to maintaining an IP-to-country database;
  6. * to summarizing client connections by country to entry guards, bridges,
  7. * and directory servers; and for statistics on answering network status
  8. * requests.
  9. *
  10. * There are two main kinds of functions in this module: geoip functions,
  11. * which map groups of IPv4 and IPv6 addresses to country codes, and
  12. * statistical functions, which collect statistics about different kinds of
  13. * per-country usage.
  14. *
  15. * The geoip lookup tables are implemented as sorted lists of disjoint address
  16. * ranges, each mapping to a singleton geoip_country_t. These country objects
  17. * are also indexed by their names in a hashtable.
  18. *
  19. * The tables are populated from disk at startup by the geoip_load_file()
  20. * function. For more information on the file format they read, see that
  21. * function. See the scripts and the README file in src/config for more
  22. * information about how those files are generated.
  23. *
  24. * Tor uses GeoIP information in order to implement user requests (such as
  25. * ExcludeNodes {cc}), and to keep track of how much usage relays are getting
  26. * for each country.
  27. */
  28. #define GEOIP_PRIVATE
  29. #include "or/or.h"
  30. #include "ht.h"
  31. #include "lib/container/buffers.h"
  32. #include "or/config.h"
  33. #include "or/control.h"
  34. #include "or/dnsserv.h"
  35. #include "or/dos.h"
  36. #include "or/geoip.h"
  37. #include "or/routerlist.h"
  38. #include "lib/container/order.h"
  39. #include "lib/time/tvdiff.h"
  40. static void init_geoip_countries(void);
  41. /** An entry from the GeoIP IPv4 file: maps an IPv4 range to a country. */
  42. typedef struct geoip_ipv4_entry_t {
  43. uint32_t ip_low; /**< The lowest IP in the range, in host order */
  44. uint32_t ip_high; /**< The highest IP in the range, in host order */
  45. intptr_t country; /**< An index into geoip_countries */
  46. } geoip_ipv4_entry_t;
  47. /** An entry from the GeoIP IPv6 file: maps an IPv6 range to a country. */
  48. typedef struct geoip_ipv6_entry_t {
  49. struct in6_addr ip_low; /**< The lowest IP in the range, in host order */
  50. struct in6_addr ip_high; /**< The highest IP in the range, in host order */
  51. intptr_t country; /**< An index into geoip_countries */
  52. } geoip_ipv6_entry_t;
  53. /** A per-country record for GeoIP request history. */
  54. typedef struct geoip_country_t {
  55. char countrycode[3];
  56. uint32_t n_v3_ns_requests;
  57. } geoip_country_t;
  58. /** A list of geoip_country_t */
  59. static smartlist_t *geoip_countries = NULL;
  60. /** A map from lowercased country codes to their position in geoip_countries.
  61. * The index is encoded in the pointer, and 1 is added so that NULL can mean
  62. * not found. */
  63. static strmap_t *country_idxplus1_by_lc_code = NULL;
  64. /** Lists of all known geoip_ipv4_entry_t and geoip_ipv6_entry_t, sorted
  65. * by their respective ip_low. */
  66. static smartlist_t *geoip_ipv4_entries = NULL, *geoip_ipv6_entries = NULL;
  67. /** SHA1 digest of the GeoIP files to include in extra-info descriptors. */
  68. static char geoip_digest[DIGEST_LEN];
  69. static char geoip6_digest[DIGEST_LEN];
  70. /* Total size in bytes of the geoip client history cache. Used by the OOM
  71. * handler. */
  72. static size_t geoip_client_history_cache_size;
  73. /* Increment the geoip client history cache size counter with the given bytes.
  74. * This prevents an overflow and set it to its maximum in that case. */
  75. static inline void
  76. geoip_increment_client_history_cache_size(size_t bytes)
  77. {
  78. /* This is shockingly high, lets log it so it can be reported. */
  79. IF_BUG_ONCE(geoip_client_history_cache_size > (SIZE_MAX - bytes)) {
  80. geoip_client_history_cache_size = SIZE_MAX;
  81. return;
  82. }
  83. geoip_client_history_cache_size += bytes;
  84. }
  85. /* Decrement the geoip client history cache size counter with the given bytes.
  86. * This prevents an underflow and set it to 0 in that case. */
  87. static inline void
  88. geoip_decrement_client_history_cache_size(size_t bytes)
  89. {
  90. /* Going below 0 means that we either allocated an entry without
  91. * incrementing the counter or we have different sizes when allocating and
  92. * freeing. It shouldn't happened so log it. */
  93. IF_BUG_ONCE(geoip_client_history_cache_size < bytes) {
  94. geoip_client_history_cache_size = 0;
  95. return;
  96. }
  97. geoip_client_history_cache_size -= bytes;
  98. }
  99. /** Return the index of the <b>country</b>'s entry in the GeoIP
  100. * country list if it is a valid 2-letter country code, otherwise
  101. * return -1. */
  102. MOCK_IMPL(country_t,
  103. geoip_get_country,(const char *country))
  104. {
  105. void *idxplus1_;
  106. intptr_t idx;
  107. idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country);
  108. if (!idxplus1_)
  109. return -1;
  110. idx = ((uintptr_t)idxplus1_)-1;
  111. return (country_t)idx;
  112. }
  113. /** Add an entry to a GeoIP table, mapping all IP addresses between <b>low</b>
  114. * and <b>high</b>, inclusive, to the 2-letter country code <b>country</b>. */
  115. static void
  116. geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high,
  117. const char *country)
  118. {
  119. intptr_t idx;
  120. void *idxplus1_;
  121. IF_BUG_ONCE(tor_addr_family(low) != tor_addr_family(high))
  122. return;
  123. IF_BUG_ONCE(tor_addr_compare(high, low, CMP_EXACT) < 0)
  124. return;
  125. idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country);
  126. if (!idxplus1_) {
  127. geoip_country_t *c = tor_malloc_zero(sizeof(geoip_country_t));
  128. strlcpy(c->countrycode, country, sizeof(c->countrycode));
  129. tor_strlower(c->countrycode);
  130. smartlist_add(geoip_countries, c);
  131. idx = smartlist_len(geoip_countries) - 1;
  132. strmap_set_lc(country_idxplus1_by_lc_code, country, (void*)(idx+1));
  133. } else {
  134. idx = ((uintptr_t)idxplus1_)-1;
  135. }
  136. {
  137. geoip_country_t *c = smartlist_get(geoip_countries, (int)idx);
  138. tor_assert(!strcasecmp(c->countrycode, country));
  139. }
  140. if (tor_addr_family(low) == AF_INET) {
  141. geoip_ipv4_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv4_entry_t));
  142. ent->ip_low = tor_addr_to_ipv4h(low);
  143. ent->ip_high = tor_addr_to_ipv4h(high);
  144. ent->country = idx;
  145. smartlist_add(geoip_ipv4_entries, ent);
  146. } else if (tor_addr_family(low) == AF_INET6) {
  147. geoip_ipv6_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv6_entry_t));
  148. ent->ip_low = *tor_addr_to_in6_assert(low);
  149. ent->ip_high = *tor_addr_to_in6_assert(high);
  150. ent->country = idx;
  151. smartlist_add(geoip_ipv6_entries, ent);
  152. }
  153. }
  154. /** Add an entry to the GeoIP table indicated by <b>family</b>,
  155. * parsing it from <b>line</b>. The format is as for geoip_load_file(). */
  156. STATIC int
  157. geoip_parse_entry(const char *line, sa_family_t family)
  158. {
  159. tor_addr_t low_addr, high_addr;
  160. char c[3];
  161. char *country = NULL;
  162. if (!geoip_countries)
  163. init_geoip_countries();
  164. if (family == AF_INET) {
  165. if (!geoip_ipv4_entries)
  166. geoip_ipv4_entries = smartlist_new();
  167. } else if (family == AF_INET6) {
  168. if (!geoip_ipv6_entries)
  169. geoip_ipv6_entries = smartlist_new();
  170. } else {
  171. log_warn(LD_GENERAL, "Unsupported family: %d", family);
  172. return -1;
  173. }
  174. while (TOR_ISSPACE(*line))
  175. ++line;
  176. if (*line == '#')
  177. return 0;
  178. char buf[512];
  179. if (family == AF_INET) {
  180. unsigned int low, high;
  181. if (tor_sscanf(line,"%u,%u,%2s", &low, &high, c) == 3 ||
  182. tor_sscanf(line,"\"%u\",\"%u\",\"%2s\",", &low, &high, c) == 3) {
  183. tor_addr_from_ipv4h(&low_addr, low);
  184. tor_addr_from_ipv4h(&high_addr, high);
  185. } else
  186. goto fail;
  187. country = c;
  188. } else { /* AF_INET6 */
  189. char *low_str, *high_str;
  190. struct in6_addr low, high;
  191. char *strtok_state;
  192. strlcpy(buf, line, sizeof(buf));
  193. low_str = tor_strtok_r(buf, ",", &strtok_state);
  194. if (!low_str)
  195. goto fail;
  196. high_str = tor_strtok_r(NULL, ",", &strtok_state);
  197. if (!high_str)
  198. goto fail;
  199. country = tor_strtok_r(NULL, "\n", &strtok_state);
  200. if (!country)
  201. goto fail;
  202. if (strlen(country) != 2)
  203. goto fail;
  204. if (tor_inet_pton(AF_INET6, low_str, &low) <= 0)
  205. goto fail;
  206. tor_addr_from_in6(&low_addr, &low);
  207. if (tor_inet_pton(AF_INET6, high_str, &high) <= 0)
  208. goto fail;
  209. tor_addr_from_in6(&high_addr, &high);
  210. }
  211. geoip_add_entry(&low_addr, &high_addr, country);
  212. return 0;
  213. fail:
  214. log_warn(LD_GENERAL, "Unable to parse line from GEOIP %s file: %s",
  215. family == AF_INET ? "IPv4" : "IPv6", escaped(line));
  216. return -1;
  217. }
  218. /** Sorting helper: return -1, 1, or 0 based on comparison of two
  219. * geoip_ipv4_entry_t */
  220. static int
  221. geoip_ipv4_compare_entries_(const void **_a, const void **_b)
  222. {
  223. const geoip_ipv4_entry_t *a = *_a, *b = *_b;
  224. if (a->ip_low < b->ip_low)
  225. return -1;
  226. else if (a->ip_low > b->ip_low)
  227. return 1;
  228. else
  229. return 0;
  230. }
  231. /** bsearch helper: return -1, 1, or 0 based on comparison of an IP (a pointer
  232. * to a uint32_t in host order) to a geoip_ipv4_entry_t */
  233. static int
  234. geoip_ipv4_compare_key_to_entry_(const void *_key, const void **_member)
  235. {
  236. /* No alignment issue here, since _key really is a pointer to uint32_t */
  237. const uint32_t addr = *(uint32_t *)_key;
  238. const geoip_ipv4_entry_t *entry = *_member;
  239. if (addr < entry->ip_low)
  240. return -1;
  241. else if (addr > entry->ip_high)
  242. return 1;
  243. else
  244. return 0;
  245. }
  246. /** Sorting helper: return -1, 1, or 0 based on comparison of two
  247. * geoip_ipv6_entry_t */
  248. static int
  249. geoip_ipv6_compare_entries_(const void **_a, const void **_b)
  250. {
  251. const geoip_ipv6_entry_t *a = *_a, *b = *_b;
  252. return fast_memcmp(a->ip_low.s6_addr, b->ip_low.s6_addr,
  253. sizeof(struct in6_addr));
  254. }
  255. /** bsearch helper: return -1, 1, or 0 based on comparison of an IPv6
  256. * (a pointer to a in6_addr) to a geoip_ipv6_entry_t */
  257. static int
  258. geoip_ipv6_compare_key_to_entry_(const void *_key, const void **_member)
  259. {
  260. const struct in6_addr *addr = (struct in6_addr *)_key;
  261. const geoip_ipv6_entry_t *entry = *_member;
  262. if (fast_memcmp(addr->s6_addr, entry->ip_low.s6_addr,
  263. sizeof(struct in6_addr)) < 0)
  264. return -1;
  265. else if (fast_memcmp(addr->s6_addr, entry->ip_high.s6_addr,
  266. sizeof(struct in6_addr)) > 0)
  267. return 1;
  268. else
  269. return 0;
  270. }
  271. /** Return 1 if we should collect geoip stats on bridge users, and
  272. * include them in our extrainfo descriptor. Else return 0. */
  273. int
  274. should_record_bridge_info(const or_options_t *options)
  275. {
  276. return options->BridgeRelay && options->BridgeRecordUsageByCountry;
  277. }
  278. /** Set up a new list of geoip countries with no countries (yet) set in it,
  279. * except for the unknown country.
  280. */
  281. static void
  282. init_geoip_countries(void)
  283. {
  284. geoip_country_t *geoip_unresolved;
  285. geoip_countries = smartlist_new();
  286. /* Add a geoip_country_t for requests that could not be resolved to a
  287. * country as first element (index 0) to geoip_countries. */
  288. geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t));
  289. strlcpy(geoip_unresolved->countrycode, "??",
  290. sizeof(geoip_unresolved->countrycode));
  291. smartlist_add(geoip_countries, geoip_unresolved);
  292. country_idxplus1_by_lc_code = strmap_new();
  293. strmap_set_lc(country_idxplus1_by_lc_code, "??", (void*)(1));
  294. }
  295. /** Clear appropriate GeoIP database, based on <b>family</b>, and
  296. * reload it from the file <b>filename</b>. Return 0 on success, -1 on
  297. * failure.
  298. *
  299. * Recognized line formats for IPv4 are:
  300. * INTIPLOW,INTIPHIGH,CC
  301. * and
  302. * "INTIPLOW","INTIPHIGH","CC","CC3","COUNTRY NAME"
  303. * where INTIPLOW and INTIPHIGH are IPv4 addresses encoded as 4-byte unsigned
  304. * integers, and CC is a country code.
  305. *
  306. * Recognized line format for IPv6 is:
  307. * IPV6LOW,IPV6HIGH,CC
  308. * where IPV6LOW and IPV6HIGH are IPv6 addresses and CC is a country code.
  309. *
  310. * It also recognizes, and skips over, blank lines and lines that start
  311. * with '#' (comments).
  312. */
  313. int
  314. geoip_load_file(sa_family_t family, const char *filename)
  315. {
  316. FILE *f;
  317. const char *msg = "";
  318. const or_options_t *options = get_options();
  319. int severity = options_need_geoip_info(options, &msg) ? LOG_WARN : LOG_INFO;
  320. crypto_digest_t *geoip_digest_env = NULL;
  321. tor_assert(family == AF_INET || family == AF_INET6);
  322. if (!(f = tor_fopen_cloexec(filename, "r"))) {
  323. log_fn(severity, LD_GENERAL, "Failed to open GEOIP file %s. %s",
  324. filename, msg);
  325. return -1;
  326. }
  327. if (!geoip_countries)
  328. init_geoip_countries();
  329. if (family == AF_INET) {
  330. if (geoip_ipv4_entries) {
  331. SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, e,
  332. tor_free(e));
  333. smartlist_free(geoip_ipv4_entries);
  334. }
  335. geoip_ipv4_entries = smartlist_new();
  336. } else { /* AF_INET6 */
  337. if (geoip_ipv6_entries) {
  338. SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, e,
  339. tor_free(e));
  340. smartlist_free(geoip_ipv6_entries);
  341. }
  342. geoip_ipv6_entries = smartlist_new();
  343. }
  344. geoip_digest_env = crypto_digest_new();
  345. log_notice(LD_GENERAL, "Parsing GEOIP %s file %s.",
  346. (family == AF_INET) ? "IPv4" : "IPv6", filename);
  347. while (!feof(f)) {
  348. char buf[512];
  349. if (fgets(buf, (int)sizeof(buf), f) == NULL)
  350. break;
  351. crypto_digest_add_bytes(geoip_digest_env, buf, strlen(buf));
  352. /* FFFF track full country name. */
  353. geoip_parse_entry(buf, family);
  354. }
  355. /*XXXX abort and return -1 if no entries/illformed?*/
  356. fclose(f);
  357. /* Sort list and remember file digests so that we can include it in
  358. * our extra-info descriptors. */
  359. if (family == AF_INET) {
  360. smartlist_sort(geoip_ipv4_entries, geoip_ipv4_compare_entries_);
  361. /* Okay, now we need to maybe change our mind about what is in
  362. * which country. We do this for IPv4 only since that's what we
  363. * store in node->country. */
  364. refresh_all_country_info();
  365. crypto_digest_get_digest(geoip_digest_env, geoip_digest, DIGEST_LEN);
  366. } else {
  367. /* AF_INET6 */
  368. smartlist_sort(geoip_ipv6_entries, geoip_ipv6_compare_entries_);
  369. crypto_digest_get_digest(geoip_digest_env, geoip6_digest, DIGEST_LEN);
  370. }
  371. crypto_digest_free(geoip_digest_env);
  372. return 0;
  373. }
  374. /** Given an IP address in host order, return a number representing the
  375. * country to which that address belongs, -1 for "No geoip information
  376. * available", or 0 for the 'unknown country'. The return value will always
  377. * be less than geoip_get_n_countries(). To decode it, call
  378. * geoip_get_country_name().
  379. */
  380. STATIC int
  381. geoip_get_country_by_ipv4(uint32_t ipaddr)
  382. {
  383. geoip_ipv4_entry_t *ent;
  384. if (!geoip_ipv4_entries)
  385. return -1;
  386. ent = smartlist_bsearch(geoip_ipv4_entries, &ipaddr,
  387. geoip_ipv4_compare_key_to_entry_);
  388. return ent ? (int)ent->country : 0;
  389. }
  390. /** Given an IPv6 address, return a number representing the country to
  391. * which that address belongs, -1 for "No geoip information available", or
  392. * 0 for the 'unknown country'. The return value will always be less than
  393. * geoip_get_n_countries(). To decode it, call geoip_get_country_name().
  394. */
  395. STATIC int
  396. geoip_get_country_by_ipv6(const struct in6_addr *addr)
  397. {
  398. geoip_ipv6_entry_t *ent;
  399. if (!geoip_ipv6_entries)
  400. return -1;
  401. ent = smartlist_bsearch(geoip_ipv6_entries, addr,
  402. geoip_ipv6_compare_key_to_entry_);
  403. return ent ? (int)ent->country : 0;
  404. }
  405. /** Given an IP address, return a number representing the country to which
  406. * that address belongs, -1 for "No geoip information available", or 0 for
  407. * the 'unknown country'. The return value will always be less than
  408. * geoip_get_n_countries(). To decode it, call geoip_get_country_name().
  409. */
  410. MOCK_IMPL(int,
  411. geoip_get_country_by_addr,(const tor_addr_t *addr))
  412. {
  413. if (tor_addr_family(addr) == AF_INET) {
  414. return geoip_get_country_by_ipv4(tor_addr_to_ipv4h(addr));
  415. } else if (tor_addr_family(addr) == AF_INET6) {
  416. return geoip_get_country_by_ipv6(tor_addr_to_in6(addr));
  417. } else {
  418. return -1;
  419. }
  420. }
  421. /** Return the number of countries recognized by the GeoIP country list. */
  422. MOCK_IMPL(int,
  423. geoip_get_n_countries,(void))
  424. {
  425. if (!geoip_countries)
  426. init_geoip_countries();
  427. return (int) smartlist_len(geoip_countries);
  428. }
  429. /** Return the two-letter country code associated with the number <b>num</b>,
  430. * or "??" for an unknown value. */
  431. const char *
  432. geoip_get_country_name(country_t num)
  433. {
  434. if (geoip_countries && num >= 0 && num < smartlist_len(geoip_countries)) {
  435. geoip_country_t *c = smartlist_get(geoip_countries, num);
  436. return c->countrycode;
  437. } else
  438. return "??";
  439. }
  440. /** Return true iff we have loaded a GeoIP database.*/
  441. MOCK_IMPL(int,
  442. geoip_is_loaded,(sa_family_t family))
  443. {
  444. tor_assert(family == AF_INET || family == AF_INET6);
  445. if (geoip_countries == NULL)
  446. return 0;
  447. if (family == AF_INET)
  448. return geoip_ipv4_entries != NULL;
  449. else /* AF_INET6 */
  450. return geoip_ipv6_entries != NULL;
  451. }
  452. /** Return the hex-encoded SHA1 digest of the loaded GeoIP file. The
  453. * result does not need to be deallocated, but will be overwritten by the
  454. * next call of hex_str(). */
  455. const char *
  456. geoip_db_digest(sa_family_t family)
  457. {
  458. tor_assert(family == AF_INET || family == AF_INET6);
  459. if (family == AF_INET)
  460. return hex_str(geoip_digest, DIGEST_LEN);
  461. else /* AF_INET6 */
  462. return hex_str(geoip6_digest, DIGEST_LEN);
  463. }
  464. /** Largest allowable value for last_seen_in_minutes. (It's a 30-bit field,
  465. * so it can hold up to (1u<<30)-1, or 0x3fffffffu.
  466. */
  467. #define MAX_LAST_SEEN_IN_MINUTES 0X3FFFFFFFu
  468. /** Map from client IP address to last time seen. */
  469. static HT_HEAD(clientmap, clientmap_entry_t) client_history =
  470. HT_INITIALIZER();
  471. /** Hashtable helper: compute a hash of a clientmap_entry_t. */
  472. static inline unsigned
  473. clientmap_entry_hash(const clientmap_entry_t *a)
  474. {
  475. unsigned h = (unsigned) tor_addr_hash(&a->addr);
  476. if (a->transport_name)
  477. h += (unsigned) siphash24g(a->transport_name, strlen(a->transport_name));
  478. return h;
  479. }
  480. /** Hashtable helper: compare two clientmap_entry_t values for equality. */
  481. static inline int
  482. clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b)
  483. {
  484. if (strcmp_opt(a->transport_name, b->transport_name))
  485. return 0;
  486. return !tor_addr_compare(&a->addr, &b->addr, CMP_EXACT) &&
  487. a->action == b->action;
  488. }
  489. HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
  490. clientmap_entries_eq)
  491. HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
  492. clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
  493. #define clientmap_entry_free(ent) \
  494. FREE_AND_NULL(clientmap_entry_t, clientmap_entry_free_, ent)
  495. /** Return the size of a client map entry. */
  496. static inline size_t
  497. clientmap_entry_size(const clientmap_entry_t *ent)
  498. {
  499. tor_assert(ent);
  500. return (sizeof(clientmap_entry_t) +
  501. (ent->transport_name ? strlen(ent->transport_name) : 0));
  502. }
  503. /** Free all storage held by <b>ent</b>. */
  504. static void
  505. clientmap_entry_free_(clientmap_entry_t *ent)
  506. {
  507. if (!ent)
  508. return;
  509. /* This entry is about to be freed so pass it to the DoS subsystem to see if
  510. * any actions can be taken about it. */
  511. dos_geoip_entry_about_to_free(ent);
  512. geoip_decrement_client_history_cache_size(clientmap_entry_size(ent));
  513. tor_free(ent->transport_name);
  514. tor_free(ent);
  515. }
  516. /* Return a newly allocated clientmap entry with the given action and address
  517. * that are mandatory. The transport_name can be optional. This can't fail. */
  518. static clientmap_entry_t *
  519. clientmap_entry_new(geoip_client_action_t action, const tor_addr_t *addr,
  520. const char *transport_name)
  521. {
  522. clientmap_entry_t *entry;
  523. tor_assert(action == GEOIP_CLIENT_CONNECT ||
  524. action == GEOIP_CLIENT_NETWORKSTATUS);
  525. tor_assert(addr);
  526. entry = tor_malloc_zero(sizeof(clientmap_entry_t));
  527. entry->action = action;
  528. tor_addr_copy(&entry->addr, addr);
  529. if (transport_name) {
  530. entry->transport_name = tor_strdup(transport_name);
  531. }
  532. /* Allocated and initialized, note down its size for the OOM handler. */
  533. geoip_increment_client_history_cache_size(clientmap_entry_size(entry));
  534. return entry;
  535. }
  536. /** Clear history of connecting clients used by entry and bridge stats. */
  537. static void
  538. client_history_clear(void)
  539. {
  540. clientmap_entry_t **ent, **next, *this;
  541. for (ent = HT_START(clientmap, &client_history); ent != NULL;
  542. ent = next) {
  543. if ((*ent)->action == GEOIP_CLIENT_CONNECT) {
  544. this = *ent;
  545. next = HT_NEXT_RMV(clientmap, &client_history, ent);
  546. clientmap_entry_free(this);
  547. } else {
  548. next = HT_NEXT(clientmap, &client_history, ent);
  549. }
  550. }
  551. }
  552. /** Note that we've seen a client connect from the IP <b>addr</b>
  553. * at time <b>now</b>. Ignored by all but bridges and directories if
  554. * configured accordingly. */
  555. void
  556. geoip_note_client_seen(geoip_client_action_t action,
  557. const tor_addr_t *addr,
  558. const char *transport_name,
  559. time_t now)
  560. {
  561. const or_options_t *options = get_options();
  562. clientmap_entry_t *ent;
  563. if (action == GEOIP_CLIENT_CONNECT) {
  564. /* Only remember statistics if the DoS mitigation subsystem is enabled. If
  565. * not, only if as entry guard or as bridge. */
  566. if (!dos_enabled()) {
  567. if (!options->EntryStatistics && !should_record_bridge_info(options)) {
  568. return;
  569. }
  570. }
  571. } else {
  572. /* Only gather directory-request statistics if configured, and
  573. * forcibly disable them on bridge authorities. */
  574. if (!options->DirReqStatistics || options->BridgeAuthoritativeDir)
  575. return;
  576. }
  577. log_debug(LD_GENERAL, "Seen client from '%s' with transport '%s'.",
  578. safe_str_client(fmt_addr((addr))),
  579. transport_name ? transport_name : "<no transport>");
  580. ent = geoip_lookup_client(addr, transport_name, action);
  581. if (! ent) {
  582. ent = clientmap_entry_new(action, addr, transport_name);
  583. HT_INSERT(clientmap, &client_history, ent);
  584. }
  585. if (now / 60 <= (int)MAX_LAST_SEEN_IN_MINUTES && now >= 0)
  586. ent->last_seen_in_minutes = (unsigned)(now/60);
  587. else
  588. ent->last_seen_in_minutes = 0;
  589. if (action == GEOIP_CLIENT_NETWORKSTATUS) {
  590. int country_idx = geoip_get_country_by_addr(addr);
  591. if (country_idx < 0)
  592. country_idx = 0; /** unresolved requests are stored at index 0. */
  593. if (country_idx >= 0 && country_idx < smartlist_len(geoip_countries)) {
  594. geoip_country_t *country = smartlist_get(geoip_countries, country_idx);
  595. ++country->n_v3_ns_requests;
  596. }
  597. }
  598. }
  599. /** HT_FOREACH helper: remove a clientmap_entry_t from the hashtable if it's
  600. * older than a certain time. */
  601. static int
  602. remove_old_client_helper_(struct clientmap_entry_t *ent, void *_cutoff)
  603. {
  604. time_t cutoff = *(time_t*)_cutoff / 60;
  605. if (ent->last_seen_in_minutes < cutoff) {
  606. clientmap_entry_free(ent);
  607. return 1;
  608. } else {
  609. return 0;
  610. }
  611. }
  612. /** Forget about all clients that haven't connected since <b>cutoff</b>. */
  613. void
  614. geoip_remove_old_clients(time_t cutoff)
  615. {
  616. clientmap_HT_FOREACH_FN(&client_history,
  617. remove_old_client_helper_,
  618. &cutoff);
  619. }
  620. /* Return a client entry object matching the given address, transport name and
  621. * geoip action from the clientmap. NULL if not found. The transport_name can
  622. * be NULL. */
  623. clientmap_entry_t *
  624. geoip_lookup_client(const tor_addr_t *addr, const char *transport_name,
  625. geoip_client_action_t action)
  626. {
  627. clientmap_entry_t lookup;
  628. tor_assert(addr);
  629. /* We always look for a client connection with no transport. */
  630. tor_addr_copy(&lookup.addr, addr);
  631. lookup.action = action;
  632. lookup.transport_name = (char *) transport_name;
  633. return HT_FIND(clientmap, &client_history, &lookup);
  634. }
  635. /* Cleanup client entries older than the cutoff. Used for the OOM. Return the
  636. * number of bytes freed. If 0 is returned, nothing was freed. */
  637. static size_t
  638. oom_clean_client_entries(time_t cutoff)
  639. {
  640. size_t bytes = 0;
  641. clientmap_entry_t **ent, **ent_next;
  642. for (ent = HT_START(clientmap, &client_history); ent; ent = ent_next) {
  643. clientmap_entry_t *entry = *ent;
  644. if (entry->last_seen_in_minutes < (cutoff / 60)) {
  645. ent_next = HT_NEXT_RMV(clientmap, &client_history, ent);
  646. bytes += clientmap_entry_size(entry);
  647. clientmap_entry_free(entry);
  648. } else {
  649. ent_next = HT_NEXT(clientmap, &client_history, ent);
  650. }
  651. }
  652. return bytes;
  653. }
  654. /* Below this minimum lifetime, the OOM won't cleanup any entries. */
  655. #define GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF (4 * 60 * 60)
  656. /* The OOM moves the cutoff by that much every run. */
  657. #define GEOIP_CLIENT_CACHE_OOM_STEP (15 * 50)
  658. /* Cleanup the geoip client history cache called from the OOM handler. Return
  659. * the amount of bytes removed. This can return a value below or above
  660. * min_remove_bytes but will stop as oon as the min_remove_bytes has been
  661. * reached. */
  662. size_t
  663. geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes)
  664. {
  665. time_t k;
  666. size_t bytes_removed = 0;
  667. /* Our OOM handler called with 0 bytes to remove is a code flow error. */
  668. tor_assert(min_remove_bytes != 0);
  669. /* Set k to the initial cutoff of an entry. We then going to move it by step
  670. * to try to remove as much as we can. */
  671. k = WRITE_STATS_INTERVAL;
  672. do {
  673. time_t cutoff;
  674. /* If k has reached the minimum lifetime, we have to stop else we might
  675. * remove every single entries which would be pretty bad for the DoS
  676. * mitigation subsystem if by just filling the geoip cache, it was enough
  677. * to trigger the OOM and clean every single entries. */
  678. if (k <= GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF) {
  679. break;
  680. }
  681. cutoff = now - k;
  682. bytes_removed += oom_clean_client_entries(cutoff);
  683. k -= GEOIP_CLIENT_CACHE_OOM_STEP;
  684. } while (bytes_removed < min_remove_bytes);
  685. return bytes_removed;
  686. }
  687. /* Return the total size in bytes of the client history cache. */
  688. size_t
  689. geoip_client_cache_total_allocation(void)
  690. {
  691. return geoip_client_history_cache_size;
  692. }
  693. /** How many responses are we giving to clients requesting v3 network
  694. * statuses? */
  695. static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM];
  696. /** Note that we've rejected a client's request for a v3 network status
  697. * for reason <b>reason</b> at time <b>now</b>. */
  698. void
  699. geoip_note_ns_response(geoip_ns_response_t response)
  700. {
  701. static int arrays_initialized = 0;
  702. if (!get_options()->DirReqStatistics)
  703. return;
  704. if (!arrays_initialized) {
  705. memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
  706. arrays_initialized = 1;
  707. }
  708. tor_assert(response < GEOIP_NS_RESPONSE_NUM);
  709. ns_v3_responses[response]++;
  710. }
  711. /** Do not mention any country from which fewer than this number of IPs have
  712. * connected. This conceivably avoids reporting information that could
  713. * deanonymize users, though analysis is lacking. */
  714. #define MIN_IPS_TO_NOTE_COUNTRY 1
  715. /** Do not report any geoip data at all if we have fewer than this number of
  716. * IPs to report about. */
  717. #define MIN_IPS_TO_NOTE_ANYTHING 1
  718. /** When reporting geoip data about countries, round up to the nearest
  719. * multiple of this value. */
  720. #define IP_GRANULARITY 8
  721. /** Helper type: used to sort per-country totals by value. */
  722. typedef struct c_hist_t {
  723. char country[3]; /**< Two-letter country code. */
  724. unsigned total; /**< Total IP addresses seen in this country. */
  725. } c_hist_t;
  726. /** Sorting helper: return -1, 1, or 0 based on comparison of two
  727. * geoip_ipv4_entry_t. Sort in descending order of total, and then by country
  728. * code. */
  729. static int
  730. c_hist_compare_(const void **_a, const void **_b)
  731. {
  732. const c_hist_t *a = *_a, *b = *_b;
  733. if (a->total > b->total)
  734. return -1;
  735. else if (a->total < b->total)
  736. return 1;
  737. else
  738. return strcmp(a->country, b->country);
  739. }
  740. /** When there are incomplete directory requests at the end of a 24-hour
  741. * period, consider those requests running for longer than this timeout as
  742. * failed, the others as still running. */
  743. #define DIRREQ_TIMEOUT (10*60)
  744. /** Entry in a map from either chan->global_identifier for direct requests
  745. * or a unique circuit identifier for tunneled requests to request time,
  746. * response size, and completion time of a network status request. Used to
  747. * measure download times of requests to derive average client
  748. * bandwidths. */
  749. typedef struct dirreq_map_entry_t {
  750. HT_ENTRY(dirreq_map_entry_t) node;
  751. /** Unique identifier for this network status request; this is either the
  752. * chan->global_identifier of the dir channel (direct request) or a new
  753. * locally unique identifier of a circuit (tunneled request). This ID is
  754. * only unique among other direct or tunneled requests, respectively. */
  755. uint64_t dirreq_id;
  756. unsigned int state:3; /**< State of this directory request. */
  757. unsigned int type:1; /**< Is this a direct or a tunneled request? */
  758. unsigned int completed:1; /**< Is this request complete? */
  759. /** When did we receive the request and started sending the response? */
  760. struct timeval request_time;
  761. size_t response_size; /**< What is the size of the response in bytes? */
  762. struct timeval completion_time; /**< When did the request succeed? */
  763. } dirreq_map_entry_t;
  764. /** Map of all directory requests asking for v2 or v3 network statuses in
  765. * the current geoip-stats interval. Values are
  766. * of type *<b>dirreq_map_entry_t</b>. */
  767. static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map =
  768. HT_INITIALIZER();
  769. static int
  770. dirreq_map_ent_eq(const dirreq_map_entry_t *a,
  771. const dirreq_map_entry_t *b)
  772. {
  773. return a->dirreq_id == b->dirreq_id && a->type == b->type;
  774. }
  775. /* DOCDOC dirreq_map_ent_hash */
  776. static unsigned
  777. dirreq_map_ent_hash(const dirreq_map_entry_t *entry)
  778. {
  779. unsigned u = (unsigned) entry->dirreq_id;
  780. u += entry->type << 20;
  781. return u;
  782. }
  783. HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
  784. dirreq_map_ent_eq)
  785. HT_GENERATE2(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
  786. dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_)
  787. /** Helper: Put <b>entry</b> into map of directory requests using
  788. * <b>type</b> and <b>dirreq_id</b> as key parts. If there is
  789. * already an entry for that key, print out a BUG warning and return. */
  790. static void
  791. dirreq_map_put_(dirreq_map_entry_t *entry, dirreq_type_t type,
  792. uint64_t dirreq_id)
  793. {
  794. dirreq_map_entry_t *old_ent;
  795. tor_assert(entry->type == type);
  796. tor_assert(entry->dirreq_id == dirreq_id);
  797. /* XXXX we could switch this to HT_INSERT some time, since it seems that
  798. * this bug doesn't happen. But since this function doesn't seem to be
  799. * critical-path, it's sane to leave it alone. */
  800. old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry);
  801. if (old_ent && old_ent != entry) {
  802. log_warn(LD_BUG, "Error when putting directory request into local "
  803. "map. There was already an entry for the same identifier.");
  804. return;
  805. }
  806. }
  807. /** Helper: Look up and return an entry in the map of directory requests
  808. * using <b>type</b> and <b>dirreq_id</b> as key parts. If there
  809. * is no such entry, return NULL. */
  810. static dirreq_map_entry_t *
  811. dirreq_map_get_(dirreq_type_t type, uint64_t dirreq_id)
  812. {
  813. dirreq_map_entry_t lookup;
  814. lookup.type = type;
  815. lookup.dirreq_id = dirreq_id;
  816. return HT_FIND(dirreqmap, &dirreq_map, &lookup);
  817. }
  818. /** Note that an either direct or tunneled (see <b>type</b>) directory
  819. * request for a v3 network status with unique ID <b>dirreq_id</b> of size
  820. * <b>response_size</b> has started. */
  821. void
  822. geoip_start_dirreq(uint64_t dirreq_id, size_t response_size,
  823. dirreq_type_t type)
  824. {
  825. dirreq_map_entry_t *ent;
  826. if (!get_options()->DirReqStatistics)
  827. return;
  828. ent = tor_malloc_zero(sizeof(dirreq_map_entry_t));
  829. ent->dirreq_id = dirreq_id;
  830. tor_gettimeofday(&ent->request_time);
  831. ent->response_size = response_size;
  832. ent->type = type;
  833. dirreq_map_put_(ent, type, dirreq_id);
  834. }
  835. /** Change the state of the either direct or tunneled (see <b>type</b>)
  836. * directory request with <b>dirreq_id</b> to <b>new_state</b> and
  837. * possibly mark it as completed. If no entry can be found for the given
  838. * key parts (e.g., if this is a directory request that we are not
  839. * measuring, or one that was started in the previous measurement period),
  840. * or if the state cannot be advanced to <b>new_state</b>, do nothing. */
  841. void
  842. geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
  843. dirreq_state_t new_state)
  844. {
  845. dirreq_map_entry_t *ent;
  846. if (!get_options()->DirReqStatistics)
  847. return;
  848. ent = dirreq_map_get_(type, dirreq_id);
  849. if (!ent)
  850. return;
  851. if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS)
  852. return;
  853. if (new_state - 1 != ent->state)
  854. return;
  855. ent->state = new_state;
  856. if ((type == DIRREQ_DIRECT &&
  857. new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) ||
  858. (type == DIRREQ_TUNNELED &&
  859. new_state == DIRREQ_CHANNEL_BUFFER_FLUSHED)) {
  860. tor_gettimeofday(&ent->completion_time);
  861. ent->completed = 1;
  862. }
  863. }
  864. /** Return the bridge-ip-transports string that should be inserted in
  865. * our extra-info descriptor. Return NULL if the bridge-ip-transports
  866. * line should be empty. */
  867. char *
  868. geoip_get_transport_history(void)
  869. {
  870. unsigned granularity = IP_GRANULARITY;
  871. /** String hash table (name of transport) -> (number of users). */
  872. strmap_t *transport_counts = strmap_new();
  873. /** Smartlist that contains copies of the names of the transports
  874. that have been used. */
  875. smartlist_t *transports_used = smartlist_new();
  876. /* Special string to signify that no transport was used for this
  877. connection. Pluggable transport names can't have symbols in their
  878. names, so this string will never collide with a real transport. */
  879. static const char* no_transport_str = "<OR>";
  880. clientmap_entry_t **ent;
  881. smartlist_t *string_chunks = smartlist_new();
  882. char *the_string = NULL;
  883. /* If we haven't seen any clients yet, return NULL. */
  884. if (HT_EMPTY(&client_history))
  885. goto done;
  886. /** We do the following steps to form the transport history string:
  887. * a) Foreach client that uses a pluggable transport, we increase the
  888. * times that transport was used by one. If the client did not use
  889. * a transport, we increase the number of times someone connected
  890. * without obfuscation.
  891. * b) Foreach transport we observed, we write its transport history
  892. * string and push it to string_chunks. So, for example, if we've
  893. * seen 665 obfs2 clients, we write "obfs2=665".
  894. * c) We concatenate string_chunks to form the final string.
  895. */
  896. log_debug(LD_GENERAL,"Starting iteration for transport history. %d clients.",
  897. HT_SIZE(&client_history));
  898. /* Loop through all clients. */
  899. HT_FOREACH(ent, clientmap, &client_history) {
  900. uintptr_t val;
  901. void *ptr;
  902. const char *transport_name = (*ent)->transport_name;
  903. if (!transport_name)
  904. transport_name = no_transport_str;
  905. /* Increase the count for this transport name. */
  906. ptr = strmap_get(transport_counts, transport_name);
  907. val = (uintptr_t)ptr;
  908. val++;
  909. ptr = (void*)val;
  910. strmap_set(transport_counts, transport_name, ptr);
  911. /* If it's the first time we see this transport, note it. */
  912. if (val == 1)
  913. smartlist_add_strdup(transports_used, transport_name);
  914. log_debug(LD_GENERAL, "Client from '%s' with transport '%s'. "
  915. "I've now seen %d clients.",
  916. safe_str_client(fmt_addr(&(*ent)->addr)),
  917. transport_name ? transport_name : "<no transport>",
  918. (int)val);
  919. }
  920. /* Sort the transport names (helps with unit testing). */
  921. smartlist_sort_strings(transports_used);
  922. /* Loop through all seen transports. */
  923. SMARTLIST_FOREACH_BEGIN(transports_used, const char *, transport_name) {
  924. void *transport_count_ptr = strmap_get(transport_counts, transport_name);
  925. uintptr_t transport_count = (uintptr_t) transport_count_ptr;
  926. log_debug(LD_GENERAL, "We got "U64_FORMAT" clients with transport '%s'.",
  927. U64_PRINTF_ARG((uint64_t)transport_count), transport_name);
  928. smartlist_add_asprintf(string_chunks, "%s="U64_FORMAT,
  929. transport_name,
  930. U64_PRINTF_ARG(round_uint64_to_next_multiple_of(
  931. (uint64_t)transport_count,
  932. granularity)));
  933. } SMARTLIST_FOREACH_END(transport_name);
  934. the_string = smartlist_join_strings(string_chunks, ",", 0, NULL);
  935. log_debug(LD_GENERAL, "Final bridge-ip-transports string: '%s'", the_string);
  936. done:
  937. strmap_free(transport_counts, NULL);
  938. SMARTLIST_FOREACH(transports_used, char *, s, tor_free(s));
  939. smartlist_free(transports_used);
  940. SMARTLIST_FOREACH(string_chunks, char *, s, tor_free(s));
  941. smartlist_free(string_chunks);
  942. return the_string;
  943. }
  944. /** Return a newly allocated comma-separated string containing statistics
  945. * on network status downloads. The string contains the number of completed
  946. * requests, timeouts, and still running requests as well as the download
  947. * times by deciles and quartiles. Return NULL if we have not observed
  948. * requests for long enough. */
  949. static char *
  950. geoip_get_dirreq_history(dirreq_type_t type)
  951. {
  952. char *result = NULL;
  953. buf_t *buf = NULL;
  954. smartlist_t *dirreq_completed = NULL;
  955. uint32_t complete = 0, timeouts = 0, running = 0;
  956. dirreq_map_entry_t **ptr, **next;
  957. struct timeval now;
  958. tor_gettimeofday(&now);
  959. dirreq_completed = smartlist_new();
  960. for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) {
  961. dirreq_map_entry_t *ent = *ptr;
  962. if (ent->type != type) {
  963. next = HT_NEXT(dirreqmap, &dirreq_map, ptr);
  964. continue;
  965. } else {
  966. if (ent->completed) {
  967. smartlist_add(dirreq_completed, ent);
  968. complete++;
  969. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
  970. } else {
  971. if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT)
  972. timeouts++;
  973. else
  974. running++;
  975. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
  976. tor_free(ent);
  977. }
  978. }
  979. }
  980. #define DIR_REQ_GRANULARITY 4
  981. complete = round_uint32_to_next_multiple_of(complete,
  982. DIR_REQ_GRANULARITY);
  983. timeouts = round_uint32_to_next_multiple_of(timeouts,
  984. DIR_REQ_GRANULARITY);
  985. running = round_uint32_to_next_multiple_of(running,
  986. DIR_REQ_GRANULARITY);
  987. buf = buf_new_with_capacity(1024);
  988. buf_add_printf(buf, "complete=%u,timeout=%u,"
  989. "running=%u", complete, timeouts, running);
  990. #define MIN_DIR_REQ_RESPONSES 16
  991. if (complete >= MIN_DIR_REQ_RESPONSES) {
  992. uint32_t *dltimes;
  993. /* We may have rounded 'completed' up. Here we want to use the
  994. * real value. */
  995. complete = smartlist_len(dirreq_completed);
  996. dltimes = tor_calloc(complete, sizeof(uint32_t));
  997. SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) {
  998. uint32_t bytes_per_second;
  999. uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time,
  1000. &ent->completion_time);
  1001. if (time_diff == 0)
  1002. time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible
  1003. * by law of nature or something, but a millisecond
  1004. * is a bit greater than "instantly" */
  1005. bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff);
  1006. dltimes[ent_sl_idx] = bytes_per_second;
  1007. } SMARTLIST_FOREACH_END(ent);
  1008. median_uint32(dltimes, complete); /* sorts as a side effect. */
  1009. buf_add_printf(buf,
  1010. ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u,"
  1011. "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u",
  1012. dltimes[0],
  1013. dltimes[1*complete/10-1],
  1014. dltimes[2*complete/10-1],
  1015. dltimes[1*complete/4-1],
  1016. dltimes[3*complete/10-1],
  1017. dltimes[4*complete/10-1],
  1018. dltimes[5*complete/10-1],
  1019. dltimes[6*complete/10-1],
  1020. dltimes[7*complete/10-1],
  1021. dltimes[3*complete/4-1],
  1022. dltimes[8*complete/10-1],
  1023. dltimes[9*complete/10-1],
  1024. dltimes[complete-1]);
  1025. tor_free(dltimes);
  1026. }
  1027. result = buf_extract(buf, NULL);
  1028. SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent,
  1029. tor_free(ent));
  1030. smartlist_free(dirreq_completed);
  1031. buf_free(buf);
  1032. return result;
  1033. }
  1034. /** Store a newly allocated comma-separated string in
  1035. * *<a>country_str</a> containing entries for all the countries from
  1036. * which we've seen enough clients connect as a bridge, directory
  1037. * server, or entry guard. The entry format is cc=num where num is the
  1038. * number of IPs we've seen connecting from that country, and cc is a
  1039. * lowercased country code. *<a>country_str</a> is set to NULL if
  1040. * we're not ready to export per country data yet.
  1041. *
  1042. * Store a newly allocated comma-separated string in <a>ipver_str</a>
  1043. * containing entries for clients connecting over IPv4 and IPv6. The
  1044. * format is family=num where num is the nubmer of IPs we've seen
  1045. * connecting over that protocol family, and family is 'v4' or 'v6'.
  1046. *
  1047. * Return 0 on success and -1 if we're missing geoip data. */
  1048. int
  1049. geoip_get_client_history(geoip_client_action_t action,
  1050. char **country_str, char **ipver_str)
  1051. {
  1052. unsigned granularity = IP_GRANULARITY;
  1053. smartlist_t *entries = NULL;
  1054. int n_countries = geoip_get_n_countries();
  1055. int i;
  1056. clientmap_entry_t **cm_ent;
  1057. unsigned *counts = NULL;
  1058. unsigned total = 0;
  1059. unsigned ipv4_count = 0, ipv6_count = 0;
  1060. if (!geoip_is_loaded(AF_INET) && !geoip_is_loaded(AF_INET6))
  1061. return -1;
  1062. counts = tor_calloc(n_countries, sizeof(unsigned));
  1063. HT_FOREACH(cm_ent, clientmap, &client_history) {
  1064. int country;
  1065. if ((*cm_ent)->action != (int)action)
  1066. continue;
  1067. country = geoip_get_country_by_addr(&(*cm_ent)->addr);
  1068. if (country < 0)
  1069. country = 0; /** unresolved requests are stored at index 0. */
  1070. tor_assert(0 <= country && country < n_countries);
  1071. ++counts[country];
  1072. ++total;
  1073. switch (tor_addr_family(&(*cm_ent)->addr)) {
  1074. case AF_INET:
  1075. ipv4_count++;
  1076. break;
  1077. case AF_INET6:
  1078. ipv6_count++;
  1079. break;
  1080. }
  1081. }
  1082. if (ipver_str) {
  1083. smartlist_t *chunks = smartlist_new();
  1084. smartlist_add_asprintf(chunks, "v4=%u",
  1085. round_to_next_multiple_of(ipv4_count, granularity));
  1086. smartlist_add_asprintf(chunks, "v6=%u",
  1087. round_to_next_multiple_of(ipv6_count, granularity));
  1088. *ipver_str = smartlist_join_strings(chunks, ",", 0, NULL);
  1089. SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
  1090. smartlist_free(chunks);
  1091. }
  1092. /* Don't record per country data if we haven't seen enough IPs. */
  1093. if (total < MIN_IPS_TO_NOTE_ANYTHING) {
  1094. tor_free(counts);
  1095. if (country_str)
  1096. *country_str = NULL;
  1097. return 0;
  1098. }
  1099. /* Make a list of c_hist_t */
  1100. entries = smartlist_new();
  1101. for (i = 0; i < n_countries; ++i) {
  1102. unsigned c = counts[i];
  1103. const char *countrycode;
  1104. c_hist_t *ent;
  1105. /* Only report a country if it has a minimum number of IPs. */
  1106. if (c >= MIN_IPS_TO_NOTE_COUNTRY) {
  1107. c = round_to_next_multiple_of(c, granularity);
  1108. countrycode = geoip_get_country_name(i);
  1109. ent = tor_malloc(sizeof(c_hist_t));
  1110. strlcpy(ent->country, countrycode, sizeof(ent->country));
  1111. ent->total = c;
  1112. smartlist_add(entries, ent);
  1113. }
  1114. }
  1115. /* Sort entries. Note that we must do this _AFTER_ rounding, or else
  1116. * the sort order could leak info. */
  1117. smartlist_sort(entries, c_hist_compare_);
  1118. if (country_str) {
  1119. smartlist_t *chunks = smartlist_new();
  1120. SMARTLIST_FOREACH(entries, c_hist_t *, ch, {
  1121. smartlist_add_asprintf(chunks, "%s=%u", ch->country, ch->total);
  1122. });
  1123. *country_str = smartlist_join_strings(chunks, ",", 0, NULL);
  1124. SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
  1125. smartlist_free(chunks);
  1126. }
  1127. SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c));
  1128. smartlist_free(entries);
  1129. tor_free(counts);
  1130. return 0;
  1131. }
  1132. /** Return a newly allocated string holding the per-country request history
  1133. * for v3 network statuses in a format suitable for an extra-info document,
  1134. * or NULL on failure. */
  1135. char *
  1136. geoip_get_request_history(void)
  1137. {
  1138. smartlist_t *entries, *strings;
  1139. char *result;
  1140. unsigned granularity = IP_GRANULARITY;
  1141. if (!geoip_countries)
  1142. return NULL;
  1143. entries = smartlist_new();
  1144. SMARTLIST_FOREACH_BEGIN(geoip_countries, geoip_country_t *, c) {
  1145. uint32_t tot = 0;
  1146. c_hist_t *ent;
  1147. tot = c->n_v3_ns_requests;
  1148. if (!tot)
  1149. continue;
  1150. ent = tor_malloc_zero(sizeof(c_hist_t));
  1151. strlcpy(ent->country, c->countrycode, sizeof(ent->country));
  1152. ent->total = round_to_next_multiple_of(tot, granularity);
  1153. smartlist_add(entries, ent);
  1154. } SMARTLIST_FOREACH_END(c);
  1155. smartlist_sort(entries, c_hist_compare_);
  1156. strings = smartlist_new();
  1157. SMARTLIST_FOREACH(entries, c_hist_t *, ent, {
  1158. smartlist_add_asprintf(strings, "%s=%u", ent->country, ent->total);
  1159. });
  1160. result = smartlist_join_strings(strings, ",", 0, NULL);
  1161. SMARTLIST_FOREACH(strings, char *, cp, tor_free(cp));
  1162. SMARTLIST_FOREACH(entries, c_hist_t *, ent, tor_free(ent));
  1163. smartlist_free(strings);
  1164. smartlist_free(entries);
  1165. return result;
  1166. }
  1167. /** Start time of directory request stats or 0 if we're not collecting
  1168. * directory request statistics. */
  1169. static time_t start_of_dirreq_stats_interval;
  1170. /** Initialize directory request stats. */
  1171. void
  1172. geoip_dirreq_stats_init(time_t now)
  1173. {
  1174. start_of_dirreq_stats_interval = now;
  1175. }
  1176. /** Reset counters for dirreq stats. */
  1177. void
  1178. geoip_reset_dirreq_stats(time_t now)
  1179. {
  1180. SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, {
  1181. c->n_v3_ns_requests = 0;
  1182. });
  1183. {
  1184. clientmap_entry_t **ent, **next, *this;
  1185. for (ent = HT_START(clientmap, &client_history); ent != NULL;
  1186. ent = next) {
  1187. if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS) {
  1188. this = *ent;
  1189. next = HT_NEXT_RMV(clientmap, &client_history, ent);
  1190. clientmap_entry_free(this);
  1191. } else {
  1192. next = HT_NEXT(clientmap, &client_history, ent);
  1193. }
  1194. }
  1195. }
  1196. memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
  1197. {
  1198. dirreq_map_entry_t **ent, **next, *this;
  1199. for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
  1200. this = *ent;
  1201. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
  1202. tor_free(this);
  1203. }
  1204. }
  1205. start_of_dirreq_stats_interval = now;
  1206. }
  1207. /** Stop collecting directory request stats in a way that we can re-start
  1208. * doing so in geoip_dirreq_stats_init(). */
  1209. void
  1210. geoip_dirreq_stats_term(void)
  1211. {
  1212. geoip_reset_dirreq_stats(0);
  1213. }
  1214. /** Return a newly allocated string containing the dirreq statistics
  1215. * until <b>now</b>, or NULL if we're not collecting dirreq stats. Caller
  1216. * must ensure start_of_dirreq_stats_interval is in the past. */
  1217. char *
  1218. geoip_format_dirreq_stats(time_t now)
  1219. {
  1220. char t[ISO_TIME_LEN+1];
  1221. int i;
  1222. char *v3_ips_string = NULL, *v3_reqs_string = NULL,
  1223. *v3_direct_dl_string = NULL, *v3_tunneled_dl_string = NULL;
  1224. char *result = NULL;
  1225. if (!start_of_dirreq_stats_interval)
  1226. return NULL; /* Not initialized. */
  1227. tor_assert(now >= start_of_dirreq_stats_interval);
  1228. format_iso_time(t, now);
  1229. geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS, &v3_ips_string, NULL);
  1230. v3_reqs_string = geoip_get_request_history();
  1231. #define RESPONSE_GRANULARITY 8
  1232. for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) {
  1233. ns_v3_responses[i] = round_uint32_to_next_multiple_of(
  1234. ns_v3_responses[i], RESPONSE_GRANULARITY);
  1235. }
  1236. #undef RESPONSE_GRANULARITY
  1237. v3_direct_dl_string = geoip_get_dirreq_history(DIRREQ_DIRECT);
  1238. v3_tunneled_dl_string = geoip_get_dirreq_history(DIRREQ_TUNNELED);
  1239. /* Put everything together into a single string. */
  1240. tor_asprintf(&result, "dirreq-stats-end %s (%d s)\n"
  1241. "dirreq-v3-ips %s\n"
  1242. "dirreq-v3-reqs %s\n"
  1243. "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u,"
  1244. "not-found=%u,not-modified=%u,busy=%u\n"
  1245. "dirreq-v3-direct-dl %s\n"
  1246. "dirreq-v3-tunneled-dl %s\n",
  1247. t,
  1248. (unsigned) (now - start_of_dirreq_stats_interval),
  1249. v3_ips_string ? v3_ips_string : "",
  1250. v3_reqs_string ? v3_reqs_string : "",
  1251. ns_v3_responses[GEOIP_SUCCESS],
  1252. ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS],
  1253. ns_v3_responses[GEOIP_REJECT_UNAVAILABLE],
  1254. ns_v3_responses[GEOIP_REJECT_NOT_FOUND],
  1255. ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED],
  1256. ns_v3_responses[GEOIP_REJECT_BUSY],
  1257. v3_direct_dl_string ? v3_direct_dl_string : "",
  1258. v3_tunneled_dl_string ? v3_tunneled_dl_string : "");
  1259. /* Free partial strings. */
  1260. tor_free(v3_ips_string);
  1261. tor_free(v3_reqs_string);
  1262. tor_free(v3_direct_dl_string);
  1263. tor_free(v3_tunneled_dl_string);
  1264. return result;
  1265. }
  1266. /** If 24 hours have passed since the beginning of the current dirreq
  1267. * stats period, write dirreq stats to $DATADIR/stats/dirreq-stats
  1268. * (possibly overwriting an existing file) and reset counters. Return
  1269. * when we would next want to write dirreq stats or 0 if we never want to
  1270. * write. */
  1271. time_t
  1272. geoip_dirreq_stats_write(time_t now)
  1273. {
  1274. char *str = NULL;
  1275. if (!start_of_dirreq_stats_interval)
  1276. return 0; /* Not initialized. */
  1277. if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now)
  1278. goto done; /* Not ready to write. */
  1279. /* Discard all items in the client history that are too old. */
  1280. geoip_remove_old_clients(start_of_dirreq_stats_interval);
  1281. /* Generate history string .*/
  1282. str = geoip_format_dirreq_stats(now);
  1283. if (! str)
  1284. goto done;
  1285. /* Write dirreq-stats string to disk. */
  1286. if (!check_or_create_data_subdir("stats")) {
  1287. write_to_data_subdir("stats", "dirreq-stats", str, "dirreq statistics");
  1288. /* Reset measurement interval start. */
  1289. geoip_reset_dirreq_stats(now);
  1290. }
  1291. done:
  1292. tor_free(str);
  1293. return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL;
  1294. }
  1295. /** Start time of bridge stats or 0 if we're not collecting bridge
  1296. * statistics. */
  1297. static time_t start_of_bridge_stats_interval;
  1298. /** Initialize bridge stats. */
  1299. void
  1300. geoip_bridge_stats_init(time_t now)
  1301. {
  1302. start_of_bridge_stats_interval = now;
  1303. }
  1304. /** Stop collecting bridge stats in a way that we can re-start doing so in
  1305. * geoip_bridge_stats_init(). */
  1306. void
  1307. geoip_bridge_stats_term(void)
  1308. {
  1309. client_history_clear();
  1310. start_of_bridge_stats_interval = 0;
  1311. }
  1312. /** Validate a bridge statistics string as it would be written to a
  1313. * current extra-info descriptor. Return 1 if the string is valid and
  1314. * recent enough, or 0 otherwise. */
  1315. static int
  1316. validate_bridge_stats(const char *stats_str, time_t now)
  1317. {
  1318. char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1],
  1319. *eos;
  1320. const char *BRIDGE_STATS_END = "bridge-stats-end ";
  1321. const char *BRIDGE_IPS = "bridge-ips ";
  1322. const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n";
  1323. const char *BRIDGE_TRANSPORTS = "bridge-ip-transports ";
  1324. const char *BRIDGE_TRANSPORTS_EMPTY_LINE = "bridge-ip-transports\n";
  1325. const char *tmp;
  1326. time_t stats_end_time;
  1327. int seconds;
  1328. tor_assert(stats_str);
  1329. /* Parse timestamp and number of seconds from
  1330. "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */
  1331. tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END);
  1332. if (!tmp)
  1333. return 0;
  1334. tmp += strlen(BRIDGE_STATS_END);
  1335. if (strlen(tmp) < ISO_TIME_LEN + 6)
  1336. return 0;
  1337. strlcpy(stats_end_str, tmp, sizeof(stats_end_str));
  1338. if (parse_iso_time(stats_end_str, &stats_end_time) < 0)
  1339. return 0;
  1340. if (stats_end_time < now - (25*60*60) ||
  1341. stats_end_time > now + (1*60*60))
  1342. return 0;
  1343. seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10);
  1344. if (!eos || seconds < 23*60*60)
  1345. return 0;
  1346. format_iso_time(stats_start_str, stats_end_time - seconds);
  1347. /* Parse: "bridge-ips CC=N,CC=N,..." */
  1348. tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS);
  1349. if (!tmp) {
  1350. /* Look if there is an empty "bridge-ips" line */
  1351. tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE);
  1352. if (!tmp)
  1353. return 0;
  1354. }
  1355. /* Parse: "bridge-ip-transports PT=N,PT=N,..." */
  1356. tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS);
  1357. if (!tmp) {
  1358. /* Look if there is an empty "bridge-ip-transports" line */
  1359. tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS_EMPTY_LINE);
  1360. if (!tmp)
  1361. return 0;
  1362. }
  1363. return 1;
  1364. }
  1365. /** Most recent bridge statistics formatted to be written to extra-info
  1366. * descriptors. */
  1367. static char *bridge_stats_extrainfo = NULL;
  1368. /** Return a newly allocated string holding our bridge usage stats by country
  1369. * in a format suitable for inclusion in an extrainfo document. Return NULL on
  1370. * failure. */
  1371. char *
  1372. geoip_format_bridge_stats(time_t now)
  1373. {
  1374. char *out = NULL;
  1375. char *country_data = NULL, *ipver_data = NULL, *transport_data = NULL;
  1376. long duration = now - start_of_bridge_stats_interval;
  1377. char written[ISO_TIME_LEN+1];
  1378. if (duration < 0)
  1379. return NULL;
  1380. if (!start_of_bridge_stats_interval)
  1381. return NULL; /* Not initialized. */
  1382. format_iso_time(written, now);
  1383. geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
  1384. transport_data = geoip_get_transport_history();
  1385. tor_asprintf(&out,
  1386. "bridge-stats-end %s (%ld s)\n"
  1387. "bridge-ips %s\n"
  1388. "bridge-ip-versions %s\n"
  1389. "bridge-ip-transports %s\n",
  1390. written, duration,
  1391. country_data ? country_data : "",
  1392. ipver_data ? ipver_data : "",
  1393. transport_data ? transport_data : "");
  1394. tor_free(country_data);
  1395. tor_free(ipver_data);
  1396. tor_free(transport_data);
  1397. return out;
  1398. }
  1399. /** Return a newly allocated string holding our bridge usage stats by country
  1400. * in a format suitable for the answer to a controller request. Return NULL on
  1401. * failure. */
  1402. static char *
  1403. format_bridge_stats_controller(time_t now)
  1404. {
  1405. char *out = NULL, *country_data = NULL, *ipver_data = NULL;
  1406. char started[ISO_TIME_LEN+1];
  1407. (void) now;
  1408. format_iso_time(started, start_of_bridge_stats_interval);
  1409. geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
  1410. tor_asprintf(&out,
  1411. "TimeStarted=\"%s\" CountrySummary=%s IPVersions=%s",
  1412. started,
  1413. country_data ? country_data : "",
  1414. ipver_data ? ipver_data : "");
  1415. tor_free(country_data);
  1416. tor_free(ipver_data);
  1417. return out;
  1418. }
  1419. /** Return a newly allocated string holding our bridge usage stats by
  1420. * country in a format suitable for inclusion in our heartbeat
  1421. * message. Return NULL on failure. */
  1422. char *
  1423. format_client_stats_heartbeat(time_t now)
  1424. {
  1425. const int n_hours = 6;
  1426. char *out = NULL;
  1427. int n_clients = 0;
  1428. clientmap_entry_t **ent;
  1429. unsigned cutoff = (unsigned)( (now-n_hours*3600)/60 );
  1430. if (!start_of_bridge_stats_interval)
  1431. return NULL; /* Not initialized. */
  1432. /* count unique IPs */
  1433. HT_FOREACH(ent, clientmap, &client_history) {
  1434. /* only count directly connecting clients */
  1435. if ((*ent)->action != GEOIP_CLIENT_CONNECT)
  1436. continue;
  1437. if ((*ent)->last_seen_in_minutes < cutoff)
  1438. continue;
  1439. n_clients++;
  1440. }
  1441. tor_asprintf(&out, "Heartbeat: "
  1442. "In the last %d hours, I have seen %d unique clients.",
  1443. n_hours,
  1444. n_clients);
  1445. return out;
  1446. }
  1447. /** Write bridge statistics to $DATADIR/stats/bridge-stats and return
  1448. * when we should next try to write statistics. */
  1449. time_t
  1450. geoip_bridge_stats_write(time_t now)
  1451. {
  1452. char *val = NULL;
  1453. /* Check if 24 hours have passed since starting measurements. */
  1454. if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL)
  1455. return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
  1456. /* Discard all items in the client history that are too old. */
  1457. geoip_remove_old_clients(start_of_bridge_stats_interval);
  1458. /* Generate formatted string */
  1459. val = geoip_format_bridge_stats(now);
  1460. if (val == NULL)
  1461. goto done;
  1462. /* Update the stored value. */
  1463. tor_free(bridge_stats_extrainfo);
  1464. bridge_stats_extrainfo = val;
  1465. start_of_bridge_stats_interval = now;
  1466. /* Write it to disk. */
  1467. if (!check_or_create_data_subdir("stats")) {
  1468. write_to_data_subdir("stats", "bridge-stats",
  1469. bridge_stats_extrainfo, "bridge statistics");
  1470. /* Tell the controller, "hey, there are clients!" */
  1471. {
  1472. char *controller_str = format_bridge_stats_controller(now);
  1473. if (controller_str)
  1474. control_event_clients_seen(controller_str);
  1475. tor_free(controller_str);
  1476. }
  1477. }
  1478. done:
  1479. return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
  1480. }
  1481. /** Try to load the most recent bridge statistics from disk, unless we
  1482. * have finished a measurement interval lately, and check whether they
  1483. * are still recent enough. */
  1484. static void
  1485. load_bridge_stats(time_t now)
  1486. {
  1487. char *fname, *contents;
  1488. if (bridge_stats_extrainfo)
  1489. return;
  1490. fname = get_datadir_fname2("stats", "bridge-stats");
  1491. contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
  1492. if (contents && validate_bridge_stats(contents, now)) {
  1493. bridge_stats_extrainfo = contents;
  1494. } else {
  1495. tor_free(contents);
  1496. }
  1497. tor_free(fname);
  1498. }
  1499. /** Return most recent bridge statistics for inclusion in extra-info
  1500. * descriptors, or NULL if we don't have recent bridge statistics. */
  1501. const char *
  1502. geoip_get_bridge_stats_extrainfo(time_t now)
  1503. {
  1504. load_bridge_stats(now);
  1505. return bridge_stats_extrainfo;
  1506. }
  1507. /** Return a new string containing the recent bridge statistics to be returned
  1508. * to controller clients, or NULL if we don't have any bridge statistics. */
  1509. char *
  1510. geoip_get_bridge_stats_controller(time_t now)
  1511. {
  1512. return format_bridge_stats_controller(now);
  1513. }
  1514. /** Start time of entry stats or 0 if we're not collecting entry
  1515. * statistics. */
  1516. static time_t start_of_entry_stats_interval;
  1517. /** Initialize entry stats. */
  1518. void
  1519. geoip_entry_stats_init(time_t now)
  1520. {
  1521. start_of_entry_stats_interval = now;
  1522. }
  1523. /** Reset counters for entry stats. */
  1524. void
  1525. geoip_reset_entry_stats(time_t now)
  1526. {
  1527. client_history_clear();
  1528. start_of_entry_stats_interval = now;
  1529. }
  1530. /** Stop collecting entry stats in a way that we can re-start doing so in
  1531. * geoip_entry_stats_init(). */
  1532. void
  1533. geoip_entry_stats_term(void)
  1534. {
  1535. geoip_reset_entry_stats(0);
  1536. }
  1537. /** Return a newly allocated string containing the entry statistics
  1538. * until <b>now</b>, or NULL if we're not collecting entry stats. Caller
  1539. * must ensure start_of_entry_stats_interval lies in the past. */
  1540. char *
  1541. geoip_format_entry_stats(time_t now)
  1542. {
  1543. char t[ISO_TIME_LEN+1];
  1544. char *data = NULL;
  1545. char *result;
  1546. if (!start_of_entry_stats_interval)
  1547. return NULL; /* Not initialized. */
  1548. tor_assert(now >= start_of_entry_stats_interval);
  1549. geoip_get_client_history(GEOIP_CLIENT_CONNECT, &data, NULL);
  1550. format_iso_time(t, now);
  1551. tor_asprintf(&result,
  1552. "entry-stats-end %s (%u s)\n"
  1553. "entry-ips %s\n",
  1554. t, (unsigned) (now - start_of_entry_stats_interval),
  1555. data ? data : "");
  1556. tor_free(data);
  1557. return result;
  1558. }
  1559. /** If 24 hours have passed since the beginning of the current entry stats
  1560. * period, write entry stats to $DATADIR/stats/entry-stats (possibly
  1561. * overwriting an existing file) and reset counters. Return when we would
  1562. * next want to write entry stats or 0 if we never want to write. */
  1563. time_t
  1564. geoip_entry_stats_write(time_t now)
  1565. {
  1566. char *str = NULL;
  1567. if (!start_of_entry_stats_interval)
  1568. return 0; /* Not initialized. */
  1569. if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now)
  1570. goto done; /* Not ready to write. */
  1571. /* Discard all items in the client history that are too old. */
  1572. geoip_remove_old_clients(start_of_entry_stats_interval);
  1573. /* Generate history string .*/
  1574. str = geoip_format_entry_stats(now);
  1575. /* Write entry-stats string to disk. */
  1576. if (!check_or_create_data_subdir("stats")) {
  1577. write_to_data_subdir("stats", "entry-stats", str, "entry statistics");
  1578. /* Reset measurement interval start. */
  1579. geoip_reset_entry_stats(now);
  1580. }
  1581. done:
  1582. tor_free(str);
  1583. return start_of_entry_stats_interval + WRITE_STATS_INTERVAL;
  1584. }
  1585. /** Helper used to implement GETINFO ip-to-country/... controller command. */
  1586. int
  1587. getinfo_helper_geoip(control_connection_t *control_conn,
  1588. const char *question, char **answer,
  1589. const char **errmsg)
  1590. {
  1591. (void)control_conn;
  1592. if (!strcmpstart(question, "ip-to-country/")) {
  1593. int c;
  1594. sa_family_t family;
  1595. tor_addr_t addr;
  1596. question += strlen("ip-to-country/");
  1597. if (!strcmp(question, "ipv4-available") ||
  1598. !strcmp(question, "ipv6-available")) {
  1599. family = !strcmp(question, "ipv4-available") ? AF_INET : AF_INET6;
  1600. const int available = geoip_is_loaded(family);
  1601. tor_asprintf(answer, "%d", !! available);
  1602. return 0;
  1603. }
  1604. family = tor_addr_parse(&addr, question);
  1605. if (family != AF_INET && family != AF_INET6) {
  1606. *errmsg = "Invalid address family";
  1607. return -1;
  1608. }
  1609. if (!geoip_is_loaded(family)) {
  1610. *errmsg = "GeoIP data not loaded";
  1611. return -1;
  1612. }
  1613. if (family == AF_INET)
  1614. c = geoip_get_country_by_ipv4(tor_addr_to_ipv4h(&addr));
  1615. else /* AF_INET6 */
  1616. c = geoip_get_country_by_ipv6(tor_addr_to_in6(&addr));
  1617. *answer = tor_strdup(geoip_get_country_name(c));
  1618. }
  1619. return 0;
  1620. }
  1621. /** Release all storage held by the GeoIP databases and country list. */
  1622. STATIC void
  1623. clear_geoip_db(void)
  1624. {
  1625. if (geoip_countries) {
  1626. SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, tor_free(c));
  1627. smartlist_free(geoip_countries);
  1628. }
  1629. strmap_free(country_idxplus1_by_lc_code, NULL);
  1630. if (geoip_ipv4_entries) {
  1631. SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, ent,
  1632. tor_free(ent));
  1633. smartlist_free(geoip_ipv4_entries);
  1634. }
  1635. if (geoip_ipv6_entries) {
  1636. SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, ent,
  1637. tor_free(ent));
  1638. smartlist_free(geoip_ipv6_entries);
  1639. }
  1640. geoip_countries = NULL;
  1641. country_idxplus1_by_lc_code = NULL;
  1642. geoip_ipv4_entries = NULL;
  1643. geoip_ipv6_entries = NULL;
  1644. }
  1645. /** Release all storage held in this file. */
  1646. void
  1647. geoip_free_all(void)
  1648. {
  1649. {
  1650. clientmap_entry_t **ent, **next, *this;
  1651. for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) {
  1652. this = *ent;
  1653. next = HT_NEXT_RMV(clientmap, &client_history, ent);
  1654. clientmap_entry_free(this);
  1655. }
  1656. HT_CLEAR(clientmap, &client_history);
  1657. }
  1658. {
  1659. dirreq_map_entry_t **ent, **next, *this;
  1660. for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
  1661. this = *ent;
  1662. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
  1663. tor_free(this);
  1664. }
  1665. HT_CLEAR(dirreqmap, &dirreq_map);
  1666. }
  1667. clear_geoip_db();
  1668. tor_free(bridge_stats_extrainfo);
  1669. memset(geoip_digest, 0, sizeof(geoip_digest));
  1670. memset(geoip6_digest, 0, sizeof(geoip6_digest));
  1671. }