geoip.c 50 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553
  1. /* Copyright (c) 2007-2013, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. /**
  4. * \file geoip.c
  5. * \brief Functions related to maintaining an IP-to-country database;
  6. * to summarizing client connections by country to entry guards, bridges,
  7. * and directory servers; and for statistics on answering network status
  8. * requests.
  9. */
  10. #define GEOIP_PRIVATE
  11. #include "or.h"
  12. #include "ht.h"
  13. #include "config.h"
  14. #include "control.h"
  15. #include "dnsserv.h"
  16. #include "geoip.h"
  17. #include "routerlist.h"
  18. static void clear_geoip_db(void);
  19. static void init_geoip_countries(void);
  20. /** An entry from the GeoIP IPv4 file: maps an IPv4 range to a country. */
  21. typedef struct geoip_ipv4_entry_t {
  22. uint32_t ip_low; /**< The lowest IP in the range, in host order */
  23. uint32_t ip_high; /**< The highest IP in the range, in host order */
  24. intptr_t country; /**< An index into geoip_countries */
  25. } geoip_ipv4_entry_t;
  26. /** An entry from the GeoIP IPv6 file: maps an IPv6 range to a country. */
  27. typedef struct geoip_ipv6_entry_t {
  28. struct in6_addr ip_low; /**< The lowest IP in the range, in host order */
  29. struct in6_addr ip_high; /**< The highest IP in the range, in host order */
  30. intptr_t country; /**< An index into geoip_countries */
  31. } geoip_ipv6_entry_t;
  32. /** A per-country record for GeoIP request history. */
  33. typedef struct geoip_country_t {
  34. char countrycode[3];
  35. uint32_t n_v3_ns_requests;
  36. } geoip_country_t;
  37. /** A list of geoip_country_t */
  38. static smartlist_t *geoip_countries = NULL;
  39. /** A map from lowercased country codes to their position in geoip_countries.
  40. * The index is encoded in the pointer, and 1 is added so that NULL can mean
  41. * not found. */
  42. static strmap_t *country_idxplus1_by_lc_code = NULL;
  43. /** Lists of all known geoip_ipv4_entry_t and geoip_ipv6_entry_t, sorted
  44. * by their respective ip_low. */
  45. static smartlist_t *geoip_ipv4_entries = NULL, *geoip_ipv6_entries = NULL;
  46. /** SHA1 digest of the GeoIP files to include in extra-info descriptors. */
  47. static char geoip_digest[DIGEST_LEN];
  48. static char geoip6_digest[DIGEST_LEN];
  49. /** Return the index of the <b>country</b>'s entry in the GeoIP
  50. * country list if it is a valid 2-letter country code, otherwise
  51. * return -1. */
  52. country_t
  53. geoip_get_country(const char *country)
  54. {
  55. void *idxplus1_;
  56. intptr_t idx;
  57. idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country);
  58. if (!idxplus1_)
  59. return -1;
  60. idx = ((uintptr_t)idxplus1_)-1;
  61. return (country_t)idx;
  62. }
  63. /** Add an entry to a GeoIP table, mapping all IP addresses between <b>low</b>
  64. * and <b>high</b>, inclusive, to the 2-letter country code <b>country</b>. */
  65. static void
  66. geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high,
  67. const char *country)
  68. {
  69. intptr_t idx;
  70. void *idxplus1_;
  71. if (tor_addr_family(low) != tor_addr_family(high))
  72. return;
  73. if (tor_addr_compare(high, low, CMP_EXACT) < 0)
  74. return;
  75. idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country);
  76. if (!idxplus1_) {
  77. geoip_country_t *c = tor_malloc_zero(sizeof(geoip_country_t));
  78. strlcpy(c->countrycode, country, sizeof(c->countrycode));
  79. tor_strlower(c->countrycode);
  80. smartlist_add(geoip_countries, c);
  81. idx = smartlist_len(geoip_countries) - 1;
  82. strmap_set_lc(country_idxplus1_by_lc_code, country, (void*)(idx+1));
  83. } else {
  84. idx = ((uintptr_t)idxplus1_)-1;
  85. }
  86. {
  87. geoip_country_t *c = smartlist_get(geoip_countries, idx);
  88. tor_assert(!strcasecmp(c->countrycode, country));
  89. }
  90. if (tor_addr_family(low) == AF_INET) {
  91. geoip_ipv4_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv4_entry_t));
  92. ent->ip_low = tor_addr_to_ipv4h(low);
  93. ent->ip_high = tor_addr_to_ipv4h(high);
  94. ent->country = idx;
  95. smartlist_add(geoip_ipv4_entries, ent);
  96. } else if (tor_addr_family(low) == AF_INET6) {
  97. geoip_ipv6_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv6_entry_t));
  98. ent->ip_low = *tor_addr_to_in6(low);
  99. ent->ip_high = *tor_addr_to_in6(high);
  100. ent->country = idx;
  101. smartlist_add(geoip_ipv6_entries, ent);
  102. }
  103. }
  104. /** Add an entry to the GeoIP table indicated by <b>family</b>,
  105. * parsing it from <b>line</b>. The format is as for geoip_load_file(). */
  106. /*private*/ int
  107. geoip_parse_entry(const char *line, sa_family_t family)
  108. {
  109. tor_addr_t low_addr, high_addr;
  110. char c[3];
  111. char *country = NULL;
  112. char buf[512];
  113. if (!geoip_countries)
  114. init_geoip_countries();
  115. if (family == AF_INET) {
  116. if (!geoip_ipv4_entries)
  117. geoip_ipv4_entries = smartlist_new();
  118. } else if (family == AF_INET6) {
  119. if (!geoip_ipv6_entries)
  120. geoip_ipv6_entries = smartlist_new();
  121. } else {
  122. log_warn(LD_GENERAL, "Unsupported family: %d", family);
  123. return -1;
  124. }
  125. while (TOR_ISSPACE(*line))
  126. ++line;
  127. if (*line == '#')
  128. return 0;
  129. if (family == AF_INET) {
  130. unsigned int low, high;
  131. if (tor_sscanf(line,"%u,%u,%2s", &low, &high, c) == 3 ||
  132. tor_sscanf(line,"\"%u\",\"%u\",\"%2s\",", &low, &high, c) == 3) {
  133. tor_addr_from_ipv4h(&low_addr, low);
  134. tor_addr_from_ipv4h(&high_addr, high);
  135. } else
  136. goto fail;
  137. country = c;
  138. } else { /* AF_INET6 */
  139. char *low_str, *high_str;
  140. struct in6_addr low, high;
  141. char *strtok_state;
  142. strlcpy(buf, line, sizeof(buf));
  143. low_str = tor_strtok_r(buf, ",", &strtok_state);
  144. if (!low_str)
  145. goto fail;
  146. high_str = tor_strtok_r(NULL, ",", &strtok_state);
  147. if (!high_str)
  148. goto fail;
  149. country = tor_strtok_r(NULL, "\n", &strtok_state);
  150. if (!country)
  151. goto fail;
  152. if (strlen(country) != 2)
  153. goto fail;
  154. if (tor_inet_pton(AF_INET6, low_str, &low) <= 0)
  155. goto fail;
  156. tor_addr_from_in6(&low_addr, &low);
  157. if (tor_inet_pton(AF_INET6, high_str, &high) <= 0)
  158. goto fail;
  159. tor_addr_from_in6(&high_addr, &high);
  160. }
  161. geoip_add_entry(&low_addr, &high_addr, country);
  162. return 0;
  163. fail:
  164. log_warn(LD_GENERAL, "Unable to parse line from GEOIP %s file: %s",
  165. family == AF_INET ? "IPv4" : "IPv6", escaped(line));
  166. return -1;
  167. }
  168. /** Sorting helper: return -1, 1, or 0 based on comparison of two
  169. * geoip_ipv4_entry_t */
  170. static int
  171. geoip_ipv4_compare_entries_(const void **_a, const void **_b)
  172. {
  173. const geoip_ipv4_entry_t *a = *_a, *b = *_b;
  174. if (a->ip_low < b->ip_low)
  175. return -1;
  176. else if (a->ip_low > b->ip_low)
  177. return 1;
  178. else
  179. return 0;
  180. }
  181. /** bsearch helper: return -1, 1, or 0 based on comparison of an IP (a pointer
  182. * to a uint32_t in host order) to a geoip_ipv4_entry_t */
  183. static int
  184. geoip_ipv4_compare_key_to_entry_(const void *_key, const void **_member)
  185. {
  186. /* No alignment issue here, since _key really is a pointer to uint32_t */
  187. const uint32_t addr = *(uint32_t *)_key;
  188. const geoip_ipv4_entry_t *entry = *_member;
  189. if (addr < entry->ip_low)
  190. return -1;
  191. else if (addr > entry->ip_high)
  192. return 1;
  193. else
  194. return 0;
  195. }
  196. /** Sorting helper: return -1, 1, or 0 based on comparison of two
  197. * geoip_ipv6_entry_t */
  198. static int
  199. geoip_ipv6_compare_entries_(const void **_a, const void **_b)
  200. {
  201. const geoip_ipv6_entry_t *a = *_a, *b = *_b;
  202. return fast_memcmp(a->ip_low.s6_addr, b->ip_low.s6_addr,
  203. sizeof(struct in6_addr));
  204. }
  205. /** bsearch helper: return -1, 1, or 0 based on comparison of an IPv6
  206. * (a pointer to a in6_addr) to a geoip_ipv6_entry_t */
  207. static int
  208. geoip_ipv6_compare_key_to_entry_(const void *_key, const void **_member)
  209. {
  210. const struct in6_addr *addr = (struct in6_addr *)_key;
  211. const geoip_ipv6_entry_t *entry = *_member;
  212. if (fast_memcmp(addr->s6_addr, entry->ip_low.s6_addr,
  213. sizeof(struct in6_addr)) < 0)
  214. return -1;
  215. else if (fast_memcmp(addr->s6_addr, entry->ip_high.s6_addr,
  216. sizeof(struct in6_addr)) > 0)
  217. return 1;
  218. else
  219. return 0;
  220. }
  221. /** Return 1 if we should collect geoip stats on bridge users, and
  222. * include them in our extrainfo descriptor. Else return 0. */
  223. int
  224. should_record_bridge_info(const or_options_t *options)
  225. {
  226. return options->BridgeRelay && options->BridgeRecordUsageByCountry;
  227. }
  228. /** Set up a new list of geoip countries with no countries (yet) set in it,
  229. * except for the unknown country.
  230. */
  231. static void
  232. init_geoip_countries(void)
  233. {
  234. geoip_country_t *geoip_unresolved;
  235. geoip_countries = smartlist_new();
  236. /* Add a geoip_country_t for requests that could not be resolved to a
  237. * country as first element (index 0) to geoip_countries. */
  238. geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t));
  239. strlcpy(geoip_unresolved->countrycode, "??",
  240. sizeof(geoip_unresolved->countrycode));
  241. smartlist_add(geoip_countries, geoip_unresolved);
  242. country_idxplus1_by_lc_code = strmap_new();
  243. strmap_set_lc(country_idxplus1_by_lc_code, "??", (void*)(1));
  244. }
  245. /** Clear appropriate GeoIP database, based on <b>family</b>, and
  246. * reload it from the file <b>filename</b>. Return 0 on success, -1 on
  247. * failure.
  248. *
  249. * Recognized line formats for IPv4 are:
  250. * INTIPLOW,INTIPHIGH,CC
  251. * and
  252. * "INTIPLOW","INTIPHIGH","CC","CC3","COUNTRY NAME"
  253. * where INTIPLOW and INTIPHIGH are IPv4 addresses encoded as 4-byte unsigned
  254. * integers, and CC is a country code.
  255. *
  256. * Recognized line format for IPv6 is:
  257. * IPV6LOW,IPV6HIGH,CC
  258. * where IPV6LOW and IPV6HIGH are IPv6 addresses and CC is a country code.
  259. *
  260. * It also recognizes, and skips over, blank lines and lines that start
  261. * with '#' (comments).
  262. */
  263. int
  264. geoip_load_file(sa_family_t family, const char *filename)
  265. {
  266. FILE *f;
  267. const char *msg = "";
  268. const or_options_t *options = get_options();
  269. int severity = options_need_geoip_info(options, &msg) ? LOG_WARN : LOG_INFO;
  270. crypto_digest_t *geoip_digest_env = NULL;
  271. tor_assert(family == AF_INET || family == AF_INET6);
  272. if (!(f = tor_fopen_cloexec(filename, "r"))) {
  273. log_fn(severity, LD_GENERAL, "Failed to open GEOIP file %s. %s",
  274. filename, msg);
  275. return -1;
  276. }
  277. if (!geoip_countries)
  278. init_geoip_countries();
  279. if (family == AF_INET) {
  280. if (geoip_ipv4_entries) {
  281. SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, e,
  282. tor_free(e));
  283. smartlist_free(geoip_ipv4_entries);
  284. }
  285. geoip_ipv4_entries = smartlist_new();
  286. } else { /* AF_INET6 */
  287. if (geoip_ipv6_entries) {
  288. SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, e,
  289. tor_free(e));
  290. smartlist_free(geoip_ipv6_entries);
  291. }
  292. geoip_ipv6_entries = smartlist_new();
  293. }
  294. geoip_digest_env = crypto_digest_new();
  295. log_notice(LD_GENERAL, "Parsing GEOIP %s file %s.",
  296. (family == AF_INET) ? "IPv4" : "IPv6", filename);
  297. while (!feof(f)) {
  298. char buf[512];
  299. if (fgets(buf, (int)sizeof(buf), f) == NULL)
  300. break;
  301. crypto_digest_add_bytes(geoip_digest_env, buf, strlen(buf));
  302. /* FFFF track full country name. */
  303. geoip_parse_entry(buf, family);
  304. }
  305. /*XXXX abort and return -1 if no entries/illformed?*/
  306. fclose(f);
  307. /* Sort list and remember file digests so that we can include it in
  308. * our extra-info descriptors. */
  309. if (family == AF_INET) {
  310. smartlist_sort(geoip_ipv4_entries, geoip_ipv4_compare_entries_);
  311. /* Okay, now we need to maybe change our mind about what is in
  312. * which country. We do this for IPv4 only since that's what we
  313. * store in node->country. */
  314. refresh_all_country_info();
  315. crypto_digest_get_digest(geoip_digest_env, geoip_digest, DIGEST_LEN);
  316. } else {
  317. /* AF_INET6 */
  318. smartlist_sort(geoip_ipv6_entries, geoip_ipv6_compare_entries_);
  319. crypto_digest_get_digest(geoip_digest_env, geoip6_digest, DIGEST_LEN);
  320. }
  321. crypto_digest_free(geoip_digest_env);
  322. return 0;
  323. }
  324. /** Given an IP address in host order, return a number representing the
  325. * country to which that address belongs, -1 for "No geoip information
  326. * available", or 0 for the 'unknown country'. The return value will always
  327. * be less than geoip_get_n_countries(). To decode it, call
  328. * geoip_get_country_name().
  329. */
  330. int
  331. geoip_get_country_by_ipv4(uint32_t ipaddr)
  332. {
  333. geoip_ipv4_entry_t *ent;
  334. if (!geoip_ipv4_entries)
  335. return -1;
  336. ent = smartlist_bsearch(geoip_ipv4_entries, &ipaddr,
  337. geoip_ipv4_compare_key_to_entry_);
  338. return ent ? (int)ent->country : 0;
  339. }
  340. /** Given an IPv6 address, return a number representing the country to
  341. * which that address belongs, -1 for "No geoip information available", or
  342. * 0 for the 'unknown country'. The return value will always be less than
  343. * geoip_get_n_countries(). To decode it, call geoip_get_country_name().
  344. */
  345. int
  346. geoip_get_country_by_ipv6(const struct in6_addr *addr)
  347. {
  348. geoip_ipv6_entry_t *ent;
  349. if (!geoip_ipv6_entries)
  350. return -1;
  351. ent = smartlist_bsearch(geoip_ipv6_entries, addr,
  352. geoip_ipv6_compare_key_to_entry_);
  353. return ent ? (int)ent->country : 0;
  354. }
  355. /** Given an IP address, return a number representing the country to which
  356. * that address belongs, -1 for "No geoip information available", or 0 for
  357. * the 'unknown country'. The return value will always be less than
  358. * geoip_get_n_countries(). To decode it, call geoip_get_country_name().
  359. */
  360. int
  361. geoip_get_country_by_addr(const tor_addr_t *addr)
  362. {
  363. if (tor_addr_family(addr) == AF_INET) {
  364. return geoip_get_country_by_ipv4(tor_addr_to_ipv4h(addr));
  365. } else if (tor_addr_family(addr) == AF_INET6) {
  366. return geoip_get_country_by_ipv6(tor_addr_to_in6(addr));
  367. } else {
  368. return -1;
  369. }
  370. }
  371. /** Return the number of countries recognized by the GeoIP country list. */
  372. int
  373. geoip_get_n_countries(void)
  374. {
  375. if (!geoip_countries)
  376. init_geoip_countries();
  377. return (int) smartlist_len(geoip_countries);
  378. }
  379. /** Return the two-letter country code associated with the number <b>num</b>,
  380. * or "??" for an unknown value. */
  381. const char *
  382. geoip_get_country_name(country_t num)
  383. {
  384. if (geoip_countries && num >= 0 && num < smartlist_len(geoip_countries)) {
  385. geoip_country_t *c = smartlist_get(geoip_countries, num);
  386. return c->countrycode;
  387. } else
  388. return "??";
  389. }
  390. /** Return true iff we have loaded a GeoIP database.*/
  391. int
  392. geoip_is_loaded(sa_family_t family)
  393. {
  394. tor_assert(family == AF_INET || family == AF_INET6);
  395. if (geoip_countries == NULL)
  396. return 0;
  397. if (family == AF_INET)
  398. return geoip_ipv4_entries != NULL;
  399. else /* AF_INET6 */
  400. return geoip_ipv6_entries != NULL;
  401. }
  402. /** Return the hex-encoded SHA1 digest of the loaded GeoIP file. The
  403. * result does not need to be deallocated, but will be overwritten by the
  404. * next call of hex_str(). */
  405. const char *
  406. geoip_db_digest(sa_family_t family)
  407. {
  408. tor_assert(family == AF_INET || family == AF_INET6);
  409. if (family == AF_INET)
  410. return hex_str(geoip_digest, DIGEST_LEN);
  411. else /* AF_INET6 */
  412. return hex_str(geoip6_digest, DIGEST_LEN);
  413. }
  414. /** Entry in a map from IP address to the last time we've seen an incoming
  415. * connection from that IP address. Used by bridges only, to track which
  416. * countries have them blocked. */
  417. typedef struct clientmap_entry_t {
  418. HT_ENTRY(clientmap_entry_t) node;
  419. tor_addr_t addr;
  420. /** Time when we last saw this IP address, in MINUTES since the epoch.
  421. *
  422. * (This will run out of space around 4011 CE. If Tor is still in use around
  423. * 4000 CE, please remember to add more bits to last_seen_in_minutes.) */
  424. unsigned int last_seen_in_minutes:30;
  425. unsigned int action:2;
  426. } clientmap_entry_t;
  427. /** Largest allowable value for last_seen_in_minutes. (It's a 30-bit field,
  428. * so it can hold up to (1u<<30)-1, or 0x3fffffffu.
  429. */
  430. #define MAX_LAST_SEEN_IN_MINUTES 0X3FFFFFFFu
  431. /** Map from client IP address to last time seen. */
  432. static HT_HEAD(clientmap, clientmap_entry_t) client_history =
  433. HT_INITIALIZER();
  434. /** Hashtable helper: compute a hash of a clientmap_entry_t. */
  435. static INLINE unsigned
  436. clientmap_entry_hash(const clientmap_entry_t *a)
  437. {
  438. return ht_improve_hash(tor_addr_hash(&a->addr));
  439. }
  440. /** Hashtable helper: compare two clientmap_entry_t values for equality. */
  441. static INLINE int
  442. clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b)
  443. {
  444. return !tor_addr_compare(&a->addr, &b->addr, CMP_EXACT) &&
  445. a->action == b->action;
  446. }
  447. HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
  448. clientmap_entries_eq);
  449. HT_GENERATE(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
  450. clientmap_entries_eq, 0.6, malloc, realloc, free);
  451. /** Clear history of connecting clients used by entry and bridge stats. */
  452. static void
  453. client_history_clear(void)
  454. {
  455. clientmap_entry_t **ent, **next, *this;
  456. for (ent = HT_START(clientmap, &client_history); ent != NULL;
  457. ent = next) {
  458. if ((*ent)->action == GEOIP_CLIENT_CONNECT) {
  459. this = *ent;
  460. next = HT_NEXT_RMV(clientmap, &client_history, ent);
  461. tor_free(this);
  462. } else {
  463. next = HT_NEXT(clientmap, &client_history, ent);
  464. }
  465. }
  466. }
  467. /** Note that we've seen a client connect from the IP <b>addr</b>
  468. * at time <b>now</b>. Ignored by all but bridges and directories if
  469. * configured accordingly. */
  470. void
  471. geoip_note_client_seen(geoip_client_action_t action,
  472. const tor_addr_t *addr, time_t now)
  473. {
  474. const or_options_t *options = get_options();
  475. clientmap_entry_t lookup, *ent;
  476. if (action == GEOIP_CLIENT_CONNECT) {
  477. /* Only remember statistics as entry guard or as bridge. */
  478. if (!options->EntryStatistics &&
  479. (!(options->BridgeRelay && options->BridgeRecordUsageByCountry)))
  480. return;
  481. } else {
  482. if (options->BridgeRelay || options->BridgeAuthoritativeDir ||
  483. !options->DirReqStatistics)
  484. return;
  485. }
  486. tor_addr_copy(&lookup.addr, addr);
  487. lookup.action = (int)action;
  488. ent = HT_FIND(clientmap, &client_history, &lookup);
  489. if (! ent) {
  490. ent = tor_malloc_zero(sizeof(clientmap_entry_t));
  491. tor_addr_copy(&ent->addr, addr);
  492. ent->action = (int)action;
  493. HT_INSERT(clientmap, &client_history, ent);
  494. }
  495. if (now / 60 <= (int)MAX_LAST_SEEN_IN_MINUTES && now >= 0)
  496. ent->last_seen_in_minutes = (unsigned)(now/60);
  497. else
  498. ent->last_seen_in_minutes = 0;
  499. if (action == GEOIP_CLIENT_NETWORKSTATUS) {
  500. int country_idx = geoip_get_country_by_addr(addr);
  501. if (country_idx < 0)
  502. country_idx = 0; /** unresolved requests are stored at index 0. */
  503. if (country_idx >= 0 && country_idx < smartlist_len(geoip_countries)) {
  504. geoip_country_t *country = smartlist_get(geoip_countries, country_idx);
  505. ++country->n_v3_ns_requests;
  506. }
  507. }
  508. }
  509. /** HT_FOREACH helper: remove a clientmap_entry_t from the hashtable if it's
  510. * older than a certain time. */
  511. static int
  512. remove_old_client_helper_(struct clientmap_entry_t *ent, void *_cutoff)
  513. {
  514. time_t cutoff = *(time_t*)_cutoff / 60;
  515. if (ent->last_seen_in_minutes < cutoff) {
  516. tor_free(ent);
  517. return 1;
  518. } else {
  519. return 0;
  520. }
  521. }
  522. /** Forget about all clients that haven't connected since <b>cutoff</b>. */
  523. void
  524. geoip_remove_old_clients(time_t cutoff)
  525. {
  526. clientmap_HT_FOREACH_FN(&client_history,
  527. remove_old_client_helper_,
  528. &cutoff);
  529. }
  530. /** How many responses are we giving to clients requesting v3 network
  531. * statuses? */
  532. static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM];
  533. /** Note that we've rejected a client's request for a v3 network status
  534. * for reason <b>reason</b> at time <b>now</b>. */
  535. void
  536. geoip_note_ns_response(geoip_ns_response_t response)
  537. {
  538. static int arrays_initialized = 0;
  539. if (!get_options()->DirReqStatistics)
  540. return;
  541. if (!arrays_initialized) {
  542. memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
  543. arrays_initialized = 1;
  544. }
  545. tor_assert(response < GEOIP_NS_RESPONSE_NUM);
  546. ns_v3_responses[response]++;
  547. }
  548. /** Do not mention any country from which fewer than this number of IPs have
  549. * connected. This conceivably avoids reporting information that could
  550. * deanonymize users, though analysis is lacking. */
  551. #define MIN_IPS_TO_NOTE_COUNTRY 1
  552. /** Do not report any geoip data at all if we have fewer than this number of
  553. * IPs to report about. */
  554. #define MIN_IPS_TO_NOTE_ANYTHING 1
  555. /** When reporting geoip data about countries, round up to the nearest
  556. * multiple of this value. */
  557. #define IP_GRANULARITY 8
  558. /** Helper type: used to sort per-country totals by value. */
  559. typedef struct c_hist_t {
  560. char country[3]; /**< Two-letter country code. */
  561. unsigned total; /**< Total IP addresses seen in this country. */
  562. } c_hist_t;
  563. /** Sorting helper: return -1, 1, or 0 based on comparison of two
  564. * geoip_ipv4_entry_t. Sort in descending order of total, and then by country
  565. * code. */
  566. static int
  567. c_hist_compare_(const void **_a, const void **_b)
  568. {
  569. const c_hist_t *a = *_a, *b = *_b;
  570. if (a->total > b->total)
  571. return -1;
  572. else if (a->total < b->total)
  573. return 1;
  574. else
  575. return strcmp(a->country, b->country);
  576. }
  577. /** When there are incomplete directory requests at the end of a 24-hour
  578. * period, consider those requests running for longer than this timeout as
  579. * failed, the others as still running. */
  580. #define DIRREQ_TIMEOUT (10*60)
  581. /** Entry in a map from either chan->global_identifier for direct requests
  582. * or a unique circuit identifier for tunneled requests to request time,
  583. * response size, and completion time of a network status request. Used to
  584. * measure download times of requests to derive average client
  585. * bandwidths. */
  586. typedef struct dirreq_map_entry_t {
  587. HT_ENTRY(dirreq_map_entry_t) node;
  588. /** Unique identifier for this network status request; this is either the
  589. * chan->global_identifier of the dir channel (direct request) or a new
  590. * locally unique identifier of a circuit (tunneled request). This ID is
  591. * only unique among other direct or tunneled requests, respectively. */
  592. uint64_t dirreq_id;
  593. unsigned int state:3; /**< State of this directory request. */
  594. unsigned int type:1; /**< Is this a direct or a tunneled request? */
  595. unsigned int completed:1; /**< Is this request complete? */
  596. /** When did we receive the request and started sending the response? */
  597. struct timeval request_time;
  598. size_t response_size; /**< What is the size of the response in bytes? */
  599. struct timeval completion_time; /**< When did the request succeed? */
  600. } dirreq_map_entry_t;
  601. /** Map of all directory requests asking for v2 or v3 network statuses in
  602. * the current geoip-stats interval. Values are
  603. * of type *<b>dirreq_map_entry_t</b>. */
  604. static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map =
  605. HT_INITIALIZER();
  606. static int
  607. dirreq_map_ent_eq(const dirreq_map_entry_t *a,
  608. const dirreq_map_entry_t *b)
  609. {
  610. return a->dirreq_id == b->dirreq_id && a->type == b->type;
  611. }
  612. /* DOCDOC dirreq_map_ent_hash */
  613. static unsigned
  614. dirreq_map_ent_hash(const dirreq_map_entry_t *entry)
  615. {
  616. unsigned u = (unsigned) entry->dirreq_id;
  617. u += entry->type << 20;
  618. return u;
  619. }
  620. HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
  621. dirreq_map_ent_eq);
  622. HT_GENERATE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
  623. dirreq_map_ent_eq, 0.6, malloc, realloc, free);
  624. /** Helper: Put <b>entry</b> into map of directory requests using
  625. * <b>type</b> and <b>dirreq_id</b> as key parts. If there is
  626. * already an entry for that key, print out a BUG warning and return. */
  627. static void
  628. dirreq_map_put_(dirreq_map_entry_t *entry, dirreq_type_t type,
  629. uint64_t dirreq_id)
  630. {
  631. dirreq_map_entry_t *old_ent;
  632. tor_assert(entry->type == type);
  633. tor_assert(entry->dirreq_id == dirreq_id);
  634. /* XXXX we could switch this to HT_INSERT some time, since it seems that
  635. * this bug doesn't happen. But since this function doesn't seem to be
  636. * critical-path, it's sane to leave it alone. */
  637. old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry);
  638. if (old_ent && old_ent != entry) {
  639. log_warn(LD_BUG, "Error when putting directory request into local "
  640. "map. There was already an entry for the same identifier.");
  641. return;
  642. }
  643. }
  644. /** Helper: Look up and return an entry in the map of directory requests
  645. * using <b>type</b> and <b>dirreq_id</b> as key parts. If there
  646. * is no such entry, return NULL. */
  647. static dirreq_map_entry_t *
  648. dirreq_map_get_(dirreq_type_t type, uint64_t dirreq_id)
  649. {
  650. dirreq_map_entry_t lookup;
  651. lookup.type = type;
  652. lookup.dirreq_id = dirreq_id;
  653. return HT_FIND(dirreqmap, &dirreq_map, &lookup);
  654. }
  655. /** Note that an either direct or tunneled (see <b>type</b>) directory
  656. * request for a v3 network status with unique ID <b>dirreq_id</b> of size
  657. * <b>response_size</b> has started. */
  658. void
  659. geoip_start_dirreq(uint64_t dirreq_id, size_t response_size,
  660. dirreq_type_t type)
  661. {
  662. dirreq_map_entry_t *ent;
  663. if (!get_options()->DirReqStatistics)
  664. return;
  665. ent = tor_malloc_zero(sizeof(dirreq_map_entry_t));
  666. ent->dirreq_id = dirreq_id;
  667. tor_gettimeofday(&ent->request_time);
  668. ent->response_size = response_size;
  669. ent->type = type;
  670. dirreq_map_put_(ent, type, dirreq_id);
  671. }
  672. /** Change the state of the either direct or tunneled (see <b>type</b>)
  673. * directory request with <b>dirreq_id</b> to <b>new_state</b> and
  674. * possibly mark it as completed. If no entry can be found for the given
  675. * key parts (e.g., if this is a directory request that we are not
  676. * measuring, or one that was started in the previous measurement period),
  677. * or if the state cannot be advanced to <b>new_state</b>, do nothing. */
  678. void
  679. geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
  680. dirreq_state_t new_state)
  681. {
  682. dirreq_map_entry_t *ent;
  683. if (!get_options()->DirReqStatistics)
  684. return;
  685. ent = dirreq_map_get_(type, dirreq_id);
  686. if (!ent)
  687. return;
  688. if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS)
  689. return;
  690. if (new_state - 1 != ent->state)
  691. return;
  692. ent->state = new_state;
  693. if ((type == DIRREQ_DIRECT &&
  694. new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) ||
  695. (type == DIRREQ_TUNNELED &&
  696. new_state == DIRREQ_CHANNEL_BUFFER_FLUSHED)) {
  697. tor_gettimeofday(&ent->completion_time);
  698. ent->completed = 1;
  699. }
  700. }
  701. /** Return a newly allocated comma-separated string containing statistics
  702. * on network status downloads. The string contains the number of completed
  703. * requests, timeouts, and still running requests as well as the download
  704. * times by deciles and quartiles. Return NULL if we have not observed
  705. * requests for long enough. */
  706. static char *
  707. geoip_get_dirreq_history(dirreq_type_t type)
  708. {
  709. char *result = NULL;
  710. smartlist_t *dirreq_completed = NULL;
  711. uint32_t complete = 0, timeouts = 0, running = 0;
  712. int bufsize = 1024, written;
  713. dirreq_map_entry_t **ptr, **next, *ent;
  714. struct timeval now;
  715. tor_gettimeofday(&now);
  716. dirreq_completed = smartlist_new();
  717. for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) {
  718. ent = *ptr;
  719. if (ent->type != type) {
  720. next = HT_NEXT(dirreqmap, &dirreq_map, ptr);
  721. continue;
  722. } else {
  723. if (ent->completed) {
  724. smartlist_add(dirreq_completed, ent);
  725. complete++;
  726. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
  727. } else {
  728. if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT)
  729. timeouts++;
  730. else
  731. running++;
  732. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
  733. tor_free(ent);
  734. }
  735. }
  736. }
  737. #define DIR_REQ_GRANULARITY 4
  738. complete = round_uint32_to_next_multiple_of(complete,
  739. DIR_REQ_GRANULARITY);
  740. timeouts = round_uint32_to_next_multiple_of(timeouts,
  741. DIR_REQ_GRANULARITY);
  742. running = round_uint32_to_next_multiple_of(running,
  743. DIR_REQ_GRANULARITY);
  744. result = tor_malloc_zero(bufsize);
  745. written = tor_snprintf(result, bufsize, "complete=%u,timeout=%u,"
  746. "running=%u", complete, timeouts, running);
  747. if (written < 0) {
  748. tor_free(result);
  749. goto done;
  750. }
  751. #define MIN_DIR_REQ_RESPONSES 16
  752. if (complete >= MIN_DIR_REQ_RESPONSES) {
  753. uint32_t *dltimes;
  754. /* We may have rounded 'completed' up. Here we want to use the
  755. * real value. */
  756. complete = smartlist_len(dirreq_completed);
  757. dltimes = tor_malloc_zero(sizeof(uint32_t) * complete);
  758. SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) {
  759. uint32_t bytes_per_second;
  760. uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time,
  761. &ent->completion_time);
  762. if (time_diff == 0)
  763. time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible
  764. * by law of nature or something, but a milisecond
  765. * is a bit greater than "instantly" */
  766. bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff);
  767. dltimes[ent_sl_idx] = bytes_per_second;
  768. } SMARTLIST_FOREACH_END(ent);
  769. median_uint32(dltimes, complete); /* sorts as a side effect. */
  770. written = tor_snprintf(result + written, bufsize - written,
  771. ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u,"
  772. "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u",
  773. dltimes[0],
  774. dltimes[1*complete/10-1],
  775. dltimes[2*complete/10-1],
  776. dltimes[1*complete/4-1],
  777. dltimes[3*complete/10-1],
  778. dltimes[4*complete/10-1],
  779. dltimes[5*complete/10-1],
  780. dltimes[6*complete/10-1],
  781. dltimes[7*complete/10-1],
  782. dltimes[3*complete/4-1],
  783. dltimes[8*complete/10-1],
  784. dltimes[9*complete/10-1],
  785. dltimes[complete-1]);
  786. if (written<0)
  787. tor_free(result);
  788. tor_free(dltimes);
  789. }
  790. done:
  791. SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent,
  792. tor_free(ent));
  793. smartlist_free(dirreq_completed);
  794. return result;
  795. }
  796. /** Store a newly allocated comma-separated string in
  797. * *<a>country_str</a> containing entries for all the countries from
  798. * which we've seen enough clients connect as a bridge, directory
  799. * server, or entry guard. The entry format is cc=num where num is the
  800. * number of IPs we've seen connecting from that country, and cc is a
  801. * lowercased country code. *<a>country_str</a> is set to NULL if
  802. * we're not ready to export per country data yet.
  803. *
  804. * Store a newly allocated comma-separated string in <a>ipver_str</a>
  805. * containing entries for clients connecting over IPv4 and IPv6. The
  806. * format is family=num where num is the nubmer of IPs we've seen
  807. * connecting over that protocol family, and family is 'v4' or 'v6'.
  808. *
  809. * Return 0 on success and -1 if we're missing geoip data. */
  810. int
  811. geoip_get_client_history(geoip_client_action_t action,
  812. char **country_str, char **ipver_str)
  813. {
  814. unsigned granularity = IP_GRANULARITY;
  815. smartlist_t *entries = NULL;
  816. int n_countries = geoip_get_n_countries();
  817. int i;
  818. clientmap_entry_t **ent;
  819. unsigned *counts = NULL;
  820. unsigned total = 0;
  821. unsigned ipv4_count = 0, ipv6_count = 0;
  822. if (!geoip_is_loaded(AF_INET) && !geoip_is_loaded(AF_INET6))
  823. return -1;
  824. counts = tor_malloc_zero(sizeof(unsigned)*n_countries);
  825. HT_FOREACH(ent, clientmap, &client_history) {
  826. int country;
  827. if ((*ent)->action != (int)action)
  828. continue;
  829. country = geoip_get_country_by_addr(&(*ent)->addr);
  830. if (country < 0)
  831. country = 0; /** unresolved requests are stored at index 0. */
  832. tor_assert(0 <= country && country < n_countries);
  833. ++counts[country];
  834. ++total;
  835. switch (tor_addr_family(&(*ent)->addr)) {
  836. case AF_INET:
  837. ipv4_count++;
  838. break;
  839. case AF_INET6:
  840. ipv6_count++;
  841. break;
  842. }
  843. }
  844. if (ipver_str) {
  845. smartlist_t *chunks = smartlist_new();
  846. smartlist_add_asprintf(chunks, "v4=%u",
  847. round_to_next_multiple_of(ipv4_count, granularity));
  848. smartlist_add_asprintf(chunks, "v6=%u",
  849. round_to_next_multiple_of(ipv6_count, granularity));
  850. *ipver_str = smartlist_join_strings(chunks, ",", 0, NULL);
  851. SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
  852. smartlist_free(chunks);
  853. }
  854. /* Don't record per country data if we haven't seen enough IPs. */
  855. if (total < MIN_IPS_TO_NOTE_ANYTHING) {
  856. tor_free(counts);
  857. if (country_str)
  858. *country_str = NULL;
  859. return 0;
  860. }
  861. /* Make a list of c_hist_t */
  862. entries = smartlist_new();
  863. for (i = 0; i < n_countries; ++i) {
  864. unsigned c = counts[i];
  865. const char *countrycode;
  866. c_hist_t *ent;
  867. /* Only report a country if it has a minimum number of IPs. */
  868. if (c >= MIN_IPS_TO_NOTE_COUNTRY) {
  869. c = round_to_next_multiple_of(c, granularity);
  870. countrycode = geoip_get_country_name(i);
  871. ent = tor_malloc(sizeof(c_hist_t));
  872. strlcpy(ent->country, countrycode, sizeof(ent->country));
  873. ent->total = c;
  874. smartlist_add(entries, ent);
  875. }
  876. }
  877. /* Sort entries. Note that we must do this _AFTER_ rounding, or else
  878. * the sort order could leak info. */
  879. smartlist_sort(entries, c_hist_compare_);
  880. if (country_str) {
  881. smartlist_t *chunks = smartlist_new();
  882. SMARTLIST_FOREACH(entries, c_hist_t *, ch, {
  883. smartlist_add_asprintf(chunks, "%s=%u", ch->country, ch->total);
  884. });
  885. *country_str = smartlist_join_strings(chunks, ",", 0, NULL);
  886. SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
  887. smartlist_free(chunks);
  888. }
  889. SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c));
  890. smartlist_free(entries);
  891. tor_free(counts);
  892. return 0;
  893. }
  894. /** Return a newly allocated string holding the per-country request history
  895. * for v3 network statuses in a format suitable for an extra-info document,
  896. * or NULL on failure. */
  897. char *
  898. geoip_get_request_history(void)
  899. {
  900. smartlist_t *entries, *strings;
  901. char *result;
  902. unsigned granularity = IP_GRANULARITY;
  903. if (!geoip_countries)
  904. return NULL;
  905. entries = smartlist_new();
  906. SMARTLIST_FOREACH_BEGIN(geoip_countries, geoip_country_t *, c) {
  907. uint32_t tot = 0;
  908. c_hist_t *ent;
  909. tot = c->n_v3_ns_requests;
  910. if (!tot)
  911. continue;
  912. ent = tor_malloc_zero(sizeof(c_hist_t));
  913. strlcpy(ent->country, c->countrycode, sizeof(ent->country));
  914. ent->total = round_to_next_multiple_of(tot, granularity);
  915. smartlist_add(entries, ent);
  916. } SMARTLIST_FOREACH_END(c);
  917. smartlist_sort(entries, c_hist_compare_);
  918. strings = smartlist_new();
  919. SMARTLIST_FOREACH(entries, c_hist_t *, ent, {
  920. smartlist_add_asprintf(strings, "%s=%u", ent->country, ent->total);
  921. });
  922. result = smartlist_join_strings(strings, ",", 0, NULL);
  923. SMARTLIST_FOREACH(strings, char *, cp, tor_free(cp));
  924. SMARTLIST_FOREACH(entries, c_hist_t *, ent, tor_free(ent));
  925. smartlist_free(strings);
  926. smartlist_free(entries);
  927. return result;
  928. }
  929. /** Start time of directory request stats or 0 if we're not collecting
  930. * directory request statistics. */
  931. static time_t start_of_dirreq_stats_interval;
  932. /** Initialize directory request stats. */
  933. void
  934. geoip_dirreq_stats_init(time_t now)
  935. {
  936. start_of_dirreq_stats_interval = now;
  937. }
  938. /** Reset counters for dirreq stats. */
  939. void
  940. geoip_reset_dirreq_stats(time_t now)
  941. {
  942. SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, {
  943. c->n_v3_ns_requests = 0;
  944. });
  945. {
  946. clientmap_entry_t **ent, **next, *this;
  947. for (ent = HT_START(clientmap, &client_history); ent != NULL;
  948. ent = next) {
  949. if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS) {
  950. this = *ent;
  951. next = HT_NEXT_RMV(clientmap, &client_history, ent);
  952. tor_free(this);
  953. } else {
  954. next = HT_NEXT(clientmap, &client_history, ent);
  955. }
  956. }
  957. }
  958. memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
  959. {
  960. dirreq_map_entry_t **ent, **next, *this;
  961. for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
  962. this = *ent;
  963. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
  964. tor_free(this);
  965. }
  966. }
  967. start_of_dirreq_stats_interval = now;
  968. }
  969. /** Stop collecting directory request stats in a way that we can re-start
  970. * doing so in geoip_dirreq_stats_init(). */
  971. void
  972. geoip_dirreq_stats_term(void)
  973. {
  974. geoip_reset_dirreq_stats(0);
  975. }
  976. /** Return a newly allocated string containing the dirreq statistics
  977. * until <b>now</b>, or NULL if we're not collecting dirreq stats. Caller
  978. * must ensure start_of_dirreq_stats_interval is in the past. */
  979. char *
  980. geoip_format_dirreq_stats(time_t now)
  981. {
  982. char t[ISO_TIME_LEN+1];
  983. int i;
  984. char *v3_ips_string, *v3_reqs_string, *v3_direct_dl_string,
  985. *v3_tunneled_dl_string;
  986. char *result;
  987. if (!start_of_dirreq_stats_interval)
  988. return NULL; /* Not initialized. */
  989. tor_assert(now >= start_of_dirreq_stats_interval);
  990. format_iso_time(t, now);
  991. geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS, &v3_ips_string, NULL);
  992. v3_reqs_string = geoip_get_request_history();
  993. #define RESPONSE_GRANULARITY 8
  994. for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) {
  995. ns_v3_responses[i] = round_uint32_to_next_multiple_of(
  996. ns_v3_responses[i], RESPONSE_GRANULARITY);
  997. }
  998. #undef RESPONSE_GRANULARITY
  999. v3_direct_dl_string = geoip_get_dirreq_history(DIRREQ_DIRECT);
  1000. v3_tunneled_dl_string = geoip_get_dirreq_history(DIRREQ_TUNNELED);
  1001. /* Put everything together into a single string. */
  1002. tor_asprintf(&result, "dirreq-stats-end %s (%d s)\n"
  1003. "dirreq-v3-ips %s\n"
  1004. "dirreq-v3-reqs %s\n"
  1005. "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u,"
  1006. "not-found=%u,not-modified=%u,busy=%u\n"
  1007. "dirreq-v3-direct-dl %s\n"
  1008. "dirreq-v3-tunneled-dl %s\n",
  1009. t,
  1010. (unsigned) (now - start_of_dirreq_stats_interval),
  1011. v3_ips_string ? v3_ips_string : "",
  1012. v3_reqs_string ? v3_reqs_string : "",
  1013. ns_v3_responses[GEOIP_SUCCESS],
  1014. ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS],
  1015. ns_v3_responses[GEOIP_REJECT_UNAVAILABLE],
  1016. ns_v3_responses[GEOIP_REJECT_NOT_FOUND],
  1017. ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED],
  1018. ns_v3_responses[GEOIP_REJECT_BUSY],
  1019. v3_direct_dl_string ? v3_direct_dl_string : "",
  1020. v3_tunneled_dl_string ? v3_tunneled_dl_string : "");
  1021. /* Free partial strings. */
  1022. tor_free(v3_ips_string);
  1023. tor_free(v3_reqs_string);
  1024. tor_free(v3_direct_dl_string);
  1025. tor_free(v3_tunneled_dl_string);
  1026. return result;
  1027. }
  1028. /** If 24 hours have passed since the beginning of the current dirreq
  1029. * stats period, write dirreq stats to $DATADIR/stats/dirreq-stats
  1030. * (possibly overwriting an existing file) and reset counters. Return
  1031. * when we would next want to write dirreq stats or 0 if we never want to
  1032. * write. */
  1033. time_t
  1034. geoip_dirreq_stats_write(time_t now)
  1035. {
  1036. char *statsdir = NULL, *filename = NULL, *str = NULL;
  1037. if (!start_of_dirreq_stats_interval)
  1038. return 0; /* Not initialized. */
  1039. if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now)
  1040. goto done; /* Not ready to write. */
  1041. /* Discard all items in the client history that are too old. */
  1042. geoip_remove_old_clients(start_of_dirreq_stats_interval);
  1043. /* Generate history string .*/
  1044. str = geoip_format_dirreq_stats(now);
  1045. /* Write dirreq-stats string to disk. */
  1046. statsdir = get_datadir_fname("stats");
  1047. if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) {
  1048. log_warn(LD_HIST, "Unable to create stats/ directory!");
  1049. goto done;
  1050. }
  1051. filename = get_datadir_fname2("stats", "dirreq-stats");
  1052. if (write_str_to_file(filename, str, 0) < 0)
  1053. log_warn(LD_HIST, "Unable to write dirreq statistics to disk!");
  1054. /* Reset measurement interval start. */
  1055. geoip_reset_dirreq_stats(now);
  1056. done:
  1057. tor_free(statsdir);
  1058. tor_free(filename);
  1059. tor_free(str);
  1060. return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL;
  1061. }
  1062. /** Start time of bridge stats or 0 if we're not collecting bridge
  1063. * statistics. */
  1064. static time_t start_of_bridge_stats_interval;
  1065. /** Initialize bridge stats. */
  1066. void
  1067. geoip_bridge_stats_init(time_t now)
  1068. {
  1069. start_of_bridge_stats_interval = now;
  1070. }
  1071. /** Stop collecting bridge stats in a way that we can re-start doing so in
  1072. * geoip_bridge_stats_init(). */
  1073. void
  1074. geoip_bridge_stats_term(void)
  1075. {
  1076. client_history_clear();
  1077. start_of_bridge_stats_interval = 0;
  1078. }
  1079. /** Validate a bridge statistics string as it would be written to a
  1080. * current extra-info descriptor. Return 1 if the string is valid and
  1081. * recent enough, or 0 otherwise. */
  1082. static int
  1083. validate_bridge_stats(const char *stats_str, time_t now)
  1084. {
  1085. char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1],
  1086. *eos;
  1087. const char *BRIDGE_STATS_END = "bridge-stats-end ";
  1088. const char *BRIDGE_IPS = "bridge-ips ";
  1089. const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n";
  1090. const char *tmp;
  1091. time_t stats_end_time;
  1092. int seconds;
  1093. tor_assert(stats_str);
  1094. /* Parse timestamp and number of seconds from
  1095. "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */
  1096. tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END);
  1097. if (!tmp)
  1098. return 0;
  1099. tmp += strlen(BRIDGE_STATS_END);
  1100. if (strlen(tmp) < ISO_TIME_LEN + 6)
  1101. return 0;
  1102. strlcpy(stats_end_str, tmp, sizeof(stats_end_str));
  1103. if (parse_iso_time(stats_end_str, &stats_end_time) < 0)
  1104. return 0;
  1105. if (stats_end_time < now - (25*60*60) ||
  1106. stats_end_time > now + (1*60*60))
  1107. return 0;
  1108. seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10);
  1109. if (!eos || seconds < 23*60*60)
  1110. return 0;
  1111. format_iso_time(stats_start_str, stats_end_time - seconds);
  1112. /* Parse: "bridge-ips CC=N,CC=N,..." */
  1113. tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS);
  1114. if (!tmp) {
  1115. /* Look if there is an empty "bridge-ips" line */
  1116. tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE);
  1117. if (!tmp)
  1118. return 0;
  1119. }
  1120. return 1;
  1121. }
  1122. /** Most recent bridge statistics formatted to be written to extra-info
  1123. * descriptors. */
  1124. static char *bridge_stats_extrainfo = NULL;
  1125. /** Return a newly allocated string holding our bridge usage stats by country
  1126. * in a format suitable for inclusion in an extrainfo document. Return NULL on
  1127. * failure. */
  1128. char *
  1129. geoip_format_bridge_stats(time_t now)
  1130. {
  1131. char *out = NULL, *country_data = NULL, *ipver_data = NULL;
  1132. long duration = now - start_of_bridge_stats_interval;
  1133. char written[ISO_TIME_LEN+1];
  1134. if (duration < 0)
  1135. return NULL;
  1136. if (!start_of_bridge_stats_interval)
  1137. return NULL; /* Not initialized. */
  1138. format_iso_time(written, now);
  1139. geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
  1140. tor_asprintf(&out,
  1141. "bridge-stats-end %s (%ld s)\n"
  1142. "bridge-ips %s\n"
  1143. "bridge-ip-versions %s\n",
  1144. written, duration,
  1145. country_data ? country_data : "",
  1146. ipver_data ? ipver_data : "");
  1147. tor_free(country_data);
  1148. tor_free(ipver_data);
  1149. return out;
  1150. }
  1151. /** Return a newly allocated string holding our bridge usage stats by country
  1152. * in a format suitable for the answer to a controller request. Return NULL on
  1153. * failure. */
  1154. static char *
  1155. format_bridge_stats_controller(time_t now)
  1156. {
  1157. char *out = NULL, *country_data = NULL, *ipver_data = NULL;
  1158. char started[ISO_TIME_LEN+1];
  1159. (void) now;
  1160. format_iso_time(started, start_of_bridge_stats_interval);
  1161. geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
  1162. tor_asprintf(&out,
  1163. "TimeStarted=\"%s\" CountrySummary=%s IPVersions=%s",
  1164. started,
  1165. country_data ? country_data : "",
  1166. ipver_data ? ipver_data : "");
  1167. tor_free(country_data);
  1168. tor_free(ipver_data);
  1169. return out;
  1170. }
  1171. /** Write bridge statistics to $DATADIR/stats/bridge-stats and return
  1172. * when we should next try to write statistics. */
  1173. time_t
  1174. geoip_bridge_stats_write(time_t now)
  1175. {
  1176. char *filename = NULL, *val = NULL, *statsdir = NULL;
  1177. /* Check if 24 hours have passed since starting measurements. */
  1178. if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL)
  1179. return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
  1180. /* Discard all items in the client history that are too old. */
  1181. geoip_remove_old_clients(start_of_bridge_stats_interval);
  1182. /* Generate formatted string */
  1183. val = geoip_format_bridge_stats(now);
  1184. if (val == NULL)
  1185. goto done;
  1186. /* Update the stored value. */
  1187. tor_free(bridge_stats_extrainfo);
  1188. bridge_stats_extrainfo = val;
  1189. start_of_bridge_stats_interval = now;
  1190. /* Write it to disk. */
  1191. statsdir = get_datadir_fname("stats");
  1192. if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0)
  1193. goto done;
  1194. filename = get_datadir_fname2("stats", "bridge-stats");
  1195. write_str_to_file(filename, bridge_stats_extrainfo, 0);
  1196. /* Tell the controller, "hey, there are clients!" */
  1197. {
  1198. char *controller_str = format_bridge_stats_controller(now);
  1199. if (controller_str)
  1200. control_event_clients_seen(controller_str);
  1201. tor_free(controller_str);
  1202. }
  1203. done:
  1204. tor_free(filename);
  1205. tor_free(statsdir);
  1206. return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
  1207. }
  1208. /** Try to load the most recent bridge statistics from disk, unless we
  1209. * have finished a measurement interval lately, and check whether they
  1210. * are still recent enough. */
  1211. static void
  1212. load_bridge_stats(time_t now)
  1213. {
  1214. char *fname, *contents;
  1215. if (bridge_stats_extrainfo)
  1216. return;
  1217. fname = get_datadir_fname2("stats", "bridge-stats");
  1218. contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
  1219. if (contents && validate_bridge_stats(contents, now)) {
  1220. bridge_stats_extrainfo = contents;
  1221. } else {
  1222. tor_free(contents);
  1223. }
  1224. tor_free(fname);
  1225. }
  1226. /** Return most recent bridge statistics for inclusion in extra-info
  1227. * descriptors, or NULL if we don't have recent bridge statistics. */
  1228. const char *
  1229. geoip_get_bridge_stats_extrainfo(time_t now)
  1230. {
  1231. load_bridge_stats(now);
  1232. return bridge_stats_extrainfo;
  1233. }
  1234. /** Return a new string containing the recent bridge statistics to be returned
  1235. * to controller clients, or NULL if we don't have any bridge statistics. */
  1236. char *
  1237. geoip_get_bridge_stats_controller(time_t now)
  1238. {
  1239. return format_bridge_stats_controller(now);
  1240. }
  1241. /** Start time of entry stats or 0 if we're not collecting entry
  1242. * statistics. */
  1243. static time_t start_of_entry_stats_interval;
  1244. /** Initialize entry stats. */
  1245. void
  1246. geoip_entry_stats_init(time_t now)
  1247. {
  1248. start_of_entry_stats_interval = now;
  1249. }
  1250. /** Reset counters for entry stats. */
  1251. void
  1252. geoip_reset_entry_stats(time_t now)
  1253. {
  1254. client_history_clear();
  1255. start_of_entry_stats_interval = now;
  1256. }
  1257. /** Stop collecting entry stats in a way that we can re-start doing so in
  1258. * geoip_entry_stats_init(). */
  1259. void
  1260. geoip_entry_stats_term(void)
  1261. {
  1262. geoip_reset_entry_stats(0);
  1263. }
  1264. /** Return a newly allocated string containing the entry statistics
  1265. * until <b>now</b>, or NULL if we're not collecting entry stats. Caller
  1266. * must ensure start_of_entry_stats_interval lies in the past. */
  1267. char *
  1268. geoip_format_entry_stats(time_t now)
  1269. {
  1270. char t[ISO_TIME_LEN+1];
  1271. char *data = NULL;
  1272. char *result;
  1273. if (!start_of_entry_stats_interval)
  1274. return NULL; /* Not initialized. */
  1275. tor_assert(now >= start_of_entry_stats_interval);
  1276. geoip_get_client_history(GEOIP_CLIENT_CONNECT, &data, NULL);
  1277. format_iso_time(t, now);
  1278. tor_asprintf(&result,
  1279. "entry-stats-end %s (%u s)\n"
  1280. "entry-ips %s\n",
  1281. t, (unsigned) (now - start_of_entry_stats_interval),
  1282. data ? data : "");
  1283. tor_free(data);
  1284. return result;
  1285. }
  1286. /** If 24 hours have passed since the beginning of the current entry stats
  1287. * period, write entry stats to $DATADIR/stats/entry-stats (possibly
  1288. * overwriting an existing file) and reset counters. Return when we would
  1289. * next want to write entry stats or 0 if we never want to write. */
  1290. time_t
  1291. geoip_entry_stats_write(time_t now)
  1292. {
  1293. char *statsdir = NULL, *filename = NULL, *str = NULL;
  1294. if (!start_of_entry_stats_interval)
  1295. return 0; /* Not initialized. */
  1296. if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now)
  1297. goto done; /* Not ready to write. */
  1298. /* Discard all items in the client history that are too old. */
  1299. geoip_remove_old_clients(start_of_entry_stats_interval);
  1300. /* Generate history string .*/
  1301. str = geoip_format_entry_stats(now);
  1302. /* Write entry-stats string to disk. */
  1303. statsdir = get_datadir_fname("stats");
  1304. if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) {
  1305. log_warn(LD_HIST, "Unable to create stats/ directory!");
  1306. goto done;
  1307. }
  1308. filename = get_datadir_fname2("stats", "entry-stats");
  1309. if (write_str_to_file(filename, str, 0) < 0)
  1310. log_warn(LD_HIST, "Unable to write entry statistics to disk!");
  1311. /* Reset measurement interval start. */
  1312. geoip_reset_entry_stats(now);
  1313. done:
  1314. tor_free(statsdir);
  1315. tor_free(filename);
  1316. tor_free(str);
  1317. return start_of_entry_stats_interval + WRITE_STATS_INTERVAL;
  1318. }
  1319. /** Helper used to implement GETINFO ip-to-country/... controller command. */
  1320. int
  1321. getinfo_helper_geoip(control_connection_t *control_conn,
  1322. const char *question, char **answer,
  1323. const char **errmsg)
  1324. {
  1325. (void)control_conn;
  1326. if (!strcmpstart(question, "ip-to-country/")) {
  1327. int c;
  1328. sa_family_t family;
  1329. tor_addr_t addr;
  1330. question += strlen("ip-to-country/");
  1331. family = tor_addr_parse(&addr, question);
  1332. if (family != AF_INET && family != AF_INET6) {
  1333. *errmsg = "Invalid address family";
  1334. return -1;
  1335. }
  1336. if (!geoip_is_loaded(family)) {
  1337. *errmsg = "GeoIP data not loaded";
  1338. return -1;
  1339. }
  1340. if (family == AF_INET)
  1341. c = geoip_get_country_by_ipv4(tor_addr_to_ipv4h(&addr));
  1342. else /* AF_INET6 */
  1343. c = geoip_get_country_by_ipv6(tor_addr_to_in6(&addr));
  1344. *answer = tor_strdup(geoip_get_country_name(c));
  1345. }
  1346. return 0;
  1347. }
  1348. /** Release all storage held by the GeoIP databases and country list. */
  1349. static void
  1350. clear_geoip_db(void)
  1351. {
  1352. if (geoip_countries) {
  1353. SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, tor_free(c));
  1354. smartlist_free(geoip_countries);
  1355. }
  1356. strmap_free(country_idxplus1_by_lc_code, NULL);
  1357. if (geoip_ipv4_entries) {
  1358. SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, ent,
  1359. tor_free(ent));
  1360. smartlist_free(geoip_ipv4_entries);
  1361. }
  1362. if (geoip_ipv6_entries) {
  1363. SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, ent,
  1364. tor_free(ent));
  1365. smartlist_free(geoip_ipv6_entries);
  1366. }
  1367. geoip_countries = NULL;
  1368. country_idxplus1_by_lc_code = NULL;
  1369. geoip_ipv4_entries = NULL;
  1370. geoip_ipv6_entries = NULL;
  1371. }
  1372. /** Release all storage held in this file. */
  1373. void
  1374. geoip_free_all(void)
  1375. {
  1376. {
  1377. clientmap_entry_t **ent, **next, *this;
  1378. for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) {
  1379. this = *ent;
  1380. next = HT_NEXT_RMV(clientmap, &client_history, ent);
  1381. tor_free(this);
  1382. }
  1383. HT_CLEAR(clientmap, &client_history);
  1384. }
  1385. {
  1386. dirreq_map_entry_t **ent, **next, *this;
  1387. for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
  1388. this = *ent;
  1389. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
  1390. tor_free(this);
  1391. }
  1392. HT_CLEAR(dirreqmap, &dirreq_map);
  1393. }
  1394. clear_geoip_db();
  1395. }