geoip.c 46 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406
  1. /* Copyright (c) 2007-2011, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. /**
  4. * \file geoip.c
  5. * \brief Functions related to maintaining an IP-to-country database;
  6. * to summarizing client connections by country to entry guards, bridges,
  7. * and directory servers; and for statistics on answering network status
  8. * requests.
  9. */
  10. #define GEOIP_PRIVATE
  11. #include "or.h"
  12. #include "ht.h"
  13. #include "config.h"
  14. #include "control.h"
  15. #include "dnsserv.h"
  16. #include "geoip.h"
  17. #include "routerlist.h"
  18. static void clear_geoip_db(void);
  19. static void init_geoip_countries(void);
  20. /** An entry from the GeoIP file: maps an IP range to a country. */
  21. typedef struct geoip_entry_t {
  22. uint32_t ip_low; /**< The lowest IP in the range, in host order */
  23. uint32_t ip_high; /**< The highest IP in the range, in host order */
  24. intptr_t country; /**< An index into geoip_countries */
  25. } geoip_entry_t;
  26. /** A per-country record for GeoIP request history. */
  27. typedef struct geoip_country_t {
  28. char countrycode[3];
  29. uint32_t n_v2_ns_requests;
  30. uint32_t n_v3_ns_requests;
  31. } geoip_country_t;
  32. /** A list of geoip_country_t */
  33. static smartlist_t *geoip_countries = NULL;
  34. /** A map from lowercased country codes to their position in geoip_countries.
  35. * The index is encoded in the pointer, and 1 is added so that NULL can mean
  36. * not found. */
  37. static strmap_t *country_idxplus1_by_lc_code = NULL;
  38. /** A list of all known geoip_entry_t, sorted by ip_low. */
  39. static smartlist_t *geoip_entries = NULL;
  40. /** SHA1 digest of the GeoIP file to include in extra-info descriptors. */
  41. static char geoip_digest[DIGEST_LEN];
  42. /** Return the index of the <b>country</b>'s entry in the GeoIP DB
  43. * if it is a valid 2-letter country code, otherwise return -1.
  44. */
  45. country_t
  46. geoip_get_country(const char *country)
  47. {
  48. void *_idxplus1;
  49. intptr_t idx;
  50. _idxplus1 = strmap_get_lc(country_idxplus1_by_lc_code, country);
  51. if (!_idxplus1)
  52. return -1;
  53. idx = ((uintptr_t)_idxplus1)-1;
  54. return (country_t)idx;
  55. }
  56. /** Add an entry to the GeoIP table, mapping all IPs between <b>low</b> and
  57. * <b>high</b>, inclusive, to the 2-letter country code <b>country</b>.
  58. */
  59. static void
  60. geoip_add_entry(uint32_t low, uint32_t high, const char *country)
  61. {
  62. intptr_t idx;
  63. geoip_entry_t *ent;
  64. void *_idxplus1;
  65. if (high < low)
  66. return;
  67. _idxplus1 = strmap_get_lc(country_idxplus1_by_lc_code, country);
  68. if (!_idxplus1) {
  69. geoip_country_t *c = tor_malloc_zero(sizeof(geoip_country_t));
  70. strlcpy(c->countrycode, country, sizeof(c->countrycode));
  71. tor_strlower(c->countrycode);
  72. smartlist_add(geoip_countries, c);
  73. idx = smartlist_len(geoip_countries) - 1;
  74. strmap_set_lc(country_idxplus1_by_lc_code, country, (void*)(idx+1));
  75. } else {
  76. idx = ((uintptr_t)_idxplus1)-1;
  77. }
  78. {
  79. geoip_country_t *c = smartlist_get(geoip_countries, idx);
  80. tor_assert(!strcasecmp(c->countrycode, country));
  81. }
  82. ent = tor_malloc_zero(sizeof(geoip_entry_t));
  83. ent->ip_low = low;
  84. ent->ip_high = high;
  85. ent->country = idx;
  86. smartlist_add(geoip_entries, ent);
  87. }
  88. /** Add an entry to the GeoIP table, parsing it from <b>line</b>. The
  89. * format is as for geoip_load_file(). */
  90. /*private*/ int
  91. geoip_parse_entry(const char *line)
  92. {
  93. unsigned int low, high;
  94. char b[3];
  95. if (!geoip_countries)
  96. init_geoip_countries();
  97. if (!geoip_entries)
  98. geoip_entries = smartlist_create();
  99. while (TOR_ISSPACE(*line))
  100. ++line;
  101. if (*line == '#')
  102. return 0;
  103. if (sscanf(line,"%u,%u,%2s", &low, &high, b) == 3) {
  104. geoip_add_entry(low, high, b);
  105. return 0;
  106. } else if (sscanf(line,"\"%u\",\"%u\",\"%2s\",", &low, &high, b) == 3) {
  107. geoip_add_entry(low, high, b);
  108. return 0;
  109. } else {
  110. log_warn(LD_GENERAL, "Unable to parse line from GEOIP file: %s",
  111. escaped(line));
  112. return -1;
  113. }
  114. }
  115. /** Sorting helper: return -1, 1, or 0 based on comparison of two
  116. * geoip_entry_t */
  117. static int
  118. _geoip_compare_entries(const void **_a, const void **_b)
  119. {
  120. const geoip_entry_t *a = *_a, *b = *_b;
  121. if (a->ip_low < b->ip_low)
  122. return -1;
  123. else if (a->ip_low > b->ip_low)
  124. return 1;
  125. else
  126. return 0;
  127. }
  128. /** bsearch helper: return -1, 1, or 0 based on comparison of an IP (a pointer
  129. * to a uint32_t in host order) to a geoip_entry_t */
  130. static int
  131. _geoip_compare_key_to_entry(const void *_key, const void **_member)
  132. {
  133. /* No alignment issue here, since _key really is a pointer to uint32_t */
  134. const uint32_t addr = *(uint32_t *)_key;
  135. const geoip_entry_t *entry = *_member;
  136. if (addr < entry->ip_low)
  137. return -1;
  138. else if (addr > entry->ip_high)
  139. return 1;
  140. else
  141. return 0;
  142. }
  143. /** Return 1 if we should collect geoip stats on bridge users, and
  144. * include them in our extrainfo descriptor. Else return 0. */
  145. int
  146. should_record_bridge_info(or_options_t *options)
  147. {
  148. return options->BridgeRelay && options->BridgeRecordUsageByCountry;
  149. }
  150. /** Set up a new list of geoip countries with no countries (yet) set in it,
  151. * except for the unknown country.
  152. */
  153. static void
  154. init_geoip_countries(void)
  155. {
  156. geoip_country_t *geoip_unresolved;
  157. geoip_countries = smartlist_create();
  158. /* Add a geoip_country_t for requests that could not be resolved to a
  159. * country as first element (index 0) to geoip_countries. */
  160. geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t));
  161. strlcpy(geoip_unresolved->countrycode, "??",
  162. sizeof(geoip_unresolved->countrycode));
  163. smartlist_add(geoip_countries, geoip_unresolved);
  164. country_idxplus1_by_lc_code = strmap_new();
  165. strmap_set_lc(country_idxplus1_by_lc_code, "??", (void*)(1));
  166. }
  167. /** Clear the GeoIP database and reload it from the file
  168. * <b>filename</b>. Return 0 on success, -1 on failure.
  169. *
  170. * Recognized line formats are:
  171. * INTIPLOW,INTIPHIGH,CC
  172. * and
  173. * "INTIPLOW","INTIPHIGH","CC","CC3","COUNTRY NAME"
  174. * where INTIPLOW and INTIPHIGH are IPv4 addresses encoded as 4-byte unsigned
  175. * integers, and CC is a country code.
  176. *
  177. * It also recognizes, and skips over, blank lines and lines that start
  178. * with '#' (comments).
  179. */
  180. int
  181. geoip_load_file(const char *filename, or_options_t *options)
  182. {
  183. FILE *f;
  184. const char *msg = "";
  185. int severity = options_need_geoip_info(options, &msg) ? LOG_WARN : LOG_INFO;
  186. crypto_digest_env_t *geoip_digest_env = NULL;
  187. clear_geoip_db();
  188. if (!(f = tor_fopen_cloexec(filename, "r"))) {
  189. log_fn(severity, LD_GENERAL, "Failed to open GEOIP file %s. %s",
  190. filename, msg);
  191. return -1;
  192. }
  193. if (!geoip_countries)
  194. init_geoip_countries();
  195. if (geoip_entries) {
  196. SMARTLIST_FOREACH(geoip_entries, geoip_entry_t *, e, tor_free(e));
  197. smartlist_free(geoip_entries);
  198. }
  199. geoip_entries = smartlist_create();
  200. geoip_digest_env = crypto_new_digest_env();
  201. log_notice(LD_GENERAL, "Parsing GEOIP file %s.", filename);
  202. while (!feof(f)) {
  203. char buf[512];
  204. if (fgets(buf, (int)sizeof(buf), f) == NULL)
  205. break;
  206. crypto_digest_add_bytes(geoip_digest_env, buf, strlen(buf));
  207. /* FFFF track full country name. */
  208. geoip_parse_entry(buf);
  209. }
  210. /*XXXX abort and return -1 if no entries/illformed?*/
  211. fclose(f);
  212. smartlist_sort(geoip_entries, _geoip_compare_entries);
  213. /* Okay, now we need to maybe change our mind about what is in which
  214. * country. */
  215. refresh_all_country_info();
  216. /* Remember file digest so that we can include it in our extra-info
  217. * descriptors. */
  218. crypto_digest_get_digest(geoip_digest_env, geoip_digest, DIGEST_LEN);
  219. crypto_free_digest_env(geoip_digest_env);
  220. return 0;
  221. }
  222. /** Given an IP address in host order, return a number representing the
  223. * country to which that address belongs, -1 for "No geoip information
  224. * available", or 0 for the 'unknown country'. The return value will always
  225. * be less than geoip_get_n_countries(). To decode it, call
  226. * geoip_get_country_name().
  227. */
  228. int
  229. geoip_get_country_by_ip(uint32_t ipaddr)
  230. {
  231. geoip_entry_t *ent;
  232. if (!geoip_entries)
  233. return -1;
  234. ent = smartlist_bsearch(geoip_entries, &ipaddr, _geoip_compare_key_to_entry);
  235. return ent ? (int)ent->country : 0;
  236. }
  237. /** Return the number of countries recognized by the GeoIP database. */
  238. int
  239. geoip_get_n_countries(void)
  240. {
  241. if (!geoip_countries)
  242. init_geoip_countries();
  243. return (int) smartlist_len(geoip_countries);
  244. }
  245. /** Return the two-letter country code associated with the number <b>num</b>,
  246. * or "??" for an unknown value. */
  247. const char *
  248. geoip_get_country_name(country_t num)
  249. {
  250. if (geoip_countries && num >= 0 && num < smartlist_len(geoip_countries)) {
  251. geoip_country_t *c = smartlist_get(geoip_countries, num);
  252. return c->countrycode;
  253. } else
  254. return "??";
  255. }
  256. /** Return true iff we have loaded a GeoIP database.*/
  257. int
  258. geoip_is_loaded(void)
  259. {
  260. return geoip_countries != NULL && geoip_entries != NULL;
  261. }
  262. /** Return the hex-encoded SHA1 digest of the loaded GeoIP file. The
  263. * result does not need to be deallocated, but will be overwritten by the
  264. * next call of hex_str(). */
  265. const char *
  266. geoip_db_digest(void)
  267. {
  268. return hex_str(geoip_digest, DIGEST_LEN);
  269. }
  270. /** Entry in a map from IP address to the last time we've seen an incoming
  271. * connection from that IP address. Used by bridges only, to track which
  272. * countries have them blocked. */
  273. typedef struct clientmap_entry_t {
  274. HT_ENTRY(clientmap_entry_t) node;
  275. uint32_t ipaddr;
  276. /** Time when we last saw this IP address, in MINUTES since the epoch.
  277. *
  278. * (This will run out of space around 4011 CE. If Tor is still in use around
  279. * 4000 CE, please remember to add more bits to last_seen_in_minutes.) */
  280. unsigned int last_seen_in_minutes:30;
  281. unsigned int action:2;
  282. } clientmap_entry_t;
  283. /** Largest allowable value for last_seen_in_minutes. (It's a 30-bit field,
  284. * so it can hold up to (1u<<30)-1, or 0x3fffffffu.
  285. */
  286. #define MAX_LAST_SEEN_IN_MINUTES 0X3FFFFFFFu
  287. /** Map from client IP address to last time seen. */
  288. static HT_HEAD(clientmap, clientmap_entry_t) client_history =
  289. HT_INITIALIZER();
  290. /** Hashtable helper: compute a hash of a clientmap_entry_t. */
  291. static INLINE unsigned
  292. clientmap_entry_hash(const clientmap_entry_t *a)
  293. {
  294. return ht_improve_hash((unsigned) a->ipaddr);
  295. }
  296. /** Hashtable helper: compare two clientmap_entry_t values for equality. */
  297. static INLINE int
  298. clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b)
  299. {
  300. return a->ipaddr == b->ipaddr && a->action == b->action;
  301. }
  302. HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
  303. clientmap_entries_eq);
  304. HT_GENERATE(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
  305. clientmap_entries_eq, 0.6, malloc, realloc, free);
  306. /** Clear history of connecting clients used by entry and bridge stats. */
  307. static void
  308. client_history_clear(void)
  309. {
  310. clientmap_entry_t **ent, **next, *this;
  311. for (ent = HT_START(clientmap, &client_history); ent != NULL;
  312. ent = next) {
  313. if ((*ent)->action == GEOIP_CLIENT_CONNECT) {
  314. this = *ent;
  315. next = HT_NEXT_RMV(clientmap, &client_history, ent);
  316. tor_free(this);
  317. } else {
  318. next = HT_NEXT(clientmap, &client_history, ent);
  319. }
  320. }
  321. }
  322. /** How often do we update our estimate which share of v2 and v3 directory
  323. * requests is sent to us? We could as well trigger updates of shares from
  324. * network status updates, but that means adding a lot of calls into code
  325. * that is independent from geoip stats (and keeping them up-to-date). We
  326. * are perfectly fine with an approximation of 15-minute granularity. */
  327. #define REQUEST_SHARE_INTERVAL (15 * 60)
  328. /** When did we last determine which share of v2 and v3 directory requests
  329. * is sent to us? */
  330. static time_t last_time_determined_shares = 0;
  331. /** Sum of products of v2 shares times the number of seconds for which we
  332. * consider these shares as valid. */
  333. static double v2_share_times_seconds;
  334. /** Sum of products of v3 shares times the number of seconds for which we
  335. * consider these shares as valid. */
  336. static double v3_share_times_seconds;
  337. /** Number of seconds we are determining v2 and v3 shares. */
  338. static int share_seconds;
  339. /** Try to determine which fraction of v2 and v3 directory requests aimed at
  340. * caches will be sent to us at time <b>now</b> and store that value in
  341. * order to take a mean value later on. */
  342. static void
  343. geoip_determine_shares(time_t now)
  344. {
  345. double v2_share = 0.0, v3_share = 0.0;
  346. if (router_get_my_share_of_directory_requests(&v2_share, &v3_share) < 0)
  347. return;
  348. if (last_time_determined_shares) {
  349. v2_share_times_seconds += v2_share *
  350. ((double) (now - last_time_determined_shares));
  351. v3_share_times_seconds += v3_share *
  352. ((double) (now - last_time_determined_shares));
  353. share_seconds += (int)(now - last_time_determined_shares);
  354. }
  355. last_time_determined_shares = now;
  356. }
  357. /** Calculate which fraction of v2 and v3 directory requests aimed at caches
  358. * have been sent to us since the last call of this function up to time
  359. * <b>now</b>. Set *<b>v2_share_out</b> and *<b>v3_share_out</b> to the
  360. * fractions of v2 and v3 protocol shares we expect to have seen. Reset
  361. * counters afterwards. Return 0 on success, -1 on failure (e.g. when zero
  362. * seconds have passed since the last call).*/
  363. static int
  364. geoip_get_mean_shares(time_t now, double *v2_share_out,
  365. double *v3_share_out)
  366. {
  367. geoip_determine_shares(now);
  368. if (!share_seconds)
  369. return -1;
  370. *v2_share_out = v2_share_times_seconds / ((double) share_seconds);
  371. *v3_share_out = v3_share_times_seconds / ((double) share_seconds);
  372. v2_share_times_seconds = v3_share_times_seconds = 0.0;
  373. share_seconds = 0;
  374. return 0;
  375. }
  376. /** Note that we've seen a client connect from the IP <b>addr</b> (host order)
  377. * at time <b>now</b>. Ignored by all but bridges and directories if
  378. * configured accordingly. */
  379. void
  380. geoip_note_client_seen(geoip_client_action_t action,
  381. uint32_t addr, time_t now)
  382. {
  383. or_options_t *options = get_options();
  384. clientmap_entry_t lookup, *ent;
  385. if (action == GEOIP_CLIENT_CONNECT) {
  386. /* Only remember statistics as entry guard or as bridge. */
  387. if (!options->EntryStatistics &&
  388. (!(options->BridgeRelay && options->BridgeRecordUsageByCountry)))
  389. return;
  390. } else {
  391. if (options->BridgeRelay || options->BridgeAuthoritativeDir ||
  392. !options->DirReqStatistics)
  393. return;
  394. }
  395. lookup.ipaddr = addr;
  396. lookup.action = (int)action;
  397. ent = HT_FIND(clientmap, &client_history, &lookup);
  398. if (! ent) {
  399. ent = tor_malloc_zero(sizeof(clientmap_entry_t));
  400. ent->ipaddr = addr;
  401. ent->action = (int)action;
  402. HT_INSERT(clientmap, &client_history, ent);
  403. }
  404. if (now / 60 <= (int)MAX_LAST_SEEN_IN_MINUTES && now >= 0)
  405. ent->last_seen_in_minutes = (unsigned)(now/60);
  406. else
  407. ent->last_seen_in_minutes = 0;
  408. if (action == GEOIP_CLIENT_NETWORKSTATUS ||
  409. action == GEOIP_CLIENT_NETWORKSTATUS_V2) {
  410. int country_idx = geoip_get_country_by_ip(addr);
  411. if (country_idx < 0)
  412. country_idx = 0; /** unresolved requests are stored at index 0. */
  413. if (country_idx >= 0 && country_idx < smartlist_len(geoip_countries)) {
  414. geoip_country_t *country = smartlist_get(geoip_countries, country_idx);
  415. if (action == GEOIP_CLIENT_NETWORKSTATUS)
  416. ++country->n_v3_ns_requests;
  417. else
  418. ++country->n_v2_ns_requests;
  419. }
  420. /* Periodically determine share of requests that we should see */
  421. if (last_time_determined_shares + REQUEST_SHARE_INTERVAL < now)
  422. geoip_determine_shares(now);
  423. }
  424. }
  425. /** HT_FOREACH helper: remove a clientmap_entry_t from the hashtable if it's
  426. * older than a certain time. */
  427. static int
  428. _remove_old_client_helper(struct clientmap_entry_t *ent, void *_cutoff)
  429. {
  430. time_t cutoff = *(time_t*)_cutoff / 60;
  431. if (ent->last_seen_in_minutes < cutoff) {
  432. tor_free(ent);
  433. return 1;
  434. } else {
  435. return 0;
  436. }
  437. }
  438. /** Forget about all clients that haven't connected since <b>cutoff</b>. */
  439. void
  440. geoip_remove_old_clients(time_t cutoff)
  441. {
  442. clientmap_HT_FOREACH_FN(&client_history,
  443. _remove_old_client_helper,
  444. &cutoff);
  445. }
  446. /** How many responses are we giving to clients requesting v2 network
  447. * statuses? */
  448. static uint32_t ns_v2_responses[GEOIP_NS_RESPONSE_NUM];
  449. /** How many responses are we giving to clients requesting v3 network
  450. * statuses? */
  451. static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM];
  452. /** Note that we've rejected a client's request for a v2 or v3 network
  453. * status, encoded in <b>action</b> for reason <b>reason</b> at time
  454. * <b>now</b>. */
  455. void
  456. geoip_note_ns_response(geoip_client_action_t action,
  457. geoip_ns_response_t response)
  458. {
  459. static int arrays_initialized = 0;
  460. if (!get_options()->DirReqStatistics)
  461. return;
  462. if (!arrays_initialized) {
  463. memset(ns_v2_responses, 0, sizeof(ns_v2_responses));
  464. memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
  465. arrays_initialized = 1;
  466. }
  467. tor_assert(action == GEOIP_CLIENT_NETWORKSTATUS ||
  468. action == GEOIP_CLIENT_NETWORKSTATUS_V2);
  469. tor_assert(response < GEOIP_NS_RESPONSE_NUM);
  470. if (action == GEOIP_CLIENT_NETWORKSTATUS)
  471. ns_v3_responses[response]++;
  472. else
  473. ns_v2_responses[response]++;
  474. }
  475. /** Do not mention any country from which fewer than this number of IPs have
  476. * connected. This conceivably avoids reporting information that could
  477. * deanonymize users, though analysis is lacking. */
  478. #define MIN_IPS_TO_NOTE_COUNTRY 1
  479. /** Do not report any geoip data at all if we have fewer than this number of
  480. * IPs to report about. */
  481. #define MIN_IPS_TO_NOTE_ANYTHING 1
  482. /** When reporting geoip data about countries, round up to the nearest
  483. * multiple of this value. */
  484. #define IP_GRANULARITY 8
  485. /** Helper type: used to sort per-country totals by value. */
  486. typedef struct c_hist_t {
  487. char country[3]; /**< Two-letter country code. */
  488. unsigned total; /**< Total IP addresses seen in this country. */
  489. } c_hist_t;
  490. /** Sorting helper: return -1, 1, or 0 based on comparison of two
  491. * geoip_entry_t. Sort in descending order of total, and then by country
  492. * code. */
  493. static int
  494. _c_hist_compare(const void **_a, const void **_b)
  495. {
  496. const c_hist_t *a = *_a, *b = *_b;
  497. if (a->total > b->total)
  498. return -1;
  499. else if (a->total < b->total)
  500. return 1;
  501. else
  502. return strcmp(a->country, b->country);
  503. }
  504. /** When there are incomplete directory requests at the end of a 24-hour
  505. * period, consider those requests running for longer than this timeout as
  506. * failed, the others as still running. */
  507. #define DIRREQ_TIMEOUT (10*60)
  508. /** Entry in a map from either conn->global_identifier for direct requests
  509. * or a unique circuit identifier for tunneled requests to request time,
  510. * response size, and completion time of a network status request. Used to
  511. * measure download times of requests to derive average client
  512. * bandwidths. */
  513. typedef struct dirreq_map_entry_t {
  514. HT_ENTRY(dirreq_map_entry_t) node;
  515. /** Unique identifier for this network status request; this is either the
  516. * conn->global_identifier of the dir conn (direct request) or a new
  517. * locally unique identifier of a circuit (tunneled request). This ID is
  518. * only unique among other direct or tunneled requests, respectively. */
  519. uint64_t dirreq_id;
  520. unsigned int state:3; /**< State of this directory request. */
  521. unsigned int type:1; /**< Is this a direct or a tunneled request? */
  522. unsigned int completed:1; /**< Is this request complete? */
  523. unsigned int action:2; /**< Is this a v2 or v3 request? */
  524. /** When did we receive the request and started sending the response? */
  525. struct timeval request_time;
  526. size_t response_size; /**< What is the size of the response in bytes? */
  527. struct timeval completion_time; /**< When did the request succeed? */
  528. } dirreq_map_entry_t;
  529. /** Map of all directory requests asking for v2 or v3 network statuses in
  530. * the current geoip-stats interval. Values are
  531. * of type *<b>dirreq_map_entry_t</b>. */
  532. static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map =
  533. HT_INITIALIZER();
  534. static int
  535. dirreq_map_ent_eq(const dirreq_map_entry_t *a,
  536. const dirreq_map_entry_t *b)
  537. {
  538. return a->dirreq_id == b->dirreq_id && a->type == b->type;
  539. }
  540. static unsigned
  541. dirreq_map_ent_hash(const dirreq_map_entry_t *entry)
  542. {
  543. unsigned u = (unsigned) entry->dirreq_id;
  544. u += entry->type << 20;
  545. return u;
  546. }
  547. HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
  548. dirreq_map_ent_eq);
  549. HT_GENERATE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
  550. dirreq_map_ent_eq, 0.6, malloc, realloc, free);
  551. /** Helper: Put <b>entry</b> into map of directory requests using
  552. * <b>type</b> and <b>dirreq_id</b> as key parts. If there is
  553. * already an entry for that key, print out a BUG warning and return. */
  554. static void
  555. _dirreq_map_put(dirreq_map_entry_t *entry, dirreq_type_t type,
  556. uint64_t dirreq_id)
  557. {
  558. dirreq_map_entry_t *old_ent;
  559. tor_assert(entry->type == type);
  560. tor_assert(entry->dirreq_id == dirreq_id);
  561. /* XXXX022 once we're sure the bug case never happens, we can switch
  562. * to HT_INSERT */
  563. old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry);
  564. if (old_ent && old_ent != entry) {
  565. log_warn(LD_BUG, "Error when putting directory request into local "
  566. "map. There was already an entry for the same identifier.");
  567. return;
  568. }
  569. }
  570. /** Helper: Look up and return an entry in the map of directory requests
  571. * using <b>type</b> and <b>dirreq_id</b> as key parts. If there
  572. * is no such entry, return NULL. */
  573. static dirreq_map_entry_t *
  574. _dirreq_map_get(dirreq_type_t type, uint64_t dirreq_id)
  575. {
  576. dirreq_map_entry_t lookup;
  577. lookup.type = type;
  578. lookup.dirreq_id = dirreq_id;
  579. return HT_FIND(dirreqmap, &dirreq_map, &lookup);
  580. }
  581. /** Note that an either direct or tunneled (see <b>type</b>) directory
  582. * request for a network status with unique ID <b>dirreq_id</b> of size
  583. * <b>response_size</b> and action <b>action</b> (either v2 or v3) has
  584. * started. */
  585. void
  586. geoip_start_dirreq(uint64_t dirreq_id, size_t response_size,
  587. geoip_client_action_t action, dirreq_type_t type)
  588. {
  589. dirreq_map_entry_t *ent;
  590. if (!get_options()->DirReqStatistics)
  591. return;
  592. ent = tor_malloc_zero(sizeof(dirreq_map_entry_t));
  593. ent->dirreq_id = dirreq_id;
  594. tor_gettimeofday(&ent->request_time);
  595. ent->response_size = response_size;
  596. ent->action = action;
  597. ent->type = type;
  598. _dirreq_map_put(ent, type, dirreq_id);
  599. }
  600. /** Change the state of the either direct or tunneled (see <b>type</b>)
  601. * directory request with <b>dirreq_id</b> to <b>new_state</b> and
  602. * possibly mark it as completed. If no entry can be found for the given
  603. * key parts (e.g., if this is a directory request that we are not
  604. * measuring, or one that was started in the previous measurement period),
  605. * or if the state cannot be advanced to <b>new_state</b>, do nothing. */
  606. void
  607. geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
  608. dirreq_state_t new_state)
  609. {
  610. dirreq_map_entry_t *ent;
  611. if (!get_options()->DirReqStatistics)
  612. return;
  613. ent = _dirreq_map_get(type, dirreq_id);
  614. if (!ent)
  615. return;
  616. if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS)
  617. return;
  618. if (new_state - 1 != ent->state)
  619. return;
  620. ent->state = new_state;
  621. if ((type == DIRREQ_DIRECT &&
  622. new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) ||
  623. (type == DIRREQ_TUNNELED &&
  624. new_state == DIRREQ_OR_CONN_BUFFER_FLUSHED)) {
  625. tor_gettimeofday(&ent->completion_time);
  626. ent->completed = 1;
  627. }
  628. }
  629. /** Return a newly allocated comma-separated string containing statistics
  630. * on network status downloads. The string contains the number of completed
  631. * requests, timeouts, and still running requests as well as the download
  632. * times by deciles and quartiles. Return NULL if we have not observed
  633. * requests for long enough. */
  634. static char *
  635. geoip_get_dirreq_history(geoip_client_action_t action,
  636. dirreq_type_t type)
  637. {
  638. char *result = NULL;
  639. smartlist_t *dirreq_completed = NULL;
  640. uint32_t complete = 0, timeouts = 0, running = 0;
  641. int bufsize = 1024, written;
  642. dirreq_map_entry_t **ptr, **next, *ent;
  643. struct timeval now;
  644. tor_gettimeofday(&now);
  645. if (action != GEOIP_CLIENT_NETWORKSTATUS &&
  646. action != GEOIP_CLIENT_NETWORKSTATUS_V2)
  647. return NULL;
  648. dirreq_completed = smartlist_create();
  649. for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) {
  650. ent = *ptr;
  651. if (ent->action != action || ent->type != type) {
  652. next = HT_NEXT(dirreqmap, &dirreq_map, ptr);
  653. continue;
  654. } else {
  655. if (ent->completed) {
  656. smartlist_add(dirreq_completed, ent);
  657. complete++;
  658. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
  659. } else {
  660. if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT)
  661. timeouts++;
  662. else
  663. running++;
  664. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
  665. tor_free(ent);
  666. }
  667. }
  668. }
  669. #define DIR_REQ_GRANULARITY 4
  670. complete = round_uint32_to_next_multiple_of(complete,
  671. DIR_REQ_GRANULARITY);
  672. timeouts = round_uint32_to_next_multiple_of(timeouts,
  673. DIR_REQ_GRANULARITY);
  674. running = round_uint32_to_next_multiple_of(running,
  675. DIR_REQ_GRANULARITY);
  676. result = tor_malloc_zero(bufsize);
  677. written = tor_snprintf(result, bufsize, "complete=%u,timeout=%u,"
  678. "running=%u", complete, timeouts, running);
  679. if (written < 0) {
  680. tor_free(result);
  681. goto done;
  682. }
  683. #define MIN_DIR_REQ_RESPONSES 16
  684. if (complete >= MIN_DIR_REQ_RESPONSES) {
  685. uint32_t *dltimes;
  686. /* We may have rounded 'completed' up. Here we want to use the
  687. * real value. */
  688. complete = smartlist_len(dirreq_completed);
  689. dltimes = tor_malloc_zero(sizeof(uint32_t) * complete);
  690. SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) {
  691. uint32_t bytes_per_second;
  692. uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time,
  693. &ent->completion_time);
  694. if (time_diff == 0)
  695. time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible
  696. * by law of nature or something, but a milisecond
  697. * is a bit greater than "instantly" */
  698. bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff);
  699. dltimes[ent_sl_idx] = bytes_per_second;
  700. } SMARTLIST_FOREACH_END(ent);
  701. median_uint32(dltimes, complete); /* sorts as a side effect. */
  702. written = tor_snprintf(result + written, bufsize - written,
  703. ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u,"
  704. "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u",
  705. dltimes[0],
  706. dltimes[1*complete/10-1],
  707. dltimes[2*complete/10-1],
  708. dltimes[1*complete/4-1],
  709. dltimes[3*complete/10-1],
  710. dltimes[4*complete/10-1],
  711. dltimes[5*complete/10-1],
  712. dltimes[6*complete/10-1],
  713. dltimes[7*complete/10-1],
  714. dltimes[3*complete/4-1],
  715. dltimes[8*complete/10-1],
  716. dltimes[9*complete/10-1],
  717. dltimes[complete-1]);
  718. if (written<0)
  719. tor_free(result);
  720. tor_free(dltimes);
  721. }
  722. done:
  723. SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent,
  724. tor_free(ent));
  725. smartlist_free(dirreq_completed);
  726. return result;
  727. }
  728. /** Return a newly allocated comma-separated string containing entries for
  729. * all the countries from which we've seen enough clients connect as a
  730. * bridge, directory server, or entry guard. The entry format is cc=num
  731. * where num is the number of IPs we've seen connecting from that country,
  732. * and cc is a lowercased country code. Returns NULL if we don't want
  733. * to export geoip data yet. */
  734. char *
  735. geoip_get_client_history(geoip_client_action_t action)
  736. {
  737. char *result = NULL;
  738. unsigned granularity = IP_GRANULARITY;
  739. smartlist_t *chunks = NULL;
  740. smartlist_t *entries = NULL;
  741. int n_countries = geoip_get_n_countries();
  742. int i;
  743. clientmap_entry_t **ent;
  744. unsigned *counts = NULL;
  745. unsigned total = 0;
  746. if (!geoip_is_loaded())
  747. return NULL;
  748. counts = tor_malloc_zero(sizeof(unsigned)*n_countries);
  749. HT_FOREACH(ent, clientmap, &client_history) {
  750. int country;
  751. if ((*ent)->action != (int)action)
  752. continue;
  753. country = geoip_get_country_by_ip((*ent)->ipaddr);
  754. if (country < 0)
  755. country = 0; /** unresolved requests are stored at index 0. */
  756. tor_assert(0 <= country && country < n_countries);
  757. ++counts[country];
  758. ++total;
  759. }
  760. /* Don't record anything if we haven't seen enough IPs. */
  761. if (total < MIN_IPS_TO_NOTE_ANYTHING)
  762. goto done;
  763. /* Make a list of c_hist_t */
  764. entries = smartlist_create();
  765. for (i = 0; i < n_countries; ++i) {
  766. unsigned c = counts[i];
  767. const char *countrycode;
  768. c_hist_t *ent;
  769. /* Only report a country if it has a minimum number of IPs. */
  770. if (c >= MIN_IPS_TO_NOTE_COUNTRY) {
  771. c = round_to_next_multiple_of(c, granularity);
  772. countrycode = geoip_get_country_name(i);
  773. ent = tor_malloc(sizeof(c_hist_t));
  774. strlcpy(ent->country, countrycode, sizeof(ent->country));
  775. ent->total = c;
  776. smartlist_add(entries, ent);
  777. }
  778. }
  779. /* Sort entries. Note that we must do this _AFTER_ rounding, or else
  780. * the sort order could leak info. */
  781. smartlist_sort(entries, _c_hist_compare);
  782. /* Build the result. */
  783. chunks = smartlist_create();
  784. SMARTLIST_FOREACH(entries, c_hist_t *, ch, {
  785. char *buf=NULL;
  786. tor_asprintf(&buf, "%s=%u", ch->country, ch->total);
  787. smartlist_add(chunks, buf);
  788. });
  789. result = smartlist_join_strings(chunks, ",", 0, NULL);
  790. done:
  791. tor_free(counts);
  792. if (chunks) {
  793. SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
  794. smartlist_free(chunks);
  795. }
  796. if (entries) {
  797. SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c));
  798. smartlist_free(entries);
  799. }
  800. return result;
  801. }
  802. /** Return a newly allocated string holding the per-country request history
  803. * for <b>action</b> in a format suitable for an extra-info document, or NULL
  804. * on failure. */
  805. char *
  806. geoip_get_request_history(geoip_client_action_t action)
  807. {
  808. smartlist_t *entries, *strings;
  809. char *result;
  810. unsigned granularity = IP_GRANULARITY;
  811. if (action != GEOIP_CLIENT_NETWORKSTATUS &&
  812. action != GEOIP_CLIENT_NETWORKSTATUS_V2)
  813. return NULL;
  814. if (!geoip_countries)
  815. return NULL;
  816. entries = smartlist_create();
  817. SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, {
  818. uint32_t tot = 0;
  819. c_hist_t *ent;
  820. tot = (action == GEOIP_CLIENT_NETWORKSTATUS) ?
  821. c->n_v3_ns_requests : c->n_v2_ns_requests;
  822. if (!tot)
  823. continue;
  824. ent = tor_malloc_zero(sizeof(c_hist_t));
  825. strlcpy(ent->country, c->countrycode, sizeof(ent->country));
  826. ent->total = round_to_next_multiple_of(tot, granularity);
  827. smartlist_add(entries, ent);
  828. });
  829. smartlist_sort(entries, _c_hist_compare);
  830. strings = smartlist_create();
  831. SMARTLIST_FOREACH(entries, c_hist_t *, ent, {
  832. char *buf = NULL;
  833. tor_asprintf(&buf, "%s=%u", ent->country, ent->total);
  834. smartlist_add(strings, buf);
  835. });
  836. result = smartlist_join_strings(strings, ",", 0, NULL);
  837. SMARTLIST_FOREACH(strings, char *, cp, tor_free(cp));
  838. SMARTLIST_FOREACH(entries, c_hist_t *, ent, tor_free(ent));
  839. smartlist_free(strings);
  840. smartlist_free(entries);
  841. return result;
  842. }
  843. /** Start time of directory request stats or 0 if we're not collecting
  844. * directory request statistics. */
  845. static time_t start_of_dirreq_stats_interval;
  846. /** Initialize directory request stats. */
  847. void
  848. geoip_dirreq_stats_init(time_t now)
  849. {
  850. start_of_dirreq_stats_interval = now;
  851. }
  852. /** Stop collecting directory request stats in a way that we can re-start
  853. * doing so in geoip_dirreq_stats_init(). */
  854. void
  855. geoip_dirreq_stats_term(void)
  856. {
  857. SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, {
  858. c->n_v2_ns_requests = c->n_v3_ns_requests = 0;
  859. });
  860. {
  861. clientmap_entry_t **ent, **next, *this;
  862. for (ent = HT_START(clientmap, &client_history); ent != NULL;
  863. ent = next) {
  864. if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS ||
  865. (*ent)->action == GEOIP_CLIENT_NETWORKSTATUS_V2) {
  866. this = *ent;
  867. next = HT_NEXT_RMV(clientmap, &client_history, ent);
  868. tor_free(this);
  869. } else {
  870. next = HT_NEXT(clientmap, &client_history, ent);
  871. }
  872. }
  873. }
  874. v2_share_times_seconds = v3_share_times_seconds = 0.0;
  875. last_time_determined_shares = 0;
  876. share_seconds = 0;
  877. memset(ns_v2_responses, 0, sizeof(ns_v2_responses));
  878. memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
  879. {
  880. dirreq_map_entry_t **ent, **next, *this;
  881. for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
  882. this = *ent;
  883. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
  884. tor_free(this);
  885. }
  886. }
  887. start_of_dirreq_stats_interval = 0;
  888. }
  889. /** Write dirreq statistics to $DATADIR/stats/dirreq-stats and return when
  890. * we would next want to write. */
  891. time_t
  892. geoip_dirreq_stats_write(time_t now)
  893. {
  894. char *statsdir = NULL, *filename = NULL;
  895. char *data_v2 = NULL, *data_v3 = NULL;
  896. char written[ISO_TIME_LEN+1];
  897. open_file_t *open_file = NULL;
  898. double v2_share = 0.0, v3_share = 0.0;
  899. FILE *out;
  900. int i;
  901. if (!start_of_dirreq_stats_interval)
  902. return 0; /* Not initialized. */
  903. if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now)
  904. goto done; /* Not ready to write. */
  905. /* Discard all items in the client history that are too old. */
  906. geoip_remove_old_clients(start_of_dirreq_stats_interval);
  907. statsdir = get_datadir_fname("stats");
  908. if (check_private_dir(statsdir, CPD_CREATE) < 0)
  909. goto done;
  910. filename = get_datadir_fname2("stats", "dirreq-stats");
  911. data_v2 = geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS_V2);
  912. data_v3 = geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS);
  913. format_iso_time(written, now);
  914. out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND,
  915. 0600, &open_file);
  916. if (!out)
  917. goto done;
  918. if (fprintf(out, "dirreq-stats-end %s (%d s)\ndirreq-v3-ips %s\n"
  919. "dirreq-v2-ips %s\n", written,
  920. (unsigned) (now - start_of_dirreq_stats_interval),
  921. data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0)
  922. goto done;
  923. tor_free(data_v2);
  924. tor_free(data_v3);
  925. data_v2 = geoip_get_request_history(GEOIP_CLIENT_NETWORKSTATUS_V2);
  926. data_v3 = geoip_get_request_history(GEOIP_CLIENT_NETWORKSTATUS);
  927. if (fprintf(out, "dirreq-v3-reqs %s\ndirreq-v2-reqs %s\n",
  928. data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0)
  929. goto done;
  930. tor_free(data_v2);
  931. tor_free(data_v3);
  932. SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, {
  933. c->n_v2_ns_requests = c->n_v3_ns_requests = 0;
  934. });
  935. #define RESPONSE_GRANULARITY 8
  936. for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) {
  937. ns_v2_responses[i] = round_uint32_to_next_multiple_of(
  938. ns_v2_responses[i], RESPONSE_GRANULARITY);
  939. ns_v3_responses[i] = round_uint32_to_next_multiple_of(
  940. ns_v3_responses[i], RESPONSE_GRANULARITY);
  941. }
  942. #undef RESPONSE_GRANULARITY
  943. if (fprintf(out, "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u,"
  944. "not-found=%u,not-modified=%u,busy=%u\n",
  945. ns_v3_responses[GEOIP_SUCCESS],
  946. ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS],
  947. ns_v3_responses[GEOIP_REJECT_UNAVAILABLE],
  948. ns_v3_responses[GEOIP_REJECT_NOT_FOUND],
  949. ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED],
  950. ns_v3_responses[GEOIP_REJECT_BUSY]) < 0)
  951. goto done;
  952. if (fprintf(out, "dirreq-v2-resp ok=%u,unavailable=%u,"
  953. "not-found=%u,not-modified=%u,busy=%u\n",
  954. ns_v2_responses[GEOIP_SUCCESS],
  955. ns_v2_responses[GEOIP_REJECT_UNAVAILABLE],
  956. ns_v2_responses[GEOIP_REJECT_NOT_FOUND],
  957. ns_v2_responses[GEOIP_REJECT_NOT_MODIFIED],
  958. ns_v2_responses[GEOIP_REJECT_BUSY]) < 0)
  959. goto done;
  960. memset(ns_v2_responses, 0, sizeof(ns_v2_responses));
  961. memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
  962. if (!geoip_get_mean_shares(now, &v2_share, &v3_share)) {
  963. if (fprintf(out, "dirreq-v2-share %0.2lf%%\n", v2_share*100) < 0)
  964. goto done;
  965. if (fprintf(out, "dirreq-v3-share %0.2lf%%\n", v3_share*100) < 0)
  966. goto done;
  967. }
  968. data_v2 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS_V2,
  969. DIRREQ_DIRECT);
  970. data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS,
  971. DIRREQ_DIRECT);
  972. if (fprintf(out, "dirreq-v3-direct-dl %s\ndirreq-v2-direct-dl %s\n",
  973. data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0)
  974. goto done;
  975. tor_free(data_v2);
  976. tor_free(data_v3);
  977. data_v2 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS_V2,
  978. DIRREQ_TUNNELED);
  979. data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS,
  980. DIRREQ_TUNNELED);
  981. if (fprintf(out, "dirreq-v3-tunneled-dl %s\ndirreq-v2-tunneled-dl %s\n",
  982. data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0)
  983. goto done;
  984. finish_writing_to_file(open_file);
  985. open_file = NULL;
  986. start_of_dirreq_stats_interval = now;
  987. done:
  988. if (open_file)
  989. abort_writing_to_file(open_file);
  990. tor_free(filename);
  991. tor_free(statsdir);
  992. tor_free(data_v2);
  993. tor_free(data_v3);
  994. return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL;
  995. }
  996. /** Start time of bridge stats or 0 if we're not collecting bridge
  997. * statistics. */
  998. static time_t start_of_bridge_stats_interval;
  999. /** Initialize bridge stats. */
  1000. void
  1001. geoip_bridge_stats_init(time_t now)
  1002. {
  1003. start_of_bridge_stats_interval = now;
  1004. }
  1005. /** Stop collecting bridge stats in a way that we can re-start doing so in
  1006. * geoip_bridge_stats_init(). */
  1007. void
  1008. geoip_bridge_stats_term(void)
  1009. {
  1010. client_history_clear();
  1011. start_of_bridge_stats_interval = 0;
  1012. }
  1013. /** Parse the bridge statistics as they are written to extra-info
  1014. * descriptors for being returned to controller clients. Return the
  1015. * controller string if successful, or NULL otherwise. */
  1016. static char *
  1017. parse_bridge_stats_controller(const char *stats_str, time_t now)
  1018. {
  1019. char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1],
  1020. *controller_str, *eos, *eol, *summary;
  1021. const char *BRIDGE_STATS_END = "bridge-stats-end ";
  1022. const char *BRIDGE_IPS = "bridge-ips ";
  1023. const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n";
  1024. const char *tmp;
  1025. time_t stats_end_time;
  1026. int seconds;
  1027. tor_assert(stats_str);
  1028. /* Parse timestamp and number of seconds from
  1029. "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */
  1030. tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END);
  1031. if (!tmp)
  1032. return NULL;
  1033. tmp += strlen(BRIDGE_STATS_END);
  1034. if (strlen(tmp) < ISO_TIME_LEN + 6)
  1035. return NULL;
  1036. strlcpy(stats_end_str, tmp, sizeof(stats_end_str));
  1037. if (parse_iso_time(stats_end_str, &stats_end_time) < 0)
  1038. return NULL;
  1039. if (stats_end_time < now - (25*60*60) ||
  1040. stats_end_time > now + (1*60*60))
  1041. return NULL;
  1042. seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10);
  1043. if (!eos || seconds < 23*60*60)
  1044. return NULL;
  1045. format_iso_time(stats_start_str, stats_end_time - seconds);
  1046. /* Parse: "bridge-ips CC=N,CC=N,..." */
  1047. tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS);
  1048. if (tmp) {
  1049. tmp += strlen(BRIDGE_IPS);
  1050. tmp = eat_whitespace_no_nl(tmp);
  1051. eol = strchr(tmp, '\n');
  1052. if (eol)
  1053. summary = tor_strndup(tmp, eol-tmp);
  1054. else
  1055. summary = tor_strdup(tmp);
  1056. } else {
  1057. /* Look if there is an empty "bridge-ips" line */
  1058. tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE);
  1059. if (!tmp)
  1060. return NULL;
  1061. summary = tor_strdup("");
  1062. }
  1063. tor_asprintf(&controller_str,
  1064. "TimeStarted=\"%s\" CountrySummary=%s",
  1065. stats_start_str, summary);
  1066. tor_free(summary);
  1067. return controller_str;
  1068. }
  1069. /** Most recent bridge statistics formatted to be written to extra-info
  1070. * descriptors. */
  1071. static char *bridge_stats_extrainfo = NULL;
  1072. /** Most recent bridge statistics formatted to be returned to controller
  1073. * clients. */
  1074. static char *bridge_stats_controller = NULL;
  1075. /** Write bridge statistics to $DATADIR/stats/bridge-stats and return
  1076. * when we should next try to write statistics. */
  1077. time_t
  1078. geoip_bridge_stats_write(time_t now)
  1079. {
  1080. char *statsdir = NULL, *filename = NULL, *data = NULL,
  1081. written[ISO_TIME_LEN+1], *out = NULL, *controller_str;
  1082. size_t len;
  1083. /* Check if 24 hours have passed since starting measurements. */
  1084. if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL)
  1085. return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
  1086. /* Discard all items in the client history that are too old. */
  1087. geoip_remove_old_clients(start_of_bridge_stats_interval);
  1088. statsdir = get_datadir_fname("stats");
  1089. if (check_private_dir(statsdir, CPD_CREATE) < 0)
  1090. goto done;
  1091. filename = get_datadir_fname2("stats", "bridge-stats");
  1092. data = geoip_get_client_history(GEOIP_CLIENT_CONNECT);
  1093. format_iso_time(written, now);
  1094. len = strlen("bridge-stats-end (999999 s)\nbridge-ips \n") +
  1095. ISO_TIME_LEN + (data ? strlen(data) : 0) + 42;
  1096. out = tor_malloc(len);
  1097. if (tor_snprintf(out, len, "bridge-stats-end %s (%u s)\nbridge-ips %s\n",
  1098. written, (unsigned) (now - start_of_bridge_stats_interval),
  1099. data ? data : "") < 0)
  1100. goto done;
  1101. write_str_to_file(filename, out, 0);
  1102. controller_str = parse_bridge_stats_controller(out, now);
  1103. if (!controller_str)
  1104. goto done;
  1105. start_of_bridge_stats_interval = now;
  1106. tor_free(bridge_stats_extrainfo);
  1107. tor_free(bridge_stats_controller);
  1108. bridge_stats_extrainfo = out;
  1109. out = NULL;
  1110. bridge_stats_controller = controller_str;
  1111. control_event_clients_seen(controller_str);
  1112. done:
  1113. tor_free(filename);
  1114. tor_free(statsdir);
  1115. tor_free(data);
  1116. tor_free(out);
  1117. return start_of_bridge_stats_interval +
  1118. WRITE_STATS_INTERVAL;
  1119. }
  1120. /** Try to load the most recent bridge statistics from disk, unless we
  1121. * have finished a measurement interval lately. */
  1122. static void
  1123. load_bridge_stats(time_t now)
  1124. {
  1125. char *statsdir, *fname=NULL, *contents, *controller_str;
  1126. if (bridge_stats_extrainfo)
  1127. return;
  1128. statsdir = get_datadir_fname("stats");
  1129. if (check_private_dir(statsdir, CPD_CREATE) < 0)
  1130. goto done;
  1131. fname = get_datadir_fname2("stats", "bridge-stats");
  1132. contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
  1133. if (contents) {
  1134. controller_str = parse_bridge_stats_controller(contents, now);
  1135. if (controller_str) {
  1136. bridge_stats_extrainfo = contents;
  1137. bridge_stats_controller = controller_str;
  1138. } else {
  1139. tor_free(contents);
  1140. }
  1141. }
  1142. done:
  1143. tor_free(fname);
  1144. tor_free(statsdir);
  1145. }
  1146. /** Return most recent bridge statistics for inclusion in extra-info
  1147. * descriptors, or NULL if we don't have recent bridge statistics. */
  1148. const char *
  1149. geoip_get_bridge_stats_extrainfo(time_t now)
  1150. {
  1151. load_bridge_stats(now);
  1152. return bridge_stats_extrainfo;
  1153. }
  1154. /** Return most recent bridge statistics to be returned to controller
  1155. * clients, or NULL if we don't have recent bridge statistics. */
  1156. const char *
  1157. geoip_get_bridge_stats_controller(time_t now)
  1158. {
  1159. load_bridge_stats(now);
  1160. return bridge_stats_controller;
  1161. }
  1162. /** Start time of entry stats or 0 if we're not collecting entry
  1163. * statistics. */
  1164. static time_t start_of_entry_stats_interval;
  1165. /** Initialize entry stats. */
  1166. void
  1167. geoip_entry_stats_init(time_t now)
  1168. {
  1169. start_of_entry_stats_interval = now;
  1170. }
  1171. /** Stop collecting entry stats in a way that we can re-start doing so in
  1172. * geoip_entry_stats_init(). */
  1173. void
  1174. geoip_entry_stats_term(void)
  1175. {
  1176. client_history_clear();
  1177. start_of_entry_stats_interval = 0;
  1178. }
  1179. /** Write entry statistics to $DATADIR/stats/entry-stats and return time
  1180. * when we would next want to write. */
  1181. time_t
  1182. geoip_entry_stats_write(time_t now)
  1183. {
  1184. char *statsdir = NULL, *filename = NULL;
  1185. char *data = NULL;
  1186. char written[ISO_TIME_LEN+1];
  1187. open_file_t *open_file = NULL;
  1188. FILE *out;
  1189. if (!start_of_entry_stats_interval)
  1190. return 0; /* Not initialized. */
  1191. if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now)
  1192. goto done; /* Not ready to write. */
  1193. /* Discard all items in the client history that are too old. */
  1194. geoip_remove_old_clients(start_of_entry_stats_interval);
  1195. statsdir = get_datadir_fname("stats");
  1196. if (check_private_dir(statsdir, CPD_CREATE) < 0)
  1197. goto done;
  1198. filename = get_datadir_fname2("stats", "entry-stats");
  1199. data = geoip_get_client_history(GEOIP_CLIENT_CONNECT);
  1200. format_iso_time(written, now);
  1201. out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND,
  1202. 0600, &open_file);
  1203. if (!out)
  1204. goto done;
  1205. if (fprintf(out, "entry-stats-end %s (%u s)\nentry-ips %s\n",
  1206. written, (unsigned) (now - start_of_entry_stats_interval),
  1207. data ? data : "") < 0)
  1208. goto done;
  1209. start_of_entry_stats_interval = now;
  1210. finish_writing_to_file(open_file);
  1211. open_file = NULL;
  1212. done:
  1213. if (open_file)
  1214. abort_writing_to_file(open_file);
  1215. tor_free(filename);
  1216. tor_free(statsdir);
  1217. tor_free(data);
  1218. return start_of_entry_stats_interval + WRITE_STATS_INTERVAL;
  1219. }
  1220. /** Helper used to implement GETINFO ip-to-country/... controller command. */
  1221. int
  1222. getinfo_helper_geoip(control_connection_t *control_conn,
  1223. const char *question, char **answer,
  1224. const char **errmsg)
  1225. {
  1226. (void)control_conn;
  1227. if (!geoip_is_loaded()) {
  1228. *errmsg = "GeoIP data not loaded";
  1229. return -1;
  1230. }
  1231. if (!strcmpstart(question, "ip-to-country/")) {
  1232. int c;
  1233. uint32_t ip;
  1234. struct in_addr in;
  1235. question += strlen("ip-to-country/");
  1236. if (tor_inet_aton(question, &in) != 0) {
  1237. ip = ntohl(in.s_addr);
  1238. c = geoip_get_country_by_ip(ip);
  1239. *answer = tor_strdup(geoip_get_country_name(c));
  1240. }
  1241. }
  1242. return 0;
  1243. }
  1244. /** Release all storage held by the GeoIP database. */
  1245. static void
  1246. clear_geoip_db(void)
  1247. {
  1248. if (geoip_countries) {
  1249. SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, tor_free(c));
  1250. smartlist_free(geoip_countries);
  1251. }
  1252. strmap_free(country_idxplus1_by_lc_code, NULL);
  1253. if (geoip_entries) {
  1254. SMARTLIST_FOREACH(geoip_entries, geoip_entry_t *, ent, tor_free(ent));
  1255. smartlist_free(geoip_entries);
  1256. }
  1257. geoip_countries = NULL;
  1258. country_idxplus1_by_lc_code = NULL;
  1259. geoip_entries = NULL;
  1260. }
  1261. /** Release all storage held in this file. */
  1262. void
  1263. geoip_free_all(void)
  1264. {
  1265. {
  1266. clientmap_entry_t **ent, **next, *this;
  1267. for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) {
  1268. this = *ent;
  1269. next = HT_NEXT_RMV(clientmap, &client_history, ent);
  1270. tor_free(this);
  1271. }
  1272. HT_CLEAR(clientmap, &client_history);
  1273. }
  1274. {
  1275. dirreq_map_entry_t **ent, **next, *this;
  1276. for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
  1277. this = *ent;
  1278. next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
  1279. tor_free(this);
  1280. }
  1281. HT_CLEAR(dirreqmap, &dirreq_map);
  1282. }
  1283. clear_geoip_db();
  1284. }