lib.rs 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694
  1. use http::status::StatusCode;
  2. use http_body_util::{BodyExt, Empty};
  3. use hyper::{body::Bytes, Body, Client, Method, Request};
  4. use hyper_util::rt::TokioExecutor;
  5. use lazy_static::lazy_static;
  6. //use select::{document::Document, predicate::Name};
  7. use serde::{Deserialize, Serialize};
  8. use sled::Db;
  9. use std::{
  10. collections::{BTreeMap, HashMap, HashSet},
  11. fmt,
  12. };
  13. use x25519_dalek::{PublicKey, StaticSecret};
  14. pub mod analysis;
  15. pub mod bridge_verification_info;
  16. pub mod crypto;
  17. pub mod extra_info;
  18. pub mod negative_report;
  19. pub mod positive_report;
  20. pub mod request_handler;
  21. use analysis::Analyzer;
  22. use extra_info::*;
  23. use negative_report::*;
  24. use positive_report::*;
  25. lazy_static! {
  26. // known country codes based on Tor geoIP database
  27. // Produced with `cat /usr/share/tor/geoip{,6} | grep -v ^# | grep -o ..$ | sort | uniq | tr '[:upper:]' '[:lower:]' | tr '\n' ',' | sed 's/,/","/g'`
  28. pub static ref COUNTRY_CODES: HashSet<&'static str> = HashSet::from(["??","ad","ae","af","ag","ai","al","am","ao","ap","aq","ar","as","at","au","aw","ax","az","ba","bb","bd","be","bf","bg","bh","bi","bj","bl","bm","bn","bo","bq","br","bs","bt","bv","bw","by","bz","ca","cc","cd","cf","cg","ch","ci","ck","cl","cm","cn","co","cr","cs","cu","cv","cw","cx","cy","cz","de","dj","dk","dm","do","dz","ec","ee","eg","eh","er","es","et","eu","fi","fj","fk","fm","fo","fr","ga","gb","gd","ge","gf","gg","gh","gi","gl","gm","gn","gp","gq","gr","gs","gt","gu","gw","gy","hk","hm","hn","hr","ht","hu","id","ie","il","im","in","io","iq","ir","is","it","je","jm","jo","jp","ke","kg","kh","ki","km","kn","kp","kr","kw","ky","kz","la","lb","lc","li","lk","lr","ls","lt","lu","lv","ly","ma","mc","md","me","mf","mg","mh","mk","ml","mm","mn","mo","mp","mq","mr","ms","mt","mu","mv","mw","mx","my","mz","na","nc","ne","nf","ng","ni","nl","no","np","nr","nu","nz","om","pa","pe","pf","pg","ph","pk","pl","pm","pn","pr","ps","pt","pw","py","qa","re","ro","rs","ru","rw","sa","sb","sc","sd","se","sg","sh","si","sj","sk","sl","sm","sn","so","sr","ss","st","sv","sx","sy","sz","tc","td","tf","tg","th","tj","tk","tl","tm","tn","to","tr","tt","tv","tw","tz","ua","ug","um","us","uy","uz","va","vc","ve","vg","vi","vn","vu","wf","ws","ye","yt","za","zm","zw"]);
  29. }
  30. /// We will accept reports up to this many days old.
  31. pub const MAX_BACKDATE: u32 = 3;
  32. /// Get Julian date
  33. pub fn get_date() -> u32 {
  34. time::OffsetDateTime::now_utc()
  35. .date()
  36. .to_julian_day()
  37. .try_into()
  38. .unwrap()
  39. }
  40. #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
  41. pub enum BridgeDistributor {
  42. Lox,
  43. }
  44. /// All the info for a bridge, to be stored in the database
  45. #[derive(Serialize, Deserialize)]
  46. pub struct BridgeInfo {
  47. /// hashed fingerprint (SHA-1 hash of 20-byte bridge ID)
  48. pub fingerprint: [u8; 20],
  49. /// nickname of bridge (probably not necessary)
  50. pub nickname: String,
  51. /// first Julian date we started collecting data on this bridge
  52. pub first_seen: u32,
  53. /// map of countries to data for this bridge in that country
  54. pub info_by_country: HashMap<String, BridgeCountryInfo>,
  55. }
  56. impl BridgeInfo {
  57. pub fn new(fingerprint: [u8; 20], nickname: &String) -> Self {
  58. Self {
  59. fingerprint: fingerprint,
  60. nickname: nickname.to_string(),
  61. first_seen: get_date(),
  62. info_by_country: HashMap::<String, BridgeCountryInfo>::new(),
  63. }
  64. }
  65. }
  66. impl fmt::Display for BridgeInfo {
  67. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
  68. let mut str = format!(
  69. "fingerprint:{}\n",
  70. array_bytes::bytes2hex("", self.fingerprint).as_str()
  71. );
  72. str.push_str(format!("nickname: {}\n", self.nickname).as_str());
  73. str.push_str(format!("first_seen: {}\n", self.first_seen).as_str());
  74. str.push_str("info_by_country:");
  75. for country in self.info_by_country.keys() {
  76. str.push_str(format!("\n country: {}", country).as_str());
  77. let country_info = self.info_by_country.get(country).unwrap();
  78. for line in country_info.to_string().lines() {
  79. str.push_str(format!("\n {}", line).as_str());
  80. }
  81. }
  82. write!(f, "{}", str)
  83. }
  84. }
  85. #[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
  86. pub enum BridgeInfoType {
  87. BridgeIps,
  88. NegativeReports,
  89. PositiveReports,
  90. }
  91. /// Information about bridge reachability from a given country
  92. #[derive(Serialize, Deserialize)]
  93. pub struct BridgeCountryInfo {
  94. pub info_by_day: BTreeMap<u32, BTreeMap<BridgeInfoType, u32>>,
  95. pub blocked: bool,
  96. }
  97. impl BridgeCountryInfo {
  98. pub fn new() -> Self {
  99. Self {
  100. info_by_day: BTreeMap::<u32, BTreeMap<BridgeInfoType, u32>>::new(),
  101. blocked: false,
  102. }
  103. }
  104. pub fn add_info(&mut self, info_type: BridgeInfoType, date: u32, count: u32) {
  105. if self.info_by_day.contains_key(&date) {
  106. let info = self.info_by_day.get_mut(&date).unwrap();
  107. if !info.contains_key(&info_type) {
  108. info.insert(info_type, count);
  109. } else if info_type == BridgeInfoType::BridgeIps {
  110. if *info.get(&info_type).unwrap() < count {
  111. // Use highest value we've seen today
  112. info.insert(info_type, count);
  113. }
  114. } else {
  115. // Add count to previous count for reports
  116. let new_count = info.get(&info_type).unwrap() + count;
  117. info.insert(info_type, new_count);
  118. }
  119. } else {
  120. let mut info = BTreeMap::<BridgeInfoType, u32>::new();
  121. info.insert(info_type, count);
  122. self.info_by_day.insert(date, info);
  123. }
  124. }
  125. }
  126. impl fmt::Display for BridgeCountryInfo {
  127. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
  128. let mut str = String::from("info:");
  129. for date in self.info_by_day.keys() {
  130. let info = self.info_by_day.get(date).unwrap();
  131. let ip_count = match info.get(&BridgeInfoType::BridgeIps) {
  132. Some(v) => v,
  133. None => &0,
  134. };
  135. let nr_count = match info.get(&BridgeInfoType::NegativeReports) {
  136. Some(v) => v,
  137. None => &0,
  138. };
  139. let pr_count = match info.get(&BridgeInfoType::PositiveReports) {
  140. Some(v) => v,
  141. None => &0,
  142. };
  143. if ip_count > &0 || nr_count > &0 || pr_count > &0 {
  144. str.push_str(
  145. format!(
  146. "\n date: {}\n connections: {}\n negative reports: {}\n positive reports: {}",
  147. date,
  148. ip_count,
  149. nr_count,
  150. pr_count,
  151. )
  152. .as_str(),
  153. );
  154. }
  155. }
  156. write!(f, "{}", str)
  157. }
  158. }
  159. /// We store a set of all known bridges so that we can later iterate over them.
  160. /// This function just adds a bridge fingerprint to that set.
  161. pub fn add_bridge_to_db(db: &Db, fingerprint: [u8; 20]) {
  162. let mut bridges = match db.get("bridges").unwrap() {
  163. Some(v) => bincode::deserialize(&v).unwrap(),
  164. None => HashSet::<[u8; 20]>::new(),
  165. };
  166. bridges.insert(fingerprint);
  167. db.insert("bridges", bincode::serialize(&bridges).unwrap())
  168. .unwrap();
  169. }
  170. // Download a webpage and return it as a string
  171. pub async fn download(url: &str) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
  172. let https = hyper_rustls::HttpsConnectorBuilder::new()
  173. .with_native_roots()
  174. .expect("no native root CA certificates found")
  175. .https_only()
  176. .enable_http1()
  177. .build();
  178. let client: hyper_util::client::legacy::Client<_, Empty<Bytes>> =
  179. hyper_util::client::legacy::Client::builder(TokioExecutor::new()).build(https);
  180. println!("Downloading {}", url);
  181. let mut res = client.get(url.parse()?).await?;
  182. assert_eq!(res.status(), StatusCode::OK);
  183. let mut body_str = String::default();
  184. while let Some(next) = res.frame().await {
  185. let frame = next?;
  186. if let Some(chunk) = frame.data_ref() {
  187. body_str.push_str(&String::from_utf8(chunk.to_vec())?);
  188. }
  189. }
  190. Ok(body_str)
  191. }
  192. // Process extra-infos
  193. /// Adds the extra-info data for a single bridge to the database. If the
  194. /// database already contains an extra-info for this bridge for thid date,
  195. /// but this extra-info contains different data for some reason, use the
  196. /// greater count of connections from each country.
  197. pub fn add_extra_info_to_db(db: &Db, extra_info: ExtraInfo) {
  198. let fingerprint = extra_info.fingerprint;
  199. let mut bridge_info = match db.get(fingerprint).unwrap() {
  200. Some(v) => bincode::deserialize(&v).unwrap(),
  201. None => {
  202. add_bridge_to_db(&db, fingerprint);
  203. BridgeInfo::new(fingerprint, &extra_info.nickname)
  204. }
  205. };
  206. for country in extra_info.bridge_ips.keys() {
  207. if bridge_info.info_by_country.contains_key::<String>(country) {
  208. bridge_info
  209. .info_by_country
  210. .get_mut(country)
  211. .unwrap()
  212. .add_info(
  213. BridgeInfoType::BridgeIps,
  214. extra_info.date,
  215. *extra_info.bridge_ips.get(country).unwrap(),
  216. );
  217. } else {
  218. // No existing entry; make a new one.
  219. let mut bridge_country_info = BridgeCountryInfo::new();
  220. bridge_country_info.add_info(
  221. BridgeInfoType::BridgeIps,
  222. extra_info.date,
  223. *extra_info.bridge_ips.get(country).unwrap(),
  224. );
  225. bridge_info
  226. .info_by_country
  227. .insert(country.to_string(), bridge_country_info);
  228. }
  229. }
  230. // Commit changes to database
  231. db.insert(fingerprint, bincode::serialize(&bridge_info).unwrap())
  232. .unwrap();
  233. }
  234. /// Download new extra-infos files and add their data to the database
  235. pub async fn update_extra_infos(
  236. db: &Db,
  237. base_url: &str,
  238. ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
  239. // Track which files have been processed. This is slightly redundant
  240. // because we're only downloading files we don't already have, but it
  241. // might be a good idea to check in case we downloaded a file but didn't
  242. // process it for some reason.
  243. let mut processed_extra_infos_files = match db.get(b"extra_infos_files").unwrap() {
  244. Some(v) => bincode::deserialize(&v).unwrap(),
  245. None => HashSet::<String>::new(),
  246. };
  247. let dir_page = download(base_url).await?;
  248. // Causes Send issues, so use solution below instead
  249. //let doc = Document::from(dir_page.as_str());
  250. //let links = doc.find(Name("a")).filter_map(|n| n.attr("href"));
  251. // Alternative, less robust solution
  252. let mut links = HashSet::<String>::new();
  253. for line in dir_page.lines() {
  254. let begin_match = "<a href=\"";
  255. let end_match = "\">";
  256. if line.contains(begin_match) {
  257. let link = &line[line.find(begin_match).unwrap() + begin_match.len()..];
  258. if link.contains(end_match) {
  259. let link = &link[0..link.find(end_match).unwrap()];
  260. links.insert(link.to_string());
  261. }
  262. }
  263. }
  264. let mut new_extra_infos = HashSet::<ExtraInfo>::new();
  265. // We should now have an iterable collection of links to consider downloading.
  266. for link in links {
  267. if link.ends_with("-extra-infos") && !processed_extra_infos_files.contains(&link) {
  268. let extra_infos_url = format!("{}{}", base_url, link);
  269. let extra_info_str = download(&extra_infos_url).await?;
  270. //ExtraInfo::parse_file(&extra_info_str, &mut new_extra_infos);
  271. let extra_infos = ExtraInfo::parse_file(&extra_info_str);
  272. new_extra_infos.extend(extra_infos);
  273. processed_extra_infos_files.insert(link);
  274. }
  275. }
  276. // Add new extra-infos data to database
  277. for extra_info in new_extra_infos {
  278. add_extra_info_to_db(&db, extra_info);
  279. }
  280. // Store which files we've already downloaded and processed
  281. db.insert(
  282. b"extra_infos_files",
  283. bincode::serialize(&processed_extra_infos_files).unwrap(),
  284. )
  285. .unwrap();
  286. Ok(())
  287. }
  288. // Process negative reports
  289. /// If there is already a negative report ECDH key for this date, return None.
  290. /// Otherwise, generate a new keypair, save the secret part in the db, and
  291. /// return the public part.
  292. pub fn new_negative_report_key(db: &Db, date: u32) -> Option<PublicKey> {
  293. let mut nr_keys = if !db.contains_key("nr-keys").unwrap() {
  294. BTreeMap::<u32, StaticSecret>::new()
  295. } else {
  296. match bincode::deserialize(&db.get("nr-keys").unwrap().unwrap()) {
  297. Ok(v) => v,
  298. Err(_) => BTreeMap::<u32, StaticSecret>::new(),
  299. }
  300. };
  301. if nr_keys.contains_key(&date) {
  302. None
  303. } else {
  304. let mut rng = rand::thread_rng();
  305. let secret = StaticSecret::random_from_rng(&mut rng);
  306. let public = PublicKey::from(&secret);
  307. nr_keys.insert(date, secret);
  308. db.insert("nr-keys", bincode::serialize(&nr_keys).unwrap())
  309. .unwrap();
  310. Some(public)
  311. }
  312. }
  313. /// Receive an encrypted negative report. Attempt to decrypt it and if
  314. /// successful, add it to the database to be processed later.
  315. pub fn handle_encrypted_negative_report(db: &Db, enc_report: EncryptedNegativeReport) {
  316. if db.contains_key("nr-keys").unwrap() {
  317. let nr_keys: BTreeMap<u32, StaticSecret> =
  318. match bincode::deserialize(&db.get("nr-keys").unwrap().unwrap()) {
  319. Ok(map) => map,
  320. Err(_) => {
  321. return;
  322. }
  323. };
  324. if nr_keys.contains_key(&enc_report.date) {
  325. let secret = nr_keys.get(&enc_report.date).unwrap();
  326. let nr = match enc_report.decrypt(&secret) {
  327. Ok(nr) => nr,
  328. Err(_) => {
  329. return;
  330. }
  331. };
  332. save_negative_report_to_process(&db, nr);
  333. }
  334. }
  335. }
  336. /// We store to-be-processed negative reports as a vector. Add this NR
  337. /// to that vector (or create a new vector if necessary)
  338. pub fn save_negative_report_to_process(db: &Db, nr: NegativeReport) {
  339. // TODO: Purge these database entries sometimes
  340. let mut nonces = match db.get(format!("nonces_{}", &nr.date)).unwrap() {
  341. Some(v) => bincode::deserialize(&v).unwrap(),
  342. None => HashSet::<[u8; 32]>::new(),
  343. };
  344. // Just ignore the report if we've seen the nonce before
  345. if nonces.insert(nr.nonce) {
  346. db.insert(
  347. format!("nonces_{}", &nr.date),
  348. bincode::serialize(&nonces).unwrap(),
  349. )
  350. .unwrap();
  351. let mut reports = match db.get("nrs-to-process").unwrap() {
  352. Some(v) => bincode::deserialize(&v).unwrap(),
  353. None => BTreeMap::<String, Vec<SerializableNegativeReport>>::new(),
  354. };
  355. // Store to-be-processed reports with key [fingerprint]_[country]_[date]
  356. let map_key = format!(
  357. "{}_{}_{}",
  358. array_bytes::bytes2hex("", &nr.fingerprint),
  359. &nr.country,
  360. &nr.date,
  361. );
  362. if reports.contains_key(&map_key) {
  363. reports
  364. .get_mut(&map_key)
  365. .unwrap()
  366. .push(nr.to_serializable_report());
  367. } else {
  368. let mut nrs = Vec::<SerializableNegativeReport>::new();
  369. nrs.push(nr.to_serializable_report());
  370. reports.insert(map_key, nrs);
  371. }
  372. // Commit changes to database
  373. db.insert("nrs-to-process", bincode::serialize(&reports).unwrap())
  374. .unwrap();
  375. }
  376. }
  377. /// Sends a collection of negative reports to the Lox Authority and returns the
  378. /// number of valid reports returned by the server. The negative reports in the
  379. /// collection should all have the same bridge fingerprint, date, country, and
  380. /// distributor.
  381. pub async fn verify_negative_reports(
  382. distributors: &BTreeMap<BridgeDistributor, String>,
  383. reports: &Vec<SerializableNegativeReport>,
  384. ) -> u32 {
  385. // Don't make a network call if we don't have any reports anyway
  386. if reports.is_empty() {
  387. return 0;
  388. }
  389. // Get one report, assume the rest have the same distributor
  390. let first_report = &reports[0];
  391. let distributor = first_report.distributor;
  392. let client = Client::new();
  393. let uri: String = (distributors.get(&distributor).unwrap().to_owned() + "/verifynegative")
  394. .parse()
  395. .unwrap();
  396. let req = Request::builder()
  397. .method(Method::POST)
  398. .uri(uri)
  399. .body(Body::from(serde_json::to_string(&reports).unwrap()))
  400. .unwrap();
  401. let resp = client.request(req).await.unwrap();
  402. let buf = hyper::body::to_bytes(resp).await.unwrap();
  403. serde_json::from_slice(&buf).unwrap()
  404. }
  405. /// Process today's negative reports and store the count of verified reports in
  406. /// the database.
  407. pub async fn update_negative_reports(db: &Db, distributors: &BTreeMap<BridgeDistributor, String>) {
  408. let mut all_negative_reports = match db.get("nrs-to-process").unwrap() {
  409. Some(v) => bincode::deserialize(&v).unwrap(),
  410. None => BTreeMap::<String, Vec<SerializableNegativeReport>>::new(),
  411. };
  412. // Key is [fingerprint]_[country]_[date]
  413. for bridge_country_date in all_negative_reports.keys() {
  414. let reports = all_negative_reports.get(bridge_country_date).unwrap();
  415. if !reports.is_empty() {
  416. let first_report = &reports[0];
  417. let fingerprint = first_report.fingerprint;
  418. let date = first_report.date;
  419. let country = first_report.country.clone();
  420. let count_valid = verify_negative_reports(&distributors, reports).await;
  421. // Get bridge info or make new one
  422. let mut bridge_info = match db.get(fingerprint).unwrap() {
  423. Some(v) => bincode::deserialize(&v).unwrap(),
  424. None => {
  425. // This case shouldn't happen unless the bridge hasn't
  426. // published any bridge stats.
  427. add_bridge_to_db(&db, fingerprint);
  428. BridgeInfo::new(fingerprint, &String::default())
  429. }
  430. };
  431. // Add the new report count to it
  432. if bridge_info.info_by_country.contains_key(&country) {
  433. let bridge_country_info = bridge_info.info_by_country.get_mut(&country).unwrap();
  434. bridge_country_info.add_info(BridgeInfoType::NegativeReports, date, count_valid);
  435. } else {
  436. // No existing entry; make a new one.
  437. let mut bridge_country_info = BridgeCountryInfo::new();
  438. bridge_country_info.add_info(BridgeInfoType::NegativeReports, date, count_valid);
  439. bridge_info
  440. .info_by_country
  441. .insert(country, bridge_country_info);
  442. }
  443. // Commit changes to database
  444. db.insert(fingerprint, bincode::serialize(&bridge_info).unwrap())
  445. .unwrap();
  446. }
  447. }
  448. // TODO: Would it be cheaper to just recreate it?
  449. all_negative_reports.clear();
  450. // Remove the now-processed reports from the database
  451. db.insert(
  452. "nrs-to-process",
  453. bincode::serialize(&all_negative_reports).unwrap(),
  454. )
  455. .unwrap();
  456. }
  457. // Process positive reports
  458. /// We store to-be-processed positive reports as a vector. Add this PR
  459. /// to that vector (or create a new vector if necessary).
  460. pub fn save_positive_report_to_process(db: &Db, pr: PositiveReport) {
  461. let mut reports = match db.get("prs-to-process").unwrap() {
  462. Some(v) => bincode::deserialize(&v).unwrap(),
  463. None => BTreeMap::<String, Vec<SerializablePositiveReport>>::new(),
  464. };
  465. // Store to-be-processed reports with key [fingerprint]_[country]_[date]
  466. let map_key = format!(
  467. "{}_{}_{}",
  468. array_bytes::bytes2hex("", &pr.fingerprint),
  469. &pr.country,
  470. &pr.date,
  471. );
  472. if reports.contains_key(&map_key) {
  473. reports
  474. .get_mut(&map_key)
  475. .unwrap()
  476. .push(pr.to_serializable_report());
  477. } else {
  478. let mut prs = Vec::<SerializablePositiveReport>::new();
  479. prs.push(pr.to_serializable_report());
  480. reports.insert(map_key, prs);
  481. }
  482. // Commit changes to database
  483. db.insert("prs-to-process", bincode::serialize(&reports).unwrap())
  484. .unwrap();
  485. }
  486. /// Sends a collection of positive reports to the Lox Authority and returns the
  487. /// number of valid reports returned by the server. The positive reports in the
  488. /// collection should all have the same bridge fingerprint, date, and country.
  489. pub async fn verify_positive_reports(
  490. distributors: &BTreeMap<BridgeDistributor, String>,
  491. reports: &Vec<SerializablePositiveReport>,
  492. ) -> u32 {
  493. // Don't make a network call if we don't have any reports anyway
  494. if reports.is_empty() {
  495. return 0;
  496. }
  497. let client = Client::new();
  498. let uri: String = (distributors
  499. .get(&BridgeDistributor::Lox)
  500. .unwrap()
  501. .to_owned()
  502. + "/verifypositive")
  503. .parse()
  504. .unwrap();
  505. let req = Request::builder()
  506. .method(Method::POST)
  507. .uri(uri)
  508. .body(Body::from(serde_json::to_string(&reports).unwrap()))
  509. .unwrap();
  510. let resp = client.request(req).await.unwrap();
  511. let buf = hyper::body::to_bytes(resp).await.unwrap();
  512. serde_json::from_slice(&buf).unwrap()
  513. }
  514. /// Process today's positive reports and store the count of verified reports in
  515. /// the database.
  516. pub async fn update_positive_reports(db: &Db, distributors: &BTreeMap<BridgeDistributor, String>) {
  517. let mut all_positive_reports = match db.get("prs-to-process").unwrap() {
  518. Some(v) => bincode::deserialize(&v).unwrap(),
  519. None => BTreeMap::<String, Vec<SerializablePositiveReport>>::new(),
  520. };
  521. // Key is [fingerprint]_[country]_[date]
  522. for bridge_country_date in all_positive_reports.keys() {
  523. let reports = all_positive_reports.get(bridge_country_date).unwrap();
  524. if !reports.is_empty() {
  525. let first_report = &reports[0];
  526. let fingerprint = first_report.fingerprint;
  527. let date = first_report.date;
  528. let country = first_report.country.clone();
  529. let count_valid = verify_positive_reports(&distributors, reports).await;
  530. // Get bridge info or make new one
  531. let mut bridge_info = match db.get(fingerprint).unwrap() {
  532. Some(v) => bincode::deserialize(&v).unwrap(),
  533. None => {
  534. // This case shouldn't happen unless the bridge hasn't
  535. // published any bridge stats.
  536. add_bridge_to_db(&db, fingerprint);
  537. BridgeInfo::new(fingerprint, &String::default())
  538. }
  539. };
  540. // Add the new report count to it
  541. if bridge_info.info_by_country.contains_key(&country) {
  542. let bridge_country_info = bridge_info.info_by_country.get_mut(&country).unwrap();
  543. bridge_country_info.add_info(BridgeInfoType::PositiveReports, date, count_valid);
  544. } else {
  545. // No existing entry; make a new one.
  546. let mut bridge_country_info = BridgeCountryInfo::new();
  547. bridge_country_info.add_info(BridgeInfoType::PositiveReports, date, count_valid);
  548. bridge_info
  549. .info_by_country
  550. .insert(country, bridge_country_info);
  551. }
  552. // Commit changes to database
  553. db.insert(fingerprint, bincode::serialize(&bridge_info).unwrap())
  554. .unwrap();
  555. }
  556. }
  557. // TODO: Would it be cheaper to just recreate it?
  558. all_positive_reports.clear();
  559. // Remove the now-processed reports from the database
  560. db.insert(
  561. "prs-to-process",
  562. bincode::serialize(&all_positive_reports).unwrap(),
  563. )
  564. .unwrap();
  565. }
  566. // Verdict on bridge reachability
  567. /// Guess which countries block a bridge. This function returns a map of new
  568. /// blockages (fingerprint : set of countries which block the bridge)
  569. pub fn guess_blockages(
  570. db: &Db,
  571. analyzer: &dyn Analyzer,
  572. confidence: f64,
  573. ) -> HashMap<[u8; 20], HashSet<String>> {
  574. // Map of bridge fingerprint to set of countries which newly block it
  575. let mut blockages = HashMap::<[u8; 20], HashSet<String>>::new();
  576. // Get list of bridges from database
  577. let bridges = match db.get("bridges").unwrap() {
  578. Some(v) => bincode::deserialize(&v).unwrap(),
  579. None => HashSet::<[u8; 20]>::new(),
  580. };
  581. // Guess for each bridge
  582. for fingerprint in bridges {
  583. let mut bridge_info: BridgeInfo =
  584. bincode::deserialize(&db.get(fingerprint).unwrap().unwrap()).unwrap();
  585. let mut new_blockages = HashSet::<String>::new();
  586. let blocked_in = analysis::blocked_in(analyzer, &bridge_info, confidence, get_date());
  587. for country in blocked_in {
  588. let bridge_country_info = bridge_info.info_by_country.get_mut(&country).unwrap();
  589. if !bridge_country_info.blocked {
  590. new_blockages.insert(country.to_string());
  591. // Mark bridge as blocked when db gets updated
  592. bridge_country_info.blocked = true;
  593. }
  594. }
  595. blockages.insert(fingerprint, new_blockages);
  596. // Commit changes to database
  597. db.insert(fingerprint, bincode::serialize(&bridge_info).unwrap())
  598. .unwrap();
  599. }
  600. // Return map of new blockages
  601. blockages
  602. }
  603. /// Report blocked bridges to bridge distributor
  604. pub async fn report_blockages(
  605. distributors: &BTreeMap<BridgeDistributor, String>,
  606. blockages: HashMap<[u8; 20], HashSet<String>>,
  607. ) {
  608. // For now, only report to Lox
  609. // TODO: Support more distributors
  610. let uri: String = (distributors
  611. .get(&BridgeDistributor::Lox)
  612. .unwrap()
  613. .to_owned()
  614. + "/reportblocked")
  615. .parse()
  616. .unwrap();
  617. // Convert map keys from [u8; 20] to 40-character hex strings
  618. let mut blockages_str = HashMap::<String, HashSet<String>>::new();
  619. for (fingerprint, countries) in blockages {
  620. let fpr_string = array_bytes::bytes2hex("", fingerprint);
  621. blockages_str.insert(fpr_string, countries);
  622. }
  623. // Report blocked bridges to bridge distributor
  624. let client = Client::new();
  625. let req = Request::builder()
  626. .method(Method::POST)
  627. .uri(uri)
  628. .body(Body::from(serde_json::to_string(&blockages_str).unwrap()))
  629. .unwrap();
  630. let resp = client.request(req).await.unwrap();
  631. let buf = hyper::body::to_bytes(resp).await.unwrap();
  632. let resp_str: String = serde_json::from_slice(&buf).unwrap();
  633. assert_eq!("OK", resp_str);
  634. }
  635. // Unit tests
  636. #[cfg(test)]
  637. mod tests;