directory.c 213 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011
  1. /* Copyright (c) 2001-2004, Roger Dingledine.
  2. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
  3. * Copyright (c) 2007-2017, The Tor Project, Inc. */
  4. /* See LICENSE for licensing information */
  5. #define DIRECTORY_PRIVATE
  6. #include "or.h"
  7. #include "backtrace.h"
  8. #include "bridges.h"
  9. #include "buffers.h"
  10. #include "circuitbuild.h"
  11. #include "config.h"
  12. #include "connection.h"
  13. #include "connection_edge.h"
  14. #include "conscache.h"
  15. #include "consdiff.h"
  16. #include "consdiffmgr.h"
  17. #include "control.h"
  18. #include "compat.h"
  19. #include "directory.h"
  20. #include "dirserv.h"
  21. #include "dirvote.h"
  22. #include "entrynodes.h"
  23. #include "geoip.h"
  24. #include "hs_cache.h"
  25. #include "hs_common.h"
  26. #include "hs_control.h"
  27. #include "hs_client.h"
  28. #include "main.h"
  29. #include "microdesc.h"
  30. #include "networkstatus.h"
  31. #include "nodelist.h"
  32. #include "policies.h"
  33. #include "relay.h"
  34. #include "rendclient.h"
  35. #include "rendcommon.h"
  36. #include "rendservice.h"
  37. #include "rephist.h"
  38. #include "router.h"
  39. #include "routerlist.h"
  40. #include "routerparse.h"
  41. #include "routerset.h"
  42. #include "shared_random.h"
  43. #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO)
  44. #if !defined(OpenBSD)
  45. #include <malloc.h>
  46. #endif
  47. #endif
  48. /**
  49. * \file directory.c
  50. * \brief Code to send and fetch information from directory authorities and
  51. * caches via HTTP.
  52. *
  53. * Directory caches and authorities use dirserv.c to generate the results of a
  54. * query and stream them to the connection; clients use routerparse.c to parse
  55. * them.
  56. *
  57. * Every directory request has a dir_connection_t on the client side and on
  58. * the server side. In most cases, the dir_connection_t object is a linked
  59. * connection, tunneled through an edge_connection_t so that it can be a
  60. * stream on the Tor network. The only non-tunneled connections are those
  61. * that are used to upload material (descriptors and votes) to authorities.
  62. * Among tunneled connections, some use one-hop circuits, and others use
  63. * multi-hop circuits for anonymity.
  64. *
  65. * Directory requests are launched by calling
  66. * directory_initiate_request(). This
  67. * launch the connection, will construct an HTTP request with
  68. * directory_send_command(), send the and wait for a response. The client
  69. * later handles the response with connection_dir_client_reached_eof(),
  70. * which passes the information received to another part of Tor.
  71. *
  72. * On the server side, requests are read in directory_handle_command(),
  73. * which dispatches first on the request type (GET or POST), and then on
  74. * the URL requested. GET requests are processed with a table-based
  75. * dispatcher in url_table[]. The process of handling larger GET requests
  76. * is complicated because we need to avoid allocating a copy of all the
  77. * data to be sent to the client in one huge buffer. Instead, we spool the
  78. * data into the buffer using logic in connection_dirserv_flushed_some() in
  79. * dirserv.c. (TODO: If we extended buf.c to have a zero-copy
  80. * reference-based buffer type, we could remove most of that code, at the
  81. * cost of a bit more reference counting.)
  82. **/
  83. /* In-points to directory.c:
  84. *
  85. * - directory_post_to_dirservers(), called from
  86. * router_upload_dir_desc_to_dirservers() in router.c
  87. * upload_service_descriptor() in rendservice.c
  88. * - directory_get_from_dirserver(), called from
  89. * rend_client_refetch_renddesc() in rendclient.c
  90. * run_scheduled_events() in main.c
  91. * do_hup() in main.c
  92. * - connection_dir_process_inbuf(), called from
  93. * connection_process_inbuf() in connection.c
  94. * - connection_dir_finished_flushing(), called from
  95. * connection_finished_flushing() in connection.c
  96. * - connection_dir_finished_connecting(), called from
  97. * connection_finished_connecting() in connection.c
  98. */
  99. static void directory_send_command(dir_connection_t *conn,
  100. int direct,
  101. const directory_request_t *request);
  102. static int body_is_plausible(const char *body, size_t body_len, int purpose);
  103. static void http_set_address_origin(const char *headers, connection_t *conn);
  104. static void connection_dir_download_routerdesc_failed(dir_connection_t *conn);
  105. static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn);
  106. static void connection_dir_download_cert_failed(
  107. dir_connection_t *conn, int status_code);
  108. static void connection_dir_retry_bridges(smartlist_t *descs);
  109. static void dir_routerdesc_download_failed(smartlist_t *failed,
  110. int status_code,
  111. int router_purpose,
  112. int was_extrainfo,
  113. int was_descriptor_digests);
  114. static void dir_microdesc_download_failed(smartlist_t *failed,
  115. int status_code,
  116. const char *dir_id);
  117. static int client_likes_consensus(const struct consensus_cache_entry_t *ent,
  118. const char *want_url);
  119. static void connection_dir_close_consensus_fetches(
  120. dir_connection_t *except_this_one, const char *resource);
  121. /********* START VARIABLES **********/
  122. /** How far in the future do we allow a directory server to tell us it is
  123. * before deciding that one of us has the wrong time? */
  124. #define ALLOW_DIRECTORY_TIME_SKEW (30*60)
  125. #define X_ADDRESS_HEADER "X-Your-Address-Is: "
  126. #define X_OR_DIFF_FROM_CONSENSUS_HEADER "X-Or-Diff-From-Consensus: "
  127. /** HTTP cache control: how long do we tell proxies they can cache each
  128. * kind of document we serve? */
  129. #define FULL_DIR_CACHE_LIFETIME (60*60)
  130. #define RUNNINGROUTERS_CACHE_LIFETIME (20*60)
  131. #define DIRPORTFRONTPAGE_CACHE_LIFETIME (20*60)
  132. #define NETWORKSTATUS_CACHE_LIFETIME (5*60)
  133. #define ROUTERDESC_CACHE_LIFETIME (30*60)
  134. #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
  135. #define ROBOTS_CACHE_LIFETIME (24*60*60)
  136. #define MICRODESC_CACHE_LIFETIME (48*60*60)
  137. /********* END VARIABLES ************/
  138. /** Return false if the directory purpose <b>dir_purpose</b>
  139. * does not require an anonymous (three-hop) connection.
  140. *
  141. * Return true 1) by default, 2) if all directory actions have
  142. * specifically been configured to be over an anonymous connection,
  143. * or 3) if the router is a bridge */
  144. int
  145. purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose,
  146. const char *resource)
  147. {
  148. if (get_options()->AllDirActionsPrivate)
  149. return 1;
  150. if (router_purpose == ROUTER_PURPOSE_BRIDGE) {
  151. if (dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC
  152. && resource && !strcmp(resource, "authority.z")) {
  153. /* We are asking a bridge for its own descriptor. That doesn't need
  154. anonymity. */
  155. return 0;
  156. }
  157. /* Assume all other bridge stuff needs anonymity. */
  158. return 1; /* if no circuits yet, this might break bootstrapping, but it's
  159. * needed to be safe. */
  160. }
  161. switch (dir_purpose)
  162. {
  163. case DIR_PURPOSE_UPLOAD_DIR:
  164. case DIR_PURPOSE_UPLOAD_VOTE:
  165. case DIR_PURPOSE_UPLOAD_SIGNATURES:
  166. case DIR_PURPOSE_FETCH_STATUS_VOTE:
  167. case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
  168. case DIR_PURPOSE_FETCH_CONSENSUS:
  169. case DIR_PURPOSE_FETCH_CERTIFICATE:
  170. case DIR_PURPOSE_FETCH_SERVERDESC:
  171. case DIR_PURPOSE_FETCH_EXTRAINFO:
  172. case DIR_PURPOSE_FETCH_MICRODESC:
  173. return 0;
  174. case DIR_PURPOSE_HAS_FETCHED_HSDESC:
  175. case DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2:
  176. case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
  177. case DIR_PURPOSE_FETCH_RENDDESC_V2:
  178. case DIR_PURPOSE_FETCH_HSDESC:
  179. case DIR_PURPOSE_UPLOAD_HSDESC:
  180. return 1;
  181. case DIR_PURPOSE_SERVER:
  182. default:
  183. log_warn(LD_BUG, "Called with dir_purpose=%d, router_purpose=%d",
  184. dir_purpose, router_purpose);
  185. tor_assert_nonfatal_unreached();
  186. return 1; /* Assume it needs anonymity; better safe than sorry. */
  187. }
  188. }
  189. /** Return a newly allocated string describing <b>auth</b>. Only describes
  190. * authority features. */
  191. STATIC char *
  192. authdir_type_to_string(dirinfo_type_t auth)
  193. {
  194. char *result;
  195. smartlist_t *lst = smartlist_new();
  196. if (auth & V3_DIRINFO)
  197. smartlist_add(lst, (void*)"V3");
  198. if (auth & BRIDGE_DIRINFO)
  199. smartlist_add(lst, (void*)"Bridge");
  200. if (smartlist_len(lst)) {
  201. result = smartlist_join_strings(lst, ", ", 0, NULL);
  202. } else {
  203. result = tor_strdup("[Not an authority]");
  204. }
  205. smartlist_free(lst);
  206. return result;
  207. }
  208. /** Return a string describing a given directory connection purpose. */
  209. STATIC const char *
  210. dir_conn_purpose_to_string(int purpose)
  211. {
  212. switch (purpose)
  213. {
  214. case DIR_PURPOSE_UPLOAD_DIR:
  215. return "server descriptor upload";
  216. case DIR_PURPOSE_UPLOAD_VOTE:
  217. return "server vote upload";
  218. case DIR_PURPOSE_UPLOAD_SIGNATURES:
  219. return "consensus signature upload";
  220. case DIR_PURPOSE_FETCH_SERVERDESC:
  221. return "server descriptor fetch";
  222. case DIR_PURPOSE_FETCH_EXTRAINFO:
  223. return "extra-info fetch";
  224. case DIR_PURPOSE_FETCH_CONSENSUS:
  225. return "consensus network-status fetch";
  226. case DIR_PURPOSE_FETCH_CERTIFICATE:
  227. return "authority cert fetch";
  228. case DIR_PURPOSE_FETCH_STATUS_VOTE:
  229. return "status vote fetch";
  230. case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
  231. return "consensus signature fetch";
  232. case DIR_PURPOSE_FETCH_RENDDESC_V2:
  233. return "hidden-service v2 descriptor fetch";
  234. case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
  235. return "hidden-service v2 descriptor upload";
  236. case DIR_PURPOSE_FETCH_HSDESC:
  237. return "hidden-service descriptor fetch";
  238. case DIR_PURPOSE_UPLOAD_HSDESC:
  239. return "hidden-service descriptor upload";
  240. case DIR_PURPOSE_FETCH_MICRODESC:
  241. return "microdescriptor fetch";
  242. }
  243. log_warn(LD_BUG, "Called with unknown purpose %d", purpose);
  244. return "(unknown)";
  245. }
  246. /** Return the requisite directory information types. */
  247. STATIC dirinfo_type_t
  248. dir_fetch_type(int dir_purpose, int router_purpose, const char *resource)
  249. {
  250. dirinfo_type_t type;
  251. switch (dir_purpose) {
  252. case DIR_PURPOSE_FETCH_EXTRAINFO:
  253. type = EXTRAINFO_DIRINFO;
  254. if (router_purpose == ROUTER_PURPOSE_BRIDGE)
  255. type |= BRIDGE_DIRINFO;
  256. else
  257. type |= V3_DIRINFO;
  258. break;
  259. case DIR_PURPOSE_FETCH_SERVERDESC:
  260. if (router_purpose == ROUTER_PURPOSE_BRIDGE)
  261. type = BRIDGE_DIRINFO;
  262. else
  263. type = V3_DIRINFO;
  264. break;
  265. case DIR_PURPOSE_FETCH_STATUS_VOTE:
  266. case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
  267. case DIR_PURPOSE_FETCH_CERTIFICATE:
  268. type = V3_DIRINFO;
  269. break;
  270. case DIR_PURPOSE_FETCH_CONSENSUS:
  271. type = V3_DIRINFO;
  272. if (resource && !strcmp(resource, "microdesc"))
  273. type |= MICRODESC_DIRINFO;
  274. break;
  275. case DIR_PURPOSE_FETCH_MICRODESC:
  276. type = MICRODESC_DIRINFO;
  277. break;
  278. default:
  279. log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose);
  280. type = NO_DIRINFO;
  281. break;
  282. }
  283. return type;
  284. }
  285. /** Return true iff <b>identity_digest</b> is the digest of a router which
  286. * says that it caches extrainfos. (If <b>is_authority</b> we always
  287. * believe that to be true.) */
  288. int
  289. router_supports_extrainfo(const char *identity_digest, int is_authority)
  290. {
  291. const node_t *node = node_get_by_id(identity_digest);
  292. if (node && node->ri) {
  293. if (node->ri->caches_extra_info)
  294. return 1;
  295. }
  296. if (is_authority) {
  297. return 1;
  298. }
  299. return 0;
  300. }
  301. /** Return true iff any trusted directory authority has accepted our
  302. * server descriptor.
  303. *
  304. * We consider any authority sufficient because waiting for all of
  305. * them means it never happens while any authority is down; we don't
  306. * go for something more complex in the middle (like \>1/3 or \>1/2 or
  307. * \>=1/2) because that doesn't seem necessary yet.
  308. */
  309. int
  310. directories_have_accepted_server_descriptor(void)
  311. {
  312. const smartlist_t *servers = router_get_trusted_dir_servers();
  313. const or_options_t *options = get_options();
  314. SMARTLIST_FOREACH(servers, dir_server_t *, d, {
  315. if ((d->type & options->PublishServerDescriptor_) &&
  316. d->has_accepted_serverdesc) {
  317. return 1;
  318. }
  319. });
  320. return 0;
  321. }
  322. /** Start a connection to every suitable directory authority, using
  323. * connection purpose <b>dir_purpose</b> and uploading <b>payload</b>
  324. * (of length <b>payload_len</b>). The dir_purpose should be one of
  325. * 'DIR_PURPOSE_UPLOAD_{DIR|VOTE|SIGNATURES}'.
  326. *
  327. * <b>router_purpose</b> describes the type of descriptor we're
  328. * publishing, if we're publishing a descriptor -- e.g. general or bridge.
  329. *
  330. * <b>type</b> specifies what sort of dir authorities (V3,
  331. * BRIDGE, etc) we should upload to.
  332. *
  333. * If <b>extrainfo_len</b> is nonzero, the first <b>payload_len</b> bytes of
  334. * <b>payload</b> hold a router descriptor, and the next <b>extrainfo_len</b>
  335. * bytes of <b>payload</b> hold an extra-info document. Upload the descriptor
  336. * to all authorities, and the extra-info document to all authorities that
  337. * support it.
  338. */
  339. void
  340. directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
  341. dirinfo_type_t type,
  342. const char *payload,
  343. size_t payload_len, size_t extrainfo_len)
  344. {
  345. const or_options_t *options = get_options();
  346. dir_indirection_t indirection;
  347. const smartlist_t *dirservers = router_get_trusted_dir_servers();
  348. int found = 0;
  349. const int exclude_self = (dir_purpose == DIR_PURPOSE_UPLOAD_VOTE ||
  350. dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES);
  351. tor_assert(dirservers);
  352. /* This tries dirservers which we believe to be down, but ultimately, that's
  353. * harmless, and we may as well err on the side of getting things uploaded.
  354. */
  355. SMARTLIST_FOREACH_BEGIN(dirservers, dir_server_t *, ds) {
  356. routerstatus_t *rs = &(ds->fake_status);
  357. size_t upload_len = payload_len;
  358. if ((type & ds->type) == 0)
  359. continue;
  360. if (exclude_self && router_digest_is_me(ds->digest)) {
  361. /* we don't upload to ourselves, but at least there's now at least
  362. * one authority of this type that has what we wanted to upload. */
  363. found = 1;
  364. continue;
  365. }
  366. if (options->StrictNodes &&
  367. routerset_contains_routerstatus(options->ExcludeNodes, rs, -1)) {
  368. log_warn(LD_DIR, "Wanted to contact authority '%s' for %s, but "
  369. "it's in our ExcludedNodes list and StrictNodes is set. "
  370. "Skipping.",
  371. ds->nickname,
  372. dir_conn_purpose_to_string(dir_purpose));
  373. continue;
  374. }
  375. found = 1; /* at least one authority of this type was listed */
  376. if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR)
  377. ds->has_accepted_serverdesc = 0;
  378. if (extrainfo_len && router_supports_extrainfo(ds->digest, 1)) {
  379. upload_len += extrainfo_len;
  380. log_info(LD_DIR, "Uploading an extrainfo too (length %d)",
  381. (int) extrainfo_len);
  382. }
  383. if (purpose_needs_anonymity(dir_purpose, router_purpose, NULL)) {
  384. indirection = DIRIND_ANONYMOUS;
  385. } else if (!fascist_firewall_allows_dir_server(ds,
  386. FIREWALL_DIR_CONNECTION,
  387. 0)) {
  388. if (fascist_firewall_allows_dir_server(ds, FIREWALL_OR_CONNECTION, 0))
  389. indirection = DIRIND_ONEHOP;
  390. else
  391. indirection = DIRIND_ANONYMOUS;
  392. } else {
  393. indirection = DIRIND_DIRECT_CONN;
  394. }
  395. directory_request_t *req = directory_request_new(dir_purpose);
  396. directory_request_set_routerstatus(req, rs);
  397. directory_request_set_router_purpose(req, router_purpose);
  398. directory_request_set_indirection(req, indirection);
  399. directory_request_set_payload(req, payload, upload_len);
  400. directory_initiate_request(req);
  401. directory_request_free(req);
  402. } SMARTLIST_FOREACH_END(ds);
  403. if (!found) {
  404. char *s = authdir_type_to_string(type);
  405. log_warn(LD_DIR, "Publishing server descriptor to directory authorities "
  406. "of type '%s', but no authorities of that type listed!", s);
  407. tor_free(s);
  408. }
  409. }
  410. /** Return true iff, according to the values in <b>options</b>, we should be
  411. * using directory guards for direct downloads of directory information. */
  412. STATIC int
  413. should_use_directory_guards(const or_options_t *options)
  414. {
  415. /* Public (non-bridge) servers never use directory guards. */
  416. if (public_server_mode(options))
  417. return 0;
  418. /* If guards are disabled, we can't use directory guards.
  419. */
  420. if (!options->UseEntryGuards)
  421. return 0;
  422. /* If we're configured to fetch directory info aggressively or of a
  423. * nonstandard type, don't use directory guards. */
  424. if (options->DownloadExtraInfo || options->FetchDirInfoEarly ||
  425. options->FetchDirInfoExtraEarly || options->FetchUselessDescriptors)
  426. return 0;
  427. return 1;
  428. }
  429. /** Pick an unconstrained directory server from among our guards, the latest
  430. * networkstatus, or the fallback dirservers, for use in downloading
  431. * information of type <b>type</b>, and return its routerstatus. */
  432. static const routerstatus_t *
  433. directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags,
  434. uint8_t dir_purpose,
  435. circuit_guard_state_t **guard_state_out)
  436. {
  437. const routerstatus_t *rs = NULL;
  438. const or_options_t *options = get_options();
  439. if (options->UseBridges)
  440. log_warn(LD_BUG, "Called when we have UseBridges set.");
  441. if (should_use_directory_guards(options)) {
  442. const node_t *node = guards_choose_dirguard(dir_purpose, guard_state_out);
  443. if (node)
  444. rs = node->rs;
  445. } else {
  446. /* anybody with a non-zero dirport will do */
  447. rs = router_pick_directory_server(type, pds_flags);
  448. }
  449. if (!rs) {
  450. log_info(LD_DIR, "No router found for %s; falling back to "
  451. "dirserver list.", dir_conn_purpose_to_string(dir_purpose));
  452. rs = router_pick_fallback_dirserver(type, pds_flags);
  453. }
  454. return rs;
  455. }
  456. /**
  457. * Set the extra fields in <b>req</b> that are used when requesting a
  458. * consensus of type <b>resource</b>.
  459. *
  460. * Right now, these fields are if-modified-since and x-or-diff-from-consensus.
  461. */
  462. static void
  463. dir_consensus_request_set_additional_headers(directory_request_t *req,
  464. const char *resource)
  465. {
  466. time_t if_modified_since = 0;
  467. uint8_t or_diff_from[DIGEST256_LEN];
  468. int or_diff_from_is_set = 0;
  469. /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus
  470. * period of 1 hour.
  471. */
  472. const int DEFAULT_IF_MODIFIED_SINCE_DELAY = 180;
  473. const int32_t DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER = 72;
  474. const int32_t MIN_TRY_DIFF_FOR_CONSENSUS_NEWER = 0;
  475. const int32_t MAX_TRY_DIFF_FOR_CONSENSUS_NEWER = 8192;
  476. const char TRY_DIFF_FOR_CONSENSUS_NEWER_NAME[] =
  477. "try-diff-for-consensus-newer-than";
  478. int flav = FLAV_NS;
  479. if (resource)
  480. flav = networkstatus_parse_flavor_name(resource);
  481. int32_t max_age_for_diff = 3600 *
  482. networkstatus_get_param(NULL,
  483. TRY_DIFF_FOR_CONSENSUS_NEWER_NAME,
  484. DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER,
  485. MIN_TRY_DIFF_FOR_CONSENSUS_NEWER,
  486. MAX_TRY_DIFF_FOR_CONSENSUS_NEWER);
  487. if (flav != -1) {
  488. /* IF we have a parsed consensus of this type, we can do an
  489. * if-modified-time based on it. */
  490. networkstatus_t *v;
  491. v = networkstatus_get_latest_consensus_by_flavor(flav);
  492. if (v) {
  493. /* In networks with particularly short V3AuthVotingIntervals,
  494. * ask for the consensus if it's been modified since half the
  495. * V3AuthVotingInterval of the most recent consensus. */
  496. time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY;
  497. if (v->fresh_until > v->valid_after
  498. && ims_delay > (v->fresh_until - v->valid_after)/2) {
  499. ims_delay = (v->fresh_until - v->valid_after)/2;
  500. }
  501. if_modified_since = v->valid_after + ims_delay;
  502. if (v->valid_after >= approx_time() - max_age_for_diff) {
  503. memcpy(or_diff_from, v->digest_sha3_as_signed, DIGEST256_LEN);
  504. or_diff_from_is_set = 1;
  505. }
  506. }
  507. } else {
  508. /* Otherwise it might be a consensus we don't parse, but which we
  509. * do cache. Look at the cached copy, perhaps. */
  510. cached_dir_t *cd = dirserv_get_consensus(resource);
  511. /* We have no method of determining the voting interval from an
  512. * unparsed consensus, so we use the default. */
  513. if (cd) {
  514. if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY;
  515. if (cd->published >= approx_time() - max_age_for_diff) {
  516. memcpy(or_diff_from, cd->digest_sha3_as_signed, DIGEST256_LEN);
  517. or_diff_from_is_set = 1;
  518. }
  519. }
  520. }
  521. if (if_modified_since > 0)
  522. directory_request_set_if_modified_since(req, if_modified_since);
  523. if (or_diff_from_is_set) {
  524. char hex[HEX_DIGEST256_LEN + 1];
  525. base16_encode(hex, sizeof(hex),
  526. (const char*)or_diff_from, sizeof(or_diff_from));
  527. directory_request_add_header(req, X_OR_DIFF_FROM_CONSENSUS_HEADER, hex);
  528. }
  529. }
  530. /** Start a connection to a random running directory server, using
  531. * connection purpose <b>dir_purpose</b>, intending to fetch descriptors
  532. * of purpose <b>router_purpose</b>, and requesting <b>resource</b>.
  533. * Use <b>pds_flags</b> as arguments to router_pick_directory_server()
  534. * or router_pick_trusteddirserver().
  535. */
  536. MOCK_IMPL(void,
  537. directory_get_from_dirserver,(
  538. uint8_t dir_purpose,
  539. uint8_t router_purpose,
  540. const char *resource,
  541. int pds_flags,
  542. download_want_authority_t want_authority))
  543. {
  544. const routerstatus_t *rs = NULL;
  545. const or_options_t *options = get_options();
  546. int prefer_authority = (directory_fetches_from_authorities(options)
  547. || want_authority == DL_WANT_AUTHORITY);
  548. int require_authority = 0;
  549. int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose,
  550. resource);
  551. dirinfo_type_t type = dir_fetch_type(dir_purpose, router_purpose, resource);
  552. if (type == NO_DIRINFO)
  553. return;
  554. if (!options->FetchServerDescriptors)
  555. return;
  556. circuit_guard_state_t *guard_state = NULL;
  557. if (!get_via_tor) {
  558. if (options->UseBridges && !(type & BRIDGE_DIRINFO)) {
  559. /* We want to ask a running bridge for which we have a descriptor.
  560. *
  561. * When we ask choose_random_entry() for a bridge, we specify what
  562. * sort of dir fetch we'll be doing, so it won't return a bridge
  563. * that can't answer our question.
  564. */
  565. const node_t *node = guards_choose_dirguard(dir_purpose, &guard_state);
  566. if (node && node->ri) {
  567. /* every bridge has a routerinfo. */
  568. routerinfo_t *ri = node->ri;
  569. /* clients always make OR connections to bridges */
  570. tor_addr_port_t or_ap;
  571. directory_request_t *req = directory_request_new(dir_purpose);
  572. /* we are willing to use a non-preferred address if we need to */
  573. fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
  574. &or_ap);
  575. directory_request_set_or_addr_port(req, &or_ap);
  576. directory_request_set_directory_id_digest(req,
  577. ri->cache_info.identity_digest);
  578. directory_request_set_router_purpose(req, router_purpose);
  579. directory_request_set_resource(req, resource);
  580. if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS)
  581. dir_consensus_request_set_additional_headers(req, resource);
  582. directory_request_set_guard_state(req, guard_state);
  583. directory_initiate_request(req);
  584. directory_request_free(req);
  585. } else {
  586. if (guard_state) {
  587. entry_guard_cancel(&guard_state);
  588. }
  589. log_notice(LD_DIR, "Ignoring directory request, since no bridge "
  590. "nodes are available yet.");
  591. }
  592. return;
  593. } else {
  594. if (prefer_authority || (type & BRIDGE_DIRINFO)) {
  595. /* only ask authdirservers, and don't ask myself */
  596. rs = router_pick_trusteddirserver(type, pds_flags);
  597. if (rs == NULL && (pds_flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
  598. PDS_NO_EXISTING_MICRODESC_FETCH))) {
  599. /* We don't want to fetch from any authorities that we're currently
  600. * fetching server descriptors from, and we got no match. Did we
  601. * get no match because all the authorities have connections
  602. * fetching server descriptors (in which case we should just
  603. * return,) or because all the authorities are down or on fire or
  604. * unreachable or something (in which case we should go on with
  605. * our fallback code)? */
  606. pds_flags &= ~(PDS_NO_EXISTING_SERVERDESC_FETCH|
  607. PDS_NO_EXISTING_MICRODESC_FETCH);
  608. rs = router_pick_trusteddirserver(type, pds_flags);
  609. if (rs) {
  610. log_debug(LD_DIR, "Deferring serverdesc fetch: all authorities "
  611. "are in use.");
  612. return;
  613. }
  614. }
  615. if (rs == NULL && require_authority) {
  616. log_info(LD_DIR, "No authorities were available for %s: will try "
  617. "later.", dir_conn_purpose_to_string(dir_purpose));
  618. return;
  619. }
  620. }
  621. if (!rs && !(type & BRIDGE_DIRINFO)) {
  622. rs = directory_pick_generic_dirserver(type, pds_flags,
  623. dir_purpose,
  624. &guard_state);
  625. if (!rs)
  626. get_via_tor = 1; /* last resort: try routing it via Tor */
  627. }
  628. }
  629. }
  630. if (get_via_tor) {
  631. /* Never use fascistfirewall; we're going via Tor. */
  632. pds_flags |= PDS_IGNORE_FASCISTFIREWALL;
  633. rs = router_pick_directory_server(type, pds_flags);
  634. }
  635. /* If we have any hope of building an indirect conn, we know some router
  636. * descriptors. If (rs==NULL), we can't build circuits anyway, so
  637. * there's no point in falling back to the authorities in this case. */
  638. if (rs) {
  639. const dir_indirection_t indirection =
  640. get_via_tor ? DIRIND_ANONYMOUS : DIRIND_ONEHOP;
  641. directory_request_t *req = directory_request_new(dir_purpose);
  642. directory_request_set_routerstatus(req, rs);
  643. directory_request_set_router_purpose(req, router_purpose);
  644. directory_request_set_indirection(req, indirection);
  645. directory_request_set_resource(req, resource);
  646. if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS)
  647. dir_consensus_request_set_additional_headers(req, resource);
  648. if (guard_state)
  649. directory_request_set_guard_state(req, guard_state);
  650. directory_initiate_request(req);
  651. directory_request_free(req);
  652. } else {
  653. log_notice(LD_DIR,
  654. "While fetching directory info, "
  655. "no running dirservers known. Will try again later. "
  656. "(purpose %d)", dir_purpose);
  657. if (!purpose_needs_anonymity(dir_purpose, router_purpose, resource)) {
  658. /* remember we tried them all and failed. */
  659. directory_all_unreachable(time(NULL));
  660. }
  661. }
  662. }
  663. /** As directory_get_from_dirserver, but initiates a request to <i>every</i>
  664. * directory authority other than ourself. Only for use by authorities when
  665. * searching for missing information while voting. */
  666. void
  667. directory_get_from_all_authorities(uint8_t dir_purpose,
  668. uint8_t router_purpose,
  669. const char *resource)
  670. {
  671. tor_assert(dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE ||
  672. dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES);
  673. SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
  674. dir_server_t *, ds) {
  675. if (router_digest_is_me(ds->digest))
  676. continue;
  677. if (!(ds->type & V3_DIRINFO))
  678. continue;
  679. const routerstatus_t *rs = &ds->fake_status;
  680. directory_request_t *req = directory_request_new(dir_purpose);
  681. directory_request_set_routerstatus(req, rs);
  682. directory_request_set_router_purpose(req, router_purpose);
  683. directory_request_set_resource(req, resource);
  684. directory_initiate_request(req);
  685. directory_request_free(req);
  686. } SMARTLIST_FOREACH_END(ds);
  687. }
  688. /** Return true iff <b>ind</b> requires a multihop circuit. */
  689. static int
  690. dirind_is_anon(dir_indirection_t ind)
  691. {
  692. return ind == DIRIND_ANON_DIRPORT || ind == DIRIND_ANONYMOUS;
  693. }
  694. /* Choose reachable OR and Dir addresses and ports from status, copying them
  695. * into use_or_ap and use_dir_ap. If indirection is anonymous, then we're
  696. * connecting via another relay, so choose the primary IPv4 address and ports.
  697. *
  698. * status should have at least one reachable address, if we can't choose a
  699. * reachable address, warn and return -1. Otherwise, return 0.
  700. */
  701. static int
  702. directory_choose_address_routerstatus(const routerstatus_t *status,
  703. dir_indirection_t indirection,
  704. tor_addr_port_t *use_or_ap,
  705. tor_addr_port_t *use_dir_ap)
  706. {
  707. tor_assert(status != NULL);
  708. tor_assert(use_or_ap != NULL);
  709. tor_assert(use_dir_ap != NULL);
  710. const or_options_t *options = get_options();
  711. int have_or = 0, have_dir = 0;
  712. /* We expect status to have at least one reachable address if we're
  713. * connecting to it directly.
  714. *
  715. * Therefore, we can simply use the other address if the one we want isn't
  716. * allowed by the firewall.
  717. *
  718. * (When Tor uploads and downloads a hidden service descriptor, it uses
  719. * DIRIND_ANONYMOUS, except for Tor2Web, which uses DIRIND_ONEHOP.
  720. * So this code will only modify the address for Tor2Web's HS descriptor
  721. * fetches. Even Single Onion Servers (NYI) use DIRIND_ANONYMOUS, to avoid
  722. * HSDirs denying service by rejecting descriptors.)
  723. */
  724. /* Initialise the OR / Dir addresses */
  725. tor_addr_make_null(&use_or_ap->addr, AF_UNSPEC);
  726. use_or_ap->port = 0;
  727. tor_addr_make_null(&use_dir_ap->addr, AF_UNSPEC);
  728. use_dir_ap->port = 0;
  729. /* ORPort connections */
  730. if (indirection == DIRIND_ANONYMOUS) {
  731. if (status->addr) {
  732. /* Since we're going to build a 3-hop circuit and ask the 2nd relay
  733. * to extend to this address, always use the primary (IPv4) OR address */
  734. tor_addr_from_ipv4h(&use_or_ap->addr, status->addr);
  735. use_or_ap->port = status->or_port;
  736. have_or = 1;
  737. }
  738. } else if (indirection == DIRIND_ONEHOP) {
  739. /* We use an IPv6 address if we have one and we prefer it.
  740. * Use the preferred address and port if they are reachable, otherwise,
  741. * use the alternate address and port (if any).
  742. */
  743. have_or = fascist_firewall_choose_address_rs(status,
  744. FIREWALL_OR_CONNECTION, 0,
  745. use_or_ap);
  746. }
  747. /* DirPort connections
  748. * DIRIND_ONEHOP uses ORPort, but may fall back to the DirPort on relays */
  749. if (indirection == DIRIND_DIRECT_CONN ||
  750. indirection == DIRIND_ANON_DIRPORT ||
  751. (indirection == DIRIND_ONEHOP
  752. && !directory_must_use_begindir(options))) {
  753. have_dir = fascist_firewall_choose_address_rs(status,
  754. FIREWALL_DIR_CONNECTION, 0,
  755. use_dir_ap);
  756. }
  757. /* We rejected all addresses in the relay's status. This means we can't
  758. * connect to it. */
  759. if (!have_or && !have_dir) {
  760. static int logged_backtrace = 0;
  761. log_info(LD_BUG, "Rejected all OR and Dir addresses from %s when "
  762. "launching an outgoing directory connection to: IPv4 %s OR %d "
  763. "Dir %d IPv6 %s OR %d Dir %d", routerstatus_describe(status),
  764. fmt_addr32(status->addr), status->or_port,
  765. status->dir_port, fmt_addr(&status->ipv6_addr),
  766. status->ipv6_orport, status->dir_port);
  767. if (!logged_backtrace) {
  768. log_backtrace(LOG_INFO, LD_BUG, "Addresses came from");
  769. logged_backtrace = 1;
  770. }
  771. return -1;
  772. }
  773. return 0;
  774. }
  775. /** Return true iff <b>conn</b> is the client side of a directory connection
  776. * we launched to ourself in order to determine the reachability of our
  777. * dir_port. */
  778. static int
  779. directory_conn_is_self_reachability_test(dir_connection_t *conn)
  780. {
  781. if (conn->requested_resource &&
  782. !strcmpstart(conn->requested_resource,"authority")) {
  783. const routerinfo_t *me = router_get_my_routerinfo();
  784. if (me &&
  785. router_digest_is_me(conn->identity_digest) &&
  786. tor_addr_eq_ipv4h(&conn->base_.addr, me->addr) && /*XXXX prop 118*/
  787. me->dir_port == conn->base_.port)
  788. return 1;
  789. }
  790. return 0;
  791. }
  792. /** Called when we are unable to complete the client's request to a directory
  793. * server due to a network error: Mark the router as down and try again if
  794. * possible.
  795. */
  796. static void
  797. connection_dir_request_failed(dir_connection_t *conn)
  798. {
  799. if (conn->guard_state) {
  800. /* We haven't seen a success on this guard state, so consider it to have
  801. * failed. */
  802. entry_guard_failed(&conn->guard_state);
  803. }
  804. if (directory_conn_is_self_reachability_test(conn)) {
  805. return; /* this was a test fetch. don't retry. */
  806. }
  807. if (!entry_list_is_constrained(get_options()))
  808. router_set_status(conn->identity_digest, 0); /* don't try this one again */
  809. if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
  810. conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
  811. log_info(LD_DIR, "Giving up on serverdesc/extrainfo fetch from "
  812. "directory server at '%s'; retrying",
  813. conn->base_.address);
  814. if (conn->router_purpose == ROUTER_PURPOSE_BRIDGE)
  815. connection_dir_bridge_routerdesc_failed(conn);
  816. connection_dir_download_routerdesc_failed(conn);
  817. } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
  818. if (conn->requested_resource)
  819. networkstatus_consensus_download_failed(0, conn->requested_resource);
  820. } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
  821. log_info(LD_DIR, "Giving up on certificate fetch from directory server "
  822. "at '%s'; retrying",
  823. conn->base_.address);
  824. connection_dir_download_cert_failed(conn, 0);
  825. } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) {
  826. log_info(LD_DIR, "Giving up downloading detached signatures from '%s'",
  827. conn->base_.address);
  828. } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) {
  829. log_info(LD_DIR, "Giving up downloading votes from '%s'",
  830. conn->base_.address);
  831. } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC) {
  832. log_info(LD_DIR, "Giving up on downloading microdescriptors from "
  833. "directory server at '%s'; will retry", conn->base_.address);
  834. connection_dir_download_routerdesc_failed(conn);
  835. }
  836. }
  837. /** Helper: Attempt to fetch directly the descriptors of each bridge
  838. * listed in <b>failed</b>.
  839. */
  840. static void
  841. connection_dir_retry_bridges(smartlist_t *descs)
  842. {
  843. char digest[DIGEST_LEN];
  844. SMARTLIST_FOREACH(descs, const char *, cp,
  845. {
  846. if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) != DIGEST_LEN) {
  847. log_warn(LD_BUG, "Malformed fingerprint in list: %s",
  848. escaped(cp));
  849. continue;
  850. }
  851. retry_bridge_descriptor_fetch_directly(digest);
  852. });
  853. }
  854. /** Called when an attempt to download one or more router descriptors
  855. * or extra-info documents on connection <b>conn</b> failed.
  856. */
  857. static void
  858. connection_dir_download_routerdesc_failed(dir_connection_t *conn)
  859. {
  860. /* No need to increment the failure count for routerdescs, since
  861. * it's not their fault. */
  862. /* No need to relaunch descriptor downloads here: we already do it
  863. * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
  864. tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
  865. conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
  866. conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
  867. (void) conn;
  868. }
  869. /** Called when an attempt to download a bridge's routerdesc from
  870. * one of the authorities failed due to a network error. If
  871. * possible attempt to download descriptors from the bridge directly.
  872. */
  873. static void
  874. connection_dir_bridge_routerdesc_failed(dir_connection_t *conn)
  875. {
  876. smartlist_t *which = NULL;
  877. /* Requests for bridge descriptors are in the form 'fp/', so ignore
  878. anything else. */
  879. if (!conn->requested_resource || strcmpstart(conn->requested_resource,"fp/"))
  880. return;
  881. which = smartlist_new();
  882. dir_split_resource_into_fingerprints(conn->requested_resource
  883. + strlen("fp/"),
  884. which, NULL, 0);
  885. tor_assert(conn->base_.purpose != DIR_PURPOSE_FETCH_EXTRAINFO);
  886. if (smartlist_len(which)) {
  887. connection_dir_retry_bridges(which);
  888. SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
  889. }
  890. smartlist_free(which);
  891. }
  892. /** Called when an attempt to fetch a certificate fails. */
  893. static void
  894. connection_dir_download_cert_failed(dir_connection_t *conn, int status)
  895. {
  896. const char *fp_pfx = "fp/";
  897. const char *fpsk_pfx = "fp-sk/";
  898. smartlist_t *failed;
  899. tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE);
  900. if (!conn->requested_resource)
  901. return;
  902. failed = smartlist_new();
  903. /*
  904. * We have two cases download by fingerprint (resource starts
  905. * with "fp/") or download by fingerprint/signing key pair
  906. * (resource starts with "fp-sk/").
  907. */
  908. if (!strcmpstart(conn->requested_resource, fp_pfx)) {
  909. /* Download by fingerprint case */
  910. dir_split_resource_into_fingerprints(conn->requested_resource +
  911. strlen(fp_pfx),
  912. failed, NULL, DSR_HEX);
  913. SMARTLIST_FOREACH_BEGIN(failed, char *, cp) {
  914. /* Null signing key digest indicates download by fp only */
  915. authority_cert_dl_failed(cp, NULL, status);
  916. tor_free(cp);
  917. } SMARTLIST_FOREACH_END(cp);
  918. } else if (!strcmpstart(conn->requested_resource, fpsk_pfx)) {
  919. /* Download by (fp,sk) pairs */
  920. dir_split_resource_into_fingerprint_pairs(conn->requested_resource +
  921. strlen(fpsk_pfx), failed);
  922. SMARTLIST_FOREACH_BEGIN(failed, fp_pair_t *, cp) {
  923. authority_cert_dl_failed(cp->first, cp->second, status);
  924. tor_free(cp);
  925. } SMARTLIST_FOREACH_END(cp);
  926. } else {
  927. log_warn(LD_DIR,
  928. "Don't know what to do with failure for cert fetch %s",
  929. conn->requested_resource);
  930. }
  931. smartlist_free(failed);
  932. update_certificate_downloads(time(NULL));
  933. }
  934. /* Should this tor instance only use begindir for all its directory requests?
  935. */
  936. int
  937. directory_must_use_begindir(const or_options_t *options)
  938. {
  939. /* Clients, onion services, and bridges must use begindir,
  940. * relays and authorities do not have to */
  941. return !public_server_mode(options);
  942. }
  943. /** Evaluate the situation and decide if we should use an encrypted
  944. * "begindir-style" connection for this directory request.
  945. * 0) If there is no DirPort, yes.
  946. * 1) If or_port is 0, or it's a direct conn and or_port is firewalled
  947. * or we're a dir mirror, no.
  948. * 2) If we prefer to avoid begindir conns, and we're not fetching or
  949. * publishing a bridge relay descriptor, no.
  950. * 3) Else yes.
  951. * If returning 0, return in *reason why we can't use begindir.
  952. * reason must not be NULL.
  953. */
  954. static int
  955. directory_command_should_use_begindir(const or_options_t *options,
  956. const directory_request_t *req,
  957. const char **reason)
  958. {
  959. const tor_addr_t *or_addr = &req->or_addr_port.addr;
  960. //const tor_addr_t *dir_addr = &req->dir_addr_port.addr;
  961. const int or_port = req->or_addr_port.port;
  962. const int dir_port = req->dir_addr_port.port;
  963. const dir_indirection_t indirection = req->indirection;
  964. tor_assert(reason);
  965. *reason = NULL;
  966. /* Reasons why we must use begindir */
  967. if (!dir_port) {
  968. *reason = "(using begindir - directory with no DirPort)";
  969. return 1; /* We don't know a DirPort -- must begindir. */
  970. }
  971. /* Reasons why we can't possibly use begindir */
  972. if (!or_port) {
  973. *reason = "directory with unknown ORPort";
  974. return 0; /* We don't know an ORPort -- no chance. */
  975. }
  976. if (indirection == DIRIND_DIRECT_CONN ||
  977. indirection == DIRIND_ANON_DIRPORT) {
  978. *reason = "DirPort connection";
  979. return 0;
  980. }
  981. if (indirection == DIRIND_ONEHOP) {
  982. /* We're firewalled and want a direct OR connection */
  983. if (!fascist_firewall_allows_address_addr(or_addr, or_port,
  984. FIREWALL_OR_CONNECTION, 0, 0)) {
  985. *reason = "ORPort not reachable";
  986. return 0;
  987. }
  988. }
  989. /* Reasons why we want to avoid using begindir */
  990. if (indirection == DIRIND_ONEHOP) {
  991. if (!directory_must_use_begindir(options)) {
  992. *reason = "in relay mode";
  993. return 0;
  994. }
  995. }
  996. /* DIRIND_ONEHOP on a client, or DIRIND_ANONYMOUS
  997. */
  998. *reason = "(using begindir)";
  999. return 1;
  1000. }
  1001. /**
  1002. * Create and return a new directory_request_t with purpose
  1003. * <b>dir_purpose</b>.
  1004. */
  1005. directory_request_t *
  1006. directory_request_new(uint8_t dir_purpose)
  1007. {
  1008. tor_assert(dir_purpose >= DIR_PURPOSE_MIN_);
  1009. tor_assert(dir_purpose <= DIR_PURPOSE_MAX_);
  1010. tor_assert(dir_purpose != DIR_PURPOSE_SERVER);
  1011. tor_assert(dir_purpose != DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2);
  1012. tor_assert(dir_purpose != DIR_PURPOSE_HAS_FETCHED_HSDESC);
  1013. directory_request_t *result = tor_malloc_zero(sizeof(*result));
  1014. tor_addr_make_null(&result->or_addr_port.addr, AF_INET);
  1015. result->or_addr_port.port = 0;
  1016. tor_addr_make_null(&result->dir_addr_port.addr, AF_INET);
  1017. result->dir_addr_port.port = 0;
  1018. result->dir_purpose = dir_purpose;
  1019. result->router_purpose = ROUTER_PURPOSE_GENERAL;
  1020. result->indirection = DIRIND_ONEHOP;
  1021. return result;
  1022. }
  1023. /**
  1024. * Release all resources held by <b>req</b>.
  1025. */
  1026. void
  1027. directory_request_free_(directory_request_t *req)
  1028. {
  1029. if (req == NULL)
  1030. return;
  1031. config_free_lines(req->additional_headers);
  1032. tor_free(req);
  1033. }
  1034. /**
  1035. * Set the address and OR port to use for this directory request. If there is
  1036. * no OR port, we'll have to connect over the dirport. (If there are both,
  1037. * the indirection setting determines which to use.)
  1038. */
  1039. void
  1040. directory_request_set_or_addr_port(directory_request_t *req,
  1041. const tor_addr_port_t *p)
  1042. {
  1043. memcpy(&req->or_addr_port, p, sizeof(*p));
  1044. }
  1045. /**
  1046. * Set the address and dirport to use for this directory request. If there
  1047. * is no dirport, we'll have to connect over the OR port. (If there are both,
  1048. * the indirection setting determines which to use.)
  1049. */
  1050. void
  1051. directory_request_set_dir_addr_port(directory_request_t *req,
  1052. const tor_addr_port_t *p)
  1053. {
  1054. memcpy(&req->dir_addr_port, p, sizeof(*p));
  1055. }
  1056. /**
  1057. * Set the RSA identity digest of the directory to use for this directory
  1058. * request.
  1059. */
  1060. void
  1061. directory_request_set_directory_id_digest(directory_request_t *req,
  1062. const char *digest)
  1063. {
  1064. memcpy(req->digest, digest, DIGEST_LEN);
  1065. }
  1066. /**
  1067. * Set the router purpose associated with uploaded and downloaded router
  1068. * descriptors and extrainfo documents in this directory request. The purpose
  1069. * must be one of ROUTER_PURPOSE_GENERAL (the default) or
  1070. * ROUTER_PURPOSE_BRIDGE.
  1071. */
  1072. void
  1073. directory_request_set_router_purpose(directory_request_t *req,
  1074. uint8_t router_purpose)
  1075. {
  1076. tor_assert(router_purpose == ROUTER_PURPOSE_GENERAL ||
  1077. router_purpose == ROUTER_PURPOSE_BRIDGE);
  1078. // assert that it actually makes sense to set this purpose, given
  1079. // the dir_purpose.
  1080. req->router_purpose = router_purpose;
  1081. }
  1082. /**
  1083. * Set the indirection to be used for the directory request. The indirection
  1084. * parameter configures whether to connect to a DirPort or ORPort, and whether
  1085. * to anonymize the connection. DIRIND_ONEHOP (use ORPort, don't anonymize)
  1086. * is the default. See dir_indirection_t for more information.
  1087. */
  1088. void
  1089. directory_request_set_indirection(directory_request_t *req,
  1090. dir_indirection_t indirection)
  1091. {
  1092. req->indirection = indirection;
  1093. }
  1094. /**
  1095. * Set a pointer to the resource to request from a directory. Different
  1096. * request types use resources to indicate different components of their URL.
  1097. * Note that only an alias to <b>resource</b> is stored, so the
  1098. * <b>resource</b> must outlive the request.
  1099. */
  1100. void
  1101. directory_request_set_resource(directory_request_t *req,
  1102. const char *resource)
  1103. {
  1104. req->resource = resource;
  1105. }
  1106. /**
  1107. * Set a pointer to the payload to include with this directory request, along
  1108. * with its length. Note that only an alias to <b>payload</b> is stored, so
  1109. * the <b>payload</b> must outlive the request.
  1110. */
  1111. void
  1112. directory_request_set_payload(directory_request_t *req,
  1113. const char *payload,
  1114. size_t payload_len)
  1115. {
  1116. tor_assert(DIR_PURPOSE_IS_UPLOAD(req->dir_purpose));
  1117. req->payload = payload;
  1118. req->payload_len = payload_len;
  1119. }
  1120. /**
  1121. * Set an if-modified-since date to send along with the request. The
  1122. * default is 0 (meaning, send no if-modified-since header).
  1123. */
  1124. void
  1125. directory_request_set_if_modified_since(directory_request_t *req,
  1126. time_t if_modified_since)
  1127. {
  1128. req->if_modified_since = if_modified_since;
  1129. }
  1130. /** Include a header of name <b>key</b> with content <b>val</b> in the
  1131. * request. Neither may include newlines or other odd characters. Their
  1132. * ordering is not currently guaranteed.
  1133. *
  1134. * Note that, as elsewhere in this module, header keys include a trailing
  1135. * colon and space.
  1136. */
  1137. void
  1138. directory_request_add_header(directory_request_t *req,
  1139. const char *key,
  1140. const char *val)
  1141. {
  1142. config_line_prepend(&req->additional_headers, key, val);
  1143. }
  1144. /**
  1145. * Set an object containing HS data to be associated with this request. Note
  1146. * that only an alias to <b>query</b> is stored, so the <b>query</b> object
  1147. * must outlive the request.
  1148. */
  1149. void
  1150. directory_request_set_rend_query(directory_request_t *req,
  1151. const rend_data_t *query)
  1152. {
  1153. if (query) {
  1154. tor_assert(req->dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 ||
  1155. req->dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2);
  1156. }
  1157. req->rend_query = query;
  1158. }
  1159. /**
  1160. * Set an object containing HS connection identifier to be associated with
  1161. * this request. Note that only an alias to <b>ident</b> is stored, so the
  1162. * <b>ident</b> object must outlive the request.
  1163. */
  1164. void
  1165. directory_request_upload_set_hs_ident(directory_request_t *req,
  1166. const hs_ident_dir_conn_t *ident)
  1167. {
  1168. if (ident) {
  1169. tor_assert(req->dir_purpose == DIR_PURPOSE_UPLOAD_HSDESC);
  1170. }
  1171. req->hs_ident = ident;
  1172. }
  1173. /**
  1174. * Set an object containing HS connection identifier to be associated with
  1175. * this fetch request. Note that only an alias to <b>ident</b> is stored, so
  1176. * the <b>ident</b> object must outlive the request.
  1177. */
  1178. void
  1179. directory_request_fetch_set_hs_ident(directory_request_t *req,
  1180. const hs_ident_dir_conn_t *ident)
  1181. {
  1182. if (ident) {
  1183. tor_assert(req->dir_purpose == DIR_PURPOSE_FETCH_HSDESC);
  1184. }
  1185. req->hs_ident = ident;
  1186. }
  1187. /** Set a static circuit_guard_state_t object to affliate with the request in
  1188. * <b>req</b>. This object will receive notification when the attempt to
  1189. * connect to the guard either succeeds or fails. */
  1190. void
  1191. directory_request_set_guard_state(directory_request_t *req,
  1192. circuit_guard_state_t *state)
  1193. {
  1194. req->guard_state = state;
  1195. }
  1196. /**
  1197. * Internal: Return true if any information for contacting the directory in
  1198. * <b>req</b> has been set, other than by the routerstatus. */
  1199. static int
  1200. directory_request_dir_contact_info_specified(const directory_request_t *req)
  1201. {
  1202. /* We only check for ports here, since we don't use an addr unless the port
  1203. * is set */
  1204. return (req->or_addr_port.port ||
  1205. req->dir_addr_port.port ||
  1206. ! tor_digest_is_zero(req->digest));
  1207. }
  1208. /**
  1209. * Set the routerstatus to use for the directory associated with this
  1210. * request. If this option is set, then no other function to set the
  1211. * directory's address or identity should be called.
  1212. */
  1213. void
  1214. directory_request_set_routerstatus(directory_request_t *req,
  1215. const routerstatus_t *status)
  1216. {
  1217. req->routerstatus = status;
  1218. }
  1219. /**
  1220. * Helper: update the addresses, ports, and identities in <b>req</b>
  1221. * from the routerstatus object in <b>req</b>. Return 0 on success.
  1222. * On failure, warn and return -1.
  1223. */
  1224. static int
  1225. directory_request_set_dir_from_routerstatus(directory_request_t *req)
  1226. {
  1227. const routerstatus_t *status = req->routerstatus;
  1228. if (BUG(status == NULL))
  1229. return -1;
  1230. const or_options_t *options = get_options();
  1231. const node_t *node;
  1232. tor_addr_port_t use_or_ap, use_dir_ap;
  1233. const int anonymized_connection = dirind_is_anon(req->indirection);
  1234. tor_assert(status != NULL);
  1235. node = node_get_by_id(status->identity_digest);
  1236. /* XXX The below check is wrong: !node means it's not in the consensus,
  1237. * but we haven't checked if we have a descriptor for it -- and also,
  1238. * we only care about the descriptor if it's a begindir-style anonymized
  1239. * connection. */
  1240. if (!node && anonymized_connection) {
  1241. log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we "
  1242. "don't have its router descriptor.",
  1243. routerstatus_describe(status));
  1244. return -1;
  1245. }
  1246. if (options->ExcludeNodes && options->StrictNodes &&
  1247. routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) {
  1248. log_warn(LD_DIR, "Wanted to contact directory mirror %s for %s, but "
  1249. "it's in our ExcludedNodes list and StrictNodes is set. "
  1250. "Skipping. This choice might make your Tor not work.",
  1251. routerstatus_describe(status),
  1252. dir_conn_purpose_to_string(req->dir_purpose));
  1253. return -1;
  1254. }
  1255. /* At this point, if we are a client making a direct connection to a
  1256. * directory server, we have selected a server that has at least one address
  1257. * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This
  1258. * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if
  1259. * possible. (If UseBridges is set, clients always use IPv6, and prefer it
  1260. * by default.)
  1261. *
  1262. * Now choose an address that we can use to connect to the directory server.
  1263. */
  1264. if (directory_choose_address_routerstatus(status,
  1265. req->indirection, &use_or_ap,
  1266. &use_dir_ap) < 0) {
  1267. return -1;
  1268. }
  1269. directory_request_set_or_addr_port(req, &use_or_ap);
  1270. directory_request_set_dir_addr_port(req, &use_dir_ap);
  1271. directory_request_set_directory_id_digest(req, status->identity_digest);
  1272. return 0;
  1273. }
  1274. /**
  1275. * Launch the provided directory request, configured in <b>request</b>.
  1276. * After this function is called, you can free <b>request</b>.
  1277. */
  1278. MOCK_IMPL(void,
  1279. directory_initiate_request,(directory_request_t *request))
  1280. {
  1281. tor_assert(request);
  1282. if (request->routerstatus) {
  1283. tor_assert_nonfatal(
  1284. ! directory_request_dir_contact_info_specified(request));
  1285. if (directory_request_set_dir_from_routerstatus(request) < 0) {
  1286. return;
  1287. }
  1288. }
  1289. const tor_addr_port_t *or_addr_port = &request->or_addr_port;
  1290. const tor_addr_port_t *dir_addr_port = &request->dir_addr_port;
  1291. const char *digest = request->digest;
  1292. const uint8_t dir_purpose = request->dir_purpose;
  1293. const uint8_t router_purpose = request->router_purpose;
  1294. const dir_indirection_t indirection = request->indirection;
  1295. const char *resource = request->resource;
  1296. const rend_data_t *rend_query = request->rend_query;
  1297. const hs_ident_dir_conn_t *hs_ident = request->hs_ident;
  1298. circuit_guard_state_t *guard_state = request->guard_state;
  1299. tor_assert(or_addr_port->port || dir_addr_port->port);
  1300. tor_assert(digest);
  1301. dir_connection_t *conn;
  1302. const or_options_t *options = get_options();
  1303. int socket_error = 0;
  1304. const char *begindir_reason = NULL;
  1305. /* Should the connection be to a relay's OR port (and inside that we will
  1306. * send our directory request)? */
  1307. const int use_begindir =
  1308. directory_command_should_use_begindir(options, request, &begindir_reason);
  1309. /* Will the connection go via a three-hop Tor circuit? Note that this
  1310. * is separate from whether it will use_begindir. */
  1311. const int anonymized_connection = dirind_is_anon(indirection);
  1312. /* What is the address we want to make the directory request to? If
  1313. * we're making a begindir request this is the ORPort of the relay
  1314. * we're contacting; if not a begindir request, this is its DirPort.
  1315. * Note that if anonymized_connection is true, we won't be initiating
  1316. * a connection directly to this address. */
  1317. tor_addr_t addr;
  1318. tor_addr_copy(&addr, &(use_begindir ? or_addr_port : dir_addr_port)->addr);
  1319. uint16_t port = (use_begindir ? or_addr_port : dir_addr_port)->port;
  1320. log_debug(LD_DIR, "anonymized %d, use_begindir %d.",
  1321. anonymized_connection, use_begindir);
  1322. log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose));
  1323. if (purpose_needs_anonymity(dir_purpose, router_purpose, resource)) {
  1324. tor_assert(anonymized_connection ||
  1325. rend_non_anonymous_mode_enabled(options));
  1326. }
  1327. /* use encrypted begindir connections for everything except relays
  1328. * this provides better protection for directory fetches */
  1329. if (!use_begindir && directory_must_use_begindir(options)) {
  1330. log_warn(LD_BUG, "Client could not use begindir connection: %s",
  1331. begindir_reason ? begindir_reason : "(NULL)");
  1332. return;
  1333. }
  1334. /* ensure that we don't make direct connections when a SOCKS server is
  1335. * configured. */
  1336. if (!anonymized_connection && !use_begindir && !options->HTTPProxy &&
  1337. (options->Socks4Proxy || options->Socks5Proxy)) {
  1338. log_warn(LD_DIR, "Cannot connect to a directory server through a "
  1339. "SOCKS proxy!");
  1340. return;
  1341. }
  1342. /* Make sure that the destination addr and port we picked is viable. */
  1343. if (!port || tor_addr_is_null(&addr)) {
  1344. static int logged_backtrace = 0;
  1345. log_warn(LD_DIR,
  1346. "Cannot make an outgoing %sconnection without a remote %sPort.",
  1347. use_begindir ? "begindir " : "",
  1348. use_begindir ? "OR" : "Dir");
  1349. if (!logged_backtrace) {
  1350. log_backtrace(LOG_INFO, LD_BUG, "Address came from");
  1351. logged_backtrace = 1;
  1352. }
  1353. return;
  1354. }
  1355. conn = dir_connection_new(tor_addr_family(&addr));
  1356. /* set up conn so it's got all the data we need to remember */
  1357. tor_addr_copy(&conn->base_.addr, &addr);
  1358. conn->base_.port = port;
  1359. conn->base_.address = tor_addr_to_str_dup(&addr);
  1360. memcpy(conn->identity_digest, digest, DIGEST_LEN);
  1361. conn->base_.purpose = dir_purpose;
  1362. conn->router_purpose = router_purpose;
  1363. /* give it an initial state */
  1364. conn->base_.state = DIR_CONN_STATE_CONNECTING;
  1365. /* decide whether we can learn our IP address from this conn */
  1366. /* XXXX This is a bad name for this field now. */
  1367. conn->dirconn_direct = !anonymized_connection;
  1368. /* copy rendezvous data, if any */
  1369. if (rend_query) {
  1370. /* We can't have both v2 and v3+ identifier. */
  1371. tor_assert_nonfatal(!hs_ident);
  1372. conn->rend_data = rend_data_dup(rend_query);
  1373. }
  1374. if (hs_ident) {
  1375. /* We can't have both v2 and v3+ identifier. */
  1376. tor_assert_nonfatal(!rend_query);
  1377. conn->hs_ident = hs_ident_dir_conn_dup(hs_ident);
  1378. }
  1379. if (!anonymized_connection && !use_begindir) {
  1380. /* then we want to connect to dirport directly */
  1381. if (options->HTTPProxy) {
  1382. tor_addr_copy(&addr, &options->HTTPProxyAddr);
  1383. port = options->HTTPProxyPort;
  1384. }
  1385. // In this case we should not have picked a directory guard.
  1386. if (BUG(guard_state)) {
  1387. entry_guard_cancel(&guard_state);
  1388. }
  1389. switch (connection_connect(TO_CONN(conn), conn->base_.address, &addr,
  1390. port, &socket_error)) {
  1391. case -1:
  1392. connection_mark_for_close(TO_CONN(conn));
  1393. return;
  1394. case 1:
  1395. /* start flushing conn */
  1396. conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
  1397. /* fall through */
  1398. case 0:
  1399. /* queue the command on the outbuf */
  1400. directory_send_command(conn, 1, request);
  1401. connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT);
  1402. /* writable indicates finish, readable indicates broken link,
  1403. error indicates broken link in windowsland. */
  1404. }
  1405. } else {
  1406. /* We will use a Tor circuit (maybe 1-hop, maybe 3-hop, maybe with
  1407. * begindir, maybe not with begindir) */
  1408. entry_connection_t *linked_conn;
  1409. /* Anonymized tunneled connections can never share a circuit.
  1410. * One-hop directory connections can share circuits with each other
  1411. * but nothing else. */
  1412. int iso_flags = anonymized_connection ? ISO_STREAM : ISO_SESSIONGRP;
  1413. /* If it's an anonymized connection, remember the fact that we
  1414. * wanted it for later: maybe we'll want it again soon. */
  1415. if (anonymized_connection && use_begindir)
  1416. rep_hist_note_used_internal(time(NULL), 0, 1);
  1417. else if (anonymized_connection && !use_begindir)
  1418. rep_hist_note_used_port(time(NULL), conn->base_.port);
  1419. // In this case we should not have a directory guard; we'll
  1420. // get a regular guard later when we build the circuit.
  1421. if (BUG(anonymized_connection && guard_state)) {
  1422. entry_guard_cancel(&guard_state);
  1423. }
  1424. conn->guard_state = guard_state;
  1425. /* make an AP connection
  1426. * populate it and add it at the right state
  1427. * hook up both sides
  1428. */
  1429. linked_conn =
  1430. connection_ap_make_link(TO_CONN(conn),
  1431. conn->base_.address, conn->base_.port,
  1432. digest,
  1433. SESSION_GROUP_DIRCONN, iso_flags,
  1434. use_begindir, !anonymized_connection);
  1435. if (!linked_conn) {
  1436. log_warn(LD_NET,"Making tunnel to dirserver failed.");
  1437. connection_mark_for_close(TO_CONN(conn));
  1438. return;
  1439. }
  1440. if (connection_add(TO_CONN(conn)) < 0) {
  1441. log_warn(LD_NET,"Unable to add connection for link to dirserver.");
  1442. connection_mark_for_close(TO_CONN(conn));
  1443. return;
  1444. }
  1445. conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
  1446. /* queue the command on the outbuf */
  1447. directory_send_command(conn, 0, request);
  1448. connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT);
  1449. connection_start_reading(ENTRY_TO_CONN(linked_conn));
  1450. }
  1451. }
  1452. /** Return true iff anything we say on <b>conn</b> is being encrypted before
  1453. * we send it to the client/server. */
  1454. int
  1455. connection_dir_is_encrypted(const dir_connection_t *conn)
  1456. {
  1457. /* Right now it's sufficient to see if conn is or has been linked, since
  1458. * the only thing it could be linked to is an edge connection on a
  1459. * circuit, and the only way it could have been unlinked is at the edge
  1460. * connection getting closed.
  1461. */
  1462. return TO_CONN(conn)->linked;
  1463. }
  1464. /** Helper for sorting
  1465. *
  1466. * sort strings alphabetically
  1467. */
  1468. static int
  1469. compare_strs_(const void **a, const void **b)
  1470. {
  1471. const char *s1 = *a, *s2 = *b;
  1472. return strcmp(s1, s2);
  1473. }
  1474. #define CONDITIONAL_CONSENSUS_FPR_LEN 3
  1475. #if (CONDITIONAL_CONSENSUS_FPR_LEN > DIGEST_LEN)
  1476. #error "conditional consensus fingerprint length is larger than digest length"
  1477. #endif
  1478. /** Return the URL we should use for a consensus download.
  1479. *
  1480. * Use the "conditional consensus downloading" feature described in
  1481. * dir-spec.txt, i.e.
  1482. * GET .../consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>
  1483. *
  1484. * If 'resource' is provided, it is the name of a consensus flavor to request.
  1485. */
  1486. static char *
  1487. directory_get_consensus_url(const char *resource)
  1488. {
  1489. char *url = NULL;
  1490. const char *hyphen, *flavor;
  1491. if (resource==NULL || strcmp(resource, "ns")==0) {
  1492. flavor = ""; /* Request ns consensuses as "", so older servers will work*/
  1493. hyphen = "";
  1494. } else {
  1495. flavor = resource;
  1496. hyphen = "-";
  1497. }
  1498. {
  1499. char *authority_id_list;
  1500. smartlist_t *authority_digests = smartlist_new();
  1501. SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
  1502. dir_server_t *, ds) {
  1503. char *hex;
  1504. if (!(ds->type & V3_DIRINFO))
  1505. continue;
  1506. hex = tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN+1);
  1507. base16_encode(hex, 2*CONDITIONAL_CONSENSUS_FPR_LEN+1,
  1508. ds->v3_identity_digest, CONDITIONAL_CONSENSUS_FPR_LEN);
  1509. smartlist_add(authority_digests, hex);
  1510. } SMARTLIST_FOREACH_END(ds);
  1511. smartlist_sort(authority_digests, compare_strs_);
  1512. authority_id_list = smartlist_join_strings(authority_digests,
  1513. "+", 0, NULL);
  1514. tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s/%s.z",
  1515. hyphen, flavor, authority_id_list);
  1516. SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp));
  1517. smartlist_free(authority_digests);
  1518. tor_free(authority_id_list);
  1519. }
  1520. return url;
  1521. }
  1522. /**
  1523. * Copies the ipv6 from source to destination, subject to buffer size limit
  1524. * size. If decorate is true, makes sure the copied address is decorated.
  1525. */
  1526. static void
  1527. copy_ipv6_address(char* destination, const char* source, size_t len,
  1528. int decorate) {
  1529. tor_assert(destination);
  1530. tor_assert(source);
  1531. if (decorate && source[0] != '[') {
  1532. tor_snprintf(destination, len, "[%s]", source);
  1533. } else {
  1534. strlcpy(destination, source, len);
  1535. }
  1536. }
  1537. /** Queue an appropriate HTTP command for <b>request</b> on
  1538. * <b>conn</b>-\>outbuf. If <b>direct</b> is true, we're making a
  1539. * non-anonymized connection to the dirport.
  1540. */
  1541. static void
  1542. directory_send_command(dir_connection_t *conn,
  1543. const int direct,
  1544. const directory_request_t *req)
  1545. {
  1546. tor_assert(req);
  1547. const int purpose = req->dir_purpose;
  1548. const char *resource = req->resource;
  1549. const char *payload = req->payload;
  1550. const size_t payload_len = req->payload_len;
  1551. const time_t if_modified_since = req->if_modified_since;
  1552. const int anonymized_connection = dirind_is_anon(req->indirection);
  1553. char proxystring[256];
  1554. char hoststring[128];
  1555. /* NEEDS to be the same size hoststring.
  1556. Will be decorated with brackets around it if it is ipv6. */
  1557. char decorated_address[128];
  1558. smartlist_t *headers = smartlist_new();
  1559. char *url;
  1560. char *accept_encoding;
  1561. size_t url_len;
  1562. char request[8192];
  1563. size_t request_len, total_request_len = 0;
  1564. const char *httpcommand = NULL;
  1565. tor_assert(conn);
  1566. tor_assert(conn->base_.type == CONN_TYPE_DIR);
  1567. tor_free(conn->requested_resource);
  1568. if (resource)
  1569. conn->requested_resource = tor_strdup(resource);
  1570. /* decorate the ip address if it is ipv6 */
  1571. if (strchr(conn->base_.address, ':')) {
  1572. copy_ipv6_address(decorated_address, conn->base_.address,
  1573. sizeof(decorated_address), 1);
  1574. } else {
  1575. strlcpy(decorated_address, conn->base_.address, sizeof(decorated_address));
  1576. }
  1577. /* come up with a string for which Host: we want */
  1578. if (conn->base_.port == 80) {
  1579. strlcpy(hoststring, decorated_address, sizeof(hoststring));
  1580. } else {
  1581. tor_snprintf(hoststring, sizeof(hoststring), "%s:%d",
  1582. decorated_address, conn->base_.port);
  1583. }
  1584. /* Format if-modified-since */
  1585. if (if_modified_since) {
  1586. char b[RFC1123_TIME_LEN+1];
  1587. format_rfc1123_time(b, if_modified_since);
  1588. smartlist_add_asprintf(headers, "If-Modified-Since: %s\r\n", b);
  1589. }
  1590. /* come up with some proxy lines, if we're using one. */
  1591. if (direct && get_options()->HTTPProxy) {
  1592. char *base64_authenticator=NULL;
  1593. const char *authenticator = get_options()->HTTPProxyAuthenticator;
  1594. tor_snprintf(proxystring, sizeof(proxystring),"http://%s", hoststring);
  1595. if (authenticator) {
  1596. base64_authenticator = alloc_http_authenticator(authenticator);
  1597. if (!base64_authenticator)
  1598. log_warn(LD_BUG, "Encoding http authenticator failed");
  1599. }
  1600. if (base64_authenticator) {
  1601. smartlist_add_asprintf(headers,
  1602. "Proxy-Authorization: Basic %s\r\n",
  1603. base64_authenticator);
  1604. tor_free(base64_authenticator);
  1605. }
  1606. } else {
  1607. proxystring[0] = 0;
  1608. }
  1609. if (! anonymized_connection) {
  1610. /* Add Accept-Encoding. */
  1611. accept_encoding = accept_encoding_header();
  1612. smartlist_add_asprintf(headers, "Accept-Encoding: %s\r\n",
  1613. accept_encoding);
  1614. tor_free(accept_encoding);
  1615. }
  1616. /* Add additional headers, if any */
  1617. {
  1618. config_line_t *h;
  1619. for (h = req->additional_headers; h; h = h->next) {
  1620. smartlist_add_asprintf(headers, "%s%s\r\n", h->key, h->value);
  1621. }
  1622. }
  1623. switch (purpose) {
  1624. case DIR_PURPOSE_FETCH_CONSENSUS:
  1625. /* resource is optional. If present, it's a flavor name */
  1626. tor_assert(!payload);
  1627. httpcommand = "GET";
  1628. url = directory_get_consensus_url(resource);
  1629. log_info(LD_DIR, "Downloading consensus from %s using %s",
  1630. hoststring, url);
  1631. break;
  1632. case DIR_PURPOSE_FETCH_CERTIFICATE:
  1633. tor_assert(resource);
  1634. tor_assert(!payload);
  1635. httpcommand = "GET";
  1636. tor_asprintf(&url, "/tor/keys/%s", resource);
  1637. break;
  1638. case DIR_PURPOSE_FETCH_STATUS_VOTE:
  1639. tor_assert(resource);
  1640. tor_assert(!payload);
  1641. httpcommand = "GET";
  1642. tor_asprintf(&url, "/tor/status-vote/next/%s.z", resource);
  1643. break;
  1644. case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
  1645. tor_assert(!resource);
  1646. tor_assert(!payload);
  1647. httpcommand = "GET";
  1648. url = tor_strdup("/tor/status-vote/next/consensus-signatures.z");
  1649. break;
  1650. case DIR_PURPOSE_FETCH_SERVERDESC:
  1651. tor_assert(resource);
  1652. httpcommand = "GET";
  1653. tor_asprintf(&url, "/tor/server/%s", resource);
  1654. break;
  1655. case DIR_PURPOSE_FETCH_EXTRAINFO:
  1656. tor_assert(resource);
  1657. httpcommand = "GET";
  1658. tor_asprintf(&url, "/tor/extra/%s", resource);
  1659. break;
  1660. case DIR_PURPOSE_FETCH_MICRODESC:
  1661. tor_assert(resource);
  1662. httpcommand = "GET";
  1663. tor_asprintf(&url, "/tor/micro/%s", resource);
  1664. break;
  1665. case DIR_PURPOSE_UPLOAD_DIR: {
  1666. const char *why = router_get_descriptor_gen_reason();
  1667. tor_assert(!resource);
  1668. tor_assert(payload);
  1669. httpcommand = "POST";
  1670. url = tor_strdup("/tor/");
  1671. if (why) {
  1672. smartlist_add_asprintf(headers, "X-Desc-Gen-Reason: %s\r\n", why);
  1673. }
  1674. break;
  1675. }
  1676. case DIR_PURPOSE_UPLOAD_VOTE:
  1677. tor_assert(!resource);
  1678. tor_assert(payload);
  1679. httpcommand = "POST";
  1680. url = tor_strdup("/tor/post/vote");
  1681. break;
  1682. case DIR_PURPOSE_UPLOAD_SIGNATURES:
  1683. tor_assert(!resource);
  1684. tor_assert(payload);
  1685. httpcommand = "POST";
  1686. url = tor_strdup("/tor/post/consensus-signature");
  1687. break;
  1688. case DIR_PURPOSE_FETCH_RENDDESC_V2:
  1689. tor_assert(resource);
  1690. tor_assert(strlen(resource) <= REND_DESC_ID_V2_LEN_BASE32);
  1691. tor_assert(!payload);
  1692. httpcommand = "GET";
  1693. tor_asprintf(&url, "/tor/rendezvous2/%s", resource);
  1694. break;
  1695. case DIR_PURPOSE_FETCH_HSDESC:
  1696. tor_assert(resource);
  1697. tor_assert(strlen(resource) <= ED25519_BASE64_LEN);
  1698. tor_assert(!payload);
  1699. httpcommand = "GET";
  1700. tor_asprintf(&url, "/tor/hs/3/%s", resource);
  1701. break;
  1702. case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
  1703. tor_assert(!resource);
  1704. tor_assert(payload);
  1705. httpcommand = "POST";
  1706. url = tor_strdup("/tor/rendezvous2/publish");
  1707. break;
  1708. case DIR_PURPOSE_UPLOAD_HSDESC:
  1709. tor_assert(resource);
  1710. tor_assert(payload);
  1711. httpcommand = "POST";
  1712. tor_asprintf(&url, "/tor/hs/%s/publish", resource);
  1713. break;
  1714. default:
  1715. tor_assert(0);
  1716. return;
  1717. }
  1718. /* warn in the non-tunneled case */
  1719. if (direct && (strlen(proxystring) + strlen(url) >= 4096)) {
  1720. log_warn(LD_BUG,
  1721. "Squid does not like URLs longer than 4095 bytes, and this "
  1722. "one is %d bytes long: %s%s",
  1723. (int)(strlen(proxystring) + strlen(url)), proxystring, url);
  1724. }
  1725. tor_snprintf(request, sizeof(request), "%s %s", httpcommand, proxystring);
  1726. request_len = strlen(request);
  1727. total_request_len += request_len;
  1728. connection_buf_add(request, request_len, TO_CONN(conn));
  1729. url_len = strlen(url);
  1730. total_request_len += url_len;
  1731. connection_buf_add(url, url_len, TO_CONN(conn));
  1732. tor_free(url);
  1733. if (!strcmp(httpcommand, "POST") || payload) {
  1734. smartlist_add_asprintf(headers, "Content-Length: %lu\r\n",
  1735. payload ? (unsigned long)payload_len : 0);
  1736. }
  1737. {
  1738. char *header = smartlist_join_strings(headers, "", 0, NULL);
  1739. tor_snprintf(request, sizeof(request), " HTTP/1.0\r\nHost: %s\r\n%s\r\n",
  1740. hoststring, header);
  1741. tor_free(header);
  1742. }
  1743. request_len = strlen(request);
  1744. total_request_len += request_len;
  1745. connection_buf_add(request, request_len, TO_CONN(conn));
  1746. if (payload) {
  1747. /* then send the payload afterwards too */
  1748. connection_buf_add(payload, payload_len, TO_CONN(conn));
  1749. total_request_len += payload_len;
  1750. }
  1751. SMARTLIST_FOREACH(headers, char *, h, tor_free(h));
  1752. smartlist_free(headers);
  1753. log_debug(LD_DIR,
  1754. "Sent request to directory server '%s:%d': "
  1755. "(purpose: %d, request size: " U64_FORMAT ", "
  1756. "payload size: " U64_FORMAT ")",
  1757. conn->base_.address, conn->base_.port,
  1758. conn->base_.purpose,
  1759. U64_PRINTF_ARG(total_request_len),
  1760. U64_PRINTF_ARG(payload ? payload_len : 0));
  1761. }
  1762. /** Parse an HTTP request string <b>headers</b> of the form
  1763. * \verbatim
  1764. * "\%s [http[s]://]\%s HTTP/1..."
  1765. * \endverbatim
  1766. * If it's well-formed, strdup the second \%s into *<b>url</b>, and
  1767. * nul-terminate it. If the url doesn't start with "/tor/", rewrite it
  1768. * so it does. Return 0.
  1769. * Otherwise, return -1.
  1770. */
  1771. STATIC int
  1772. parse_http_url(const char *headers, char **url)
  1773. {
  1774. char *command = NULL;
  1775. if (parse_http_command(headers, &command, url) < 0) {
  1776. return -1;
  1777. }
  1778. if (strcmpstart(*url, "/tor/")) {
  1779. char *new_url = NULL;
  1780. tor_asprintf(&new_url, "/tor%s%s",
  1781. *url[0] == '/' ? "" : "/",
  1782. *url);
  1783. tor_free(*url);
  1784. *url = new_url;
  1785. }
  1786. tor_free(command);
  1787. return 0;
  1788. }
  1789. /** Parse an HTTP request line at the start of a headers string. On failure,
  1790. * return -1. On success, set *<b>command_out</b> to a copy of the HTTP
  1791. * command ("get", "post", etc), set *<b>url_out</b> to a copy of the URL, and
  1792. * return 0. */
  1793. int
  1794. parse_http_command(const char *headers, char **command_out, char **url_out)
  1795. {
  1796. const char *command, *end_of_command;
  1797. char *s, *start, *tmp;
  1798. s = (char *)eat_whitespace_no_nl(headers);
  1799. if (!*s) return -1;
  1800. command = s;
  1801. s = (char *)find_whitespace(s); /* get past GET/POST */
  1802. if (!*s) return -1;
  1803. end_of_command = s;
  1804. s = (char *)eat_whitespace_no_nl(s);
  1805. if (!*s) return -1;
  1806. start = s; /* this is the URL, assuming it's valid */
  1807. s = (char *)find_whitespace(start);
  1808. if (!*s) return -1;
  1809. /* tolerate the http[s] proxy style of putting the hostname in the url */
  1810. if (s-start >= 4 && !strcmpstart(start,"http")) {
  1811. tmp = start + 4;
  1812. if (*tmp == 's')
  1813. tmp++;
  1814. if (s-tmp >= 3 && !strcmpstart(tmp,"://")) {
  1815. tmp = strchr(tmp+3, '/');
  1816. if (tmp && tmp < s) {
  1817. log_debug(LD_DIR,"Skipping over 'http[s]://hostname/' string");
  1818. start = tmp;
  1819. }
  1820. }
  1821. }
  1822. /* Check if the header is well formed (next sequence
  1823. * should be HTTP/1.X\r\n). Assumes we're supporting 1.0? */
  1824. {
  1825. unsigned minor_ver;
  1826. char ch;
  1827. char *e = (char *)eat_whitespace_no_nl(s);
  1828. if (2 != tor_sscanf(e, "HTTP/1.%u%c", &minor_ver, &ch)) {
  1829. return -1;
  1830. }
  1831. if (ch != '\r')
  1832. return -1;
  1833. }
  1834. *url_out = tor_memdup_nulterm(start, s-start);
  1835. *command_out = tor_memdup_nulterm(command, end_of_command - command);
  1836. return 0;
  1837. }
  1838. /** Return a copy of the first HTTP header in <b>headers</b> whose key is
  1839. * <b>which</b>. The key should be given with a terminating colon and space;
  1840. * this function copies everything after, up to but not including the
  1841. * following \\r\\n. */
  1842. char *
  1843. http_get_header(const char *headers, const char *which)
  1844. {
  1845. const char *cp = headers;
  1846. while (cp) {
  1847. if (!strcasecmpstart(cp, which)) {
  1848. char *eos;
  1849. cp += strlen(which);
  1850. if ((eos = strchr(cp,'\r')))
  1851. return tor_strndup(cp, eos-cp);
  1852. else
  1853. return tor_strdup(cp);
  1854. }
  1855. cp = strchr(cp, '\n');
  1856. if (cp)
  1857. ++cp;
  1858. }
  1859. return NULL;
  1860. }
  1861. /** If <b>headers</b> indicates that a proxy was involved, then rewrite
  1862. * <b>conn</b>-\>address to describe our best guess of the address that
  1863. * originated this HTTP request. */
  1864. static void
  1865. http_set_address_origin(const char *headers, connection_t *conn)
  1866. {
  1867. char *fwd;
  1868. fwd = http_get_header(headers, "Forwarded-For: ");
  1869. if (!fwd)
  1870. fwd = http_get_header(headers, "X-Forwarded-For: ");
  1871. if (fwd) {
  1872. tor_addr_t toraddr;
  1873. if (tor_addr_parse(&toraddr,fwd) == -1 ||
  1874. tor_addr_is_internal(&toraddr,0)) {
  1875. log_debug(LD_DIR, "Ignoring local/internal IP %s", escaped(fwd));
  1876. tor_free(fwd);
  1877. return;
  1878. }
  1879. tor_free(conn->address);
  1880. conn->address = tor_strdup(fwd);
  1881. tor_free(fwd);
  1882. }
  1883. }
  1884. /** Parse an HTTP response string <b>headers</b> of the form
  1885. * \verbatim
  1886. * "HTTP/1.\%d \%d\%s\r\n...".
  1887. * \endverbatim
  1888. *
  1889. * If it's well-formed, assign the status code to *<b>code</b> and
  1890. * return 0. Otherwise, return -1.
  1891. *
  1892. * On success: If <b>date</b> is provided, set *date to the Date
  1893. * header in the http headers, or 0 if no such header is found. If
  1894. * <b>compression</b> is provided, set *<b>compression</b> to the
  1895. * compression method given in the Content-Encoding header, or 0 if no
  1896. * such header is found, or -1 if the value of the header is not
  1897. * recognized. If <b>reason</b> is provided, strdup the reason string
  1898. * into it.
  1899. */
  1900. int
  1901. parse_http_response(const char *headers, int *code, time_t *date,
  1902. compress_method_t *compression, char **reason)
  1903. {
  1904. unsigned n1, n2;
  1905. char datestr[RFC1123_TIME_LEN+1];
  1906. smartlist_t *parsed_headers;
  1907. tor_assert(headers);
  1908. tor_assert(code);
  1909. while (TOR_ISSPACE(*headers)) headers++; /* tolerate leading whitespace */
  1910. if (tor_sscanf(headers, "HTTP/1.%u %u", &n1, &n2) < 2 ||
  1911. (n1 != 0 && n1 != 1) ||
  1912. (n2 < 100 || n2 >= 600)) {
  1913. log_warn(LD_HTTP,"Failed to parse header %s",escaped(headers));
  1914. return -1;
  1915. }
  1916. *code = n2;
  1917. parsed_headers = smartlist_new();
  1918. smartlist_split_string(parsed_headers, headers, "\n",
  1919. SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
  1920. if (reason) {
  1921. smartlist_t *status_line_elements = smartlist_new();
  1922. tor_assert(smartlist_len(parsed_headers));
  1923. smartlist_split_string(status_line_elements,
  1924. smartlist_get(parsed_headers, 0),
  1925. " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
  1926. tor_assert(smartlist_len(status_line_elements) <= 3);
  1927. if (smartlist_len(status_line_elements) == 3) {
  1928. *reason = smartlist_get(status_line_elements, 2);
  1929. smartlist_set(status_line_elements, 2, NULL); /* Prevent free */
  1930. }
  1931. SMARTLIST_FOREACH(status_line_elements, char *, cp, tor_free(cp));
  1932. smartlist_free(status_line_elements);
  1933. }
  1934. if (date) {
  1935. *date = 0;
  1936. SMARTLIST_FOREACH(parsed_headers, const char *, s,
  1937. if (!strcmpstart(s, "Date: ")) {
  1938. strlcpy(datestr, s+6, sizeof(datestr));
  1939. /* This will do nothing on failure, so we don't need to check
  1940. the result. We shouldn't warn, since there are many other valid
  1941. date formats besides the one we use. */
  1942. parse_rfc1123_time(datestr, date);
  1943. break;
  1944. });
  1945. }
  1946. if (compression) {
  1947. const char *enc = NULL;
  1948. SMARTLIST_FOREACH(parsed_headers, const char *, s,
  1949. if (!strcmpstart(s, "Content-Encoding: ")) {
  1950. enc = s+18; break;
  1951. });
  1952. if (enc == NULL)
  1953. *compression = NO_METHOD;
  1954. else {
  1955. *compression = compression_method_get_by_name(enc);
  1956. if (*compression == UNKNOWN_METHOD)
  1957. log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.",
  1958. escaped(enc));
  1959. }
  1960. }
  1961. SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s));
  1962. smartlist_free(parsed_headers);
  1963. return 0;
  1964. }
  1965. /** Return true iff <b>body</b> doesn't start with a plausible router or
  1966. * network-status or microdescriptor opening. This is a sign of possible
  1967. * compression. */
  1968. static int
  1969. body_is_plausible(const char *body, size_t len, int purpose)
  1970. {
  1971. int i;
  1972. if (len == 0)
  1973. return 1; /* empty bodies don't need decompression */
  1974. if (len < 32)
  1975. return 0;
  1976. if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
  1977. return (!strcmpstart(body,"onion-key"));
  1978. }
  1979. if (!strcmpstart(body,"router") ||
  1980. !strcmpstart(body,"network-status"))
  1981. return 1;
  1982. for (i=0;i<32;++i) {
  1983. if (!TOR_ISPRINT(body[i]) && !TOR_ISSPACE(body[i]))
  1984. return 0;
  1985. }
  1986. return 1;
  1987. }
  1988. /** Called when we've just fetched a bunch of router descriptors in
  1989. * <b>body</b>. The list <b>which</b>, if present, holds digests for
  1990. * descriptors we requested: descriptor digests if <b>descriptor_digests</b>
  1991. * is true, or identity digests otherwise. Parse the descriptors, validate
  1992. * them, and annotate them as having purpose <b>purpose</b> and as having been
  1993. * downloaded from <b>source</b>.
  1994. *
  1995. * Return the number of routers actually added. */
  1996. static int
  1997. load_downloaded_routers(const char *body, smartlist_t *which,
  1998. int descriptor_digests,
  1999. int router_purpose,
  2000. const char *source)
  2001. {
  2002. char buf[256];
  2003. char time_buf[ISO_TIME_LEN+1];
  2004. int added = 0;
  2005. int general = router_purpose == ROUTER_PURPOSE_GENERAL;
  2006. format_iso_time(time_buf, time(NULL));
  2007. tor_assert(source);
  2008. if (tor_snprintf(buf, sizeof(buf),
  2009. "@downloaded-at %s\n"
  2010. "@source %s\n"
  2011. "%s%s%s", time_buf, escaped(source),
  2012. !general ? "@purpose " : "",
  2013. !general ? router_purpose_to_string(router_purpose) : "",
  2014. !general ? "\n" : "")<0)
  2015. return added;
  2016. added = router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
  2017. descriptor_digests, buf);
  2018. if (added && general)
  2019. control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS,
  2020. count_loading_descriptors_progress());
  2021. return added;
  2022. }
  2023. static int handle_response_fetch_certificate(dir_connection_t *,
  2024. const response_handler_args_t *);
  2025. static int handle_response_fetch_status_vote(dir_connection_t *,
  2026. const response_handler_args_t *);
  2027. static int handle_response_fetch_detached_signatures(dir_connection_t *,
  2028. const response_handler_args_t *);
  2029. static int handle_response_fetch_desc(dir_connection_t *,
  2030. const response_handler_args_t *);
  2031. static int handle_response_upload_dir(dir_connection_t *,
  2032. const response_handler_args_t *);
  2033. static int handle_response_upload_vote(dir_connection_t *,
  2034. const response_handler_args_t *);
  2035. static int handle_response_upload_signatures(dir_connection_t *,
  2036. const response_handler_args_t *);
  2037. static int handle_response_fetch_renddesc_v2(dir_connection_t *,
  2038. const response_handler_args_t *);
  2039. static int handle_response_upload_renddesc_v2(dir_connection_t *,
  2040. const response_handler_args_t *);
  2041. static int handle_response_upload_hsdesc(dir_connection_t *,
  2042. const response_handler_args_t *);
  2043. static int
  2044. dir_client_decompress_response_body(char **bodyp, size_t *bodylenp,
  2045. dir_connection_t *conn,
  2046. compress_method_t compression,
  2047. int anonymized_connection)
  2048. {
  2049. int rv = 0;
  2050. const char *body = *bodyp;
  2051. size_t body_len = *bodylenp;
  2052. int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
  2053. conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
  2054. conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
  2055. int plausible = body_is_plausible(body, body_len, conn->base_.purpose);
  2056. if (plausible && compression == NO_METHOD) {
  2057. return 0;
  2058. }
  2059. int severity = LOG_DEBUG;
  2060. char *new_body = NULL;
  2061. size_t new_len = 0;
  2062. const char *description1, *description2;
  2063. int want_to_try_both = 0;
  2064. int tried_both = 0;
  2065. compress_method_t guessed = detect_compression_method(body, body_len);
  2066. description1 = compression_method_get_human_name(compression);
  2067. if (BUG(description1 == NULL))
  2068. description1 = compression_method_get_human_name(UNKNOWN_METHOD);
  2069. if (guessed == UNKNOWN_METHOD && !plausible)
  2070. description2 = "confusing binary junk";
  2071. else
  2072. description2 = compression_method_get_human_name(guessed);
  2073. /* Tell the user if we don't believe what we're told about compression.*/
  2074. want_to_try_both = (compression == UNKNOWN_METHOD ||
  2075. guessed != compression);
  2076. if (want_to_try_both) {
  2077. severity = LOG_PROTOCOL_WARN;
  2078. }
  2079. tor_log(severity, LD_HTTP,
  2080. "HTTP body from server '%s:%d' was labeled as %s, "
  2081. "%s it seems to be %s.%s",
  2082. conn->base_.address, conn->base_.port, description1,
  2083. guessed != compression?"but":"and",
  2084. description2,
  2085. (compression>0 && guessed>0 && want_to_try_both)?
  2086. " Trying both.":"");
  2087. /* Try declared compression first if we can.
  2088. * tor_compress_supports_method() also returns true for NO_METHOD.
  2089. * Ensure that the server is not sending us data compressed using a
  2090. * compression method that is not allowed for anonymous connections. */
  2091. if (anonymized_connection &&
  2092. ! allowed_anonymous_connection_compression_method(compression)) {
  2093. warn_disallowed_anonymous_compression_method(compression);
  2094. rv = -1;
  2095. goto done;
  2096. }
  2097. if (tor_compress_supports_method(compression)) {
  2098. tor_uncompress(&new_body, &new_len, body, body_len, compression,
  2099. !allow_partial, LOG_PROTOCOL_WARN);
  2100. if (new_body) {
  2101. /* We succeeded with the declared compression method. Great! */
  2102. rv = 0;
  2103. goto done;
  2104. }
  2105. }
  2106. /* Okay, if that didn't work, and we think that it was compressed
  2107. * differently, try that. */
  2108. if (anonymized_connection &&
  2109. ! allowed_anonymous_connection_compression_method(guessed)) {
  2110. warn_disallowed_anonymous_compression_method(guessed);
  2111. rv = -1;
  2112. goto done;
  2113. }
  2114. if (tor_compress_supports_method(guessed) &&
  2115. compression != guessed) {
  2116. tor_uncompress(&new_body, &new_len, body, body_len, guessed,
  2117. !allow_partial, LOG_INFO);
  2118. tried_both = 1;
  2119. }
  2120. /* If we're pretty sure that we have a compressed directory, and
  2121. * we didn't manage to uncompress it, then warn and bail. */
  2122. if (!plausible && !new_body) {
  2123. log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
  2124. "Unable to decompress HTTP body (tried %s%s%s, server '%s:%d').",
  2125. description1,
  2126. tried_both?" and ":"",
  2127. tried_both?description2:"",
  2128. conn->base_.address, conn->base_.port);
  2129. rv = -1;
  2130. goto done;
  2131. }
  2132. done:
  2133. if (new_body) {
  2134. if (rv == 0) {
  2135. /* success! */
  2136. tor_free(*bodyp);
  2137. *bodyp = new_body;
  2138. *bodylenp = new_len;
  2139. } else {
  2140. tor_free(new_body);
  2141. }
  2142. }
  2143. return rv;
  2144. }
  2145. /** We are a client, and we've finished reading the server's
  2146. * response. Parse it and act appropriately.
  2147. *
  2148. * If we're still happy with using this directory server in the future, return
  2149. * 0. Otherwise return -1; and the caller should consider trying the request
  2150. * again.
  2151. *
  2152. * The caller will take care of marking the connection for close.
  2153. */
  2154. static int
  2155. connection_dir_client_reached_eof(dir_connection_t *conn)
  2156. {
  2157. char *body = NULL;
  2158. char *headers = NULL;
  2159. char *reason = NULL;
  2160. size_t body_len = 0;
  2161. int status_code;
  2162. time_t date_header = 0;
  2163. long apparent_skew;
  2164. compress_method_t compression;
  2165. int skewed = 0;
  2166. int rv;
  2167. int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
  2168. conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
  2169. conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
  2170. size_t received_bytes;
  2171. const int anonymized_connection =
  2172. purpose_needs_anonymity(conn->base_.purpose,
  2173. conn->router_purpose,
  2174. conn->requested_resource);
  2175. received_bytes = connection_get_inbuf_len(TO_CONN(conn));
  2176. switch (connection_fetch_from_buf_http(TO_CONN(conn),
  2177. &headers, MAX_HEADERS_SIZE,
  2178. &body, &body_len, MAX_DIR_DL_SIZE,
  2179. allow_partial)) {
  2180. case -1: /* overflow */
  2181. log_warn(LD_PROTOCOL,
  2182. "'fetch' response too large (server '%s:%d'). Closing.",
  2183. conn->base_.address, conn->base_.port);
  2184. return -1;
  2185. case 0:
  2186. log_info(LD_HTTP,
  2187. "'fetch' response not all here, but we're at eof. Closing.");
  2188. return -1;
  2189. /* case 1, fall through */
  2190. }
  2191. if (parse_http_response(headers, &status_code, &date_header,
  2192. &compression, &reason) < 0) {
  2193. log_warn(LD_HTTP,"Unparseable headers (server '%s:%d'). Closing.",
  2194. conn->base_.address, conn->base_.port);
  2195. rv = -1;
  2196. goto done;
  2197. }
  2198. if (!reason) reason = tor_strdup("[no reason given]");
  2199. tor_log(LOG_DEBUG, LD_DIR,
  2200. "Received response from directory server '%s:%d': %d %s "
  2201. "(purpose: %d, response size: " U64_FORMAT
  2202. #ifdef MEASUREMENTS_21206
  2203. ", data cells received: %d, data cells sent: %d"
  2204. #endif
  2205. ", compression: %d)",
  2206. conn->base_.address, conn->base_.port, status_code,
  2207. escaped(reason), conn->base_.purpose,
  2208. U64_PRINTF_ARG(received_bytes),
  2209. #ifdef MEASUREMENTS_21206
  2210. conn->data_cells_received, conn->data_cells_sent,
  2211. #endif
  2212. compression);
  2213. if (conn->guard_state) {
  2214. /* we count the connection as successful once we can read from it. We do
  2215. * not, however, delay use of the circuit here, since it's just for a
  2216. * one-hop directory request. */
  2217. /* XXXXprop271 note that this will not do the right thing for other
  2218. * waiting circuits that would be triggered by this circuit becoming
  2219. * complete/usable. But that's ok, I think.
  2220. */
  2221. entry_guard_succeeded(&conn->guard_state);
  2222. circuit_guard_state_free(conn->guard_state);
  2223. conn->guard_state = NULL;
  2224. }
  2225. /* now check if it's got any hints for us about our IP address. */
  2226. if (conn->dirconn_direct) {
  2227. char *guess = http_get_header(headers, X_ADDRESS_HEADER);
  2228. if (guess) {
  2229. router_new_address_suggestion(guess, conn);
  2230. tor_free(guess);
  2231. }
  2232. }
  2233. if (date_header > 0) {
  2234. /* The date header was written very soon after we sent our request,
  2235. * so compute the skew as the difference between sending the request
  2236. * and the date header. (We used to check now-date_header, but that's
  2237. * inaccurate if we spend a lot of time downloading.)
  2238. */
  2239. apparent_skew = conn->base_.timestamp_lastwritten - date_header;
  2240. if (labs(apparent_skew)>ALLOW_DIRECTORY_TIME_SKEW) {
  2241. int trusted = router_digest_is_trusted_dir(conn->identity_digest);
  2242. clock_skew_warning(TO_CONN(conn), apparent_skew, trusted, LD_HTTP,
  2243. "directory", "DIRSERV");
  2244. skewed = 1; /* don't check the recommended-versions line */
  2245. } else {
  2246. log_debug(LD_HTTP, "Time on received directory is within tolerance; "
  2247. "we are %ld seconds skewed. (That's okay.)", apparent_skew);
  2248. }
  2249. }
  2250. (void) skewed; /* skewed isn't used yet. */
  2251. if (status_code == 503) {
  2252. routerstatus_t *rs;
  2253. dir_server_t *ds;
  2254. const char *id_digest = conn->identity_digest;
  2255. log_info(LD_DIR,"Received http status code %d (%s) from server "
  2256. "'%s:%d'. I'll try again soon.",
  2257. status_code, escaped(reason), conn->base_.address,
  2258. conn->base_.port);
  2259. time_t now = approx_time();
  2260. if ((rs = router_get_mutable_consensus_status_by_id(id_digest)))
  2261. rs->last_dir_503_at = now;
  2262. if ((ds = router_get_fallback_dirserver_by_digest(id_digest)))
  2263. ds->fake_status.last_dir_503_at = now;
  2264. rv = -1;
  2265. goto done;
  2266. }
  2267. if (dir_client_decompress_response_body(&body, &body_len,
  2268. conn, compression, anonymized_connection) < 0) {
  2269. rv = -1;
  2270. goto done;
  2271. }
  2272. response_handler_args_t args;
  2273. memset(&args, 0, sizeof(args));
  2274. args.status_code = status_code;
  2275. args.reason = reason;
  2276. args.body = body;
  2277. args.body_len = body_len;
  2278. args.headers = headers;
  2279. switch (conn->base_.purpose) {
  2280. case DIR_PURPOSE_FETCH_CONSENSUS:
  2281. rv = handle_response_fetch_consensus(conn, &args);
  2282. break;
  2283. case DIR_PURPOSE_FETCH_CERTIFICATE:
  2284. rv = handle_response_fetch_certificate(conn, &args);
  2285. break;
  2286. case DIR_PURPOSE_FETCH_STATUS_VOTE:
  2287. rv = handle_response_fetch_status_vote(conn, &args);
  2288. break;
  2289. case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
  2290. rv = handle_response_fetch_detached_signatures(conn, &args);
  2291. break;
  2292. case DIR_PURPOSE_FETCH_SERVERDESC:
  2293. case DIR_PURPOSE_FETCH_EXTRAINFO:
  2294. rv = handle_response_fetch_desc(conn, &args);
  2295. break;
  2296. case DIR_PURPOSE_FETCH_MICRODESC:
  2297. rv = handle_response_fetch_microdesc(conn, &args);
  2298. break;
  2299. case DIR_PURPOSE_FETCH_RENDDESC_V2:
  2300. rv = handle_response_fetch_renddesc_v2(conn, &args);
  2301. break;
  2302. case DIR_PURPOSE_UPLOAD_DIR:
  2303. rv = handle_response_upload_dir(conn, &args);
  2304. break;
  2305. case DIR_PURPOSE_UPLOAD_SIGNATURES:
  2306. rv = handle_response_upload_signatures(conn, &args);
  2307. break;
  2308. case DIR_PURPOSE_UPLOAD_VOTE:
  2309. rv = handle_response_upload_vote(conn, &args);
  2310. break;
  2311. case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
  2312. rv = handle_response_upload_renddesc_v2(conn, &args);
  2313. break;
  2314. case DIR_PURPOSE_UPLOAD_HSDESC:
  2315. rv = handle_response_upload_hsdesc(conn, &args);
  2316. break;
  2317. case DIR_PURPOSE_FETCH_HSDESC:
  2318. rv = handle_response_fetch_hsdesc_v3(conn, &args);
  2319. break;
  2320. default:
  2321. tor_assert_nonfatal_unreached();
  2322. rv = -1;
  2323. break;
  2324. }
  2325. done:
  2326. tor_free(body);
  2327. tor_free(headers);
  2328. tor_free(reason);
  2329. return rv;
  2330. }
  2331. /**
  2332. * Handler function: processes a response to a request for a networkstatus
  2333. * consensus document by checking the consensus, storing it, and marking
  2334. * router requests as reachable.
  2335. **/
  2336. STATIC int
  2337. handle_response_fetch_consensus(dir_connection_t *conn,
  2338. const response_handler_args_t *args)
  2339. {
  2340. tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS);
  2341. const int status_code = args->status_code;
  2342. const char *body = args->body;
  2343. const size_t body_len = args->body_len;
  2344. const char *reason = args->reason;
  2345. const time_t now = approx_time();
  2346. const char *consensus;
  2347. char *new_consensus = NULL;
  2348. const char *sourcename;
  2349. int r;
  2350. const char *flavname = conn->requested_resource;
  2351. if (status_code != 200) {
  2352. int severity = (status_code == 304) ? LOG_INFO : LOG_WARN;
  2353. tor_log(severity, LD_DIR,
  2354. "Received http status code %d (%s) from server "
  2355. "'%s:%d' while fetching consensus directory.",
  2356. status_code, escaped(reason), conn->base_.address,
  2357. conn->base_.port);
  2358. networkstatus_consensus_download_failed(status_code, flavname);
  2359. return -1;
  2360. }
  2361. if (looks_like_a_consensus_diff(body, body_len)) {
  2362. /* First find our previous consensus. Maybe it's in ram, maybe not. */
  2363. cached_dir_t *cd = dirserv_get_consensus(flavname);
  2364. const char *consensus_body;
  2365. char *owned_consensus = NULL;
  2366. if (cd) {
  2367. consensus_body = cd->dir;
  2368. } else {
  2369. owned_consensus = networkstatus_read_cached_consensus(flavname);
  2370. consensus_body = owned_consensus;
  2371. }
  2372. if (!consensus_body) {
  2373. log_warn(LD_DIR, "Received a consensus diff, but we can't find "
  2374. "any %s-flavored consensus in our current cache.",flavname);
  2375. networkstatus_consensus_download_failed(0, flavname);
  2376. // XXXX if this happens too much, see below
  2377. return -1;
  2378. }
  2379. new_consensus = consensus_diff_apply(consensus_body, body);
  2380. tor_free(owned_consensus);
  2381. if (new_consensus == NULL) {
  2382. log_warn(LD_DIR, "Could not apply consensus diff received from server "
  2383. "'%s:%d'", conn->base_.address, conn->base_.port);
  2384. // XXXX If this happens too many times, we should maybe not use
  2385. // XXXX this directory for diffs any more?
  2386. networkstatus_consensus_download_failed(0, flavname);
  2387. return -1;
  2388. }
  2389. log_info(LD_DIR, "Applied consensus diff (size %d) from server "
  2390. "'%s:%d', resulting in a new consensus document (size %d).",
  2391. (int)body_len, conn->base_.address, conn->base_.port,
  2392. (int)strlen(new_consensus));
  2393. consensus = new_consensus;
  2394. sourcename = "generated based on a diff";
  2395. } else {
  2396. log_info(LD_DIR,"Received consensus directory (body size %d) from server "
  2397. "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port);
  2398. consensus = body;
  2399. sourcename = "downloaded";
  2400. }
  2401. if ((r=networkstatus_set_current_consensus(consensus, flavname, 0,
  2402. conn->identity_digest))<0) {
  2403. log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
  2404. "Unable to load %s consensus directory %s from "
  2405. "server '%s:%d'. I'll try again soon.",
  2406. flavname, sourcename, conn->base_.address, conn->base_.port);
  2407. networkstatus_consensus_download_failed(0, flavname);
  2408. tor_free(new_consensus);
  2409. return -1;
  2410. }
  2411. /* If we launched other fetches for this consensus, cancel them. */
  2412. connection_dir_close_consensus_fetches(conn, flavname);
  2413. /* update the list of routers and directory guards */
  2414. routers_update_all_from_networkstatus(now, 3);
  2415. update_microdescs_from_networkstatus(now);
  2416. directory_info_has_arrived(now, 0, 0);
  2417. if (authdir_mode_v3(get_options())) {
  2418. sr_act_post_consensus(
  2419. networkstatus_get_latest_consensus_by_flavor(FLAV_NS));
  2420. }
  2421. log_info(LD_DIR, "Successfully loaded consensus.");
  2422. tor_free(new_consensus);
  2423. return 0;
  2424. }
  2425. /**
  2426. * Handler function: processes a response to a request for one or more
  2427. * authority certificates
  2428. **/
  2429. static int
  2430. handle_response_fetch_certificate(dir_connection_t *conn,
  2431. const response_handler_args_t *args)
  2432. {
  2433. tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE);
  2434. const int status_code = args->status_code;
  2435. const char *reason = args->reason;
  2436. const char *body = args->body;
  2437. const size_t body_len = args->body_len;
  2438. if (status_code != 200) {
  2439. log_warn(LD_DIR,
  2440. "Received http status code %d (%s) from server "
  2441. "'%s:%d' while fetching \"/tor/keys/%s\".",
  2442. status_code, escaped(reason), conn->base_.address,
  2443. conn->base_.port, conn->requested_resource);
  2444. connection_dir_download_cert_failed(conn, status_code);
  2445. return -1;
  2446. }
  2447. log_info(LD_DIR,"Received authority certificates (body size %d) from "
  2448. "server '%s:%d'",
  2449. (int)body_len, conn->base_.address, conn->base_.port);
  2450. /*
  2451. * Tell trusted_dirs_load_certs_from_string() whether it was by fp
  2452. * or fp-sk pair.
  2453. */
  2454. int src_code = -1;
  2455. if (!strcmpstart(conn->requested_resource, "fp/")) {
  2456. src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST;
  2457. } else if (!strcmpstart(conn->requested_resource, "fp-sk/")) {
  2458. src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST;
  2459. }
  2460. if (src_code != -1) {
  2461. if (trusted_dirs_load_certs_from_string(body, src_code, 1,
  2462. conn->identity_digest)<0) {
  2463. log_warn(LD_DIR, "Unable to parse fetched certificates");
  2464. /* if we fetched more than one and only some failed, the successful
  2465. * ones got flushed to disk so it's safe to call this on them */
  2466. connection_dir_download_cert_failed(conn, status_code);
  2467. } else {
  2468. time_t now = approx_time();
  2469. directory_info_has_arrived(now, 0, 0);
  2470. log_info(LD_DIR, "Successfully loaded certificates from fetch.");
  2471. }
  2472. } else {
  2473. log_warn(LD_DIR,
  2474. "Couldn't figure out what to do with fetched certificates for "
  2475. "unknown resource %s",
  2476. conn->requested_resource);
  2477. connection_dir_download_cert_failed(conn, status_code);
  2478. }
  2479. return 0;
  2480. }
  2481. /**
  2482. * Handler function: processes a response to a request for an authority's
  2483. * current networkstatus vote.
  2484. **/
  2485. static int
  2486. handle_response_fetch_status_vote(dir_connection_t *conn,
  2487. const response_handler_args_t *args)
  2488. {
  2489. tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE);
  2490. const int status_code = args->status_code;
  2491. const char *reason = args->reason;
  2492. const char *body = args->body;
  2493. const size_t body_len = args->body_len;
  2494. const char *msg;
  2495. int st;
  2496. log_info(LD_DIR,"Got votes (body size %d) from server %s:%d",
  2497. (int)body_len, conn->base_.address, conn->base_.port);
  2498. if (status_code != 200) {
  2499. log_warn(LD_DIR,
  2500. "Received http status code %d (%s) from server "
  2501. "'%s:%d' while fetching \"/tor/status-vote/next/%s.z\".",
  2502. status_code, escaped(reason), conn->base_.address,
  2503. conn->base_.port, conn->requested_resource);
  2504. return -1;
  2505. }
  2506. dirvote_add_vote(body, &msg, &st);
  2507. if (st > 299) {
  2508. log_warn(LD_DIR, "Error adding retrieved vote: %s", msg);
  2509. } else {
  2510. log_info(LD_DIR, "Added vote(s) successfully [msg: %s]", msg);
  2511. }
  2512. return 0;
  2513. }
  2514. /**
  2515. * Handler function: processes a response to a request for the signatures
  2516. * that an authority knows about on a given consensus.
  2517. **/
  2518. static int
  2519. handle_response_fetch_detached_signatures(dir_connection_t *conn,
  2520. const response_handler_args_t *args)
  2521. {
  2522. tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES);
  2523. const int status_code = args->status_code;
  2524. const char *reason = args->reason;
  2525. const char *body = args->body;
  2526. const size_t body_len = args->body_len;
  2527. const char *msg = NULL;
  2528. log_info(LD_DIR,"Got detached signatures (body size %d) from server %s:%d",
  2529. (int)body_len, conn->base_.address, conn->base_.port);
  2530. if (status_code != 200) {
  2531. log_warn(LD_DIR,
  2532. "Received http status code %d (%s) from server '%s:%d' while fetching "
  2533. "\"/tor/status-vote/next/consensus-signatures.z\".",
  2534. status_code, escaped(reason), conn->base_.address,
  2535. conn->base_.port);
  2536. return -1;
  2537. }
  2538. if (dirvote_add_signatures(body, conn->base_.address, &msg)<0) {
  2539. log_warn(LD_DIR, "Problem adding detached signatures from %s:%d: %s",
  2540. conn->base_.address, conn->base_.port, msg?msg:"???");
  2541. }
  2542. return 0;
  2543. }
  2544. /**
  2545. * Handler function: processes a response to a request for a group of server
  2546. * descriptors or an extrainfo documents.
  2547. **/
  2548. static int
  2549. handle_response_fetch_desc(dir_connection_t *conn,
  2550. const response_handler_args_t *args)
  2551. {
  2552. tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
  2553. conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO);
  2554. const int status_code = args->status_code;
  2555. const char *reason = args->reason;
  2556. const char *body = args->body;
  2557. const size_t body_len = args->body_len;
  2558. int was_ei = conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO;
  2559. smartlist_t *which = NULL;
  2560. int n_asked_for = 0;
  2561. int descriptor_digests = conn->requested_resource &&
  2562. !strcmpstart(conn->requested_resource,"d/");
  2563. log_info(LD_DIR,"Received %s (body size %d) from server '%s:%d'",
  2564. was_ei ? "extra server info" : "server info",
  2565. (int)body_len, conn->base_.address, conn->base_.port);
  2566. if (conn->requested_resource &&
  2567. (!strcmpstart(conn->requested_resource,"d/") ||
  2568. !strcmpstart(conn->requested_resource,"fp/"))) {
  2569. which = smartlist_new();
  2570. dir_split_resource_into_fingerprints(conn->requested_resource +
  2571. (descriptor_digests ? 2 : 3),
  2572. which, NULL, 0);
  2573. n_asked_for = smartlist_len(which);
  2574. }
  2575. if (status_code != 200) {
  2576. int dir_okay = status_code == 404 ||
  2577. (status_code == 400 && !strcmp(reason, "Servers unavailable."));
  2578. /* 404 means that it didn't have them; no big deal.
  2579. * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */
  2580. log_fn(dir_okay ? LOG_INFO : LOG_WARN, LD_DIR,
  2581. "Received http status code %d (%s) from server '%s:%d' "
  2582. "while fetching \"/tor/server/%s\". I'll try again soon.",
  2583. status_code, escaped(reason), conn->base_.address,
  2584. conn->base_.port, conn->requested_resource);
  2585. if (!which) {
  2586. connection_dir_download_routerdesc_failed(conn);
  2587. } else {
  2588. dir_routerdesc_download_failed(which, status_code,
  2589. conn->router_purpose,
  2590. was_ei, descriptor_digests);
  2591. SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
  2592. smartlist_free(which);
  2593. }
  2594. return dir_okay ? 0 : -1;
  2595. }
  2596. /* Learn the routers, assuming we requested by fingerprint or "all"
  2597. * or "authority".
  2598. *
  2599. * We use "authority" to fetch our own descriptor for
  2600. * testing, and to fetch bridge descriptors for bootstrapping. Ignore
  2601. * the output of "authority" requests unless we are using bridges,
  2602. * since otherwise they'll be the response from reachability tests,
  2603. * and we don't really want to add that to our routerlist. */
  2604. if (which || (conn->requested_resource &&
  2605. (!strcmpstart(conn->requested_resource, "all") ||
  2606. (!strcmpstart(conn->requested_resource, "authority") &&
  2607. get_options()->UseBridges)))) {
  2608. /* as we learn from them, we remove them from 'which' */
  2609. if (was_ei) {
  2610. router_load_extrainfo_from_string(body, NULL, SAVED_NOWHERE, which,
  2611. descriptor_digests);
  2612. } else {
  2613. //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
  2614. // descriptor_digests, conn->router_purpose);
  2615. if (load_downloaded_routers(body, which, descriptor_digests,
  2616. conn->router_purpose,
  2617. conn->base_.address)) {
  2618. time_t now = approx_time();
  2619. directory_info_has_arrived(now, 0, 1);
  2620. }
  2621. }
  2622. }
  2623. if (which) { /* mark remaining ones as failed */
  2624. log_info(LD_DIR, "Received %d/%d %s requested from %s:%d",
  2625. n_asked_for-smartlist_len(which), n_asked_for,
  2626. was_ei ? "extra-info documents" : "router descriptors",
  2627. conn->base_.address, (int)conn->base_.port);
  2628. if (smartlist_len(which)) {
  2629. dir_routerdesc_download_failed(which, status_code,
  2630. conn->router_purpose,
  2631. was_ei, descriptor_digests);
  2632. }
  2633. SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
  2634. smartlist_free(which);
  2635. }
  2636. if (directory_conn_is_self_reachability_test(conn))
  2637. router_dirport_found_reachable();
  2638. return 0;
  2639. }
  2640. /**
  2641. * Handler function: processes a response to a request for a group of
  2642. * microdescriptors
  2643. **/
  2644. STATIC int
  2645. handle_response_fetch_microdesc(dir_connection_t *conn,
  2646. const response_handler_args_t *args)
  2647. {
  2648. tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
  2649. const int status_code = args->status_code;
  2650. const char *reason = args->reason;
  2651. const char *body = args->body;
  2652. const size_t body_len = args->body_len;
  2653. smartlist_t *which = NULL;
  2654. log_info(LD_DIR,"Received answer to microdescriptor request (status %d, "
  2655. "body size %d) from server '%s:%d'",
  2656. status_code, (int)body_len, conn->base_.address,
  2657. conn->base_.port);
  2658. tor_assert(conn->requested_resource &&
  2659. !strcmpstart(conn->requested_resource, "d/"));
  2660. tor_assert_nonfatal(!tor_mem_is_zero(conn->identity_digest, DIGEST_LEN));
  2661. which = smartlist_new();
  2662. dir_split_resource_into_fingerprints(conn->requested_resource+2,
  2663. which, NULL,
  2664. DSR_DIGEST256|DSR_BASE64);
  2665. if (status_code != 200) {
  2666. log_info(LD_DIR, "Received status code %d (%s) from server "
  2667. "'%s:%d' while fetching \"/tor/micro/%s\". I'll try again "
  2668. "soon.",
  2669. status_code, escaped(reason), conn->base_.address,
  2670. (int)conn->base_.port, conn->requested_resource);
  2671. dir_microdesc_download_failed(which, status_code, conn->identity_digest);
  2672. SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
  2673. smartlist_free(which);
  2674. return 0;
  2675. } else {
  2676. smartlist_t *mds;
  2677. time_t now = approx_time();
  2678. mds = microdescs_add_to_cache(get_microdesc_cache(),
  2679. body, body+body_len, SAVED_NOWHERE, 0,
  2680. now, which);
  2681. if (smartlist_len(which)) {
  2682. /* Mark remaining ones as failed. */
  2683. dir_microdesc_download_failed(which, status_code, conn->identity_digest);
  2684. }
  2685. if (mds && smartlist_len(mds)) {
  2686. control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS,
  2687. count_loading_descriptors_progress());
  2688. directory_info_has_arrived(now, 0, 1);
  2689. }
  2690. SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
  2691. smartlist_free(which);
  2692. smartlist_free(mds);
  2693. }
  2694. return 0;
  2695. }
  2696. /**
  2697. * Handler function: processes a response to a POST request to upload our
  2698. * router descriptor.
  2699. **/
  2700. static int
  2701. handle_response_upload_dir(dir_connection_t *conn,
  2702. const response_handler_args_t *args)
  2703. {
  2704. tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_DIR);
  2705. const int status_code = args->status_code;
  2706. const char *reason = args->reason;
  2707. const char *headers = args->headers;
  2708. switch (status_code) {
  2709. case 200: {
  2710. dir_server_t *ds =
  2711. router_get_trusteddirserver_by_digest(conn->identity_digest);
  2712. char *rejected_hdr = http_get_header(headers,
  2713. "X-Descriptor-Not-New: ");
  2714. if (rejected_hdr) {
  2715. if (!strcmp(rejected_hdr, "Yes")) {
  2716. log_info(LD_GENERAL,
  2717. "Authority '%s' declined our descriptor (not new)",
  2718. ds->nickname);
  2719. /* XXXX use this information; be sure to upload next one
  2720. * sooner. -NM */
  2721. /* XXXX++ On further thought, the task above implies that we're
  2722. * basing our regenerate-descriptor time on when we uploaded the
  2723. * last descriptor, not on the published time of the last
  2724. * descriptor. If those are different, that's a bad thing to
  2725. * do. -NM */
  2726. }
  2727. tor_free(rejected_hdr);
  2728. }
  2729. log_info(LD_GENERAL,"eof (status 200) after uploading server "
  2730. "descriptor: finished.");
  2731. control_event_server_status(
  2732. LOG_NOTICE, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d",
  2733. conn->base_.address, conn->base_.port);
  2734. ds->has_accepted_serverdesc = 1;
  2735. if (directories_have_accepted_server_descriptor())
  2736. control_event_server_status(LOG_NOTICE, "GOOD_SERVER_DESCRIPTOR");
  2737. }
  2738. break;
  2739. case 400:
  2740. log_warn(LD_GENERAL,"http status 400 (%s) response from "
  2741. "dirserver '%s:%d'. Please correct.",
  2742. escaped(reason), conn->base_.address, conn->base_.port);
  2743. control_event_server_status(LOG_WARN,
  2744. "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"",
  2745. conn->base_.address, conn->base_.port, escaped(reason));
  2746. break;
  2747. default:
  2748. log_warn(LD_GENERAL,
  2749. "HTTP status %d (%s) was unexpected while uploading "
  2750. "descriptor to server '%s:%d'. Possibly the server is "
  2751. "misconfigured?",
  2752. status_code, escaped(reason), conn->base_.address,
  2753. conn->base_.port);
  2754. break;
  2755. }
  2756. /* return 0 in all cases, since we don't want to mark any
  2757. * dirservers down just because they don't like us. */
  2758. return 0;
  2759. }
  2760. /**
  2761. * Handler function: processes a response to POST request to upload our
  2762. * own networkstatus vote.
  2763. **/
  2764. static int
  2765. handle_response_upload_vote(dir_connection_t *conn,
  2766. const response_handler_args_t *args)
  2767. {
  2768. tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_VOTE);
  2769. const int status_code = args->status_code;
  2770. const char *reason = args->reason;
  2771. switch (status_code) {
  2772. case 200: {
  2773. log_notice(LD_DIR,"Uploaded a vote to dirserver %s:%d",
  2774. conn->base_.address, conn->base_.port);
  2775. }
  2776. break;
  2777. case 400:
  2778. log_warn(LD_DIR,"http status 400 (%s) response after uploading "
  2779. "vote to dirserver '%s:%d'. Please correct.",
  2780. escaped(reason), conn->base_.address, conn->base_.port);
  2781. break;
  2782. default:
  2783. log_warn(LD_GENERAL,
  2784. "HTTP status %d (%s) was unexpected while uploading "
  2785. "vote to server '%s:%d'.",
  2786. status_code, escaped(reason), conn->base_.address,
  2787. conn->base_.port);
  2788. break;
  2789. }
  2790. /* return 0 in all cases, since we don't want to mark any
  2791. * dirservers down just because they don't like us. */
  2792. return 0;
  2793. }
  2794. /**
  2795. * Handler function: processes a response to POST request to upload our
  2796. * view of the signatures on the current consensus.
  2797. **/
  2798. static int
  2799. handle_response_upload_signatures(dir_connection_t *conn,
  2800. const response_handler_args_t *args)
  2801. {
  2802. tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_SIGNATURES);
  2803. const int status_code = args->status_code;
  2804. const char *reason = args->reason;
  2805. switch (status_code) {
  2806. case 200: {
  2807. log_notice(LD_DIR,"Uploaded signature(s) to dirserver %s:%d",
  2808. conn->base_.address, conn->base_.port);
  2809. }
  2810. break;
  2811. case 400:
  2812. log_warn(LD_DIR,"http status 400 (%s) response after uploading "
  2813. "signatures to dirserver '%s:%d'. Please correct.",
  2814. escaped(reason), conn->base_.address, conn->base_.port);
  2815. break;
  2816. default:
  2817. log_warn(LD_GENERAL,
  2818. "HTTP status %d (%s) was unexpected while uploading "
  2819. "signatures to server '%s:%d'.",
  2820. status_code, escaped(reason), conn->base_.address,
  2821. conn->base_.port);
  2822. break;
  2823. }
  2824. /* return 0 in all cases, since we don't want to mark any
  2825. * dirservers down just because they don't like us. */
  2826. return 0;
  2827. }
  2828. /**
  2829. * Handler function: processes a response to a request for a v3 hidden service
  2830. * descriptor.
  2831. **/
  2832. STATIC int
  2833. handle_response_fetch_hsdesc_v3(dir_connection_t *conn,
  2834. const response_handler_args_t *args)
  2835. {
  2836. const int status_code = args->status_code;
  2837. const char *reason = args->reason;
  2838. const char *body = args->body;
  2839. const size_t body_len = args->body_len;
  2840. tor_assert(conn->hs_ident);
  2841. log_info(LD_REND,"Received v3 hsdesc (body size %d, status %d (%s))",
  2842. (int)body_len, status_code, escaped(reason));
  2843. switch (status_code) {
  2844. case 200:
  2845. /* We got something: Try storing it in the cache. */
  2846. if (hs_cache_store_as_client(body, &conn->hs_ident->identity_pk) < 0) {
  2847. log_warn(LD_REND, "Failed to store hidden service descriptor");
  2848. /* Fire control port FAILED event. */
  2849. hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
  2850. "BAD_DESC");
  2851. hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
  2852. NULL);
  2853. } else {
  2854. log_info(LD_REND, "Stored hidden service descriptor successfully.");
  2855. TO_CONN(conn)->purpose = DIR_PURPOSE_HAS_FETCHED_HSDESC;
  2856. hs_client_desc_has_arrived(conn->hs_ident);
  2857. /* Fire control port RECEIVED event. */
  2858. hs_control_desc_event_received(conn->hs_ident, conn->identity_digest);
  2859. hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
  2860. body);
  2861. }
  2862. break;
  2863. case 404:
  2864. /* Not there. We'll retry when connection_about_to_close_connection()
  2865. * tries to clean this conn up. */
  2866. log_info(LD_REND, "Fetching hidden service v3 descriptor not found: "
  2867. "Retrying at another directory.");
  2868. /* Fire control port FAILED event. */
  2869. hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
  2870. "NOT_FOUND");
  2871. hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
  2872. NULL);
  2873. break;
  2874. case 400:
  2875. log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
  2876. "http status 400 (%s). Dirserver didn't like our "
  2877. "query? Retrying at another directory.",
  2878. escaped(reason));
  2879. /* Fire control port FAILED event. */
  2880. hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
  2881. "QUERY_REJECTED");
  2882. hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
  2883. NULL);
  2884. break;
  2885. default:
  2886. log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
  2887. "http status %d (%s) response unexpected from HSDir server "
  2888. "'%s:%d'. Retrying at another directory.",
  2889. status_code, escaped(reason), TO_CONN(conn)->address,
  2890. TO_CONN(conn)->port);
  2891. /* Fire control port FAILED event. */
  2892. hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
  2893. "UNEXPECTED");
  2894. hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
  2895. NULL);
  2896. break;
  2897. }
  2898. return 0;
  2899. }
  2900. /**
  2901. * Handler function: processes a response to a request for a v2 hidden service
  2902. * descriptor.
  2903. **/
  2904. static int
  2905. handle_response_fetch_renddesc_v2(dir_connection_t *conn,
  2906. const response_handler_args_t *args)
  2907. {
  2908. tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2);
  2909. const int status_code = args->status_code;
  2910. const char *reason = args->reason;
  2911. const char *body = args->body;
  2912. const size_t body_len = args->body_len;
  2913. #define SEND_HS_DESC_FAILED_EVENT(reason) \
  2914. (control_event_hsv2_descriptor_failed(conn->rend_data, \
  2915. conn->identity_digest, \
  2916. reason))
  2917. #define SEND_HS_DESC_FAILED_CONTENT() \
  2918. (control_event_hs_descriptor_content( \
  2919. rend_data_get_address(conn->rend_data), \
  2920. conn->requested_resource, \
  2921. conn->identity_digest, \
  2922. NULL))
  2923. tor_assert(conn->rend_data);
  2924. log_info(LD_REND,"Received rendezvous descriptor (body size %d, status %d "
  2925. "(%s))",
  2926. (int)body_len, status_code, escaped(reason));
  2927. switch (status_code) {
  2928. case 200:
  2929. {
  2930. rend_cache_entry_t *entry = NULL;
  2931. if (rend_cache_store_v2_desc_as_client(body,
  2932. conn->requested_resource,
  2933. conn->rend_data, &entry) < 0) {
  2934. log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. "
  2935. "Retrying at another directory.");
  2936. /* We'll retry when connection_about_to_close_connection()
  2937. * cleans this dir conn up. */
  2938. SEND_HS_DESC_FAILED_EVENT("BAD_DESC");
  2939. SEND_HS_DESC_FAILED_CONTENT();
  2940. } else {
  2941. char service_id[REND_SERVICE_ID_LEN_BASE32 + 1];
  2942. /* Should never be NULL here if we found the descriptor. */
  2943. tor_assert(entry);
  2944. rend_get_service_id(entry->parsed->pk, service_id);
  2945. /* success. notify pending connections about this. */
  2946. log_info(LD_REND, "Successfully fetched v2 rendezvous "
  2947. "descriptor.");
  2948. control_event_hsv2_descriptor_received(service_id,
  2949. conn->rend_data,
  2950. conn->identity_digest);
  2951. control_event_hs_descriptor_content(service_id,
  2952. conn->requested_resource,
  2953. conn->identity_digest,
  2954. body);
  2955. conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2;
  2956. rend_client_desc_trynow(service_id);
  2957. memwipe(service_id, 0, sizeof(service_id));
  2958. }
  2959. break;
  2960. }
  2961. case 404:
  2962. /* Not there. We'll retry when
  2963. * connection_about_to_close_connection() cleans this conn up. */
  2964. log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: "
  2965. "Retrying at another directory.");
  2966. SEND_HS_DESC_FAILED_EVENT("NOT_FOUND");
  2967. SEND_HS_DESC_FAILED_CONTENT();
  2968. break;
  2969. case 400:
  2970. log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
  2971. "http status 400 (%s). Dirserver didn't like our "
  2972. "v2 rendezvous query? Retrying at another directory.",
  2973. escaped(reason));
  2974. SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED");
  2975. SEND_HS_DESC_FAILED_CONTENT();
  2976. break;
  2977. default:
  2978. log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
  2979. "http status %d (%s) response unexpected while "
  2980. "fetching v2 hidden service descriptor (server '%s:%d'). "
  2981. "Retrying at another directory.",
  2982. status_code, escaped(reason), conn->base_.address,
  2983. conn->base_.port);
  2984. SEND_HS_DESC_FAILED_EVENT("UNEXPECTED");
  2985. SEND_HS_DESC_FAILED_CONTENT();
  2986. break;
  2987. }
  2988. return 0;
  2989. }
  2990. /**
  2991. * Handler function: processes a response to a POST request to upload a v2
  2992. * hidden service descriptor.
  2993. **/
  2994. static int
  2995. handle_response_upload_renddesc_v2(dir_connection_t *conn,
  2996. const response_handler_args_t *args)
  2997. {
  2998. tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2);
  2999. const int status_code = args->status_code;
  3000. const char *reason = args->reason;
  3001. #define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason) \
  3002. (control_event_hs_descriptor_upload_failed( \
  3003. conn->identity_digest, \
  3004. rend_data_get_address(conn->rend_data), \
  3005. reason))
  3006. log_info(LD_REND,"Uploaded rendezvous descriptor (status %d "
  3007. "(%s))",
  3008. status_code, escaped(reason));
  3009. /* Without the rend data, we'll have a problem identifying what has been
  3010. * uploaded for which service. */
  3011. tor_assert(conn->rend_data);
  3012. switch (status_code) {
  3013. case 200:
  3014. log_info(LD_REND,
  3015. "Uploading rendezvous descriptor: finished with status "
  3016. "200 (%s)", escaped(reason));
  3017. control_event_hs_descriptor_uploaded(conn->identity_digest,
  3018. rend_data_get_address(conn->rend_data));
  3019. rend_service_desc_has_uploaded(conn->rend_data);
  3020. break;
  3021. case 400:
  3022. log_warn(LD_REND,"http status 400 (%s) response from dirserver "
  3023. "'%s:%d'. Malformed rendezvous descriptor?",
  3024. escaped(reason), conn->base_.address, conn->base_.port);
  3025. SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED");
  3026. break;
  3027. default:
  3028. log_warn(LD_REND,"http status %d (%s) response unexpected (server "
  3029. "'%s:%d').",
  3030. status_code, escaped(reason), conn->base_.address,
  3031. conn->base_.port);
  3032. SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED");
  3033. break;
  3034. }
  3035. return 0;
  3036. }
  3037. /**
  3038. * Handler function: processes a response to a POST request to upload an
  3039. * hidden service descriptor.
  3040. **/
  3041. static int
  3042. handle_response_upload_hsdesc(dir_connection_t *conn,
  3043. const response_handler_args_t *args)
  3044. {
  3045. const int status_code = args->status_code;
  3046. const char *reason = args->reason;
  3047. tor_assert(conn);
  3048. tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_HSDESC);
  3049. log_info(LD_REND, "Uploaded hidden service descriptor (status %d "
  3050. "(%s))",
  3051. status_code, escaped(reason));
  3052. /* For this directory response, it MUST have an hidden service identifier on
  3053. * this connection. */
  3054. tor_assert(conn->hs_ident);
  3055. switch (status_code) {
  3056. case 200:
  3057. log_info(LD_REND, "Uploading hidden service descriptor: "
  3058. "finished with status 200 (%s)", escaped(reason));
  3059. hs_control_desc_event_uploaded(conn->hs_ident, conn->identity_digest);
  3060. break;
  3061. case 400:
  3062. log_fn(LOG_PROTOCOL_WARN, LD_REND,
  3063. "Uploading hidden service descriptor: http "
  3064. "status 400 (%s) response from dirserver "
  3065. "'%s:%d'. Malformed hidden service descriptor?",
  3066. escaped(reason), conn->base_.address, conn->base_.port);
  3067. hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
  3068. "UPLOAD_REJECTED");
  3069. break;
  3070. default:
  3071. log_warn(LD_REND, "Uploading hidden service descriptor: http "
  3072. "status %d (%s) response unexpected (server "
  3073. "'%s:%d').",
  3074. status_code, escaped(reason), conn->base_.address,
  3075. conn->base_.port);
  3076. hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
  3077. "UNEXPECTED");
  3078. break;
  3079. }
  3080. return 0;
  3081. }
  3082. /** Called when a directory connection reaches EOF. */
  3083. int
  3084. connection_dir_reached_eof(dir_connection_t *conn)
  3085. {
  3086. int retval;
  3087. if (conn->base_.state != DIR_CONN_STATE_CLIENT_READING) {
  3088. log_info(LD_HTTP,"conn reached eof, not reading. [state=%d] Closing.",
  3089. conn->base_.state);
  3090. connection_close_immediate(TO_CONN(conn)); /* error: give up on flushing */
  3091. connection_mark_for_close(TO_CONN(conn));
  3092. return -1;
  3093. }
  3094. retval = connection_dir_client_reached_eof(conn);
  3095. if (retval == 0) /* success */
  3096. conn->base_.state = DIR_CONN_STATE_CLIENT_FINISHED;
  3097. connection_mark_for_close(TO_CONN(conn));
  3098. return retval;
  3099. }
  3100. /** If any directory object is arriving, and it's over 10MB large, we're
  3101. * getting DoS'd. (As of 0.1.2.x, raw directories are about 1MB, and we never
  3102. * ask for more than 96 router descriptors at a time.)
  3103. */
  3104. #define MAX_DIRECTORY_OBJECT_SIZE (10*(1<<20))
  3105. #define MAX_VOTE_DL_SIZE (MAX_DIRECTORY_OBJECT_SIZE * 5)
  3106. /** Read handler for directory connections. (That's connections <em>to</em>
  3107. * directory servers and connections <em>at</em> directory servers.)
  3108. */
  3109. int
  3110. connection_dir_process_inbuf(dir_connection_t *conn)
  3111. {
  3112. size_t max_size;
  3113. tor_assert(conn);
  3114. tor_assert(conn->base_.type == CONN_TYPE_DIR);
  3115. /* Directory clients write, then read data until they receive EOF;
  3116. * directory servers read data until they get an HTTP command, then
  3117. * write their response (when it's finished flushing, they mark for
  3118. * close).
  3119. */
  3120. /* If we're on the dirserver side, look for a command. */
  3121. if (conn->base_.state == DIR_CONN_STATE_SERVER_COMMAND_WAIT) {
  3122. if (directory_handle_command(conn) < 0) {
  3123. connection_mark_for_close(TO_CONN(conn));
  3124. return -1;
  3125. }
  3126. return 0;
  3127. }
  3128. max_size =
  3129. (TO_CONN(conn)->purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) ?
  3130. MAX_VOTE_DL_SIZE : MAX_DIRECTORY_OBJECT_SIZE;
  3131. if (connection_get_inbuf_len(TO_CONN(conn)) > max_size) {
  3132. log_warn(LD_HTTP,
  3133. "Too much data received from directory connection (%s): "
  3134. "denial of service attempt, or you need to upgrade?",
  3135. conn->base_.address);
  3136. connection_mark_for_close(TO_CONN(conn));
  3137. return -1;
  3138. }
  3139. if (!conn->base_.inbuf_reached_eof)
  3140. log_debug(LD_HTTP,"Got data, not eof. Leaving on inbuf.");
  3141. return 0;
  3142. }
  3143. /** We are closing a dir connection: If <b>dir_conn</b> is a dir connection
  3144. * that tried to fetch an HS descriptor, check if it successfully fetched it,
  3145. * or if we need to try again. */
  3146. static void
  3147. refetch_hsdesc_if_needed(dir_connection_t *dir_conn)
  3148. {
  3149. connection_t *conn = TO_CONN(dir_conn);
  3150. /* If we were trying to fetch a v2 rend desc and did not succeed, retry as
  3151. * needed. (If a fetch is successful, the connection state is changed to
  3152. * DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 or DIR_PURPOSE_HAS_FETCHED_HSDESC to
  3153. * mark that refetching is unnecessary.) */
  3154. if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 &&
  3155. dir_conn->rend_data &&
  3156. rend_valid_v2_service_id(
  3157. rend_data_get_address(dir_conn->rend_data))) {
  3158. rend_client_refetch_v2_renddesc(dir_conn->rend_data);
  3159. }
  3160. /* Check for v3 rend desc fetch */
  3161. if (conn->purpose == DIR_PURPOSE_FETCH_HSDESC &&
  3162. dir_conn->hs_ident &&
  3163. !ed25519_public_key_is_zero(&dir_conn->hs_ident->identity_pk)) {
  3164. hs_client_refetch_hsdesc(&dir_conn->hs_ident->identity_pk);
  3165. }
  3166. }
  3167. /** Called when we're about to finally unlink and free a directory connection:
  3168. * perform necessary accounting and cleanup */
  3169. void
  3170. connection_dir_about_to_close(dir_connection_t *dir_conn)
  3171. {
  3172. connection_t *conn = TO_CONN(dir_conn);
  3173. if (conn->state < DIR_CONN_STATE_CLIENT_FINISHED) {
  3174. /* It's a directory connection and connecting or fetching
  3175. * failed: forget about this router, and maybe try again. */
  3176. connection_dir_request_failed(dir_conn);
  3177. }
  3178. refetch_hsdesc_if_needed(dir_conn);
  3179. }
  3180. /** Create an http response for the client <b>conn</b> out of
  3181. * <b>status</b> and <b>reason_phrase</b>. Write it to <b>conn</b>.
  3182. */
  3183. static void
  3184. write_short_http_response(dir_connection_t *conn, int status,
  3185. const char *reason_phrase)
  3186. {
  3187. char *buf = NULL;
  3188. char *datestring = NULL;
  3189. IF_BUG_ONCE(!reason_phrase) { /* bullet-proofing */
  3190. reason_phrase = "unspecified";
  3191. }
  3192. if (server_mode(get_options())) {
  3193. /* include the Date: header, but only if we're a relay or bridge */
  3194. char datebuf[RFC1123_TIME_LEN+1];
  3195. format_rfc1123_time(datebuf, time(NULL));
  3196. tor_asprintf(&datestring, "Date: %s\r\n", datebuf);
  3197. }
  3198. tor_asprintf(&buf, "HTTP/1.0 %d %s\r\n%s\r\n",
  3199. status, reason_phrase, datestring?datestring:"");
  3200. log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase);
  3201. connection_buf_add(buf, strlen(buf), TO_CONN(conn));
  3202. tor_free(datestring);
  3203. tor_free(buf);
  3204. }
  3205. /** Write the header for an HTTP/1.0 response onto <b>conn</b>-\>outbuf,
  3206. * with <b>type</b> as the Content-Type.
  3207. *
  3208. * If <b>length</b> is nonnegative, it is the Content-Length.
  3209. * If <b>encoding</b> is provided, it is the Content-Encoding.
  3210. * If <b>cache_lifetime</b> is greater than 0, the content may be cached for
  3211. * up to cache_lifetime seconds. Otherwise, the content may not be cached. */
  3212. static void
  3213. write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
  3214. const char *type, const char *encoding,
  3215. const char *extra_headers,
  3216. long cache_lifetime)
  3217. {
  3218. char date[RFC1123_TIME_LEN+1];
  3219. time_t now = time(NULL);
  3220. buf_t *buf = buf_new_with_capacity(1024);
  3221. tor_assert(conn);
  3222. format_rfc1123_time(date, now);
  3223. buf_add_printf(buf, "HTTP/1.0 200 OK\r\nDate: %s\r\n", date);
  3224. if (type) {
  3225. buf_add_printf(buf, "Content-Type: %s\r\n", type);
  3226. }
  3227. if (!is_local_addr(&conn->base_.addr)) {
  3228. /* Don't report the source address for a nearby/private connection.
  3229. * Otherwise we tend to mis-report in cases where incoming ports are
  3230. * being forwarded to a Tor server running behind the firewall. */
  3231. buf_add_printf(buf, X_ADDRESS_HEADER "%s\r\n", conn->base_.address);
  3232. }
  3233. if (encoding) {
  3234. buf_add_printf(buf, "Content-Encoding: %s\r\n", encoding);
  3235. }
  3236. if (length >= 0) {
  3237. buf_add_printf(buf, "Content-Length: %ld\r\n", (long)length);
  3238. }
  3239. if (cache_lifetime > 0) {
  3240. char expbuf[RFC1123_TIME_LEN+1];
  3241. format_rfc1123_time(expbuf, (time_t)(now + cache_lifetime));
  3242. /* We could say 'Cache-control: max-age=%d' here if we start doing
  3243. * http/1.1 */
  3244. buf_add_printf(buf, "Expires: %s\r\n", expbuf);
  3245. } else if (cache_lifetime == 0) {
  3246. /* We could say 'Cache-control: no-cache' here if we start doing
  3247. * http/1.1 */
  3248. buf_add_string(buf, "Pragma: no-cache\r\n");
  3249. }
  3250. if (extra_headers) {
  3251. buf_add_string(buf, extra_headers);
  3252. }
  3253. buf_add_string(buf, "\r\n");
  3254. connection_buf_add_buf(TO_CONN(conn), buf);
  3255. buf_free(buf);
  3256. }
  3257. /** As write_http_response_header_impl, but sets encoding and content-typed
  3258. * based on whether the response will be <b>compressed</b> or not. */
  3259. static void
  3260. write_http_response_headers(dir_connection_t *conn, ssize_t length,
  3261. compress_method_t method,
  3262. const char *extra_headers, long cache_lifetime)
  3263. {
  3264. const char *methodname = compression_method_get_name(method);
  3265. const char *doctype;
  3266. if (method == NO_METHOD)
  3267. doctype = "text/plain";
  3268. else
  3269. doctype = "application/octet-stream";
  3270. write_http_response_header_impl(conn, length,
  3271. doctype,
  3272. methodname,
  3273. extra_headers,
  3274. cache_lifetime);
  3275. }
  3276. /** As write_http_response_headers, but assumes extra_headers is NULL */
  3277. static void
  3278. write_http_response_header(dir_connection_t *conn, ssize_t length,
  3279. compress_method_t method,
  3280. long cache_lifetime)
  3281. {
  3282. write_http_response_headers(conn, length, method, NULL, cache_lifetime);
  3283. }
  3284. /** Array of compression methods to use (if supported) for serving
  3285. * precompressed data, ordered from best to worst. */
  3286. static compress_method_t srv_meth_pref_precompressed[] = {
  3287. LZMA_METHOD,
  3288. ZSTD_METHOD,
  3289. ZLIB_METHOD,
  3290. GZIP_METHOD,
  3291. NO_METHOD
  3292. };
  3293. /** Array of compression methods to use (if supported) for serving
  3294. * streamed data, ordered from best to worst. */
  3295. static compress_method_t srv_meth_pref_streaming_compression[] = {
  3296. ZSTD_METHOD,
  3297. ZLIB_METHOD,
  3298. GZIP_METHOD,
  3299. NO_METHOD
  3300. };
  3301. /** Array of allowed compression methods to use (if supported) when receiving a
  3302. * response from a request that was required to be anonymous. */
  3303. static compress_method_t client_meth_allowed_anonymous_compression[] = {
  3304. ZLIB_METHOD,
  3305. GZIP_METHOD,
  3306. NO_METHOD
  3307. };
  3308. /** Parse the compression methods listed in an Accept-Encoding header <b>h</b>,
  3309. * and convert them to a bitfield where compression method x is supported if
  3310. * and only if 1 &lt;&lt; x is set in the bitfield. */
  3311. STATIC unsigned
  3312. parse_accept_encoding_header(const char *h)
  3313. {
  3314. unsigned result = (1u << NO_METHOD);
  3315. smartlist_t *methods = smartlist_new();
  3316. smartlist_split_string(methods, h, ",",
  3317. SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE|SPLIT_IGNORE_BLANK, 0);
  3318. SMARTLIST_FOREACH_BEGIN(methods, const char *, m) {
  3319. compress_method_t method = compression_method_get_by_name(m);
  3320. if (method != UNKNOWN_METHOD) {
  3321. tor_assert(((unsigned)method) < 8*sizeof(unsigned));
  3322. result |= (1u << method);
  3323. }
  3324. } SMARTLIST_FOREACH_END(m);
  3325. SMARTLIST_FOREACH_BEGIN(methods, char *, m) {
  3326. tor_free(m);
  3327. } SMARTLIST_FOREACH_END(m);
  3328. smartlist_free(methods);
  3329. return result;
  3330. }
  3331. /** Array of compression methods to use (if supported) for requesting
  3332. * compressed data, ordered from best to worst. */
  3333. static compress_method_t client_meth_pref[] = {
  3334. LZMA_METHOD,
  3335. ZSTD_METHOD,
  3336. ZLIB_METHOD,
  3337. GZIP_METHOD,
  3338. NO_METHOD
  3339. };
  3340. /** Return a newly allocated string containing a comma separated list of
  3341. * supported encodings. */
  3342. STATIC char *
  3343. accept_encoding_header(void)
  3344. {
  3345. smartlist_t *methods = smartlist_new();
  3346. char *header = NULL;
  3347. compress_method_t method;
  3348. unsigned i;
  3349. for (i = 0; i < ARRAY_LENGTH(client_meth_pref); ++i) {
  3350. method = client_meth_pref[i];
  3351. if (tor_compress_supports_method(method))
  3352. smartlist_add(methods, (char *)compression_method_get_name(method));
  3353. }
  3354. header = smartlist_join_strings(methods, ", ", 0, NULL);
  3355. smartlist_free(methods);
  3356. return header;
  3357. }
  3358. /** Decide whether a client would accept the consensus we have.
  3359. *
  3360. * Clients can say they only want a consensus if it's signed by more
  3361. * than half the authorities in a list. They pass this list in
  3362. * the url as "...consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>".
  3363. *
  3364. * <b>fpr</b> may be an abbreviated fingerprint, i.e. only a left substring
  3365. * of the full authority identity digest. (Only strings of even length,
  3366. * i.e. encodings of full bytes, are handled correctly. In the case
  3367. * of an odd number of hex digits the last one is silently ignored.)
  3368. *
  3369. * Returns 1 if more than half of the requested authorities signed the
  3370. * consensus, 0 otherwise.
  3371. */
  3372. int
  3373. client_likes_consensus(const struct consensus_cache_entry_t *ent,
  3374. const char *want_url)
  3375. {
  3376. smartlist_t *voters = smartlist_new();
  3377. int need_at_least;
  3378. int have = 0;
  3379. if (consensus_cache_entry_get_voter_id_digests(ent, voters) != 0) {
  3380. return 1; // We don't know the voters; assume the client won't mind. */
  3381. }
  3382. smartlist_t *want_authorities = smartlist_new();
  3383. dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0);
  3384. need_at_least = smartlist_len(want_authorities)/2+1;
  3385. SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, want_digest) {
  3386. SMARTLIST_FOREACH_BEGIN(voters, const char *, digest) {
  3387. if (!strcasecmpstart(digest, want_digest)) {
  3388. have++;
  3389. break;
  3390. };
  3391. } SMARTLIST_FOREACH_END(digest);
  3392. /* early exit, if we already have enough */
  3393. if (have >= need_at_least)
  3394. break;
  3395. } SMARTLIST_FOREACH_END(want_digest);
  3396. SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d));
  3397. smartlist_free(want_authorities);
  3398. SMARTLIST_FOREACH(voters, char *, cp, tor_free(cp));
  3399. smartlist_free(voters);
  3400. return (have >= need_at_least);
  3401. }
  3402. /** Return the compression level we should use for sending a compressed
  3403. * response of size <b>n_bytes</b>. */
  3404. STATIC compression_level_t
  3405. choose_compression_level(ssize_t n_bytes)
  3406. {
  3407. if (! have_been_under_memory_pressure()) {
  3408. return HIGH_COMPRESSION; /* we have plenty of RAM. */
  3409. } else if (n_bytes < 0) {
  3410. return HIGH_COMPRESSION; /* unknown; might be big. */
  3411. } else if (n_bytes < 1024) {
  3412. return LOW_COMPRESSION;
  3413. } else if (n_bytes < 2048) {
  3414. return MEDIUM_COMPRESSION;
  3415. } else {
  3416. return HIGH_COMPRESSION;
  3417. }
  3418. }
  3419. /** Information passed to handle a GET request. */
  3420. typedef struct get_handler_args_t {
  3421. /** Bitmask of compression methods that the client said (or implied) it
  3422. * supported. */
  3423. unsigned compression_supported;
  3424. /** If nonzero, the time included an if-modified-since header with this
  3425. * value. */
  3426. time_t if_modified_since;
  3427. /** String containing the requested URL or resource. */
  3428. const char *url;
  3429. /** String containing the HTTP headers */
  3430. const char *headers;
  3431. } get_handler_args_t;
  3432. /** Entry for handling an HTTP GET request.
  3433. *
  3434. * This entry matches a request if "string" is equal to the requested
  3435. * resource, or if "is_prefix" is true and "string" is a prefix of the
  3436. * requested resource.
  3437. *
  3438. * The 'handler' function is called to handle the request. It receives
  3439. * an arguments structure, and must return 0 on success or -1 if we should
  3440. * close the connection.
  3441. **/
  3442. typedef struct url_table_ent_s {
  3443. const char *string;
  3444. int is_prefix;
  3445. int (*handler)(dir_connection_t *conn, const get_handler_args_t *args);
  3446. } url_table_ent_t;
  3447. static int handle_get_frontpage(dir_connection_t *conn,
  3448. const get_handler_args_t *args);
  3449. static int handle_get_current_consensus(dir_connection_t *conn,
  3450. const get_handler_args_t *args);
  3451. static int handle_get_status_vote(dir_connection_t *conn,
  3452. const get_handler_args_t *args);
  3453. static int handle_get_microdesc(dir_connection_t *conn,
  3454. const get_handler_args_t *args);
  3455. static int handle_get_descriptor(dir_connection_t *conn,
  3456. const get_handler_args_t *args);
  3457. static int handle_get_keys(dir_connection_t *conn,
  3458. const get_handler_args_t *args);
  3459. static int handle_get_hs_descriptor_v2(dir_connection_t *conn,
  3460. const get_handler_args_t *args);
  3461. static int handle_get_robots(dir_connection_t *conn,
  3462. const get_handler_args_t *args);
  3463. static int handle_get_networkstatus_bridges(dir_connection_t *conn,
  3464. const get_handler_args_t *args);
  3465. /** Table for handling GET requests. */
  3466. static const url_table_ent_t url_table[] = {
  3467. { "/tor/", 0, handle_get_frontpage },
  3468. { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus },
  3469. { "/tor/status-vote/current/", 1, handle_get_status_vote },
  3470. { "/tor/status-vote/next/", 1, handle_get_status_vote },
  3471. { "/tor/micro/d/", 1, handle_get_microdesc },
  3472. { "/tor/server/", 1, handle_get_descriptor },
  3473. { "/tor/extra/", 1, handle_get_descriptor },
  3474. { "/tor/keys/", 1, handle_get_keys },
  3475. { "/tor/rendezvous2/", 1, handle_get_hs_descriptor_v2 },
  3476. { "/tor/hs/3/", 1, handle_get_hs_descriptor_v3 },
  3477. { "/tor/robots.txt", 0, handle_get_robots },
  3478. { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges },
  3479. { NULL, 0, NULL },
  3480. };
  3481. /** Helper function: called when a dirserver gets a complete HTTP GET
  3482. * request. Look for a request for a directory or for a rendezvous
  3483. * service descriptor. On finding one, write a response into
  3484. * conn-\>outbuf. If the request is unrecognized, send a 404.
  3485. * Return 0 if we handled this successfully, or -1 if we need to close
  3486. * the connection. */
  3487. MOCK_IMPL(STATIC int,
  3488. directory_handle_command_get,(dir_connection_t *conn, const char *headers,
  3489. const char *req_body, size_t req_body_len))
  3490. {
  3491. char *url, *url_mem, *header;
  3492. time_t if_modified_since = 0;
  3493. int zlib_compressed_in_url;
  3494. unsigned compression_methods_supported;
  3495. /* We ignore the body of a GET request. */
  3496. (void)req_body;
  3497. (void)req_body_len;
  3498. log_debug(LD_DIRSERV,"Received GET command.");
  3499. conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
  3500. if (parse_http_url(headers, &url) < 0) {
  3501. write_short_http_response(conn, 400, "Bad request");
  3502. return 0;
  3503. }
  3504. if ((header = http_get_header(headers, "If-Modified-Since: "))) {
  3505. struct tm tm;
  3506. if (parse_http_time(header, &tm) == 0) {
  3507. if (tor_timegm(&tm, &if_modified_since)<0) {
  3508. if_modified_since = 0;
  3509. } else {
  3510. log_debug(LD_DIRSERV, "If-Modified-Since is '%s'.", escaped(header));
  3511. }
  3512. }
  3513. /* The correct behavior on a malformed If-Modified-Since header is to
  3514. * act as if no If-Modified-Since header had been given. */
  3515. tor_free(header);
  3516. }
  3517. log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
  3518. url_mem = url;
  3519. {
  3520. size_t url_len = strlen(url);
  3521. zlib_compressed_in_url = url_len > 2 && !strcmp(url+url_len-2, ".z");
  3522. if (zlib_compressed_in_url) {
  3523. url[url_len-2] = '\0';
  3524. }
  3525. }
  3526. if ((header = http_get_header(headers, "Accept-Encoding: "))) {
  3527. compression_methods_supported = parse_accept_encoding_header(header);
  3528. tor_free(header);
  3529. } else {
  3530. compression_methods_supported = (1u << NO_METHOD);
  3531. }
  3532. if (zlib_compressed_in_url) {
  3533. compression_methods_supported |= (1u << ZLIB_METHOD);
  3534. }
  3535. /* Remove all methods that we don't both support. */
  3536. compression_methods_supported &= tor_compress_get_supported_method_bitmask();
  3537. get_handler_args_t args;
  3538. args.url = url;
  3539. args.headers = headers;
  3540. args.if_modified_since = if_modified_since;
  3541. args.compression_supported = compression_methods_supported;
  3542. int i, result = -1;
  3543. for (i = 0; url_table[i].string; ++i) {
  3544. int match;
  3545. if (url_table[i].is_prefix) {
  3546. match = !strcmpstart(url, url_table[i].string);
  3547. } else {
  3548. match = !strcmp(url, url_table[i].string);
  3549. }
  3550. if (match) {
  3551. result = url_table[i].handler(conn, &args);
  3552. goto done;
  3553. }
  3554. }
  3555. /* we didn't recognize the url */
  3556. write_short_http_response(conn, 404, "Not found");
  3557. result = 0;
  3558. done:
  3559. tor_free(url_mem);
  3560. return result;
  3561. }
  3562. /** Helper function for GET / or GET /tor/
  3563. */
  3564. static int
  3565. handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
  3566. {
  3567. (void) args; /* unused */
  3568. const char *frontpage = get_dirportfrontpage();
  3569. if (frontpage) {
  3570. size_t dlen;
  3571. dlen = strlen(frontpage);
  3572. /* Let's return a disclaimer page (users shouldn't use V1 anymore,
  3573. and caches don't fetch '/', so this is safe). */
  3574. /* [We don't check for write_bucket_low here, since we want to serve
  3575. * this page no matter what.] */
  3576. write_http_response_header_impl(conn, dlen, "text/html", "identity",
  3577. NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME);
  3578. connection_buf_add(frontpage, dlen, TO_CONN(conn));
  3579. } else {
  3580. write_short_http_response(conn, 404, "Not found");
  3581. }
  3582. return 0;
  3583. }
  3584. /** Warn that the cached consensus <b>consensus</b> of type
  3585. * <b>flavor</b> is too old and will not be served to clients. Rate-limit the
  3586. * warning to avoid logging an entry on every request.
  3587. */
  3588. static void
  3589. warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus,
  3590. const char *flavor, time_t now)
  3591. {
  3592. #define TOO_OLD_WARNING_INTERVAL (60*60)
  3593. static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL);
  3594. char timestamp[ISO_TIME_LEN+1];
  3595. time_t valid_until;
  3596. char *dupes;
  3597. if (consensus_cache_entry_get_valid_until(consensus, &valid_until))
  3598. return;
  3599. if ((dupes = rate_limit_log(&warned, now))) {
  3600. format_local_iso_time(timestamp, valid_until);
  3601. log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not "
  3602. "serve it to clients. It was valid until %s local time and we "
  3603. "continued to serve it for up to 24 hours after it expired.%s",
  3604. flavor ? flavor : "", flavor ? " " : "", timestamp, dupes);
  3605. tor_free(dupes);
  3606. }
  3607. }
  3608. /**
  3609. * Parse a single hex-encoded sha3-256 digest from <b>hex</b> into
  3610. * <b>digest</b>. Return 0 on success. On failure, report that the hash came
  3611. * from <b>location</b>, report that we are taking <b>action</b> with it, and
  3612. * return -1.
  3613. */
  3614. static int
  3615. parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location,
  3616. const char *action)
  3617. {
  3618. if (base16_decode((char*)digest, DIGEST256_LEN, hex, strlen(hex)) ==
  3619. DIGEST256_LEN) {
  3620. return 0;
  3621. } else {
  3622. log_fn(LOG_PROTOCOL_WARN, LD_DIR,
  3623. "%s contained bogus digest %s; %s.",
  3624. location, escaped(hex), action);
  3625. return -1;
  3626. }
  3627. }
  3628. /** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>,
  3629. * set <b>digest_out<b> to a new smartlist containing every 256-bit
  3630. * hex-encoded digest listed in that header and return 0. Otherwise return
  3631. * -1. */
  3632. static int
  3633. parse_or_diff_from_header(smartlist_t **digests_out, const char *headers)
  3634. {
  3635. char *hdr = http_get_header(headers, X_OR_DIFF_FROM_CONSENSUS_HEADER);
  3636. if (hdr == NULL) {
  3637. return -1;
  3638. }
  3639. smartlist_t *hex_digests = smartlist_new();
  3640. *digests_out = smartlist_new();
  3641. smartlist_split_string(hex_digests, hdr, " ",
  3642. SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
  3643. SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) {
  3644. uint8_t digest[DIGEST256_LEN];
  3645. if (!parse_one_diff_hash(digest, hex, "X-Or-Diff-From-Consensus header",
  3646. "ignoring")) {
  3647. smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest)));
  3648. }
  3649. } SMARTLIST_FOREACH_END(hex);
  3650. SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp));
  3651. smartlist_free(hex_digests);
  3652. tor_free(hdr);
  3653. return 0;
  3654. }
  3655. /** Fallback compression method. The fallback compression method is used in
  3656. * case a client requests a non-compressed document. We only store compressed
  3657. * documents, so we use this compression method to fetch the document and let
  3658. * the spooling system do the streaming decompression.
  3659. */
  3660. #define FALLBACK_COMPRESS_METHOD ZLIB_METHOD
  3661. /**
  3662. * Try to find the best consensus diff possible in order to serve a client
  3663. * request for a diff from one of the consensuses in <b>digests</b> to the
  3664. * current consensus of flavor <b>flav</b>. The client supports the
  3665. * compression methods listed in the <b>compression_methods</b> bitfield:
  3666. * place the method chosen (if any) into <b>compression_used_out</b>.
  3667. */
  3668. static struct consensus_cache_entry_t *
  3669. find_best_diff(const smartlist_t *digests, int flav,
  3670. unsigned compression_methods,
  3671. compress_method_t *compression_used_out)
  3672. {
  3673. struct consensus_cache_entry_t *result = NULL;
  3674. SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
  3675. unsigned u;
  3676. for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
  3677. compress_method_t method = srv_meth_pref_precompressed[u];
  3678. if (0 == (compression_methods & (1u<<method)))
  3679. continue; // client doesn't like this one, or we don't have it.
  3680. if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256,
  3681. diff_from, DIGEST256_LEN,
  3682. method) == CONSDIFF_AVAILABLE) {
  3683. tor_assert_nonfatal(result);
  3684. *compression_used_out = method;
  3685. return result;
  3686. }
  3687. }
  3688. } SMARTLIST_FOREACH_END(diff_from);
  3689. SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
  3690. if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, diff_from,
  3691. DIGEST256_LEN, FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
  3692. tor_assert_nonfatal(result);
  3693. *compression_used_out = FALLBACK_COMPRESS_METHOD;
  3694. return result;
  3695. }
  3696. } SMARTLIST_FOREACH_END(diff_from);
  3697. return NULL;
  3698. }
  3699. /** Lookup the cached consensus document by the flavor found in <b>flav</b>.
  3700. * The preferred set of compression methods should be listed in the
  3701. * <b>compression_methods</b> bitfield. The compression method chosen (if any)
  3702. * is stored in <b>compression_used_out</b>. */
  3703. static struct consensus_cache_entry_t *
  3704. find_best_consensus(int flav,
  3705. unsigned compression_methods,
  3706. compress_method_t *compression_used_out)
  3707. {
  3708. struct consensus_cache_entry_t *result = NULL;
  3709. unsigned u;
  3710. for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
  3711. compress_method_t method = srv_meth_pref_precompressed[u];
  3712. if (0 == (compression_methods & (1u<<method)))
  3713. continue;
  3714. if (consdiffmgr_find_consensus(&result, flav,
  3715. method) == CONSDIFF_AVAILABLE) {
  3716. tor_assert_nonfatal(result);
  3717. *compression_used_out = method;
  3718. return result;
  3719. }
  3720. }
  3721. if (consdiffmgr_find_consensus(&result, flav,
  3722. FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
  3723. tor_assert_nonfatal(result);
  3724. *compression_used_out = FALLBACK_COMPRESS_METHOD;
  3725. return result;
  3726. }
  3727. return NULL;
  3728. }
  3729. /** Try to find the best supported compression method possible from a given
  3730. * <b>compression_methods</b>. Return NO_METHOD if no mutually supported
  3731. * compression method could be found. */
  3732. static compress_method_t
  3733. find_best_compression_method(unsigned compression_methods, int stream)
  3734. {
  3735. unsigned u;
  3736. compress_method_t *methods;
  3737. size_t length;
  3738. if (stream) {
  3739. methods = srv_meth_pref_streaming_compression;
  3740. length = ARRAY_LENGTH(srv_meth_pref_streaming_compression);
  3741. } else {
  3742. methods = srv_meth_pref_precompressed;
  3743. length = ARRAY_LENGTH(srv_meth_pref_precompressed);
  3744. }
  3745. for (u = 0; u < length; ++u) {
  3746. compress_method_t method = methods[u];
  3747. if (compression_methods & (1u<<method))
  3748. return method;
  3749. }
  3750. return NO_METHOD;
  3751. }
  3752. /** Check if any of the digests in <b>digests</b> matches the latest consensus
  3753. * flavor (given in <b>flavor</b>) that we have available. */
  3754. static int
  3755. digest_list_contains_best_consensus(consensus_flavor_t flavor,
  3756. const smartlist_t *digests)
  3757. {
  3758. const networkstatus_t *ns = NULL;
  3759. if (digests == NULL)
  3760. return 0;
  3761. ns = networkstatus_get_latest_consensus_by_flavor(flavor);
  3762. if (ns == NULL)
  3763. return 0;
  3764. SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, digest) {
  3765. if (tor_memeq(ns->digest_sha3_as_signed, digest, DIGEST256_LEN))
  3766. return 1;
  3767. } SMARTLIST_FOREACH_END(digest);
  3768. return 0;
  3769. }
  3770. /** Check if the given compression method is allowed for a connection that is
  3771. * supposed to be anonymous. Returns 1 if the compression method is allowed,
  3772. * otherwise 0. */
  3773. STATIC int
  3774. allowed_anonymous_connection_compression_method(compress_method_t method)
  3775. {
  3776. unsigned u;
  3777. for (u = 0; u < ARRAY_LENGTH(client_meth_allowed_anonymous_compression);
  3778. ++u) {
  3779. compress_method_t allowed_method =
  3780. client_meth_allowed_anonymous_compression[u];
  3781. if (! tor_compress_supports_method(allowed_method))
  3782. continue;
  3783. if (method == allowed_method)
  3784. return 1;
  3785. }
  3786. return 0;
  3787. }
  3788. /** Log a warning when a remote server has sent us a document using a
  3789. * compression method that is not allowed for anonymous directory requests. */
  3790. STATIC void
  3791. warn_disallowed_anonymous_compression_method(compress_method_t method)
  3792. {
  3793. log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
  3794. "Received a %s HTTP response, which is not "
  3795. "allowed for anonymous directory requests.",
  3796. compression_method_get_human_name(method));
  3797. }
  3798. /** Encodes the results of parsing a consensus request to figure out what
  3799. * consensus, and possibly what diffs, the user asked for. */
  3800. typedef struct {
  3801. /** name of the flavor to retrieve. */
  3802. char *flavor;
  3803. /** flavor to retrive, as enum. */
  3804. consensus_flavor_t flav;
  3805. /** plus-separated list of authority fingerprints; see
  3806. * client_likes_consensus(). Aliases the URL in the request passed to
  3807. * parse_consensus_request(). */
  3808. const char *want_fps;
  3809. /** Optionally, a smartlist of sha3 digests-as-signed of the consensuses
  3810. * to return a diff from. */
  3811. smartlist_t *diff_from_digests;
  3812. /** If true, never send a full consensus. If there is no diff, send
  3813. * a 404 instead. */
  3814. int diff_only;
  3815. } parsed_consensus_request_t;
  3816. /** Remove all data held in <b>req</b>. Do not free <b>req</b> itself, since
  3817. * it is stack-allocated. */
  3818. static void
  3819. parsed_consensus_request_clear(parsed_consensus_request_t *req)
  3820. {
  3821. if (!req)
  3822. return;
  3823. tor_free(req->flavor);
  3824. if (req->diff_from_digests) {
  3825. SMARTLIST_FOREACH(req->diff_from_digests, uint8_t *, d, tor_free(d));
  3826. smartlist_free(req->diff_from_digests);
  3827. }
  3828. memset(req, 0, sizeof(parsed_consensus_request_t));
  3829. }
  3830. /**
  3831. * Parse the URL and relevant headers of <b>args</b> for a current-consensus
  3832. * request to learn what flavor of consensus we want, what keys it must be
  3833. * signed with, and what diffs we would accept (or demand) instead. Return 0
  3834. * on success and -1 on failure.
  3835. */
  3836. static int
  3837. parse_consensus_request(parsed_consensus_request_t *out,
  3838. const get_handler_args_t *args)
  3839. {
  3840. const char *url = args->url;
  3841. memset(out, 0, sizeof(parsed_consensus_request_t));
  3842. out->flav = FLAV_NS;
  3843. const char CONSENSUS_URL_PREFIX[] = "/tor/status-vote/current/consensus/";
  3844. const char CONSENSUS_FLAVORED_PREFIX[] =
  3845. "/tor/status-vote/current/consensus-";
  3846. /* figure out the flavor if any, and who we wanted to sign the thing */
  3847. const char *after_flavor = NULL;
  3848. if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) {
  3849. const char *f, *cp;
  3850. f = url + strlen(CONSENSUS_FLAVORED_PREFIX);
  3851. cp = strchr(f, '/');
  3852. if (cp) {
  3853. after_flavor = cp+1;
  3854. out->flavor = tor_strndup(f, cp-f);
  3855. } else {
  3856. out->flavor = tor_strdup(f);
  3857. }
  3858. int flav = networkstatus_parse_flavor_name(out->flavor);
  3859. if (flav < 0)
  3860. flav = FLAV_NS;
  3861. out->flav = flav;
  3862. } else {
  3863. if (!strcmpstart(url, CONSENSUS_URL_PREFIX))
  3864. after_flavor = url+strlen(CONSENSUS_URL_PREFIX);
  3865. }
  3866. /* see whether we've been asked explicitly for a diff from an older
  3867. * consensus. (The user might also have said that a diff would be okay,
  3868. * via X-Or-Diff-From-Consensus */
  3869. const char DIFF_COMPONENT[] = "diff/";
  3870. char *diff_hash_in_url = NULL;
  3871. if (after_flavor && !strcmpstart(after_flavor, DIFF_COMPONENT)) {
  3872. after_flavor += strlen(DIFF_COMPONENT);
  3873. const char *cp = strchr(after_flavor, '/');
  3874. if (cp) {
  3875. diff_hash_in_url = tor_strndup(after_flavor, cp-after_flavor);
  3876. out->want_fps = cp+1;
  3877. } else {
  3878. diff_hash_in_url = tor_strdup(after_flavor);
  3879. out->want_fps = NULL;
  3880. }
  3881. } else {
  3882. out->want_fps = after_flavor;
  3883. }
  3884. if (diff_hash_in_url) {
  3885. uint8_t diff_from[DIGEST256_LEN];
  3886. out->diff_from_digests = smartlist_new();
  3887. out->diff_only = 1;
  3888. int ok = !parse_one_diff_hash(diff_from, diff_hash_in_url, "URL",
  3889. "rejecting");
  3890. tor_free(diff_hash_in_url);
  3891. if (ok) {
  3892. smartlist_add(out->diff_from_digests,
  3893. tor_memdup(diff_from, DIGEST256_LEN));
  3894. } else {
  3895. return -1;
  3896. }
  3897. } else {
  3898. parse_or_diff_from_header(&out->diff_from_digests, args->headers);
  3899. }
  3900. return 0;
  3901. }
  3902. /** Helper function for GET /tor/status-vote/current/consensus
  3903. */
  3904. static int
  3905. handle_get_current_consensus(dir_connection_t *conn,
  3906. const get_handler_args_t *args)
  3907. {
  3908. const compress_method_t compress_method =
  3909. find_best_compression_method(args->compression_supported, 0);
  3910. const time_t if_modified_since = args->if_modified_since;
  3911. int clear_spool = 0;
  3912. /* v3 network status fetch. */
  3913. long lifetime = NETWORKSTATUS_CACHE_LIFETIME;
  3914. time_t now = time(NULL);
  3915. parsed_consensus_request_t req;
  3916. if (parse_consensus_request(&req, args) < 0) {
  3917. write_short_http_response(conn, 404, "Couldn't parse request");
  3918. goto done;
  3919. }
  3920. if (digest_list_contains_best_consensus(req.flav,
  3921. req.diff_from_digests)) {
  3922. write_short_http_response(conn, 304, "Not modified");
  3923. geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
  3924. goto done;
  3925. }
  3926. struct consensus_cache_entry_t *cached_consensus = NULL;
  3927. compress_method_t compression_used = NO_METHOD;
  3928. if (req.diff_from_digests) {
  3929. cached_consensus = find_best_diff(req.diff_from_digests, req.flav,
  3930. args->compression_supported,
  3931. &compression_used);
  3932. }
  3933. if (req.diff_only && !cached_consensus) {
  3934. write_short_http_response(conn, 404, "No such diff available");
  3935. // XXXX warn_consensus_is_too_old(v, req.flavor, now);
  3936. geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
  3937. goto done;
  3938. }
  3939. if (! cached_consensus) {
  3940. cached_consensus = find_best_consensus(req.flav,
  3941. args->compression_supported,
  3942. &compression_used);
  3943. }
  3944. time_t fresh_until, valid_until;
  3945. int have_fresh_until = 0, have_valid_until = 0;
  3946. if (cached_consensus) {
  3947. have_fresh_until =
  3948. !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until);
  3949. have_valid_until =
  3950. !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until);
  3951. }
  3952. if (cached_consensus && have_valid_until &&
  3953. !networkstatus_valid_until_is_reasonably_live(valid_until, now)) {
  3954. write_short_http_response(conn, 404, "Consensus is too old");
  3955. warn_consensus_is_too_old(cached_consensus, req.flavor, now);
  3956. geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
  3957. goto done;
  3958. }
  3959. if (cached_consensus && req.want_fps &&
  3960. !client_likes_consensus(cached_consensus, req.want_fps)) {
  3961. write_short_http_response(conn, 404, "Consensus not signed by sufficient "
  3962. "number of requested authorities");
  3963. geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS);
  3964. goto done;
  3965. }
  3966. conn->spool = smartlist_new();
  3967. clear_spool = 1;
  3968. {
  3969. spooled_resource_t *spooled;
  3970. if (cached_consensus) {
  3971. spooled = spooled_resource_new_from_cache_entry(cached_consensus);
  3972. smartlist_add(conn->spool, spooled);
  3973. }
  3974. }
  3975. lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0;
  3976. size_t size_guess = 0;
  3977. int n_expired = 0;
  3978. dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since,
  3979. compress_method != NO_METHOD,
  3980. &size_guess,
  3981. &n_expired);
  3982. if (!smartlist_len(conn->spool) && !n_expired) {
  3983. write_short_http_response(conn, 404, "Not found");
  3984. geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
  3985. goto done;
  3986. } else if (!smartlist_len(conn->spool)) {
  3987. write_short_http_response(conn, 304, "Not modified");
  3988. geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
  3989. goto done;
  3990. }
  3991. if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
  3992. log_debug(LD_DIRSERV,
  3993. "Client asked for network status lists, but we've been "
  3994. "writing too many bytes lately. Sending 503 Dir busy.");
  3995. write_short_http_response(conn, 503, "Directory busy, try again later");
  3996. geoip_note_ns_response(GEOIP_REJECT_BUSY);
  3997. goto done;
  3998. }
  3999. tor_addr_t addr;
  4000. if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) {
  4001. geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS,
  4002. &addr, NULL,
  4003. time(NULL));
  4004. geoip_note_ns_response(GEOIP_SUCCESS);
  4005. /* Note that a request for a network status has started, so that we
  4006. * can measure the download time later on. */
  4007. if (conn->dirreq_id)
  4008. geoip_start_dirreq(conn->dirreq_id, size_guess, DIRREQ_TUNNELED);
  4009. else
  4010. geoip_start_dirreq(TO_CONN(conn)->global_identifier, size_guess,
  4011. DIRREQ_DIRECT);
  4012. }
  4013. /* Use this header to tell caches that the response depends on the
  4014. * X-Or-Diff-From-Consensus header (or lack thereof). */
  4015. const char vary_header[] = "Vary: X-Or-Diff-From-Consensus\r\n";
  4016. clear_spool = 0;
  4017. // The compress_method might have been NO_METHOD, but we store the data
  4018. // compressed. Decompress them using `compression_used`. See fallback code in
  4019. // find_best_consensus() and find_best_diff().
  4020. write_http_response_headers(conn, -1,
  4021. compress_method == NO_METHOD ?
  4022. NO_METHOD : compression_used,
  4023. vary_header,
  4024. smartlist_len(conn->spool) == 1 ? lifetime : 0);
  4025. if (compress_method == NO_METHOD && smartlist_len(conn->spool))
  4026. conn->compress_state = tor_compress_new(0, compression_used,
  4027. HIGH_COMPRESSION);
  4028. /* Prime the connection with some data. */
  4029. const int initial_flush_result = connection_dirserv_flushed_some(conn);
  4030. tor_assert_nonfatal(initial_flush_result == 0);
  4031. goto done;
  4032. done:
  4033. parsed_consensus_request_clear(&req);
  4034. if (clear_spool) {
  4035. dir_conn_clear_spool(conn);
  4036. }
  4037. return 0;
  4038. }
  4039. /** Helper function for GET /tor/status-vote/{current,next}/...
  4040. */
  4041. static int
  4042. handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
  4043. {
  4044. const char *url = args->url;
  4045. {
  4046. int current;
  4047. ssize_t body_len = 0;
  4048. ssize_t estimated_len = 0;
  4049. /* This smartlist holds strings that we can compress on the fly. */
  4050. smartlist_t *items = smartlist_new();
  4051. /* This smartlist holds cached_dir_t objects that have a precompressed
  4052. * deflated version. */
  4053. smartlist_t *dir_items = smartlist_new();
  4054. int lifetime = 60; /* XXXX?? should actually use vote intervals. */
  4055. url += strlen("/tor/status-vote/");
  4056. current = !strcmpstart(url, "current/");
  4057. url = strchr(url, '/');
  4058. tor_assert(url);
  4059. ++url;
  4060. if (!strcmp(url, "consensus")) {
  4061. const char *item;
  4062. tor_assert(!current); /* we handle current consensus specially above,
  4063. * since it wants to be spooled. */
  4064. if ((item = dirvote_get_pending_consensus(FLAV_NS)))
  4065. smartlist_add(items, (char*)item);
  4066. } else if (!current && !strcmp(url, "consensus-signatures")) {
  4067. /* XXXX the spec says that we should implement
  4068. * current/consensus-signatures too. It doesn't seem to be needed,
  4069. * though. */
  4070. const char *item;
  4071. if ((item=dirvote_get_pending_detached_signatures()))
  4072. smartlist_add(items, (char*)item);
  4073. } else if (!strcmp(url, "authority")) {
  4074. const cached_dir_t *d;
  4075. int flags = DGV_BY_ID |
  4076. (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING);
  4077. if ((d=dirvote_get_vote(NULL, flags)))
  4078. smartlist_add(dir_items, (cached_dir_t*)d);
  4079. } else {
  4080. const cached_dir_t *d;
  4081. smartlist_t *fps = smartlist_new();
  4082. int flags;
  4083. if (!strcmpstart(url, "d/")) {
  4084. url += 2;
  4085. flags = DGV_INCLUDE_PENDING | DGV_INCLUDE_PREVIOUS;
  4086. } else {
  4087. flags = DGV_BY_ID |
  4088. (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING);
  4089. }
  4090. dir_split_resource_into_fingerprints(url, fps, NULL,
  4091. DSR_HEX|DSR_SORT_UNIQ);
  4092. SMARTLIST_FOREACH(fps, char *, fp, {
  4093. if ((d = dirvote_get_vote(fp, flags)))
  4094. smartlist_add(dir_items, (cached_dir_t*)d);
  4095. tor_free(fp);
  4096. });
  4097. smartlist_free(fps);
  4098. }
  4099. if (!smartlist_len(dir_items) && !smartlist_len(items)) {
  4100. write_short_http_response(conn, 404, "Not found");
  4101. goto vote_done;
  4102. }
  4103. /* We're sending items from at most one kind of source */
  4104. tor_assert_nonfatal(smartlist_len(items) == 0 ||
  4105. smartlist_len(dir_items) == 0);
  4106. int streaming;
  4107. unsigned mask;
  4108. if (smartlist_len(items)) {
  4109. /* We're taking strings and compressing them on the fly. */
  4110. streaming = 1;
  4111. mask = ~0u;
  4112. } else {
  4113. /* We're taking cached_dir_t objects. We only have them uncompressed
  4114. * or deflated. */
  4115. streaming = 0;
  4116. mask = (1u<<NO_METHOD) | (1u<<ZLIB_METHOD);
  4117. }
  4118. const compress_method_t compress_method = find_best_compression_method(
  4119. args->compression_supported&mask, streaming);
  4120. SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
  4121. body_len += compress_method != NO_METHOD ?
  4122. d->dir_compressed_len : d->dir_len);
  4123. estimated_len += body_len;
  4124. SMARTLIST_FOREACH(items, const char *, item, {
  4125. size_t ln = strlen(item);
  4126. if (compress_method != NO_METHOD) {
  4127. estimated_len += ln/2;
  4128. } else {
  4129. body_len += ln; estimated_len += ln;
  4130. }
  4131. });
  4132. if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) {
  4133. write_short_http_response(conn, 503, "Directory busy, try again later");
  4134. goto vote_done;
  4135. }
  4136. write_http_response_header(conn, body_len ? body_len : -1,
  4137. compress_method,
  4138. lifetime);
  4139. if (smartlist_len(items)) {
  4140. if (compress_method != NO_METHOD) {
  4141. conn->compress_state = tor_compress_new(1, compress_method,
  4142. choose_compression_level(estimated_len));
  4143. SMARTLIST_FOREACH(items, const char *, c,
  4144. connection_buf_add_compress(c, strlen(c), conn, 0));
  4145. connection_buf_add_compress("", 0, conn, 1);
  4146. } else {
  4147. SMARTLIST_FOREACH(items, const char *, c,
  4148. connection_buf_add(c, strlen(c), TO_CONN(conn)));
  4149. }
  4150. } else {
  4151. SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
  4152. connection_buf_add(compress_method != NO_METHOD ?
  4153. d->dir_compressed : d->dir,
  4154. compress_method != NO_METHOD ?
  4155. d->dir_compressed_len : d->dir_len,
  4156. TO_CONN(conn)));
  4157. }
  4158. vote_done:
  4159. smartlist_free(items);
  4160. smartlist_free(dir_items);
  4161. goto done;
  4162. }
  4163. done:
  4164. return 0;
  4165. }
  4166. /** Helper function for GET /tor/micro/d/...
  4167. */
  4168. static int
  4169. handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args)
  4170. {
  4171. const char *url = args->url;
  4172. const compress_method_t compress_method =
  4173. find_best_compression_method(args->compression_supported, 1);
  4174. int clear_spool = 1;
  4175. {
  4176. conn->spool = smartlist_new();
  4177. dir_split_resource_into_spoolable(url+strlen("/tor/micro/d/"),
  4178. DIR_SPOOL_MICRODESC,
  4179. conn->spool, NULL,
  4180. DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ);
  4181. size_t size_guess = 0;
  4182. dirserv_spool_remove_missing_and_guess_size(conn, 0,
  4183. compress_method != NO_METHOD,
  4184. &size_guess, NULL);
  4185. if (smartlist_len(conn->spool) == 0) {
  4186. write_short_http_response(conn, 404, "Not found");
  4187. goto done;
  4188. }
  4189. if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
  4190. log_info(LD_DIRSERV,
  4191. "Client asked for server descriptors, but we've been "
  4192. "writing too many bytes lately. Sending 503 Dir busy.");
  4193. write_short_http_response(conn, 503, "Directory busy, try again later");
  4194. goto done;
  4195. }
  4196. clear_spool = 0;
  4197. write_http_response_header(conn, -1,
  4198. compress_method,
  4199. MICRODESC_CACHE_LIFETIME);
  4200. if (compress_method != NO_METHOD)
  4201. conn->compress_state = tor_compress_new(1, compress_method,
  4202. choose_compression_level(size_guess));
  4203. const int initial_flush_result = connection_dirserv_flushed_some(conn);
  4204. tor_assert_nonfatal(initial_flush_result == 0);
  4205. goto done;
  4206. }
  4207. done:
  4208. if (clear_spool) {
  4209. dir_conn_clear_spool(conn);
  4210. }
  4211. return 0;
  4212. }
  4213. /** Helper function for GET /tor/{server,extra}/...
  4214. */
  4215. static int
  4216. handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args)
  4217. {
  4218. const char *url = args->url;
  4219. const compress_method_t compress_method =
  4220. find_best_compression_method(args->compression_supported, 1);
  4221. const or_options_t *options = get_options();
  4222. int clear_spool = 1;
  4223. if (!strcmpstart(url,"/tor/server/") ||
  4224. (!options->BridgeAuthoritativeDir &&
  4225. !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) {
  4226. int res;
  4227. const char *msg = NULL;
  4228. int cache_lifetime = 0;
  4229. int is_extra = !strcmpstart(url,"/tor/extra/");
  4230. url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/");
  4231. dir_spool_source_t source;
  4232. time_t publish_cutoff = 0;
  4233. if (!strcmpstart(url, "d/")) {
  4234. source =
  4235. is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST;
  4236. } else {
  4237. source =
  4238. is_extra ? DIR_SPOOL_EXTRA_BY_FP : DIR_SPOOL_SERVER_BY_FP;
  4239. /* We only want to apply a publish cutoff when we're requesting
  4240. * resources by fingerprint. */
  4241. publish_cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
  4242. }
  4243. conn->spool = smartlist_new();
  4244. res = dirserv_get_routerdesc_spool(conn->spool, url,
  4245. source,
  4246. connection_dir_is_encrypted(conn),
  4247. &msg);
  4248. if (!strcmpstart(url, "all")) {
  4249. cache_lifetime = FULL_DIR_CACHE_LIFETIME;
  4250. } else if (smartlist_len(conn->spool) == 1) {
  4251. cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME;
  4252. }
  4253. size_t size_guess = 0;
  4254. int n_expired = 0;
  4255. dirserv_spool_remove_missing_and_guess_size(conn, publish_cutoff,
  4256. compress_method != NO_METHOD,
  4257. &size_guess, &n_expired);
  4258. /* If we are the bridge authority and the descriptor is a bridge
  4259. * descriptor, remember that we served this descriptor for desc stats. */
  4260. /* XXXX it's a bit of a kludge to have this here. */
  4261. if (get_options()->BridgeAuthoritativeDir &&
  4262. source == DIR_SPOOL_SERVER_BY_FP) {
  4263. SMARTLIST_FOREACH_BEGIN(conn->spool, spooled_resource_t *, spooled) {
  4264. const routerinfo_t *router =
  4265. router_get_by_id_digest((const char *)spooled->digest);
  4266. /* router can be NULL here when the bridge auth is asked for its own
  4267. * descriptor. */
  4268. if (router && router->purpose == ROUTER_PURPOSE_BRIDGE)
  4269. rep_hist_note_desc_served(router->cache_info.identity_digest);
  4270. } SMARTLIST_FOREACH_END(spooled);
  4271. }
  4272. if (res < 0 || size_guess == 0 || smartlist_len(conn->spool) == 0) {
  4273. if (msg == NULL)
  4274. msg = "Not found";
  4275. write_short_http_response(conn, 404, msg);
  4276. } else {
  4277. if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
  4278. log_info(LD_DIRSERV,
  4279. "Client asked for server descriptors, but we've been "
  4280. "writing too many bytes lately. Sending 503 Dir busy.");
  4281. write_short_http_response(conn, 503,
  4282. "Directory busy, try again later");
  4283. dir_conn_clear_spool(conn);
  4284. goto done;
  4285. }
  4286. write_http_response_header(conn, -1, compress_method, cache_lifetime);
  4287. if (compress_method != NO_METHOD)
  4288. conn->compress_state = tor_compress_new(1, compress_method,
  4289. choose_compression_level(size_guess));
  4290. clear_spool = 0;
  4291. /* Prime the connection with some data. */
  4292. int initial_flush_result = connection_dirserv_flushed_some(conn);
  4293. tor_assert_nonfatal(initial_flush_result == 0);
  4294. }
  4295. goto done;
  4296. }
  4297. done:
  4298. if (clear_spool)
  4299. dir_conn_clear_spool(conn);
  4300. return 0;
  4301. }
  4302. /** Helper function for GET /tor/keys/...
  4303. */
  4304. static int
  4305. handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
  4306. {
  4307. const char *url = args->url;
  4308. const compress_method_t compress_method =
  4309. find_best_compression_method(args->compression_supported, 1);
  4310. const time_t if_modified_since = args->if_modified_since;
  4311. {
  4312. smartlist_t *certs = smartlist_new();
  4313. ssize_t len = -1;
  4314. if (!strcmp(url, "/tor/keys/all")) {
  4315. authority_cert_get_all(certs);
  4316. } else if (!strcmp(url, "/tor/keys/authority")) {
  4317. authority_cert_t *cert = get_my_v3_authority_cert();
  4318. if (cert)
  4319. smartlist_add(certs, cert);
  4320. } else if (!strcmpstart(url, "/tor/keys/fp/")) {
  4321. smartlist_t *fps = smartlist_new();
  4322. dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"),
  4323. fps, NULL,
  4324. DSR_HEX|DSR_SORT_UNIQ);
  4325. SMARTLIST_FOREACH(fps, char *, d, {
  4326. authority_cert_t *c = authority_cert_get_newest_by_id(d);
  4327. if (c) smartlist_add(certs, c);
  4328. tor_free(d);
  4329. });
  4330. smartlist_free(fps);
  4331. } else if (!strcmpstart(url, "/tor/keys/sk/")) {
  4332. smartlist_t *fps = smartlist_new();
  4333. dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"),
  4334. fps, NULL,
  4335. DSR_HEX|DSR_SORT_UNIQ);
  4336. SMARTLIST_FOREACH(fps, char *, d, {
  4337. authority_cert_t *c = authority_cert_get_by_sk_digest(d);
  4338. if (c) smartlist_add(certs, c);
  4339. tor_free(d);
  4340. });
  4341. smartlist_free(fps);
  4342. } else if (!strcmpstart(url, "/tor/keys/fp-sk/")) {
  4343. smartlist_t *fp_sks = smartlist_new();
  4344. dir_split_resource_into_fingerprint_pairs(url+strlen("/tor/keys/fp-sk/"),
  4345. fp_sks);
  4346. SMARTLIST_FOREACH(fp_sks, fp_pair_t *, pair, {
  4347. authority_cert_t *c = authority_cert_get_by_digests(pair->first,
  4348. pair->second);
  4349. if (c) smartlist_add(certs, c);
  4350. tor_free(pair);
  4351. });
  4352. smartlist_free(fp_sks);
  4353. } else {
  4354. write_short_http_response(conn, 400, "Bad request");
  4355. goto keys_done;
  4356. }
  4357. if (!smartlist_len(certs)) {
  4358. write_short_http_response(conn, 404, "Not found");
  4359. goto keys_done;
  4360. }
  4361. SMARTLIST_FOREACH(certs, authority_cert_t *, c,
  4362. if (c->cache_info.published_on < if_modified_since)
  4363. SMARTLIST_DEL_CURRENT(certs, c));
  4364. if (!smartlist_len(certs)) {
  4365. write_short_http_response(conn, 304, "Not modified");
  4366. goto keys_done;
  4367. }
  4368. len = 0;
  4369. SMARTLIST_FOREACH(certs, authority_cert_t *, c,
  4370. len += c->cache_info.signed_descriptor_len);
  4371. if (global_write_bucket_low(TO_CONN(conn),
  4372. compress_method != NO_METHOD ? len/2 : len,
  4373. 2)) {
  4374. write_short_http_response(conn, 503, "Directory busy, try again later");
  4375. goto keys_done;
  4376. }
  4377. write_http_response_header(conn,
  4378. compress_method != NO_METHOD ? -1 : len,
  4379. compress_method,
  4380. 60*60);
  4381. if (compress_method != NO_METHOD) {
  4382. conn->compress_state = tor_compress_new(1, compress_method,
  4383. choose_compression_level(len));
  4384. SMARTLIST_FOREACH(certs, authority_cert_t *, c,
  4385. connection_buf_add_compress(
  4386. c->cache_info.signed_descriptor_body,
  4387. c->cache_info.signed_descriptor_len,
  4388. conn, 0));
  4389. connection_buf_add_compress("", 0, conn, 1);
  4390. } else {
  4391. SMARTLIST_FOREACH(certs, authority_cert_t *, c,
  4392. connection_buf_add(c->cache_info.signed_descriptor_body,
  4393. c->cache_info.signed_descriptor_len,
  4394. TO_CONN(conn)));
  4395. }
  4396. keys_done:
  4397. smartlist_free(certs);
  4398. goto done;
  4399. }
  4400. done:
  4401. return 0;
  4402. }
  4403. /** Helper function for GET /tor/rendezvous2/
  4404. */
  4405. static int
  4406. handle_get_hs_descriptor_v2(dir_connection_t *conn,
  4407. const get_handler_args_t *args)
  4408. {
  4409. const char *url = args->url;
  4410. if (connection_dir_is_encrypted(conn)) {
  4411. /* Handle v2 rendezvous descriptor fetch request. */
  4412. const char *descp;
  4413. const char *query = url + strlen("/tor/rendezvous2/");
  4414. if (rend_valid_descriptor_id(query)) {
  4415. log_info(LD_REND, "Got a v2 rendezvous descriptor request for ID '%s'",
  4416. safe_str(escaped(query)));
  4417. switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) {
  4418. case 1: /* valid */
  4419. write_http_response_header(conn, strlen(descp), NO_METHOD, 0);
  4420. connection_buf_add(descp, strlen(descp), TO_CONN(conn));
  4421. break;
  4422. case 0: /* well-formed but not present */
  4423. write_short_http_response(conn, 404, "Not found");
  4424. break;
  4425. case -1: /* not well-formed */
  4426. write_short_http_response(conn, 400, "Bad request");
  4427. break;
  4428. }
  4429. } else { /* not well-formed */
  4430. write_short_http_response(conn, 400, "Bad request");
  4431. }
  4432. goto done;
  4433. } else {
  4434. /* Not encrypted! */
  4435. write_short_http_response(conn, 404, "Not found");
  4436. }
  4437. done:
  4438. return 0;
  4439. }
  4440. /** Helper function for GET /tor/hs/3/<z>. Only for version 3.
  4441. */
  4442. STATIC int
  4443. handle_get_hs_descriptor_v3(dir_connection_t *conn,
  4444. const get_handler_args_t *args)
  4445. {
  4446. int retval;
  4447. const char *desc_str = NULL;
  4448. const char *pubkey_str = NULL;
  4449. const char *url = args->url;
  4450. /* Reject unencrypted dir connections */
  4451. if (!connection_dir_is_encrypted(conn)) {
  4452. write_short_http_response(conn, 404, "Not found");
  4453. goto done;
  4454. }
  4455. /* After the path prefix follows the base64 encoded blinded pubkey which we
  4456. * use to get the descriptor from the cache. Skip the prefix and get the
  4457. * pubkey. */
  4458. tor_assert(!strcmpstart(url, "/tor/hs/3/"));
  4459. pubkey_str = url + strlen("/tor/hs/3/");
  4460. retval = hs_cache_lookup_as_dir(HS_VERSION_THREE,
  4461. pubkey_str, &desc_str);
  4462. if (retval <= 0 || desc_str == NULL) {
  4463. write_short_http_response(conn, 404, "Not found");
  4464. goto done;
  4465. }
  4466. /* Found requested descriptor! Pass it to this nice client. */
  4467. write_http_response_header(conn, strlen(desc_str), NO_METHOD, 0);
  4468. connection_buf_add(desc_str, strlen(desc_str), TO_CONN(conn));
  4469. done:
  4470. return 0;
  4471. }
  4472. /** Helper function for GET /tor/networkstatus-bridges
  4473. */
  4474. static int
  4475. handle_get_networkstatus_bridges(dir_connection_t *conn,
  4476. const get_handler_args_t *args)
  4477. {
  4478. const char *headers = args->headers;
  4479. const or_options_t *options = get_options();
  4480. if (options->BridgeAuthoritativeDir &&
  4481. options->BridgePassword_AuthDigest_ &&
  4482. connection_dir_is_encrypted(conn)) {
  4483. char *status;
  4484. char digest[DIGEST256_LEN];
  4485. char *header = http_get_header(headers, "Authorization: Basic ");
  4486. if (header)
  4487. crypto_digest256(digest, header, strlen(header), DIGEST_SHA256);
  4488. /* now make sure the password is there and right */
  4489. if (!header ||
  4490. tor_memneq(digest,
  4491. options->BridgePassword_AuthDigest_, DIGEST256_LEN)) {
  4492. write_short_http_response(conn, 404, "Not found");
  4493. tor_free(header);
  4494. goto done;
  4495. }
  4496. tor_free(header);
  4497. /* all happy now. send an answer. */
  4498. status = networkstatus_getinfo_by_purpose("bridge", time(NULL));
  4499. size_t dlen = strlen(status);
  4500. write_http_response_header(conn, dlen, NO_METHOD, 0);
  4501. connection_buf_add(status, dlen, TO_CONN(conn));
  4502. tor_free(status);
  4503. goto done;
  4504. }
  4505. done:
  4506. return 0;
  4507. }
  4508. /** Helper function for GET robots.txt or /tor/robots.txt */
  4509. static int
  4510. handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args)
  4511. {
  4512. (void)args;
  4513. {
  4514. const char robots[] = "User-agent: *\r\nDisallow: /\r\n";
  4515. size_t len = strlen(robots);
  4516. write_http_response_header(conn, len, NO_METHOD, ROBOTS_CACHE_LIFETIME);
  4517. connection_buf_add(robots, len, TO_CONN(conn));
  4518. }
  4519. return 0;
  4520. }
  4521. /* Given the <b>url</b> from a POST request, try to extract the version number
  4522. * using the provided <b>prefix</b>. The version should be after the prefix and
  4523. * ending with the separator "/". For instance:
  4524. * /tor/hs/3/publish
  4525. *
  4526. * On success, <b>end_pos</b> points to the position right after the version
  4527. * was found. On error, it is set to NULL.
  4528. *
  4529. * Return version on success else negative value. */
  4530. STATIC int
  4531. parse_hs_version_from_post(const char *url, const char *prefix,
  4532. const char **end_pos)
  4533. {
  4534. int ok;
  4535. unsigned long version;
  4536. const char *start;
  4537. char *end = NULL;
  4538. tor_assert(url);
  4539. tor_assert(prefix);
  4540. tor_assert(end_pos);
  4541. /* Check if the prefix does start the url. */
  4542. if (strcmpstart(url, prefix)) {
  4543. goto err;
  4544. }
  4545. /* Move pointer to the end of the prefix string. */
  4546. start = url + strlen(prefix);
  4547. /* Try this to be the HS version and if we are still at the separator, next
  4548. * will be move to the right value. */
  4549. version = tor_parse_long(start, 10, 0, INT_MAX, &ok, &end);
  4550. if (!ok) {
  4551. goto err;
  4552. }
  4553. *end_pos = end;
  4554. return (int) version;
  4555. err:
  4556. *end_pos = NULL;
  4557. return -1;
  4558. }
  4559. /* Handle the POST request for a hidden service descripror. The request is in
  4560. * <b>url</b>, the body of the request is in <b>body</b>. Return 200 on success
  4561. * else return 400 indicating a bad request. */
  4562. STATIC int
  4563. handle_post_hs_descriptor(const char *url, const char *body)
  4564. {
  4565. int version;
  4566. const char *end_pos;
  4567. tor_assert(url);
  4568. tor_assert(body);
  4569. version = parse_hs_version_from_post(url, "/tor/hs/", &end_pos);
  4570. if (version < 0) {
  4571. goto err;
  4572. }
  4573. /* We have a valid version number, now make sure it's a publish request. Use
  4574. * the end position just after the version and check for the command. */
  4575. if (strcmpstart(end_pos, "/publish")) {
  4576. goto err;
  4577. }
  4578. switch (version) {
  4579. case HS_VERSION_THREE:
  4580. if (hs_cache_store_as_dir(body) < 0) {
  4581. goto err;
  4582. }
  4583. log_info(LD_REND, "Publish request for HS descriptor handled "
  4584. "successfully.");
  4585. break;
  4586. default:
  4587. /* Unsupported version, return a bad request. */
  4588. goto err;
  4589. }
  4590. return 200;
  4591. err:
  4592. /* Bad request. */
  4593. return 400;
  4594. }
  4595. /** Helper function: called when a dirserver gets a complete HTTP POST
  4596. * request. Look for an uploaded server descriptor or rendezvous
  4597. * service descriptor. On finding one, process it and write a
  4598. * response into conn-\>outbuf. If the request is unrecognized, send a
  4599. * 400. Always return 0. */
  4600. MOCK_IMPL(STATIC int,
  4601. directory_handle_command_post,(dir_connection_t *conn, const char *headers,
  4602. const char *body, size_t body_len))
  4603. {
  4604. char *url = NULL;
  4605. const or_options_t *options = get_options();
  4606. log_debug(LD_DIRSERV,"Received POST command.");
  4607. conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
  4608. if (!public_server_mode(options)) {
  4609. log_info(LD_DIR, "Rejected dir post request from %s "
  4610. "since we're not a public relay.", conn->base_.address);
  4611. write_short_http_response(conn, 503, "Not acting as a public relay");
  4612. goto done;
  4613. }
  4614. if (parse_http_url(headers, &url) < 0) {
  4615. write_short_http_response(conn, 400, "Bad request");
  4616. return 0;
  4617. }
  4618. log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
  4619. /* Handle v2 rendezvous service publish request. */
  4620. if (connection_dir_is_encrypted(conn) &&
  4621. !strcmpstart(url,"/tor/rendezvous2/publish")) {
  4622. if (rend_cache_store_v2_desc_as_dir(body) < 0) {
  4623. log_warn(LD_REND, "Rejected v2 rend descriptor (body size %d) from %s.",
  4624. (int)body_len, conn->base_.address);
  4625. write_short_http_response(conn, 400,
  4626. "Invalid v2 service descriptor rejected");
  4627. } else {
  4628. write_short_http_response(conn, 200, "Service descriptor (v2) stored");
  4629. log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted");
  4630. }
  4631. goto done;
  4632. }
  4633. /* Handle HS descriptor publish request. */
  4634. /* XXX: This should be disabled with a consensus param until we want to
  4635. * the prop224 be deployed and thus use. */
  4636. if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) {
  4637. const char *msg = "HS descriptor stored successfully.";
  4638. /* We most probably have a publish request for an HS descriptor. */
  4639. int code = handle_post_hs_descriptor(url, body);
  4640. if (code != 200) {
  4641. msg = "Invalid HS descriptor. Rejected.";
  4642. }
  4643. write_short_http_response(conn, code, msg);
  4644. goto done;
  4645. }
  4646. if (!authdir_mode(options)) {
  4647. /* we just provide cached directories; we don't want to
  4648. * receive anything. */
  4649. write_short_http_response(conn, 400, "Nonauthoritative directory does not "
  4650. "accept posted server descriptors");
  4651. goto done;
  4652. }
  4653. if (authdir_mode(options) &&
  4654. !strcmp(url,"/tor/")) { /* server descriptor post */
  4655. const char *msg = "[None]";
  4656. uint8_t purpose = authdir_mode_bridge(options) ?
  4657. ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
  4658. was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose,
  4659. conn->base_.address, &msg);
  4660. tor_assert(msg);
  4661. if (r == ROUTER_ADDED_SUCCESSFULLY) {
  4662. write_short_http_response(conn, 200, msg);
  4663. } else if (WRA_WAS_OUTDATED(r)) {
  4664. write_http_response_header_impl(conn, -1, NULL, NULL,
  4665. "X-Descriptor-Not-New: Yes\r\n", -1);
  4666. } else {
  4667. log_info(LD_DIRSERV,
  4668. "Rejected router descriptor or extra-info from %s "
  4669. "(\"%s\").",
  4670. conn->base_.address, msg);
  4671. write_short_http_response(conn, 400, msg);
  4672. }
  4673. goto done;
  4674. }
  4675. if (authdir_mode_v3(options) &&
  4676. !strcmp(url,"/tor/post/vote")) { /* v3 networkstatus vote */
  4677. const char *msg = "OK";
  4678. int status;
  4679. if (dirvote_add_vote(body, &msg, &status)) {
  4680. write_short_http_response(conn, status, "Vote stored");
  4681. } else {
  4682. tor_assert(msg);
  4683. log_warn(LD_DIRSERV, "Rejected vote from %s (\"%s\").",
  4684. conn->base_.address, msg);
  4685. write_short_http_response(conn, status, msg);
  4686. }
  4687. goto done;
  4688. }
  4689. if (authdir_mode_v3(options) &&
  4690. !strcmp(url,"/tor/post/consensus-signature")) { /* sigs on consensus. */
  4691. const char *msg = NULL;
  4692. if (dirvote_add_signatures(body, conn->base_.address, &msg)>=0) {
  4693. write_short_http_response(conn, 200, msg?msg:"Signatures stored");
  4694. } else {
  4695. log_warn(LD_DIR, "Unable to store signatures posted by %s: %s",
  4696. conn->base_.address, msg?msg:"???");
  4697. write_short_http_response(conn, 400,
  4698. msg?msg:"Unable to store signatures");
  4699. }
  4700. goto done;
  4701. }
  4702. /* we didn't recognize the url */
  4703. write_short_http_response(conn, 404, "Not found");
  4704. done:
  4705. tor_free(url);
  4706. return 0;
  4707. }
  4708. /** Called when a dirserver receives data on a directory connection;
  4709. * looks for an HTTP request. If the request is complete, remove it
  4710. * from the inbuf, try to process it; otherwise, leave it on the
  4711. * buffer. Return a 0 on success, or -1 on error.
  4712. */
  4713. STATIC int
  4714. directory_handle_command(dir_connection_t *conn)
  4715. {
  4716. char *headers=NULL, *body=NULL;
  4717. size_t body_len=0;
  4718. int r;
  4719. tor_assert(conn);
  4720. tor_assert(conn->base_.type == CONN_TYPE_DIR);
  4721. switch (connection_fetch_from_buf_http(TO_CONN(conn),
  4722. &headers, MAX_HEADERS_SIZE,
  4723. &body, &body_len, MAX_DIR_UL_SIZE, 0)) {
  4724. case -1: /* overflow */
  4725. log_warn(LD_DIRSERV,
  4726. "Request too large from address '%s' to DirPort. Closing.",
  4727. safe_str(conn->base_.address));
  4728. return -1;
  4729. case 0:
  4730. log_debug(LD_DIRSERV,"command not all here yet.");
  4731. return 0;
  4732. /* case 1, fall through */
  4733. }
  4734. http_set_address_origin(headers, TO_CONN(conn));
  4735. // we should escape headers here as well,
  4736. // but we can't call escaped() twice, as it uses the same buffer
  4737. //log_debug(LD_DIRSERV,"headers %s, body %s.", headers, escaped(body));
  4738. if (!strncasecmp(headers,"GET",3))
  4739. r = directory_handle_command_get(conn, headers, body, body_len);
  4740. else if (!strncasecmp(headers,"POST",4))
  4741. r = directory_handle_command_post(conn, headers, body, body_len);
  4742. else {
  4743. log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
  4744. "Got headers %s with unknown command. Closing.",
  4745. escaped(headers));
  4746. r = -1;
  4747. }
  4748. tor_free(headers); tor_free(body);
  4749. return r;
  4750. }
  4751. /** Write handler for directory connections; called when all data has
  4752. * been flushed. Close the connection or wait for a response as
  4753. * appropriate.
  4754. */
  4755. int
  4756. connection_dir_finished_flushing(dir_connection_t *conn)
  4757. {
  4758. tor_assert(conn);
  4759. tor_assert(conn->base_.type == CONN_TYPE_DIR);
  4760. /* Note that we have finished writing the directory response. For direct
  4761. * connections this means we're done; for tunneled connections it's only
  4762. * an intermediate step. */
  4763. if (conn->dirreq_id)
  4764. geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED,
  4765. DIRREQ_FLUSHING_DIR_CONN_FINISHED);
  4766. else
  4767. geoip_change_dirreq_state(TO_CONN(conn)->global_identifier,
  4768. DIRREQ_DIRECT,
  4769. DIRREQ_FLUSHING_DIR_CONN_FINISHED);
  4770. switch (conn->base_.state) {
  4771. case DIR_CONN_STATE_CONNECTING:
  4772. case DIR_CONN_STATE_CLIENT_SENDING:
  4773. log_debug(LD_DIR,"client finished sending command.");
  4774. conn->base_.state = DIR_CONN_STATE_CLIENT_READING;
  4775. return 0;
  4776. case DIR_CONN_STATE_SERVER_WRITING:
  4777. if (conn->spool) {
  4778. log_warn(LD_BUG, "Emptied a dirserv buffer, but it's still spooling!");
  4779. connection_mark_for_close(TO_CONN(conn));
  4780. } else {
  4781. log_debug(LD_DIRSERV, "Finished writing server response. Closing.");
  4782. connection_mark_for_close(TO_CONN(conn));
  4783. }
  4784. return 0;
  4785. default:
  4786. log_warn(LD_BUG,"called in unexpected state %d.",
  4787. conn->base_.state);
  4788. tor_fragile_assert();
  4789. return -1;
  4790. }
  4791. return 0;
  4792. }
  4793. /* We just got a new consensus! If there are other in-progress requests
  4794. * for this consensus flavor (for example because we launched several in
  4795. * parallel), cancel them.
  4796. *
  4797. * We do this check here (not just in
  4798. * connection_ap_handshake_attach_circuit()) to handle the edge case where
  4799. * a consensus fetch begins and ends before some other one tries to attach to
  4800. * a circuit, in which case the other one won't know that we're all happy now.
  4801. *
  4802. * Don't mark the conn that just gave us the consensus -- otherwise we
  4803. * would end up double-marking it when it cleans itself up.
  4804. */
  4805. static void
  4806. connection_dir_close_consensus_fetches(dir_connection_t *except_this_one,
  4807. const char *resource)
  4808. {
  4809. smartlist_t *conns_to_close =
  4810. connection_dir_list_by_purpose_and_resource(DIR_PURPOSE_FETCH_CONSENSUS,
  4811. resource);
  4812. SMARTLIST_FOREACH_BEGIN(conns_to_close, dir_connection_t *, d) {
  4813. if (d == except_this_one)
  4814. continue;
  4815. log_info(LD_DIR, "Closing consensus fetch (to %s) since one "
  4816. "has just arrived.", TO_CONN(d)->address);
  4817. connection_mark_for_close(TO_CONN(d));
  4818. } SMARTLIST_FOREACH_END(d);
  4819. smartlist_free(conns_to_close);
  4820. }
  4821. /** Connected handler for directory connections: begin sending data to the
  4822. * server, and return 0.
  4823. * Only used when connections don't immediately connect. */
  4824. int
  4825. connection_dir_finished_connecting(dir_connection_t *conn)
  4826. {
  4827. tor_assert(conn);
  4828. tor_assert(conn->base_.type == CONN_TYPE_DIR);
  4829. tor_assert(conn->base_.state == DIR_CONN_STATE_CONNECTING);
  4830. log_debug(LD_HTTP,"Dir connection to router %s:%u established.",
  4831. conn->base_.address,conn->base_.port);
  4832. /* start flushing conn */
  4833. conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
  4834. return 0;
  4835. }
  4836. /** Decide which download schedule we want to use based on descriptor type
  4837. * in <b>dls</b> and <b>options</b>.
  4838. * Then return a list of int pointers defining download delays in seconds.
  4839. * Helper function for download_status_increment_failure(),
  4840. * download_status_reset(), and download_status_increment_attempt(). */
  4841. STATIC const smartlist_t *
  4842. find_dl_schedule(const download_status_t *dls, const or_options_t *options)
  4843. {
  4844. switch (dls->schedule) {
  4845. case DL_SCHED_GENERIC:
  4846. /* Any other directory document */
  4847. if (dir_server_mode(options)) {
  4848. /* A directory authority or directory mirror */
  4849. return options->TestingServerDownloadSchedule;
  4850. } else {
  4851. return options->TestingClientDownloadSchedule;
  4852. }
  4853. case DL_SCHED_CONSENSUS:
  4854. if (!networkstatus_consensus_can_use_multiple_directories(options)) {
  4855. /* A public relay */
  4856. return options->TestingServerConsensusDownloadSchedule;
  4857. } else {
  4858. /* A client or bridge */
  4859. if (networkstatus_consensus_is_bootstrapping(time(NULL))) {
  4860. /* During bootstrapping */
  4861. if (!networkstatus_consensus_can_use_extra_fallbacks(options)) {
  4862. /* A bootstrapping client without extra fallback directories */
  4863. return
  4864. options->ClientBootstrapConsensusAuthorityOnlyDownloadSchedule;
  4865. } else if (dls->want_authority) {
  4866. /* A bootstrapping client with extra fallback directories, but
  4867. * connecting to an authority */
  4868. return
  4869. options->ClientBootstrapConsensusAuthorityDownloadSchedule;
  4870. } else {
  4871. /* A bootstrapping client connecting to extra fallback directories
  4872. */
  4873. return
  4874. options->ClientBootstrapConsensusFallbackDownloadSchedule;
  4875. }
  4876. } else {
  4877. /* A client with a reasonably live consensus, with or without
  4878. * certificates */
  4879. return options->TestingClientConsensusDownloadSchedule;
  4880. }
  4881. }
  4882. case DL_SCHED_BRIDGE:
  4883. if (options->UseBridges && num_bridges_usable(0) > 0) {
  4884. /* A bridge client that is sure that one or more of its bridges are
  4885. * running can afford to wait longer to update bridge descriptors. */
  4886. return options->TestingBridgeDownloadSchedule;
  4887. } else {
  4888. /* A bridge client which might have no running bridges, must try to
  4889. * get bridge descriptors straight away. */
  4890. return options->TestingBridgeBootstrapDownloadSchedule;
  4891. }
  4892. default:
  4893. tor_assert(0);
  4894. }
  4895. /* Impossible, but gcc will fail with -Werror without a `return`. */
  4896. return NULL;
  4897. }
  4898. /** Decide which minimum and maximum delay step we want to use based on
  4899. * descriptor type in <b>dls</b> and <b>options</b>.
  4900. * Helper function for download_status_schedule_get_delay(). */
  4901. STATIC void
  4902. find_dl_min_and_max_delay(download_status_t *dls, const or_options_t *options,
  4903. int *min, int *max)
  4904. {
  4905. tor_assert(dls);
  4906. tor_assert(options);
  4907. tor_assert(min);
  4908. tor_assert(max);
  4909. /*
  4910. * For now, just use the existing schedule config stuff and pick the
  4911. * first/last entries off to get min/max delay for backoff purposes
  4912. */
  4913. const smartlist_t *schedule = find_dl_schedule(dls, options);
  4914. tor_assert(schedule != NULL && smartlist_len(schedule) >= 2);
  4915. *min = *((int *)(smartlist_get(schedule, 0)));
  4916. /* Increment on failure schedules always use exponential backoff, but they
  4917. * have a smaller limit when they're deterministic */
  4918. if (dls->backoff == DL_SCHED_DETERMINISTIC)
  4919. *max = *((int *)((smartlist_get(schedule, smartlist_len(schedule) - 1))));
  4920. else
  4921. *max = INT_MAX;
  4922. }
  4923. /** As next_random_exponential_delay() below, but does not compute a random
  4924. * value. Instead, compute the range of values that
  4925. * next_random_exponential_delay() should use when computing its random value.
  4926. * Store the low bound into *<b>low_bound_out</b>, and the high bound into
  4927. * *<b>high_bound_out</b>. Guarantees that the low bound is strictly less
  4928. * than the high bound. */
  4929. STATIC void
  4930. next_random_exponential_delay_range(int *low_bound_out,
  4931. int *high_bound_out,
  4932. int delay,
  4933. int base_delay)
  4934. {
  4935. // This is the "decorrelated jitter" approach, from
  4936. // https://www.awsarchitectureblog.com/2015/03/backoff.html
  4937. // The formula is
  4938. // sleep = min(cap, random_between(base, sleep * 3))
  4939. const int delay_times_3 = delay < INT_MAX/3 ? delay * 3 : INT_MAX;
  4940. *low_bound_out = base_delay;
  4941. if (delay_times_3 > base_delay) {
  4942. *high_bound_out = delay_times_3;
  4943. } else {
  4944. *high_bound_out = base_delay+1;
  4945. }
  4946. }
  4947. /** Advance one delay step. The algorithm will generate a random delay,
  4948. * such that each failure is possibly (random) longer than the ones before.
  4949. *
  4950. * We then clamp that value to be no larger than max_delay, and return it.
  4951. *
  4952. * The <b>base_delay</b> parameter is lowest possible delay time (can't be
  4953. * zero); the <b>backoff_position</b> parameter is the number of times we've
  4954. * generated a delay; and the <b>delay</b> argument is the most recently used
  4955. * delay.
  4956. *
  4957. * Requires that delay is less than INT_MAX, and delay is in [0,max_delay].
  4958. */
  4959. STATIC int
  4960. next_random_exponential_delay(int delay,
  4961. int base_delay,
  4962. int max_delay)
  4963. {
  4964. /* Check preconditions */
  4965. if (BUG(max_delay < 0))
  4966. max_delay = 0;
  4967. if (BUG(delay > max_delay))
  4968. delay = max_delay;
  4969. if (BUG(delay < 0))
  4970. delay = 0;
  4971. if (base_delay < 1)
  4972. base_delay = 1;
  4973. int low_bound=0, high_bound=max_delay;
  4974. next_random_exponential_delay_range(&low_bound, &high_bound,
  4975. delay, base_delay);
  4976. int rand_delay = crypto_rand_int_range(low_bound, high_bound);
  4977. return MIN(rand_delay, max_delay);
  4978. }
  4979. /** Find the current delay for dls based on schedule or min_delay/
  4980. * max_delay if we're using exponential backoff. If dls->backoff is
  4981. * DL_SCHED_RANDOM_EXPONENTIAL, we must have 0 <= min_delay <= max_delay <=
  4982. * INT_MAX, but schedule may be set to NULL; otherwise schedule is required.
  4983. * This function sets dls->next_attempt_at based on now, and returns the delay.
  4984. * Helper for download_status_increment_failure and
  4985. * download_status_increment_attempt. */
  4986. STATIC int
  4987. download_status_schedule_get_delay(download_status_t *dls,
  4988. const smartlist_t *schedule,
  4989. int min_delay, int max_delay,
  4990. time_t now)
  4991. {
  4992. tor_assert(dls);
  4993. /* We don't need a schedule if we're using random exponential backoff */
  4994. tor_assert(dls->backoff == DL_SCHED_RANDOM_EXPONENTIAL ||
  4995. schedule != NULL);
  4996. /* If we're using random exponential backoff, we do need min/max delay */
  4997. tor_assert(dls->backoff != DL_SCHED_RANDOM_EXPONENTIAL ||
  4998. (min_delay >= 0 && max_delay >= min_delay));
  4999. int delay = INT_MAX;
  5000. uint8_t dls_schedule_position = (dls->increment_on
  5001. == DL_SCHED_INCREMENT_ATTEMPT
  5002. ? dls->n_download_attempts
  5003. : dls->n_download_failures);
  5004. if (dls->backoff == DL_SCHED_DETERMINISTIC) {
  5005. if (dls_schedule_position < smartlist_len(schedule))
  5006. delay = *(int *)smartlist_get(schedule, dls_schedule_position);
  5007. else if (dls_schedule_position == IMPOSSIBLE_TO_DOWNLOAD)
  5008. delay = INT_MAX;
  5009. else
  5010. delay = *(int *)smartlist_get(schedule, smartlist_len(schedule) - 1);
  5011. } else if (dls->backoff == DL_SCHED_RANDOM_EXPONENTIAL) {
  5012. /* Check if we missed a reset somehow */
  5013. IF_BUG_ONCE(dls->last_backoff_position > dls_schedule_position) {
  5014. dls->last_backoff_position = 0;
  5015. dls->last_delay_used = 0;
  5016. }
  5017. if (dls_schedule_position > 0) {
  5018. delay = dls->last_delay_used;
  5019. while (dls->last_backoff_position < dls_schedule_position) {
  5020. /* Do one increment step */
  5021. delay = next_random_exponential_delay(delay, min_delay, max_delay);
  5022. /* Update our position */
  5023. ++(dls->last_backoff_position);
  5024. }
  5025. } else {
  5026. /* If we're just starting out, use the minimum delay */
  5027. delay = min_delay;
  5028. }
  5029. /* Clamp it within min/max if we have them */
  5030. if (min_delay >= 0 && delay < min_delay) delay = min_delay;
  5031. if (max_delay != INT_MAX && delay > max_delay) delay = max_delay;
  5032. /* Store it for next time */
  5033. dls->last_backoff_position = dls_schedule_position;
  5034. dls->last_delay_used = delay;
  5035. }
  5036. /* A negative delay makes no sense. Knowing that delay is
  5037. * non-negative allows us to safely do the wrapping check below. */
  5038. tor_assert(delay >= 0);
  5039. /* Avoid now+delay overflowing TIME_MAX, by comparing with a subtraction
  5040. * that won't overflow (since delay is non-negative). */
  5041. if (delay < INT_MAX && now <= TIME_MAX - delay) {
  5042. dls->next_attempt_at = now+delay;
  5043. } else {
  5044. dls->next_attempt_at = TIME_MAX;
  5045. }
  5046. return delay;
  5047. }
  5048. /* Log a debug message about item, which increments on increment_action, has
  5049. * incremented dls_n_download_increments times. The message varies based on
  5050. * was_schedule_incremented (if not, not_incremented_response is logged), and
  5051. * the values of increment, dls_next_attempt_at, and now.
  5052. * Helper for download_status_increment_failure and
  5053. * download_status_increment_attempt. */
  5054. static void
  5055. download_status_log_helper(const char *item, int was_schedule_incremented,
  5056. const char *increment_action,
  5057. const char *not_incremented_response,
  5058. uint8_t dls_n_download_increments, int increment,
  5059. time_t dls_next_attempt_at, time_t now)
  5060. {
  5061. if (item) {
  5062. if (!was_schedule_incremented)
  5063. log_debug(LD_DIR, "%s %s %d time(s); I'll try again %s.",
  5064. item, increment_action, (int)dls_n_download_increments,
  5065. not_incremented_response);
  5066. else if (increment == 0)
  5067. log_debug(LD_DIR, "%s %s %d time(s); I'll try again immediately.",
  5068. item, increment_action, (int)dls_n_download_increments);
  5069. else if (dls_next_attempt_at < TIME_MAX)
  5070. log_debug(LD_DIR, "%s %s %d time(s); I'll try again in %d seconds.",
  5071. item, increment_action, (int)dls_n_download_increments,
  5072. (int)(dls_next_attempt_at-now));
  5073. else
  5074. log_debug(LD_DIR, "%s %s %d time(s); Giving up for a while.",
  5075. item, increment_action, (int)dls_n_download_increments);
  5076. }
  5077. }
  5078. /** Determine when a failed download attempt should be retried.
  5079. * Called when an attempt to download <b>dls</b> has failed with HTTP status
  5080. * <b>status_code</b>. Increment the failure count (if the code indicates a
  5081. * real failure, or if we're a server) and set <b>dls</b>-\>next_attempt_at to
  5082. * an appropriate time in the future and return it.
  5083. * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_ATTEMPT, increment the
  5084. * failure count, and return a time in the far future for the next attempt (to
  5085. * avoid an immediate retry). */
  5086. time_t
  5087. download_status_increment_failure(download_status_t *dls, int status_code,
  5088. const char *item, int server, time_t now)
  5089. {
  5090. (void) status_code; // XXXX no longer used.
  5091. (void) server; // XXXX no longer used.
  5092. int increment = -1;
  5093. int min_delay = 0, max_delay = INT_MAX;
  5094. tor_assert(dls);
  5095. /* dls wasn't reset before it was used */
  5096. if (dls->next_attempt_at == 0) {
  5097. download_status_reset(dls);
  5098. }
  5099. /* count the failure */
  5100. if (dls->n_download_failures < IMPOSSIBLE_TO_DOWNLOAD-1) {
  5101. ++dls->n_download_failures;
  5102. }
  5103. if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) {
  5104. /* We don't find out that a failure-based schedule has attempted a
  5105. * connection until that connection fails.
  5106. * We'll never find out about successful connections, but this doesn't
  5107. * matter, because schedules are reset after a successful download.
  5108. */
  5109. if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1)
  5110. ++dls->n_download_attempts;
  5111. /* only return a failure retry time if this schedule increments on failures
  5112. */
  5113. const smartlist_t *schedule = find_dl_schedule(dls, get_options());
  5114. find_dl_min_and_max_delay(dls, get_options(), &min_delay, &max_delay);
  5115. increment = download_status_schedule_get_delay(dls, schedule,
  5116. min_delay, max_delay, now);
  5117. }
  5118. download_status_log_helper(item, !dls->increment_on, "failed",
  5119. "concurrently", dls->n_download_failures,
  5120. increment,
  5121. download_status_get_next_attempt_at(dls),
  5122. now);
  5123. if (dls->increment_on == DL_SCHED_INCREMENT_ATTEMPT) {
  5124. /* stop this schedule retrying on failure, it will launch concurrent
  5125. * connections instead */
  5126. return TIME_MAX;
  5127. } else {
  5128. return download_status_get_next_attempt_at(dls);
  5129. }
  5130. }
  5131. /** Determine when the next download attempt should be made when using an
  5132. * attempt-based (potentially concurrent) download schedule.
  5133. * Called when an attempt to download <b>dls</b> is being initiated.
  5134. * Increment the attempt count and set <b>dls</b>-\>next_attempt_at to an
  5135. * appropriate time in the future and return it.
  5136. * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_FAILURE, don't increment
  5137. * the attempts, and return a time in the far future (to avoid launching a
  5138. * concurrent attempt). */
  5139. time_t
  5140. download_status_increment_attempt(download_status_t *dls, const char *item,
  5141. time_t now)
  5142. {
  5143. int delay = -1;
  5144. int min_delay = 0, max_delay = INT_MAX;
  5145. tor_assert(dls);
  5146. /* dls wasn't reset before it was used */
  5147. if (dls->next_attempt_at == 0) {
  5148. download_status_reset(dls);
  5149. }
  5150. if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) {
  5151. /* this schedule should retry on failure, and not launch any concurrent
  5152. attempts */
  5153. log_warn(LD_BUG, "Tried to launch an attempt-based connection on a "
  5154. "failure-based schedule.");
  5155. return TIME_MAX;
  5156. }
  5157. if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1)
  5158. ++dls->n_download_attempts;
  5159. const smartlist_t *schedule = find_dl_schedule(dls, get_options());
  5160. find_dl_min_and_max_delay(dls, get_options(), &min_delay, &max_delay);
  5161. delay = download_status_schedule_get_delay(dls, schedule,
  5162. min_delay, max_delay, now);
  5163. download_status_log_helper(item, dls->increment_on, "attempted",
  5164. "on failure", dls->n_download_attempts,
  5165. delay, download_status_get_next_attempt_at(dls),
  5166. now);
  5167. return download_status_get_next_attempt_at(dls);
  5168. }
  5169. static time_t
  5170. download_status_get_initial_delay_from_now(const download_status_t *dls)
  5171. {
  5172. const smartlist_t *schedule = find_dl_schedule(dls, get_options());
  5173. /* We use constant initial delays, even in exponential backoff
  5174. * schedules. */
  5175. return time(NULL) + *(int *)smartlist_get(schedule, 0);
  5176. }
  5177. /** Reset <b>dls</b> so that it will be considered downloadable
  5178. * immediately, and/or to show that we don't need it anymore.
  5179. *
  5180. * Must be called to initialise a download schedule, otherwise the zeroth item
  5181. * in the schedule will never be used.
  5182. *
  5183. * (We find the zeroth element of the download schedule, and set
  5184. * next_attempt_at to be the appropriate offset from 'now'. In most
  5185. * cases this means setting it to 'now', so the item will be immediately
  5186. * downloadable; when using authorities with fallbacks, there is a few seconds'
  5187. * delay.) */
  5188. void
  5189. download_status_reset(download_status_t *dls)
  5190. {
  5191. if (dls->n_download_failures == IMPOSSIBLE_TO_DOWNLOAD
  5192. || dls->n_download_attempts == IMPOSSIBLE_TO_DOWNLOAD)
  5193. return; /* Don't reset this. */
  5194. dls->n_download_failures = 0;
  5195. dls->n_download_attempts = 0;
  5196. dls->next_attempt_at = download_status_get_initial_delay_from_now(dls);
  5197. dls->last_backoff_position = 0;
  5198. dls->last_delay_used = 0;
  5199. /* Don't reset dls->want_authority or dls->increment_on */
  5200. }
  5201. /** Return the number of failures on <b>dls</b> since the last success (if
  5202. * any). */
  5203. int
  5204. download_status_get_n_failures(const download_status_t *dls)
  5205. {
  5206. return dls->n_download_failures;
  5207. }
  5208. /** Return the number of attempts to download <b>dls</b> since the last success
  5209. * (if any). This can differ from download_status_get_n_failures() due to
  5210. * outstanding concurrent attempts. */
  5211. int
  5212. download_status_get_n_attempts(const download_status_t *dls)
  5213. {
  5214. return dls->n_download_attempts;
  5215. }
  5216. /** Return the next time to attempt to download <b>dls</b>. */
  5217. time_t
  5218. download_status_get_next_attempt_at(const download_status_t *dls)
  5219. {
  5220. /* dls wasn't reset before it was used */
  5221. if (dls->next_attempt_at == 0) {
  5222. /* so give the answer we would have given if it had been */
  5223. return download_status_get_initial_delay_from_now(dls);
  5224. }
  5225. return dls->next_attempt_at;
  5226. }
  5227. /** Called when one or more routerdesc (or extrainfo, if <b>was_extrainfo</b>)
  5228. * fetches have failed (with uppercase fingerprints listed in <b>failed</b>,
  5229. * either as descriptor digests or as identity digests based on
  5230. * <b>was_descriptor_digests</b>).
  5231. */
  5232. static void
  5233. dir_routerdesc_download_failed(smartlist_t *failed, int status_code,
  5234. int router_purpose,
  5235. int was_extrainfo, int was_descriptor_digests)
  5236. {
  5237. char digest[DIGEST_LEN];
  5238. time_t now = time(NULL);
  5239. int server = directory_fetches_from_authorities(get_options());
  5240. if (!was_descriptor_digests) {
  5241. if (router_purpose == ROUTER_PURPOSE_BRIDGE) {
  5242. tor_assert(!was_extrainfo);
  5243. connection_dir_retry_bridges(failed);
  5244. }
  5245. return; /* FFFF should implement for other-than-router-purpose someday */
  5246. }
  5247. SMARTLIST_FOREACH_BEGIN(failed, const char *, cp) {
  5248. download_status_t *dls = NULL;
  5249. if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) != DIGEST_LEN) {
  5250. log_warn(LD_BUG, "Malformed fingerprint in list: %s", escaped(cp));
  5251. continue;
  5252. }
  5253. if (was_extrainfo) {
  5254. signed_descriptor_t *sd =
  5255. router_get_by_extrainfo_digest(digest);
  5256. if (sd)
  5257. dls = &sd->ei_dl_status;
  5258. } else {
  5259. dls = router_get_dl_status_by_descriptor_digest(digest);
  5260. }
  5261. if (!dls || dls->n_download_failures >=
  5262. get_options()->TestingDescriptorMaxDownloadTries)
  5263. continue;
  5264. download_status_increment_failure(dls, status_code, cp, server, now);
  5265. } SMARTLIST_FOREACH_END(cp);
  5266. /* No need to relaunch descriptor downloads here: we already do it
  5267. * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
  5268. }
  5269. /** Called when a connection to download microdescriptors from relay with
  5270. * <b>dir_id</b> has failed in whole or in part. <b>failed</b> is a list
  5271. * of every microdesc digest we didn't get. <b>status_code</b> is the http
  5272. * status code we received. Reschedule the microdesc downloads as
  5273. * appropriate. */
  5274. static void
  5275. dir_microdesc_download_failed(smartlist_t *failed,
  5276. int status_code, const char *dir_id)
  5277. {
  5278. networkstatus_t *consensus
  5279. = networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
  5280. routerstatus_t *rs;
  5281. download_status_t *dls;
  5282. time_t now = time(NULL);
  5283. int server = directory_fetches_from_authorities(get_options());
  5284. if (! consensus)
  5285. return;
  5286. /* We failed to fetch a microdescriptor from 'dir_id', note it down
  5287. * so that we don't try the same relay next time... */
  5288. microdesc_note_outdated_dirserver(dir_id);
  5289. SMARTLIST_FOREACH_BEGIN(failed, const char *, d) {
  5290. rs = router_get_mutable_consensus_status_by_descriptor_digest(consensus,d);
  5291. if (!rs)
  5292. continue;
  5293. dls = &rs->dl_status;
  5294. if (dls->n_download_failures >=
  5295. get_options()->TestingMicrodescMaxDownloadTries) {
  5296. continue;
  5297. }
  5298. { /* Increment the failure count for this md fetch */
  5299. char buf[BASE64_DIGEST256_LEN+1];
  5300. digest256_to_base64(buf, d);
  5301. log_info(LD_DIR, "Failed to download md %s from %s",
  5302. buf, hex_str(dir_id, DIGEST_LEN));
  5303. download_status_increment_failure(dls, status_code, buf,
  5304. server, now);
  5305. }
  5306. } SMARTLIST_FOREACH_END(d);
  5307. }
  5308. /** Helper. Compare two fp_pair_t objects, and return negative, 0, or
  5309. * positive as appropriate. */
  5310. static int
  5311. compare_pairs_(const void **a, const void **b)
  5312. {
  5313. const fp_pair_t *fp1 = *a, *fp2 = *b;
  5314. int r;
  5315. if ((r = fast_memcmp(fp1->first, fp2->first, DIGEST_LEN)))
  5316. return r;
  5317. else
  5318. return fast_memcmp(fp1->second, fp2->second, DIGEST_LEN);
  5319. }
  5320. /** Divide a string <b>res</b> of the form FP1-FP2+FP3-FP4...[.z], where each
  5321. * FP is a hex-encoded fingerprint, into a sequence of distinct sorted
  5322. * fp_pair_t. Skip malformed pairs. On success, return 0 and add those
  5323. * fp_pair_t into <b>pairs_out</b>. On failure, return -1. */
  5324. int
  5325. dir_split_resource_into_fingerprint_pairs(const char *res,
  5326. smartlist_t *pairs_out)
  5327. {
  5328. smartlist_t *pairs_tmp = smartlist_new();
  5329. smartlist_t *pairs_result = smartlist_new();
  5330. smartlist_split_string(pairs_tmp, res, "+", 0, 0);
  5331. if (smartlist_len(pairs_tmp)) {
  5332. char *last = smartlist_get(pairs_tmp,smartlist_len(pairs_tmp)-1);
  5333. size_t last_len = strlen(last);
  5334. if (last_len > 2 && !strcmp(last+last_len-2, ".z")) {
  5335. last[last_len-2] = '\0';
  5336. }
  5337. }
  5338. SMARTLIST_FOREACH_BEGIN(pairs_tmp, char *, cp) {
  5339. if (strlen(cp) != HEX_DIGEST_LEN*2+1) {
  5340. log_info(LD_DIR,
  5341. "Skipping digest pair %s with non-standard length.", escaped(cp));
  5342. } else if (cp[HEX_DIGEST_LEN] != '-') {
  5343. log_info(LD_DIR,
  5344. "Skipping digest pair %s with missing dash.", escaped(cp));
  5345. } else {
  5346. fp_pair_t pair;
  5347. if (base16_decode(pair.first, DIGEST_LEN,
  5348. cp, HEX_DIGEST_LEN) != DIGEST_LEN ||
  5349. base16_decode(pair.second,DIGEST_LEN,
  5350. cp+HEX_DIGEST_LEN+1, HEX_DIGEST_LEN) != DIGEST_LEN) {
  5351. log_info(LD_DIR, "Skipping non-decodable digest pair %s", escaped(cp));
  5352. } else {
  5353. smartlist_add(pairs_result, tor_memdup(&pair, sizeof(pair)));
  5354. }
  5355. }
  5356. tor_free(cp);
  5357. } SMARTLIST_FOREACH_END(cp);
  5358. smartlist_free(pairs_tmp);
  5359. /* Uniq-and-sort */
  5360. smartlist_sort(pairs_result, compare_pairs_);
  5361. smartlist_uniq(pairs_result, compare_pairs_, tor_free_);
  5362. smartlist_add_all(pairs_out, pairs_result);
  5363. smartlist_free(pairs_result);
  5364. return 0;
  5365. }
  5366. /** Given a directory <b>resource</b> request, containing zero
  5367. * or more strings separated by plus signs, followed optionally by ".z", store
  5368. * the strings, in order, into <b>fp_out</b>. If <b>compressed_out</b> is
  5369. * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0.
  5370. *
  5371. * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and
  5372. * decode the rest. If (flags & DSR_BASE64), then use "-" rather than "+" as
  5373. * a separator, delete all the elements that aren't base64-encoded digests,
  5374. * and decode the rest. If (flags & DSR_DIGEST256), these digests should be
  5375. * 256 bits long; else they should be 160.
  5376. *
  5377. * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates.
  5378. */
  5379. int
  5380. dir_split_resource_into_fingerprints(const char *resource,
  5381. smartlist_t *fp_out, int *compressed_out,
  5382. int flags)
  5383. {
  5384. const int decode_hex = flags & DSR_HEX;
  5385. const int decode_base64 = flags & DSR_BASE64;
  5386. const int digests_are_256 = flags & DSR_DIGEST256;
  5387. const int sort_uniq = flags & DSR_SORT_UNIQ;
  5388. const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN;
  5389. const int hex_digest_len = digests_are_256 ?
  5390. HEX_DIGEST256_LEN : HEX_DIGEST_LEN;
  5391. const int base64_digest_len = digests_are_256 ?
  5392. BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN;
  5393. smartlist_t *fp_tmp = smartlist_new();
  5394. tor_assert(!(decode_hex && decode_base64));
  5395. tor_assert(fp_out);
  5396. smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0);
  5397. if (compressed_out)
  5398. *compressed_out = 0;
  5399. if (smartlist_len(fp_tmp)) {
  5400. char *last = smartlist_get(fp_tmp,smartlist_len(fp_tmp)-1);
  5401. size_t last_len = strlen(last);
  5402. if (last_len > 2 && !strcmp(last+last_len-2, ".z")) {
  5403. last[last_len-2] = '\0';
  5404. if (compressed_out)
  5405. *compressed_out = 1;
  5406. }
  5407. }
  5408. if (decode_hex || decode_base64) {
  5409. const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len;
  5410. int i;
  5411. char *cp, *d = NULL;
  5412. for (i = 0; i < smartlist_len(fp_tmp); ++i) {
  5413. cp = smartlist_get(fp_tmp, i);
  5414. if (strlen(cp) != encoded_len) {
  5415. log_info(LD_DIR,
  5416. "Skipping digest %s with non-standard length.", escaped(cp));
  5417. smartlist_del_keeporder(fp_tmp, i--);
  5418. goto again;
  5419. }
  5420. d = tor_malloc_zero(digest_len);
  5421. if (decode_hex ?
  5422. (base16_decode(d, digest_len, cp, hex_digest_len) != digest_len) :
  5423. (base64_decode(d, digest_len, cp, base64_digest_len)
  5424. != digest_len)) {
  5425. log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp));
  5426. smartlist_del_keeporder(fp_tmp, i--);
  5427. goto again;
  5428. }
  5429. smartlist_set(fp_tmp, i, d);
  5430. d = NULL;
  5431. again:
  5432. tor_free(cp);
  5433. tor_free(d);
  5434. }
  5435. }
  5436. if (sort_uniq) {
  5437. if (decode_hex || decode_base64) {
  5438. if (digests_are_256) {
  5439. smartlist_sort_digests256(fp_tmp);
  5440. smartlist_uniq_digests256(fp_tmp);
  5441. } else {
  5442. smartlist_sort_digests(fp_tmp);
  5443. smartlist_uniq_digests(fp_tmp);
  5444. }
  5445. } else {
  5446. smartlist_sort_strings(fp_tmp);
  5447. smartlist_uniq_strings(fp_tmp);
  5448. }
  5449. }
  5450. smartlist_add_all(fp_out, fp_tmp);
  5451. smartlist_free(fp_tmp);
  5452. return 0;
  5453. }
  5454. /** As dir_split_resource_into_fingerprints, but instead fills
  5455. * <b>spool_out</b> with a list of spoolable_resource_t for the resource
  5456. * identified through <b>source</b>. */
  5457. int
  5458. dir_split_resource_into_spoolable(const char *resource,
  5459. dir_spool_source_t source,
  5460. smartlist_t *spool_out,
  5461. int *compressed_out,
  5462. int flags)
  5463. {
  5464. smartlist_t *fingerprints = smartlist_new();
  5465. tor_assert(flags & (DSR_HEX|DSR_BASE64));
  5466. const size_t digest_len =
  5467. (flags & DSR_DIGEST256) ? DIGEST256_LEN : DIGEST_LEN;
  5468. int r = dir_split_resource_into_fingerprints(resource, fingerprints,
  5469. compressed_out, flags);
  5470. /* This is not a very efficient implementation XXXX */
  5471. SMARTLIST_FOREACH_BEGIN(fingerprints, uint8_t *, digest) {
  5472. spooled_resource_t *spooled =
  5473. spooled_resource_new(source, digest, digest_len);
  5474. if (spooled)
  5475. smartlist_add(spool_out, spooled);
  5476. tor_free(digest);
  5477. } SMARTLIST_FOREACH_END(digest);
  5478. smartlist_free(fingerprints);
  5479. return r;
  5480. }