fetch-all-v3 3.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111
  1. #!/bin/bash
  2. # Download all current v3 directory status votes and the consensus document,
  3. # then download the descriptors and extra info documents.
  4. # Copyright (c) 2005, 2006, 2007, 2008 Peter Palfrader
  5. #
  6. # Permission is hereby granted, free of charge, to any person obtaining a copy
  7. # of this software and associated documentation files (the "Software"), to deal
  8. # in the Software without restriction, including without limitation the rights
  9. # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10. # copies of the Software, and to permit persons to whom the Software is
  11. # furnished to do so, subject to the following conditions:
  12. #
  13. # The above copyright notice and this permission notice shall be included in
  14. # all copies or substantial portions of the Software.
  15. #
  16. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  19. # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  22. # SOFTWARE.
  23. TZ=UTC
  24. export TZ
  25. DIRSERVERS=""
  26. DIRSERVERS="$DIRSERVERS 86.59.21.38:80" # tor26
  27. DIRSERVERS="$DIRSERVERS 128.31.0.34:9031" # moria1
  28. DIRSERVERS="$DIRSERVERS 216.224.124.114:9030" # ides
  29. DIRSERVERS="$DIRSERVERS 80.190.246.100:80" # gabelmoo
  30. #DIRSERVERS="$DIRSERVERS 140.247.60.64:80" # lefkada
  31. DIRSERVERS="$DIRSERVERS 194.109.206.212:80" # dizum
  32. #DIRSERVERS="$DIRSERVERS 128.31.0.34:9032" # moria2
  33. DIRSERVERS="$DIRSERVERS 213.73.91.31:80" # dannenberg
  34. DIRSERVERS="$DIRSERVERS 208.83.223.34:443" # urras
  35. TIME=$(date "+%Y%m%d-%H%M%S")
  36. . fetch-all-functions
  37. consensus=""
  38. tmpdir="consensus/tmp"
  39. [ -d "$tmpdir" ] || mkdir -p "$tmpdir"
  40. for dirserver in $DIRSERVERS; do
  41. wget -q -O "$tmpdir/$TIME-consensus" http://$dirserver/tor/status-vote/current/consensus
  42. if [ "$?" != 0 ]; then
  43. rm -f "$tmpdir/$TIME-consensus"
  44. continue
  45. fi
  46. freshconsensus="$tmpdir/$TIME-consensus"
  47. timestamp=$(awk '$1=="valid-after" {printf "%s-%s", $2, $3}' < "$freshconsensus")
  48. datedir=$(awk '$1=="valid-after" {printf "%s", $2}' < "$freshconsensus" | tr '-' '/')
  49. dir="consensus/$datedir"
  50. [ -d "$dir" ] || mkdir -p "$dir"
  51. consensus="$dir/$timestamp-consensus.bz2"
  52. if ! [ -e "$consensus" ]; then
  53. # the consensus is new, or at least we don't have it yet
  54. bzip2 "$freshconsensus"
  55. mv "$freshconsensus.bz2" "$consensus"
  56. break
  57. fi
  58. rm -f "$freshconsensus"
  59. echo "Consensus from $timestamp (gotten from $dirserver) already exists!" >&2
  60. # maybe there is a newer one on a different authority, so try again.
  61. done
  62. if [ "$consensus" = "" ]; then
  63. echo "No consensus available" 2>&1
  64. exit 1
  65. fi
  66. votes=$(bzcat $consensus | awk '$1 == "vote-digest" {print $2}')
  67. for vote in $votes; do
  68. for dirserver in $DIRSERVERS; do
  69. wget -q -O "$dir/$TIME-vote-$vote" http://$dirserver/tor/status-vote/current/d/$vote
  70. if [ "$?" != 0 ]; then
  71. rm -f "$dir/$TIME-vote-$vote"
  72. continue
  73. fi
  74. break
  75. done
  76. if [ -e "$dir/$TIME-vote-$vote" ]; then
  77. voteridentity=$(awk '$1=="fingerprint" {print $2}' < "$dir/$TIME-vote-$vote")
  78. if [ -e "$dir/$timestamp-vote-$voteridentity-$vote.bz2" ]; then
  79. echo "Vote $vote from $voteridentity already exists!" >&2
  80. rm -f "$dir/$TIME-vote-$vote"
  81. continue;
  82. fi
  83. mv "$dir/$TIME-vote-$vote" "$dir/$timestamp-vote-$voteridentity-$vote"
  84. bzip2 "$dir/$timestamp-vote-$voteridentity-$vote"
  85. else
  86. echo "Failed to get vote $vote!" >&2
  87. fi
  88. done
  89. digests=$( for i in ` bzcat $consensus | awk '$1 == "r" {printf "%s=\n", $4}' | sort -u `; do
  90. echo $i | \
  91. base64-decode | \
  92. perl -e 'undef $/; $a=<>; print unpack("H\*", $a),"\n";';
  93. done )
  94. for digest in $digests; do
  95. fetch_digest "$digest" "server-descriptor"
  96. done