fetch-all-v3 3.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110
  1. #!/bin/bash
  2. # Download all current v3 directory status votes and the consensus document,
  3. # then download the descriptors and extra info documents.
  4. # Copyright (c) 2005, 2006, 2007, 2008 Peter Palfrader
  5. #
  6. # Permission is hereby granted, free of charge, to any person obtaining a copy
  7. # of this software and associated documentation files (the "Software"), to deal
  8. # in the Software without restriction, including without limitation the rights
  9. # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10. # copies of the Software, and to permit persons to whom the Software is
  11. # furnished to do so, subject to the following conditions:
  12. #
  13. # The above copyright notice and this permission notice shall be included in
  14. # all copies or substantial portions of the Software.
  15. #
  16. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  19. # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  22. # SOFTWARE.
  23. TZ=UTC
  24. export TZ
  25. DIRSERVERS=""
  26. DIRSERVERS="$DIRSERVERS 86.59.21.38:80" # tor26
  27. DIRSERVERS="$DIRSERVERS 128.31.0.34:9031" # moria1
  28. DIRSERVERS="$DIRSERVERS 216.224.124.114:9030" # ides
  29. DIRSERVERS="$DIRSERVERS 80.190.246.100:80" # gabelmoo
  30. #DIRSERVERS="$DIRSERVERS 140.247.60.64:80" # lefkada
  31. DIRSERVERS="$DIRSERVERS 194.109.206.212:80" # dizum
  32. #DIRSERVERS="$DIRSERVERS 128.31.0.34:9032" # moria2
  33. DIRSERVERS="$DIRSERVERS 213.73.91.31:80" # dannenberg
  34. TIME=$(date "+%Y%m%d-%H%M%S")
  35. . fetch-all-functions
  36. consensus=""
  37. tmpdir="consensus/tmp"
  38. [ -d "$tmpdir" ] || mkdir -p "$tmpdir"
  39. for dirserver in $DIRSERVERS; do
  40. wget -q -O "$tmpdir/$TIME-consensus" http://$dirserver/tor/status-vote/current/consensus
  41. if [ "$?" != 0 ]; then
  42. rm -f "$tmpdir/$TIME-consensus"
  43. continue
  44. fi
  45. freshconsensus="$tmpdir/$TIME-consensus"
  46. timestamp=$(awk '$1=="valid-after" {printf "%s-%s", $2, $3}' < "$freshconsensus")
  47. datedir=$(awk '$1=="valid-after" {printf "%s", $2}' < "$freshconsensus" | tr '-' '/')
  48. dir="consensus/$datedir"
  49. [ -d "$dir" ] || mkdir -p "$dir"
  50. consensus="$dir/$timestamp-consensus.bz2"
  51. if ! [ -e "$consensus" ]; then
  52. # the consensus is new, or at least we don't have it yet
  53. bzip2 "$freshconsensus"
  54. mv "$freshconsensus.bz2" "$consensus"
  55. break
  56. fi
  57. rm -f "$freshconsensus"
  58. echo "Consensus from $timestamp (gotten from $dirserver) already exists!" >&2
  59. # maybe there is a newer one on a different authority, so try again.
  60. done
  61. if [ "$consensus" = "" ]; then
  62. echo "No consensus available" 2>&1
  63. exit 1
  64. fi
  65. votes=$(bzcat $consensus | awk '$1 == "vote-digest" {print $2}')
  66. for vote in $votes; do
  67. for dirserver in $DIRSERVERS; do
  68. wget -q -O "$dir/$TIME-vote-$vote" http://$dirserver/tor/status-vote/current/d/$vote
  69. if [ "$?" != 0 ]; then
  70. rm -f "$dir/$TIME-vote-$vote"
  71. continue
  72. fi
  73. break
  74. done
  75. if [ -e "$dir/$TIME-vote-$vote" ]; then
  76. voteridentity=$(awk '$1=="fingerprint" {print $2}' < "$dir/$TIME-vote-$vote")
  77. if [ -e "$dir/$timestamp-vote-$voteridentity-$vote.bz2" ]; then
  78. echo "Vote $vote from $voteridentity already exists!" >&2
  79. rm -f "$dir/$TIME-vote-$vote"
  80. continue;
  81. fi
  82. mv "$dir/$TIME-vote-$vote" "$dir/$timestamp-vote-$voteridentity-$vote"
  83. bzip2 "$dir/$timestamp-vote-$voteridentity-$vote"
  84. else
  85. echo "Failed to get vote $vote!" >&2
  86. fi
  87. done
  88. digests=$( for i in ` bzcat $consensus | awk '$1 == "r" {printf "%s=\n", $4}' | sort -u `; do
  89. echo $i | \
  90. base64-decode | \
  91. perl -e 'undef $/; $a=<>; print unpack("H\*", $a),"\n";';
  92. done )
  93. for digest in $digests; do
  94. fetch_digest "$digest" "server-descriptor"
  95. done