redox.py 7.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228
  1. #!/usr/bin/python
  2. #
  3. # Copyright (c) 2008-2013, The Tor Project, Inc.
  4. # See LICENSE for licensing information.
  5. #
  6. # Hi!
  7. # I'm redox.py, the Tor redocumentation tool!
  8. # I am a horrible hack!
  9. # I read the output of doxygen from stderr, and add missing DOCDOC comments
  10. # to tell you where documentation should go!
  11. # To use me, edit the stuff below...
  12. # ...and run 'make doxygen 2>doxygen.stderr' ...
  13. # ...and run ./scripts/maint/redox.py < doxygen.stderr !
  14. # I'll make a bunch of new files by adding missing DOCDOC comments to your
  15. # source. Those files will have names like ./src/common/util.c.newdoc.
  16. # You will want to look over the changes by hand before checking them in.
  17. #
  18. # So, here's your workflow:
  19. #
  20. # 0. Make sure you're running a bourne shell for the redirects below.
  21. # 1. make doxygen 1>doxygen.stdout 2>doxygen.stderr.
  22. # 2. grep Warning doxygen.stderr | grep -v 'is not documented' | less
  23. # [This will tell you about all the bogus doxygen output you have]
  24. # 3. python ./scripts/maint/redox.py <doxygen.stderr
  25. # [This will make lots of .newdoc files with DOCDOC comments for
  26. # whatever was missing documentation.]
  27. # 4. Look over those .newdoc files, and see which docdoc comments you
  28. # want to merge into the main file. If it's all good, just run
  29. # "mv fname.c.newdoc fname.c". Otherwise, you'll need to merge
  30. # the parts you like by hand.
  31. # Which files should we ignore warning from? Mostly, these are external
  32. # files that we've snarfed in from somebody else, whose C we do no intend
  33. # to document for them.
  34. SKIP_FILES = [ "OpenBSD_malloc_Linux.c",
  35. "eventdns.c",
  36. "eventdns.h",
  37. "strlcat.c",
  38. "strlcpy.c",
  39. "sha256.c",
  40. "sha256.h",
  41. "aes.c",
  42. "aes.h" ]
  43. # What names of things never need javadoc
  44. SKIP_NAME_PATTERNS = [ r'^.*_c_id$',
  45. r'^.*_H_ID$' ]
  46. # Which types of things should get DOCDOC comments added if they are
  47. # missing documentation? Recognized types are in KINDS below.
  48. ADD_DOCDOCS_TO_TYPES = [ 'function', 'type', 'typedef' ]
  49. ADD_DOCDOCS_TO_TYPES += [ 'variable', ]
  50. # ====================
  51. # The rest of this should not need hacking.
  52. import re
  53. import sys
  54. KINDS = [ "type", "field", "typedef", "define", "function", "variable",
  55. "enumeration" ]
  56. NODOC_LINE_RE = re.compile(r'^([^:]+):(\d+): (\w+): (.*) is not documented\.$')
  57. THING_RE = re.compile(r'^Member ([a-zA-Z0-9_]+).*\((typedef|define|function|variable|enumeration|macro definition)\) of (file|class) ')
  58. SKIP_NAMES = [re.compile(s) for s in SKIP_NAME_PATTERNS]
  59. def parsething(thing):
  60. """I figure out what 'foobar baz in quux quum is not documented' means,
  61. and return: the name of the foobar, and the kind of the foobar.
  62. """
  63. if thing.startswith("Compound "):
  64. tp, name = "type", thing.split()[1]
  65. else:
  66. m = THING_RE.match(thing)
  67. if not m:
  68. print thing, "???? Format didn't match."
  69. return None, None
  70. else:
  71. name, tp, parent = m.groups()
  72. if parent == 'class':
  73. if tp == 'variable' or tp == 'function':
  74. tp = 'field'
  75. return name, tp
  76. def read():
  77. """I snarf doxygen stderr from stdin, and parse all the "foo has no
  78. documentation messages. I return a map from filename to lists
  79. of tuples of (alleged line number, name of thing, kind of thing)
  80. """
  81. errs = {}
  82. for line in sys.stdin:
  83. m = NODOC_LINE_RE.match(line)
  84. if m:
  85. file, line, tp, thing = m.groups()
  86. assert tp.lower() == 'warning'
  87. name, kind = parsething(thing)
  88. errs.setdefault(file, []).append((int(line), name, kind))
  89. return errs
  90. def findline(lines, lineno, ident):
  91. """Given a list of all the lines in the file (adjusted so 1-indexing works),
  92. a line number that ident is alledgedly on, and ident, I figure out
  93. the line where ident was really declared."""
  94. lno = lineno
  95. for lineno in xrange(lineno, 0, -1):
  96. try:
  97. if ident in lines[lineno]:
  98. return lineno
  99. except IndexError:
  100. continue
  101. return None
  102. FUNC_PAT = re.compile(r"^[A-Za-z0-9_]+\(")
  103. def hascomment(lines, lineno, kind):
  104. """I return true if it looks like there's already a good comment about
  105. the thing on lineno of lines of type kind. """
  106. if "*/" in lines[lineno-1]:
  107. return True
  108. if kind == 'function' and FUNC_PAT.match(lines[lineno]):
  109. if "*/" in lines[lineno-2]:
  110. return True
  111. return False
  112. def hasdocdoc(lines, lineno, kind):
  113. """I return true if it looks like there's already a docdoc comment about
  114. the thing on lineno of lines of type kind."""
  115. try:
  116. if "DOCDOC" in lines[lineno]:
  117. return True
  118. except IndexError:
  119. pass
  120. try:
  121. if "DOCDOC" in lines[lineno-1]:
  122. return True
  123. except IndexError:
  124. pass
  125. if kind == 'function' and FUNC_PAT.match(lines[lineno]):
  126. if "DOCDOC" in lines[lineno-2]:
  127. return True
  128. return False
  129. def checkf(fn, errs):
  130. """I go through the output of read() for a single file, and build a list
  131. of tuples of things that want DOCDOC comments. Each tuple has:
  132. the line number where the comment goes; the kind of thing; its name.
  133. """
  134. for skip in SKIP_FILES:
  135. if fn.endswith(skip):
  136. print "Skipping",fn
  137. return
  138. comments = []
  139. lines = [ None ]
  140. try:
  141. lines.extend( open(fn, 'r').readlines() )
  142. except IOError:
  143. return
  144. for line, name, kind in errs:
  145. if any(pat.match(name) for pat in SKIP_NAMES):
  146. continue
  147. if kind not in ADD_DOCDOCS_TO_TYPES:
  148. continue
  149. ln = findline(lines, line, name)
  150. if ln == None:
  151. print "Couldn't find the definition of %s allegedly on %s of %s"%(
  152. name, line, fn)
  153. else:
  154. if hasdocdoc(lines, line, kind):
  155. # print "Has a DOCDOC"
  156. # print fn, line, name, kind
  157. # print "\t",lines[line-2],
  158. # print "\t",lines[line-1],
  159. # print "\t",lines[line],
  160. # print "-------"
  161. pass
  162. else:
  163. if kind == 'function' and FUNC_PAT.match(lines[ln]):
  164. ln = ln - 1
  165. comments.append((ln, kind, name))
  166. return comments
  167. def applyComments(fn, entries):
  168. """I apply lots of comments to the file in fn, making a new .newdoc file.
  169. """
  170. N = 0
  171. lines = [ None ]
  172. try:
  173. lines.extend( open(fn, 'r').readlines() )
  174. except IOError:
  175. return
  176. # Process the comments in reverse order by line number, so that
  177. # the line numbers for the ones we haven't added yet remain valid
  178. # until we add them. Standard trick.
  179. entries.sort()
  180. entries.reverse()
  181. for ln, kind, name in entries:
  182. lines.insert(ln, "/* DOCDOC %s */\n"%name)
  183. N += 1
  184. outf = open(fn+".newdoc", 'w')
  185. for line in lines[1:]:
  186. outf.write(line)
  187. outf.close()
  188. print "Added %s DOCDOCs to %s" %(N, fn)
  189. e = read()
  190. for fn, errs in e.iteritems():
  191. print `(fn, errs)`
  192. comments = checkf(fn, errs)
  193. if comments:
  194. applyComments(fn, comments)