redox.py 7.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226
  1. #!/usr/bin/python
  2. #
  3. # Copyright (c) 2008-2017, The Tor Project, Inc.
  4. # See LICENSE for licensing information.
  5. #
  6. # Hi!
  7. # I'm redox.py, the Tor redocumentation tool!
  8. # I am a horrible hack!
  9. # I read the output of doxygen from stderr, and add missing DOCDOC comments
  10. # to tell you where documentation should go!
  11. # To use me, edit the stuff below...
  12. # ...and run 'make doxygen 2>doxygen.stderr' ...
  13. # ...and run ./scripts/maint/redox.py < doxygen.stderr !
  14. # I'll make a bunch of new files by adding missing DOCDOC comments to your
  15. # source. Those files will have names like ./src/common/util.c.newdoc.
  16. # You will want to look over the changes by hand before checking them in.
  17. #
  18. # So, here's your workflow:
  19. #
  20. # 0. Make sure you're running a bourne shell for the redirects below.
  21. # 1. make doxygen 1>doxygen.stdout 2>doxygen.stderr.
  22. # 2. grep Warning doxygen.stderr | grep -v 'is not documented' | less
  23. # [This will tell you about all the bogus doxygen output you have]
  24. # 3. python ./scripts/maint/redox.py <doxygen.stderr
  25. # [This will make lots of .newdoc files with DOCDOC comments for
  26. # whatever was missing documentation.]
  27. # 4. Look over those .newdoc files, and see which docdoc comments you
  28. # want to merge into the main file. If it's all good, just run
  29. # "mv fname.c.newdoc fname.c". Otherwise, you'll need to merge
  30. # the parts you like by hand.
  31. # Which files should we ignore warning from? Mostly, these are external
  32. # files that we've snarfed in from somebody else, whose C we do no intend
  33. # to document for them.
  34. SKIP_FILES = [ "OpenBSD_malloc_Linux.c",
  35. "strlcat.c",
  36. "strlcpy.c",
  37. "sha256.c",
  38. "sha256.h",
  39. "aes.c",
  40. "aes.h" ]
  41. # What names of things never need javadoc
  42. SKIP_NAME_PATTERNS = [ r'^.*_c_id$',
  43. r'^.*_H_ID$' ]
  44. # Which types of things should get DOCDOC comments added if they are
  45. # missing documentation? Recognized types are in KINDS below.
  46. ADD_DOCDOCS_TO_TYPES = [ 'function', 'type', 'typedef' ]
  47. ADD_DOCDOCS_TO_TYPES += [ 'variable', ]
  48. # ====================
  49. # The rest of this should not need hacking.
  50. import re
  51. import sys
  52. KINDS = [ "type", "field", "typedef", "define", "function", "variable",
  53. "enumeration" ]
  54. NODOC_LINE_RE = re.compile(r'^([^:]+):(\d+): (\w+): (.*) is not documented\.$')
  55. THING_RE = re.compile(r'^Member ([a-zA-Z0-9_]+).*\((typedef|define|function|variable|enumeration|macro definition)\) of (file|class) ')
  56. SKIP_NAMES = [re.compile(s) for s in SKIP_NAME_PATTERNS]
  57. def parsething(thing):
  58. """I figure out what 'foobar baz in quux quum is not documented' means,
  59. and return: the name of the foobar, and the kind of the foobar.
  60. """
  61. if thing.startswith("Compound "):
  62. tp, name = "type", thing.split()[1]
  63. else:
  64. m = THING_RE.match(thing)
  65. if not m:
  66. print thing, "???? Format didn't match."
  67. return None, None
  68. else:
  69. name, tp, parent = m.groups()
  70. if parent == 'class':
  71. if tp == 'variable' or tp == 'function':
  72. tp = 'field'
  73. return name, tp
  74. def read():
  75. """I snarf doxygen stderr from stdin, and parse all the "foo has no
  76. documentation messages. I return a map from filename to lists
  77. of tuples of (alleged line number, name of thing, kind of thing)
  78. """
  79. errs = {}
  80. for line in sys.stdin:
  81. m = NODOC_LINE_RE.match(line)
  82. if m:
  83. file, line, tp, thing = m.groups()
  84. assert tp.lower() == 'warning'
  85. name, kind = parsething(thing)
  86. errs.setdefault(file, []).append((int(line), name, kind))
  87. return errs
  88. def findline(lines, lineno, ident):
  89. """Given a list of all the lines in the file (adjusted so 1-indexing works),
  90. a line number that ident is allegedly on, and ident, I figure out
  91. the line where ident was really declared."""
  92. lno = lineno
  93. for lineno in xrange(lineno, 0, -1):
  94. try:
  95. if ident in lines[lineno]:
  96. return lineno
  97. except IndexError:
  98. continue
  99. return None
  100. FUNC_PAT = re.compile(r"^[A-Za-z0-9_]+\(")
  101. def hascomment(lines, lineno, kind):
  102. """I return true if it looks like there's already a good comment about
  103. the thing on lineno of lines of type kind. """
  104. if "*/" in lines[lineno-1]:
  105. return True
  106. if kind == 'function' and FUNC_PAT.match(lines[lineno]):
  107. if "*/" in lines[lineno-2]:
  108. return True
  109. return False
  110. def hasdocdoc(lines, lineno, kind):
  111. """I return true if it looks like there's already a docdoc comment about
  112. the thing on lineno of lines of type kind."""
  113. try:
  114. if "DOCDOC" in lines[lineno]:
  115. return True
  116. except IndexError:
  117. pass
  118. try:
  119. if "DOCDOC" in lines[lineno-1]:
  120. return True
  121. except IndexError:
  122. pass
  123. if kind == 'function' and FUNC_PAT.match(lines[lineno]):
  124. if "DOCDOC" in lines[lineno-2]:
  125. return True
  126. return False
  127. def checkf(fn, errs):
  128. """I go through the output of read() for a single file, and build a list
  129. of tuples of things that want DOCDOC comments. Each tuple has:
  130. the line number where the comment goes; the kind of thing; its name.
  131. """
  132. for skip in SKIP_FILES:
  133. if fn.endswith(skip):
  134. print "Skipping",fn
  135. return
  136. comments = []
  137. lines = [ None ]
  138. try:
  139. lines.extend( open(fn, 'r').readlines() )
  140. except IOError:
  141. return
  142. for line, name, kind in errs:
  143. if any(pat.match(name) for pat in SKIP_NAMES):
  144. continue
  145. if kind not in ADD_DOCDOCS_TO_TYPES:
  146. continue
  147. ln = findline(lines, line, name)
  148. if ln == None:
  149. print "Couldn't find the definition of %s allegedly on %s of %s"%(
  150. name, line, fn)
  151. else:
  152. if hasdocdoc(lines, line, kind):
  153. # print "Has a DOCDOC"
  154. # print fn, line, name, kind
  155. # print "\t",lines[line-2],
  156. # print "\t",lines[line-1],
  157. # print "\t",lines[line],
  158. # print "-------"
  159. pass
  160. else:
  161. if kind == 'function' and FUNC_PAT.match(lines[ln]):
  162. ln = ln - 1
  163. comments.append((ln, kind, name))
  164. return comments
  165. def applyComments(fn, entries):
  166. """I apply lots of comments to the file in fn, making a new .newdoc file.
  167. """
  168. N = 0
  169. lines = [ None ]
  170. try:
  171. lines.extend( open(fn, 'r').readlines() )
  172. except IOError:
  173. return
  174. # Process the comments in reverse order by line number, so that
  175. # the line numbers for the ones we haven't added yet remain valid
  176. # until we add them. Standard trick.
  177. entries.sort()
  178. entries.reverse()
  179. for ln, kind, name in entries:
  180. lines.insert(ln, "/* DOCDOC %s */\n"%name)
  181. N += 1
  182. outf = open(fn+".newdoc", 'w')
  183. for line in lines[1:]:
  184. outf.write(line)
  185. outf.close()
  186. print "Added %s DOCDOCs to %s" %(N, fn)
  187. e = read()
  188. for fn, errs in e.iteritems():
  189. print `(fn, errs)`
  190. comments = checkf(fn, errs)
  191. if comments:
  192. applyComments(fn, comments)