format_changelog.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567
  1. #!/usr/bin/python
  2. # Copyright (c) 2014-2015, The Tor Project, Inc.
  3. # See LICENSE for licensing information
  4. #
  5. # This script reformats a section of the changelog to wrap everything to
  6. # the right width and put blank lines in the right places. Eventually,
  7. # it might include a linter.
  8. #
  9. # To run it, pipe a section of the changelog (starting with "Changes
  10. # in Tor 0.x.y.z-alpha" through the script.)
  11. import os
  12. import re
  13. import sys
  14. import optparse
  15. # ==============================
  16. # Oh, look! It's a cruddy approximation to Knuth's elegant text wrapping
  17. # algorithm, with totally ad hoc parameters!
  18. #
  19. # We're trying to minimize:
  20. # The total of the cubes of ragged space on underflowed intermediate lines,
  21. # PLUS
  22. # 100 * the fourth power of overflowed characters
  23. # PLUS
  24. # .1 * a bit more than the cube of ragged space on the last line.
  25. # PLUS
  26. # OPENPAREN_PENALTY for each line that starts with (
  27. #
  28. # We use an obvious dynamic programming algorithm to sorta approximate this.
  29. # It's not coded right or optimally, but it's fast enough for changelogs
  30. #
  31. # (Code found in an old directory of mine, lightly cleaned. -NM)
  32. NO_HYPHENATE=set("""
  33. pf-divert
  34. tor-resolve
  35. tor-gencert
  36. """.split())
  37. LASTLINE_UNDERFLOW_EXPONENT = 1
  38. LASTLINE_UNDERFLOW_PENALTY = 1
  39. UNDERFLOW_EXPONENT = 3
  40. UNDERFLOW_PENALTY = 1
  41. OVERFLOW_EXPONENT = 4
  42. OVERFLOW_PENALTY = 2000
  43. ORPHAN_PENALTY = 10000
  44. OPENPAREN_PENALTY = 200
  45. def generate_wrapping(words, divisions):
  46. lines = []
  47. last = 0
  48. for i in divisions:
  49. w = words[last:i]
  50. last = i
  51. line = " ".join(w).replace("\xff ","-").replace("\xff","-")
  52. lines.append(line.strip())
  53. return lines
  54. def wrapping_quality(words, divisions, width1, width2):
  55. total = 0.0
  56. lines = generate_wrapping(words, divisions)
  57. for line in lines:
  58. length = len(line)
  59. if line is lines[0]:
  60. width = width1
  61. else:
  62. width = width2
  63. if line[0:1] == '(':
  64. total += OPENPAREN_PENALTY
  65. if length > width:
  66. total += OVERFLOW_PENALTY * (
  67. (length - width) ** OVERFLOW_EXPONENT )
  68. else:
  69. if line is lines[-1]:
  70. e,p = (LASTLINE_UNDERFLOW_EXPONENT, LASTLINE_UNDERFLOW_PENALTY)
  71. if " " not in line:
  72. total += ORPHAN_PENALTY
  73. else:
  74. e,p = (UNDERFLOW_EXPONENT, UNDERFLOW_PENALTY)
  75. total += p * ((width - length) ** e)
  76. return total
  77. def wrap_graf(words, prefix_len1=0, prefix_len2=0, width=72):
  78. wrapping_after = [ (0,), ]
  79. w1 = width - prefix_len1
  80. w2 = width - prefix_len2
  81. for i in range(1, len(words)+1):
  82. best_so_far = None
  83. best_score = 1e300
  84. for j in range(i):
  85. t = wrapping_after[j]
  86. t1 = t[:-1] + (i,)
  87. t2 = t + (i,)
  88. wq1 = wrapping_quality(words, t1, w1, w2)
  89. wq2 = wrapping_quality(words, t2, w1, w2)
  90. if wq1 < best_score:
  91. best_so_far = t1
  92. best_score = wq1
  93. if wq2 < best_score:
  94. best_so_far = t2
  95. best_score = wq2
  96. wrapping_after.append( best_so_far )
  97. lines = generate_wrapping(words, wrapping_after[-1])
  98. return lines
  99. def hyphenatable(word):
  100. if "--" in word:
  101. return False
  102. if re.match(r'^[^\d\-]\D*-', word):
  103. stripped = re.sub(r'^\W+','',word)
  104. stripped = re.sub(r'\W+$','',word)
  105. return stripped not in NO_HYPHENATE
  106. else:
  107. return False
  108. def split_paragraph(s):
  109. "Split paragraph into words; tuned for Tor."
  110. r = []
  111. for word in s.split():
  112. if hyphenatable(word):
  113. while "-" in word:
  114. a,word = word.split("-",1)
  115. r.append(a+"\xff")
  116. r.append(word)
  117. return r
  118. def fill(text, width, initial_indent, subsequent_indent):
  119. words = split_paragraph(text)
  120. lines = wrap_graf(words, len(initial_indent), len(subsequent_indent),
  121. width)
  122. res = [ initial_indent, lines[0], "\n" ]
  123. for line in lines[1:]:
  124. res.append(subsequent_indent)
  125. res.append(line)
  126. res.append("\n")
  127. return "".join(res)
  128. # ==============================
  129. TP_MAINHEAD = 0
  130. TP_HEADTEXT = 1
  131. TP_BLANK = 2
  132. TP_SECHEAD = 3
  133. TP_ITEMFIRST = 4
  134. TP_ITEMBODY = 5
  135. TP_END = 6
  136. TP_PREHEAD = 7
  137. def head_parser(line):
  138. if re.match(r'^Changes in', line):
  139. return TP_MAINHEAD
  140. elif re.match(r'^[A-Za-z]', line):
  141. return TP_PREHEAD
  142. elif re.match(r'^ o ', line):
  143. return TP_SECHEAD
  144. elif re.match(r'^\s*$', line):
  145. return TP_BLANK
  146. else:
  147. return TP_HEADTEXT
  148. def body_parser(line):
  149. if re.match(r'^ o ', line):
  150. return TP_SECHEAD
  151. elif re.match(r'^ -',line):
  152. return TP_ITEMFIRST
  153. elif re.match(r'^ \S', line):
  154. return TP_ITEMBODY
  155. elif re.match(r'^\s*$', line):
  156. return TP_BLANK
  157. elif re.match(r'^Changes in', line):
  158. return TP_END
  159. elif re.match(r'^\s+\S', line):
  160. return TP_HEADTEXT
  161. else:
  162. print "Weird line %r"%line
  163. def clean_head(head):
  164. return head
  165. def head_score(s):
  166. m = re.match(r'^ +o (.*)', s)
  167. if not m:
  168. print >>sys.stderr, "Can't score %r"%s
  169. return 99999
  170. lw = m.group(1).lower()
  171. if lw.startswith("security") and "feature" not in lw:
  172. score = -300
  173. elif lw.startswith("deprecated version"):
  174. score = -200
  175. elif (('new' in lw and 'requirement' in lw) or
  176. ('new' in lw and 'dependenc' in lw) or
  177. ('build' in lw and 'requirement' in lw) or
  178. ('removed' in lw and 'platform' in lw)):
  179. score = -100
  180. elif lw.startswith("major feature"):
  181. score = 00
  182. elif lw.startswith("major bug"):
  183. score = 50
  184. elif lw.startswith("major"):
  185. score = 70
  186. elif lw.startswith("minor feature"):
  187. score = 200
  188. elif lw.startswith("minor bug"):
  189. score = 250
  190. elif lw.startswith("minor"):
  191. score = 270
  192. else:
  193. score = 1000
  194. if 'secur' in lw:
  195. score -= 2
  196. if "(other)" in lw:
  197. score += 2
  198. if '(' not in lw:
  199. score -= 1
  200. return score
  201. class ChangeLog(object):
  202. def __init__(self, wrapText=True, blogOrder=True, drupalBreak=False):
  203. self.prehead = []
  204. self.mainhead = None
  205. self.headtext = []
  206. self.curgraf = None
  207. self.sections = []
  208. self.cursection = None
  209. self.lineno = 0
  210. self.wrapText = wrapText
  211. self.blogOrder = blogOrder
  212. self.drupalBreak = drupalBreak
  213. def addLine(self, tp, line):
  214. self.lineno += 1
  215. if tp == TP_MAINHEAD:
  216. assert not self.mainhead
  217. self.mainhead = line
  218. elif tp == TP_PREHEAD:
  219. self.prehead.append(line)
  220. elif tp == TP_HEADTEXT:
  221. if self.curgraf is None:
  222. self.curgraf = []
  223. self.headtext.append(self.curgraf)
  224. self.curgraf.append(line)
  225. elif tp == TP_BLANK:
  226. self.curgraf = None
  227. elif tp == TP_SECHEAD:
  228. self.cursection = [ self.lineno, line, [] ]
  229. self.sections.append(self.cursection)
  230. elif tp == TP_ITEMFIRST:
  231. item = ( self.lineno, [ [line] ])
  232. self.curgraf = item[1][0]
  233. self.cursection[2].append(item)
  234. elif tp == TP_ITEMBODY:
  235. if self.curgraf is None:
  236. self.curgraf = []
  237. self.cursection[2][-1][1].append(self.curgraf)
  238. self.curgraf.append(line)
  239. else:
  240. assert "This" is "unreachable"
  241. def lint_head(self, line, head):
  242. m = re.match(r'^ *o ([^\(]+)((?:\([^\)]+\))?):', head)
  243. if not m:
  244. print >>sys.stderr, "Weird header format on line %s"%line
  245. def lint_item(self, line, grafs, head_type):
  246. pass
  247. def lint(self):
  248. self.head_lines = {}
  249. for sec_line, sec_head, items in self.sections:
  250. head_type = self.lint_head(sec_line, sec_head)
  251. for item_line, grafs in items:
  252. self.lint_item(item_line, grafs, head_type)
  253. def dumpGraf(self,par,indent1,indent2=-1):
  254. if not self.wrapText:
  255. for line in par:
  256. print line
  257. return
  258. if indent2 == -1:
  259. indent2 = indent1
  260. text = " ".join(re.sub(r'\s+', ' ', line.strip()) for line in par)
  261. sys.stdout.write(fill(text,
  262. width=72,
  263. initial_indent=" "*indent1,
  264. subsequent_indent=" "*indent2))
  265. def dumpPreheader(self, graf):
  266. self.dumpGraf(graf, 0)
  267. print
  268. def dumpMainhead(self, head):
  269. print head
  270. def dumpHeadGraf(self, graf):
  271. self.dumpGraf(graf, 2)
  272. print
  273. def dumpSectionHeader(self, header):
  274. print header
  275. def dumpStartOfSections(self):
  276. pass
  277. def dumpEndOfSections(self):
  278. pass
  279. def dumpEndOfSection(self):
  280. print
  281. def dumpEndOfChangelog(self):
  282. print
  283. def dumpDrupalBreak(self):
  284. pass
  285. def dumpItem(self, grafs):
  286. self.dumpGraf(grafs[0],4,6)
  287. for par in grafs[1:]:
  288. print
  289. self.dumpGraf(par,6,6)
  290. def collateAndSortSections(self):
  291. heads = []
  292. sectionsByHead = { }
  293. for _, head, items in self.sections:
  294. head = clean_head(head)
  295. try:
  296. s = sectionsByHead[head]
  297. except KeyError:
  298. s = sectionsByHead[head] = []
  299. heads.append( (head_score(head), head.lower(), head, s) )
  300. s.extend(items)
  301. heads.sort()
  302. self.sections = [ (0, head, items) for _1,_2,head,items in heads ]
  303. def dump(self):
  304. if self.prehead:
  305. self.dumpPreheader(self.prehead)
  306. if not self.blogOrder:
  307. self.dumpMainhead(self.mainhead)
  308. for par in self.headtext:
  309. self.dumpHeadGraf(par)
  310. if self.blogOrder:
  311. self.dumpMainhead(self.mainhead)
  312. drupalBreakAfter = None
  313. if self.drupalBreak and len(self.sections) > 4:
  314. drupalBreakAfter = self.sections[1][2]
  315. self.dumpStartOfSections()
  316. for _,head,items in self.sections:
  317. if not head.endswith(':'):
  318. print >>sys.stderr, "adding : to %r"%head
  319. head = head + ":"
  320. self.dumpSectionHeader(head)
  321. for _,grafs in items:
  322. self.dumpItem(grafs)
  323. self.dumpEndOfSection()
  324. if items is drupalBreakAfter:
  325. self.dumpDrupalBreak()
  326. self.dumpEndOfSections()
  327. self.dumpEndOfChangelog()
  328. # Let's turn bugs to html.
  329. BUG_PAT = re.compile('(bug|ticket|feature)\s+(\d{4,5})', re.I)
  330. def bug_html(m):
  331. return "%s <a href='https://trac.torproject.org/projects/tor/ticket/%s'>%s</a>" % (m.group(1), m.group(2), m.group(2))
  332. class HTMLChangeLog(ChangeLog):
  333. def __init__(self, *args, **kwargs):
  334. ChangeLog.__init__(self, *args, **kwargs)
  335. def htmlText(self, graf):
  336. output = []
  337. for line in graf:
  338. line = line.rstrip().replace("&","&amp;")
  339. line = line.rstrip().replace("<","&lt;").replace(">","&gt;")
  340. output.append(line.strip())
  341. output = " ".join(output)
  342. output = BUG_PAT.sub(bug_html, output)
  343. sys.stdout.write(output)
  344. def htmlPar(self, graf):
  345. sys.stdout.write("<p>")
  346. self.htmlText(graf)
  347. sys.stdout.write("</p>\n")
  348. def dumpPreheader(self, graf):
  349. self.htmlPar(graf)
  350. def dumpMainhead(self, head):
  351. sys.stdout.write("<h2>%s</h2>"%head)
  352. def dumpHeadGraf(self, graf):
  353. self.htmlPar(graf)
  354. def dumpSectionHeader(self, header):
  355. header = header.replace(" o ", "", 1).lstrip()
  356. sys.stdout.write(" <li>%s\n"%header)
  357. sys.stdout.write(" <ul>\n")
  358. def dumpEndOfSection(self):
  359. sys.stdout.write(" </ul>\n\n")
  360. def dumpEndOfChangelog(self):
  361. pass
  362. def dumpStartOfSections(self):
  363. print "<ul>\n"
  364. def dumpEndOfSections(self):
  365. print "</ul>\n"
  366. def dumpDrupalBreak(self):
  367. print "\n</ul>\n"
  368. print "<p>&nbsp;</p>"
  369. print "\n<!--break-->\n\n"
  370. print "<ul>"
  371. def dumpItem(self, grafs):
  372. grafs[0][0] = grafs[0][0].replace(" - ", "", 1).lstrip()
  373. sys.stdout.write(" <li>")
  374. if len(grafs) > 1:
  375. for par in grafs:
  376. self.htmlPar(par)
  377. else:
  378. self.htmlText(grafs[0])
  379. print
  380. op = optparse.OptionParser(usage="usage: %prog [options] [filename]")
  381. op.add_option('-W', '--no-wrap', action='store_false',
  382. dest='wrapText', default=True,
  383. help='Do not re-wrap paragraphs')
  384. op.add_option('-S', '--no-sort', action='store_false',
  385. dest='sort', default=True,
  386. help='Do not sort or collate sections')
  387. op.add_option('-o', '--output', dest='output',
  388. default='-', metavar='FILE', help="write output to FILE")
  389. op.add_option('-H', '--html', action='store_true',
  390. dest='html', default=False,
  391. help="generate an HTML fragment")
  392. op.add_option('-1', '--first', action='store_true',
  393. dest='firstOnly', default=False,
  394. help="write only the first section")
  395. op.add_option('-b', '--blog-header', action='store_true',
  396. dest='blogOrder', default=False,
  397. help="Write the header in blog order")
  398. op.add_option('-B', '--blog', action='store_true',
  399. dest='blogFormat', default=False,
  400. help="Set all other options as appropriate for a blog post")
  401. op.add_option('--inplace', action='store_true',
  402. dest='inplace', default=False,
  403. help="Alter the ChangeLog in place")
  404. op.add_option('--drupal-break', action='store_true',
  405. dest='drupalBreak', default=False,
  406. help='Insert a drupal-friendly <!--break--> as needed')
  407. options,args = op.parse_args()
  408. if options.blogFormat:
  409. options.blogOrder = True
  410. options.html = True
  411. options.sort = False
  412. options.wrapText = False
  413. options.firstOnly = True
  414. options.drupalBreak = True
  415. if len(args) > 1:
  416. op.error("Too many arguments")
  417. elif len(args) == 0:
  418. fname = 'ChangeLog'
  419. else:
  420. fname = args[0]
  421. if options.inplace:
  422. assert options.output == '-'
  423. options.output = fname
  424. if fname != '-':
  425. sys.stdin = open(fname, 'r')
  426. nextline = None
  427. if options.html:
  428. ChangeLogClass = HTMLChangeLog
  429. else:
  430. ChangeLogClass = ChangeLog
  431. CL = ChangeLogClass(wrapText=options.wrapText,
  432. blogOrder=options.blogOrder,
  433. drupalBreak=options.drupalBreak)
  434. parser = head_parser
  435. for line in sys.stdin:
  436. line = line.rstrip()
  437. tp = parser(line)
  438. if tp == TP_SECHEAD:
  439. parser = body_parser
  440. elif tp == TP_END:
  441. nextline = line
  442. break
  443. CL.addLine(tp,line)
  444. CL.lint()
  445. if options.output != '-':
  446. fname_new = options.output+".new"
  447. fname_out = options.output
  448. sys.stdout = open(fname_new, 'w')
  449. else:
  450. fname_new = fname_out = None
  451. if options.sort:
  452. CL.collateAndSortSections()
  453. CL.dump()
  454. if options.firstOnly:
  455. sys.exit(0)
  456. if nextline is not None:
  457. print nextline
  458. for line in sys.stdin:
  459. sys.stdout.write(line)
  460. if fname_new is not None:
  461. os.rename(fname_new, fname_out)