Templating.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377
  1. #!/usr/bin/env python2
  2. #
  3. # Copyright 2011 Nick Mathewson, Michael Stone
  4. #
  5. # You may do anything with this work that copyright law would normally
  6. # restrict, so long as you retain the above notice(s) and this license
  7. # in all redistributed copies and derived works. There is no warranty.
  8. """
  9. This module contins a general-purpose specialization-based
  10. templating mechanism. Chutney uses it for string-substitution to
  11. build torrc files and other such things.
  12. First, there are a few classes to implement "Environments". An
  13. "Environment" is like a dictionary, but delegates keys that it can't
  14. find to a "parent" object, which can be an environment or a dict.
  15. >>> base = Environ(foo=99, bar=600)
  16. >>> derived1 = Environ(parent=base, bar=700, quux=32)
  17. >>> base["foo"]
  18. 99
  19. >>> sorted(base.keys())
  20. ['bar', 'foo']
  21. >>> derived1["foo"]
  22. 99
  23. >>> base["bar"]
  24. 600
  25. >>> derived1["bar"]
  26. 700
  27. >>> derived1["quux"]
  28. 32
  29. >>> sorted(derived1.keys())
  30. ['bar', 'foo', 'quux']
  31. You can declare an environment subclass with methods named
  32. _get_foo() to implement a dictionary whose 'foo' item is
  33. calculated on the fly.
  34. >>> class Specialized(Environ):
  35. ... def __init__(self, p=None, **kw):
  36. ... Environ.__init__(self, p, **kw)
  37. ... def _get_expensive_value(self, my):
  38. ... return "Let's pretend this is hard to compute"
  39. ...
  40. >>> s = Specialized(base, quux="hi")
  41. >>> s["quux"]
  42. 'hi'
  43. >>> s['expensive_value']
  44. "Let's pretend this is hard to compute"
  45. >>> sorted(s.keys())
  46. ['bar', 'expensive_value', 'foo', 'quux']
  47. There's an internal class that extends Python's string.Template
  48. with a slightly more useful syntax for us. (It allows more characters
  49. in its key types.)
  50. >>> bt = _BetterTemplate("Testing ${hello}, $goodbye$$, $foo , ${a:b:c}")
  51. >>> bt.safe_substitute({'a:b:c': "4"}, hello=1, goodbye=2, foo=3)
  52. 'Testing 1, 2$, 3 , 4'
  53. Finally, there's a Template class that implements templates for
  54. simple string substitution. Variables with names like $abc or
  55. ${abc} are replaced by their values; $$ is replaced by $, and
  56. ${include:foo} is replaced by the contents of the file "foo".
  57. >>> t = Template("${include:/dev/null} $hi_there")
  58. >>> sorted(t.freevars())
  59. ['hi_there']
  60. >>> t.format(dict(hi_there=99))
  61. ' 99'
  62. >>> t2 = Template("X$${include:$fname} $bar $baz")
  63. >>> t2.format(dict(fname="/dev/null", bar=33, baz="$foo", foo=1337))
  64. 'X 33 1337'
  65. >>> sorted(t2.freevars({'fname':"/dev/null"}))
  66. ['bar', 'baz', 'fname']
  67. """
  68. from __future__ import with_statement
  69. import string
  70. import os
  71. # class _KeyError(KeyError):
  72. # pass
  73. _KeyError = KeyError
  74. class _DictWrapper(object):
  75. """Base class to implement a dictionary-like object with delegation.
  76. To use it, implement the _getitem method, and pass the optional
  77. 'parent' argument to the constructor.
  78. Lookups are satisfied first by calling _getitem(). If that
  79. fails with KeyError but the parent is present, lookups delegate
  80. to _parent.
  81. """
  82. # Fields
  83. # _parent: the parent object that lookups delegate to.
  84. def __init__(self, parent=None):
  85. self._parent = parent
  86. def _getitem(self, key, my):
  87. raise NotImplemented()
  88. def __getitem__(self, key):
  89. return self.lookup(key, self)
  90. def lookup(self, key, my):
  91. """As self[key], but parents are told when doing their lookups that
  92. the lookup is relative to a specialized environment 'my'. This
  93. is helpful when a parent environment has a value that depends
  94. on other values.
  95. """
  96. try:
  97. return self._getitem(key, my)
  98. except KeyError:
  99. pass
  100. if self._parent is None:
  101. raise _KeyError(key)
  102. try:
  103. lookup = self._parent.lookup
  104. except AttributeError:
  105. try:
  106. return self._parent[key]
  107. except KeyError:
  108. raise _KeyError(key)
  109. try:
  110. return lookup(key, my)
  111. except KeyError:
  112. raise _KeyError(key)
  113. class Environ(_DictWrapper):
  114. """An 'Environ' is used to define a set of key-value mappings with a
  115. fall-back parent Environ. When looking for keys in the
  116. Environ, any keys not found in this Environ are searched for in
  117. the parent.
  118. >>> animal = Environ(mobile=True,legs=4,can_program=False,can_meow=False)
  119. >>> biped = Environ(animal,legs=2)
  120. >>> cat = Environ(animal,can_meow=True)
  121. >>> human = Environ(biped,feathers=False,can_program=True)
  122. >>> chicken = Environ(biped,feathers=True)
  123. >>> human['legs']
  124. 2
  125. >>> human['can_meow']
  126. False
  127. >>> human['can_program']
  128. True
  129. >>> cat['legs']
  130. 4
  131. >>> cat['can_meow']
  132. True
  133. You can extend Environ to support values calculated at run-time by
  134. defining methods with names in the format _get_KEY():
  135. >>> class HomeEnv(Environ):
  136. ... def __init__(self, p=None, **kw):
  137. ... Environ.__init__(self, p, **kw)
  138. ... def _get_dotemacs(self, my):
  139. ... return os.path.join(my['homedir'], ".emacs")
  140. >>> h = HomeEnv(homedir="/tmp")
  141. >>> h['dotemacs']
  142. '/tmp/.emacs'
  143. The 'my' argument passed to these functions is the top-level
  144. dictionary that we're using for our lookup. This is useful
  145. when defining values that depend on other values which might in
  146. turn be overridden:
  147. >>> class Animal(Environ):
  148. ... def __init__(self, p=None, **kw):
  149. ... Environ.__init__(self, p, **kw)
  150. ... def _get_limbs(self, my):
  151. ... return my['legs'] + my['arms']
  152. >>> a = Animal(legs=2,arms=2)
  153. >>> spider = Environ(a, legs=8,arms=0)
  154. >>> squid = Environ(a, legs=0,arms=10)
  155. >>> squid['limbs']
  156. 10
  157. >>> spider['limbs']
  158. 8
  159. Note that if _get_limbs() had been defined as
  160. 'return self['legs']+self['arms']',
  161. both spider['limbs'] and squid['limbs'] would be given
  162. (incorrectly) as 4.
  163. """
  164. # Fields
  165. # _dict: dictionary holding the contents of this Environ that are
  166. # not inherited from the parent and are not computed on the fly.
  167. def __init__(self, parent=None, **kw):
  168. _DictWrapper.__init__(self, parent)
  169. self._dict = kw
  170. def _getitem(self, key, my):
  171. try:
  172. return self._dict[key]
  173. except KeyError:
  174. pass
  175. fn = getattr(self, "_get_%s" % key, None)
  176. if fn is not None:
  177. try:
  178. rv = fn(my)
  179. return rv
  180. except _KeyError:
  181. raise KeyError(key)
  182. raise KeyError(key)
  183. def __setitem__(self, key, val):
  184. self._dict[key] = val
  185. def keys(self):
  186. s = set()
  187. s.update(self._dict.keys())
  188. if self._parent is not None:
  189. s.update(self._parent.keys())
  190. s.update(name[5:] for name in dir(self) if name.startswith("_get_"))
  191. return s
  192. class IncluderDict(_DictWrapper):
  193. """Helper to implement ${include:} template substitution. Acts as a
  194. dictionary that maps include:foo to the contents of foo (relative to
  195. a search path if foo is a relative path), and delegates everything else
  196. to a parent.
  197. """
  198. # Fields
  199. # _includePath: a list of directories to consider when searching
  200. # for files given as relative paths.
  201. # _st_mtime: the most recent time when any of the files included
  202. # so far has been updated. (As seconds since the epoch).
  203. def __init__(self, parent, includePath=(".",)):
  204. """Create a new IncluderDict. Non-include entries are delegated to
  205. parent. Non-absolute paths are searched for relative to the
  206. paths listed in includePath.
  207. """
  208. _DictWrapper.__init__(self, parent)
  209. self._includePath = includePath
  210. self._st_mtime = 0
  211. def _getitem(self, key, my):
  212. if not key.startswith("include:"):
  213. raise KeyError(key)
  214. filename = key[len("include:"):]
  215. if os.path.isabs(filename):
  216. with open(filename, 'r') as f:
  217. stat = os.fstat(f.fileno())
  218. if stat.st_mtime > self._st_mtime:
  219. self._st_mtime = stat.st_mtime
  220. return f.read()
  221. for elt in self._includePath:
  222. fullname = os.path.join(elt, filename)
  223. if os.path.exists(fullname):
  224. with open(fullname, 'r') as f:
  225. stat = os.fstat(f.fileno())
  226. if stat.st_mtime > self._st_mtime:
  227. self._st_mtime = stat.st_mtime
  228. return f.read()
  229. raise KeyError(key)
  230. def getUpdateTime(self):
  231. return self._st_mtime
  232. class _BetterTemplate(string.Template):
  233. """Subclass of the standard string.Template that allows a wider range of
  234. characters in variable names.
  235. """
  236. idpattern = r'[a-z0-9:_/\.\-\/]+'
  237. def __init__(self, template):
  238. string.Template.__init__(self, template)
  239. class _FindVarsHelper(object):
  240. """Helper dictionary for finding the free variables in a template.
  241. It answers all requests for key lookups affirmatively, and remembers
  242. what it was asked for.
  243. """
  244. # Fields
  245. # _dflts: a dictionary of default values to treat specially
  246. # _vars: a set of all the keys that we've been asked to look up so far.
  247. def __init__(self, dflts):
  248. self._dflts = dflts
  249. self._vars = set()
  250. def __getitem__(self, var):
  251. return self.lookup(var, self)
  252. def lookup(self, var, my):
  253. self._vars.add(var)
  254. try:
  255. return self._dflts[var]
  256. except KeyError:
  257. return ""
  258. class Template(object):
  259. """A Template is a string pattern that allows repeated variable
  260. substitutions. These syntaxes are supported:
  261. $var -- expands to the value of var
  262. ${var} -- expands to the value of var
  263. $$ -- expands to a single $
  264. ${include:filename} -- expands to the contents of filename
  265. Substitutions are performed iteratively until no more are possible.
  266. """
  267. # Okay, actually, we stop after this many substitutions to avoid
  268. # infinite loops
  269. MAX_ITERATIONS = 32
  270. # Fields
  271. # _pat: The base pattern string to start our substitutions from
  272. # _includePath: a list of directories to search when including a file
  273. # by relative path.
  274. def __init__(self, pattern, includePath=(".",)):
  275. self._pat = pattern
  276. self._includePath = includePath
  277. def freevars(self, defaults=None):
  278. """Return a set containing all the free variables in this template"""
  279. if defaults is None:
  280. defaults = {}
  281. d = _FindVarsHelper(defaults)
  282. self.format(d)
  283. return d._vars
  284. def format(self, values):
  285. """Return a string containing this template, filled in with the
  286. values in the mapping 'values'.
  287. """
  288. values = IncluderDict(values, self._includePath)
  289. orig_val = self._pat
  290. nIterations = 0
  291. while True:
  292. v = _BetterTemplate(orig_val).substitute(values)
  293. if v == orig_val:
  294. return v
  295. orig_val = v
  296. nIterations += 1
  297. if nIterations > self.MAX_ITERATIONS:
  298. raise ValueError("Too many iterations in expanding template!")
  299. if __name__ == '__main__':
  300. import sys
  301. if len(sys.argv) == 1:
  302. import doctest
  303. doctest.testmod()
  304. print "done"
  305. else:
  306. for fn in sys.argv[1:]:
  307. with open(fn, 'r') as f:
  308. t = Template(f.read())
  309. print fn, t.freevars()