Templating.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408
  1. #!/usr/bin/env python
  2. #
  3. # Copyright 2011 Nick Mathewson, Michael Stone
  4. #
  5. # You may do anything with this work that copyright law would normally
  6. # restrict, so long as you retain the above notice(s) and this license
  7. # in all redistributed copies and derived works. There is no warranty.
  8. """
  9. This module contins a general-purpose specialization-based
  10. templating mechanism. Chutney uses it for string-substitution to
  11. build torrc files and other such things.
  12. First, there are a few classes to implement "Environments". An
  13. "Environment" is like a dictionary, but delegates keys that it can't
  14. find to a "parent" object, which can be an environment or a dict.
  15. >>> base = Environ(foo=99, bar=600)
  16. >>> derived1 = Environ(parent=base, bar=700, quux=32)
  17. >>> base["foo"]
  18. 99
  19. >>> sorted(base.keys())
  20. ['bar', 'foo']
  21. >>> derived1["foo"]
  22. 99
  23. >>> base["bar"]
  24. 600
  25. >>> derived1["bar"]
  26. 700
  27. >>> derived1["quux"]
  28. 32
  29. >>> sorted(derived1.keys())
  30. ['bar', 'foo', 'quux']
  31. You can declare an environment subclass with methods named
  32. _get_foo() to implement a dictionary whose 'foo' item is
  33. calculated on the fly.
  34. >>> class Specialized(Environ):
  35. ... def __init__(self, p=None, **kw):
  36. ... Environ.__init__(self, p, **kw)
  37. ... def _get_expensive_value(self, my):
  38. ... return "Let's pretend this is hard to compute"
  39. ...
  40. >>> s = Specialized(base, quux="hi")
  41. >>> s["quux"]
  42. 'hi'
  43. >>> s['expensive_value']
  44. "Let's pretend this is hard to compute"
  45. >>> sorted(s.keys())
  46. ['bar', 'expensive_value', 'foo', 'quux']
  47. There's an internal class that extends Python's string.Template
  48. with a slightly more useful syntax for us. (It allows more characters
  49. in its key types.)
  50. >>> bt = _BetterTemplate("Testing ${hello}, $goodbye$$, $foo , ${a:b:c}")
  51. >>> bt.safe_substitute({'a:b:c': "4"}, hello=1, goodbye=2, foo=3)
  52. 'Testing 1, 2$, 3 , 4'
  53. Finally, there's a Template class that implements templates for
  54. simple string substitution. Variables with names like $abc or
  55. ${abc} are replaced by their values; $$ is replaced by $, and
  56. ${include:foo} is replaced by the contents of the file "foo".
  57. >>> t = Template("${include:/dev/null} $hi_there")
  58. >>> sorted(t.freevars())
  59. ['hi_there']
  60. >>> t.format(dict(hi_there=99))
  61. ' 99'
  62. >>> t2 = Template("X$${include:$fname} $bar $baz")
  63. >>> t2.format(dict(fname="/dev/null", bar=33, baz="$foo", foo=1337))
  64. 'X 33 1337'
  65. >>> sorted(t2.freevars({'fname':"/dev/null"}))
  66. ['bar', 'baz', 'fname']
  67. """
  68. # Future imports for Python 2.7, mandatory in 3.0
  69. from __future__ import division
  70. from __future__ import print_function
  71. from __future__ import unicode_literals
  72. import string
  73. import os
  74. # class _KeyError(KeyError):
  75. # pass
  76. _KeyError = KeyError
  77. class _DictWrapper(object):
  78. """Base class to implement a dictionary-like object with delegation.
  79. To use it, implement the _getitem method, and pass the optional
  80. 'parent' argument to the constructor.
  81. Lookups are satisfied first by calling _getitem(). If that
  82. fails with KeyError but the parent is present, lookups delegate
  83. to _parent.
  84. """
  85. # Fields
  86. # _parent: the parent object that lookups delegate to.
  87. def __init__(self, parent=None):
  88. self._parent = parent
  89. def _getitem(self, key, my):
  90. raise NotImplemented()
  91. def __getitem__(self, key):
  92. return self.lookup(key, self)
  93. def lookup(self, key, my):
  94. """As self[key], but parents are told when doing their lookups that
  95. the lookup is relative to a specialized environment 'my'. This
  96. is helpful when a parent environment has a value that depends
  97. on other values.
  98. """
  99. try:
  100. return self._getitem(key, my)
  101. except KeyError:
  102. pass
  103. if self._parent is None:
  104. raise _KeyError(key)
  105. try:
  106. lookup = self._parent.lookup
  107. except AttributeError:
  108. try:
  109. return self._parent[key]
  110. except KeyError:
  111. raise _KeyError(key)
  112. try:
  113. return lookup(key, my)
  114. except KeyError:
  115. raise _KeyError(key)
  116. class Environ(_DictWrapper):
  117. """An 'Environ' is used to define a set of key-value mappings with a
  118. fall-back parent Environ. When looking for keys in the
  119. Environ, any keys not found in this Environ are searched for in
  120. the parent.
  121. >>> animal = Environ(mobile=True,legs=4,can_program=False,
  122. can_meow=False)
  123. >>> biped = Environ(animal,legs=2)
  124. >>> cat = Environ(animal,can_meow=True)
  125. >>> human = Environ(biped,feathers=False,can_program=True)
  126. >>> chicken = Environ(biped,feathers=True)
  127. >>> human['legs']
  128. 2
  129. >>> human['can_meow']
  130. False
  131. >>> human['can_program']
  132. True
  133. >>> cat['legs']
  134. 4
  135. >>> cat['can_meow']
  136. True
  137. You can extend Environ to support values calculated at run-time by
  138. defining methods with names in the format _get_KEY():
  139. >>> class HomeEnv(Environ):
  140. ... def __init__(self, p=None, **kw):
  141. ... Environ.__init__(self, p, **kw)
  142. ... def _get_dotemacs(self, my):
  143. ... return os.path.join(my['homedir'], ".emacs")
  144. >>> h = HomeEnv(homedir="/tmp")
  145. >>> h['dotemacs']
  146. '/tmp/.emacs'
  147. The 'my' argument passed to these functions is the top-level
  148. dictionary that we're using for our lookup. This is useful
  149. when defining values that depend on other values which might in
  150. turn be overridden:
  151. >>> class Animal(Environ):
  152. ... def __init__(self, p=None, **kw):
  153. ... Environ.__init__(self, p, **kw)
  154. ... def _get_limbs(self, my):
  155. ... return my['legs'] + my['arms']
  156. >>> a = Animal(legs=2,arms=2)
  157. >>> spider = Environ(a, legs=8,arms=0)
  158. >>> squid = Environ(a, legs=0,arms=10)
  159. >>> squid['limbs']
  160. 10
  161. >>> spider['limbs']
  162. 8
  163. Note that if _get_limbs() had been defined as
  164. 'return self['legs']+self['arms']',
  165. both spider['limbs'] and squid['limbs'] would be given
  166. (incorrectly) as 4.
  167. """
  168. # Fields
  169. # _dict: dictionary holding the contents of this Environ that are
  170. # not inherited from the parent and are not computed on the fly.
  171. def __init__(self, parent=None, **kw):
  172. _DictWrapper.__init__(self, parent)
  173. self._dict = kw
  174. def _getitem(self, key, my):
  175. try:
  176. return self._dict[key]
  177. except KeyError:
  178. pass
  179. fn = getattr(self, "_get_%s" % key, None)
  180. if fn is not None:
  181. try:
  182. rv = fn(my)
  183. return rv
  184. except _KeyError:
  185. raise KeyError(key)
  186. raise KeyError(key)
  187. def __setitem__(self, key, val):
  188. self._dict[key] = val
  189. def keys(self):
  190. s = set()
  191. s.update(self._dict.keys())
  192. if self._parent is not None:
  193. s.update(self._parent.keys())
  194. s.update(name[5:] for name in dir(self) if name.startswith("_get_"))
  195. return s
  196. class IncluderDict(_DictWrapper):
  197. """Helper to implement ${include:} template substitution. Acts as a
  198. dictionary that maps include:foo to the contents of foo (relative to
  199. a search path if foo is a relative path), and delegates everything else
  200. to a parent.
  201. """
  202. # Fields
  203. # _includePath: a list of directories to consider when searching
  204. # for files given as relative paths.
  205. # _st_mtime: the most recent time when any of the files included
  206. # so far has been updated. (As seconds since the epoch).
  207. def __init__(self, parent, includePath=(".",)):
  208. """Create a new IncluderDict. Non-include entries are delegated to
  209. parent. Non-absolute paths are searched for relative to the
  210. paths listed in includePath.
  211. """
  212. _DictWrapper.__init__(self, parent)
  213. self._includePath = includePath
  214. self._st_mtime = 0
  215. def _getitem(self, key, my):
  216. if not key.startswith("include:"):
  217. raise KeyError(key)
  218. filename = key[len("include:"):]
  219. if os.path.isabs(filename):
  220. with open(filename, 'r') as f:
  221. stat = os.fstat(f.fileno())
  222. if stat.st_mtime > self._st_mtime:
  223. self._st_mtime = stat.st_mtime
  224. return f.read()
  225. for elt in self._includePath:
  226. fullname = os.path.join(elt, filename)
  227. if os.path.exists(fullname):
  228. with open(fullname, 'r') as f:
  229. stat = os.fstat(f.fileno())
  230. if stat.st_mtime > self._st_mtime:
  231. self._st_mtime = stat.st_mtime
  232. return f.read()
  233. raise KeyError(key)
  234. def getUpdateTime(self):
  235. return self._st_mtime
  236. class PathDict(_DictWrapper):
  237. """
  238. Implements ${path:} patterns, which map ${path:foo} to the location
  239. of 'foo' in the PATH environment variable.
  240. """
  241. def __init__(self, parent, path=None):
  242. _DictWrapper.__init__(self, parent)
  243. if path is None:
  244. path = os.getenv('PATH').split(":")
  245. self._path = path
  246. def _getitem(self, key, my):
  247. if not key.startswith("path:"):
  248. raise KeyError(key)
  249. key = key[len("path:"):]
  250. for location in self._path:
  251. p = os.path.join(location, key)
  252. try:
  253. s = os.stat(p)
  254. if s and s.st_mode & 0x111:
  255. return p
  256. except OSError:
  257. pass
  258. raise KeyError(key)
  259. class _BetterTemplate(string.Template):
  260. """Subclass of the standard string.Template that allows a wider range of
  261. characters in variable names.
  262. """
  263. idpattern = r'[a-z0-9:_/\.\-\/]+'
  264. def __init__(self, template):
  265. string.Template.__init__(self, template)
  266. class _FindVarsHelper(object):
  267. """Helper dictionary for finding the free variables in a template.
  268. It answers all requests for key lookups affirmatively, and remembers
  269. what it was asked for.
  270. """
  271. # Fields
  272. # _dflts: a dictionary of default values to treat specially
  273. # _vars: a set of all the keys that we've been asked to look up so far.
  274. def __init__(self, dflts):
  275. self._dflts = dflts
  276. self._vars = set()
  277. def __getitem__(self, var):
  278. return self.lookup(var, self)
  279. def lookup(self, var, my):
  280. self._vars.add(var)
  281. try:
  282. return self._dflts[var]
  283. except KeyError:
  284. return ""
  285. class Template(object):
  286. """A Template is a string pattern that allows repeated variable
  287. substitutions. These syntaxes are supported:
  288. $var -- expands to the value of var
  289. ${var} -- expands to the value of var
  290. $$ -- expands to a single $
  291. ${include:filename} -- expands to the contents of filename
  292. Substitutions are performed iteratively until no more are possible.
  293. """
  294. # Okay, actually, we stop after this many substitutions to avoid
  295. # infinite loops
  296. MAX_ITERATIONS = 32
  297. # Fields
  298. # _pat: The base pattern string to start our substitutions from
  299. # _includePath: a list of directories to search when including a file
  300. # by relative path.
  301. def __init__(self, pattern, includePath=(".",)):
  302. self._pat = pattern
  303. self._includePath = includePath
  304. def freevars(self, defaults=None):
  305. """Return a set containing all the free variables in this template"""
  306. if defaults is None:
  307. defaults = {}
  308. d = _FindVarsHelper(defaults)
  309. self.format(d)
  310. return d._vars
  311. def format(self, values):
  312. """Return a string containing this template, filled in with the
  313. values in the mapping 'values'.
  314. """
  315. values = IncluderDict(values, self._includePath)
  316. values = PathDict(values)
  317. orig_val = self._pat
  318. nIterations = 0
  319. while True:
  320. v = _BetterTemplate(orig_val).substitute(values)
  321. if v == orig_val:
  322. return v
  323. orig_val = v
  324. nIterations += 1
  325. if nIterations > self.MAX_ITERATIONS:
  326. raise ValueError("Too many iterations in expanding template!")
  327. if __name__ == '__main__':
  328. import sys
  329. if len(sys.argv) == 1:
  330. import doctest
  331. doctest.testmod()
  332. print("done")
  333. else:
  334. for fn in sys.argv[1:]:
  335. with open(fn, 'r') as f:
  336. t = Template(f.read())
  337. print(fn, t.freevars())