Templating.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406
  1. #!/usr/bin/env python
  2. #
  3. # Copyright 2011 Nick Mathewson, Michael Stone
  4. #
  5. # You may do anything with this work that copyright law would normally
  6. # restrict, so long as you retain the above notice(s) and this license
  7. # in all redistributed copies and derived works. There is no warranty.
  8. """
  9. This module contins a general-purpose specialization-based
  10. templating mechanism. Chutney uses it for string-substitution to
  11. build torrc files and other such things.
  12. First, there are a few classes to implement "Environments". An
  13. "Environment" is like a dictionary, but delegates keys that it can't
  14. find to a "parent" object, which can be an environment or a dict.
  15. >>> base = Environ(foo=99, bar=600)
  16. >>> derived1 = Environ(parent=base, bar=700, quux=32)
  17. >>> base["foo"]
  18. 99
  19. >>> sorted(base.keys())
  20. ['bar', 'foo']
  21. >>> derived1["foo"]
  22. 99
  23. >>> base["bar"]
  24. 600
  25. >>> derived1["bar"]
  26. 700
  27. >>> derived1["quux"]
  28. 32
  29. >>> sorted(derived1.keys())
  30. ['bar', 'foo', 'quux']
  31. You can declare an environment subclass with methods named
  32. _get_foo() to implement a dictionary whose 'foo' item is
  33. calculated on the fly.
  34. >>> class Specialized(Environ):
  35. ... def __init__(self, p=None, **kw):
  36. ... Environ.__init__(self, p, **kw)
  37. ... def _get_expensive_value(self, my):
  38. ... return "Let's pretend this is hard to compute"
  39. ...
  40. >>> s = Specialized(base, quux="hi")
  41. >>> s["quux"]
  42. 'hi'
  43. >>> s['expensive_value']
  44. "Let's pretend this is hard to compute"
  45. >>> sorted(s.keys())
  46. ['bar', 'expensive_value', 'foo', 'quux']
  47. There's an internal class that extends Python's string.Template
  48. with a slightly more useful syntax for us. (It allows more characters
  49. in its key types.)
  50. >>> bt = _BetterTemplate("Testing ${hello}, $goodbye$$, $foo , ${a:b:c}")
  51. >>> bt.safe_substitute({'a:b:c': "4"}, hello=1, goodbye=2, foo=3)
  52. 'Testing 1, 2$, 3 , 4'
  53. Finally, there's a Template class that implements templates for
  54. simple string substitution. Variables with names like $abc or
  55. ${abc} are replaced by their values; $$ is replaced by $, and
  56. ${include:foo} is replaced by the contents of the file "foo".
  57. >>> t = Template("${include:/dev/null} $hi_there")
  58. >>> sorted(t.freevars())
  59. ['hi_there']
  60. >>> t.format(dict(hi_there=99))
  61. ' 99'
  62. >>> t2 = Template("X$${include:$fname} $bar $baz")
  63. >>> t2.format(dict(fname="/dev/null", bar=33, baz="$foo", foo=1337))
  64. 'X 33 1337'
  65. >>> sorted(t2.freevars({'fname':"/dev/null"}))
  66. ['bar', 'baz', 'fname']
  67. """
  68. from __future__ import print_function
  69. from __future__ import with_statement
  70. import string
  71. import os
  72. # class _KeyError(KeyError):
  73. # pass
  74. _KeyError = KeyError
  75. class _DictWrapper(object):
  76. """Base class to implement a dictionary-like object with delegation.
  77. To use it, implement the _getitem method, and pass the optional
  78. 'parent' argument to the constructor.
  79. Lookups are satisfied first by calling _getitem(). If that
  80. fails with KeyError but the parent is present, lookups delegate
  81. to _parent.
  82. """
  83. # Fields
  84. # _parent: the parent object that lookups delegate to.
  85. def __init__(self, parent=None):
  86. self._parent = parent
  87. def _getitem(self, key, my):
  88. raise NotImplemented()
  89. def __getitem__(self, key):
  90. return self.lookup(key, self)
  91. def lookup(self, key, my):
  92. """As self[key], but parents are told when doing their lookups that
  93. the lookup is relative to a specialized environment 'my'. This
  94. is helpful when a parent environment has a value that depends
  95. on other values.
  96. """
  97. try:
  98. return self._getitem(key, my)
  99. except KeyError:
  100. pass
  101. if self._parent is None:
  102. raise _KeyError(key)
  103. try:
  104. lookup = self._parent.lookup
  105. except AttributeError:
  106. try:
  107. return self._parent[key]
  108. except KeyError:
  109. raise _KeyError(key)
  110. try:
  111. return lookup(key, my)
  112. except KeyError:
  113. raise _KeyError(key)
  114. class Environ(_DictWrapper):
  115. """An 'Environ' is used to define a set of key-value mappings with a
  116. fall-back parent Environ. When looking for keys in the
  117. Environ, any keys not found in this Environ are searched for in
  118. the parent.
  119. >>> animal = Environ(mobile=True,legs=4,can_program=False,
  120. can_meow=False)
  121. >>> biped = Environ(animal,legs=2)
  122. >>> cat = Environ(animal,can_meow=True)
  123. >>> human = Environ(biped,feathers=False,can_program=True)
  124. >>> chicken = Environ(biped,feathers=True)
  125. >>> human['legs']
  126. 2
  127. >>> human['can_meow']
  128. False
  129. >>> human['can_program']
  130. True
  131. >>> cat['legs']
  132. 4
  133. >>> cat['can_meow']
  134. True
  135. You can extend Environ to support values calculated at run-time by
  136. defining methods with names in the format _get_KEY():
  137. >>> class HomeEnv(Environ):
  138. ... def __init__(self, p=None, **kw):
  139. ... Environ.__init__(self, p, **kw)
  140. ... def _get_dotemacs(self, my):
  141. ... return os.path.join(my['homedir'], ".emacs")
  142. >>> h = HomeEnv(homedir="/tmp")
  143. >>> h['dotemacs']
  144. '/tmp/.emacs'
  145. The 'my' argument passed to these functions is the top-level
  146. dictionary that we're using for our lookup. This is useful
  147. when defining values that depend on other values which might in
  148. turn be overridden:
  149. >>> class Animal(Environ):
  150. ... def __init__(self, p=None, **kw):
  151. ... Environ.__init__(self, p, **kw)
  152. ... def _get_limbs(self, my):
  153. ... return my['legs'] + my['arms']
  154. >>> a = Animal(legs=2,arms=2)
  155. >>> spider = Environ(a, legs=8,arms=0)
  156. >>> squid = Environ(a, legs=0,arms=10)
  157. >>> squid['limbs']
  158. 10
  159. >>> spider['limbs']
  160. 8
  161. Note that if _get_limbs() had been defined as
  162. 'return self['legs']+self['arms']',
  163. both spider['limbs'] and squid['limbs'] would be given
  164. (incorrectly) as 4.
  165. """
  166. # Fields
  167. # _dict: dictionary holding the contents of this Environ that are
  168. # not inherited from the parent and are not computed on the fly.
  169. def __init__(self, parent=None, **kw):
  170. _DictWrapper.__init__(self, parent)
  171. self._dict = kw
  172. def _getitem(self, key, my):
  173. try:
  174. return self._dict[key]
  175. except KeyError:
  176. pass
  177. fn = getattr(self, "_get_%s" % key, None)
  178. if fn is not None:
  179. try:
  180. rv = fn(my)
  181. return rv
  182. except _KeyError:
  183. raise KeyError(key)
  184. raise KeyError(key)
  185. def __setitem__(self, key, val):
  186. self._dict[key] = val
  187. def keys(self):
  188. s = set()
  189. s.update(self._dict.keys())
  190. if self._parent is not None:
  191. s.update(self._parent.keys())
  192. s.update(name[5:] for name in dir(self) if name.startswith("_get_"))
  193. return s
  194. class IncluderDict(_DictWrapper):
  195. """Helper to implement ${include:} template substitution. Acts as a
  196. dictionary that maps include:foo to the contents of foo (relative to
  197. a search path if foo is a relative path), and delegates everything else
  198. to a parent.
  199. """
  200. # Fields
  201. # _includePath: a list of directories to consider when searching
  202. # for files given as relative paths.
  203. # _st_mtime: the most recent time when any of the files included
  204. # so far has been updated. (As seconds since the epoch).
  205. def __init__(self, parent, includePath=(".",)):
  206. """Create a new IncluderDict. Non-include entries are delegated to
  207. parent. Non-absolute paths are searched for relative to the
  208. paths listed in includePath.
  209. """
  210. _DictWrapper.__init__(self, parent)
  211. self._includePath = includePath
  212. self._st_mtime = 0
  213. def _getitem(self, key, my):
  214. if not key.startswith("include:"):
  215. raise KeyError(key)
  216. filename = key[len("include:"):]
  217. if os.path.isabs(filename):
  218. with open(filename, 'r') as f:
  219. stat = os.fstat(f.fileno())
  220. if stat.st_mtime > self._st_mtime:
  221. self._st_mtime = stat.st_mtime
  222. return f.read()
  223. for elt in self._includePath:
  224. fullname = os.path.join(elt, filename)
  225. if os.path.exists(fullname):
  226. with open(fullname, 'r') as f:
  227. stat = os.fstat(f.fileno())
  228. if stat.st_mtime > self._st_mtime:
  229. self._st_mtime = stat.st_mtime
  230. return f.read()
  231. raise KeyError(key)
  232. def getUpdateTime(self):
  233. return self._st_mtime
  234. class PathDict(_DictWrapper):
  235. """
  236. Implements ${path:} patterns, which map ${path:foo} to the location
  237. of 'foo' in the PATH environment variable.
  238. """
  239. def __init__(self, parent, path=None):
  240. _DictWrapper.__init__(self, parent)
  241. if path is None:
  242. path = os.getenv('PATH').split(":")
  243. self._path = path
  244. def _getitem(self, key, my):
  245. if not key.startswith("path:"):
  246. raise KeyError(key)
  247. key = key[len("path:"):]
  248. for location in self._path:
  249. p = os.path.join(location, key)
  250. try:
  251. s = os.stat(p)
  252. if s and s.st_mode & 0x111:
  253. return p
  254. except OSError:
  255. pass
  256. raise KeyError(key)
  257. class _BetterTemplate(string.Template):
  258. """Subclass of the standard string.Template that allows a wider range of
  259. characters in variable names.
  260. """
  261. idpattern = r'[a-z0-9:_/\.\-\/]+'
  262. def __init__(self, template):
  263. string.Template.__init__(self, template)
  264. class _FindVarsHelper(object):
  265. """Helper dictionary for finding the free variables in a template.
  266. It answers all requests for key lookups affirmatively, and remembers
  267. what it was asked for.
  268. """
  269. # Fields
  270. # _dflts: a dictionary of default values to treat specially
  271. # _vars: a set of all the keys that we've been asked to look up so far.
  272. def __init__(self, dflts):
  273. self._dflts = dflts
  274. self._vars = set()
  275. def __getitem__(self, var):
  276. return self.lookup(var, self)
  277. def lookup(self, var, my):
  278. self._vars.add(var)
  279. try:
  280. return self._dflts[var]
  281. except KeyError:
  282. return ""
  283. class Template(object):
  284. """A Template is a string pattern that allows repeated variable
  285. substitutions. These syntaxes are supported:
  286. $var -- expands to the value of var
  287. ${var} -- expands to the value of var
  288. $$ -- expands to a single $
  289. ${include:filename} -- expands to the contents of filename
  290. Substitutions are performed iteratively until no more are possible.
  291. """
  292. # Okay, actually, we stop after this many substitutions to avoid
  293. # infinite loops
  294. MAX_ITERATIONS = 32
  295. # Fields
  296. # _pat: The base pattern string to start our substitutions from
  297. # _includePath: a list of directories to search when including a file
  298. # by relative path.
  299. def __init__(self, pattern, includePath=(".",)):
  300. self._pat = pattern
  301. self._includePath = includePath
  302. def freevars(self, defaults=None):
  303. """Return a set containing all the free variables in this template"""
  304. if defaults is None:
  305. defaults = {}
  306. d = _FindVarsHelper(defaults)
  307. self.format(d)
  308. return d._vars
  309. def format(self, values):
  310. """Return a string containing this template, filled in with the
  311. values in the mapping 'values'.
  312. """
  313. values = IncluderDict(values, self._includePath)
  314. values = PathDict(values)
  315. orig_val = self._pat
  316. nIterations = 0
  317. while True:
  318. v = _BetterTemplate(orig_val).substitute(values)
  319. if v == orig_val:
  320. return v
  321. orig_val = v
  322. nIterations += 1
  323. if nIterations > self.MAX_ITERATIONS:
  324. raise ValueError("Too many iterations in expanding template!")
  325. if __name__ == '__main__':
  326. import sys
  327. if len(sys.argv) == 1:
  328. import doctest
  329. doctest.testmod()
  330. print("done")
  331. else:
  332. for fn in sys.argv[1:]:
  333. with open(fn, 'r') as f:
  334. t = Template(f.read())
  335. print(fn, t.freevars())