diff options
| author | Inna Palant <ipalant@google.com> | 2023-12-14 16:05:36 -0800 |
|---|---|---|
| committer | Inna Palant <ipalant@google.com> | 2023-12-14 16:05:36 -0800 |
| commit | be49714f29a7595341e2702c54ebfc8a105901d3 (patch) | |
| tree | d682d5d42a5167f52b471cc56c0989750f98b2e2 | |
| parent | 3df926ed966e5b7f7ba4e34453af6c864afff741 (diff) | |
| parent | bdc7b459f8c2003e3a44468891efbc2d8d6b75bf (diff) | |
| download | mako-be49714f29a7595341e2702c54ebfc8a105901d3.tar.gz | |
Merge remote-tracking branch 'origin/upstream'platform-tools-34.0.5simpleperf-release
Import b/312187805
154 files changed, 27772 insertions, 0 deletions
diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..bd18eca --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,6 @@ +# These are supported funding model platforms + +github: sqlalchemy +patreon: zzzeek +tidelift: "pypi/SQLAlchemy" + diff --git a/.github/workflows/run-on-pr.yaml b/.github/workflows/run-on-pr.yaml new file mode 100644 index 0000000..fa6c759 --- /dev/null +++ b/.github/workflows/run-on-pr.yaml @@ -0,0 +1,46 @@ +name: Run tests on a pr + +on: + # run on pull request to main excluding changes that are only on doc or example folders + pull_request: + branches: + - main + paths-ignore: + - "doc/**" + +jobs: + run-test: + name: ${{ matrix.python-version }}-${{ matrix.os }}-${{matrix.tox-env}} + runs-on: ${{ matrix.os }} + strategy: + # run this job using this matrix + matrix: + os: + - "ubuntu-latest" + python-version: + - "3.10" + tox-env: + - "" + - "-e pep8" + + fail-fast: false + + # steps to run in each job. Some are github actions, others run shell commands + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + architecture: ${{ matrix.architecture }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade tox setuptools + pip list + + - name: Run tests + run: tox ${{ matrix.tox-env }} diff --git a/.github/workflows/run-test.yaml b/.github/workflows/run-test.yaml new file mode 100644 index 0000000..88e5a96 --- /dev/null +++ b/.github/workflows/run-test.yaml @@ -0,0 +1,59 @@ +name: Run tests + +on: + # run on push in main or rel_* branches excluding changes are only on doc or example folders + push: + branches: + - main + - "rel_*" + # branches used to test the workflow + - "workflow_test_*" + paths-ignore: + - "docs/**" + +jobs: + run-test: + name: ${{ matrix.python-version }}-${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + # run this job using this matrix + matrix: + os: + - "ubuntu-latest" + - "windows-latest" + - "macos-latest" + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" + + exclude: + # beaker raises warning on 3.10. only windows seems affected + # See https://github.com/bbangert/beaker/pull/213 + - os: "windows-latest" + python-version: "3.10" + + fail-fast: false + + # steps to run in each job. Some are github actions, others run shell commands + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + architecture: ${{ matrix.architecture }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade tox setuptools + pip list + + - name: Run tests + run: tox diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..870312a --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +/build +/dist +/.coverage +/doc/build/output +*.pyc +*.orig +*.egg-info +*.sw[opq] +/.Python +/bin +/include +/lib +/man +.tox/ +.cache/ +.vscode diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..1a5944b --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=gerrit.sqlalchemy.org +project=sqlalchemy/mako +defaultbranch=main diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..36ec5fa --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,14 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/python/black + rev: 23.9.1 + hooks: + - id: black + +- repo: https://github.com/sqlalchemyorg/zimports + rev: v0.6.0 + hooks: + - id: zimports + + @@ -0,0 +1,13 @@ +Mako was created by Michael Bayer. + +Major contributing authors include: + +- Michael Bayer <mike_mp@zzzcomputing.com> +- Geoffrey T. Dairiki <dairiki@dairiki.org> +- Philip Jenvey <pjenvey@underboss.org> +- David Peckam +- Armin Ronacher +- Ben Bangert <ben@groovie.org> +- Ben Trofatter + + diff --git a/Android.bp b/Android.bp new file mode 100644 index 0000000..bf56fe6 --- /dev/null +++ b/Android.bp @@ -0,0 +1,43 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["external_python_mako_license"], +} + +// Added automatically by a large-scale-change +// See: http://go/android-license-faq +license { + name: "external_python_mako_license", + visibility: [":__subpackages__"], + license_kinds: [ + "SPDX-license-identifier-MIT", + ], + license_text: [ + "LICENSE", + ], +} + +python_library { + name: "mako", + host_supported: true, + srcs: [ + "mako/*.py", + "mako/ext/*.py", + ], + libs: [ + "py-setuptools", + "py-markupsafe", + ] +} @@ -0,0 +1,15 @@ +===== +MOVED +===== + +Please see: + + /docs/changelog.html + + /docs/build/changelog.rst + +or + + https://docs.makotemplates.org/en/latest/changelog.html + +for the current CHANGES. @@ -0,0 +1,19 @@ +Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file>. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..25324e3 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,8 @@ +recursive-include doc *.html *.css *.txt *.js *.png *.py Makefile *.rst *.mako +recursive-include examples *.py *.xml *.mako *.myt *.kid *.tmpl +recursive-include test *.py *.html *.mako *.cfg + +include README* AUTHORS LICENSE CHANGES* tox.ini + +prune doc/build/output + diff --git a/METADATA b/METADATA new file mode 100644 index 0000000..846d94a --- /dev/null +++ b/METADATA @@ -0,0 +1,16 @@ +name: "mako" +description: + "Mako is a template library written in Python. It provides a familiar, " + "non-XML syntax which compiles into Python modules for maximum performance." + +third_party { +homepage: "https://github.com/sqlalchemy/mako" + identifier { + type: "Git" + value: "https://github.com/sqlalchemy/mako" + primary_source: true + } + version: "rel_1_3_0" + last_upgrade_date { year: 2023 month: 12 day: 6 } + license_type: NOTICE +} diff --git a/MODULE_LICENSE_MIT b/MODULE_LICENSE_MIT new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/MODULE_LICENSE_MIT @@ -0,0 +1 @@ +enh@google.com diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..9aa2f23 --- /dev/null +++ b/README.rst @@ -0,0 +1,52 @@ +========================= +Mako Templates for Python +========================= + +Mako is a template library written in Python. It provides a familiar, non-XML +syntax which compiles into Python modules for maximum performance. Mako's +syntax and API borrows from the best ideas of many others, including Django +templates, Cheetah, Myghty, and Genshi. Conceptually, Mako is an embedded +Python (i.e. Python Server Page) language, which refines the familiar ideas +of componentized layout and inheritance to produce one of the most +straightforward and flexible models available, while also maintaining close +ties to Python calling and scoping semantics. + +Nutshell +======== + +:: + + <%inherit file="base.html"/> + <% + rows = [[v for v in range(0,10)] for row in range(0,10)] + %> + <table> + % for row in rows: + ${makerow(row)} + % endfor + </table> + + <%def name="makerow(row)"> + <tr> + % for name in row: + <td>${name}</td>\ + % endfor + </tr> + </%def> + +Philosophy +=========== + +Python is a great scripting language. Don't reinvent the wheel...your templates can handle it ! + +Documentation +============== + +See documentation for Mako at https://docs.makotemplates.org/en/latest/ + +License +======== + +Mako is licensed under an MIT-style license (see LICENSE). +Other incorporated projects may be licensed under different licenses. +All licenses allow for non-commercial and commercial use. diff --git a/doc/build/Makefile b/doc/build/Makefile new file mode 100644 index 0000000..beed529 --- /dev/null +++ b/doc/build/Makefile @@ -0,0 +1,137 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = -T +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = output + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest dist-html site-mako + +help: + @echo "Please use \`make <target>' where <target> is one of" + @echo " html to make standalone HTML files" + @echo " dist-html same as html, but places files in /doc" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html -A mako_layout=html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dist-html: + $(SPHINXBUILD) -b html -A mako_layout=html $(ALLSPHINXOPTS) .. + @echo + @echo "Build finished. The HTML pages are in ../." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/SQLAlchemy.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/SQLAlchemy.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/SQLAlchemy" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/SQLAlchemy" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + cp texinputs/* $(BUILDDIR)/latex/ + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + make -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) . + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/doc/build/caching.rst b/doc/build/caching.rst new file mode 100644 index 0000000..9f63750 --- /dev/null +++ b/doc/build/caching.rst @@ -0,0 +1,387 @@ +.. _caching_toplevel: + +======= +Caching +======= + +Any template or component can be cached using the ``cache`` +argument to the ``<%page>``, ``<%def>`` or ``<%block>`` directives: + +.. sourcecode:: mako + + <%page cached="True"/> + + template text + +The above template, after being executed the first time, will +store its content within a cache that by default is scoped +within memory. Subsequent calls to the template's :meth:`~.Template.render` +method will return content directly from the cache. When the +:class:`.Template` object itself falls out of scope, its corresponding +cache is garbage collected along with the template. + +The caching system requires that a cache backend be installed; this +includes either the `Beaker <http://beaker.readthedocs.org/>`_ package +or the `dogpile.cache <http://dogpilecache.readthedocs.org>`_, as well as +any other third-party caching libraries that feature Mako integration. + +By default, caching will attempt to make use of Beaker. +To use dogpile.cache, the +``cache_impl`` argument must be set; see this argument in the +section :ref:`cache_arguments`. + +In addition to being available on the ``<%page>`` tag, the caching flag and all +its options can be used with the ``<%def>`` tag as well: + +.. sourcecode:: mako + + <%def name="mycomp" cached="True" cache_timeout="60"> + other text + </%def> + +... and equivalently with the ``<%block>`` tag, anonymous or named: + +.. sourcecode:: mako + + <%block cached="True" cache_timeout="60"> + other text + </%block> + + +.. _cache_arguments: + +Cache Arguments +=============== + +Mako has two cache arguments available on tags that are +available in all cases. The rest of the arguments +available are specific to a backend. + +The two generic tags arguments are: + +* ``cached="True"`` - enable caching for this ``<%page>``, + ``<%def>``, or ``<%block>``. +* ``cache_key`` - the "key" used to uniquely identify this content + in the cache. Usually, this key is chosen automatically + based on the name of the rendering callable (i.e. ``body`` + when used in ``<%page>``, the name of the def when using ``<%def>``, + the explicit or internally-generated name when using ``<%block>``). + Using the ``cache_key`` parameter, the key can be overridden + using a fixed or programmatically generated value. + + For example, here's a page + that caches any page which inherits from it, based on the + filename of the calling template: + + .. sourcecode:: mako + + <%page cached="True" cache_key="${self.filename}"/> + + ${next.body()} + + ## rest of template + +On a :class:`.Template` or :class:`.TemplateLookup`, the +caching can be configured using these arguments: + +* ``cache_enabled`` - Setting this + to ``False`` will disable all caching functionality + when the template renders. Defaults to ``True``. + e.g.: + + .. sourcecode:: python + + lookup = TemplateLookup( + directories='/path/to/templates', + cache_enabled = False + ) + +* ``cache_impl`` - The string name of the cache backend + to use. This defaults to ``'beaker'``, indicating + that the 'beaker' backend will be used. + +* ``cache_args`` - A dictionary of cache parameters that + will be consumed by the cache backend. See + :ref:`beaker_backend` and :ref:`dogpile.cache_backend` for examples. + + +Backend-Specific Cache Arguments +-------------------------------- + +The ``<%page>``, ``<%def>``, and ``<%block>`` tags +accept any named argument that starts with the prefix ``"cache_"``. +Those arguments are then packaged up and passed along to the +underlying caching implementation, minus the ``"cache_"`` prefix. + +The actual arguments understood are determined by the backend. + +* :ref:`beaker_backend` - Includes arguments understood by + Beaker. +* :ref:`dogpile.cache_backend` - Includes arguments understood by + dogpile.cache. + +.. _beaker_backend: + +Using the Beaker Cache Backend +------------------------------ + +When using Beaker, new implementations will want to make usage +of **cache regions** so that cache configurations can be maintained +externally to templates. These configurations live under +named "regions" that can be referred to within templates themselves. + +.. versionadded:: 0.6.0 + Support for Beaker cache regions. + +For example, suppose we would like two regions. One is a "short term" +region that will store content in a memory-based dictionary, +expiring after 60 seconds. The other is a Memcached region, +where values should expire in five minutes. To configure +our :class:`.TemplateLookup`, first we get a handle to a +:class:`beaker.cache.CacheManager`: + +.. sourcecode:: python + + from beaker.cache import CacheManager + + manager = CacheManager(cache_regions={ + 'short_term':{ + 'type': 'memory', + 'expire': 60 + }, + 'long_term':{ + 'type': 'ext:memcached', + 'url': '127.0.0.1:11211', + 'expire': 300 + } + }) + + lookup = TemplateLookup( + directories=['/path/to/templates'], + module_directory='/path/to/modules', + cache_impl='beaker', + cache_args={ + 'manager':manager + } + ) + +Our templates can then opt to cache data in one of either region, +using the ``cache_region`` argument. Such as using ``short_term`` +at the ``<%page>`` level: + +.. sourcecode:: mako + + <%page cached="True" cache_region="short_term"> + + ## ... + +Or, ``long_term`` at the ``<%block>`` level: + +.. sourcecode:: mako + + <%block name="header" cached="True" cache_region="long_term"> + other text + </%block> + +The Beaker backend also works without regions. There are a +variety of arguments that can be passed to the ``cache_args`` +dictionary, which are also allowable in templates via the +``<%page>``, ``<%block>``, +and ``<%def>`` tags specific to those sections. The values +given override those specified at the :class:`.TemplateLookup` +or :class:`.Template` level. + +With the possible exception +of ``cache_timeout``, these arguments are probably better off +staying at the template configuration level. Each argument +specified as ``cache_XYZ`` in a template tag is specified +without the ``cache_`` prefix in the ``cache_args`` dictionary: + +* ``cache_timeout`` - number of seconds in which to invalidate the + cached data. After this timeout, the content is re-generated + on the next call. Available as ``timeout`` in the ``cache_args`` + dictionary. +* ``cache_type`` - type of caching. ``'memory'``, ``'file'``, ``'dbm'``, or + ``'ext:memcached'`` (note that the string ``memcached`` is + also accepted by the dogpile.cache Mako plugin, though not by Beaker itself). + Available as ``type`` in the ``cache_args`` dictionary. +* ``cache_url`` - (only used for ``memcached`` but required) a single + IP address or a semi-colon separated list of IP address of + memcache servers to use. Available as ``url`` in the ``cache_args`` + dictionary. +* ``cache_dir`` - in the case of the ``'file'`` and ``'dbm'`` cache types, + this is the filesystem directory with which to store data + files. If this option is not present, the value of + ``module_directory`` is used (i.e. the directory where compiled + template modules are stored). If neither option is available + an exception is thrown. Available as ``dir`` in the + ``cache_args`` dictionary. + +.. _dogpile.cache_backend: + +Using the dogpile.cache Backend +------------------------------- + +`dogpile.cache`_ is a new replacement for Beaker. It provides +a modernized, slimmed down interface and is generally easier to use +than Beaker. As of this writing it has not yet been released. dogpile.cache +includes its own Mako cache plugin -- see :mod:`dogpile.cache.plugins.mako_cache` in the +dogpile.cache documentation. + +Programmatic Cache Access +========================= + +The :class:`.Template`, as well as any template-derived :class:`.Namespace`, has +an accessor called ``cache`` which returns the :class:`.Cache` object +for that template. This object is a facade on top of the underlying +:class:`.CacheImpl` object, and provides some very rudimental +capabilities, such as the ability to get and put arbitrary +values: + +.. sourcecode:: mako + + <% + local.cache.set("somekey", type="memory", "somevalue") + %> + +Above, the cache associated with the ``local`` namespace is +accessed and a key is placed within a memory cache. + +More commonly, the ``cache`` object is used to invalidate cached +sections programmatically: + +.. sourcecode:: python + + template = lookup.get_template('/sometemplate.html') + + # invalidate the "body" of the template + template.cache.invalidate_body() + + # invalidate an individual def + template.cache.invalidate_def('somedef') + + # invalidate an arbitrary key + template.cache.invalidate('somekey') + +You can access any special method or attribute of the :class:`.CacheImpl` +itself using the :attr:`impl <.Cache.impl>` attribute: + +.. sourcecode:: python + + template.cache.impl.do_something_special() + +Note that using implementation-specific methods will mean you can't +swap in a different kind of :class:`.CacheImpl` implementation at a +later time. + +.. _cache_plugins: + +Cache Plugins +============= + +The mechanism used by caching can be plugged in +using a :class:`.CacheImpl` subclass. This class implements +the rudimental methods Mako needs to implement the caching +API. Mako includes the :class:`.BeakerCacheImpl` class to +provide the default implementation. A :class:`.CacheImpl` class +is acquired by Mako using a ``importlib.metatada`` entrypoint, using +the name given as the ``cache_impl`` argument to :class:`.Template` +or :class:`.TemplateLookup`. This entry point can be +installed via the standard `setuptools`/``setup()`` procedure, underneath +the `EntryPoint` group named ``"mako.cache"``. It can also be +installed at runtime via a convenience installer :func:`.register_plugin` +which accomplishes essentially the same task. + +An example plugin that implements a local dictionary cache: + +.. sourcecode:: python + + from mako.cache import Cacheimpl, register_plugin + + class SimpleCacheImpl(CacheImpl): + def __init__(self, cache): + super(SimpleCacheImpl, self).__init__(cache) + self._cache = {} + + def get_or_create(self, key, creation_function, **kw): + if key in self._cache: + return self._cache[key] + else: + self._cache[key] = value = creation_function() + return value + + def set(self, key, value, **kwargs): + self._cache[key] = value + + def get(self, key, **kwargs): + return self._cache.get(key) + + def invalidate(self, key, **kwargs): + self._cache.pop(key, None) + + # optional - register the class locally + register_plugin("simple", __name__, "SimpleCacheImpl") + +Enabling the above plugin in a template would look like: + +.. sourcecode:: python + + t = Template("mytemplate", + file="mytemplate.html", + cache_impl='simple') + +Guidelines for Writing Cache Plugins +------------------------------------ + +* The :class:`.CacheImpl` is created on a per-:class:`.Template` basis. The + class should ensure that only data for the parent :class:`.Template` is + persisted or returned by the cache methods. The actual :class:`.Template` + is available via the ``self.cache.template`` attribute. The ``self.cache.id`` + attribute, which is essentially the unique modulename of the template, is + a good value to use in order to represent a unique namespace of keys specific + to the template. +* Templates only use the :meth:`.CacheImpl.get_or_create()` method + in an implicit fashion. The :meth:`.CacheImpl.set`, + :meth:`.CacheImpl.get`, and :meth:`.CacheImpl.invalidate` methods are + only used in response to direct programmatic access to the corresponding + methods on the :class:`.Cache` object. +* :class:`.CacheImpl` will be accessed in a multithreaded fashion if the + :class:`.Template` itself is used multithreaded. Care should be taken + to ensure caching implementations are threadsafe. +* A library like `Dogpile <http://pypi.python.org/pypi/dogpile.core>`_, which + is a minimal locking system derived from Beaker, can be used to help + implement the :meth:`.CacheImpl.get_or_create` method in a threadsafe + way that can maximize effectiveness across multiple threads as well + as processes. :meth:`.CacheImpl.get_or_create` is the + key method used by templates. +* All arguments passed to ``**kw`` come directly from the parameters + inside the ``<%def>``, ``<%block>``, or ``<%page>`` tags directly, + minus the ``"cache_"`` prefix, as strings, with the exception of + the argument ``cache_timeout``, which is passed to the plugin + as the name ``timeout`` with the value converted to an integer. + Arguments present in ``cache_args`` on :class:`.Template` or + :class:`.TemplateLookup` are passed directly, but are superseded + by those present in the most specific template tag. +* The directory where :class:`.Template` places module files can + be acquired using the accessor ``self.cache.template.module_directory``. + This directory can be a good place to throw cache-related work + files, underneath a prefix like ``_my_cache_work`` so that name + conflicts with generated modules don't occur. + +API Reference +============= + +.. autoclass:: mako.cache.Cache + :members: + :show-inheritance: + +.. autoclass:: mako.cache.CacheImpl + :members: + :show-inheritance: + +.. autofunction:: mako.cache.register_plugin + +.. autoclass:: mako.ext.beaker_cache.BeakerCacheImpl + :members: + :show-inheritance: + diff --git a/doc/build/changelog.rst b/doc/build/changelog.rst new file mode 100644 index 0000000..61019e6 --- /dev/null +++ b/doc/build/changelog.rst @@ -0,0 +1,2534 @@ + +========= +Changelog +========= + +1.3 +=== + +.. changelog:: + :version: 1.3.0 + :released: Wed Nov 8 2023 + + .. change:: + :tags: change, installation + + Mako 1.3.0 bumps the minimum Python version to 3.8, as 3.7 is EOL as of + 2023-06-27. Python 3.12 is now supported explicitly. + +1.2 +=== + + +.. changelog:: + :version: 1.2.4 + :released: Tue Nov 15 2022 + + .. change:: + :tags: bug, codegen + :tickets: 368 + + Fixed issue where unpacking nested tuples in a for loop using would raise a + "couldn't apply loop context" error if the loop context was used. The regex + used to match the for loop expression now allows the list of loop variables + to contain parenthesized sub-tuples. Pull request courtesy Matt Trescott. + + +.. changelog:: + :version: 1.2.3 + :released: Thu Sep 22 2022 + + .. change:: + :tags: bug, lexer + :tickets: 367 + + Fixed issue in lexer in the same category as that of :ticket:`366` where + the regexp used to match an end tag didn't correctly organize for matching + characters surrounded by whitespace, leading to high memory / interpreter + hang if a closing tag incorrectly had a large amount of unterminated space + in it. Credit to Sebastian Chnelik for locating the issue. + + As Mako templates inherently render and directly invoke arbitrary Python + code from the template source, it is **never** appropriate to create + templates that contain untrusted input. + +.. changelog:: + :version: 1.2.2 + :released: Mon Aug 29 2022 + + .. change:: + :tags: bug, lexer + :tickets: 366 + + Fixed issue in lexer where the regexp used to match tags would not + correctly interpret quoted sections individually. While this parsing issue + still produced the same expected tag structure later on, the mis-handling + of quoted sections was also subject to a regexp crash if a tag had a large + number of quotes within its quoted sections. Credit to Sebastian + Chnelik for locating the issue. + + As Mako templates inherently render and directly invoke arbitrary Python + code from the template source, it is **never** appropriate to create + templates that contain untrusted input. + +.. changelog:: + :version: 1.2.1 + :released: Thu Jun 30 2022 + + .. change:: + :tags: performance + :tickets: 361 + + Optimized some codepaths within the lexer/Python code generation process, + improving performance for generation of templates prior to their being + cached. Pull request courtesy Takuto Ikuta. + + .. change:: + :tags: bug, tests + :tickets: 360 + + Various fixes to the test suite in the area of exception message rendering + to accommodate for variability in Python versions as well as Pygments. + +.. changelog:: + :version: 1.2.0 + :released: Thu Mar 10 2022 + + .. change:: + :tags: changed, py3k + :tickets: 351 + + Corrected "universal wheel" directive in ``setup.cfg`` so that building a + wheel does not target Python 2. + + .. change:: + :tags: changed, py3k + + The ``bytestring_passthrough`` template argument is removed, as this + flag only applied to Python 2. + + .. change:: + :tags: changed, py3k + + With the removal of Python 2's ``cStringIO``, Mako now uses its own + internal ``FastEncodingBuffer`` exclusively. + + .. change:: + :tags: changed, py3k + + Removed ``disable_unicode`` flag, that's no longer used in Python 3. + + .. change:: + :tags: changed + :tickets: 349 + + Refactored test utilities into ``mako.testing`` module. Removed + ``unittest.TestCase`` dependency in favor of ``pytest``. + + .. change:: + :tags: changed, setup + + Replaced the use of ``pkg_resources`` with the ``importlib`` library. + For Python < 3.8 the library ``importlib_metadata`` is used. + + .. change:: + :tags: changed, py3k + + Removed support for Python 2 and Python 3.6. Mako now requires Python >= + 3.7. + + .. change:: + :tags: bug, py3k + + Mako now performs exception chaining using ``raise from``, correctly + identifying underlying exception conditions when it raises its own + exceptions. Pull request courtesy Ram Rachum. + +1.1 +=== + +.. changelog:: + :version: 1.1.6 + :released: Wed Nov 17 2021 + + .. change:: + :tags: bug, lexer + :tickets: 346 + :versions: 1.2.0, 1.1.6 + + Fixed issue where control statements on multi lines with a backslash would + not parse correctly if the template itself contained CR/LF pairs as on + Windows. Pull request courtesy Charles Pigott. + + +.. changelog:: + :version: 1.1.5 + :released: Fri Aug 20 2021 + + .. change:: + :tags: bug, tests + :tickets: 338 + + Fixed some issues with running the test suite which would be revealed by + running tests in random order. + + + +.. changelog:: + :version: 1.1.4 + :released: Thu Jan 14 2021 + + .. change:: + :tags: bug, py3k + :tickets: 328 + + Fixed Python deprecation issues related to module importing, as well as + file access within the Lingua plugin, for deprecated APIs that began to + emit warnings under Python 3.10. Pull request courtesy Petr Viktorin. + +.. changelog:: + :version: 1.1.3 + :released: Fri May 29 2020 + + .. change:: + :tags: bug, templates + :tickets: 267 + + The default template encoding is now utf-8. Previously, the encoding was + "ascii", which was standard throughout Python 2. This allows that + "magic encoding comment" for utf-8 templates is no longer required. + + +.. changelog:: + :version: 1.1.2 + :released: Sun Mar 1 2020 + + .. change:: + :tags: feature, commands + :tickets: 283 + + Added --output-file argument to the Mako command line runner, which allows + a specific output file to be selected. Pull request courtesy Björn + Dahlgren. + +.. changelog:: + :version: 1.1.1 + :released: Mon Jan 20 2020 + + .. change:: + :tags: bug, py3k + :tickets: 310 + + Replaced usage of the long-superseded "parser.suite" module in the + mako.util package for parsing the python magic encoding comment with the + "ast.parse" function introduced many years ago in Python 2.5, as + "parser.suite" is emitting deprecation warnings in Python 3.9. + + + + .. change:: + :tags: bug, ext + :tickets: 304 + + Added "babel" and "lingua" dependency entries to the setuptools entrypoints + for the babel and lingua extensions, so that pkg_resources can check that + these extra dependencies are available, raising an informative + exception if not. Pull request courtesy sinoroc. + + + +.. changelog:: + :version: 1.1.0 + :released: Thu Aug 1 2019 + + .. change:: + :tags: bug, py3k, windows + :tickets: 301 + + Replaced usage of time.clock() on windows as well as time.time() elsewhere + for microsecond timestamps with timeit.default_timer(), as time.clock() is + being removed in Python 3.8. Pull request courtesy Christoph Reiter. + + + .. change:: + :tags: bug, py3k + :tickets: 295 + + Replaced usage of ``inspect.getfullargspec()`` with the vendored version + used by SQLAlchemy, Alembic to avoid future deprecation warnings. Also + cleans up an additional version of the same function that's apparently + been floating around for some time. + + + .. change:: + :tags: changed, setup + :tickets: 303 + + Removed the "python setup.py test" feature in favor of a straight run of + "tox". Per Pypa / pytest developers, "setup.py" commands are in general + headed towards deprecation in favor of tox. The tox.ini script has been + updated such that running "tox" with no arguments will perform a single run + of the test suite against the default installed Python interpreter. + + .. seealso:: + + https://github.com/pypa/setuptools/issues/1684 + + https://github.com/pytest-dev/pytest/issues/5534 + + .. change:: + :tags: changed, py3k, installer + :tickets: 249 + + Mako 1.1 now supports Python versions: + + * 2.7 + * 3.4 and higher + + This includes that setup.py no longer includes any conditionals, allowing + for a pure Python wheel build, however this is not necessarily part of the + Pypi release process as of yet. The test suite also raises for Python + deprecation warnings. + + +1.0 +=== + +.. changelog:: + :version: 1.0.14 + :released: Sat Jul 20 2019 + + .. change:: + :tags: feature, template + + The ``n`` filter is now supported in the ``<%page>`` tag. This allows a + template to omit the default expression filters throughout a whole + template, for those cases where a template-wide filter needs to have + default filtering disabled. Pull request courtesy Martin von Gagern. + + .. seealso:: + + :ref:`expression_filtering_nfilter` + + + + .. change:: + :tags: bug, exceptions + + Fixed issue where the correct file URI would not be shown in the + template-formatted exception traceback if the template filename were not + known. Additionally fixes an issue where stale filenames would be + displayed if a stack trace alternated between different templates. Pull + request courtesy Martin von Gagern. + + +.. changelog:: + :version: 1.0.13 + :released: Mon Jul 1 2019 + + .. change:: + :tags: bug, exceptions + + Improved the line-number tracking for source lines inside of Python ``<% + ... %>`` blocks, such that text- and HTML-formatted exception traces such + as that of :func:`.html_error_template` now report the correct source line + inside the block, rather than the first line of the block itself. + Exceptions in ``<%! ... %>`` blocks which get raised while loading the + module are still not reported correctly, as these are handled before the + Mako code is generated. Pull request courtesy Martin von Gagern. + +.. changelog:: + :version: 1.0.12 + :released: Wed Jun 5 2019 + + .. change:: + :tags: bug, py3k + :tickets: 296 + + Fixed regression where import refactors in Mako 1.0.11 caused broken + imports on Python 3.8. + + +.. changelog:: + :version: 1.0.11 + :released: Fri May 31 2019 + + .. change:: + :tags: changed + + Updated for additional project metadata in setup.py. Additionally, + the code has been reformatted using Black and zimports. + +.. changelog:: + :version: 1.0.10 + :released: Fri May 10 2019 + + .. change:: + :tags: bug, py3k + :tickets: 293 + + Added a default encoding of "utf-8" when the :class:`.RichTraceback` + object retrieves Python source lines from a Python traceback; as these + are bytes in Python 3 they need to be decoded so that they can be + formatted in the template. + +.. changelog:: + :version: 1.0.9 + :released: Mon Apr 15 2019 + + .. change:: + :tags: bug + :tickets: 287 + + Further corrected the previous fix for :ticket:`287` as it relied upon + an attribute that is monkeypatched by Python's ``ast`` module for some + reason, which fails if ``ast`` hasn't been imported; the correct + attribute ``Constant.value`` is now used. Also note the issue + was mis-numbered in the previous changelog note. + +.. changelog:: + :version: 1.0.8 + :released: Wed Mar 20 2019 + :released: Wed Mar 20 2019 + + .. change:: + :tags: bug + :tickets: 287 + + Fixed an element in the AST Python generator which changed + for Python 3.8, causing expression generation to fail. + + .. change:: + :tags: feature + :tickets: 271 + + Added ``--output-encoding`` flag to the mako-render script. + Pull request courtesy lacsaP. + + .. change:: + :tags: bug + + Removed unnecessary "usage" prefix from mako-render script. + Pull request courtesy Hugo. + +.. changelog:: + :version: 1.0.7 + :released: Thu Jul 13 2017 + + .. change:: + :tags: bug + + Changed the "print" in the mako-render command to + sys.stdout.write(), avoiding the extra newline at the end + of the template output. Pull request courtesy + Yves Chevallier. + +.. changelog:: + :version: 1.0.6 + :released: Wed Nov 9 2016 + + .. change:: + :tags: feature + + Added new parameter :paramref:`.Template.include_error_handler` . + This works like :paramref:`.Template.error_handler` but indicates the + handler should take place when this template is included within another + template via the ``<%include>`` tag. Pull request courtesy + Huayi Zhang. + +.. changelog:: + :version: 1.0.5 + :released: Wed Nov 2 2016 + + .. change:: + :tags: bug + + Updated the Sphinx documentation builder to work with recent + versions of Sphinx. + +.. changelog:: + :version: 1.0.4 + :released: Thu Mar 10 2016 + + .. change:: + :tags: feature, test + + The default test runner is now py.test. Running "python setup.py test" + will make use of py.test instead of nose. nose still works as a test + runner as well, however. + + .. change:: + :tags: bug, lexer + :pullreq: github:19 + + Major improvements to lexing of intricate Python sections which may + contain complex backslash sequences, as well as support for the bitwise + operator (e.g. pipe symbol) inside of expression sections distinct + from the Mako "filter" operator, provided the operator is enclosed + within parentheses or brackets. Pull request courtesy Daniel Martin. + + .. change:: + :tags: feature + + Added new method :meth:`.Template.list_defs`. Pull request courtesy + Jonathan Vanasco. + +.. changelog:: + :version: 1.0.3 + :released: Tue Oct 27 2015 + + .. change:: + :tags: bug, babel + + Fixed an issue where the Babel plugin would not handle a translation + symbol that contained non-ascii characters. Pull request courtesy + Roman Imankulov. + +.. changelog:: + :version: 1.0.2 + :released: Wed Aug 26 2015 + + .. change:: + :tags: bug, installation + :tickets: 249 + + The "universal wheel" marker is removed from setup.cfg, because + our setup.py currently makes use of conditional dependencies. + In :ticket:`249`, the discussion is ongoing on how to correct our + setup.cfg / setup.py fully so that we can handle the per-version + dependency changes while still maintaining optimal wheel settings, + so this issue is not yet fully resolved. + + .. change:: + :tags: bug, py3k + :tickets: 250 + + Repair some calls within the ast module that no longer work on Python3.5; + additionally replace the use of ``inspect.getargspec()`` under + Python 3 (seems to be called from the TG plugin) to avoid deprecation + warnings. + + .. change:: + :tags: bug + + Update the Lingua translation extraction plugin to correctly + handle templates mixing Python control statements (such as if, + for and while) with template fragments. Pull request courtesy + Laurent Daverio. + + .. change:: + :tags: feature + :tickets: 236 + + Added ``STOP_RENDERING`` keyword for returning/exiting from a + template early, which is a synonym for an empty string ``""``. + Previously, the docs suggested a bare + ``return``, but this could cause ``None`` to appear in the + rendered template result. + + .. seealso:: + + :ref:`syntax_exiting_early` + +.. changelog:: + :version: 1.0.1 + :released: Thu Jan 22 2015 + + .. change:: + :tags: feature + + Added support for Lingua, a translation extraction system as an + alternative to Babel. Pull request courtesy Wichert Akkerman. + + .. change:: + :tags: bug, py3k + + Modernized the examples/wsgi/run_wsgi.py file for Py3k. + Pull requset courtesy Cody Taylor. + +.. changelog:: + :version: 1.0.0 + :released: Sun Jun 8 2014 + + .. change:: + :tags: bug, py2k + + Improved the error re-raise operation when a custom + :paramref:`.Template.error_handler` is used that does not handle + the exception; the original stack trace etc. is now preserved. + Pull request courtesy Manfred Haltner. + + .. change:: + :tags: bug, py2k, filters + + Added an html_escape filter that works in "non unicode" mode. + Previously, when using ``disable_unicode=True``, the ``u`` filter + would fail to handle non-ASCII bytes properly. Pull request + courtesy George Xie. + + .. change:: + :tags: general + + Compatibility changes; in order to modernize the codebase, Mako + is now dropping support for Python 2.4 and Python 2.5 altogether. + The source base is now targeted at Python 2.6 and forwards. + + .. change:: + :tags: feature + + Template modules now generate a JSON "metadata" structure at the bottom + of the source file which includes parseable information about the + templates' source file, encoding etc. as well as a mapping of module + source lines to template lines, thus replacing the "# SOURCE LINE" + markers throughout the source code. The structure also indicates those + lines that are explicitly not part of the template's source; the goal + here is to allow better integration with coverage and other tools. + + .. change:: + :tags: bug, py3k + + Fixed bug in ``decode.<encoding>`` filter where a non-string object + would not be correctly interpreted in Python 3. + + .. change:: + :tags: bug, py3k + :tickets: 227 + + Fixed bug in Python parsing logic which would fail on Python 3 + when a "try/except" targeted a tuple of exception types, rather + than a single exception. + + .. change:: + :tags: feature + + mako-render is now implemented as a setuptools entrypoint script; + a standalone mako.cmd.cmdline() callable is now available, and the + system also uses argparse now instead of optparse. Pull request + courtesy Derek Harland. + + .. change:: + :tags: feature + + The mako-render script will now catch exceptions and run them + into the text error handler, and exit with a non-zero exit code. + Pull request courtesy Derek Harland. + + .. change:: + :tags: bug + + A rework of the mako-render script allows the script to run + correctly when given a file pathname that is outside of the current + directory, e.g. ``mako-render ../some_template.mako``. In this case, + the "template root" defaults to the directory in which the template + is located, instead of ".". The script also accepts a new argument + ``--template-dir`` which can be specified multiple times to establish + template lookup directories. Standard input for templates also works + now too. Pull request courtesy Derek Harland. + + .. change:: + :tags: feature, py3k + :pullreq: github:7 + + Support is added for Python 3 "keyword only" arguments, as used in + defs. Pull request courtesy Eevee. + + +0.9 +=== + +.. changelog:: + :version: 0.9.1 + :released: Thu Dec 26 2013 + + .. change:: + :tags: bug + :tickets: 225 + + Fixed bug in Babel plugin where translator comments + would be lost if intervening text nodes were encountered. + Fix courtesy Ned Batchelder. + + .. change:: + :tags: bug + :tickets: + + Fixed TGPlugin.render method to support unicode template + names in Py2K - courtesy Vladimir Magamedov. + + .. change:: + :tags: bug + :tickets: + + Fixed an AST issue that was preventing correct operation + under alpha versions of Python 3.4. Pullreq courtesy Zer0-. + + .. change:: + :tags: bug + :tickets: + + Changed the format of the "source encoding" header output + by the code generator to use the format ``# -*- coding:%s -*-`` + instead of ``# -*- encoding:%s -*-``; the former is more common + and compatible with emacs. Courtesy Martin Geisler. + + .. change:: + :tags: bug + :tickets: 224 + + Fixed issue where an old lexer rule prevented a template line + which looked like "#*" from being correctly parsed. + +.. changelog:: + :version: 0.9.0 + :released: Tue Aug 27 2013 + + .. change:: + :tags: bug + :tickets: 219 + + The Context.locals_() method becomes a private underscored + method, as this method has a specific internal use. The purpose + of Context.kwargs has been clarified, in that it only delivers + top level keyword arguments originally passed to template.render(). + + .. change:: + :tags: bug + :tickets: + + Fixed the babel plugin to properly interpret ${} sections + inside of a "call" tag, i.e. <%self:some_tag attr="${_('foo')}"/>. + Code that's subject to babel escapes in here needs to be + specified as a Python expression, not a literal. This change + is backwards incompatible vs. code that is relying upon a _('') + translation to be working within a call tag. + + .. change:: + :tags: bug + :tickets: 187 + + The Babel plugin has been repaired to work on Python 3. + + .. change:: + :tags: bug + :tickets: 207 + + Using <%namespace import="*" module="somemodule"/> now + skips over module elements that are not explcitly callable, + avoiding TypeError when trying to produce partials. + + .. change:: + :tags: bug + :tickets: 190 + + Fixed Py3K bug where a "lambda" expression was not + interpreted correctly within a template tag; also + fixed in Py2.4. + +0.8 +=== + +.. changelog:: + :version: 0.8.1 + :released: Fri May 24 2013 + + .. change:: + :tags: bug + :tickets: 216 + + Changed setup.py to skip installing markupsafe + if Python version is < 2.6 or is between 3.0 and + less than 3.3, as Markupsafe now only supports 2.6->2.X, + 3.3->3.X. + + .. change:: + :tags: bug + :tickets: 214 + + Fixed regression where "entity" filter wasn't + converted for py3k properly (added tests.) + + .. change:: + :tags: bug + :tickets: 212 + + Fixed bug where mako-render script wasn't + compatible with Py3k. + + .. change:: + :tags: bug + :tickets: 213 + + Cleaned up all the various deprecation/ + file warnings when running the tests under + various Pythons with warnings turned on. + +.. changelog:: + :version: 0.8.0 + :released: Wed Apr 10 2013 + + .. change:: + :tags: feature + :tickets: + + Performance improvement to the + "legacy" HTML escape feature, used for XML + escaping and when markupsafe isn't present, + courtesy George Xie. + + .. change:: + :tags: bug + :tickets: 209 + + Fixed bug whereby an exception in Python 3 + against a module compiled to the filesystem would + fail trying to produce a RichTraceback due to the + content being in bytes. + + .. change:: + :tags: bug + :tickets: 208 + + Change default for compile()->reserved_names + from tuple to frozenset, as this is expected to be + a set by default. + + .. change:: + :tags: feature + :tickets: + + Code has been reworked to support Python 2.4-> + Python 3.xx in place. 2to3 no longer needed. + + .. change:: + :tags: feature + :tickets: + + Added lexer_cls argument to Template, + TemplateLookup, allows alternate Lexer classes + to be used. + + .. change:: + :tags: feature + :tickets: + + Added future_imports parameter to Template + and TemplateLookup, renders the __future__ header + with desired capabilities at the top of the generated + template module. Courtesy Ben Trofatter. + +0.7 +=== + +.. changelog:: + :version: 0.7.3 + :released: Wed Nov 7 2012 + + .. change:: + :tags: bug + :tickets: + + legacy_html_escape function, used when + Markupsafe isn't installed, was using an inline-compiled + regexp which causes major slowdowns on Python 3.3; + is now precompiled. + + .. change:: + :tags: bug + :tickets: 201 + + AST supporting now supports tuple-packed + function arguments inside pure-python def + or lambda expressions. + + .. change:: + :tags: bug + :tickets: + + Fixed Py3K bug in the Babel extension. + + .. change:: + :tags: bug + :tickets: + + Fixed the "filter" attribute of the + <%text> tag so that it pulls locally specified + identifiers from the context the same + way as that of <%block> and <%filter>. + + .. change:: + :tags: bug + :tickets: + + Fixed bug in plugin loader to correctly + raise exception when non-existent plugin + is specified. + +.. changelog:: + :version: 0.7.2 + :released: Fri Jul 20 2012 + + .. change:: + :tags: bug + :tickets: 193 + + Fixed regression in 0.7.1 where AST + parsing for Py2.4 was broken. + +.. changelog:: + :version: 0.7.1 + :released: Sun Jul 8 2012 + + .. change:: + :tags: feature + :tickets: 146 + + Control lines with no bodies will + now succeed, as "pass" is added for these + when no statements are otherwise present. + Courtesy Ben Trofatter + + .. change:: + :tags: bug + :tickets: 192 + + Fixed some long-broken scoping behavior + involving variables declared in defs and such, + which only became apparent when + the strict_undefined flag was turned on. + + .. change:: + :tags: bug + :tickets: 191 + + Can now use strict_undefined at the + same time args passed to def() are used + by other elements of the <%def> tag. + +.. changelog:: + :version: 0.7.0 + :released: Fri Mar 30 2012 + + .. change:: + :tags: feature + :tickets: 125 + + Added new "loop" variable to templates, + is provided within a % for block to provide + info about the loop such as index, first/last, + odd/even, etc. A migration path is also provided + for legacy templates via the "enable_loop" argument + available on Template, TemplateLookup, and <%page>. + Thanks to Ben Trofatter for all + the work on this + + .. change:: + :tags: feature + :tickets: + + Added a real check for "reserved" + names, that is names which are never pulled + from the context and cannot be passed to + the template.render() method. Current names + are "context", "loop", "UNDEFINED". + + .. change:: + :tags: feature + :tickets: 95 + + The html_error_template() will now + apply Pygments highlighting to the source + code displayed in the traceback, if Pygments + if available. Courtesy Ben Trofatter + + .. change:: + :tags: feature + :tickets: 147 + + Added support for context managers, + i.e. "% with x as e:/ % endwith" support. + Courtesy Ben Trofatter + + .. change:: + :tags: feature + :tickets: 185 + + Added class-level flag to CacheImpl + "pass_context"; when True, the keyword argument + 'context' will be passed to get_or_create() + containing the Mako Context object. + + .. change:: + :tags: bug + :tickets: 182 + + Fixed some Py3K resource warnings due + to filehandles being implicitly closed. + + .. change:: + :tags: bug + :tickets: 186 + + Fixed endless recursion bug when + nesting multiple def-calls with content. + Thanks to Jeff Dairiki. + + .. change:: + :tags: feature + :tickets: + + Added Jinja2 to the example + benchmark suite, courtesy Vincent Férotin + +Older Versions +============== + +.. changelog:: + :version: 0.6.2 + :released: Thu Feb 2 2012 + + .. change:: + :tags: bug + :tickets: 86, 20 + + The ${{"foo":"bar"}} parsing issue is fixed!! + The legendary Eevee has slain the dragon!. Also fixes quoting issue + at. + +.. changelog:: + :version: 0.6.1 + :released: Sat Jan 28 2012 + + .. change:: + :tags: bug + :tickets: + + Added special compatibility for the 0.5.0 + Cache() constructor, which was preventing file + version checks and not allowing Mako 0.6 to + recompile the module files. + +.. changelog:: + :version: 0.6.0 + :released: Sat Jan 21 2012 + + .. change:: + :tags: feature + :tickets: + + Template caching has been converted into a plugin + system, whereby the usage of Beaker is just the + default plugin. Template and TemplateLookup + now accept a string "cache_impl" parameter which + refers to the name of a cache plugin, defaulting + to the name 'beaker'. New plugins can be + registered as pkg_resources entrypoints under + the group "mako.cache", or registered directly + using mako.cache.register_plugin(). The + core plugin is the mako.cache.CacheImpl + class. + + .. change:: + :tags: feature + :tickets: + + Added support for Beaker cache regions + in templates. Usage of regions should be considered + as superseding the very obsolete idea of passing in + backend options, timeouts, etc. within templates. + + .. change:: + :tags: feature + :tickets: + + The 'put' method on Cache is now + 'set'. 'put' is there for backwards compatibility. + + .. change:: + :tags: feature + :tickets: + + The <%def>, <%block> and <%page> tags now accept + any argument named "cache_*", and the key + minus the "cache_" prefix will be passed as keyword + arguments to the CacheImpl methods. + + .. change:: + :tags: feature + :tickets: + + Template and TemplateLookup now accept an argument + cache_args, which refers to a dictionary containing + cache parameters. The cache_dir, cache_url, cache_type, + cache_timeout arguments are deprecated (will probably + never be removed, however) and can be passed + now as cache_args={'url':<some url>, 'type':'memcached', + 'timeout':50, 'dir':'/path/to/some/directory'} + + .. change:: + :tags: feature/bug + :tickets: 180 + + Can now refer to context variables + within extra arguments to <%block>, <%def>, i.e. + <%block name="foo" cache_key="${somekey}">. + Filters can also be used in this way, i.e. + <%def name="foo()" filter="myfilter"> + then template.render(myfilter=some_callable) + + .. change:: + :tags: feature + :tickets: 178 + + Added "--var name=value" option to the mako-render + script, allows passing of kw to the template from + the command line. + + .. change:: + :tags: feature + :tickets: 181 + + Added module_writer argument to Template, + TemplateLookup, allows a callable to be passed which + takes over the writing of the template's module source + file, so that special environment-specific steps + can be taken. + + .. change:: + :tags: bug + :tickets: 142 + + The exception message in the html_error_template + is now escaped with the HTML filter. + + .. change:: + :tags: bug + :tickets: 173 + + Added "white-space:pre" style to html_error_template() + for code blocks so that indentation is preserved + + .. change:: + :tags: bug + :tickets: 175 + + The "benchmark" example is now Python 3 compatible + (even though several of those old template libs aren't + available on Py3K, so YMMV) + + +.. changelog:: + :version: 0.5.0 + :released: Wed Sep 28 2011 + + .. change:: + :tags: + :tickets: 174 + + A Template is explicitly disallowed + from having a url that normalizes to relative outside + of the root. That is, if the Lookup is based + at /home/mytemplates, an include that would place + the ultimate template at + /home/mytemplates/../some_other_directory, + i.e. outside of /home/mytemplates, + is disallowed. This usage was never intended + despite the lack of an explicit check. + The main issue this causes + is that module files can be written outside + of the module root (or raise an error, if file perms aren't + set up), and can also lead to the same template being + cached in the lookup under multiple, relative roots. + TemplateLookup instead has always supported multiple + file roots for this purpose. + + +.. changelog:: + :version: 0.4.2 + :released: Fri Aug 5 2011 + + .. change:: + :tags: + :tickets: 170 + + Fixed bug regarding <%call>/def calls w/ content + whereby the identity of the "caller" callable + inside the <%def> would be corrupted by the + presence of another <%call> in the same block. + + .. change:: + :tags: + :tickets: 169 + + Fixed the babel plugin to accommodate <%block> + +.. changelog:: + :version: 0.4.1 + :released: Wed Apr 6 2011 + + .. change:: + :tags: + :tickets: 164 + + New tag: <%block>. A variant on <%def> that + evaluates its contents in-place. + Can be named or anonymous, + the named version is intended for inheritance + layouts where any given section can be + surrounded by the <%block> tag in order for + it to become overrideable by inheriting + templates, without the need to specify a + top-level <%def> plus explicit call. + Modified scoping and argument rules as well as a + more strictly enforced usage scheme make it ideal + for this purpose without at all replacing most + other things that defs are still good for. + Lots of new docs. + + .. change:: + :tags: + :tickets: 165 + + a slight adjustment to the "highlight" logic + for generating template bound stacktraces. + Will stick to known template source lines + without any extra guessing. + +.. changelog:: + :version: 0.4.0 + :released: Sun Mar 6 2011 + + .. change:: + :tags: + :tickets: + + A 20% speedup for a basic two-page + inheritance setup rendering + a table of escaped data + (see http://techspot.zzzeek.org/2010/11/19/quick-mako-vs.-jinja-speed-test/). + A few configurational changes which + affect those in the I-don't-do-unicode + camp should be noted below. + + .. change:: + :tags: + :tickets: + + The FastEncodingBuffer is now used + by default instead of cStringIO or StringIO, + regardless of whether output_encoding + is set to None or not. FEB is faster than + both. Only StringIO allows bytestrings + of unknown encoding to pass right + through, however - while it is of course + not recommended to send bytestrings of unknown + encoding to the output stream, this + mode of usage can be re-enabled by + setting the flag bytestring_passthrough + to True. + + .. change:: + :tags: + :tickets: + + disable_unicode mode requires that + output_encoding be set to None - it also + forces the bytestring_passthrough flag + to True. + + .. change:: + :tags: + :tickets: 156 + + the <%namespace> tag raises an error + if the 'template' and 'module' attributes + are specified at the same time in + one tag. A different class is used + for each case which allows a reduction in + runtime conditional logic and function + call overhead. + + .. change:: + :tags: + :tickets: 159 + + the keys() in the Context, as well as + it's internal _data dictionary, now + include just what was specified to + render() as well as Mako builtins + 'caller', 'capture'. The contents + of __builtin__ are no longer copied. + Thanks to Daniel Lopez for pointing + this out. + + +.. changelog:: + :version: 0.3.6 + :released: Sat Nov 13 2010 + + .. change:: + :tags: + :tickets: 126 + + Documentation is on Sphinx. + + .. change:: + :tags: + :tickets: 154 + + Beaker is now part of "extras" in + setup.py instead of "install_requires". + This to produce a lighter weight install + for those who don't use the caching + as well as to conform to Pyramid + deployment practices. + + .. change:: + :tags: + :tickets: 153 + + The Beaker import (or attempt thereof) + is delayed until actually needed; + this to remove the performance penalty + from startup, particularly for + "single execution" environments + such as shell scripts. + + .. change:: + :tags: + :tickets: 155 + + Patch to lexer to not generate an empty + '' write in the case of backslash-ended + lines. + + .. change:: + :tags: + :tickets: 148 + + Fixed missing \**extra collection in + setup.py which prevented setup.py + from running 2to3 on install. + + .. change:: + :tags: + :tickets: + + New flag on Template, TemplateLookup - + strict_undefined=True, will cause + variables not found in the context to + raise a NameError immediately, instead of + defaulting to the UNDEFINED value. + + .. change:: + :tags: + :tickets: + + The range of Python identifiers that + are considered "undefined", meaning they + are pulled from the context, has been + trimmed back to not include variables + declared inside of expressions (i.e. from + list comprehensions), as well as + in the argument list of lambdas. This + to better support the strict_undefined + feature. The change should be + fully backwards-compatible but involved + a little bit of tinkering in the AST code, + which hadn't really been touched for + a couple of years, just FYI. + +.. changelog:: + :version: 0.3.5 + :released: Sun Oct 24 2010 + + .. change:: + :tags: + :tickets: 141 + + The <%namespace> tag allows expressions + for the `file` argument, i.e. with ${}. + The `context` variable, if needed, + must be referenced explicitly. + + .. change:: + :tags: + :tickets: + + ${} expressions embedded in tags, + such as <%foo:bar x="${...}">, now + allow multiline Python expressions. + + .. change:: + :tags: + :tickets: + + Fixed previously non-covered regular + expression, such that using a ${} expression + inside of a tag element that doesn't allow + them raises a CompileException instead of + silently failing. + + .. change:: + :tags: + :tickets: 151 + + Added a try/except around "import markupsafe". + This to support GAE which can't run markupsafe. No idea whatsoever if the + install_requires in setup.py also breaks GAE, + couldn't get an answer on this. + +.. changelog:: + :version: 0.3.4 + :released: Tue Jun 22 2010 + + .. change:: + :tags: + :tickets: + + Now using MarkupSafe for HTML escaping, + i.e. in place of cgi.escape(). Faster + C-based implementation and also escapes + single quotes for additional security. + Supports the __html__ attribute for + the given expression as well. + + When using "disable_unicode" mode, + a pure Python HTML escaper function + is used which also quotes single quotes. + + Note that Pylons by default doesn't + use Mako's filter - check your + environment.py file. + + .. change:: + :tags: + :tickets: 137 + + Fixed call to "unicode.strip" in + exceptions.text_error_template which + is not Py3k compatible. + +.. changelog:: + :version: 0.3.3 + :released: Mon May 31 2010 + + .. change:: + :tags: + :tickets: 135 + + Added conditional to RichTraceback + such that if no traceback is passed + and sys.exc_info() has been reset, + the formatter just returns blank + for the "traceback" portion. + + .. change:: + :tags: + :tickets: 131 + + Fixed sometimes incorrect usage of + exc.__class__.__name__ + in html/text error templates when using + Python 2.4 + + .. change:: + :tags: + :tickets: + + Fixed broken @property decorator on + template.last_modified + + .. change:: + :tags: + :tickets: 132 + + Fixed error formatting when a stacktrace + line contains no line number, as in when + inside an eval/exec-generated function. + + .. change:: + :tags: + :tickets: + + When a .py is being created, the tempfile + where the source is stored temporarily is + now made in the same directory as that of + the .py file. This ensures that the two + files share the same filesystem, thus + avoiding cross-filesystem synchronization + issues. Thanks to Charles Cazabon. + +.. changelog:: + :version: 0.3.2 + :released: Thu Mar 11 2010 + + .. change:: + :tags: + :tickets: 116 + + Calling a def from the top, via + template.get_def(...).render() now checks the + argument signature the same way as it did in + 0.2.5, so that TypeError is not raised. + reopen of + +.. changelog:: + :version: 0.3.1 + :released: Sun Mar 7 2010 + + .. change:: + :tags: + :tickets: 129 + + Fixed incorrect dir name in setup.py + +.. changelog:: + :version: 0.3.0 + :released: Fri Mar 5 2010 + + .. change:: + :tags: + :tickets: 123 + + Python 2.3 support is dropped. + + .. change:: + :tags: + :tickets: 119 + + Python 3 support is added ! See README.py3k + for installation and testing notes. + + .. change:: + :tags: + :tickets: 127 + + Unit tests now run with nose. + + .. change:: + :tags: + :tickets: 99 + + Source code escaping has been simplified. + In particular, module source files are now + generated with the Python "magic encoding + comment", and source code is passed through + mostly unescaped, except for that code which + is regenerated from parsed Python source. + This fixes usage of unicode in + <%namespace:defname> tags. + + .. change:: + :tags: + :tickets: 122 + + RichTraceback(), html_error_template().render(), + text_error_template().render() now accept "error" + and "traceback" as optional arguments, and + these are now actually used. + + .. change:: + :tags: + :tickets: + + The exception output generated when + format_exceptions=True will now be as a Python + unicode if it occurred during render_unicode(), + or an encoded string if during render(). + + .. change:: + :tags: + :tickets: 112 + + A percent sign can be emitted as the first + non-whitespace character on a line by escaping + it as in "%%". + + .. change:: + :tags: + :tickets: 94 + + Template accepts empty control structure, i.e. + % if: %endif, etc. + + .. change:: + :tags: + :tickets: 116 + + The <%page args> tag can now be used in a base + inheriting template - the full set of render() + arguments are passed down through the inherits + chain. Undeclared arguments go into \**pageargs + as usual. + + .. change:: + :tags: + :tickets: 109 + + defs declared within a <%namespace> section, an + uncommon feature, have been improved. The defs + no longer get doubly-rendered in the body() scope, + and now allow local variable assignment without + breakage. + + .. change:: + :tags: + :tickets: 128 + + Windows paths are handled correctly if a Template + is passed only an absolute filename (i.e. with c: + drive etc.) and no URI - the URI is converted + to a forward-slash path and module_directory + is treated as a windows path. + + .. change:: + :tags: + :tickets: 73 + + TemplateLookup raises TopLevelLookupException for + a given path that is a directory, not a filename, + instead of passing through to the template to + generate IOError. + + +.. changelog:: + :version: 0.2.6 + :released: + + .. change:: + :tags: + :tickets: + + Fix mako function decorators to preserve the + original function's name in all cases. Patch + from Scott Torborg. + + .. change:: + :tags: + :tickets: 118 + + Support the <%namespacename:defname> syntax in + the babel extractor. + + .. change:: + :tags: + :tickets: 88 + + Further fixes to unicode handling of .py files with the + html_error_template. + +.. changelog:: + :version: 0.2.5 + :released: Mon Sep 7 2009 + + .. change:: + :tags: + :tickets: + + Added a "decorator" kw argument to <%def>, + allows custom decoration functions to wrap + rendering callables. Mainly intended for + custom caching algorithms, not sure what + other uses there may be (but there may be). + Examples are in the "filtering" docs. + + .. change:: + :tags: + :tickets: 101 + + When Mako creates subdirectories in which + to store templates, it uses the more + permissive mode of 0775 instead of 0750, + helping out with certain multi-process + scenarios. Note that the mode is always + subject to the restrictions of the existing + umask. + + .. change:: + :tags: + :tickets: 104 + + Fixed namespace.__getattr__() to raise + AttributeError on attribute not found + instead of RuntimeError. + + .. change:: + :tags: + :tickets: 97 + + Added last_modified accessor to Template, + returns the time.time() when the module + was created. + + .. change:: + :tags: + :tickets: 102 + + Fixed lexing support for whitespace + around '=' sign in defs. + + .. change:: + :tags: + :tickets: 108 + + Removed errant "lower()" in the lexer which + was causing tags to compile with + case-insensitive names, thus messing up + custom <%call> names. + + .. change:: + :tags: + :tickets: 110 + + added "mako.__version__" attribute to + the base module. + +.. changelog:: + :version: 0.2.4 + :released: Tue Dec 23 2008 + + .. change:: + :tags: + :tickets: + + Fixed compatibility with Jython 2.5b1. + +.. changelog:: + :version: 0.2.3 + :released: Sun Nov 23 2008 + + .. change:: + :tags: + :tickets: + + the <%namespacename:defname> syntax described at + http://techspot.zzzeek.org/?p=28 has now + been added as a built in syntax, and is recommended + as a more modern syntax versus <%call expr="expression">. + The %call tag itself will always remain, + with <%namespacename:defname> presenting a more HTML-like + alternative to calling defs, both plain and + nested. Many examples of the new syntax are in the + "Calling a def with embedded content" section + of the docs. + + .. change:: + :tags: + :tickets: + + added support for Jython 2.5. + + .. change:: + :tags: + :tickets: + + cache module now uses Beaker's CacheManager + object directly, so that all cache types are included. + memcached is available as both "ext:memcached" and + "memcached", the latter for backwards compatibility. + + .. change:: + :tags: + :tickets: + + added "cache" accessor to Template, Namespace. + e.g. ${local.cache.get('somekey')} or + template.cache.invalidate_body() + + .. change:: + :tags: + :tickets: + + added "cache_enabled=True" flag to Template, + TemplateLookup. Setting this to False causes cache + operations to "pass through" and execute every time; + this flag should be integrated in Pylons with its own + cache_enabled configuration setting. + + .. change:: + :tags: + :tickets: 92 + + the Cache object now supports invalidate_def(name), + invalidate_body(), invalidate_closure(name), + invalidate(key), which will remove the given key + from the cache, if it exists. The cache arguments + (i.e. storage type) are derived from whatever has + been already persisted for that template. + + .. change:: + :tags: + :tickets: + + For cache changes to work fully, Beaker 1.1 is required. + 1.0.1 and up will work as well with the exception of + cache expiry. Note that Beaker 1.1 is **required** + for applications which use dynamically generated keys, + since previous versions will permanently store state in memory + for each individual key, thus consuming all available + memory for an arbitrarily large number of distinct + keys. + + .. change:: + :tags: + :tickets: 93 + + fixed bug whereby an <%included> template with + <%page> args named the same as a __builtin__ would not + honor the default value specified in <%page> + + .. change:: + :tags: + :tickets: 88 + + fixed the html_error_template not handling tracebacks from + normal .py files with a magic encoding comment + + .. change:: + :tags: + :tickets: + + RichTraceback() now accepts an optional traceback object + to be used in place of sys.exc_info()[2]. html_error_template() + and text_error_template() accept an optional + render()-time argument "traceback" which is passed to the + RichTraceback object. + + .. change:: + :tags: + :tickets: + + added ModuleTemplate class, which allows the construction + of a Template given a Python module generated by a previous + Template. This allows Python modules alone to be used + as templates with no compilation step. Source code + and template source are optional but allow error reporting + to work correctly. + + .. change:: + :tags: + :tickets: 90 + + fixed Python 2.3 compat. in mako.pyparser + + .. change:: + :tags: + :tickets: + + fix Babel 0.9.3 compatibility; stripping comment tags is now + optional (and enabled by default). + +.. changelog:: + :version: 0.2.2 + :released: Mon Jun 23 2008 + + .. change:: + :tags: + :tickets: 87 + + cached blocks now use the current context when rendering + an expired section, instead of the original context + passed in + + .. change:: + :tags: + :tickets: + + fixed a critical issue regarding caching, whereby + a cached block would raise an error when called within a + cache-refresh operation that was initiated after the + initiating template had completed rendering. + +.. changelog:: + :version: 0.2.1 + :released: Mon Jun 16 2008 + + .. change:: + :tags: + :tickets: + + fixed bug where 'output_encoding' parameter would prevent + render_unicode() from returning a unicode object. + + .. change:: + :tags: + :tickets: + + bumped magic number, which forces template recompile for + this version (fixes incompatible compile symbols from 0.1 + series). + + .. change:: + :tags: + :tickets: + + added a few docs for cache options, specifically those that + help with memcached. + +.. changelog:: + :version: 0.2.0 + :released: Tue Jun 3 2008 + + .. change:: + :tags: + :tickets: + + Speed improvements (as though we needed them, but people + contributed and there you go): + + .. change:: + :tags: + :tickets: 77 + + added "bytestring passthru" mode, via + `disable_unicode=True` argument passed to Template or + TemplateLookup. All unicode-awareness and filtering is + turned off, and template modules are generated with + the appropriate magic encoding comment. In this mode, + template expressions can only receive raw bytestrings + or Unicode objects which represent straight ASCII, and + render_unicode() may not be used if multibyte + characters are present. When enabled, speed + improvement around 10-20%. (courtesy + anonymous guest) + + .. change:: + :tags: + :tickets: 76 + + inlined the "write" function of Context into a local + template variable. This affords a 12-30% speedup in + template render time. (idea courtesy same anonymous + guest) + + .. change:: + :tags: + :tickets: + + New Features, API changes: + + .. change:: + :tags: + :tickets: 62 + + added "attr" accessor to namespaces. Returns + attributes configured as module level attributes, i.e. + within <%! %> sections. i.e.:: + + # somefile.html + <%! + foo = 27 + %> + + # some other template + <%namespace name="myns" file="somefile.html"/> + ${myns.attr.foo} + + The slight backwards incompatibility here is, you + can't have namespace defs named "attr" since the + "attr" descriptor will occlude it. + + .. change:: + :tags: + :tickets: 78 + + cache_key argument can now render arguments passed + directly to the %page or %def, i.e. <%def + name="foo(x)" cached="True" cache_key="${x}"/> + + .. change:: + :tags: + :tickets: + + some functions on Context are now private: + _push_buffer(), _pop_buffer(), + caller_stack._push_frame(), caller_stack._pop_frame(). + + .. change:: + :tags: + :tickets: 56, 81 + + added a runner script "mako-render" which renders + standard input as a template to stdout + + .. change:: + :tags: bugfixes + :tickets: 83, 84 + + can now use most names from __builtins__ as variable + names without explicit declaration (i.e. 'id', + 'exception', 'range', etc.) + + .. change:: + :tags: bugfixes + :tickets: 84 + + can also use builtin names as local variable names + (i.e. dict, locals) (came from fix for) + + .. change:: + :tags: bugfixes + :tickets: 68 + + fixed bug in python generation when variable names are + used with identifiers like "else", "finally", etc. + inside them + + .. change:: + :tags: bugfixes + :tickets: 69 + + fixed codegen bug which occurred when using <%page> + level caching, combined with an expression-based + cache_key, combined with the usage of <%namespace + import="*"/> - fixed lexer exceptions not cleaning up + temporary files, which could lead to a maximum number + of file descriptors used in the process + + .. change:: + :tags: bugfixes + :tickets: 71 + + fixed issue with inline format_exceptions that was + producing blank exception pages when an inheriting + template is present + + .. change:: + :tags: bugfixes + :tickets: + + format_exceptions will apply the encoding options of + html_error_template() to the buffered output + + .. change:: + :tags: bugfixes + :tickets: 75 + + rewrote the "whitespace adjuster" function to work + with more elaborate combinations of quotes and + comments + + +.. changelog:: + :version: 0.1.10 + :released: + + .. change:: + :tags: + :tickets: + + fixed propagation of 'caller' such that nested %def calls + within a <%call> tag's argument list propigates 'caller' + to the %call function itself (propigates to the inner + calls too, this is a slight side effect which previously + existed anyway) + + .. change:: + :tags: + :tickets: + + fixed bug where local.get_namespace() could put an + incorrect "self" in the current context + + .. change:: + :tags: + :tickets: + + fixed another namespace bug where the namespace functions + did not have access to the correct context containing + their 'self' and 'parent' + +.. changelog:: + :version: 0.1.9 + :released: + + .. change:: + :tags: + :tickets: 47 + + filters.Decode filter can also accept a non-basestring + object and will call str() + unicode() on it + + .. change:: + :tags: + :tickets: 53 + + comments can be placed at the end of control lines, + i.e. if foo: # a comment,, thanks to + Paul Colomiets + + .. change:: + :tags: + :tickets: 16 + + fixed expressions and page tag arguments and with embedded + newlines in CRLF templates, follow up to, thanks + Eric Woroshow + + .. change:: + :tags: + :tickets: 51 + + added an IOError catch for source file not found in RichTraceback + exception reporter + +.. changelog:: + :version: 0.1.8 + :released: Tue Jun 26 2007 + + .. change:: + :tags: + :tickets: + + variable names declared in render methods by internal + codegen prefixed by "__M_" to prevent name collisions + with user code + + .. change:: + :tags: + :tickets: 45 + + added a Babel (http://babel.edgewall.org/) extractor entry + point, allowing extraction of gettext messages directly from + mako templates via Babel + + .. change:: + :tags: + :tickets: + + fix to turbogears plugin to work with dot-separated names + (i.e. load_template('foo.bar')). also takes file extension + as a keyword argument (default is 'mak'). + + .. change:: + :tags: + :tickets: 35 + + more tg fix: fixed, allowing string-based + templates with tgplugin even if non-compatible args were sent + +.. changelog:: + :version: 0.1.7 + :released: Wed Jun 13 2007 + + .. change:: + :tags: + :tickets: + + one small fix to the unit tests to support python 2.3 + + .. change:: + :tags: + :tickets: + + a slight hack to how cache.py detects Beaker's memcached, + works around unexplained import behavior observed on some + python 2.3 installations + +.. changelog:: + :version: 0.1.6 + :released: Fri May 18 2007 + + .. change:: + :tags: + :tickets: + + caching is now supplied directly by Beaker, which has + all of MyghtyUtils merged into it now. The latest Beaker + (0.7.1) also fixes a bug related to how Mako was using the + cache API. + + .. change:: + :tags: + :tickets: 34 + + fix to module_directory path generation when the path is "./" + + .. change:: + :tags: + :tickets: 35 + + TGPlugin passes options to string-based templates + + .. change:: + :tags: + :tickets: 28 + + added an explicit stack frame step to template runtime, which + allows much simpler and hopefully bug-free tracking of 'caller', + fixes + + .. change:: + :tags: + :tickets: + + if plain Python defs are used with <%call>, a decorator + @runtime.supports_callable exists to ensure that the "caller" + stack is properly handled for the def. + + .. change:: + :tags: + :tickets: 37 + + fix to RichTraceback and exception reporting to get template + source code as a unicode object + + .. change:: + :tags: + :tickets: 39 + + html_error_template includes options "full=True", "css=True" + which control generation of HTML tags, CSS + + .. change:: + :tags: + :tickets: 40 + + added the 'encoding_errors' parameter to Template/TemplateLookup + for specifying the error handler associated with encoding to + 'output_encoding' + + .. change:: + :tags: + :tickets: 37 + + the Template returned by html_error_template now defaults to + output_encoding=sys.getdefaultencoding(), + encoding_errors='htmlentityreplace' + + .. change:: + :tags: + :tickets: + + control lines, i.e. % lines, support backslashes to continue long + lines (#32) + + .. change:: + :tags: + :tickets: + + fixed codegen bug when defining <%def> within <%call> within <%call> + + .. change:: + :tags: + :tickets: + + leading utf-8 BOM in template files is honored according to pep-0263 + +.. changelog:: + :version: 0.1.5 + :released: Sat Mar 31 2007 + + .. change:: + :tags: + :tickets: 26 + + AST expression generation - added in just about everything + expression-wise from the AST module + + .. change:: + :tags: + :tickets: 27 + + AST parsing, properly detects imports of the form "import foo.bar" + + .. change:: + :tags: + :tickets: + + fix to lexing of <%docs> tag nested in other tags + + .. change:: + :tags: + :tickets: 29 + + fix to context-arguments inside of <%include> tag which broke + during 0.1.4 + + .. change:: + :tags: + :tickets: + + added "n" filter, disables *all* filters normally applied to an expression + via <%page> or default_filters (but not those within the filter) + + .. change:: + :tags: + :tickets: + + added buffer_filters argument, defines filters applied to the return value + of buffered/cached/filtered %defs, after all filters defined with the %def + itself have been applied. allows the creation of default expression filters + that let the output of return-valued %defs "opt out" of that filtering + via passing special attributes or objects. + +.. changelog:: + :version: 0.1.4 + :released: Sat Mar 10 2007 + + .. change:: + :tags: + :tickets: + + got defs-within-defs to be cacheable + + .. change:: + :tags: + :tickets: 23 + + fixes to code parsing/whitespace adjusting where plain python comments + may contain quote characters + + .. change:: + :tags: + :tickets: + + fix to variable scoping for identifiers only referenced within + functions + + .. change:: + :tags: + :tickets: + + added a path normalization step to lookup so URIs like + "/foo/bar/../etc/../foo" pre-process the ".." tokens before checking + the filesystem + + .. change:: + :tags: + :tickets: + + fixed/improved "caller" semantics so that undefined caller is + "UNDEFINED", propigates __nonzero__ method so it evaulates to False if + not present, True otherwise. this way you can say % if caller:\n + ${caller.body()}\n% endif + + .. change:: + :tags: + :tickets: + + <%include> has an "args" attribute that can pass arguments to the + called template (keyword arguments only, must be declared in that + page's <%page> tag.) + + .. change:: + :tags: + :tickets: + + <%include> plus arguments is also programmatically available via + self.include_file(<filename>, \**kwargs) + + .. change:: + :tags: + :tickets: 24 + + further escaping added for multibyte expressions in %def, %call + attributes + +.. changelog:: + :version: 0.1.3 + :released: Wed Feb 21 2007 + + .. change:: + :tags: + :tickets: + + ***Small Syntax Change*** - the single line comment character is now + *two* hash signs, i.e. "## this is a comment". This avoids a common + collection with CSS selectors. + + .. change:: + :tags: + :tickets: + + the magic "coding" comment (i.e. # coding:utf-8) will still work with + either one "#" sign or two for now; two is preferred going forward, i.e. + ## coding:<someencoding>. + + .. change:: + :tags: + :tickets: + + new multiline comment form: "<%doc> a comment </%doc>" + + .. change:: + :tags: + :tickets: + + UNDEFINED evaluates to False + + .. change:: + :tags: + :tickets: + + improvement to scoping of "caller" variable when using <%call> tag + + .. change:: + :tags: + :tickets: + + added lexer error for unclosed control-line (%) line + + .. change:: + :tags: + :tickets: + + added "preprocessor" argument to Template, TemplateLookup - is a single + callable or list of callables which will be applied to the template text + before lexing. given the text as an argument, returns the new text. + + .. change:: + :tags: + :tickets: + + added mako.ext.preprocessors package, contains one preprocessor so far: + 'convert_comments', which will convert single # comments to the new ## + format + +.. changelog:: + :version: 0.1.2 + :released: Thu Feb 1 2007 + + .. change:: + :tags: + :tickets: 11 + + fix to parsing of code/expression blocks to insure that non-ascii + characters, combined with a template that indicates a non-standard + encoding, are expanded into backslash-escaped glyphs before being AST + parsed + + .. change:: + :tags: + :tickets: + + all template lexing converts the template to unicode first, to + immediately catch any encoding issues and ensure internal unicode + representation. + + .. change:: + :tags: + :tickets: + + added module_filename argument to Template to allow specification of a + specific module file + + .. change:: + :tags: + :tickets: 14 + + added modulename_callable to TemplateLookup to allow a function to + determine module filenames (takes filename, uri arguments). used for + + .. change:: + :tags: + :tickets: + + added optional input_encoding flag to Template, to allow sending a + unicode() object with no magic encoding comment + + .. change:: + :tags: + :tickets: + + "expression_filter" argument in <%page> applies only to expressions + + .. change:: + :tags: "unicode" + :tickets: + + added "default_filters" argument to Template, TemplateLookup. applies only + to expressions, gets prepended to "expression_filter" arg from <%page>. + defaults to, so that all expressions get stringified into u'' + by default (this is what Mako already does). By setting to [], expressions + are passed through raw. + + .. change:: + :tags: + :tickets: + + added "imports" argument to Template, TemplateLookup. so you can predefine + a list of import statements at the top of the template. can be used in + conjunction with default_filters. + + .. change:: + :tags: + :tickets: 16 + + support for CRLF templates...whoops ! welcome to all the windows users. + + .. change:: + :tags: + :tickets: + + small fix to local variable propigation for locals that are conditionally + declared + + .. change:: + :tags: + :tickets: + + got "top level" def calls to work, i.e. template.get_def("somedef").render() + +.. changelog:: + :version: 0.1.1 + :released: Sun Jan 14 2007 + + .. change:: + :tags: + :tickets: 8 + + buffet plugin supports string-based templates, allows ToscaWidgets to work + + .. change:: + :tags: + :tickets: + + AST parsing fixes: fixed TryExcept identifier parsing + + .. change:: + :tags: + :tickets: + + removed textmate tmbundle from contrib and into separate SVN location; + windows users cant handle those files, setuptools not very good at + "pruning" certain directories + + .. change:: + :tags: + :tickets: + + fix so that "cache_timeout" parameter is propigated + + .. change:: + :tags: + :tickets: + + fix to expression filters so that string conversion (actually unicode) + properly occurs before filtering + + .. change:: + :tags: + :tickets: + + better error message when a lookup is attempted with a template that has no + lookup + + .. change:: + :tags: + :tickets: + + implemented "module" attribute for namespace + + .. change:: + :tags: + :tickets: + + fix to code generation to correctly track multiple defs with the same name + + .. change:: + :tags: + :tickets: 9 + + "directories" can be passed to TemplateLookup as a scalar in which case it + gets converted to a list diff --git a/doc/build/conf.py b/doc/build/conf.py new file mode 100644 index 0000000..6c75698 --- /dev/null +++ b/doc/build/conf.py @@ -0,0 +1,311 @@ +# +# Mako documentation build configuration file +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import os +import sys + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath("../..")) +sys.path.insert(0, os.path.abspath(".")) + +if True: + import mako # noqa + + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +# extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', +# 'sphinx.ext.doctest', 'builder.builders'] + +extensions = [ + "sphinx.ext.autodoc", + "changelog", + "sphinx_paramlinks", + "zzzeeksphinx", +] + +changelog_render_ticket = "https://github.com/sqlalchemy/mako/issues/%s" + +changelog_render_pullreq = { + "default": "https://github.com/sqlalchemy/mako/pull/%s", + "github": "https://github.com/sqlalchemy/mako/pull/%s", +} + +# tags to sort on inside of sections +changelog_sections = [ + "changed", + "feature", + "bug", + "usecase", + "moved", + "removed", +] + + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["templates"] + +nitpicky = True + + +site_base = os.environ.get("RTD_SITE_BASE", "http://www.makotemplates.org") +site_adapter_template = "docs_adapter.mako" +site_adapter_py = "docs_adapter.py" + +# The suffix of source filenames. +source_suffix = ".rst" + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "Mako" +copyright = "the Mako authors and contributors" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = mako.__version__ +# The full version, including alpha/beta/rc tags. +release = "1.3.0" +release_date = "Wed Nov 8 2023" +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ["build"] + +# The reST default role (used for this markup: `text`) to use for all documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = "zsmako" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The style sheet to use for HTML and HTML Help pages. A file of that name +# must exist either in Sphinx' static/ path, or in one of the custom paths +# given in html_static_path. +html_style = "default.css" + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +html_title = "%s %s Documentation" % (project, release) + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["static"] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +html_last_updated_fmt = "%m/%d/%Y %H:%M:%S" + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +html_domain_indices = False + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, the reST sources are included in the HTML build as _sources/<name>. +# html_copy_source = True +html_copy_source = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True +html_show_sourcelink = False + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = "Makodoc" + +# autoclass_content = 'both' + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +# latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +# latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ( + "index", + "mako_%s.tex" % release.replace(".", "_"), + "Mako Documentation", + "Mike Bayer", + "manual", + ) +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Additional stuff for the LaTeX preamble. +# sets TOC depth to 2. +latex_preamble = r"\setcounter{tocdepth}{3}" + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + +# latex_elements = { +# 'papersize': 'letterpaper', +# 'pointsize': '10pt', +# } + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [("index", "mako", "Mako Documentation", ["Mako authors"], 1)] + + +# -- Options for Epub output --------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = "Mako" +epub_author = "Mako authors" +epub_publisher = "Mako authors" +epub_copyright = "Mako authors" + +# The language of the text. It defaults to the language option +# or en if the language is not set. +# epub_language = '' + +# The scheme of the identifier. Typical schemes are ISBN or URL. +# epub_scheme = '' + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# epub_identifier = '' + +# A unique identification for the text. +# epub_uid = '' + +# HTML files that should be inserted before the pages created by sphinx. +# The format is a list of tuples containing the path and title. +# epub_pre_files = [] + +# HTML files shat should be inserted after the pages created by sphinx. +# The format is a list of tuples containing the path and title. +# epub_post_files = [] + +# A list of files that should not be packed into the epub file. +# epub_exclude_files = [] + +# The depth of the table of contents in toc.ncx. +# epub_tocdepth = 3 + +# Allow duplicate toc entries. +# epub_tocdup = True diff --git a/doc/build/defs.rst b/doc/build/defs.rst new file mode 100644 index 0000000..314e9b9 --- /dev/null +++ b/doc/build/defs.rst @@ -0,0 +1,622 @@ +.. _defs_toplevel: + +=============== +Defs and Blocks +=============== + +``<%def>`` and ``<%block>`` are two tags that both demarcate any block of text +and/or code. They both exist within generated Python as a callable function, +i.e., a Python ``def``. They differ in their scope and calling semantics. +Whereas ``<%def>`` provides a construct that is very much like a named Python +``def``, the ``<%block>`` is more layout oriented. + +Using Defs +========== + +The ``<%def>`` tag requires a ``name`` attribute, where the ``name`` references +a Python function signature: + +.. sourcecode:: mako + + <%def name="hello()"> + hello world + </%def> + +To invoke the ``<%def>``, it is normally called as an expression: + +.. sourcecode:: mako + + the def: ${hello()} + +If the ``<%def>`` is not nested inside of another ``<%def>``, +it's known as a **top level def** and can be accessed anywhere in +the template, including above where it was defined. + +All defs, top level or not, have access to the current +contextual namespace in exactly the same way their containing +template does. Suppose the template below is executed with the +variables ``username`` and ``accountdata`` inside the context: + +.. sourcecode:: mako + + Hello there ${username}, how are ya. Lets see what your account says: + + ${account()} + + <%def name="account()"> + Account for ${username}:<br/> + + % for row in accountdata: + Value: ${row}<br/> + % endfor + </%def> + +The ``username`` and ``accountdata`` variables are present +within the main template body as well as the body of the +``account()`` def. + +Since defs are just Python functions, you can define and pass +arguments to them as well: + +.. sourcecode:: mako + + ${account(accountname='john')} + + <%def name="account(accountname, type='regular')"> + account name: ${accountname}, type: ${type} + </%def> + +When you declare an argument signature for your def, they are +required to follow normal Python conventions (i.e., all +arguments are required except keyword arguments with a default +value). This is in contrast to using context-level variables, +which evaluate to ``UNDEFINED`` if you reference a name that +does not exist. + +Calling Defs from Other Files +----------------------------- + +Top level ``<%def>``\ s are **exported** by your template's +module, and can be called from the outside; including from other +templates, as well as normal Python code. Calling a ``<%def>`` +from another template is something like using an ``<%include>`` +-- except you are calling a specific function within the +template, not the whole template. + +The remote ``<%def>`` call is also a little bit like calling +functions from other modules in Python. There is an "import" +step to pull the names from another template into your own +template; then the function or functions are available. + +To import another template, use the ``<%namespace>`` tag: + +.. sourcecode:: mako + + <%namespace name="mystuff" file="mystuff.html"/> + +The above tag adds a local variable ``mystuff`` to the current +scope. + +Then, just call the defs off of ``mystuff``: + +.. sourcecode:: mako + + ${mystuff.somedef(x=5,y=7)} + +The ``<%namespace>`` tag also supports some of the other +semantics of Python's ``import`` statement, including pulling +names into the local variable space, or using ``*`` to represent +all names, using the ``import`` attribute: + +.. sourcecode:: mako + + <%namespace file="mystuff.html" import="foo, bar"/> + +This is just a quick intro to the concept of a **namespace**, +which is a central Mako concept that has its own chapter in +these docs. For more detail and examples, see +:ref:`namespaces_toplevel`. + +Calling Defs Programmatically +----------------------------- + +You can call defs programmatically from any :class:`.Template` object +using the :meth:`~.Template.get_def()` method, which returns a :class:`.DefTemplate` +object. This is a :class:`.Template` subclass which the parent +:class:`.Template` creates, and is usable like any other template: + +.. sourcecode:: python + + from mako.template import Template + + template = Template(""" + <%def name="hi(name)"> + hi ${name}! + </%def> + + <%def name="bye(name)"> + bye ${name}! + </%def> + """) + + print(template.get_def("hi").render(name="ed")) + print(template.get_def("bye").render(name="ed")) + +Defs within Defs +---------------- + +The def model follows regular Python rules for closures. +Declaring ``<%def>`` inside another ``<%def>`` declares it +within the parent's **enclosing scope**: + +.. sourcecode:: mako + + <%def name="mydef()"> + <%def name="subdef()"> + a sub def + </%def> + + i'm the def, and the subcomponent is ${subdef()} + </%def> + +Just like Python, names that exist outside the inner ``<%def>`` +exist inside it as well: + +.. sourcecode:: mako + + <% + x = 12 + %> + <%def name="outer()"> + <% + y = 15 + %> + <%def name="inner()"> + inner, x is ${x}, y is ${y} + </%def> + + outer, x is ${x}, y is ${y} + </%def> + +Assigning to a name inside of a def declares that name as local +to the scope of that def (again, like Python itself). This means +the following code will raise an error: + +.. sourcecode:: mako + + <% + x = 10 + %> + <%def name="somedef()"> + ## error ! + somedef, x is ${x} + <% + x = 27 + %> + </%def> + +...because the assignment to ``x`` declares ``x`` as local to the +scope of ``somedef``, rendering the "outer" version unreachable +in the expression that tries to render it. + +.. _defs_with_content: + +Calling a Def with Embedded Content and/or Other Defs +----------------------------------------------------- + +A flip-side to def within def is a def call with content. This +is where you call a def, and at the same time declare a block of +content (or multiple blocks) that can be used by the def being +called. The main point of such a call is to create custom, +nestable tags, just like any other template language's +custom-tag creation system -- where the external tag controls the +execution of the nested tags and can communicate state to them. +Only with Mako, you don't have to use any external Python +modules, you can define arbitrarily nestable tags right in your +templates. + +To achieve this, the target def is invoked using the form +``<%namespacename:defname>`` instead of the normal ``${}`` +syntax. This syntax, introduced in Mako 0.2.3, is functionally +equivalent to another tag known as ``%call``, which takes the form +``<%call expr='namespacename.defname(args)'>``. While ``%call`` +is available in all versions of Mako, the newer style is +probably more familiar looking. The ``namespace`` portion of the +call is the name of the **namespace** in which the def is +defined -- in the most simple cases, this can be ``local`` or +``self`` to reference the current template's namespace (the +difference between ``local`` and ``self`` is one of inheritance +-- see :ref:`namespaces_builtin` for details). + +When the target def is invoked, a variable ``caller`` is placed +in its context which contains another namespace containing the +body and other defs defined by the caller. The body itself is +referenced by the method ``body()``. Below, we build a ``%def`` +that operates upon ``caller.body()`` to invoke the body of the +custom tag: + +.. sourcecode:: mako + + <%def name="buildtable()"> + <table> + <tr><td> + ${caller.body()} + </td></tr> + </table> + </%def> + + <%self:buildtable> + I am the table body. + </%self:buildtable> + +This produces the output (whitespace formatted): + +.. sourcecode:: html + + <table> + <tr><td> + I am the table body. + </td></tr> + </table> + +Using the older ``%call`` syntax looks like: + +.. sourcecode:: mako + + <%def name="buildtable()"> + <table> + <tr><td> + ${caller.body()} + </td></tr> + </table> + </%def> + + <%call expr="buildtable()"> + I am the table body. + </%call> + +The ``body()`` can be executed multiple times or not at all. +This means you can use def-call-with-content to build iterators, +conditionals, etc: + +.. sourcecode:: mako + + <%def name="lister(count)"> + % for x in range(count): + ${caller.body()} + % endfor + </%def> + + <%self:lister count="${3}"> + hi + </%self:lister> + +Produces: + +.. sourcecode:: html + + hi + hi + hi + +Notice above we pass ``3`` as a Python expression, so that it +remains as an integer. + +A custom "conditional" tag: + +.. sourcecode:: mako + + <%def name="conditional(expression)"> + % if expression: + ${caller.body()} + % endif + </%def> + + <%self:conditional expression="${4==4}"> + i'm the result + </%self:conditional> + +Produces: + +.. sourcecode:: html + + i'm the result + +But that's not all. The ``body()`` function also can handle +arguments, which will augment the local namespace of the body +callable. The caller must define the arguments which it expects +to receive from its target def using the ``args`` attribute, +which is a comma-separated list of argument names. Below, our +``<%def>`` calls the ``body()`` of its caller, passing in an +element of data from its argument: + +.. sourcecode:: mako + + <%def name="layoutdata(somedata)"> + <table> + % for item in somedata: + <tr> + % for col in item: + <td>${caller.body(col=col)}</td> + % endfor + </tr> + % endfor + </table> + </%def> + + <%self:layoutdata somedata="${[[1,2,3],[4,5,6],[7,8,9]]}" args="col">\ + Body data: ${col}\ + </%self:layoutdata> + +Produces: + +.. sourcecode:: html + + <table> + <tr> + <td>Body data: 1</td> + <td>Body data: 2</td> + <td>Body data: 3</td> + </tr> + <tr> + <td>Body data: 4</td> + <td>Body data: 5</td> + <td>Body data: 6</td> + </tr> + <tr> + <td>Body data: 7</td> + <td>Body data: 8</td> + <td>Body data: 9</td> + </tr> + </table> + +You don't have to stick to calling just the ``body()`` function. +The caller can define any number of callables, allowing the +``<%call>`` tag to produce whole layouts: + +.. sourcecode:: mako + + <%def name="layout()"> + ## a layout def + <div class="mainlayout"> + <div class="header"> + ${caller.header()} + </div> + + <div class="sidebar"> + ${caller.sidebar()} + </div> + + <div class="content"> + ${caller.body()} + </div> + </div> + </%def> + + ## calls the layout def + <%self:layout> + <%def name="header()"> + I am the header + </%def> + <%def name="sidebar()"> + <ul> + <li>sidebar 1</li> + <li>sidebar 2</li> + </ul> + </%def> + + this is the body + </%self:layout> + +The above layout would produce: + +.. sourcecode:: html + + <div class="mainlayout"> + <div class="header"> + I am the header + </div> + + <div class="sidebar"> + <ul> + <li>sidebar 1</li> + <li>sidebar 2</li> + </ul> + </div> + + <div class="content"> + this is the body + </div> + </div> + +The number of things you can do with ``<%call>`` and/or the +``<%namespacename:defname>`` calling syntax is enormous. You can +create form widget libraries, such as an enclosing ``<FORM>`` +tag and nested HTML input elements, or portable wrapping schemes +using ``<div>`` or other elements. You can create tags that +interpret rows of data, such as from a database, providing the +individual columns of each row to a ``body()`` callable which +lays out the row any way it wants. Basically anything you'd do +with a "custom tag" or tag library in some other system, Mako +provides via ``<%def>`` tags and plain Python callables which are +invoked via ``<%namespacename:defname>`` or ``<%call>``. + +.. _blocks: + +Using Blocks +============ + +The ``<%block>`` tag introduces some new twists on the +``<%def>`` tag which make it more closely tailored towards layout. + +.. versionadded:: 0.4.1 + +An example of a block: + +.. sourcecode:: mako + + <html> + <body> + <%block> + this is a block. + </%block> + </body> + </html> + +In the above example, we define a simple block. The block renders its content in the place +that it's defined. Since the block is called for us, it doesn't need a name and the above +is referred to as an **anonymous block**. So the output of the above template will be: + +.. sourcecode:: html + + <html> + <body> + this is a block. + </body> + </html> + +So in fact the above block has absolutely no effect. Its usefulness comes when we start +using modifiers. Such as, we can apply a filter to our block: + +.. sourcecode:: mako + + <html> + <body> + <%block filter="h"> + <html>this is some escaped html.</html> + </%block> + </body> + </html> + +or perhaps a caching directive: + +.. sourcecode:: mako + + <html> + <body> + <%block cached="True" cache_timeout="60"> + This content will be cached for 60 seconds. + </%block> + </body> + </html> + +Blocks also work in iterations, conditionals, just like defs: + +.. sourcecode:: mako + + % if some_condition: + <%block>condition is met</%block> + % endif + +While the block renders at the point it is defined in the template, +the underlying function is present in the generated Python code only +once, so there's no issue with placing a block inside of a loop or +similar. Anonymous blocks are defined as closures in the local +rendering body, so have access to local variable scope: + +.. sourcecode:: mako + + % for i in range(1, 4): + <%block>i is ${i}</%block> + % endfor + +Using Named Blocks +------------------ + +Possibly the more important area where blocks are useful is when we +do actually give them names. Named blocks are tailored to behave +somewhat closely to Jinja2's block tag, in that they define an area +of a layout which can be overridden by an inheriting template. In +sharp contrast to the ``<%def>`` tag, the name given to a block is +global for the entire template regardless of how deeply it's nested: + +.. sourcecode:: mako + + <html> + <%block name="header"> + <head> + <title> + <%block name="title">Title</%block> + </title> + </head> + </%block> + <body> + ${next.body()} + </body> + </html> + +The above example has two named blocks "``header``" and "``title``", both of which can be referred to +by an inheriting template. A detailed walkthrough of this usage can be found at :ref:`inheritance_toplevel`. + +Note above that named blocks don't have any argument declaration the way defs do. They still implement themselves +as Python functions, however, so they can be invoked additional times beyond their initial definition: + +.. sourcecode:: mako + + <div name="page"> + <%block name="pagecontrol"> + <a href="">previous page</a> | + <a href="">next page</a> + </%block> + + <table> + ## some content + </table> + + ${pagecontrol()} + </div> + +The content referenced by ``pagecontrol`` above will be rendered both above and below the ``<table>`` tags. + +To keep things sane, named blocks have restrictions that defs do not: + +* The ``<%block>`` declaration cannot have any argument signature. +* The name of a ``<%block>`` can only be defined once in a template -- an error is raised if two blocks of the same + name occur anywhere in a single template, regardless of nesting. A similar error is raised if a top level def + shares the same name as that of a block. +* A named ``<%block>`` cannot be defined within a ``<%def>``, or inside the body of a "call", i.e. + ``<%call>`` or ``<%namespacename:defname>`` tag. Anonymous blocks can, however. + +Using Page Arguments in Named Blocks +------------------------------------ + +A named block is very much like a top level def. It has a similar +restriction to these types of defs in that arguments passed to the +template via the ``<%page>`` tag aren't automatically available. +Using arguments with the ``<%page>`` tag is described in the section +:ref:`namespaces_body`, and refers to scenarios such as when the +``body()`` method of a template is called from an inherited template passing +arguments, or the template is invoked from an ``<%include>`` tag +with arguments. To allow a named block to share the same arguments +passed to the page, the ``args`` attribute can be used: + +.. sourcecode:: mako + + <%page args="post"/> + + <a name="${post.title}" /> + + <span class="post_prose"> + <%block name="post_prose" args="post"> + ${post.content} + </%block> + </span> + +Where above, if the template is called via a directive like +``<%include file="post.mako" args="post=post" />``, the ``post`` +variable is available both in the main body as well as the +``post_prose`` block. + +Similarly, the ``**pageargs`` variable is present, in named blocks only, +for those arguments not explicit in the ``<%page>`` tag: + +.. sourcecode:: mako + + <%block name="post_prose"> + ${pageargs['post'].content} + </%block> + +The ``args`` attribute is only allowed with named blocks. With +anonymous blocks, the Python function is always rendered in the same +scope as the call itself, so anything available directly outside the +anonymous block is available inside as well. diff --git a/doc/build/filtering.rst b/doc/build/filtering.rst new file mode 100644 index 0000000..b4c3323 --- /dev/null +++ b/doc/build/filtering.rst @@ -0,0 +1,368 @@ +.. _filtering_toplevel: + +======================= +Filtering and Buffering +======================= + +.. _expression_filtering: + +Expression Filtering +==================== + +As described in the chapter :ref:`syntax_toplevel`, the "``|``" operator can be +applied to a "``${}``" expression to apply escape filters to the +output: + +.. sourcecode:: mako + + ${"this is some text" | u} + +The above expression applies URL escaping to the expression, and +produces ``this+is+some+text``. + +The built-in escape flags are: + +* ``u`` : URL escaping, provided by + ``urllib.quote_plus(string.encode('utf-8'))`` +* ``h`` : HTML escaping, provided by + ``markupsafe.escape(string)`` + + .. versionadded:: 0.3.4 + Prior versions use ``cgi.escape(string, True)``. + +* ``x`` : XML escaping +* ``trim`` : whitespace trimming, provided by ``string.strip()`` +* ``entity`` : produces HTML entity references for applicable + strings, derived from ``htmlentitydefs`` +* ``str`` : produces a Python unicode + string (this function is applied by default) +* ``unicode`` : aliased to ``str`` above + + .. versionchanged:: 1.2.0 + Prior versions applied the ``unicode`` built-in when running in Python 2; + in 1.2.0 Mako applies the Python 3 ``str`` built-in. + +* ``decode.<some encoding>`` : decode input into a Python + unicode with the specified encoding +* ``n`` : disable all default filtering; only filters specified + in the local expression tag will be applied. + +To apply more than one filter, separate them by a comma: + +.. sourcecode:: mako + + ${" <tag>some value</tag> " | h,trim} + +The above produces ``<tag>some value</tag>``, with +no leading or trailing whitespace. The HTML escaping function is +applied first, the "trim" function second. + +Naturally, you can make your own filters too. A filter is just a +Python function that accepts a single string argument, and +returns the filtered result. The expressions after the ``|`` +operator draw upon the local namespace of the template in which +they appear, meaning you can define escaping functions locally: + +.. sourcecode:: mako + + <%! + def myescape(text): + return "<TAG>" + text + "</TAG>" + %> + + Here's some tagged text: ${"text" | myescape} + +Or from any Python module: + +.. sourcecode:: mako + + <%! + import myfilters + %> + + Here's some tagged text: ${"text" | myfilters.tagfilter} + +A page can apply a default set of filters to all expression tags +using the ``expression_filter`` argument to the ``%page`` tag: + +.. sourcecode:: mako + + <%page expression_filter="h"/> + + Escaped text: ${"<html>some html</html>"} + +Result: + +.. sourcecode:: html + + Escaped text: <html>some html</html> + +.. _filtering_default_filters: + +The ``default_filters`` Argument +-------------------------------- + +In addition to the ``expression_filter`` argument, the +``default_filters`` argument to both :class:`.Template` and +:class:`.TemplateLookup` can specify filtering for all expression tags +at the programmatic level. This array-based argument, when given +its default argument of ``None``, will be internally set to +``["str"]``: + +.. sourcecode:: python + + t = TemplateLookup(directories=['/tmp'], default_filters=['str']) + +To replace the usual ``str`` function with a +specific encoding, the ``decode`` filter can be substituted: + +.. sourcecode:: python + + t = TemplateLookup(directories=['/tmp'], default_filters=['decode.utf8']) + +To disable ``default_filters`` entirely, set it to an empty +list: + +.. sourcecode:: python + + t = TemplateLookup(directories=['/tmp'], default_filters=[]) + +Any string name can be added to ``default_filters`` where it +will be added to all expressions as a filter. The filters are +applied from left to right, meaning the leftmost filter is +applied first. + +.. sourcecode:: python + + t = Template(templatetext, default_filters=['str', 'myfilter']) + +To ease the usage of ``default_filters`` with custom filters, +you can also add imports (or other code) to all templates using +the ``imports`` argument: + +.. sourcecode:: python + + t = TemplateLookup(directories=['/tmp'], + default_filters=['str', 'myfilter'], + imports=['from mypackage import myfilter']) + +The above will generate templates something like this: + +.. sourcecode:: python + + # .... + from mypackage import myfilter + + def render_body(context): + context.write(myfilter(str("some text"))) + +.. _expression_filtering_nfilter: + +Turning off Filtering with the ``n`` Filter +------------------------------------------- + +In all cases the special ``n`` filter, used locally within an +expression, will **disable** all filters declared in the +``<%page>`` tag as well as in ``default_filters``. Such as: + +.. sourcecode:: mako + + ${'myexpression' | n} + +will render ``myexpression`` with no filtering of any kind, and: + +.. sourcecode:: mako + + ${'myexpression' | n,trim} + +will render ``myexpression`` using the ``trim`` filter only. + +Including the ``n`` filter in a ``<%page>`` tag will only disable +``default_filters``. In effect this makes the filters from the tag replace +default filters instead of adding to them. For example: + +.. sourcecode:: mako + + <%page expression_filter="n, json.dumps"/> + data = {a: ${123}, b: ${"123"}}; + +will suppress turning the values into strings using the default filter, so that +``json.dumps`` (which requires ``imports=["import json"]`` or something +equivalent) can take the value type into account, formatting numbers as numeric +literals and strings as string literals. + +.. versionadded:: 1.0.14 The ``n`` filter can now be used in the ``<%page>`` tag. + +Filtering Defs and Blocks +========================= + +The ``%def`` and ``%block`` tags have an argument called ``filter`` which will apply the +given list of filter functions to the output of the ``%def``: + +.. sourcecode:: mako + + <%def name="foo()" filter="h, trim"> + <b>this is bold</b> + </%def> + +When the ``filter`` attribute is applied to a def as above, the def +is automatically **buffered** as well. This is described next. + +Buffering +========= + +One of Mako's central design goals is speed. To this end, all of +the textual content within a template and its various callables +is by default piped directly to the single buffer that is stored +within the :class:`.Context` object. While this normally is easy to +miss, it has certain side effects. The main one is that when you +call a def using the normal expression syntax, i.e. +``${somedef()}``, it may appear that the return value of the +function is the content it produced, which is then delivered to +your template just like any other expression substitution, +except that normally, this is not the case; the return value of +``${somedef()}`` is simply the empty string ``''``. By the time +you receive this empty string, the output of ``somedef()`` has +been sent to the underlying buffer. + +You may not want this effect, if for example you are doing +something like this: + +.. sourcecode:: mako + + ${" results " + somedef() + " more results "} + +If the ``somedef()`` function produced the content "``somedef's +results``", the above template would produce this output: + +.. sourcecode:: html + + somedef's results results more results + +This is because ``somedef()`` fully executes before the +expression returns the results of its concatenation; the +concatenation in turn receives just the empty string as its +middle expression. + +Mako provides two ways to work around this. One is by applying +buffering to the ``%def`` itself: + +.. sourcecode:: mako + + <%def name="somedef()" buffered="True"> + somedef's results + </%def> + +The above definition will generate code similar to this: + +.. sourcecode:: python + + def somedef(): + context.push_buffer() + try: + context.write("somedef's results") + finally: + buf = context.pop_buffer() + return buf.getvalue() + +So that the content of ``somedef()`` is sent to a second buffer, +which is then popped off the stack and its value returned. The +speed hit inherent in buffering the output of a def is also +apparent. + +Note that the ``filter`` argument on ``%def`` also causes the def to +be buffered. This is so that the final content of the ``%def`` can +be delivered to the escaping function in one batch, which +reduces method calls and also produces more deterministic +behavior for the filtering function itself, which can possibly +be useful for a filtering function that wishes to apply a +transformation to the text as a whole. + +The other way to buffer the output of a def or any Mako callable +is by using the built-in ``capture`` function. This function +performs an operation similar to the above buffering operation +except it is specified by the caller. + +.. sourcecode:: mako + + ${" results " + capture(somedef) + " more results "} + +Note that the first argument to the ``capture`` function is +**the function itself**, not the result of calling it. This is +because the ``capture`` function takes over the job of actually +calling the target function, after setting up a buffered +environment. To send arguments to the function, just send them +to ``capture`` instead: + +.. sourcecode:: mako + + ${capture(somedef, 17, 'hi', use_paging=True)} + +The above call is equivalent to the unbuffered call: + +.. sourcecode:: mako + + ${somedef(17, 'hi', use_paging=True)} + +Decorating +========== + +.. versionadded:: 0.2.5 + +Somewhat like a filter for a ``%def`` but more flexible, the ``decorator`` +argument to ``%def`` allows the creation of a function that will +work in a similar manner to a Python decorator. The function can +control whether or not the function executes. The original +intent of this function is to allow the creation of custom cache +logic, but there may be other uses as well. + +``decorator`` is intended to be used with a regular Python +function, such as one defined in a library module. Here we'll +illustrate the python function defined in the template for +simplicities' sake: + +.. sourcecode:: mako + + <%! + def bar(fn): + def decorate(context, *args, **kw): + context.write("BAR") + fn(*args, **kw) + context.write("BAR") + return '' + return decorate + %> + + <%def name="foo()" decorator="bar"> + this is foo + </%def> + + ${foo()} + +The above template will return, with more whitespace than this, +``"BAR this is foo BAR"``. The function is the render callable +itself (or possibly a wrapper around it), and by default will +write to the context. To capture its output, use the :func:`.capture` +callable in the ``mako.runtime`` module (available in templates +as just ``runtime``): + +.. sourcecode:: mako + + <%! + def bar(fn): + def decorate(context, *args, **kw): + return "BAR" + runtime.capture(context, fn, *args, **kw) + "BAR" + return decorate + %> + + <%def name="foo()" decorator="bar"> + this is foo + </%def> + + ${foo()} + +The decorator can be used with top-level defs as well as nested +defs, and blocks too. Note that when calling a top-level def from the +:class:`.Template` API, i.e. ``template.get_def('somedef').render()``, +the decorator has to write the output to the ``context``, i.e. +as in the first example. The return value gets discarded. diff --git a/doc/build/index.rst b/doc/build/index.rst new file mode 100644 index 0000000..3104ca9 --- /dev/null +++ b/doc/build/index.rst @@ -0,0 +1,23 @@ +Table of Contents +================= + +.. toctree:: + :maxdepth: 2 + + usage + syntax + defs + runtime + namespaces + inheritance + filtering + unicode + caching + changelog + +Indices and Tables +------------------ + +* :ref:`genindex` +* :ref:`search` + diff --git a/doc/build/inheritance.rst b/doc/build/inheritance.rst new file mode 100644 index 0000000..842b8fc --- /dev/null +++ b/doc/build/inheritance.rst @@ -0,0 +1,647 @@ +.. _inheritance_toplevel: + +=========== +Inheritance +=========== + +.. note:: Most of the inheritance examples here take advantage of a feature that's + new in Mako as of version 0.4.1 called the "block". This tag is very similar to + the "def" tag but is more streamlined for usage with inheritance. Note that + all of the examples here which use blocks can also use defs instead. Contrasting + usages will be illustrated. + +Using template inheritance, two or more templates can organize +themselves into an **inheritance chain**, where content and +functions from all involved templates can be intermixed. The +general paradigm of template inheritance is this: if a template +``A`` inherits from template ``B``, then template ``A`` agrees +to send the executional control to template ``B`` at runtime +(``A`` is called the **inheriting** template). Template ``B``, +the **inherited** template, then makes decisions as to what +resources from ``A`` shall be executed. + +In practice, it looks like this. Here's a hypothetical inheriting +template, ``index.html``: + +.. sourcecode:: mako + + ## index.html + <%inherit file="base.html"/> + + <%block name="header"> + this is some header content + </%block> + + this is the body content. + +And ``base.html``, the inherited template: + +.. sourcecode:: mako + + ## base.html + <html> + <body> + <div class="header"> + <%block name="header"/> + </div> + + ${self.body()} + + <div class="footer"> + <%block name="footer"> + this is the footer + </%block> + </div> + </body> + </html> + +Here is a breakdown of the execution: + +#. When ``index.html`` is rendered, control immediately passes to + ``base.html``. +#. ``base.html`` then renders the top part of an HTML document, + then invokes the ``<%block name="header">`` block. It invokes the + underlying ``header()`` function off of a built-in namespace + called ``self`` (this namespace was first introduced in the + :doc:`Namespaces chapter <namespaces>` in :ref:`namespace_self`). Since + ``index.html`` is the topmost template and also defines a block + called ``header``, it's this ``header`` block that ultimately gets + executed -- instead of the one that's present in ``base.html``. +#. Control comes back to ``base.html``. Some more HTML is + rendered. +#. ``base.html`` executes ``self.body()``. The ``body()`` + function on all template-based namespaces refers to the main + body of the template, therefore the main body of + ``index.html`` is rendered. +#. When ``<%block name="header">`` is encountered in ``index.html`` + during the ``self.body()`` call, a conditional is checked -- does the + current inherited template, i.e. ``base.html``, also define this block? If yes, + the ``<%block>`` is **not** executed here -- the inheritance + mechanism knows that the parent template is responsible for rendering + this block (and in fact it already has). In other words a block + only renders in its *basemost scope*. +#. Control comes back to ``base.html``. More HTML is rendered, + then the ``<%block name="footer">`` expression is invoked. +#. The ``footer`` block is only defined in ``base.html``, so being + the topmost definition of ``footer``, it's the one that + executes. If ``index.html`` also specified ``footer``, then + its version would **override** that of the base. +#. ``base.html`` finishes up rendering its HTML and the template + is complete, producing: + + .. sourcecode:: html + + <html> + <body> + <div class="header"> + this is some header content + </div> + + this is the body content. + + <div class="footer"> + this is the footer + </div> + </body> + </html> + +...and that is template inheritance in a nutshell. The main idea +is that the methods that you call upon ``self`` always +correspond to the topmost definition of that method. Very much +the way ``self`` works in a Python class, even though Mako is +not actually using Python class inheritance to implement this +functionality. (Mako doesn't take the "inheritance" metaphor too +seriously; while useful to setup some commonly recognized +semantics, a textual template is not very much like an +object-oriented class construct in practice). + +Nesting Blocks +============== + +The named blocks defined in an inherited template can also be nested within +other blocks. The name given to each block is globally accessible via any inheriting +template. We can add a new block ``title`` to our ``header`` block: + +.. sourcecode:: mako + + ## base.html + <html> + <body> + <div class="header"> + <%block name="header"> + <h2> + <%block name="title"/> + </h2> + </%block> + </div> + + ${self.body()} + + <div class="footer"> + <%block name="footer"> + this is the footer + </%block> + </div> + </body> + </html> + +The inheriting template can name either or both of ``header`` and ``title``, separately +or nested themselves: + +.. sourcecode:: mako + + ## index.html + <%inherit file="base.html"/> + + <%block name="header"> + this is some header content + ${parent.header()} + </%block> + + <%block name="title"> + this is the title + </%block> + + this is the body content. + +Note when we overrode ``header``, we added an extra call ``${parent.header()}`` in order to invoke +the parent's ``header`` block in addition to our own. That's described in more detail below, +in :ref:`parent_namespace`. + +Rendering a Named Block Multiple Times +====================================== + +Recall from the section :ref:`blocks` that a named block is just like a ``<%def>``, +with some different usage rules. We can call one of our named sections distinctly, for example +a section that is used more than once, such as the title of a page: + +.. sourcecode:: mako + + <html> + <head> + <title>${self.title()}</title> + </head> + <body> + <%block name="header"> + <h2><%block name="title"/></h2> + </%block> + ${self.body()} + </body> + </html> + +Where above an inheriting template can define ``<%block name="title">`` just once, and it will be +used in the base template both in the ``<title>`` section as well as the ``<h2>``. + + + +But what about Defs? +==================== + +The previous example used the ``<%block>`` tag to produce areas of content +to be overridden. Before Mako 0.4.1, there wasn't any such tag -- instead +there was only the ``<%def>`` tag. As it turns out, named blocks and defs are +largely interchangeable. The def simply doesn't call itself automatically, +and has more open-ended naming and scoping rules that are more flexible and similar +to Python itself, but less suited towards layout. The first example from +this chapter using defs would look like: + +.. sourcecode:: mako + + ## index.html + <%inherit file="base.html"/> + + <%def name="header()"> + this is some header content + </%def> + + this is the body content. + +And ``base.html``, the inherited template: + +.. sourcecode:: mako + + ## base.html + <html> + <body> + <div class="header"> + ${self.header()} + </div> + + ${self.body()} + + <div class="footer"> + ${self.footer()} + </div> + </body> + </html> + + <%def name="header()"/> + <%def name="footer()"> + this is the footer + </%def> + +Above, we illustrate that defs differ from blocks in that their definition +and invocation are defined in two separate places, instead of at once. You can *almost* do exactly what a +block does if you put the two together: + +.. sourcecode:: mako + + <div class="header"> + <%def name="header()"></%def>${self.header()} + </div> + +The ``<%block>`` is obviously more streamlined than the ``<%def>`` for this kind +of usage. In addition, +the above "inline" approach with ``<%def>`` does not work with nesting: + +.. sourcecode:: mako + + <head> + <%def name="header()"> + <title> + ## this won't work ! + <%def name="title()">default title</%def>${self.title()} + </title> + </%def>${self.header()} + </head> + +Where above, the ``title()`` def, because it's a def within a def, is not part of the +template's exported namespace and will not be part of ``self``. If the inherited template +did define its own ``title`` def at the top level, it would be called, but the "default title" +above is not present at all on ``self`` no matter what. For this to work as expected +you'd instead need to say: + +.. sourcecode:: mako + + <head> + <%def name="header()"> + <title> + ${self.title()} + </title> + </%def>${self.header()} + + <%def name="title()"/> + </head> + +That is, ``title`` is defined outside of any other defs so that it is in the ``self`` namespace. +It works, but the definition needs to be potentially far away from the point of render. + +A named block is always placed in the ``self`` namespace, regardless of nesting, +so this restriction is lifted: + +.. sourcecode:: mako + + ## base.html + <head> + <%block name="header"> + <title> + <%block name="title"/> + </title> + </%block> + </head> + +The above template defines ``title`` inside of ``header``, and an inheriting template can define +one or both in **any** configuration, nested inside each other or not, in order for them to be used: + +.. sourcecode:: mako + + ## index.html + <%inherit file="base.html"/> + <%block name="title"> + the title + </%block> + <%block name="header"> + the header + </%block> + +So while the ``<%block>`` tag lifts the restriction of nested blocks not being available externally, +in order to achieve this it *adds* the restriction that all block names in a single template need +to be globally unique within the template, and additionally that a ``<%block>`` can't be defined +inside of a ``<%def>``. It's a more restricted tag suited towards a more specific use case than ``<%def>``. + +Using the ``next`` Namespace to Produce Content Wrapping +======================================================== + +Sometimes you have an inheritance chain that spans more than two +templates. Or maybe you don't, but you'd like to build your +system such that extra inherited templates can be inserted in +the middle of a chain where they would be smoothly integrated. +If each template wants to define its layout just within its main +body, you can't just call ``self.body()`` to get at the +inheriting template's body, since that is only the topmost body. +To get at the body of the *next* template, you call upon the +namespace ``next``, which is the namespace of the template +**immediately following** the current template. + +Lets change the line in ``base.html`` which calls upon +``self.body()`` to instead call upon ``next.body()``: + +.. sourcecode:: mako + + ## base.html + <html> + <body> + <div class="header"> + <%block name="header"/> + </div> + + ${next.body()} + + <div class="footer"> + <%block name="footer"> + this is the footer + </%block> + </div> + </body> + </html> + + +Lets also add an intermediate template called ``layout.html``, +which inherits from ``base.html``: + +.. sourcecode:: mako + + ## layout.html + <%inherit file="base.html"/> + <ul> + <%block name="toolbar"> + <li>selection 1</li> + <li>selection 2</li> + <li>selection 3</li> + </%block> + </ul> + <div class="mainlayout"> + ${next.body()} + </div> + +And finally change ``index.html`` to inherit from +``layout.html`` instead: + +.. sourcecode:: mako + + ## index.html + <%inherit file="layout.html"/> + + ## .. rest of template + +In this setup, each call to ``next.body()`` will render the body +of the next template in the inheritance chain (which can be +written as ``base.html -> layout.html -> index.html``). Control +is still first passed to the bottommost template ``base.html``, +and ``self`` still references the topmost definition of any +particular def. + +The output we get would be: + +.. sourcecode:: html + + <html> + <body> + <div class="header"> + this is some header content + </div> + + <ul> + <li>selection 1</li> + <li>selection 2</li> + <li>selection 3</li> + </ul> + + <div class="mainlayout"> + this is the body content. + </div> + + <div class="footer"> + this is the footer + </div> + </body> + </html> + +So above, we have the ``<html>``, ``<body>`` and +``header``/``footer`` layout of ``base.html``, we have the +``<ul>`` and ``mainlayout`` section of ``layout.html``, and the +main body of ``index.html`` as well as its overridden ``header`` +def. The ``layout.html`` template is inserted into the middle of +the chain without ``base.html`` having to change anything. +Without the ``next`` namespace, only the main body of +``index.html`` could be used; there would be no way to call +``layout.html``'s body content. + +.. _parent_namespace: + +Using the ``parent`` Namespace to Augment Defs +============================================== + +Lets now look at the other inheritance-specific namespace, the +opposite of ``next`` called ``parent``. ``parent`` is the +namespace of the template **immediately preceding** the current +template. What's useful about this namespace is that +defs or blocks can call upon their overridden versions. +This is not as hard as it sounds and +is very much like using the ``super`` keyword in Python. Lets +modify ``index.html`` to augment the list of selections provided +by the ``toolbar`` function in ``layout.html``: + +.. sourcecode:: mako + + ## index.html + <%inherit file="layout.html"/> + + <%block name="header"> + this is some header content + </%block> + + <%block name="toolbar"> + ## call the parent's toolbar first + ${parent.toolbar()} + <li>selection 4</li> + <li>selection 5</li> + </%block> + + this is the body content. + +Above, we implemented a ``toolbar()`` function, which is meant +to override the definition of ``toolbar`` within the inherited +template ``layout.html``. However, since we want the content +from that of ``layout.html`` as well, we call it via the +``parent`` namespace whenever we want it's content, in this case +before we add our own selections. So the output for the whole +thing is now: + +.. sourcecode:: html + + <html> + <body> + <div class="header"> + this is some header content + </div> + + <ul> + <li>selection 1</li> + <li>selection 2</li> + <li>selection 3</li> + <li>selection 4</li> + <li>selection 5</li> + </ul> + + <div class="mainlayout"> + this is the body content. + </div> + + <div class="footer"> + this is the footer + </div> + </body> + </html> + +and you're now a template inheritance ninja! + +Using ``<%include>`` with Template Inheritance +============================================== + +A common source of confusion is the behavior of the ``<%include>`` tag, +often in conjunction with its interaction within template inheritance. +Key to understanding the ``<%include>`` tag is that it is a *dynamic*, e.g. +runtime, include, and not a static include. The ``<%include>`` is only processed +as the template renders, and not at inheritance setup time. When encountered, +the referenced template is run fully as an entirely separate template with no +linkage to any current inheritance structure. + +If the tag were on the other hand a *static* include, this would allow source +within the included template to interact within the same inheritance context +as the calling template, but currently Mako has no static include facility. + +In practice, this means that ``<%block>`` elements defined in an ``<%include>`` +file will not interact with corresponding ``<%block>`` elements in the calling +template. + +A common mistake is along these lines: + +.. sourcecode:: mako + + ## partials.mako + <%block name="header"> + Global Header + </%block> + + ## parent.mako + <%include file="partials.mako" /> + + ## child.mako + <%inherit file="parent.mako" /> + <%block name="header"> + Custom Header + </%block> + +Above, one might expect that the ``"header"`` block declared in ``child.mako`` +might be invoked, as a result of it overriding the same block present in +``parent.mako`` via the include for ``partials.mako``. But this is not the case. +Instead, ``parent.mako`` will invoke ``partials.mako``, which then invokes +``"header"`` in ``partials.mako``, and then is finished rendering. Nothing +from ``child.mako`` will render; there is no interaction between the ``"header"`` +block in ``child.mako`` and the ``"header"`` block in ``partials.mako``. + +Instead, ``parent.mako`` must explicitly state the inheritance structure. +In order to call upon specific elements of ``partials.mako``, we will call upon +it as a namespace: + +.. sourcecode:: mako + + ## partials.mako + <%block name="header"> + Global Header + </%block> + + ## parent.mako + <%namespace name="partials" file="partials.mako"/> + <%block name="header"> + ${partials.header()} + </%block> + + ## child.mako + <%inherit file="parent.mako" /> + <%block name="header"> + Custom Header + </%block> + +Where above, ``parent.mako`` states the inheritance structure that ``child.mako`` +is to participate within. ``partials.mako`` only defines defs/blocks that can be +used on a per-name basis. + +Another scenario is below, which results in both ``"SectionA"`` blocks being rendered for the ``child.mako`` document: + +.. sourcecode:: mako + + ## base.mako + ${self.body()} + <%block name="SectionA"> + base.mako + </%block> + + ## parent.mako + <%inherit file="base.mako" /> + <%include file="child.mako" /> + + ## child.mako + <%block name="SectionA"> + child.mako + </%block> + +The resolution is similar; instead of using ``<%include>``, we call upon the blocks +of ``child.mako`` using a namespace: + +.. sourcecode:: mako + + ## parent.mako + <%inherit file="base.mako" /> + <%namespace name="child" file="child.mako" /> + + <%block name="SectionA"> + ${child.SectionA()} + </%block> + + +.. _inheritance_attr: + +Inheritable Attributes +====================== + +The :attr:`attr <.Namespace.attr>` accessor of the :class:`.Namespace` object +allows access to module level variables declared in a template. By accessing +``self.attr``, you can access regular attributes from the +inheritance chain as declared in ``<%! %>`` sections. Such as: + +.. sourcecode:: mako + + <%! + class_ = "grey" + %> + + <div class="${self.attr.class_}"> + ${self.body()} + </div> + +If an inheriting template overrides ``class_`` to be +``"white"``, as in: + +.. sourcecode:: mako + + <%! + class_ = "white" + %> + <%inherit file="parent.html"/> + + This is the body + +you'll get output like: + +.. sourcecode:: html + + <div class="white"> + This is the body + </div> + +.. seealso:: + + :ref:`namespace_attr_for_includes` - a more sophisticated example using + :attr:`.Namespace.attr`. diff --git a/doc/build/namespaces.rst b/doc/build/namespaces.rst new file mode 100644 index 0000000..afd323d --- /dev/null +++ b/doc/build/namespaces.rst @@ -0,0 +1,475 @@ +.. _namespaces_toplevel: + +========== +Namespaces +========== + +Namespaces are used to organize groups of defs into +categories, and also to "import" defs from other files. + +If the file ``components.html`` defines these two defs: + +.. sourcecode:: mako + + ## components.html + <%def name="comp1()"> + this is comp1 + </%def> + + <%def name="comp2(x)"> + this is comp2, x is ${x} + </%def> + +you can make another file, for example ``index.html``, that +pulls those two defs into a namespace called ``comp``: + +.. sourcecode:: mako + + ## index.html + <%namespace name="comp" file="components.html"/> + + Here's comp1: ${comp.comp1()} + Here's comp2: ${comp.comp2(x=5)} + +The ``comp`` variable above is an instance of +:class:`.Namespace`, a **proxy object** which delivers +method calls to the underlying template callable using the +current context. + +``<%namespace>`` also provides an ``import`` attribute which can +be used to pull the names into the local namespace, removing the +need to call it via the "``.``" operator. When ``import`` is used, the +``name`` attribute is optional. + +.. sourcecode:: mako + + <%namespace file="components.html" import="comp1, comp2"/> + + Heres comp1: ${comp1()} + Heres comp2: ${comp2(x=5)} + +``import`` also supports the "``*``" operator: + +.. sourcecode:: mako + + <%namespace file="components.html" import="*"/> + + Heres comp1: ${comp1()} + Heres comp2: ${comp2(x=5)} + +The names imported by the ``import`` attribute take precedence +over any names that exist within the current context. + +.. note:: In current versions of Mako, usage of ``import='*'`` is + known to decrease performance of the template. This will be + fixed in a future release. + +The ``file`` argument allows expressions -- if looking for +context variables, the ``context`` must be named explicitly: + +.. sourcecode:: mako + + <%namespace name="dyn" file="${context['namespace_name']}"/> + +Ways to Call Namespaces +======================= + +There are essentially four ways to call a function from a +namespace. + +The "expression" format, as described previously. Namespaces are +just Python objects with functions on them, and can be used in +expressions like any other function: + +.. sourcecode:: mako + + ${mynamespace.somefunction('some arg1', 'some arg2', arg3='some arg3', arg4='some arg4')} + +Synonymous with the "expression" format is the "custom tag" +format, when a "closed" tag is used. This format, introduced in +Mako 0.2.3, allows the usage of a "custom" Mako tag, with the +function arguments passed in using named attributes: + +.. sourcecode:: mako + + <%mynamespace:somefunction arg1="some arg1" arg2="some arg2" arg3="some arg3" arg4="some arg4"/> + +When using tags, the values of the arguments are taken as +literal strings by default. To embed Python expressions as +arguments, use the embedded expression format: + +.. sourcecode:: mako + + <%mynamespace:somefunction arg1="${someobject.format()}" arg2="${somedef(5, 12)}"/> + +The "custom tag" format is intended mainly for namespace +functions which recognize body content, which in Mako is known +as a "def with embedded content": + +.. sourcecode:: mako + + <%mynamespace:somefunction arg1="some argument" args="x, y"> + Some record: ${x}, ${y} + </%mynamespace:somefunction> + +The "classic" way to call defs with embedded content is the ``<%call>`` tag: + +.. sourcecode:: mako + + <%call expr="mynamespace.somefunction(arg1='some argument')" args="x, y"> + Some record: ${x}, ${y} + </%call> + +For information on how to construct defs that embed content from +the caller, see :ref:`defs_with_content`. + +.. _namespaces_python_modules: + +Namespaces from Regular Python Modules +====================================== + +Namespaces can also import regular Python functions from +modules. These callables need to take at least one argument, +``context``, an instance of :class:`.Context`. A module file +``some/module.py`` might contain the callable: + +.. sourcecode:: python + + def my_tag(context): + context.write("hello world") + return '' + +A template can use this module via: + +.. sourcecode:: mako + + <%namespace name="hw" module="some.module"/> + + ${hw.my_tag()} + +Note that the ``context`` argument is not needed in the call; +the :class:`.Namespace` tag creates a locally-scoped callable which +takes care of it. The ``return ''`` is so that the def does not +dump a ``None`` into the output stream -- the return value of any +def is rendered after the def completes, in addition to whatever +was passed to :meth:`.Context.write` within its body. + +If your def is to be called in an "embedded content" context, +that is as described in :ref:`defs_with_content`, you should use +the :func:`.supports_caller` decorator, which will ensure that Mako +will ensure the correct "caller" variable is available when your +def is called, supporting embedded content: + +.. sourcecode:: python + + from mako.runtime import supports_caller + + @supports_caller + def my_tag(context): + context.write("<div>") + context['caller'].body() + context.write("</div>") + return '' + +Capturing of output is available as well, using the +outside-of-templates version of the :func:`.capture` function, +which accepts the "context" as its first argument: + +.. sourcecode:: python + + from mako.runtime import supports_caller, capture + + @supports_caller + def my_tag(context): + return "<div>%s</div>" % \ + capture(context, context['caller'].body, x="foo", y="bar") + +Declaring Defs in Namespaces +============================ + +The ``<%namespace>`` tag supports the definition of ``<%def>``\ s +directly inside the tag. These defs become part of the namespace +like any other function, and will override the definitions +pulled in from a remote template or module: + +.. sourcecode:: mako + + ## define a namespace + <%namespace name="stuff"> + <%def name="comp1()"> + comp1 + </%def> + </%namespace> + + ## then call it + ${stuff.comp1()} + +.. _namespaces_body: + +The ``body()`` Method +===================== + +Every namespace that is generated from a template contains a +method called ``body()``. This method corresponds to the main +body of the template, and plays its most important roles when +using inheritance relationships as well as +def-calls-with-content. + +Since the ``body()`` method is available from a namespace just +like all the other defs defined in a template, what happens if +you send arguments to it? By default, the ``body()`` method +accepts no positional arguments, and for usefulness in +inheritance scenarios will by default dump all keyword arguments +into a dictionary called ``pageargs``. But if you actually want +to get at the keyword arguments, Mako recommends you define your +own argument signature explicitly. You do this via using the +``<%page>`` tag: + +.. sourcecode:: mako + + <%page args="x, y, someval=8, scope='foo', **kwargs"/> + +A template which defines the above signature requires that the +variables ``x`` and ``y`` are defined, defines default values +for ``someval`` and ``scope``, and sets up ``**kwargs`` to +receive all other keyword arguments. If ``**kwargs`` or similar +is not present, the argument ``**pageargs`` gets tacked on by +Mako. When the template is called as a top-level template (i.e. +via :meth:`~.Template.render`) or via the ``<%include>`` tag, the +values for these arguments will be pulled from the ``Context``. +In all other cases, i.e. via calling the ``body()`` method, the +arguments are taken as ordinary arguments from the method call. +So above, the body might be called as: + +.. sourcecode:: mako + + ${self.body(5, y=10, someval=15, delta=7)} + +The :class:`.Context` object also supplies a :attr:`~.Context.kwargs` +accessor, for cases when you'd like to pass along the top level context +arguments to a ``body()`` callable: + +.. sourcecode:: mako + + ${next.body(**context.kwargs)} + +The usefulness of calls like the above become more apparent when +one works with inheriting templates. For more information on +this, as well as the meanings of the names ``self`` and +``next``, see :ref:`inheritance_toplevel`. + +.. _namespaces_builtin: + +Built-in Namespaces +=================== + +The namespace is so great that Mako gives your template one (or +two) for free. The names of these namespaces are ``local`` and +``self``. Other built-in namespaces include ``parent`` and +``next``, which are optional and are described in +:ref:`inheritance_toplevel`. + +.. _namespace_local: + +``local`` +--------- + +The ``local`` namespace is basically the namespace for the +currently executing template. This means that all of the top +level defs defined in your template, as well as your template's +``body()`` function, are also available off of the ``local`` +namespace. + +The ``local`` namespace is also where properties like ``uri``, +``filename``, and ``module`` and the ``get_namespace`` method +can be particularly useful. + +.. _namespace_self: + +``self`` +-------- + +The ``self`` namespace, in the case of a template that does not +use inheritance, is synonymous with ``local``. If inheritance is +used, then ``self`` references the topmost template in the +inheritance chain, where it is most useful for providing the +ultimate form of various "method" calls which may have been +overridden at various points in an inheritance chain. See +:ref:`inheritance_toplevel`. + +Inheritable Namespaces +====================== + +The ``<%namespace>`` tag includes an optional attribute +``inheritable="True"``, which will cause the namespace to be +attached to the ``self`` namespace. Since ``self`` is globally +available throughout an inheritance chain (described in the next +section), all the templates in an inheritance chain can get at +the namespace imported in a super-template via ``self``. + +.. sourcecode:: mako + + ## base.html + <%namespace name="foo" file="foo.html" inheritable="True"/> + + ${next.body()} + + ## somefile.html + <%inherit file="base.html"/> + + ${self.foo.bar()} + +This allows a super-template to load a whole bunch of namespaces +that its inheriting templates can get to, without them having to +explicitly load those namespaces themselves. + +The ``import="*"`` part of the ``<%namespace>`` tag doesn't yet +interact with the ``inheritable`` flag, so currently you have to +use the explicit namespace name off of ``self``, followed by the +desired function name. But more on this in a future release. + +Namespace API Usage Example - Static Dependencies +================================================== + +The ``<%namespace>`` tag at runtime produces an instance of +:class:`.Namespace`. Programmatic access of :class:`.Namespace` can be used +to build various kinds of scaffolding in templates and between templates. + +A common request is the ability for a particular template to declare +"static includes" - meaning, the usage of a particular set of defs requires +that certain Javascript/CSS files are present. Using :class:`.Namespace` as the +object that holds together the various templates present, we can build a variety +of such schemes. In particular, the :class:`.Context` has a ``namespaces`` +attribute, which is a dictionary of all :class:`.Namespace` objects declared. +Iterating the values of this dictionary will provide a :class:`.Namespace` +object for each time the ``<%namespace>`` tag was used, anywhere within the +inheritance chain. + + +.. _namespace_attr_for_includes: + +Version One - Use :attr:`.Namespace.attr` +----------------------------------------- + +The :attr:`.Namespace.attr` attribute allows us to locate any variables declared +in the ``<%! %>`` of a template. + +.. sourcecode:: mako + + ## base.mako + ## base-most template, renders layout etc. + <html> + <head> + ## traverse through all namespaces present, + ## look for an attribute named 'includes' + % for ns in context.namespaces.values(): + % for incl in getattr(ns.attr, 'includes', []): + ${incl} + % endfor + % endfor + </head> + <body> + ${next.body()} + </body + </html> + + ## library.mako + ## library functions. + <%! + includes = [ + '<link rel="stylesheet" type="text/css" href="mystyle.css"/>', + '<script type="text/javascript" src="functions.js"></script>' + ] + %> + + <%def name="mytag()"> + <form> + ${caller.body()} + </form> + </%def> + + ## index.mako + ## calling template. + <%inherit file="base.mako"/> + <%namespace name="foo" file="library.mako"/> + + <%foo:mytag> + a form + </%foo:mytag> + + +Above, the file ``library.mako`` declares an attribute ``includes`` inside its global ``<%! %>`` section. +``index.mako`` includes this template using the ``<%namespace>`` tag. The base template ``base.mako``, which is the inherited parent of ``index.mako`` and is responsible for layout, then locates this attribute and iterates through its contents to produce the includes that are specific to ``library.mako``. + +Version Two - Use a specific named def +----------------------------------------- + +In this version, we put the includes into a ``<%def>`` that +follows a naming convention. + +.. sourcecode:: mako + + ## base.mako + ## base-most template, renders layout etc. + <html> + <head> + ## traverse through all namespaces present, + ## look for a %def named 'includes' + % for ns in context.namespaces.values(): + % if hasattr(ns, 'includes'): + ${ns.includes()} + % endif + % endfor + </head> + <body> + ${next.body()} + </body + </html> + + ## library.mako + ## library functions. + + <%def name="includes()"> + <link rel="stylesheet" type="text/css" href="mystyle.css"/> + <script type="text/javascript" src="functions.js"></script> + </%def> + + <%def name="mytag()"> + <form> + ${caller.body()} + </form> + </%def> + + + ## index.mako + ## calling template. + <%inherit file="base.mako"/> + <%namespace name="foo" file="library.mako"/> + + <%foo:mytag> + a form + </%foo:mytag> + +In this version, ``library.mako`` declares a ``<%def>`` named ``includes``. The example works +identically to the previous one, except that ``base.mako`` looks for defs named ``include`` +on each namespace it examines. + +API Reference +============= + +.. autoclass:: mako.runtime.Namespace + :show-inheritance: + :members: + +.. autoclass:: mako.runtime.TemplateNamespace + :show-inheritance: + :members: + +.. autoclass:: mako.runtime.ModuleNamespace + :show-inheritance: + :members: + +.. autofunction:: mako.runtime.supports_caller + +.. autofunction:: mako.runtime.capture + diff --git a/doc/build/requirements.txt b/doc/build/requirements.txt new file mode 100644 index 0000000..f3e40e0 --- /dev/null +++ b/doc/build/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/sqlalchemyorg/changelog.git#egg=changelog +git+https://github.com/sqlalchemyorg/sphinx-paramlinks.git#egg=sphinx-paramlinks +git+https://github.com/sqlalchemyorg/zzzeeksphinx.git#egg=zzzeeksphinx diff --git a/doc/build/runtime.rst b/doc/build/runtime.rst new file mode 100644 index 0000000..17c9b99 --- /dev/null +++ b/doc/build/runtime.rst @@ -0,0 +1,448 @@ +.. _runtime_toplevel: + +============================ +The Mako Runtime Environment +============================ + +This section describes a little bit about the objects and +built-in functions that are available in templates. + +.. _context: + +Context +======= + +The :class:`.Context` is the central object that is created when +a template is first executed, and is responsible for handling +all communication with the outside world. Within the template +environment, it is available via the :ref:`reserved name <reserved_names>` +``context``. The :class:`.Context` includes two +major components, one of which is the output buffer, which is a +file-like object such as Python's ``StringIO`` or similar, and +the other a dictionary of variables that can be freely +referenced within a template; this dictionary is a combination +of the arguments sent to the :meth:`~.Template.render` function and +some built-in variables provided by Mako's runtime environment. + +The Buffer +---------- + +The buffer is stored within the :class:`.Context`, and writing +to it is achieved by calling the :meth:`~.Context.write` method +-- in a template this looks like ``context.write('some string')``. +You usually don't need to care about this, as all text within a template, as +well as all expressions provided by ``${}``, automatically send +everything to this method. The cases you might want to be aware +of its existence are if you are dealing with various +filtering/buffering scenarios, which are described in +:ref:`filtering_toplevel`, or if you want to programmatically +send content to the output stream, such as within a ``<% %>`` +block. + +.. sourcecode:: mako + + <% + context.write("some programmatic text") + %> + +The actual buffer may or may not be the original buffer sent to +the :class:`.Context` object, as various filtering/caching +scenarios may "push" a new buffer onto the context's underlying +buffer stack. For this reason, just stick with +``context.write()`` and content will always go to the topmost +buffer. + +.. _context_vars: + +Context Variables +----------------- + +When your template is compiled into a Python module, the body +content is enclosed within a Python function called +``render_body``. Other top-level defs defined in the template are +defined within their own function bodies which are named after +the def's name with the prefix ``render_`` (i.e. ``render_mydef``). +One of the first things that happens within these functions is +that all variable names that are referenced within the function +which are not defined in some other way (i.e. such as via +assignment, module level imports, etc.) are pulled from the +:class:`.Context` object's dictionary of variables. This is how you're +able to freely reference variable names in a template which +automatically correspond to what was passed into the current +:class:`.Context`. + +* **What happens if I reference a variable name that is not in + the current context?** - The value you get back is a special + value called ``UNDEFINED``, or if the ``strict_undefined=True`` flag + is used a ``NameError`` is raised. ``UNDEFINED`` is just a simple global + variable with the class :class:`mako.runtime.Undefined`. The + ``UNDEFINED`` object throws an error when you call ``str()`` on + it, which is what happens if you try to use it in an + expression. +* **UNDEFINED makes it hard for me to find what name is missing** - An alternative + is to specify the option ``strict_undefined=True`` + to the :class:`.Template` or :class:`.TemplateLookup`. This will cause + any non-present variables to raise an immediate ``NameError`` + which includes the name of the variable in its message + when :meth:`~.Template.render` is called -- ``UNDEFINED`` is not used. + + .. versionadded:: 0.3.6 + +* **Why not just return None?** Using ``UNDEFINED``, or + raising a ``NameError`` is more + explicit and allows differentiation between a value of ``None`` + that was explicitly passed to the :class:`.Context` and a value that + wasn't present at all. +* **Why raise an exception when you call str() on it ? Why not + just return a blank string?** - Mako tries to stick to the + Python philosophy of "explicit is better than implicit". In + this case, it's decided that the template author should be made + to specifically handle a missing value rather than + experiencing what may be a silent failure. Since ``UNDEFINED`` + is a singleton object just like Python's ``True`` or ``False``, + you can use the ``is`` operator to check for it: + + .. sourcecode:: mako + + % if someval is UNDEFINED: + someval is: no value + % else: + someval is: ${someval} + % endif + +Another facet of the :class:`.Context` is that its dictionary of +variables is **immutable**. Whatever is set when +:meth:`~.Template.render` is called is what stays. Of course, since +its Python, you can hack around this and change values in the +context's internal dictionary, but this will probably will not +work as well as you'd think. The reason for this is that Mako in +many cases creates copies of the :class:`.Context` object, which +get sent to various elements of the template and inheriting +templates used in an execution. So changing the value in your +local :class:`.Context` will not necessarily make that value +available in other parts of the template's execution. Examples +of where Mako creates copies of the :class:`.Context` include +within top-level def calls from the main body of the template +(the context is used to propagate locally assigned variables +into the scope of defs; since in the template's body they appear +as inlined functions, Mako tries to make them act that way), and +within an inheritance chain (each template in an inheritance +chain has a different notion of ``parent`` and ``next``, which +are all stored in unique :class:`.Context` instances). + +* **So what if I want to set values that are global to everyone + within a template request?** - All you have to do is provide a + dictionary to your :class:`.Context` when the template first + runs, and everyone can just get/set variables from that. Lets + say its called ``attributes``. + + Running the template looks like: + + .. sourcecode:: python + + output = template.render(attributes={}) + + Within a template, just reference the dictionary: + + .. sourcecode:: mako + + <% + attributes['foo'] = 'bar' + %> + 'foo' attribute is: ${attributes['foo']} + +* **Why can't "attributes" be a built-in feature of the + Context?** - This is an area where Mako is trying to make as + few decisions about your application as it possibly can. + Perhaps you don't want your templates to use this technique of + assigning and sharing data, or perhaps you have a different + notion of the names and kinds of data structures that should + be passed around. Once again Mako would rather ask the user to + be explicit. + +Context Methods and Accessors +----------------------------- + +Significant members of :class:`.Context` include: + +* ``context[key]`` / ``context.get(key, default=None)`` - + dictionary-like accessors for the context. Normally, any + variable you use in your template is automatically pulled from + the context if it isn't defined somewhere already. Use the + dictionary accessor and/or ``get`` method when you want a + variable that *is* already defined somewhere else, such as in + the local arguments sent to a ``%def`` call. If a key is not + present, like a dictionary it raises ``KeyError``. +* ``keys()`` - all the names defined within this context. +* ``kwargs`` - this returns a **copy** of the context's + dictionary of variables. This is useful when you want to + propagate the variables in the current context to a function + as keyword arguments, i.e.: + + .. sourcecode:: mako + + ${next.body(**context.kwargs)} + +* ``write(text)`` - write some text to the current output + stream. +* ``lookup`` - returns the :class:`.TemplateLookup` instance that is + used for all file-lookups within the current execution (even + though individual :class:`.Template` instances can conceivably have + different instances of a :class:`.TemplateLookup`, only the + :class:`.TemplateLookup` of the originally-called :class:`.Template` gets + used in a particular execution). + +.. _loop_context: + +The Loop Context +================ + +Within ``% for`` blocks, the :ref:`reserved name<reserved_names>` ``loop`` +is available. ``loop`` tracks the progress of +the ``for`` loop and makes it easy to use the iteration state to control +template behavior: + +.. sourcecode:: mako + + <ul> + % for a in ("one", "two", "three"): + <li>Item ${loop.index}: ${a}</li> + % endfor + </ul> + +.. versionadded:: 0.7 + +Iterations +---------- + +Regardless of the type of iterable you're looping over, ``loop`` always tracks +the 0-indexed iteration count (available at ``loop.index``), its parity +(through the ``loop.even`` and ``loop.odd`` bools), and ``loop.first``, a bool +indicating whether the loop is on its first iteration. If your iterable +provides a ``__len__`` method, ``loop`` also provides access to +a count of iterations remaining at ``loop.reverse_index`` and ``loop.last``, +a bool indicating whether the loop is on its last iteration; accessing these +without ``__len__`` will raise a ``TypeError``. + +Cycling +------- + +Cycling is available regardless of whether the iterable you're using provides +a ``__len__`` method. Prior to Mako 0.7, you might have generated a simple +zebra striped list using ``enumerate``: + +.. sourcecode:: mako + + <ul> + % for i, item in enumerate(('spam', 'ham', 'eggs')): + <li class="${'odd' if i % 2 else 'even'}">${item}</li> + % endfor + </ul> + +With ``loop.cycle``, you get the same results with cleaner code and less prep work: + +.. sourcecode:: mako + + <ul> + % for item in ('spam', 'ham', 'eggs'): + <li class="${loop.cycle('even', 'odd')}">${item}</li> + % endfor + </ul> + +Both approaches produce output like the following: + +.. sourcecode:: html + + <ul> + <li class="even">spam</li> + <li class="odd">ham</li> + <li class="even">eggs</li> + </ul> + +Parent Loops +------------ + +Loop contexts can also be transparently nested, and the Mako runtime will do +the right thing and manage the scope for you. You can access the parent loop +context through ``loop.parent``. + +This allows you to reach all the way back up through the loop stack by +chaining ``parent`` attribute accesses, i.e. ``loop.parent.parent....`` as +long as the stack depth isn't exceeded. For example, you can use the parent +loop to make a checkered table: + +.. sourcecode:: mako + + <table> + % for consonant in 'pbj': + <tr> + % for vowel in 'iou': + <td class="${'black' if (loop.parent.even == loop.even) else 'red'}"> + ${consonant + vowel}t + </td> + % endfor + </tr> + % endfor + </table> + +.. sourcecode:: html + + <table> + <tr> + <td class="black"> + pit + </td> + <td class="red"> + pot + </td> + <td class="black"> + put + </td> + </tr> + <tr> + <td class="red"> + bit + </td> + <td class="black"> + bot + </td> + <td class="red"> + but + </td> + </tr> + <tr> + <td class="black"> + jit + </td> + <td class="red"> + jot + </td> + <td class="black"> + jut + </td> + </tr> + </table> + +.. _migrating_loop: + +Migrating Legacy Templates that Use the Word "loop" +--------------------------------------------------- + +.. versionchanged:: 0.7 + The ``loop`` name is now :ref:`reserved <reserved_names>` in Mako, + which means a template that refers to a variable named ``loop`` + won't function correctly when used in Mako 0.7. + +To ease the transition for such systems, the feature can be disabled across the board for +all templates, then re-enabled on a per-template basis for those templates which wish +to make use of the new system. + +First, the ``enable_loop=False`` flag is passed to either the :class:`.TemplateLookup` +or :class:`.Template` object in use: + +.. sourcecode:: python + + lookup = TemplateLookup(directories=['/docs'], enable_loop=False) + +or: + +.. sourcecode:: python + + template = Template("some template", enable_loop=False) + +An individual template can make usage of the feature when ``enable_loop`` is set to +``False`` by switching it back on within the ``<%page>`` tag: + +.. sourcecode:: mako + + <%page enable_loop="True"/> + + % for i in collection: + ${i} ${loop.index} + % endfor + +Using the above scheme, it's safe to pass the name ``loop`` to the :meth:`.Template.render` +method as well as to freely make usage of a variable named ``loop`` within a template, provided +the ``<%page>`` tag doesn't override it. New templates that want to use the ``loop`` context +can then set up ``<%page enable_loop="True"/>`` to use the new feature without affecting +old templates. + +All the Built-in Names +====================== + +A one-stop shop for all the names Mako defines. Most of these +names are instances of :class:`.Namespace`, which are described +in the next section, :ref:`namespaces_toplevel`. Also, most of +these names other than ``context``, ``UNDEFINED``, and ``loop`` are +also present *within* the :class:`.Context` itself. The names +``context``, ``loop`` and ``UNDEFINED`` themselves can't be passed +to the context and can't be substituted -- see the section :ref:`reserved_names`. + +* ``context`` - this is the :class:`.Context` object, introduced + at :ref:`context`. +* ``local`` - the namespace of the current template, described + in :ref:`namespaces_builtin`. +* ``self`` - the namespace of the topmost template in an + inheritance chain (if any, otherwise the same as ``local``), + mostly described in :ref:`inheritance_toplevel`. +* ``parent`` - the namespace of the parent template in an + inheritance chain (otherwise undefined); see + :ref:`inheritance_toplevel`. +* ``next`` - the namespace of the next template in an + inheritance chain (otherwise undefined); see + :ref:`inheritance_toplevel`. +* ``caller`` - a "mini" namespace created when using the + ``<%call>`` tag to define a "def call with content"; described + in :ref:`defs_with_content`. +* ``loop`` - this provides access to :class:`.LoopContext` objects when + they are requested within ``% for`` loops, introduced at :ref:`loop_context`. +* ``capture`` - a function that calls a given def and captures + its resulting content into a string, which is returned. Usage + is described in :ref:`filtering_toplevel`. +* ``UNDEFINED`` - a global singleton that is applied to all + otherwise uninitialized template variables that were not + located within the :class:`.Context` when rendering began, + unless the :class:`.Template` flag ``strict_undefined`` + is set to ``True``. ``UNDEFINED`` is + an instance of :class:`.Undefined`, and raises an + exception when its ``__str__()`` method is called. +* ``pageargs`` - this is a dictionary which is present in a + template which does not define any ``**kwargs`` section in its + ``<%page>`` tag. All keyword arguments sent to the ``body()`` + function of a template (when used via namespaces) go here by + default unless otherwise defined as a page argument. If this + makes no sense, it shouldn't; read the section + :ref:`namespaces_body`. + +.. _reserved_names: + +Reserved Names +-------------- + +Mako has a few names that are considered to be "reserved" and can't be used +as variable names. + +.. versionchanged:: 0.7 + Mako raises an error if these words are found passed to the template + as context arguments, whereas in previous versions they'd be silently + ignored or lead to other error messages. + +* ``context`` - see :ref:`context`. +* ``UNDEFINED`` - see :ref:`context_vars`. +* ``loop`` - see :ref:`loop_context`. Note this can be disabled for legacy templates + via the ``enable_loop=False`` argument; see :ref:`migrating_loop`. + +API Reference +============= + +.. autoclass:: mako.runtime.Context + :show-inheritance: + :members: + +.. autoclass:: mako.runtime.LoopContext + :show-inheritance: + :members: + +.. autoclass:: mako.runtime.Undefined + :show-inheritance: + diff --git a/doc/build/syntax.rst b/doc/build/syntax.rst new file mode 100644 index 0000000..2873584 --- /dev/null +++ b/doc/build/syntax.rst @@ -0,0 +1,495 @@ +.. _syntax_toplevel: + +====== +Syntax +====== + +A Mako template is parsed from a text stream containing any kind +of content, XML, HTML, email text, etc. The template can further +contain Mako-specific directives which represent variable and/or +expression substitutions, control structures (i.e. conditionals +and loops), server-side comments, full blocks of Python code, as +well as various tags that offer additional functionality. All of +these constructs compile into real Python code. This means that +you can leverage the full power of Python in almost every aspect +of a Mako template. + +Expression Substitution +======================= + +The simplest expression is just a variable substitution. The +syntax for this is the ``${}`` construct, which is inspired by +Perl, Genshi, JSP EL, and others: + +.. sourcecode:: mako + + this is x: ${x} + +Above, the string representation of ``x`` is applied to the +template's output stream. If you're wondering where ``x`` comes +from, it's usually from the :class:`.Context` supplied to the +template's rendering function. If ``x`` was not supplied to the +template and was not otherwise assigned locally, it evaluates to +a special value ``UNDEFINED``. More on that later. + +The contents within the ``${}`` tag are evaluated by Python +directly, so full expressions are OK: + +.. sourcecode:: mako + + pythagorean theorem: ${pow(x,2) + pow(y,2)} + +The results of the expression are evaluated into a string result +in all cases before being rendered to the output stream, such as +the above example where the expression produces a numeric +result. + +Expression Escaping +=================== + +Mako includes a number of built-in escaping mechanisms, +including HTML, URI and XML escaping, as well as a "trim" +function. These escapes can be added to an expression +substitution using the ``|`` operator: + +.. sourcecode:: mako + + ${"this is some text" | u} + +The above expression applies URL escaping to the expression, and +produces ``this+is+some+text``. The ``u`` name indicates URL +escaping, whereas ``h`` represents HTML escaping, ``x`` +represents XML escaping, and ``trim`` applies a trim function. + +Read more about built-in filtering functions, including how to +make your own filter functions, in :ref:`filtering_toplevel`. + +Control Structures +================== + +A control structure refers to all those things that control the +flow of a program -- conditionals (i.e. ``if``/``else``), loops (like +``while`` and ``for``), as well as things like ``try``/``except``. In Mako, +control structures are written using the ``%`` marker followed +by a regular Python control expression, and are "closed" by +using another ``%`` marker with the tag "``end<name>``", where +"``<name>``" is the keyword of the expression: + +.. sourcecode:: mako + + % if x==5: + this is some output + % endif + +The ``%`` can appear anywhere on the line as long as no text +precedes it; indentation is not significant. The full range of +Python "colon" expressions are allowed here, including +``if``/``elif``/``else``, ``while``, ``for``, ``with``, and even ``def``, +although Mako has a built-in tag for defs which is more full-featured. + +.. sourcecode:: mako + + % for a in ['one', 'two', 'three', 'four', 'five']: + % if a[0] == 't': + its two or three + % elif a[0] == 'f': + four/five + % else: + one + % endif + % endfor + +The ``%`` sign can also be "escaped", if you actually want to +emit a percent sign as the first non whitespace character on a +line, by escaping it as in ``%%``: + +.. sourcecode:: mako + + %% some text + + %% some more text + +The Loop Context +---------------- + +The **loop context** provides additional information about a loop +while inside of a ``% for`` structure: + +.. sourcecode:: mako + + <ul> + % for a in ("one", "two", "three"): + <li>Item ${loop.index}: ${a}</li> + % endfor + </ul> + +See :ref:`loop_context` for more information on this feature. + +.. versionadded:: 0.7 + +Comments +======== + +Comments come in two varieties. The single line comment uses +``##`` as the first non-space characters on a line: + +.. sourcecode:: mako + + ## this is a comment. + ...text ... + +A multiline version exists using ``<%doc> ...text... </%doc>``: + +.. sourcecode:: mako + + <%doc> + these are comments + more comments + </%doc> + +Newline Filters +=============== + +The backslash ("``\``") character, placed at the end of any +line, will consume the newline character before continuing to +the next line: + +.. sourcecode:: mako + + here is a line that goes onto \ + another line. + +The above text evaluates to: + +.. sourcecode:: text + + here is a line that goes onto another line. + +Python Blocks +============= + +Any arbitrary block of python can be dropped in using the ``<% +%>`` tags: + +.. sourcecode:: mako + + this is a template + <% + x = db.get_resource('foo') + y = [z.element for z in x if x.frobnizzle==5] + %> + % for elem in y: + element: ${elem} + % endfor + +Within ``<% %>``, you're writing a regular block of Python code. +While the code can appear with an arbitrary level of preceding +whitespace, it has to be consistently formatted with itself. +Mako's compiler will adjust the block of Python to be consistent +with the surrounding generated Python code. + +Module-level Blocks +=================== + +A variant on ``<% %>`` is the module-level code block, denoted +by ``<%! %>``. Code within these tags is executed at the module +level of the template, and not within the rendering function of +the template. Therefore, this code does not have access to the +template's context and is only executed when the template is +loaded into memory (which can be only once per application, or +more, depending on the runtime environment). Use the ``<%! %>`` +tags to declare your template's imports, as well as any +pure-Python functions you might want to declare: + +.. sourcecode:: mako + + <%! + import mylib + import re + + def filter(text): + return re.sub(r'^@', '', text) + %> + +Any number of ``<%! %>`` blocks can be declared anywhere in a +template; they will be rendered in the resulting module +in a single contiguous block above all render callables, +in the order in which they appear in the source template. + +Tags +==== + +The rest of what Mako offers takes place in the form of tags. +All tags use the same syntax, which is similar to an XML tag +except that the first character of the tag name is a ``%`` +character. The tag is closed either by a contained slash +character, or an explicit closing tag: + +.. sourcecode:: mako + + <%include file="foo.txt"/> + + <%def name="foo" buffered="True"> + this is a def + </%def> + +All tags have a set of attributes which are defined for each +tag. Some of these attributes are required. Also, many +attributes support **evaluation**, meaning you can embed an +expression (using ``${}``) inside the attribute text: + +.. sourcecode:: mako + + <%include file="/foo/bar/${myfile}.txt"/> + +Whether or not an attribute accepts runtime evaluation depends +on the type of tag and how that tag is compiled into the +template. The best way to find out if you can stick an +expression in is to try it! The lexer will tell you if it's not +valid. + +Heres a quick summary of all the tags: + +``<%page>`` +----------- + +This tag defines general characteristics of the template, +including caching arguments, and optional lists of arguments +which the template expects when invoked. + +.. sourcecode:: mako + + <%page args="x, y, z='default'"/> + +Or a page tag that defines caching characteristics: + +.. sourcecode:: mako + + <%page cached="True" cache_type="memory"/> + +Currently, only one ``<%page>`` tag gets used per template, the +rest get ignored. While this will be improved in a future +release, for now make sure you have only one ``<%page>`` tag +defined in your template, else you may not get the results you +want. Further details on what ``<%page>`` is used for are described +in the following sections: + +* :ref:`namespaces_body` - ``<%page>`` is used to define template-level + arguments and defaults + +* :ref:`expression_filtering` - expression filters can be applied to all + expressions throughout a template using the ``<%page>`` tag + +* :ref:`caching_toplevel` - options to control template-level caching + may be applied in the ``<%page>`` tag. + +``<%include>`` +-------------- + +A tag that is familiar from other template languages, ``%include`` +is a regular joe that just accepts a file argument and calls in +the rendered result of that file: + +.. sourcecode:: mako + + <%include file="header.html"/> + + hello world + + <%include file="footer.html"/> + +Include also accepts arguments which are available as ``<%page>`` arguments in the receiving template: + +.. sourcecode:: mako + + <%include file="toolbar.html" args="current_section='members', username='ed'"/> + +``<%def>`` +---------- + +The ``%def`` tag defines a Python function which contains a set +of content, that can be called at some other point in the +template. The basic idea is simple: + +.. sourcecode:: mako + + <%def name="myfunc(x)"> + this is myfunc, x is ${x} + </%def> + + ${myfunc(7)} + +The ``%def`` tag is a lot more powerful than a plain Python ``def``, as +the Mako compiler provides many extra services with ``%def`` that +you wouldn't normally have, such as the ability to export defs +as template "methods", automatic propagation of the current +:class:`.Context`, buffering/filtering/caching flags, and def calls +with content, which enable packages of defs to be sent as +arguments to other def calls (not as hard as it sounds). Get the +full deal on what ``%def`` can do in :ref:`defs_toplevel`. + +``<%block>`` +------------ + +``%block`` is a tag that is close to a ``%def``, +except executes itself immediately in its base-most scope, +and can also be anonymous (i.e. with no name): + +.. sourcecode:: mako + + <%block filter="h"> + some <html> stuff. + </%block> + +Inspired by Jinja2 blocks, named blocks offer a syntactically pleasing way +to do inheritance: + +.. sourcecode:: mako + + <html> + <body> + <%block name="header"> + <h2><%block name="title"/></h2> + </%block> + ${self.body()} + </body> + </html> + +Blocks are introduced in :ref:`blocks` and further described in :ref:`inheritance_toplevel`. + +.. versionadded:: 0.4.1 + +``<%namespace>`` +---------------- + +``%namespace`` is Mako's equivalent of Python's ``import`` +statement. It allows access to all the rendering functions and +metadata of other template files, plain Python modules, as well +as locally defined "packages" of functions. + +.. sourcecode:: mako + + <%namespace file="functions.html" import="*"/> + +The underlying object generated by ``%namespace``, an instance of +:class:`.mako.runtime.Namespace`, is a central construct used in +templates to reference template-specific information such as the +current URI, inheritance structures, and other things that are +not as hard as they sound right here. Namespaces are described +in :ref:`namespaces_toplevel`. + +``<%inherit>`` +-------------- + +Inherit allows templates to arrange themselves in **inheritance +chains**. This is a concept familiar in many other template +languages. + +.. sourcecode:: mako + + <%inherit file="base.html"/> + +When using the ``%inherit`` tag, control is passed to the topmost +inherited template first, which then decides how to handle +calling areas of content from its inheriting templates. Mako +offers a lot of flexibility in this area, including dynamic +inheritance, content wrapping, and polymorphic method calls. +Check it out in :ref:`inheritance_toplevel`. + +``<%``\ nsname\ ``:``\ defname\ ``>`` +------------------------------------- + +Any user-defined "tag" can be created against +a namespace by using a tag with a name of the form +``<%<namespacename>:<defname>>``. The closed and open formats of such a +tag are equivalent to an inline expression and the ``<%call>`` +tag, respectively. + +.. sourcecode:: mako + + <%mynamespace:somedef param="some value"> + this is the body + </%mynamespace:somedef> + +To create custom tags which accept a body, see +:ref:`defs_with_content`. + +.. versionadded:: 0.2.3 + +``<%call>`` +----------- + +The call tag is the "classic" form of a user-defined tag, and is +roughly equivalent to the ``<%namespacename:defname>`` syntax +described above. This tag is also described in :ref:`defs_with_content`. + +``<%doc>`` +---------- + +The ``%doc`` tag handles multiline comments: + +.. sourcecode:: mako + + <%doc> + these are comments + more comments + </%doc> + +Also the ``##`` symbol as the first non-space characters on a line can be used for single line comments. + +``<%text>`` +----------- + +This tag suspends the Mako lexer's normal parsing of Mako +template directives, and returns its entire body contents as +plain text. It is used pretty much to write documentation about +Mako: + +.. sourcecode:: mako + + <%text filter="h"> + heres some fake mako ${syntax} + <%def name="x()">${x}</%def> + </%text> + +.. _syntax_exiting_early: + +Exiting Early from a Template +============================= + +Sometimes you want to stop processing a template or ``<%def>`` +method in the middle and just use the text you've accumulated so +far. This is accomplished by using ``return`` statement inside +a Python block. It's a good idea for the ``return`` statement +to return an empty string, which prevents the Python default return +value of ``None`` from being rendered by the template. This +return value is for semantic purposes provided in templates via +the ``STOP_RENDERING`` symbol: + +.. sourcecode:: mako + + % if not len(records): + No records found. + <% return STOP_RENDERING %> + % endif + +Or perhaps: + +.. sourcecode:: mako + + <% + if not len(records): + return STOP_RENDERING + %> + +In older versions of Mako, an empty string can be substituted for +the ``STOP_RENDERING`` symbol: + +.. sourcecode:: mako + + <% return '' %> + +.. versionadded:: 1.0.2 - added the ``STOP_RENDERING`` symbol which serves + as a semantic identifier for the empty string ``""`` used by a + Python ``return`` statement. + diff --git a/doc/build/unicode.rst b/doc/build/unicode.rst new file mode 100644 index 0000000..060e113 --- /dev/null +++ b/doc/build/unicode.rst @@ -0,0 +1,153 @@ +.. _unicode_toplevel: + +=================== +The Unicode Chapter +=================== + +In normal Mako operation, all parsed template constructs and +output streams are handled internally as Python 3 ``str`` (Unicode) +objects. It's only at the point of :meth:`~.Template.render` that this stream of Unicode objects may be rendered into whatever the desired output encoding +is. The implication here is that the template developer must +:ensure that :ref:`the encoding of all non-ASCII templates is explicit +<set_template_file_encoding>` (still required in Python 3, although Mako defaults to ``utf-8``), +that :ref:`all non-ASCII-encoded expressions are in one way or another +converted to unicode <handling_non_ascii_expressions>` +(not much of a burden in Python 3), and that :ref:`the output stream of the +template is handled as a unicode stream being encoded to some +encoding <defining_output_encoding>` (still required in Python 3). + +.. _set_template_file_encoding: + +Specifying the Encoding of a Template File +========================================== + +.. versionchanged:: 1.1.3 + + As of Mako 1.1.3, the default template encoding is "utf-8". Previously, a + Python "magic encoding comment" was required for templates that were not + using ASCII. + +Mako templates support Python's "magic encoding comment" syntax +described in `pep-0263 <http://www.python.org/dev/peps/pep-0263/>`_: + +.. sourcecode:: mako + + ## -*- coding: utf-8 -*- + + Alors vous imaginez ma surprise, au lever du jour, quand + une drôle de petite voix m’a réveillé. Elle disait: + « S’il vous plaît… dessine-moi un mouton! » + +As an alternative, the template encoding can be specified +programmatically to either :class:`.Template` or :class:`.TemplateLookup` via +the ``input_encoding`` parameter: + +.. sourcecode:: python + + t = TemplateLookup(directories=['./'], input_encoding='utf-8') + +The above will assume all located templates specify ``utf-8`` +encoding, unless the template itself contains its own magic +encoding comment, which takes precedence. + +.. _handling_non_ascii_expressions: + +Handling Expressions +==================== + +The next area that encoding comes into play is in expression +constructs. By default, Mako's treatment of an expression like +this: + +.. sourcecode:: mako + + ${"hello world"} + +looks something like this: + +.. sourcecode:: python + + context.write(str("hello world")) + +That is, **the output of all expressions is run through the +``str`` built-in**. This is the default setting, and can be +modified to expect various encodings. The ``str`` step serves +both the purpose of rendering non-string expressions into +strings (such as integers or objects which contain ``__str()__`` +methods), and to ensure that the final output stream is +constructed as a Unicode object. The main implication of this is +that **any raw byte-strings that contain an encoding other than +ASCII must first be decoded to a Python unicode object**. + +Similarly, if you are reading data from a file that is streaming +bytes, or returning data from some object that is returning a +Python byte-string containing a non-ASCII encoding, you have to +explicitly decode to Unicode first, such as: + +.. sourcecode:: mako + + ${call_my_object().decode('utf-8')} + +Note that filehandles acquired by ``open()`` in Python 3 default +to returning "text": that is, the decoding is done for you. See +Python 3's documentation for the ``open()`` built-in for details on +this. + +If you want a certain encoding applied to *all* expressions, +override the ``str`` builtin with the ``decode`` built-in at the +:class:`.Template` or :class:`.TemplateLookup` level: + +.. sourcecode:: python + + t = Template(templatetext, default_filters=['decode.utf8']) + +Note that the built-in ``decode`` object is slower than the +``str`` function, since unlike ``str`` it's not a Python +built-in, and it also checks the type of the incoming data to +determine if string conversion is needed first. + +The ``default_filters`` argument can be used to entirely customize +the filtering process of expressions. This argument is described +in :ref:`filtering_default_filters`. + +.. _defining_output_encoding: + +Defining Output Encoding +======================== + +Now that we have a template which produces a pure Unicode output +stream, all the hard work is done. We can take the output and do +anything with it. + +As stated in the :doc:`"Usage" chapter <usage>`, both :class:`.Template` and +:class:`.TemplateLookup` accept ``output_encoding`` and ``encoding_errors`` +parameters which can be used to encode the output in any Python +supported codec: + +.. sourcecode:: python + + from mako.template import Template + from mako.lookup import TemplateLookup + + mylookup = TemplateLookup(directories=['/docs'], output_encoding='utf-8', encoding_errors='replace') + + mytemplate = mylookup.get_template("foo.txt") + print(mytemplate.render()) + +:meth:`~.Template.render` will return a ``bytes`` object in Python 3 if an output +encoding is specified. By default it performs no encoding and +returns a native string. + +:meth:`~.Template.render_unicode` will return the template output as a Python +``str`` object: + +.. sourcecode:: python + + print(mytemplate.render_unicode()) + +The above method disgards the output encoding keyword argument; +you can encode yourself by saying: + +.. sourcecode:: python + + print(mytemplate.render_unicode().encode('utf-8', 'replace')) diff --git a/doc/build/unreleased/README.txt b/doc/build/unreleased/README.txt new file mode 100644 index 0000000..f7bc72a --- /dev/null +++ b/doc/build/unreleased/README.txt @@ -0,0 +1,13 @@ +individual per-changelog files go here +in .rst format, which are pulled in by +changelog to +be rendered into the changelog.rst file. +At release time, the files here are removed and written +directly into the changelog. + +Rationale is so that multiple changes being merged +into gerrit don't produce conflicts. Note that +gerrit does not support custom merge handlers unlike +git itself. + + diff --git a/doc/build/usage.rst b/doc/build/usage.rst new file mode 100644 index 0000000..22b6bac --- /dev/null +++ b/doc/build/usage.rst @@ -0,0 +1,519 @@ +.. _usage_toplevel: + +===== +Usage +===== + +Basic Usage +=========== + +This section describes the Python API for Mako templates. If you +are using Mako within a web framework such as Pylons, the work +of integrating Mako's API is already done for you, in which case +you can skip to the next section, :ref:`syntax_toplevel`. + +The most basic way to create a template and render it is through +the :class:`.Template` class: + +.. sourcecode:: python + + from mako.template import Template + + mytemplate = Template("hello world!") + print(mytemplate.render()) + +Above, the text argument to :class:`.Template` is **compiled** into a +Python module representation. This module contains a function +called ``render_body()``, which produces the output of the +template. When ``mytemplate.render()`` is called, Mako sets up a +runtime environment for the template and calls the +``render_body()`` function, capturing the output into a buffer and +returning its string contents. + + +The code inside the ``render_body()`` function has access to a +namespace of variables. You can specify these variables by +sending them as additional keyword arguments to the :meth:`~.Template.render` +method: + +.. sourcecode:: python + + from mako.template import Template + + mytemplate = Template("hello, ${name}!") + print(mytemplate.render(name="jack")) + +The :meth:`~.Template.render` method calls upon Mako to create a +:class:`.Context` object, which stores all the variable names accessible +to the template and also stores a buffer used to capture output. +You can create this :class:`.Context` yourself and have the template +render with it, using the :meth:`~.Template.render_context` method: + +.. sourcecode:: python + + from mako.template import Template + from mako.runtime import Context + from io import StringIO + + mytemplate = Template("hello, ${name}!") + buf = StringIO() + ctx = Context(buf, name="jack") + mytemplate.render_context(ctx) + print(buf.getvalue()) + +Using File-Based Templates +========================== + +A :class:`.Template` can also load its template source code from a file, +using the ``filename`` keyword argument: + +.. sourcecode:: python + + from mako.template import Template + + mytemplate = Template(filename='/docs/mytmpl.txt') + print(mytemplate.render()) + +For improved performance, a :class:`.Template` which is loaded from a +file can also cache the source code to its generated module on +the filesystem as a regular Python module file (i.e. a ``.py`` +file). To do this, just add the ``module_directory`` argument to +the template: + +.. sourcecode:: python + + from mako.template import Template + + mytemplate = Template(filename='/docs/mytmpl.txt', module_directory='/tmp/mako_modules') + print(mytemplate.render()) + +When the above code is rendered, a file +``/tmp/mako_modules/docs/mytmpl.txt.py`` is created containing the +source code for the module. The next time a :class:`.Template` with the +same arguments is created, this module file will be +automatically re-used. + +.. _usage_templatelookup: + +Using ``TemplateLookup`` +======================== + +All of the examples thus far have dealt with the usage of a +single :class:`.Template` object. If the code within those templates +tries to locate another template resource, it will need some way +to find them, using simple URI strings. For this need, the +resolution of other templates from within a template is +accomplished by the :class:`.TemplateLookup` class. This class is +constructed given a list of directories in which to search for +templates, as well as keyword arguments that will be passed to +the :class:`.Template` objects it creates: + +.. sourcecode:: python + + from mako.template import Template + from mako.lookup import TemplateLookup + + mylookup = TemplateLookup(directories=['/docs']) + mytemplate = Template("""<%include file="header.txt"/> hello world!""", lookup=mylookup) + +Above, we created a textual template which includes the file +``"header.txt"``. In order for it to have somewhere to look for +``"header.txt"``, we passed a :class:`.TemplateLookup` object to it, which +will search in the directory ``/docs`` for the file ``"header.txt"``. + +Usually, an application will store most or all of its templates +as text files on the filesystem. So far, all of our examples +have been a little bit contrived in order to illustrate the +basic concepts. But a real application would get most or all of +its templates directly from the :class:`.TemplateLookup`, using the +aptly named :meth:`~.TemplateLookup.get_template` method, which accepts the URI of the +desired template: + +.. sourcecode:: python + + from mako.template import Template + from mako.lookup import TemplateLookup + + mylookup = TemplateLookup(directories=['/docs'], module_directory='/tmp/mako_modules') + + def serve_template(templatename, **kwargs): + mytemplate = mylookup.get_template(templatename) + print(mytemplate.render(**kwargs)) + +In the example above, we create a :class:`.TemplateLookup` which will +look for templates in the ``/docs`` directory, and will store +generated module files in the ``/tmp/mako_modules`` directory. The +lookup locates templates by appending the given URI to each of +its search directories; so if you gave it a URI of +``/etc/beans/info.txt``, it would search for the file +``/docs/etc/beans/info.txt``, else raise a :class:`.TopLevelNotFound` +exception, which is a custom Mako exception. + +When the lookup locates templates, it will also assign a ``uri`` +property to the :class:`.Template` which is the URI passed to the +:meth:`~.TemplateLookup.get_template()` call. :class:`.Template` uses this URI to calculate the +name of its module file. So in the above example, a +``templatename`` argument of ``/etc/beans/info.txt`` will create a +module file ``/tmp/mako_modules/etc/beans/info.txt.py``. + +Setting the Collection Size +--------------------------- + +The :class:`.TemplateLookup` also serves the important need of caching a +fixed set of templates in memory at a given time, so that +successive URI lookups do not result in full template +compilations and/or module reloads on each request. By default, +the :class:`.TemplateLookup` size is unbounded. You can specify a fixed +size using the ``collection_size`` argument: + +.. sourcecode:: python + + mylookup = TemplateLookup(directories=['/docs'], + module_directory='/tmp/mako_modules', collection_size=500) + +The above lookup will continue to load templates into memory +until it reaches a count of around 500. At that point, it will +clean out a certain percentage of templates using a least +recently used scheme. + +Setting Filesystem Checks +------------------------- + +Another important flag on :class:`.TemplateLookup` is +``filesystem_checks``. This defaults to ``True``, and says that each +time a template is returned by the :meth:`~.TemplateLookup.get_template()` method, the +revision time of the original template file is checked against +the last time the template was loaded, and if the file is newer +will reload its contents and recompile the template. On a +production system, setting ``filesystem_checks`` to ``False`` can +afford a small to moderate performance increase (depending on +the type of filesystem used). + +.. _usage_unicode: + +Using Unicode and Encoding +========================== + +Both :class:`.Template` and :class:`.TemplateLookup` accept ``output_encoding`` +and ``encoding_errors`` parameters which can be used to encode the +output in any Python supported codec: + +.. sourcecode:: python + + from mako.template import Template + from mako.lookup import TemplateLookup + + mylookup = TemplateLookup(directories=['/docs'], output_encoding='utf-8', encoding_errors='replace') + + mytemplate = mylookup.get_template("foo.txt") + print(mytemplate.render()) + +When using Python 3, the :meth:`~.Template.render` method will return a ``bytes`` +object, **if** ``output_encoding`` is set. Otherwise it returns a +``string``. + +Additionally, the :meth:`~.Template.render_unicode()` method exists which will +return the template output as a Python ``unicode`` object, or in +Python 3 a ``string``: + +.. sourcecode:: python + + print(mytemplate.render_unicode()) + +The above method disregards the output encoding keyword +argument; you can encode yourself by saying: + +.. sourcecode:: python + + print(mytemplate.render_unicode().encode('utf-8', 'replace')) + +Note that Mako's ability to return data in any encoding and/or +``unicode`` implies that the underlying output stream of the +template is a Python unicode object. This behavior is described +fully in :ref:`unicode_toplevel`. + +.. _handling_exceptions: + +Handling Exceptions +=================== + +Template exceptions can occur in two distinct places. One is +when you **lookup, parse and compile** the template, the other +is when you **run** the template. Within the running of a +template, exceptions are thrown normally from whatever Python +code originated the issue. Mako has its own set of exception +classes which mostly apply to the lookup and lexer/compiler +stages of template construction. Mako provides some library +routines that can be used to help provide Mako-specific +information about any exception's stack trace, as well as +formatting the exception within textual or HTML format. In all +cases, the main value of these handlers is that of converting +Python filenames, line numbers, and code samples into Mako +template filenames, line numbers, and code samples. All lines +within a stack trace which correspond to a Mako template module +will be converted to be against the originating template file. + +To format exception traces, the :func:`.text_error_template` and +:func:`.html_error_template` functions are provided. They make usage of +``sys.exc_info()`` to get at the most recently thrown exception. +Usage of these handlers usually looks like: + +.. sourcecode:: python + + from mako import exceptions + + try: + template = lookup.get_template(uri) + print(template.render()) + except: + print(exceptions.text_error_template().render()) + +Or for the HTML render function: + +.. sourcecode:: python + + from mako import exceptions + + try: + template = lookup.get_template(uri) + print(template.render()) + except: + print(exceptions.html_error_template().render()) + +The :func:`.html_error_template` template accepts two options: +specifying ``full=False`` causes only a section of an HTML +document to be rendered. Specifying ``css=False`` will disable the +default stylesheet from being rendered. + +E.g.: + +.. sourcecode:: python + + print(exceptions.html_error_template().render(full=False)) + +The HTML render function is also available built-in to +:class:`.Template` using the ``format_exceptions`` flag. In this case, any +exceptions raised within the **render** stage of the template +will result in the output being substituted with the output of +:func:`.html_error_template`: + +.. sourcecode:: python + + template = Template(filename="/foo/bar", format_exceptions=True) + print(template.render()) + +Note that the compile stage of the above template occurs when +you construct the :class:`.Template` itself, and no output stream is +defined. Therefore exceptions which occur within the +lookup/parse/compile stage will not be handled and will +propagate normally. While the pre-render traceback usually will +not include any Mako-specific lines anyway, it will mean that +exceptions which occur previous to rendering and those which +occur within rendering will be handled differently... so the +``try``/``except`` patterns described previously are probably of more +general use. + +The underlying object used by the error template functions is +the :class:`.RichTraceback` object. This object can also be used +directly to provide custom error views. Here's an example usage +which describes its general API: + +.. sourcecode:: python + + from mako.exceptions import RichTraceback + + try: + template = lookup.get_template(uri) + print(template.render()) + except: + traceback = RichTraceback() + for (filename, lineno, function, line) in traceback.traceback: + print("File %s, line %s, in %s" % (filename, lineno, function)) + print(line, "\n") + print("%s: %s" % (str(traceback.error.__class__.__name__), traceback.error)) + +Common Framework Integrations +============================= + +The Mako distribution includes a little bit of helper code for +the purpose of using Mako in some popular web framework +scenarios. This is a brief description of what's included. + +WSGI +---- + +A sample WSGI application is included in the distribution in the +file ``examples/wsgi/run_wsgi.py``. This runner is set up to pull +files from a `templates` as well as an `htdocs` directory and +includes a rudimental two-file layout. The WSGI runner acts as a +fully functional standalone web server, using ``wsgiutils`` to run +itself, and propagates GET and POST arguments from the request +into the :class:`.Context`, can serve images, CSS files and other kinds +of files, and also displays errors using Mako's included +exception-handling utilities. + +Pygments +-------- + +A `Pygments <https://pygments.org/>`_-compatible syntax +highlighting module is included under :mod:`mako.ext.pygmentplugin`. +This module is used in the generation of Mako documentation and +also contains various `setuptools` entry points under the heading +``pygments.lexers``, including ``mako``, ``html+mako``, ``xml+mako`` +(see the ``setup.py`` file for all the entry points). + +Babel +----- + +Mako provides support for extracting `gettext` messages from +templates via a `Babel`_ extractor +entry point under ``mako.ext.babelplugin``. + +`Gettext` messages are extracted from all Python code sections, +including those of control lines and expressions embedded +in tags. + +`Translator +comments <http://babel.edgewall.org/wiki/Documentation/messages.html#comments-tags-and-translator-comments-explanation>`_ +may also be extracted from Mako templates when a comment tag is +specified to `Babel`_ (such as with +the ``-c`` option). + +For example, a project ``"myproj"`` contains the following Mako +template at ``myproj/myproj/templates/name.html``: + +.. sourcecode:: mako + + <div id="name"> + Name: + ## TRANSLATORS: This is a proper name. See the gettext + ## manual, section Names. + ${_('Francois Pinard')} + </div> + +To extract gettext messages from this template the project needs +a Mako section in its `Babel Extraction Method Mapping +file <http://babel.edgewall.org/wiki/Documentation/messages.html#extraction-method-mapping-and-configuration>`_ +(typically located at ``myproj/babel.cfg``): + +.. sourcecode:: cfg + + # Extraction from Python source files + + [python: myproj/**.py] + + # Extraction from Mako templates + + [mako: myproj/templates/**.html] + input_encoding = utf-8 + +The Mako extractor supports an optional ``input_encoding`` +parameter specifying the encoding of the templates (identical to +:class:`.Template`/:class:`.TemplateLookup`'s ``input_encoding`` parameter). + +Invoking `Babel`_'s extractor at the +command line in the project's root directory: + +.. sourcecode:: sh + + myproj$ pybabel extract -F babel.cfg -c "TRANSLATORS:" . + +will output a `gettext` catalog to `stdout` including the following: + +.. sourcecode:: pot + + #. TRANSLATORS: This is a proper name. See the gettext + #. manual, section Names. + #: myproj/templates/name.html:5 + msgid "Francois Pinard" + msgstr "" + +This is only a basic example: +`Babel`_ can be invoked from ``setup.py`` +and its command line options specified in the accompanying +``setup.cfg`` via `Babel Distutils/Setuptools +Integration <http://babel.edgewall.org/wiki/Documentation/setup.html>`_. + +Comments must immediately precede a `gettext` message to be +extracted. In the following case the ``TRANSLATORS:`` comment would +not have been extracted: + +.. sourcecode:: mako + + <div id="name"> + ## TRANSLATORS: This is a proper name. See the gettext + ## manual, section Names. + Name: ${_('Francois Pinard')} + </div> + +See the `Babel User +Guide <http://babel.edgewall.org/wiki/Documentation/index.html>`_ +for more information. + +.. _babel: http://babel.edgewall.org/ + + +API Reference +============= + +.. autoclass:: mako.template.Template + :show-inheritance: + :members: + +.. autoclass:: mako.template.DefTemplate + :show-inheritance: + :members: + +.. autoclass:: mako.lookup.TemplateCollection + :show-inheritance: + :members: + +.. autoclass:: mako.lookup.TemplateLookup + :show-inheritance: + :members: + +.. autoclass:: mako.exceptions.RichTraceback + :show-inheritance: + + .. py:attribute:: error + + the exception instance. + + .. py:attribute:: message + + the exception error message as unicode. + + .. py:attribute:: source + + source code of the file where the error occurred. + If the error occurred within a compiled template, + this is the template source. + + .. py:attribute:: lineno + + line number where the error occurred. If the error + occurred within a compiled template, the line number + is adjusted to that of the template source. + + .. py:attribute:: records + + a list of 8-tuples containing the original + python traceback elements, plus the + filename, line number, source line, and full template source + for the traceline mapped back to its originating source + template, if any for that traceline (else the fields are ``None``). + + .. py:attribute:: reverse_records + + the list of records in reverse + traceback -- a list of 4-tuples, in the same format as a regular + python traceback, with template-corresponding + traceback records replacing the originals. + + .. py:attribute:: reverse_traceback + + the traceback list in reverse. + +.. autofunction:: mako.exceptions.html_error_template + +.. autofunction:: mako.exceptions.text_error_template diff --git a/examples/bench/basic.py b/examples/bench/basic.py new file mode 100644 index 0000000..fc36527 --- /dev/null +++ b/examples/bench/basic.py @@ -0,0 +1,224 @@ +# basic.py - basic benchmarks adapted from Genshi +# Copyright (C) 2006 Edgewall Software +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. The name of the author may not be used to endorse or promote +# products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from io import StringIO +import sys +import timeit + + +__all__ = [ + "mako", + "mako_inheritance", + "jinja2", + "jinja2_inheritance", + "cheetah", + "django", + "myghty", + "genshi", + "kid", +] + +# Templates content and constants +TITLE = "Just a test" +USER = "joe" +ITEMS = ["Number %d" % num for num in range(1, 15)] + + +def genshi(dirname, verbose=False): + from genshi.template import TemplateLoader + + loader = TemplateLoader([dirname], auto_reload=False) + template = loader.load("template.html") + + def render(): + data = dict(title=TITLE, user=USER, items=ITEMS) + return template.generate(**data).render("xhtml") + + if verbose: + print(render()) + return render + + +def myghty(dirname, verbose=False): + from myghty import interp + + interpreter = interp.Interpreter(component_root=dirname) + + def render(): + data = dict(title=TITLE, user=USER, items=ITEMS) + buffer = StringIO() + interpreter.execute( + "template.myt", request_args=data, out_buffer=buffer + ) + return buffer.getvalue() + + if verbose: + print(render()) + return render + + +def mako(dirname, verbose=False): + from mako.template import Template + from mako.lookup import TemplateLookup + + lookup = TemplateLookup(directories=[dirname], filesystem_checks=False) + template = lookup.get_template("template.html") + + def render(): + return template.render(title=TITLE, user=USER, list_items=ITEMS) + + if verbose: + print(template.code + " " + render()) + return render + + +mako_inheritance = mako + + +def jinja2(dirname, verbose=False): + from jinja2 import Environment, FileSystemLoader + + env = Environment(loader=FileSystemLoader(dirname)) + template = env.get_template("template.html") + + def render(): + return template.render(title=TITLE, user=USER, list_items=ITEMS) + + if verbose: + print(render()) + return render + + +jinja2_inheritance = jinja2 + + +def cheetah(dirname, verbose=False): + from Cheetah.Template import Template + + filename = os.path.join(dirname, "template.tmpl") + template = Template(file=filename) + + def render(): + template.__dict__.update( + {"title": TITLE, "user": USER, "list_items": ITEMS} + ) + return template.respond() + + if verbose: + print(dir(template)) + print(template.generatedModuleCode()) + print(render()) + return render + + +def django(dirname, verbose=False): + from django.conf import settings + + settings.configure(TEMPLATE_DIRS=[os.path.join(dirname, "templates")]) + from django import template, templatetags + from django.template import loader + + templatetags.__path__.append(os.path.join(dirname, "templatetags")) + tmpl = loader.get_template("template.html") + + def render(): + data = {"title": TITLE, "user": USER, "items": ITEMS} + return tmpl.render(template.Context(data)) + + if verbose: + print(render()) + return render + + +def kid(dirname, verbose=False): + import kid + + kid.path = kid.TemplatePath([dirname]) + template = kid.Template(file="template.kid") + + def render(): + template = kid.Template( + file="template.kid", title=TITLE, user=USER, items=ITEMS + ) + return template.serialize(output="xhtml") + + if verbose: + print(render()) + return render + + +def run(engines, number=2000, verbose=False): + basepath = os.path.abspath(os.path.dirname(__file__)) + for engine in engines: + dirname = os.path.join(basepath, engine) + if verbose: + print("%s:" % engine.capitalize()) + print("--------------------------------------------------------") + else: + sys.stdout.write("%s:" % engine.capitalize()) + t = timeit.Timer( + setup='from __main__ import %s; render = %s(r"%s", %s)' + % (engine, engine, dirname, verbose), + stmt="render()", + ) + + time = t.timeit(number=number) / number + if verbose: + print("--------------------------------------------------------") + print("%.2f ms" % (1000 * time)) + if verbose: + print("--------------------------------------------------------") + + +if __name__ == "__main__": + engines = [arg for arg in sys.argv[1:] if arg[0] != "-"] + if not engines: + engines = __all__ + + verbose = "-v" in sys.argv + + if "-p" in sys.argv: + try: + import hotshot, hotshot.stats + + prof = hotshot.Profile("template.prof") + benchtime = prof.runcall(run, engines, number=100, verbose=verbose) + stats = hotshot.stats.load("template.prof") + except ImportError: + import cProfile, pstats + + stmt = "run(%r, number=%r, verbose=%r)" % (engines, 1000, verbose) + cProfile.runctx(stmt, globals(), {}, "template.prof") + stats = pstats.Stats("template.prof") + stats.strip_dirs() + stats.sort_stats("time", "calls") + stats.print_stats() + else: + run(engines, verbose=verbose) diff --git a/examples/bench/cheetah/footer.tmpl b/examples/bench/cheetah/footer.tmpl new file mode 100644 index 0000000..1b00330 --- /dev/null +++ b/examples/bench/cheetah/footer.tmpl @@ -0,0 +1,2 @@ +<div id="footer"> +</div> diff --git a/examples/bench/cheetah/header.tmpl b/examples/bench/cheetah/header.tmpl new file mode 100644 index 0000000..432487f --- /dev/null +++ b/examples/bench/cheetah/header.tmpl @@ -0,0 +1,5 @@ +<div id="header"> + <h1>$title</h1> +</div> + + diff --git a/examples/bench/cheetah/template.tmpl b/examples/bench/cheetah/template.tmpl new file mode 100644 index 0000000..f1c2243 --- /dev/null +++ b/examples/bench/cheetah/template.tmpl @@ -0,0 +1,31 @@ +<!DOCTYPE html + PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" lang="en"> + <head> + <title>${title}</title> + </head> + <body> + + #def greeting(name) + <p>hello ${name}!</p> + #end def + + #include "cheetah/header.tmpl" + + $greeting($user) + $greeting('me') + $greeting('world') + + <h2>Loop</h2> + #if $list_items + <ul> + #for $list_item in $list_items + <li #if $list_item is $list_items[-1] then "class='last'" else ""#>$list_item</li> + #end for + </ul> + #end if + + #include "cheetah/footer.tmpl" + </body> +</html> diff --git a/examples/bench/django/templates/base.html b/examples/bench/django/templates/base.html new file mode 100644 index 0000000..8e64906 --- /dev/null +++ b/examples/bench/django/templates/base.html @@ -0,0 +1,14 @@ +<!DOCTYPE html + PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" lang="en"> + + {% block body %} + <div id="header"> + <h1>{{ title|escape }}</h1> + </div> + {{ block.super }} + <div id="footer"></div> + {% endblock %} + +</html> diff --git a/examples/bench/django/templates/template.html b/examples/bench/django/templates/template.html new file mode 100644 index 0000000..cae6c5e --- /dev/null +++ b/examples/bench/django/templates/template.html @@ -0,0 +1,22 @@ +{% extends "base.html" %} +{% load bench %} + +<head> + <title>${title|escape}</title> +</head> + +{% block body %} + <div>{% greeting user %}</div> + <div>{% greeting "me" %}</div> + <div>{% greeting "world" %}</div> + + <h2>Loop</h2> + {% if items %} + <ul> + {% for item in items %} + <li{% if forloop.last %} class="last"{% endif %}>{{ item }}</li> + {% endfor %} + </ul> + {% endif %} + +{% endblock %} diff --git a/examples/bench/django/templatetags/__init__.py b/examples/bench/django/templatetags/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/examples/bench/django/templatetags/__init__.py diff --git a/examples/bench/django/templatetags/bench.py b/examples/bench/django/templatetags/bench.py new file mode 100644 index 0000000..b5bfe26 --- /dev/null +++ b/examples/bench/django/templatetags/bench.py @@ -0,0 +1,11 @@ +from django.template import Library +from django.utils.html import escape + +register = Library() + + +def greeting(name): + return "Hello, %s!" % escape(name) + + +greeting = register.simple_tag(greeting) diff --git a/examples/bench/genshi/base.html b/examples/bench/genshi/base.html new file mode 100644 index 0000000..f53abf2 --- /dev/null +++ b/examples/bench/genshi/base.html @@ -0,0 +1,17 @@ +<html xmlns="http://www.w3.org/1999/xhtml" + xmlns:py="http://genshi.edgewall.org/" + py:strip=""> + + <p py:def="greeting(name)"> + Hello, ${name}! + </p> + + <body py:match="body"> + <div id="header"> + <h1>${title}</h1> + </div> + ${select('*')} + <div id="footer" /> + </body> + +</html> diff --git a/examples/bench/genshi/template.html b/examples/bench/genshi/template.html new file mode 100644 index 0000000..cdcc327 --- /dev/null +++ b/examples/bench/genshi/template.html @@ -0,0 +1,24 @@ +<!DOCTYPE html + PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" + xmlns:py="http://genshi.edgewall.org/" + xmlns:xi="http://www.w3.org/2001/XInclude" + lang="en"> + <xi:include href="base.html" /> + <head> + <title>${title}</title> + </head> + <body> + <div>${greeting(user)}</div> + <div>${greeting('me')}</div> + <div>${greeting('world')}</div> + + <h2>Loop</h2> + <ul py:if="items"> + <li py:for="idx, item in enumerate(items)" py:content="item" + class="${idx + 1 == len(items) and 'last' or None}" /> + </ul> + + </body> +</html> diff --git a/examples/bench/jinja2/footer.html b/examples/bench/jinja2/footer.html new file mode 100644 index 0000000..1b00330 --- /dev/null +++ b/examples/bench/jinja2/footer.html @@ -0,0 +1,2 @@ +<div id="footer"> +</div> diff --git a/examples/bench/jinja2/header.html b/examples/bench/jinja2/header.html new file mode 100644 index 0000000..ccb2fb7 --- /dev/null +++ b/examples/bench/jinja2/header.html @@ -0,0 +1,3 @@ +<div id="header"> + <h1>{{ title }}</h1> +</div> diff --git a/examples/bench/jinja2/template.html b/examples/bench/jinja2/template.html new file mode 100644 index 0000000..5965e0d --- /dev/null +++ b/examples/bench/jinja2/template.html @@ -0,0 +1,31 @@ +<!DOCTYPE html + PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" lang="en"> + <head> + <title>{{ title }}</title> + </head> + <body> + + {%- macro greeting(name) %} + <p>hello {{ name }}</p> + {%- endmacro %} + + {% include "header.html" %} + + {{ greeting(user) }} + {{ greeting('me') }} + {{ greeting('world') }} + + <h2>Loop</h2> + {%- if list_items %} + <ul> + {%- for list_item in list_items %} + <li {{ "class='last'" if loop.last else "" }}>{{ list_item }}</li> + {%- endfor %} + </ul> + {%- endif %} + + {% include "footer.html" %} + </body> +</html> diff --git a/examples/bench/jinja2_inheritance/base.html b/examples/bench/jinja2_inheritance/base.html new file mode 100644 index 0000000..c10a434 --- /dev/null +++ b/examples/bench/jinja2_inheritance/base.html @@ -0,0 +1,24 @@ +<!DOCTYPE html + PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" lang="en"> + <head> + <title>{{ title }}</title> + </head> + <body> + +{%- macro greeting(name) %} + <p>hello {{ name }}</p> +{%- endmacro %} + + <div id="header"> + <h1>{{ title }}</h1> + </div> + +{%- block body %}{%- endblock %} + + <div id="footer"> + </div> + + </body> +</html> diff --git a/examples/bench/jinja2_inheritance/template.html b/examples/bench/jinja2_inheritance/template.html new file mode 100644 index 0000000..7e1b781 --- /dev/null +++ b/examples/bench/jinja2_inheritance/template.html @@ -0,0 +1,15 @@ +{% extends "base.html" %} +{% block body %} + {{ greeting(user) }} + {{ greeting('me') }} + {{ greeting('world') }} + + <h2>Loop</h2> + {%- if list_items %} + <ul> + {%- for list_item in list_items %} + <li {{ "class='last'" if loop.last else ""}}>{{ list_item }}</li> + {%- endfor %} + </ul> + {%- endif %} +{% endblock body %} diff --git a/examples/bench/kid/base.kid b/examples/bench/kid/base.kid new file mode 100644 index 0000000..061e9dd --- /dev/null +++ b/examples/bench/kid/base.kid @@ -0,0 +1,15 @@ +<html xmlns="http://www.w3.org/1999/xhtml" + xmlns:py="http://purl.org/kid/ns#"> + + <p py:def="greeting(name)"> + Hello, ${name}! + </p> + + <body py:match="item.tag == '{http://www.w3.org/1999/xhtml}body'" py:strip=""> + <div id="header"> + <h1>${title}</h1> + </div> + ${item} + <div id="footer" /> + </body> +</html> diff --git a/examples/bench/kid/template.kid b/examples/bench/kid/template.kid new file mode 100644 index 0000000..7f79d7a --- /dev/null +++ b/examples/bench/kid/template.kid @@ -0,0 +1,22 @@ +<!DOCTYPE html + PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" + xmlns:py="http://purl.org/kid/ns#" + py:extends="'base.kid'" + lang="en"> + <head> + <title>${title}</title> + </head> + <body> + <div>${greeting(user)}</div> + <div>${greeting('me')}</div> + <div>${greeting('world')}</div> + + <h2>Loop</h2> + <ul py:if="items"> + <li py:for="idx, item in enumerate(items)" py:content="item" + class="${idx + 1 == len(items) and 'last' or None}" /> + </ul> + </body> +</html> diff --git a/examples/bench/mako/footer.html b/examples/bench/mako/footer.html new file mode 100644 index 0000000..1b00330 --- /dev/null +++ b/examples/bench/mako/footer.html @@ -0,0 +1,2 @@ +<div id="footer"> +</div> diff --git a/examples/bench/mako/header.html b/examples/bench/mako/header.html new file mode 100644 index 0000000..e4f3382 --- /dev/null +++ b/examples/bench/mako/header.html @@ -0,0 +1,5 @@ +<div id="header"> + <h1>${title}</h1> +</div> + + diff --git a/examples/bench/mako/template.html b/examples/bench/mako/template.html new file mode 100644 index 0000000..d5ded9a --- /dev/null +++ b/examples/bench/mako/template.html @@ -0,0 +1,31 @@ +<!DOCTYPE html + PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" lang="en"> + <head> + <title>${title}</title> + </head> + <body> + +<%def name="greeting(name)"> + <p>hello ${name}!</p> +</%def> + + <%include file="header.html"/> + + ${greeting(user)} + ${greeting('me')} + ${greeting('world')} + + <h2>Loop</h2> + % if list_items: + <ul> + % for i, list_item in enumerate(list_items): + <li ${i+1==len(list_items) and "class='last'" or ""}>${list_item}</li> + % endfor + </ul> + % endif + + <%include file="footer.html"/> + </body> +</html> diff --git a/examples/bench/mako_inheritance/base.html b/examples/bench/mako_inheritance/base.html new file mode 100644 index 0000000..84b2930 --- /dev/null +++ b/examples/bench/mako_inheritance/base.html @@ -0,0 +1,24 @@ +<!DOCTYPE html + PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" lang="en"> + <head> + <title>${title}</title> + </head> + <body> + +<%def name="greeting(name)"> + <p>hello ${name}!</p> +</%def> + +<div id="header"> + <h1>${title}</h1> +</div> + + ${self.body()} + + <div id="footer"> + </div> + + </body> +</html> diff --git a/examples/bench/mako_inheritance/template.html b/examples/bench/mako_inheritance/template.html new file mode 100644 index 0000000..7c53bf1 --- /dev/null +++ b/examples/bench/mako_inheritance/template.html @@ -0,0 +1,15 @@ +<%inherit file="base.html"/> + + ${parent.greeting(user)} + ${parent.greeting('me')} + ${parent.greeting('world')} + + <h2>Loop</h2> + % if list_items: + <ul> + % for list_item in list_items: + <li>${list_item}</li> + % endfor + </ul> + % endif + diff --git a/examples/bench/myghty/base.myt b/examples/bench/myghty/base.myt new file mode 100644 index 0000000..af0474a --- /dev/null +++ b/examples/bench/myghty/base.myt @@ -0,0 +1,29 @@ +<!DOCTYPE html + PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" lang="en"> +<%args scope="request"> + title +</%args> + +<& REQUEST:header &> + +<body> +<div id="header"> + <h1><% title %></h1> +</div> + +% m.call_next() + +<div id="footer"></div> + +</body> +</html> + + +<%method greeting> +<%args> + name +</%args> +Hello, <% name | h %> +</%method> diff --git a/examples/bench/myghty/template.myt b/examples/bench/myghty/template.myt new file mode 100644 index 0000000..138e0ae --- /dev/null +++ b/examples/bench/myghty/template.myt @@ -0,0 +1,30 @@ +<%flags>inherit="base.myt"</%flags> +<%args> + title + items + user +</%args> + +<%method header> + <%args scope="request"> + title + </%args> +<head> + <title><% title %></title> +</head> +</%method> + + <div><& base.myt:greeting, name=user &></div> + <div><& base.myt:greeting, name="me"&></div> + <div><& base.myt:greeting, name="world" &></div> + + <h2>Loop</h2> +%if items: + <ul> +% for i, item in enumerate(items): + <li <% i+1==len(items) and "class='last'" or ""%>><% item %></li> +% + </ul> +% + + diff --git a/examples/wsgi/htdocs/index.html b/examples/wsgi/htdocs/index.html new file mode 100644 index 0000000..ef2df4d --- /dev/null +++ b/examples/wsgi/htdocs/index.html @@ -0,0 +1,8 @@ +<% +%> + +<%inherit file="root.html"/> + +This is index.html + +c is ${c is not UNDEFINED and c or "undefined"} diff --git a/examples/wsgi/run_wsgi.py b/examples/wsgi/run_wsgi.py new file mode 100644 index 0000000..a51a463 --- /dev/null +++ b/examples/wsgi/run_wsgi.py @@ -0,0 +1,89 @@ +#!/usr/bin/python +import cgi +import mimetypes +import os +import posixpath +import re + +from mako import exceptions +from mako.lookup import TemplateLookup + +root = "./" +port = 8000 + +lookup = TemplateLookup( + directories=[root + "templates", root + "htdocs"], + filesystem_checks=True, + module_directory="./modules", + # even better would be to use 'charset' in start_response + output_encoding="ascii", + encoding_errors="replace", +) + + +def serve(environ, start_response): + """serves requests using the WSGI callable interface.""" + fieldstorage = cgi.FieldStorage( + fp=environ["wsgi.input"], environ=environ, keep_blank_values=True + ) + d = dict([(k, getfield(fieldstorage[k])) for k in fieldstorage]) + + uri = environ.get("PATH_INFO", "/") + if not uri: + uri = "/index.html" + else: + uri = re.sub(r"^/$", "/index.html", uri) + + if re.match(r".*\.html$", uri): + try: + template = lookup.get_template(uri) + except exceptions.TopLevelLookupException: + start_response("404 Not Found", []) + return [str.encode("Cant find template '%s'" % uri)] + + start_response("200 OK", [("Content-type", "text/html")]) + + try: + return [template.render(**d)] + except: + return [exceptions.html_error_template().render()] + else: + u = re.sub(r"^\/+", "", uri) + filename = os.path.join(root, u) + if os.path.isfile(filename): + start_response("200 OK", [("Content-type", guess_type(uri))]) + return [open(filename, "rb").read()] + else: + start_response("404 Not Found", []) + return [str.encode("File not found: '%s'" % filename)] + + +def getfield(f): + """convert values from cgi.Field objects to plain values.""" + if isinstance(f, list): + return [getfield(x) for x in f] + else: + return f.value + + +extensions_map = mimetypes.types_map.copy() + + +def guess_type(path): + """return a mimetype for the given path based on file extension.""" + base, ext = posixpath.splitext(path) + if ext in extensions_map: + return extensions_map[ext] + ext = ext.lower() + if ext in extensions_map: + return extensions_map[ext] + else: + return "text/html" + + +if __name__ == "__main__": + import wsgiref.simple_server + + server = wsgiref.simple_server.make_server("", port, serve) + print("Server listening on port %d" % port) + server.serve_forever() diff --git a/examples/wsgi/templates/root.html b/examples/wsgi/templates/root.html new file mode 100644 index 0000000..6b57fc3 --- /dev/null +++ b/examples/wsgi/templates/root.html @@ -0,0 +1,7 @@ +<html> + +<head><title>hi</title></head> +<body> + ${next.body()} +</body> +</html>
\ No newline at end of file diff --git a/mako/__init__.py b/mako/__init__.py new file mode 100644 index 0000000..25d577d --- /dev/null +++ b/mako/__init__.py @@ -0,0 +1,8 @@ +# mako/__init__.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + + +__version__ = "1.3.0" diff --git a/mako/_ast_util.py b/mako/_ast_util.py new file mode 100644 index 0000000..7dcdb7f --- /dev/null +++ b/mako/_ast_util.py @@ -0,0 +1,713 @@ +# mako/_ast_util.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +""" + ast + ~~~ + + This is a stripped down version of Armin Ronacher's ast module. + + :copyright: Copyright 2008 by Armin Ronacher. + :license: Python License. +""" + + +from _ast import Add +from _ast import And +from _ast import AST +from _ast import BitAnd +from _ast import BitOr +from _ast import BitXor +from _ast import Div +from _ast import Eq +from _ast import FloorDiv +from _ast import Gt +from _ast import GtE +from _ast import If +from _ast import In +from _ast import Invert +from _ast import Is +from _ast import IsNot +from _ast import LShift +from _ast import Lt +from _ast import LtE +from _ast import Mod +from _ast import Mult +from _ast import Name +from _ast import Not +from _ast import NotEq +from _ast import NotIn +from _ast import Or +from _ast import PyCF_ONLY_AST +from _ast import RShift +from _ast import Sub +from _ast import UAdd +from _ast import USub + + +BOOLOP_SYMBOLS = {And: "and", Or: "or"} + +BINOP_SYMBOLS = { + Add: "+", + Sub: "-", + Mult: "*", + Div: "/", + FloorDiv: "//", + Mod: "%", + LShift: "<<", + RShift: ">>", + BitOr: "|", + BitAnd: "&", + BitXor: "^", +} + +CMPOP_SYMBOLS = { + Eq: "==", + Gt: ">", + GtE: ">=", + In: "in", + Is: "is", + IsNot: "is not", + Lt: "<", + LtE: "<=", + NotEq: "!=", + NotIn: "not in", +} + +UNARYOP_SYMBOLS = {Invert: "~", Not: "not", UAdd: "+", USub: "-"} + +ALL_SYMBOLS = {} +ALL_SYMBOLS.update(BOOLOP_SYMBOLS) +ALL_SYMBOLS.update(BINOP_SYMBOLS) +ALL_SYMBOLS.update(CMPOP_SYMBOLS) +ALL_SYMBOLS.update(UNARYOP_SYMBOLS) + + +def parse(expr, filename="<unknown>", mode="exec"): + """Parse an expression into an AST node.""" + return compile(expr, filename, mode, PyCF_ONLY_AST) + + +def iter_fields(node): + """Iterate over all fields of a node, only yielding existing fields.""" + + for field in node._fields: + try: + yield field, getattr(node, field) + except AttributeError: + pass + + +class NodeVisitor: + + """ + Walks the abstract syntax tree and call visitor functions for every node + found. The visitor functions may return values which will be forwarded + by the `visit` method. + + Per default the visitor functions for the nodes are ``'visit_'`` + + class name of the node. So a `TryFinally` node visit function would + be `visit_TryFinally`. This behavior can be changed by overriding + the `get_visitor` function. If no visitor function exists for a node + (return value `None`) the `generic_visit` visitor is used instead. + + Don't use the `NodeVisitor` if you want to apply changes to nodes during + traversing. For this a special visitor exists (`NodeTransformer`) that + allows modifications. + """ + + def get_visitor(self, node): + """ + Return the visitor function for this node or `None` if no visitor + exists for this node. In that case the generic visit function is + used instead. + """ + method = "visit_" + node.__class__.__name__ + return getattr(self, method, None) + + def visit(self, node): + """Visit a node.""" + f = self.get_visitor(node) + if f is not None: + return f(node) + return self.generic_visit(node) + + def generic_visit(self, node): + """Called if no explicit visitor function exists for a node.""" + for field, value in iter_fields(node): + if isinstance(value, list): + for item in value: + if isinstance(item, AST): + self.visit(item) + elif isinstance(value, AST): + self.visit(value) + + +class NodeTransformer(NodeVisitor): + + """ + Walks the abstract syntax tree and allows modifications of nodes. + + The `NodeTransformer` will walk the AST and use the return value of the + visitor functions to replace or remove the old node. If the return + value of the visitor function is `None` the node will be removed + from the previous location otherwise it's replaced with the return + value. The return value may be the original node in which case no + replacement takes place. + + Here an example transformer that rewrites all `foo` to `data['foo']`:: + + class RewriteName(NodeTransformer): + + def visit_Name(self, node): + return copy_location(Subscript( + value=Name(id='data', ctx=Load()), + slice=Index(value=Str(s=node.id)), + ctx=node.ctx + ), node) + + Keep in mind that if the node you're operating on has child nodes + you must either transform the child nodes yourself or call the generic + visit function for the node first. + + Nodes that were part of a collection of statements (that applies to + all statement nodes) may also return a list of nodes rather than just + a single node. + + Usually you use the transformer like this:: + + node = YourTransformer().visit(node) + """ + + def generic_visit(self, node): + for field, old_value in iter_fields(node): + old_value = getattr(node, field, None) + if isinstance(old_value, list): + new_values = [] + for value in old_value: + if isinstance(value, AST): + value = self.visit(value) + if value is None: + continue + elif not isinstance(value, AST): + new_values.extend(value) + continue + new_values.append(value) + old_value[:] = new_values + elif isinstance(old_value, AST): + new_node = self.visit(old_value) + if new_node is None: + delattr(node, field) + else: + setattr(node, field, new_node) + return node + + +class SourceGenerator(NodeVisitor): + + """ + This visitor is able to transform a well formed syntax tree into python + sourcecode. For more details have a look at the docstring of the + `node_to_source` function. + """ + + def __init__(self, indent_with): + self.result = [] + self.indent_with = indent_with + self.indentation = 0 + self.new_lines = 0 + + def write(self, x): + if self.new_lines: + if self.result: + self.result.append("\n" * self.new_lines) + self.result.append(self.indent_with * self.indentation) + self.new_lines = 0 + self.result.append(x) + + def newline(self, n=1): + self.new_lines = max(self.new_lines, n) + + def body(self, statements): + self.new_line = True + self.indentation += 1 + for stmt in statements: + self.visit(stmt) + self.indentation -= 1 + + def body_or_else(self, node): + self.body(node.body) + if node.orelse: + self.newline() + self.write("else:") + self.body(node.orelse) + + def signature(self, node): + want_comma = [] + + def write_comma(): + if want_comma: + self.write(", ") + else: + want_comma.append(True) + + padding = [None] * (len(node.args) - len(node.defaults)) + for arg, default in zip(node.args, padding + node.defaults): + write_comma() + self.visit(arg) + if default is not None: + self.write("=") + self.visit(default) + if node.vararg is not None: + write_comma() + self.write("*" + node.vararg.arg) + if node.kwarg is not None: + write_comma() + self.write("**" + node.kwarg.arg) + + def decorators(self, node): + for decorator in node.decorator_list: + self.newline() + self.write("@") + self.visit(decorator) + + # Statements + + def visit_Assign(self, node): + self.newline() + for idx, target in enumerate(node.targets): + if idx: + self.write(", ") + self.visit(target) + self.write(" = ") + self.visit(node.value) + + def visit_AugAssign(self, node): + self.newline() + self.visit(node.target) + self.write(BINOP_SYMBOLS[type(node.op)] + "=") + self.visit(node.value) + + def visit_ImportFrom(self, node): + self.newline() + self.write("from %s%s import " % ("." * node.level, node.module)) + for idx, item in enumerate(node.names): + if idx: + self.write(", ") + self.write(item) + + def visit_Import(self, node): + self.newline() + for item in node.names: + self.write("import ") + self.visit(item) + + def visit_Expr(self, node): + self.newline() + self.generic_visit(node) + + def visit_FunctionDef(self, node): + self.newline(n=2) + self.decorators(node) + self.newline() + self.write("def %s(" % node.name) + self.signature(node.args) + self.write("):") + self.body(node.body) + + def visit_ClassDef(self, node): + have_args = [] + + def paren_or_comma(): + if have_args: + self.write(", ") + else: + have_args.append(True) + self.write("(") + + self.newline(n=3) + self.decorators(node) + self.newline() + self.write("class %s" % node.name) + for base in node.bases: + paren_or_comma() + self.visit(base) + # XXX: the if here is used to keep this module compatible + # with python 2.6. + if hasattr(node, "keywords"): + for keyword in node.keywords: + paren_or_comma() + self.write(keyword.arg + "=") + self.visit(keyword.value) + if getattr(node, "starargs", None): + paren_or_comma() + self.write("*") + self.visit(node.starargs) + if getattr(node, "kwargs", None): + paren_or_comma() + self.write("**") + self.visit(node.kwargs) + self.write(have_args and "):" or ":") + self.body(node.body) + + def visit_If(self, node): + self.newline() + self.write("if ") + self.visit(node.test) + self.write(":") + self.body(node.body) + while True: + else_ = node.orelse + if len(else_) == 1 and isinstance(else_[0], If): + node = else_[0] + self.newline() + self.write("elif ") + self.visit(node.test) + self.write(":") + self.body(node.body) + else: + self.newline() + self.write("else:") + self.body(else_) + break + + def visit_For(self, node): + self.newline() + self.write("for ") + self.visit(node.target) + self.write(" in ") + self.visit(node.iter) + self.write(":") + self.body_or_else(node) + + def visit_While(self, node): + self.newline() + self.write("while ") + self.visit(node.test) + self.write(":") + self.body_or_else(node) + + def visit_With(self, node): + self.newline() + self.write("with ") + self.visit(node.context_expr) + if node.optional_vars is not None: + self.write(" as ") + self.visit(node.optional_vars) + self.write(":") + self.body(node.body) + + def visit_Pass(self, node): + self.newline() + self.write("pass") + + def visit_Print(self, node): + # XXX: python 2.6 only + self.newline() + self.write("print ") + want_comma = False + if node.dest is not None: + self.write(" >> ") + self.visit(node.dest) + want_comma = True + for value in node.values: + if want_comma: + self.write(", ") + self.visit(value) + want_comma = True + if not node.nl: + self.write(",") + + def visit_Delete(self, node): + self.newline() + self.write("del ") + for idx, target in enumerate(node): + if idx: + self.write(", ") + self.visit(target) + + def visit_TryExcept(self, node): + self.newline() + self.write("try:") + self.body(node.body) + for handler in node.handlers: + self.visit(handler) + + def visit_TryFinally(self, node): + self.newline() + self.write("try:") + self.body(node.body) + self.newline() + self.write("finally:") + self.body(node.finalbody) + + def visit_Global(self, node): + self.newline() + self.write("global " + ", ".join(node.names)) + + def visit_Nonlocal(self, node): + self.newline() + self.write("nonlocal " + ", ".join(node.names)) + + def visit_Return(self, node): + self.newline() + self.write("return ") + self.visit(node.value) + + def visit_Break(self, node): + self.newline() + self.write("break") + + def visit_Continue(self, node): + self.newline() + self.write("continue") + + def visit_Raise(self, node): + # XXX: Python 2.6 / 3.0 compatibility + self.newline() + self.write("raise") + if hasattr(node, "exc") and node.exc is not None: + self.write(" ") + self.visit(node.exc) + if node.cause is not None: + self.write(" from ") + self.visit(node.cause) + elif hasattr(node, "type") and node.type is not None: + self.visit(node.type) + if node.inst is not None: + self.write(", ") + self.visit(node.inst) + if node.tback is not None: + self.write(", ") + self.visit(node.tback) + + # Expressions + + def visit_Attribute(self, node): + self.visit(node.value) + self.write("." + node.attr) + + def visit_Call(self, node): + want_comma = [] + + def write_comma(): + if want_comma: + self.write(", ") + else: + want_comma.append(True) + + self.visit(node.func) + self.write("(") + for arg in node.args: + write_comma() + self.visit(arg) + for keyword in node.keywords: + write_comma() + self.write(keyword.arg + "=") + self.visit(keyword.value) + if getattr(node, "starargs", None): + write_comma() + self.write("*") + self.visit(node.starargs) + if getattr(node, "kwargs", None): + write_comma() + self.write("**") + self.visit(node.kwargs) + self.write(")") + + def visit_Name(self, node): + self.write(node.id) + + def visit_NameConstant(self, node): + self.write(str(node.value)) + + def visit_arg(self, node): + self.write(node.arg) + + def visit_Str(self, node): + self.write(repr(node.s)) + + def visit_Bytes(self, node): + self.write(repr(node.s)) + + def visit_Num(self, node): + self.write(repr(node.n)) + + # newly needed in Python 3.8 + def visit_Constant(self, node): + self.write(repr(node.value)) + + def visit_Tuple(self, node): + self.write("(") + idx = -1 + for idx, item in enumerate(node.elts): + if idx: + self.write(", ") + self.visit(item) + self.write(idx and ")" or ",)") + + def sequence_visit(left, right): + def visit(self, node): + self.write(left) + for idx, item in enumerate(node.elts): + if idx: + self.write(", ") + self.visit(item) + self.write(right) + + return visit + + visit_List = sequence_visit("[", "]") + visit_Set = sequence_visit("{", "}") + del sequence_visit + + def visit_Dict(self, node): + self.write("{") + for idx, (key, value) in enumerate(zip(node.keys, node.values)): + if idx: + self.write(", ") + self.visit(key) + self.write(": ") + self.visit(value) + self.write("}") + + def visit_BinOp(self, node): + self.write("(") + self.visit(node.left) + self.write(" %s " % BINOP_SYMBOLS[type(node.op)]) + self.visit(node.right) + self.write(")") + + def visit_BoolOp(self, node): + self.write("(") + for idx, value in enumerate(node.values): + if idx: + self.write(" %s " % BOOLOP_SYMBOLS[type(node.op)]) + self.visit(value) + self.write(")") + + def visit_Compare(self, node): + self.write("(") + self.visit(node.left) + for op, right in zip(node.ops, node.comparators): + self.write(" %s " % CMPOP_SYMBOLS[type(op)]) + self.visit(right) + self.write(")") + + def visit_UnaryOp(self, node): + self.write("(") + op = UNARYOP_SYMBOLS[type(node.op)] + self.write(op) + if op == "not": + self.write(" ") + self.visit(node.operand) + self.write(")") + + def visit_Subscript(self, node): + self.visit(node.value) + self.write("[") + self.visit(node.slice) + self.write("]") + + def visit_Slice(self, node): + if node.lower is not None: + self.visit(node.lower) + self.write(":") + if node.upper is not None: + self.visit(node.upper) + if node.step is not None: + self.write(":") + if not (isinstance(node.step, Name) and node.step.id == "None"): + self.visit(node.step) + + def visit_ExtSlice(self, node): + for idx, item in node.dims: + if idx: + self.write(", ") + self.visit(item) + + def visit_Yield(self, node): + self.write("yield ") + self.visit(node.value) + + def visit_Lambda(self, node): + self.write("lambda ") + self.signature(node.args) + self.write(": ") + self.visit(node.body) + + def visit_Ellipsis(self, node): + self.write("Ellipsis") + + def generator_visit(left, right): + def visit(self, node): + self.write(left) + self.visit(node.elt) + for comprehension in node.generators: + self.visit(comprehension) + self.write(right) + + return visit + + visit_ListComp = generator_visit("[", "]") + visit_GeneratorExp = generator_visit("(", ")") + visit_SetComp = generator_visit("{", "}") + del generator_visit + + def visit_DictComp(self, node): + self.write("{") + self.visit(node.key) + self.write(": ") + self.visit(node.value) + for comprehension in node.generators: + self.visit(comprehension) + self.write("}") + + def visit_IfExp(self, node): + self.visit(node.body) + self.write(" if ") + self.visit(node.test) + self.write(" else ") + self.visit(node.orelse) + + def visit_Starred(self, node): + self.write("*") + self.visit(node.value) + + def visit_Repr(self, node): + # XXX: python 2.6 only + self.write("`") + self.visit(node.value) + self.write("`") + + # Helper Nodes + + def visit_alias(self, node): + self.write(node.name) + if node.asname is not None: + self.write(" as " + node.asname) + + def visit_comprehension(self, node): + self.write(" for ") + self.visit(node.target) + self.write(" in ") + self.visit(node.iter) + if node.ifs: + for if_ in node.ifs: + self.write(" if ") + self.visit(if_) + + def visit_excepthandler(self, node): + self.newline() + self.write("except") + if node.type is not None: + self.write(" ") + self.visit(node.type) + if node.name is not None: + self.write(" as ") + self.visit(node.name) + self.write(":") + self.body(node.body) diff --git a/mako/ast.py b/mako/ast.py new file mode 100644 index 0000000..3076e2e --- /dev/null +++ b/mako/ast.py @@ -0,0 +1,202 @@ +# mako/ast.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""utilities for analyzing expressions and blocks of Python +code, as well as generating Python from AST nodes""" + +import re + +from mako import exceptions +from mako import pyparser + + +class PythonCode: + + """represents information about a string containing Python code""" + + def __init__(self, code, **exception_kwargs): + self.code = code + + # represents all identifiers which are assigned to at some point in + # the code + self.declared_identifiers = set() + + # represents all identifiers which are referenced before their + # assignment, if any + self.undeclared_identifiers = set() + + # note that an identifier can be in both the undeclared and declared + # lists. + + # using AST to parse instead of using code.co_varnames, + # code.co_names has several advantages: + # - we can locate an identifier as "undeclared" even if + # its declared later in the same block of code + # - AST is less likely to break with version changes + # (for example, the behavior of co_names changed a little bit + # in python version 2.5) + if isinstance(code, str): + expr = pyparser.parse(code.lstrip(), "exec", **exception_kwargs) + else: + expr = code + + f = pyparser.FindIdentifiers(self, **exception_kwargs) + f.visit(expr) + + +class ArgumentList: + + """parses a fragment of code as a comma-separated list of expressions""" + + def __init__(self, code, **exception_kwargs): + self.codeargs = [] + self.args = [] + self.declared_identifiers = set() + self.undeclared_identifiers = set() + if isinstance(code, str): + if re.match(r"\S", code) and not re.match(r",\s*$", code): + # if theres text and no trailing comma, insure its parsed + # as a tuple by adding a trailing comma + code += "," + expr = pyparser.parse(code, "exec", **exception_kwargs) + else: + expr = code + + f = pyparser.FindTuple(self, PythonCode, **exception_kwargs) + f.visit(expr) + + +class PythonFragment(PythonCode): + + """extends PythonCode to provide identifier lookups in partial control + statements + + e.g.:: + + for x in 5: + elif y==9: + except (MyException, e): + + """ + + def __init__(self, code, **exception_kwargs): + m = re.match(r"^(\w+)(?:\s+(.*?))?:\s*(#|$)", code.strip(), re.S) + if not m: + raise exceptions.CompileException( + "Fragment '%s' is not a partial control statement" % code, + **exception_kwargs, + ) + if m.group(3): + code = code[: m.start(3)] + (keyword, expr) = m.group(1, 2) + if keyword in ["for", "if", "while"]: + code = code + "pass" + elif keyword == "try": + code = code + "pass\nexcept:pass" + elif keyword in ["elif", "else"]: + code = "if False:pass\n" + code + "pass" + elif keyword == "except": + code = "try:pass\n" + code + "pass" + elif keyword == "with": + code = code + "pass" + else: + raise exceptions.CompileException( + "Unsupported control keyword: '%s'" % keyword, + **exception_kwargs, + ) + super().__init__(code, **exception_kwargs) + + +class FunctionDecl: + + """function declaration""" + + def __init__(self, code, allow_kwargs=True, **exception_kwargs): + self.code = code + expr = pyparser.parse(code, "exec", **exception_kwargs) + + f = pyparser.ParseFunc(self, **exception_kwargs) + f.visit(expr) + if not hasattr(self, "funcname"): + raise exceptions.CompileException( + "Code '%s' is not a function declaration" % code, + **exception_kwargs, + ) + if not allow_kwargs and self.kwargs: + raise exceptions.CompileException( + "'**%s' keyword argument not allowed here" + % self.kwargnames[-1], + **exception_kwargs, + ) + + def get_argument_expressions(self, as_call=False): + """Return the argument declarations of this FunctionDecl as a printable + list. + + By default the return value is appropriate for writing in a ``def``; + set `as_call` to true to build arguments to be passed to the function + instead (assuming locals with the same names as the arguments exist). + """ + + namedecls = [] + + # Build in reverse order, since defaults and slurpy args come last + argnames = self.argnames[::-1] + kwargnames = self.kwargnames[::-1] + defaults = self.defaults[::-1] + kwdefaults = self.kwdefaults[::-1] + + # Named arguments + if self.kwargs: + namedecls.append("**" + kwargnames.pop(0)) + + for name in kwargnames: + # Keyword-only arguments must always be used by name, so even if + # this is a call, print out `foo=foo` + if as_call: + namedecls.append("%s=%s" % (name, name)) + elif kwdefaults: + default = kwdefaults.pop(0) + if default is None: + # The AST always gives kwargs a default, since you can do + # `def foo(*, a=1, b, c=3)` + namedecls.append(name) + else: + namedecls.append( + "%s=%s" + % (name, pyparser.ExpressionGenerator(default).value()) + ) + else: + namedecls.append(name) + + # Positional arguments + if self.varargs: + namedecls.append("*" + argnames.pop(0)) + + for name in argnames: + if as_call or not defaults: + namedecls.append(name) + else: + default = defaults.pop(0) + namedecls.append( + "%s=%s" + % (name, pyparser.ExpressionGenerator(default).value()) + ) + + namedecls.reverse() + return namedecls + + @property + def allargnames(self): + return tuple(self.argnames) + tuple(self.kwargnames) + + +class FunctionArgs(FunctionDecl): + + """the argument portion of a function declaration""" + + def __init__(self, code, **kwargs): + super().__init__("def ANON(%s):pass" % code, **kwargs) diff --git a/mako/cache.py b/mako/cache.py new file mode 100644 index 0000000..b4e32d0 --- /dev/null +++ b/mako/cache.py @@ -0,0 +1,239 @@ +# mako/cache.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +from mako import util + +_cache_plugins = util.PluginLoader("mako.cache") + +register_plugin = _cache_plugins.register +register_plugin("beaker", "mako.ext.beaker_cache", "BeakerCacheImpl") + + +class Cache: + + """Represents a data content cache made available to the module + space of a specific :class:`.Template` object. + + .. versionadded:: 0.6 + :class:`.Cache` by itself is mostly a + container for a :class:`.CacheImpl` object, which implements + a fixed API to provide caching services; specific subclasses exist to + implement different + caching strategies. Mako includes a backend that works with + the Beaker caching system. Beaker itself then supports + a number of backends (i.e. file, memory, memcached, etc.) + + The construction of a :class:`.Cache` is part of the mechanics + of a :class:`.Template`, and programmatic access to this + cache is typically via the :attr:`.Template.cache` attribute. + + """ + + impl = None + """Provide the :class:`.CacheImpl` in use by this :class:`.Cache`. + + This accessor allows a :class:`.CacheImpl` with additional + methods beyond that of :class:`.Cache` to be used programmatically. + + """ + + id = None + """Return the 'id' that identifies this cache. + + This is a value that should be globally unique to the + :class:`.Template` associated with this cache, and can + be used by a caching system to name a local container + for data specific to this template. + + """ + + starttime = None + """Epochal time value for when the owning :class:`.Template` was + first compiled. + + A cache implementation may wish to invalidate data earlier than + this timestamp; this has the effect of the cache for a specific + :class:`.Template` starting clean any time the :class:`.Template` + is recompiled, such as when the original template file changed on + the filesystem. + + """ + + def __init__(self, template, *args): + # check for a stale template calling the + # constructor + if isinstance(template, str) and args: + return + self.template = template + self.id = template.module.__name__ + self.starttime = template.module._modified_time + self._def_regions = {} + self.impl = self._load_impl(self.template.cache_impl) + + def _load_impl(self, name): + return _cache_plugins.load(name)(self) + + def get_or_create(self, key, creation_function, **kw): + """Retrieve a value from the cache, using the given creation function + to generate a new value.""" + + return self._ctx_get_or_create(key, creation_function, None, **kw) + + def _ctx_get_or_create(self, key, creation_function, context, **kw): + """Retrieve a value from the cache, using the given creation function + to generate a new value.""" + + if not self.template.cache_enabled: + return creation_function() + + return self.impl.get_or_create( + key, creation_function, **self._get_cache_kw(kw, context) + ) + + def set(self, key, value, **kw): + r"""Place a value in the cache. + + :param key: the value's key. + :param value: the value. + :param \**kw: cache configuration arguments. + + """ + + self.impl.set(key, value, **self._get_cache_kw(kw, None)) + + put = set + """A synonym for :meth:`.Cache.set`. + + This is here for backwards compatibility. + + """ + + def get(self, key, **kw): + r"""Retrieve a value from the cache. + + :param key: the value's key. + :param \**kw: cache configuration arguments. The + backend is configured using these arguments upon first request. + Subsequent requests that use the same series of configuration + values will use that same backend. + + """ + return self.impl.get(key, **self._get_cache_kw(kw, None)) + + def invalidate(self, key, **kw): + r"""Invalidate a value in the cache. + + :param key: the value's key. + :param \**kw: cache configuration arguments. The + backend is configured using these arguments upon first request. + Subsequent requests that use the same series of configuration + values will use that same backend. + + """ + self.impl.invalidate(key, **self._get_cache_kw(kw, None)) + + def invalidate_body(self): + """Invalidate the cached content of the "body" method for this + template. + + """ + self.invalidate("render_body", __M_defname="render_body") + + def invalidate_def(self, name): + """Invalidate the cached content of a particular ``<%def>`` within this + template. + + """ + + self.invalidate("render_%s" % name, __M_defname="render_%s" % name) + + def invalidate_closure(self, name): + """Invalidate a nested ``<%def>`` within this template. + + Caching of nested defs is a blunt tool as there is no + management of scope -- nested defs that use cache tags + need to have names unique of all other nested defs in the + template, else their content will be overwritten by + each other. + + """ + + self.invalidate(name, __M_defname=name) + + def _get_cache_kw(self, kw, context): + defname = kw.pop("__M_defname", None) + if not defname: + tmpl_kw = self.template.cache_args.copy() + tmpl_kw.update(kw) + elif defname in self._def_regions: + tmpl_kw = self._def_regions[defname] + else: + tmpl_kw = self.template.cache_args.copy() + tmpl_kw.update(kw) + self._def_regions[defname] = tmpl_kw + if context and self.impl.pass_context: + tmpl_kw = tmpl_kw.copy() + tmpl_kw.setdefault("context", context) + return tmpl_kw + + +class CacheImpl: + + """Provide a cache implementation for use by :class:`.Cache`.""" + + def __init__(self, cache): + self.cache = cache + + pass_context = False + """If ``True``, the :class:`.Context` will be passed to + :meth:`get_or_create <.CacheImpl.get_or_create>` as the name ``'context'``. + """ + + def get_or_create(self, key, creation_function, **kw): + r"""Retrieve a value from the cache, using the given creation function + to generate a new value. + + This function *must* return a value, either from + the cache, or via the given creation function. + If the creation function is called, the newly + created value should be populated into the cache + under the given key before being returned. + + :param key: the value's key. + :param creation_function: function that when called generates + a new value. + :param \**kw: cache configuration arguments. + + """ + raise NotImplementedError() + + def set(self, key, value, **kw): + r"""Place a value in the cache. + + :param key: the value's key. + :param value: the value. + :param \**kw: cache configuration arguments. + + """ + raise NotImplementedError() + + def get(self, key, **kw): + r"""Retrieve a value from the cache. + + :param key: the value's key. + :param \**kw: cache configuration arguments. + + """ + raise NotImplementedError() + + def invalidate(self, key, **kw): + r"""Invalidate a value in the cache. + + :param key: the value's key. + :param \**kw: cache configuration arguments. + + """ + raise NotImplementedError() diff --git a/mako/cmd.py b/mako/cmd.py new file mode 100755 index 0000000..6bb8197 --- /dev/null +++ b/mako/cmd.py @@ -0,0 +1,99 @@ +# mako/cmd.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php +from argparse import ArgumentParser +from os.path import dirname +from os.path import isfile +import sys + +from mako import exceptions +from mako.lookup import TemplateLookup +from mako.template import Template + + +def varsplit(var): + if "=" not in var: + return (var, "") + return var.split("=", 1) + + +def _exit(): + sys.stderr.write(exceptions.text_error_template().render()) + sys.exit(1) + + +def cmdline(argv=None): + parser = ArgumentParser() + parser.add_argument( + "--var", + default=[], + action="append", + help="variable (can be used multiple times, use name=value)", + ) + parser.add_argument( + "--template-dir", + default=[], + action="append", + help="Directory to use for template lookup (multiple " + "directories may be provided). If not given then if the " + "template is read from stdin, the value defaults to be " + "the current directory, otherwise it defaults to be the " + "parent directory of the file provided.", + ) + parser.add_argument( + "--output-encoding", default=None, help="force output encoding" + ) + parser.add_argument( + "--output-file", + default=None, + help="Write to file upon successful render instead of stdout", + ) + parser.add_argument("input", nargs="?", default="-") + + options = parser.parse_args(argv) + + output_encoding = options.output_encoding + output_file = options.output_file + + if options.input == "-": + lookup_dirs = options.template_dir or ["."] + lookup = TemplateLookup(lookup_dirs) + try: + template = Template( + sys.stdin.read(), + lookup=lookup, + output_encoding=output_encoding, + ) + except: + _exit() + else: + filename = options.input + if not isfile(filename): + raise SystemExit("error: can't find %s" % filename) + lookup_dirs = options.template_dir or [dirname(filename)] + lookup = TemplateLookup(lookup_dirs) + try: + template = Template( + filename=filename, + lookup=lookup, + output_encoding=output_encoding, + ) + except: + _exit() + + kw = dict(varsplit(var) for var in options.var) + try: + rendered = template.render(**kw) + except: + _exit() + else: + if output_file: + open(output_file, "wt", encoding=output_encoding).write(rendered) + else: + sys.stdout.write(rendered) + + +if __name__ == "__main__": + cmdline() diff --git a/mako/codegen.py b/mako/codegen.py new file mode 100644 index 0000000..a516d3b --- /dev/null +++ b/mako/codegen.py @@ -0,0 +1,1307 @@ +# mako/codegen.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""provides functionality for rendering a parsetree constructing into module +source code.""" + +import json +import re +import time + +from mako import ast +from mako import exceptions +from mako import filters +from mako import parsetree +from mako import util +from mako.pygen import PythonPrinter + + +MAGIC_NUMBER = 10 + +# names which are hardwired into the +# template and are not accessed via the +# context itself +TOPLEVEL_DECLARED = {"UNDEFINED", "STOP_RENDERING"} +RESERVED_NAMES = {"context", "loop"}.union(TOPLEVEL_DECLARED) + + +def compile( # noqa + node, + uri, + filename=None, + default_filters=None, + buffer_filters=None, + imports=None, + future_imports=None, + source_encoding=None, + generate_magic_comment=True, + strict_undefined=False, + enable_loop=True, + reserved_names=frozenset(), +): + """Generate module source code given a parsetree node, + uri, and optional source filename""" + + buf = util.FastEncodingBuffer() + + printer = PythonPrinter(buf) + _GenerateRenderMethod( + printer, + _CompileContext( + uri, + filename, + default_filters, + buffer_filters, + imports, + future_imports, + source_encoding, + generate_magic_comment, + strict_undefined, + enable_loop, + reserved_names, + ), + node, + ) + return buf.getvalue() + + +class _CompileContext: + def __init__( + self, + uri, + filename, + default_filters, + buffer_filters, + imports, + future_imports, + source_encoding, + generate_magic_comment, + strict_undefined, + enable_loop, + reserved_names, + ): + self.uri = uri + self.filename = filename + self.default_filters = default_filters + self.buffer_filters = buffer_filters + self.imports = imports + self.future_imports = future_imports + self.source_encoding = source_encoding + self.generate_magic_comment = generate_magic_comment + self.strict_undefined = strict_undefined + self.enable_loop = enable_loop + self.reserved_names = reserved_names + + +class _GenerateRenderMethod: + + """A template visitor object which generates the + full module source for a template. + + """ + + def __init__(self, printer, compiler, node): + self.printer = printer + self.compiler = compiler + self.node = node + self.identifier_stack = [None] + self.in_def = isinstance(node, (parsetree.DefTag, parsetree.BlockTag)) + + if self.in_def: + name = "render_%s" % node.funcname + args = node.get_argument_expressions() + filtered = len(node.filter_args.args) > 0 + buffered = eval(node.attributes.get("buffered", "False")) + cached = eval(node.attributes.get("cached", "False")) + defs = None + pagetag = None + if node.is_block and not node.is_anonymous: + args += ["**pageargs"] + else: + defs = self.write_toplevel() + pagetag = self.compiler.pagetag + name = "render_body" + if pagetag is not None: + args = pagetag.body_decl.get_argument_expressions() + if not pagetag.body_decl.kwargs: + args += ["**pageargs"] + cached = eval(pagetag.attributes.get("cached", "False")) + self.compiler.enable_loop = self.compiler.enable_loop or eval( + pagetag.attributes.get("enable_loop", "False") + ) + else: + args = ["**pageargs"] + cached = False + buffered = filtered = False + if args is None: + args = ["context"] + else: + args = [a for a in ["context"] + args] + + self.write_render_callable( + pagetag or node, name, args, buffered, filtered, cached + ) + + if defs is not None: + for node in defs: + _GenerateRenderMethod(printer, compiler, node) + + if not self.in_def: + self.write_metadata_struct() + + def write_metadata_struct(self): + self.printer.source_map[self.printer.lineno] = max( + self.printer.source_map + ) + struct = { + "filename": self.compiler.filename, + "uri": self.compiler.uri, + "source_encoding": self.compiler.source_encoding, + "line_map": self.printer.source_map, + } + self.printer.writelines( + '"""', + "__M_BEGIN_METADATA", + json.dumps(struct), + "__M_END_METADATA\n" '"""', + ) + + @property + def identifiers(self): + return self.identifier_stack[-1] + + def write_toplevel(self): + """Traverse a template structure for module-level directives and + generate the start of module-level code. + + """ + inherit = [] + namespaces = {} + module_code = [] + + self.compiler.pagetag = None + + class FindTopLevel: + def visitInheritTag(s, node): + inherit.append(node) + + def visitNamespaceTag(s, node): + namespaces[node.name] = node + + def visitPageTag(s, node): + self.compiler.pagetag = node + + def visitCode(s, node): + if node.ismodule: + module_code.append(node) + + f = FindTopLevel() + for n in self.node.nodes: + n.accept_visitor(f) + + self.compiler.namespaces = namespaces + + module_ident = set() + for n in module_code: + module_ident = module_ident.union(n.declared_identifiers()) + + module_identifiers = _Identifiers(self.compiler) + module_identifiers.declared = module_ident + + # module-level names, python code + if ( + self.compiler.generate_magic_comment + and self.compiler.source_encoding + ): + self.printer.writeline( + "# -*- coding:%s -*-" % self.compiler.source_encoding + ) + + if self.compiler.future_imports: + self.printer.writeline( + "from __future__ import %s" + % (", ".join(self.compiler.future_imports),) + ) + self.printer.writeline("from mako import runtime, filters, cache") + self.printer.writeline("UNDEFINED = runtime.UNDEFINED") + self.printer.writeline("STOP_RENDERING = runtime.STOP_RENDERING") + self.printer.writeline("__M_dict_builtin = dict") + self.printer.writeline("__M_locals_builtin = locals") + self.printer.writeline("_magic_number = %r" % MAGIC_NUMBER) + self.printer.writeline("_modified_time = %r" % time.time()) + self.printer.writeline("_enable_loop = %r" % self.compiler.enable_loop) + self.printer.writeline( + "_template_filename = %r" % self.compiler.filename + ) + self.printer.writeline("_template_uri = %r" % self.compiler.uri) + self.printer.writeline( + "_source_encoding = %r" % self.compiler.source_encoding + ) + if self.compiler.imports: + buf = "" + for imp in self.compiler.imports: + buf += imp + "\n" + self.printer.writeline(imp) + impcode = ast.PythonCode( + buf, + source="", + lineno=0, + pos=0, + filename="template defined imports", + ) + else: + impcode = None + + main_identifiers = module_identifiers.branch(self.node) + mit = module_identifiers.topleveldefs + module_identifiers.topleveldefs = mit.union( + main_identifiers.topleveldefs + ) + module_identifiers.declared.update(TOPLEVEL_DECLARED) + if impcode: + module_identifiers.declared.update(impcode.declared_identifiers) + + self.compiler.identifiers = module_identifiers + self.printer.writeline( + "_exports = %r" + % [n.name for n in main_identifiers.topleveldefs.values()] + ) + self.printer.write_blanks(2) + + if len(module_code): + self.write_module_code(module_code) + + if len(inherit): + self.write_namespaces(namespaces) + self.write_inherit(inherit[-1]) + elif len(namespaces): + self.write_namespaces(namespaces) + + return list(main_identifiers.topleveldefs.values()) + + def write_render_callable( + self, node, name, args, buffered, filtered, cached + ): + """write a top-level render callable. + + this could be the main render() method or that of a top-level def.""" + + if self.in_def: + decorator = node.decorator + if decorator: + self.printer.writeline( + "@runtime._decorate_toplevel(%s)" % decorator + ) + + self.printer.start_source(node.lineno) + self.printer.writelines( + "def %s(%s):" % (name, ",".join(args)), + # push new frame, assign current frame to __M_caller + "__M_caller = context.caller_stack._push_frame()", + "try:", + ) + if buffered or filtered or cached: + self.printer.writeline("context._push_buffer()") + + self.identifier_stack.append( + self.compiler.identifiers.branch(self.node) + ) + if (not self.in_def or self.node.is_block) and "**pageargs" in args: + self.identifier_stack[-1].argument_declared.add("pageargs") + + if not self.in_def and ( + len(self.identifiers.locally_assigned) > 0 + or len(self.identifiers.argument_declared) > 0 + ): + self.printer.writeline( + "__M_locals = __M_dict_builtin(%s)" + % ",".join( + [ + "%s=%s" % (x, x) + for x in self.identifiers.argument_declared + ] + ) + ) + + self.write_variable_declares(self.identifiers, toplevel=True) + + for n in self.node.nodes: + n.accept_visitor(self) + + self.write_def_finish(self.node, buffered, filtered, cached) + self.printer.writeline(None) + self.printer.write_blanks(2) + if cached: + self.write_cache_decorator( + node, name, args, buffered, self.identifiers, toplevel=True + ) + + def write_module_code(self, module_code): + """write module-level template code, i.e. that which + is enclosed in <%! %> tags in the template.""" + for n in module_code: + self.printer.write_indented_block(n.text, starting_lineno=n.lineno) + + def write_inherit(self, node): + """write the module-level inheritance-determination callable.""" + + self.printer.writelines( + "def _mako_inherit(template, context):", + "_mako_generate_namespaces(context)", + "return runtime._inherit_from(context, %s, _template_uri)" + % (node.parsed_attributes["file"]), + None, + ) + + def write_namespaces(self, namespaces): + """write the module-level namespace-generating callable.""" + self.printer.writelines( + "def _mako_get_namespace(context, name):", + "try:", + "return context.namespaces[(__name__, name)]", + "except KeyError:", + "_mako_generate_namespaces(context)", + "return context.namespaces[(__name__, name)]", + None, + None, + ) + self.printer.writeline("def _mako_generate_namespaces(context):") + + for node in namespaces.values(): + if "import" in node.attributes: + self.compiler.has_ns_imports = True + self.printer.start_source(node.lineno) + if len(node.nodes): + self.printer.writeline("def make_namespace():") + export = [] + identifiers = self.compiler.identifiers.branch(node) + self.in_def = True + + class NSDefVisitor: + def visitDefTag(s, node): + s.visitDefOrBase(node) + + def visitBlockTag(s, node): + s.visitDefOrBase(node) + + def visitDefOrBase(s, node): + if node.is_anonymous: + raise exceptions.CompileException( + "Can't put anonymous blocks inside " + "<%namespace>", + **node.exception_kwargs, + ) + self.write_inline_def(node, identifiers, nested=False) + export.append(node.funcname) + + vis = NSDefVisitor() + for n in node.nodes: + n.accept_visitor(vis) + self.printer.writeline("return [%s]" % (",".join(export))) + self.printer.writeline(None) + self.in_def = False + callable_name = "make_namespace()" + else: + callable_name = "None" + + if "file" in node.parsed_attributes: + self.printer.writeline( + "ns = runtime.TemplateNamespace(%r," + " context._clean_inheritance_tokens()," + " templateuri=%s, callables=%s, " + " calling_uri=_template_uri)" + % ( + node.name, + node.parsed_attributes.get("file", "None"), + callable_name, + ) + ) + elif "module" in node.parsed_attributes: + self.printer.writeline( + "ns = runtime.ModuleNamespace(%r," + " context._clean_inheritance_tokens()," + " callables=%s, calling_uri=_template_uri," + " module=%s)" + % ( + node.name, + callable_name, + node.parsed_attributes.get("module", "None"), + ) + ) + else: + self.printer.writeline( + "ns = runtime.Namespace(%r," + " context._clean_inheritance_tokens()," + " callables=%s, calling_uri=_template_uri)" + % (node.name, callable_name) + ) + if eval(node.attributes.get("inheritable", "False")): + self.printer.writeline("context['self'].%s = ns" % (node.name)) + + self.printer.writeline( + "context.namespaces[(__name__, %s)] = ns" % repr(node.name) + ) + self.printer.write_blanks(1) + if not len(namespaces): + self.printer.writeline("pass") + self.printer.writeline(None) + + def write_variable_declares(self, identifiers, toplevel=False, limit=None): + """write variable declarations at the top of a function. + + the variable declarations are in the form of callable + definitions for defs and/or name lookup within the + function's context argument. the names declared are based + on the names that are referenced in the function body, + which don't otherwise have any explicit assignment + operation. names that are assigned within the body are + assumed to be locally-scoped variables and are not + separately declared. + + for def callable definitions, if the def is a top-level + callable then a 'stub' callable is generated which wraps + the current Context into a closure. if the def is not + top-level, it is fully rendered as a local closure. + + """ + + # collection of all defs available to us in this scope + comp_idents = {c.funcname: c for c in identifiers.defs} + to_write = set() + + # write "context.get()" for all variables we are going to + # need that arent in the namespace yet + to_write = to_write.union(identifiers.undeclared) + + # write closure functions for closures that we define + # right here + to_write = to_write.union( + [c.funcname for c in identifiers.closuredefs.values()] + ) + + # remove identifiers that are declared in the argument + # signature of the callable + to_write = to_write.difference(identifiers.argument_declared) + + # remove identifiers that we are going to assign to. + # in this way we mimic Python's behavior, + # i.e. assignment to a variable within a block + # means that variable is now a "locally declared" var, + # which cannot be referenced beforehand. + to_write = to_write.difference(identifiers.locally_declared) + + if self.compiler.enable_loop: + has_loop = "loop" in to_write + to_write.discard("loop") + else: + has_loop = False + + # if a limiting set was sent, constraint to those items in that list + # (this is used for the caching decorator) + if limit is not None: + to_write = to_write.intersection(limit) + + if toplevel and getattr(self.compiler, "has_ns_imports", False): + self.printer.writeline("_import_ns = {}") + self.compiler.has_imports = True + for ident, ns in self.compiler.namespaces.items(): + if "import" in ns.attributes: + self.printer.writeline( + "_mako_get_namespace(context, %r)." + "_populate(_import_ns, %r)" + % ( + ident, + re.split(r"\s*,\s*", ns.attributes["import"]), + ) + ) + + if has_loop: + self.printer.writeline("loop = __M_loop = runtime.LoopStack()") + + for ident in to_write: + if ident in comp_idents: + comp = comp_idents[ident] + if comp.is_block: + if not comp.is_anonymous: + self.write_def_decl(comp, identifiers) + else: + self.write_inline_def(comp, identifiers, nested=True) + else: + if comp.is_root(): + self.write_def_decl(comp, identifiers) + else: + self.write_inline_def(comp, identifiers, nested=True) + + elif ident in self.compiler.namespaces: + self.printer.writeline( + "%s = _mako_get_namespace(context, %r)" % (ident, ident) + ) + else: + if getattr(self.compiler, "has_ns_imports", False): + if self.compiler.strict_undefined: + self.printer.writelines( + "%s = _import_ns.get(%r, UNDEFINED)" + % (ident, ident), + "if %s is UNDEFINED:" % ident, + "try:", + "%s = context[%r]" % (ident, ident), + "except KeyError:", + "raise NameError(\"'%s' is not defined\")" % ident, + None, + None, + ) + else: + self.printer.writeline( + "%s = _import_ns.get" + "(%r, context.get(%r, UNDEFINED))" + % (ident, ident, ident) + ) + else: + if self.compiler.strict_undefined: + self.printer.writelines( + "try:", + "%s = context[%r]" % (ident, ident), + "except KeyError:", + "raise NameError(\"'%s' is not defined\")" % ident, + None, + ) + else: + self.printer.writeline( + "%s = context.get(%r, UNDEFINED)" % (ident, ident) + ) + + self.printer.writeline("__M_writer = context.writer()") + + def write_def_decl(self, node, identifiers): + """write a locally-available callable referencing a top-level def""" + funcname = node.funcname + namedecls = node.get_argument_expressions() + nameargs = node.get_argument_expressions(as_call=True) + + if not self.in_def and ( + len(self.identifiers.locally_assigned) > 0 + or len(self.identifiers.argument_declared) > 0 + ): + nameargs.insert(0, "context._locals(__M_locals)") + else: + nameargs.insert(0, "context") + self.printer.writeline("def %s(%s):" % (funcname, ",".join(namedecls))) + self.printer.writeline( + "return render_%s(%s)" % (funcname, ",".join(nameargs)) + ) + self.printer.writeline(None) + + def write_inline_def(self, node, identifiers, nested): + """write a locally-available def callable inside an enclosing def.""" + + namedecls = node.get_argument_expressions() + + decorator = node.decorator + if decorator: + self.printer.writeline( + "@runtime._decorate_inline(context, %s)" % decorator + ) + self.printer.writeline( + "def %s(%s):" % (node.funcname, ",".join(namedecls)) + ) + filtered = len(node.filter_args.args) > 0 + buffered = eval(node.attributes.get("buffered", "False")) + cached = eval(node.attributes.get("cached", "False")) + self.printer.writelines( + # push new frame, assign current frame to __M_caller + "__M_caller = context.caller_stack._push_frame()", + "try:", + ) + if buffered or filtered or cached: + self.printer.writelines("context._push_buffer()") + + identifiers = identifiers.branch(node, nested=nested) + + self.write_variable_declares(identifiers) + + self.identifier_stack.append(identifiers) + for n in node.nodes: + n.accept_visitor(self) + self.identifier_stack.pop() + + self.write_def_finish(node, buffered, filtered, cached) + self.printer.writeline(None) + if cached: + self.write_cache_decorator( + node, + node.funcname, + namedecls, + False, + identifiers, + inline=True, + toplevel=False, + ) + + def write_def_finish( + self, node, buffered, filtered, cached, callstack=True + ): + """write the end section of a rendering function, either outermost or + inline. + + this takes into account if the rendering function was filtered, + buffered, etc. and closes the corresponding try: block if any, and + writes code to retrieve captured content, apply filters, send proper + return value.""" + + if not buffered and not cached and not filtered: + self.printer.writeline("return ''") + if callstack: + self.printer.writelines( + "finally:", "context.caller_stack._pop_frame()", None + ) + + if buffered or filtered or cached: + if buffered or cached: + # in a caching scenario, don't try to get a writer + # from the context after popping; assume the caching + # implemenation might be using a context with no + # extra buffers + self.printer.writelines( + "finally:", "__M_buf = context._pop_buffer()" + ) + else: + self.printer.writelines( + "finally:", + "__M_buf, __M_writer = context._pop_buffer_and_writer()", + ) + + if callstack: + self.printer.writeline("context.caller_stack._pop_frame()") + + s = "__M_buf.getvalue()" + if filtered: + s = self.create_filter_callable( + node.filter_args.args, s, False + ) + self.printer.writeline(None) + if buffered and not cached: + s = self.create_filter_callable( + self.compiler.buffer_filters, s, False + ) + if buffered or cached: + self.printer.writeline("return %s" % s) + else: + self.printer.writelines("__M_writer(%s)" % s, "return ''") + + def write_cache_decorator( + self, + node_or_pagetag, + name, + args, + buffered, + identifiers, + inline=False, + toplevel=False, + ): + """write a post-function decorator to replace a rendering + callable with a cached version of itself.""" + + self.printer.writeline("__M_%s = %s" % (name, name)) + cachekey = node_or_pagetag.parsed_attributes.get( + "cache_key", repr(name) + ) + + cache_args = {} + if self.compiler.pagetag is not None: + cache_args.update( + (pa[6:], self.compiler.pagetag.parsed_attributes[pa]) + for pa in self.compiler.pagetag.parsed_attributes + if pa.startswith("cache_") and pa != "cache_key" + ) + cache_args.update( + (pa[6:], node_or_pagetag.parsed_attributes[pa]) + for pa in node_or_pagetag.parsed_attributes + if pa.startswith("cache_") and pa != "cache_key" + ) + if "timeout" in cache_args: + cache_args["timeout"] = int(eval(cache_args["timeout"])) + + self.printer.writeline("def %s(%s):" % (name, ",".join(args))) + + # form "arg1, arg2, arg3=arg3, arg4=arg4", etc. + pass_args = [ + "%s=%s" % ((a.split("=")[0],) * 2) if "=" in a else a for a in args + ] + + self.write_variable_declares( + identifiers, + toplevel=toplevel, + limit=node_or_pagetag.undeclared_identifiers(), + ) + if buffered: + s = ( + "context.get('local')." + "cache._ctx_get_or_create(" + "%s, lambda:__M_%s(%s), context, %s__M_defname=%r)" + % ( + cachekey, + name, + ",".join(pass_args), + "".join( + ["%s=%s, " % (k, v) for k, v in cache_args.items()] + ), + name, + ) + ) + # apply buffer_filters + s = self.create_filter_callable( + self.compiler.buffer_filters, s, False + ) + self.printer.writelines("return " + s, None) + else: + self.printer.writelines( + "__M_writer(context.get('local')." + "cache._ctx_get_or_create(" + "%s, lambda:__M_%s(%s), context, %s__M_defname=%r))" + % ( + cachekey, + name, + ",".join(pass_args), + "".join( + ["%s=%s, " % (k, v) for k, v in cache_args.items()] + ), + name, + ), + "return ''", + None, + ) + + def create_filter_callable(self, args, target, is_expression): + """write a filter-applying expression based on the filters + present in the given filter names, adjusting for the global + 'default' filter aliases as needed.""" + + def locate_encode(name): + if re.match(r"decode\..+", name): + return "filters." + name + else: + return filters.DEFAULT_ESCAPES.get(name, name) + + if "n" not in args: + if is_expression: + if self.compiler.pagetag: + args = self.compiler.pagetag.filter_args.args + args + if self.compiler.default_filters and "n" not in args: + args = self.compiler.default_filters + args + for e in args: + # if filter given as a function, get just the identifier portion + if e == "n": + continue + m = re.match(r"(.+?)(\(.*\))", e) + if m: + ident, fargs = m.group(1, 2) + f = locate_encode(ident) + e = f + fargs + else: + e = locate_encode(e) + assert e is not None + target = "%s(%s)" % (e, target) + return target + + def visitExpression(self, node): + self.printer.start_source(node.lineno) + if ( + len(node.escapes) + or ( + self.compiler.pagetag is not None + and len(self.compiler.pagetag.filter_args.args) + ) + or len(self.compiler.default_filters) + ): + s = self.create_filter_callable( + node.escapes_code.args, "%s" % node.text, True + ) + self.printer.writeline("__M_writer(%s)" % s) + else: + self.printer.writeline("__M_writer(%s)" % node.text) + + def visitControlLine(self, node): + if node.isend: + self.printer.writeline(None) + if node.has_loop_context: + self.printer.writeline("finally:") + self.printer.writeline("loop = __M_loop._exit()") + self.printer.writeline(None) + else: + self.printer.start_source(node.lineno) + if self.compiler.enable_loop and node.keyword == "for": + text = mangle_mako_loop(node, self.printer) + else: + text = node.text + self.printer.writeline(text) + children = node.get_children() + # this covers the three situations where we want to insert a pass: + # 1) a ternary control line with no children, + # 2) a primary control line with nothing but its own ternary + # and end control lines, and + # 3) any control line with no content other than comments + if not children or ( + all( + isinstance(c, (parsetree.Comment, parsetree.ControlLine)) + for c in children + ) + and all( + (node.is_ternary(c.keyword) or c.isend) + for c in children + if isinstance(c, parsetree.ControlLine) + ) + ): + self.printer.writeline("pass") + + def visitText(self, node): + self.printer.start_source(node.lineno) + self.printer.writeline("__M_writer(%s)" % repr(node.content)) + + def visitTextTag(self, node): + filtered = len(node.filter_args.args) > 0 + if filtered: + self.printer.writelines( + "__M_writer = context._push_writer()", "try:" + ) + for n in node.nodes: + n.accept_visitor(self) + if filtered: + self.printer.writelines( + "finally:", + "__M_buf, __M_writer = context._pop_buffer_and_writer()", + "__M_writer(%s)" + % self.create_filter_callable( + node.filter_args.args, "__M_buf.getvalue()", False + ), + None, + ) + + def visitCode(self, node): + if not node.ismodule: + self.printer.write_indented_block( + node.text, starting_lineno=node.lineno + ) + + if not self.in_def and len(self.identifiers.locally_assigned) > 0: + # if we are the "template" def, fudge locally + # declared/modified variables into the "__M_locals" dictionary, + # which is used for def calls within the same template, + # to simulate "enclosing scope" + self.printer.writeline( + "__M_locals_builtin_stored = __M_locals_builtin()" + ) + self.printer.writeline( + "__M_locals.update(__M_dict_builtin([(__M_key," + " __M_locals_builtin_stored[__M_key]) for __M_key in" + " [%s] if __M_key in __M_locals_builtin_stored]))" + % ",".join([repr(x) for x in node.declared_identifiers()]) + ) + + def visitIncludeTag(self, node): + self.printer.start_source(node.lineno) + args = node.attributes.get("args") + if args: + self.printer.writeline( + "runtime._include_file(context, %s, _template_uri, %s)" + % (node.parsed_attributes["file"], args) + ) + else: + self.printer.writeline( + "runtime._include_file(context, %s, _template_uri)" + % (node.parsed_attributes["file"]) + ) + + def visitNamespaceTag(self, node): + pass + + def visitDefTag(self, node): + pass + + def visitBlockTag(self, node): + if node.is_anonymous: + self.printer.writeline("%s()" % node.funcname) + else: + nameargs = node.get_argument_expressions(as_call=True) + nameargs += ["**pageargs"] + self.printer.writeline( + "if 'parent' not in context._data or " + "not hasattr(context._data['parent'], '%s'):" % node.funcname + ) + self.printer.writeline( + "context['self'].%s(%s)" % (node.funcname, ",".join(nameargs)) + ) + self.printer.writeline("\n") + + def visitCallNamespaceTag(self, node): + # TODO: we can put namespace-specific checks here, such + # as ensure the given namespace will be imported, + # pre-import the namespace, etc. + self.visitCallTag(node) + + def visitCallTag(self, node): + self.printer.writeline("def ccall(caller):") + export = ["body"] + callable_identifiers = self.identifiers.branch(node, nested=True) + body_identifiers = callable_identifiers.branch(node, nested=False) + # we want the 'caller' passed to ccall to be used + # for the body() function, but for other non-body() + # <%def>s within <%call> we want the current caller + # off the call stack (if any) + body_identifiers.add_declared("caller") + + self.identifier_stack.append(body_identifiers) + + class DefVisitor: + def visitDefTag(s, node): + s.visitDefOrBase(node) + + def visitBlockTag(s, node): + s.visitDefOrBase(node) + + def visitDefOrBase(s, node): + self.write_inline_def(node, callable_identifiers, nested=False) + if not node.is_anonymous: + export.append(node.funcname) + # remove defs that are within the <%call> from the + # "closuredefs" defined in the body, so they dont render twice + if node.funcname in body_identifiers.closuredefs: + del body_identifiers.closuredefs[node.funcname] + + vis = DefVisitor() + for n in node.nodes: + n.accept_visitor(vis) + self.identifier_stack.pop() + + bodyargs = node.body_decl.get_argument_expressions() + self.printer.writeline("def body(%s):" % ",".join(bodyargs)) + + # TODO: figure out best way to specify + # buffering/nonbuffering (at call time would be better) + buffered = False + if buffered: + self.printer.writelines("context._push_buffer()", "try:") + self.write_variable_declares(body_identifiers) + self.identifier_stack.append(body_identifiers) + + for n in node.nodes: + n.accept_visitor(self) + self.identifier_stack.pop() + + self.write_def_finish(node, buffered, False, False, callstack=False) + self.printer.writelines(None, "return [%s]" % (",".join(export)), None) + + self.printer.writelines( + # push on caller for nested call + "context.caller_stack.nextcaller = " + "runtime.Namespace('caller', context, " + "callables=ccall(__M_caller))", + "try:", + ) + self.printer.start_source(node.lineno) + self.printer.writelines( + "__M_writer(%s)" + % self.create_filter_callable([], node.expression, True), + "finally:", + "context.caller_stack.nextcaller = None", + None, + ) + + +class _Identifiers: + + """tracks the status of identifier names as template code is rendered.""" + + def __init__(self, compiler, node=None, parent=None, nested=False): + if parent is not None: + # if we are the branch created in write_namespaces(), + # we don't share any context from the main body(). + if isinstance(node, parsetree.NamespaceTag): + self.declared = set() + self.topleveldefs = util.SetLikeDict() + else: + # things that have already been declared + # in an enclosing namespace (i.e. names we can just use) + self.declared = ( + set(parent.declared) + .union([c.name for c in parent.closuredefs.values()]) + .union(parent.locally_declared) + .union(parent.argument_declared) + ) + + # if these identifiers correspond to a "nested" + # scope, it means whatever the parent identifiers + # had as undeclared will have been declared by that parent, + # and therefore we have them in our scope. + if nested: + self.declared = self.declared.union(parent.undeclared) + + # top level defs that are available + self.topleveldefs = util.SetLikeDict(**parent.topleveldefs) + else: + self.declared = set() + self.topleveldefs = util.SetLikeDict() + + self.compiler = compiler + + # things within this level that are referenced before they + # are declared (e.g. assigned to) + self.undeclared = set() + + # things that are declared locally. some of these things + # could be in the "undeclared" list as well if they are + # referenced before declared + self.locally_declared = set() + + # assignments made in explicit python blocks. + # these will be propagated to + # the context of local def calls. + self.locally_assigned = set() + + # things that are declared in the argument + # signature of the def callable + self.argument_declared = set() + + # closure defs that are defined in this level + self.closuredefs = util.SetLikeDict() + + self.node = node + + if node is not None: + node.accept_visitor(self) + + illegal_names = self.compiler.reserved_names.intersection( + self.locally_declared + ) + if illegal_names: + raise exceptions.NameConflictError( + "Reserved words declared in template: %s" + % ", ".join(illegal_names) + ) + + def branch(self, node, **kwargs): + """create a new Identifiers for a new Node, with + this Identifiers as the parent.""" + + return _Identifiers(self.compiler, node, self, **kwargs) + + @property + def defs(self): + return set(self.topleveldefs.union(self.closuredefs).values()) + + def __repr__(self): + return ( + "Identifiers(declared=%r, locally_declared=%r, " + "undeclared=%r, topleveldefs=%r, closuredefs=%r, " + "argumentdeclared=%r)" + % ( + list(self.declared), + list(self.locally_declared), + list(self.undeclared), + [c.name for c in self.topleveldefs.values()], + [c.name for c in self.closuredefs.values()], + self.argument_declared, + ) + ) + + def check_declared(self, node): + """update the state of this Identifiers with the undeclared + and declared identifiers of the given node.""" + + for ident in node.undeclared_identifiers(): + if ident != "context" and ident not in self.declared.union( + self.locally_declared + ): + self.undeclared.add(ident) + for ident in node.declared_identifiers(): + self.locally_declared.add(ident) + + def add_declared(self, ident): + self.declared.add(ident) + if ident in self.undeclared: + self.undeclared.remove(ident) + + def visitExpression(self, node): + self.check_declared(node) + + def visitControlLine(self, node): + self.check_declared(node) + + def visitCode(self, node): + if not node.ismodule: + self.check_declared(node) + self.locally_assigned = self.locally_assigned.union( + node.declared_identifiers() + ) + + def visitNamespaceTag(self, node): + # only traverse into the sub-elements of a + # <%namespace> tag if we are the branch created in + # write_namespaces() + if self.node is node: + for n in node.nodes: + n.accept_visitor(self) + + def _check_name_exists(self, collection, node): + existing = collection.get(node.funcname) + collection[node.funcname] = node + if ( + existing is not None + and existing is not node + and (node.is_block or existing.is_block) + ): + raise exceptions.CompileException( + "%%def or %%block named '%s' already " + "exists in this template." % node.funcname, + **node.exception_kwargs, + ) + + def visitDefTag(self, node): + if node.is_root() and not node.is_anonymous: + self._check_name_exists(self.topleveldefs, node) + elif node is not self.node: + self._check_name_exists(self.closuredefs, node) + + for ident in node.undeclared_identifiers(): + if ident != "context" and ident not in self.declared.union( + self.locally_declared + ): + self.undeclared.add(ident) + + # visit defs only one level deep + if node is self.node: + for ident in node.declared_identifiers(): + self.argument_declared.add(ident) + + for n in node.nodes: + n.accept_visitor(self) + + def visitBlockTag(self, node): + if node is not self.node and not node.is_anonymous: + if isinstance(self.node, parsetree.DefTag): + raise exceptions.CompileException( + "Named block '%s' not allowed inside of def '%s'" + % (node.name, self.node.name), + **node.exception_kwargs, + ) + elif isinstance( + self.node, (parsetree.CallTag, parsetree.CallNamespaceTag) + ): + raise exceptions.CompileException( + "Named block '%s' not allowed inside of <%%call> tag" + % (node.name,), + **node.exception_kwargs, + ) + + for ident in node.undeclared_identifiers(): + if ident != "context" and ident not in self.declared.union( + self.locally_declared + ): + self.undeclared.add(ident) + + if not node.is_anonymous: + self._check_name_exists(self.topleveldefs, node) + self.undeclared.add(node.funcname) + elif node is not self.node: + self._check_name_exists(self.closuredefs, node) + for ident in node.declared_identifiers(): + self.argument_declared.add(ident) + for n in node.nodes: + n.accept_visitor(self) + + def visitTextTag(self, node): + for ident in node.undeclared_identifiers(): + if ident != "context" and ident not in self.declared.union( + self.locally_declared + ): + self.undeclared.add(ident) + + def visitIncludeTag(self, node): + self.check_declared(node) + + def visitPageTag(self, node): + for ident in node.declared_identifiers(): + self.argument_declared.add(ident) + self.check_declared(node) + + def visitCallNamespaceTag(self, node): + self.visitCallTag(node) + + def visitCallTag(self, node): + if node is self.node: + for ident in node.undeclared_identifiers(): + if ident != "context" and ident not in self.declared.union( + self.locally_declared + ): + self.undeclared.add(ident) + for ident in node.declared_identifiers(): + self.argument_declared.add(ident) + for n in node.nodes: + n.accept_visitor(self) + else: + for ident in node.undeclared_identifiers(): + if ident != "context" and ident not in self.declared.union( + self.locally_declared + ): + self.undeclared.add(ident) + + +_FOR_LOOP = re.compile( + r"^for\s+((?:\(?)\s*" + r"(?:\(?)\s*[A-Za-z_][A-Za-z_0-9]*" + r"(?:\s*,\s*(?:[A-Za-z_][A-Za-z_0-9]*),??)*\s*(?:\)?)" + r"(?:\s*,\s*(?:" + r"(?:\(?)\s*[A-Za-z_][A-Za-z_0-9]*" + r"(?:\s*,\s*(?:[A-Za-z_][A-Za-z_0-9]*),??)*\s*(?:\)?)" + r"),??)*\s*(?:\)?))\s+in\s+(.*):" +) + + +def mangle_mako_loop(node, printer): + """converts a for loop into a context manager wrapped around a for loop + when access to the `loop` variable has been detected in the for loop body + """ + loop_variable = LoopVariable() + node.accept_visitor(loop_variable) + if loop_variable.detected: + node.nodes[-1].has_loop_context = True + match = _FOR_LOOP.match(node.text) + if match: + printer.writelines( + "loop = __M_loop._enter(%s)" % match.group(2), + "try:" + # 'with __M_loop(%s) as loop:' % match.group(2) + ) + text = "for %s in loop:" % match.group(1) + else: + raise SyntaxError("Couldn't apply loop context: %s" % node.text) + else: + text = node.text + return text + + +class LoopVariable: + + """A node visitor which looks for the name 'loop' within undeclared + identifiers.""" + + def __init__(self): + self.detected = False + + def _loop_reference_detected(self, node): + if "loop" in node.undeclared_identifiers(): + self.detected = True + else: + for n in node.get_children(): + n.accept_visitor(self) + + def visitControlLine(self, node): + self._loop_reference_detected(node) + + def visitCode(self, node): + self._loop_reference_detected(node) + + def visitExpression(self, node): + self._loop_reference_detected(node) diff --git a/mako/compat.py b/mako/compat.py new file mode 100644 index 0000000..4de11c5 --- /dev/null +++ b/mako/compat.py @@ -0,0 +1,70 @@ +# mako/compat.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +import collections +from importlib import metadata as importlib_metadata +from importlib import util +import inspect +import sys + +win32 = sys.platform.startswith("win") +pypy = hasattr(sys, "pypy_version_info") + +ArgSpec = collections.namedtuple( + "ArgSpec", ["args", "varargs", "keywords", "defaults"] +) + + +def inspect_getargspec(func): + """getargspec based on fully vendored getfullargspec from Python 3.3.""" + + if inspect.ismethod(func): + func = func.__func__ + if not inspect.isfunction(func): + raise TypeError(f"{func!r} is not a Python function") + + co = func.__code__ + if not inspect.iscode(co): + raise TypeError(f"{co!r} is not a code object") + + nargs = co.co_argcount + names = co.co_varnames + nkwargs = co.co_kwonlyargcount + args = list(names[:nargs]) + + nargs += nkwargs + varargs = None + if co.co_flags & inspect.CO_VARARGS: + varargs = co.co_varnames[nargs] + nargs = nargs + 1 + varkw = None + if co.co_flags & inspect.CO_VARKEYWORDS: + varkw = co.co_varnames[nargs] + + return ArgSpec(args, varargs, varkw, func.__defaults__) + + +def load_module(module_id, path): + spec = util.spec_from_file_location(module_id, path) + module = util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def exception_as(): + return sys.exc_info()[1] + + +def exception_name(exc): + return exc.__class__.__name__ + + +def importlib_metadata_get(group): + ep = importlib_metadata.entry_points() + if hasattr(ep, "select"): + return ep.select(group=group) + else: + return ep.get(group, ()) diff --git a/mako/exceptions.py b/mako/exceptions.py new file mode 100644 index 0000000..2bf6a60 --- /dev/null +++ b/mako/exceptions.py @@ -0,0 +1,417 @@ +# mako/exceptions.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""exception classes""" + +import sys +import traceback + +from mako import compat +from mako import util + + +class MakoException(Exception): + pass + + +class RuntimeException(MakoException): + pass + + +def _format_filepos(lineno, pos, filename): + if filename is None: + return " at line: %d char: %d" % (lineno, pos) + else: + return " in file '%s' at line: %d char: %d" % (filename, lineno, pos) + + +class CompileException(MakoException): + def __init__(self, message, source, lineno, pos, filename): + MakoException.__init__( + self, message + _format_filepos(lineno, pos, filename) + ) + self.lineno = lineno + self.pos = pos + self.filename = filename + self.source = source + + +class SyntaxException(MakoException): + def __init__(self, message, source, lineno, pos, filename): + MakoException.__init__( + self, message + _format_filepos(lineno, pos, filename) + ) + self.lineno = lineno + self.pos = pos + self.filename = filename + self.source = source + + +class UnsupportedError(MakoException): + + """raised when a retired feature is used.""" + + +class NameConflictError(MakoException): + + """raised when a reserved word is used inappropriately""" + + +class TemplateLookupException(MakoException): + pass + + +class TopLevelLookupException(TemplateLookupException): + pass + + +class RichTraceback: + + """Pull the current exception from the ``sys`` traceback and extracts + Mako-specific template information. + + See the usage examples in :ref:`handling_exceptions`. + + """ + + def __init__(self, error=None, traceback=None): + self.source, self.lineno = "", 0 + + if error is None or traceback is None: + t, value, tback = sys.exc_info() + + if error is None: + error = value or t + + if traceback is None: + traceback = tback + + self.error = error + self.records = self._init(traceback) + + if isinstance(self.error, (CompileException, SyntaxException)): + self.source = self.error.source + self.lineno = self.error.lineno + self._has_source = True + + self._init_message() + + @property + def errorname(self): + return compat.exception_name(self.error) + + def _init_message(self): + """Find a unicode representation of self.error""" + try: + self.message = str(self.error) + except UnicodeError: + try: + self.message = str(self.error) + except UnicodeEncodeError: + # Fallback to args as neither unicode nor + # str(Exception(u'\xe6')) work in Python < 2.6 + self.message = self.error.args[0] + if not isinstance(self.message, str): + self.message = str(self.message, "ascii", "replace") + + def _get_reformatted_records(self, records): + for rec in records: + if rec[6] is not None: + yield (rec[4], rec[5], rec[2], rec[6]) + else: + yield tuple(rec[0:4]) + + @property + def traceback(self): + """Return a list of 4-tuple traceback records (i.e. normal python + format) with template-corresponding lines remapped to the originating + template. + + """ + return list(self._get_reformatted_records(self.records)) + + @property + def reverse_records(self): + return reversed(self.records) + + @property + def reverse_traceback(self): + """Return the same data as traceback, except in reverse order.""" + + return list(self._get_reformatted_records(self.reverse_records)) + + def _init(self, trcback): + """format a traceback from sys.exc_info() into 7-item tuples, + containing the regular four traceback tuple items, plus the original + template filename, the line number adjusted relative to the template + source, and code line from that line number of the template.""" + + import mako.template + + mods = {} + rawrecords = traceback.extract_tb(trcback) + new_trcback = [] + for filename, lineno, function, line in rawrecords: + if not line: + line = "" + try: + (line_map, template_lines, template_filename) = mods[filename] + except KeyError: + try: + info = mako.template._get_module_info(filename) + module_source = info.code + template_source = info.source + template_filename = ( + info.template_filename or info.template_uri or filename + ) + except KeyError: + # A normal .py file (not a Template) + new_trcback.append( + ( + filename, + lineno, + function, + line, + None, + None, + None, + None, + ) + ) + continue + + template_ln = 1 + + mtm = mako.template.ModuleInfo + source_map = mtm.get_module_source_metadata( + module_source, full_line_map=True + ) + line_map = source_map["full_line_map"] + + template_lines = [ + line_ for line_ in template_source.split("\n") + ] + mods[filename] = (line_map, template_lines, template_filename) + + template_ln = line_map[lineno - 1] + + if template_ln <= len(template_lines): + template_line = template_lines[template_ln - 1] + else: + template_line = None + new_trcback.append( + ( + filename, + lineno, + function, + line, + template_filename, + template_ln, + template_line, + template_source, + ) + ) + if not self.source: + for l in range(len(new_trcback) - 1, 0, -1): + if new_trcback[l][5]: + self.source = new_trcback[l][7] + self.lineno = new_trcback[l][5] + break + else: + if new_trcback: + try: + # A normal .py file (not a Template) + with open(new_trcback[-1][0], "rb") as fp: + encoding = util.parse_encoding(fp) + if not encoding: + encoding = "utf-8" + fp.seek(0) + self.source = fp.read() + if encoding: + self.source = self.source.decode(encoding) + except IOError: + self.source = "" + self.lineno = new_trcback[-1][1] + return new_trcback + + +def text_error_template(lookup=None): + """Provides a template that renders a stack trace in a similar format to + the Python interpreter, substituting source template filenames, line + numbers and code for that of the originating source template, as + applicable. + + """ + import mako.template + + return mako.template.Template( + r""" +<%page args="error=None, traceback=None"/> +<%! + from mako.exceptions import RichTraceback +%>\ +<% + tback = RichTraceback(error=error, traceback=traceback) +%>\ +Traceback (most recent call last): +% for (filename, lineno, function, line) in tback.traceback: + File "${filename}", line ${lineno}, in ${function or '?'} + ${line | trim} +% endfor +${tback.errorname}: ${tback.message} +""" + ) + + +def _install_pygments(): + global syntax_highlight, pygments_html_formatter + from mako.ext.pygmentplugin import syntax_highlight # noqa + from mako.ext.pygmentplugin import pygments_html_formatter # noqa + + +def _install_fallback(): + global syntax_highlight, pygments_html_formatter + from mako.filters import html_escape + + pygments_html_formatter = None + + def syntax_highlight(filename="", language=None): + return html_escape + + +def _install_highlighting(): + try: + _install_pygments() + except ImportError: + _install_fallback() + + +_install_highlighting() + + +def html_error_template(): + """Provides a template that renders a stack trace in an HTML format, + providing an excerpt of code as well as substituting source template + filenames, line numbers and code for that of the originating source + template, as applicable. + + The template's default ``encoding_errors`` value is + ``'htmlentityreplace'``. The template has two options. With the + ``full`` option disabled, only a section of an HTML document is + returned. With the ``css`` option disabled, the default stylesheet + won't be included. + + """ + import mako.template + + return mako.template.Template( + r""" +<%! + from mako.exceptions import RichTraceback, syntax_highlight,\ + pygments_html_formatter +%> +<%page args="full=True, css=True, error=None, traceback=None"/> +% if full: +<html> +<head> + <title>Mako Runtime Error</title> +% endif +% if css: + <style> + body { font-family:verdana; margin:10px 30px 10px 30px;} + .stacktrace { margin:5px 5px 5px 5px; } + .highlight { padding:0px 10px 0px 10px; background-color:#9F9FDF; } + .nonhighlight { padding:0px; background-color:#DFDFDF; } + .sample { padding:10px; margin:10px 10px 10px 10px; + font-family:monospace; } + .sampleline { padding:0px 10px 0px 10px; } + .sourceline { margin:5px 5px 10px 5px; font-family:monospace;} + .location { font-size:80%; } + .highlight { white-space:pre; } + .sampleline { white-space:pre; } + + % if pygments_html_formatter: + ${pygments_html_formatter.get_style_defs()} + .linenos { min-width: 2.5em; text-align: right; } + pre { margin: 0; } + .syntax-highlighted { padding: 0 10px; } + .syntax-highlightedtable { border-spacing: 1px; } + .nonhighlight { border-top: 1px solid #DFDFDF; + border-bottom: 1px solid #DFDFDF; } + .stacktrace .nonhighlight { margin: 5px 15px 10px; } + .sourceline { margin: 0 0; font-family:monospace; } + .code { background-color: #F8F8F8; width: 100%; } + .error .code { background-color: #FFBDBD; } + .error .syntax-highlighted { background-color: #FFBDBD; } + % endif + + </style> +% endif +% if full: +</head> +<body> +% endif + +<h2>Error !</h2> +<% + tback = RichTraceback(error=error, traceback=traceback) + src = tback.source + line = tback.lineno + if src: + lines = src.split('\n') + else: + lines = None +%> +<h3>${tback.errorname}: ${tback.message|h}</h3> + +% if lines: + <div class="sample"> + <div class="nonhighlight"> +% for index in range(max(0, line-4),min(len(lines), line+5)): + <% + if pygments_html_formatter: + pygments_html_formatter.linenostart = index + 1 + %> + % if index + 1 == line: + <% + if pygments_html_formatter: + old_cssclass = pygments_html_formatter.cssclass + pygments_html_formatter.cssclass = 'error ' + old_cssclass + %> + ${lines[index] | syntax_highlight(language='mako')} + <% + if pygments_html_formatter: + pygments_html_formatter.cssclass = old_cssclass + %> + % else: + ${lines[index] | syntax_highlight(language='mako')} + % endif +% endfor + </div> + </div> +% endif + +<div class="stacktrace"> +% for (filename, lineno, function, line) in tback.reverse_traceback: + <div class="location">${filename}, line ${lineno}:</div> + <div class="nonhighlight"> + <% + if pygments_html_formatter: + pygments_html_formatter.linenostart = lineno + %> + <div class="sourceline">${line | syntax_highlight(filename)}</div> + </div> +% endfor +</div> + +% if full: +</body> +</html> +% endif +""", + output_encoding=sys.getdefaultencoding(), + encoding_errors="htmlentityreplace", + ) diff --git a/mako/ext/__init__.py b/mako/ext/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/mako/ext/__init__.py diff --git a/mako/ext/autohandler.py b/mako/ext/autohandler.py new file mode 100644 index 0000000..c33f080 --- /dev/null +++ b/mako/ext/autohandler.py @@ -0,0 +1,70 @@ +# ext/autohandler.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""adds autohandler functionality to Mako templates. + +requires that the TemplateLookup class is used with templates. + +usage:: + + <%! + from mako.ext.autohandler import autohandler + %> + <%inherit file="${autohandler(template, context)}"/> + + +or with custom autohandler filename:: + + <%! + from mako.ext.autohandler import autohandler + %> + <%inherit file="${autohandler(template, context, name='somefilename')}"/> + +""" + +import os +import posixpath +import re + + +def autohandler(template, context, name="autohandler"): + lookup = context.lookup + _template_uri = template.module._template_uri + if not lookup.filesystem_checks: + try: + return lookup._uri_cache[(autohandler, _template_uri, name)] + except KeyError: + pass + + tokens = re.findall(r"([^/]+)", posixpath.dirname(_template_uri)) + [name] + while len(tokens): + path = "/" + "/".join(tokens) + if path != _template_uri and _file_exists(lookup, path): + if not lookup.filesystem_checks: + return lookup._uri_cache.setdefault( + (autohandler, _template_uri, name), path + ) + else: + return path + if len(tokens) == 1: + break + tokens[-2:] = [name] + + if not lookup.filesystem_checks: + return lookup._uri_cache.setdefault( + (autohandler, _template_uri, name), None + ) + else: + return None + + +def _file_exists(lookup, path): + psub = re.sub(r"^/", "", path) + for d in lookup.directories: + if os.path.exists(d + "/" + psub): + return True + else: + return False diff --git a/mako/ext/babelplugin.py b/mako/ext/babelplugin.py new file mode 100644 index 0000000..5126d6f --- /dev/null +++ b/mako/ext/babelplugin.py @@ -0,0 +1,57 @@ +# ext/babelplugin.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""gettext message extraction via Babel: https://pypi.org/project/Babel/""" +from babel.messages.extract import extract_python + +from mako.ext.extract import MessageExtractor + + +class BabelMakoExtractor(MessageExtractor): + def __init__(self, keywords, comment_tags, options): + self.keywords = keywords + self.options = options + self.config = { + "comment-tags": " ".join(comment_tags), + "encoding": options.get( + "input_encoding", options.get("encoding", None) + ), + } + super().__init__() + + def __call__(self, fileobj): + return self.process_file(fileobj) + + def process_python(self, code, code_lineno, translator_strings): + comment_tags = self.config["comment-tags"] + for ( + lineno, + funcname, + messages, + python_translator_comments, + ) in extract_python(code, self.keywords, comment_tags, self.options): + yield ( + code_lineno + (lineno - 1), + funcname, + messages, + translator_strings + python_translator_comments, + ) + + +def extract(fileobj, keywords, comment_tags, options): + """Extract messages from Mako templates. + + :param fileobj: the file-like object the messages should be extracted from + :param keywords: a list of keywords (i.e. function names) that should be + recognized as translation functions + :param comment_tags: a list of translator tags to search for and include + in the results + :param options: a dictionary of additional options (optional) + :return: an iterator over ``(lineno, funcname, message, comments)`` tuples + :rtype: ``iterator`` + """ + extractor = BabelMakoExtractor(keywords, comment_tags, options) + yield from extractor(fileobj) diff --git a/mako/ext/beaker_cache.py b/mako/ext/beaker_cache.py new file mode 100644 index 0000000..3f1f9d4 --- /dev/null +++ b/mako/ext/beaker_cache.py @@ -0,0 +1,82 @@ +# ext/beaker_cache.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""Provide a :class:`.CacheImpl` for the Beaker caching system.""" + +from mako import exceptions +from mako.cache import CacheImpl + +try: + from beaker import cache as beaker_cache +except: + has_beaker = False +else: + has_beaker = True + +_beaker_cache = None + + +class BeakerCacheImpl(CacheImpl): + + """A :class:`.CacheImpl` provided for the Beaker caching system. + + This plugin is used by default, based on the default + value of ``'beaker'`` for the ``cache_impl`` parameter of the + :class:`.Template` or :class:`.TemplateLookup` classes. + + """ + + def __init__(self, cache): + if not has_beaker: + raise exceptions.RuntimeException( + "Can't initialize Beaker plugin; Beaker is not installed." + ) + global _beaker_cache + if _beaker_cache is None: + if "manager" in cache.template.cache_args: + _beaker_cache = cache.template.cache_args["manager"] + else: + _beaker_cache = beaker_cache.CacheManager() + super().__init__(cache) + + def _get_cache(self, **kw): + expiretime = kw.pop("timeout", None) + if "dir" in kw: + kw["data_dir"] = kw.pop("dir") + elif self.cache.template.module_directory: + kw["data_dir"] = self.cache.template.module_directory + + if "manager" in kw: + kw.pop("manager") + + if kw.get("type") == "memcached": + kw["type"] = "ext:memcached" + + if "region" in kw: + region = kw.pop("region") + cache = _beaker_cache.get_cache_region(self.cache.id, region, **kw) + else: + cache = _beaker_cache.get_cache(self.cache.id, **kw) + cache_args = {"starttime": self.cache.starttime} + if expiretime: + cache_args["expiretime"] = expiretime + return cache, cache_args + + def get_or_create(self, key, creation_function, **kw): + cache, kw = self._get_cache(**kw) + return cache.get(key, createfunc=creation_function, **kw) + + def put(self, key, value, **kw): + cache, kw = self._get_cache(**kw) + cache.put(key, value, **kw) + + def get(self, key, **kw): + cache, kw = self._get_cache(**kw) + return cache.get(key, **kw) + + def invalidate(self, key, **kw): + cache, kw = self._get_cache(**kw) + cache.remove_value(key, **kw) diff --git a/mako/ext/extract.py b/mako/ext/extract.py new file mode 100644 index 0000000..fa7fffa --- /dev/null +++ b/mako/ext/extract.py @@ -0,0 +1,129 @@ +# ext/extract.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +from io import BytesIO +from io import StringIO +import re + +from mako import lexer +from mako import parsetree + + +class MessageExtractor: + use_bytes = True + + def process_file(self, fileobj): + template_node = lexer.Lexer( + fileobj.read(), input_encoding=self.config["encoding"] + ).parse() + yield from self.extract_nodes(template_node.get_children()) + + def extract_nodes(self, nodes): + translator_comments = [] + in_translator_comments = False + input_encoding = self.config["encoding"] or "ascii" + comment_tags = list( + filter(None, re.split(r"\s+", self.config["comment-tags"])) + ) + + for node in nodes: + child_nodes = None + if ( + in_translator_comments + and isinstance(node, parsetree.Text) + and not node.content.strip() + ): + # Ignore whitespace within translator comments + continue + + if isinstance(node, parsetree.Comment): + value = node.text.strip() + if in_translator_comments: + translator_comments.extend( + self._split_comment(node.lineno, value) + ) + continue + for comment_tag in comment_tags: + if value.startswith(comment_tag): + in_translator_comments = True + translator_comments.extend( + self._split_comment(node.lineno, value) + ) + continue + + if isinstance(node, parsetree.DefTag): + code = node.function_decl.code + child_nodes = node.nodes + elif isinstance(node, parsetree.BlockTag): + code = node.body_decl.code + child_nodes = node.nodes + elif isinstance(node, parsetree.CallTag): + code = node.code.code + child_nodes = node.nodes + elif isinstance(node, parsetree.PageTag): + code = node.body_decl.code + elif isinstance(node, parsetree.CallNamespaceTag): + code = node.expression + child_nodes = node.nodes + elif isinstance(node, parsetree.ControlLine): + if node.isend: + in_translator_comments = False + continue + code = node.text + elif isinstance(node, parsetree.Code): + in_translator_comments = False + code = node.code.code + elif isinstance(node, parsetree.Expression): + code = node.code.code + else: + continue + + # Comments don't apply unless they immediately precede the message + if ( + translator_comments + and translator_comments[-1][0] < node.lineno - 1 + ): + translator_comments = [] + + translator_strings = [ + comment[1] for comment in translator_comments + ] + + if isinstance(code, str) and self.use_bytes: + code = code.encode(input_encoding, "backslashreplace") + + used_translator_comments = False + # We add extra newline to work around a pybabel bug + # (see python-babel/babel#274, parse_encoding dies if the first + # input string of the input is non-ascii) + # Also, because we added it, we have to subtract one from + # node.lineno + if self.use_bytes: + code = BytesIO(b"\n" + code) + else: + code = StringIO("\n" + code) + + for message in self.process_python( + code, node.lineno - 1, translator_strings + ): + yield message + used_translator_comments = True + + if used_translator_comments: + translator_comments = [] + in_translator_comments = False + + if child_nodes: + yield from self.extract_nodes(child_nodes) + + @staticmethod + def _split_comment(lineno, comment): + """Return the multiline comment at lineno split into a list of + comment line numbers and the accompanying comment line""" + return [ + (lineno + index, line) + for index, line in enumerate(comment.splitlines()) + ] diff --git a/mako/ext/linguaplugin.py b/mako/ext/linguaplugin.py new file mode 100644 index 0000000..8058b36 --- /dev/null +++ b/mako/ext/linguaplugin.py @@ -0,0 +1,57 @@ +# ext/linguaplugin.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +import contextlib +import io + +from lingua.extractors import Extractor +from lingua.extractors import get_extractor +from lingua.extractors import Message + +from mako.ext.extract import MessageExtractor + + +class LinguaMakoExtractor(Extractor, MessageExtractor): + """Mako templates""" + + use_bytes = False + extensions = [".mako"] + default_config = {"encoding": "utf-8", "comment-tags": ""} + + def __call__(self, filename, options, fileobj=None): + self.options = options + self.filename = filename + self.python_extractor = get_extractor("x.py") + if fileobj is None: + ctx = open(filename, "r") + else: + ctx = contextlib.nullcontext(fileobj) + with ctx as file_: + yield from self.process_file(file_) + + def process_python(self, code, code_lineno, translator_strings): + source = code.getvalue().strip() + if source.endswith(":"): + if source in ("try:", "else:") or source.startswith("except"): + source = "" # Ignore try/except and else + elif source.startswith("elif"): + source = source[2:] # Replace "elif" with "if" + source += "pass" + code = io.StringIO(source) + for msg in self.python_extractor( + self.filename, self.options, code, code_lineno - 1 + ): + if translator_strings: + msg = Message( + msg.msgctxt, + msg.msgid, + msg.msgid_plural, + msg.flags, + " ".join(translator_strings + [msg.comment]), + msg.tcomment, + msg.location, + ) + yield msg diff --git a/mako/ext/preprocessors.py b/mako/ext/preprocessors.py new file mode 100644 index 0000000..d285685 --- /dev/null +++ b/mako/ext/preprocessors.py @@ -0,0 +1,20 @@ +# ext/preprocessors.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""preprocessing functions, used with the 'preprocessor' +argument on Template, TemplateLookup""" + +import re + + +def convert_comments(text): + """preprocess old style comments. + + example: + + from mako.ext.preprocessors import convert_comments + t = Template(..., preprocessor=convert_comments)""" + return re.sub(r"(?<=\n)\s*#[^#]", "##", text) diff --git a/mako/ext/pygmentplugin.py b/mako/ext/pygmentplugin.py new file mode 100644 index 0000000..7763bc8 --- /dev/null +++ b/mako/ext/pygmentplugin.py @@ -0,0 +1,150 @@ +# ext/pygmentplugin.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +from pygments import highlight +from pygments.formatters.html import HtmlFormatter +from pygments.lexer import bygroups +from pygments.lexer import DelegatingLexer +from pygments.lexer import include +from pygments.lexer import RegexLexer +from pygments.lexer import using +from pygments.lexers.agile import Python3Lexer +from pygments.lexers.agile import PythonLexer +from pygments.lexers.web import CssLexer +from pygments.lexers.web import HtmlLexer +from pygments.lexers.web import JavascriptLexer +from pygments.lexers.web import XmlLexer +from pygments.token import Comment +from pygments.token import Keyword +from pygments.token import Name +from pygments.token import Operator +from pygments.token import Other +from pygments.token import String +from pygments.token import Text + + +class MakoLexer(RegexLexer): + name = "Mako" + aliases = ["mako"] + filenames = ["*.mao"] + + tokens = { + "root": [ + ( + r"(\s*)(\%)(\s*end(?:\w+))(\n|\Z)", + bygroups(Text, Comment.Preproc, Keyword, Other), + ), + ( + r"(\s*)(\%(?!%))([^\n]*)(\n|\Z)", + bygroups(Text, Comment.Preproc, using(PythonLexer), Other), + ), + ( + r"(\s*)(##[^\n]*)(\n|\Z)", + bygroups(Text, Comment.Preproc, Other), + ), + (r"""(?s)<%doc>.*?</%doc>""", Comment.Preproc), + ( + r"(<%)([\w\.\:]+)", + bygroups(Comment.Preproc, Name.Builtin), + "tag", + ), + ( + r"(</%)([\w\.\:]+)(>)", + bygroups(Comment.Preproc, Name.Builtin, Comment.Preproc), + ), + (r"<%(?=([\w\.\:]+))", Comment.Preproc, "ondeftags"), + ( + r"(?s)(<%(?:!?))(.*?)(%>)", + bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc), + ), + ( + r"(\$\{)(.*?)(\})", + bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc), + ), + ( + r"""(?sx) + (.+?) # anything, followed by: + (?: + (?<=\n)(?=%(?!%)|\#\#) | # an eval or comment line + (?=\#\*) | # multiline comment + (?=</?%) | # a python block + # call start or end + (?=\$\{) | # a substitution + (?<=\n)(?=\s*%) | + # - don't consume + (\\\n) | # an escaped newline + \Z # end of string + ) + """, + bygroups(Other, Operator), + ), + (r"\s+", Text), + ], + "ondeftags": [ + (r"<%", Comment.Preproc), + (r"(?<=<%)(include|inherit|namespace|page)", Name.Builtin), + include("tag"), + ], + "tag": [ + (r'((?:\w+)\s*=)\s*(".*?")', bygroups(Name.Attribute, String)), + (r"/?\s*>", Comment.Preproc, "#pop"), + (r"\s+", Text), + ], + "attr": [ + ('".*?"', String, "#pop"), + ("'.*?'", String, "#pop"), + (r"[^\s>]+", String, "#pop"), + ], + } + + +class MakoHtmlLexer(DelegatingLexer): + name = "HTML+Mako" + aliases = ["html+mako"] + + def __init__(self, **options): + super().__init__(HtmlLexer, MakoLexer, **options) + + +class MakoXmlLexer(DelegatingLexer): + name = "XML+Mako" + aliases = ["xml+mako"] + + def __init__(self, **options): + super().__init__(XmlLexer, MakoLexer, **options) + + +class MakoJavascriptLexer(DelegatingLexer): + name = "JavaScript+Mako" + aliases = ["js+mako", "javascript+mako"] + + def __init__(self, **options): + super().__init__(JavascriptLexer, MakoLexer, **options) + + +class MakoCssLexer(DelegatingLexer): + name = "CSS+Mako" + aliases = ["css+mako"] + + def __init__(self, **options): + super().__init__(CssLexer, MakoLexer, **options) + + +pygments_html_formatter = HtmlFormatter( + cssclass="syntax-highlighted", linenos=True +) + + +def syntax_highlight(filename="", language=None): + mako_lexer = MakoLexer() + python_lexer = Python3Lexer() + if filename.startswith("memory:") or language == "mako": + return lambda string: highlight( + string, mako_lexer, pygments_html_formatter + ) + return lambda string: highlight( + string, python_lexer, pygments_html_formatter + ) diff --git a/mako/ext/turbogears.py b/mako/ext/turbogears.py new file mode 100644 index 0000000..28f2696 --- /dev/null +++ b/mako/ext/turbogears.py @@ -0,0 +1,61 @@ +# ext/turbogears.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +from mako import compat +from mako.lookup import TemplateLookup +from mako.template import Template + + +class TGPlugin: + + """TurboGears compatible Template Plugin.""" + + def __init__(self, extra_vars_func=None, options=None, extension="mak"): + self.extra_vars_func = extra_vars_func + self.extension = extension + if not options: + options = {} + + # Pull the options out and initialize the lookup + lookup_options = {} + for k, v in options.items(): + if k.startswith("mako."): + lookup_options[k[5:]] = v + elif k in ["directories", "filesystem_checks", "module_directory"]: + lookup_options[k] = v + self.lookup = TemplateLookup(**lookup_options) + + self.tmpl_options = {} + # transfer lookup args to template args, based on those available + # in getargspec + for kw in compat.inspect_getargspec(Template.__init__)[0]: + if kw in lookup_options: + self.tmpl_options[kw] = lookup_options[kw] + + def load_template(self, templatename, template_string=None): + """Loads a template from a file or a string""" + if template_string is not None: + return Template(template_string, **self.tmpl_options) + # Translate TG dot notation to normal / template path + if "/" not in templatename: + templatename = ( + "/" + templatename.replace(".", "/") + "." + self.extension + ) + + # Lookup template + return self.lookup.get_template(templatename) + + def render( + self, info, format="html", fragment=False, template=None # noqa + ): + if isinstance(template, str): + template = self.load_template(template) + + # Load extra vars func if provided + if self.extra_vars_func: + info.update(self.extra_vars_func()) + + return template.render(**info) diff --git a/mako/filters.py b/mako/filters.py new file mode 100644 index 0000000..b255aaf --- /dev/null +++ b/mako/filters.py @@ -0,0 +1,163 @@ +# mako/filters.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + + +import codecs +from html.entities import codepoint2name +from html.entities import name2codepoint +import re +from urllib.parse import quote_plus + +import markupsafe + +html_escape = markupsafe.escape + +xml_escapes = { + "&": "&", + ">": ">", + "<": "<", + '"': """, # also " in html-only + "'": "'", # also ' in html-only +} + + +def xml_escape(string): + return re.sub(r'([&<"\'>])', lambda m: xml_escapes[m.group()], string) + + +def url_escape(string): + # convert into a list of octets + string = string.encode("utf8") + return quote_plus(string) + + +def trim(string): + return string.strip() + + +class Decode: + def __getattr__(self, key): + def decode(x): + if isinstance(x, str): + return x + elif not isinstance(x, bytes): + return decode(str(x)) + else: + return str(x, encoding=key) + + return decode + + +decode = Decode() + + +class XMLEntityEscaper: + def __init__(self, codepoint2name, name2codepoint): + self.codepoint2entity = { + c: str("&%s;" % n) for c, n in codepoint2name.items() + } + self.name2codepoint = name2codepoint + + def escape_entities(self, text): + """Replace characters with their character entity references. + + Only characters corresponding to a named entity are replaced. + """ + return str(text).translate(self.codepoint2entity) + + def __escape(self, m): + codepoint = ord(m.group()) + try: + return self.codepoint2entity[codepoint] + except (KeyError, IndexError): + return "&#x%X;" % codepoint + + __escapable = re.compile(r'["&<>]|[^\x00-\x7f]') + + def escape(self, text): + """Replace characters with their character references. + + Replace characters by their named entity references. + Non-ASCII characters, if they do not have a named entity reference, + are replaced by numerical character references. + + The return value is guaranteed to be ASCII. + """ + return self.__escapable.sub(self.__escape, str(text)).encode("ascii") + + # XXX: This regexp will not match all valid XML entity names__. + # (It punts on details involving involving CombiningChars and Extenders.) + # + # .. __: http://www.w3.org/TR/2000/REC-xml-20001006#NT-EntityRef + __characterrefs = re.compile( + r"""& (?: + \#(\d+) + | \#x([\da-f]+) + | ( (?!\d) [:\w] [-.:\w]+ ) + ) ;""", + re.X | re.UNICODE, + ) + + def __unescape(self, m): + dval, hval, name = m.groups() + if dval: + codepoint = int(dval) + elif hval: + codepoint = int(hval, 16) + else: + codepoint = self.name2codepoint.get(name, 0xFFFD) + # U+FFFD = "REPLACEMENT CHARACTER" + if codepoint < 128: + return chr(codepoint) + return chr(codepoint) + + def unescape(self, text): + """Unescape character references. + + All character references (both entity references and numerical + character references) are unescaped. + """ + return self.__characterrefs.sub(self.__unescape, text) + + +_html_entities_escaper = XMLEntityEscaper(codepoint2name, name2codepoint) + +html_entities_escape = _html_entities_escaper.escape_entities +html_entities_unescape = _html_entities_escaper.unescape + + +def htmlentityreplace_errors(ex): + """An encoding error handler. + + This python codecs error handler replaces unencodable + characters with HTML entities, or, if no HTML entity exists for + the character, XML character references:: + + >>> 'The cost was \u20ac12.'.encode('latin1', 'htmlentityreplace') + 'The cost was €12.' + """ + if isinstance(ex, UnicodeEncodeError): + # Handle encoding errors + bad_text = ex.object[ex.start : ex.end] + text = _html_entities_escaper.escape(bad_text) + return (str(text), ex.end) + raise ex + + +codecs.register_error("htmlentityreplace", htmlentityreplace_errors) + + +DEFAULT_ESCAPES = { + "x": "filters.xml_escape", + "h": "filters.html_escape", + "u": "filters.url_escape", + "trim": "filters.trim", + "entity": "filters.html_entities_escape", + "unicode": "str", + "decode": "decode", + "str": "str", + "n": "n", +} diff --git a/mako/lexer.py b/mako/lexer.py new file mode 100644 index 0000000..34f17dc --- /dev/null +++ b/mako/lexer.py @@ -0,0 +1,469 @@ +# mako/lexer.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""provides the Lexer class for parsing template strings into parse trees.""" + +import codecs +import re + +from mako import exceptions +from mako import parsetree +from mako.pygen import adjust_whitespace + +_regexp_cache = {} + + +class Lexer: + def __init__( + self, text, filename=None, input_encoding=None, preprocessor=None + ): + self.text = text + self.filename = filename + self.template = parsetree.TemplateNode(self.filename) + self.matched_lineno = 1 + self.matched_charpos = 0 + self.lineno = 1 + self.match_position = 0 + self.tag = [] + self.control_line = [] + self.ternary_stack = [] + self.encoding = input_encoding + + if preprocessor is None: + self.preprocessor = [] + elif not hasattr(preprocessor, "__iter__"): + self.preprocessor = [preprocessor] + else: + self.preprocessor = preprocessor + + @property + def exception_kwargs(self): + return { + "source": self.text, + "lineno": self.matched_lineno, + "pos": self.matched_charpos, + "filename": self.filename, + } + + def match(self, regexp, flags=None): + """compile the given regexp, cache the reg, and call match_reg().""" + + try: + reg = _regexp_cache[(regexp, flags)] + except KeyError: + reg = re.compile(regexp, flags) if flags else re.compile(regexp) + _regexp_cache[(regexp, flags)] = reg + + return self.match_reg(reg) + + def match_reg(self, reg): + """match the given regular expression object to the current text + position. + + if a match occurs, update the current text and line position. + + """ + + mp = self.match_position + + match = reg.match(self.text, self.match_position) + if match: + (start, end) = match.span() + self.match_position = end + 1 if end == start else end + self.matched_lineno = self.lineno + cp = mp - 1 + if cp >= 0 and cp < self.textlength: + cp = self.text[: cp + 1].rfind("\n") + self.matched_charpos = mp - cp + self.lineno += self.text[mp : self.match_position].count("\n") + return match + + def parse_until_text(self, watch_nesting, *text): + startpos = self.match_position + text_re = r"|".join(text) + brace_level = 0 + paren_level = 0 + bracket_level = 0 + while True: + match = self.match(r"#.*\n") + if match: + continue + match = self.match( + r"(\"\"\"|\'\'\'|\"|\')[^\\]*?(\\.[^\\]*?)*\1", re.S + ) + if match: + continue + match = self.match(r"(%s)" % text_re) + if match and not ( + watch_nesting + and (brace_level > 0 or paren_level > 0 or bracket_level > 0) + ): + return ( + self.text[ + startpos : self.match_position - len(match.group(1)) + ], + match.group(1), + ) + elif not match: + match = self.match(r"(.*?)(?=\"|\'|#|%s)" % text_re, re.S) + if match: + brace_level += match.group(1).count("{") + brace_level -= match.group(1).count("}") + paren_level += match.group(1).count("(") + paren_level -= match.group(1).count(")") + bracket_level += match.group(1).count("[") + bracket_level -= match.group(1).count("]") + continue + raise exceptions.SyntaxException( + "Expected: %s" % ",".join(text), **self.exception_kwargs + ) + + def append_node(self, nodecls, *args, **kwargs): + kwargs.setdefault("source", self.text) + kwargs.setdefault("lineno", self.matched_lineno) + kwargs.setdefault("pos", self.matched_charpos) + kwargs["filename"] = self.filename + node = nodecls(*args, **kwargs) + if len(self.tag): + self.tag[-1].nodes.append(node) + else: + self.template.nodes.append(node) + # build a set of child nodes for the control line + # (used for loop variable detection) + # also build a set of child nodes on ternary control lines + # (used for determining if a pass needs to be auto-inserted + if self.control_line: + control_frame = self.control_line[-1] + control_frame.nodes.append(node) + if ( + not ( + isinstance(node, parsetree.ControlLine) + and control_frame.is_ternary(node.keyword) + ) + and self.ternary_stack + and self.ternary_stack[-1] + ): + self.ternary_stack[-1][-1].nodes.append(node) + if isinstance(node, parsetree.Tag): + if len(self.tag): + node.parent = self.tag[-1] + self.tag.append(node) + elif isinstance(node, parsetree.ControlLine): + if node.isend: + self.control_line.pop() + self.ternary_stack.pop() + elif node.is_primary: + self.control_line.append(node) + self.ternary_stack.append([]) + elif self.control_line and self.control_line[-1].is_ternary( + node.keyword + ): + self.ternary_stack[-1].append(node) + elif self.control_line and not self.control_line[-1].is_ternary( + node.keyword + ): + raise exceptions.SyntaxException( + "Keyword '%s' not a legal ternary for keyword '%s'" + % (node.keyword, self.control_line[-1].keyword), + **self.exception_kwargs, + ) + + _coding_re = re.compile(r"#.*coding[:=]\s*([-\w.]+).*\r?\n") + + def decode_raw_stream(self, text, decode_raw, known_encoding, filename): + """given string/unicode or bytes/string, determine encoding + from magic encoding comment, return body as unicode + or raw if decode_raw=False + + """ + if isinstance(text, str): + m = self._coding_re.match(text) + encoding = m and m.group(1) or known_encoding or "utf-8" + return encoding, text + + if text.startswith(codecs.BOM_UTF8): + text = text[len(codecs.BOM_UTF8) :] + parsed_encoding = "utf-8" + m = self._coding_re.match(text.decode("utf-8", "ignore")) + if m is not None and m.group(1) != "utf-8": + raise exceptions.CompileException( + "Found utf-8 BOM in file, with conflicting " + "magic encoding comment of '%s'" % m.group(1), + text.decode("utf-8", "ignore"), + 0, + 0, + filename, + ) + else: + m = self._coding_re.match(text.decode("utf-8", "ignore")) + parsed_encoding = m.group(1) if m else known_encoding or "utf-8" + if decode_raw: + try: + text = text.decode(parsed_encoding) + except UnicodeDecodeError: + raise exceptions.CompileException( + "Unicode decode operation of encoding '%s' failed" + % parsed_encoding, + text.decode("utf-8", "ignore"), + 0, + 0, + filename, + ) + + return parsed_encoding, text + + def parse(self): + self.encoding, self.text = self.decode_raw_stream( + self.text, True, self.encoding, self.filename + ) + + for preproc in self.preprocessor: + self.text = preproc(self.text) + + # push the match marker past the + # encoding comment. + self.match_reg(self._coding_re) + + self.textlength = len(self.text) + + while True: + if self.match_position > self.textlength: + break + + if self.match_end(): + break + if self.match_expression(): + continue + if self.match_control_line(): + continue + if self.match_comment(): + continue + if self.match_tag_start(): + continue + if self.match_tag_end(): + continue + if self.match_python_block(): + continue + if self.match_text(): + continue + + if self.match_position > self.textlength: + break + # TODO: no coverage here + raise exceptions.MakoException("assertion failed") + + if len(self.tag): + raise exceptions.SyntaxException( + "Unclosed tag: <%%%s>" % self.tag[-1].keyword, + **self.exception_kwargs, + ) + if len(self.control_line): + raise exceptions.SyntaxException( + "Unterminated control keyword: '%s'" + % self.control_line[-1].keyword, + self.text, + self.control_line[-1].lineno, + self.control_line[-1].pos, + self.filename, + ) + return self.template + + def match_tag_start(self): + reg = r""" + \<% # opening tag + + ([\w\.\:]+) # keyword + + ((?:\s+\w+|\s*=\s*|"[^"]*?"|'[^']*?'|\s*,\s*)*) # attrname, = \ + # sign, string expression + # comma is for backwards compat + # identified in #366 + + \s* # more whitespace + + (/)?> # closing + + """ + + match = self.match( + reg, + re.I | re.S | re.X, + ) + + if not match: + return False + + keyword, attr, isend = match.groups() + self.keyword = keyword + attributes = {} + if attr: + for att in re.findall( + r"\s*(\w+)\s*=\s*(?:'([^']*)'|\"([^\"]*)\")", attr + ): + key, val1, val2 = att + text = val1 or val2 + text = text.replace("\r\n", "\n") + attributes[key] = text + self.append_node(parsetree.Tag, keyword, attributes) + if isend: + self.tag.pop() + elif keyword == "text": + match = self.match(r"(.*?)(?=\</%text>)", re.S) + if not match: + raise exceptions.SyntaxException( + "Unclosed tag: <%%%s>" % self.tag[-1].keyword, + **self.exception_kwargs, + ) + self.append_node(parsetree.Text, match.group(1)) + return self.match_tag_end() + return True + + def match_tag_end(self): + match = self.match(r"\</%[\t ]*([^\t ]+?)[\t ]*>") + if match: + if not len(self.tag): + raise exceptions.SyntaxException( + "Closing tag without opening tag: </%%%s>" + % match.group(1), + **self.exception_kwargs, + ) + elif self.tag[-1].keyword != match.group(1): + raise exceptions.SyntaxException( + "Closing tag </%%%s> does not match tag: <%%%s>" + % (match.group(1), self.tag[-1].keyword), + **self.exception_kwargs, + ) + self.tag.pop() + return True + else: + return False + + def match_end(self): + match = self.match(r"\Z", re.S) + if not match: + return False + + string = match.group() + if string: + return string + else: + return True + + def match_text(self): + match = self.match( + r""" + (.*?) # anything, followed by: + ( + (?<=\n)(?=[ \t]*(?=%|\#\#)) # an eval or line-based + # comment preceded by a + # consumed newline and whitespace + | + (?=\${) # an expression + | + (?=</?[%&]) # a substitution or block or call start or end + # - don't consume + | + (\\\r?\n) # an escaped newline - throw away + | + \Z # end of string + )""", + re.X | re.S, + ) + + if match: + text = match.group(1) + if text: + self.append_node(parsetree.Text, text) + return True + else: + return False + + def match_python_block(self): + match = self.match(r"<%(!)?") + if match: + line, pos = self.matched_lineno, self.matched_charpos + text, end = self.parse_until_text(False, r"%>") + # the trailing newline helps + # compiler.parse() not complain about indentation + text = adjust_whitespace(text) + "\n" + self.append_node( + parsetree.Code, + text, + match.group(1) == "!", + lineno=line, + pos=pos, + ) + return True + else: + return False + + def match_expression(self): + match = self.match(r"\${") + if not match: + return False + + line, pos = self.matched_lineno, self.matched_charpos + text, end = self.parse_until_text(True, r"\|", r"}") + if end == "|": + escapes, end = self.parse_until_text(True, r"}") + else: + escapes = "" + text = text.replace("\r\n", "\n") + self.append_node( + parsetree.Expression, + text, + escapes.strip(), + lineno=line, + pos=pos, + ) + return True + + def match_control_line(self): + match = self.match( + r"(?<=^)[\t ]*(%(?!%)|##)[\t ]*((?:(?:\\\r?\n)|[^\r\n])*)" + r"(?:\r?\n|\Z)", + re.M, + ) + if not match: + return False + + operator = match.group(1) + text = match.group(2) + if operator == "%": + m2 = re.match(r"(end)?(\w+)\s*(.*)", text) + if not m2: + raise exceptions.SyntaxException( + "Invalid control line: '%s'" % text, + **self.exception_kwargs, + ) + isend, keyword = m2.group(1, 2) + isend = isend is not None + + if isend: + if not len(self.control_line): + raise exceptions.SyntaxException( + "No starting keyword '%s' for '%s'" % (keyword, text), + **self.exception_kwargs, + ) + elif self.control_line[-1].keyword != keyword: + raise exceptions.SyntaxException( + "Keyword '%s' doesn't match keyword '%s'" + % (text, self.control_line[-1].keyword), + **self.exception_kwargs, + ) + self.append_node(parsetree.ControlLine, keyword, isend, text) + else: + self.append_node(parsetree.Comment, text) + return True + + def match_comment(self): + """matches the multiline version of a comment""" + match = self.match(r"<%doc>(.*?)</%doc>", re.S) + if match: + self.append_node(parsetree.Comment, match.group(1)) + return True + else: + return False diff --git a/mako/lookup.py b/mako/lookup.py new file mode 100644 index 0000000..ea1aec6 --- /dev/null +++ b/mako/lookup.py @@ -0,0 +1,361 @@ +# mako/lookup.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +import os +import posixpath +import re +import stat +import threading + +from mako import exceptions +from mako import util +from mako.template import Template + + +class TemplateCollection: + + """Represent a collection of :class:`.Template` objects, + identifiable via URI. + + A :class:`.TemplateCollection` is linked to the usage of + all template tags that address other templates, such + as ``<%include>``, ``<%namespace>``, and ``<%inherit>``. + The ``file`` attribute of each of those tags refers + to a string URI that is passed to that :class:`.Template` + object's :class:`.TemplateCollection` for resolution. + + :class:`.TemplateCollection` is an abstract class, + with the usual default implementation being :class:`.TemplateLookup`. + + """ + + def has_template(self, uri): + """Return ``True`` if this :class:`.TemplateLookup` is + capable of returning a :class:`.Template` object for the + given ``uri``. + + :param uri: String URI of the template to be resolved. + + """ + try: + self.get_template(uri) + return True + except exceptions.TemplateLookupException: + return False + + def get_template(self, uri, relativeto=None): + """Return a :class:`.Template` object corresponding to the given + ``uri``. + + The default implementation raises + :class:`.NotImplementedError`. Implementations should + raise :class:`.TemplateLookupException` if the given ``uri`` + cannot be resolved. + + :param uri: String URI of the template to be resolved. + :param relativeto: if present, the given ``uri`` is assumed to + be relative to this URI. + + """ + raise NotImplementedError() + + def filename_to_uri(self, uri, filename): + """Convert the given ``filename`` to a URI relative to + this :class:`.TemplateCollection`.""" + + return uri + + def adjust_uri(self, uri, filename): + """Adjust the given ``uri`` based on the calling ``filename``. + + When this method is called from the runtime, the + ``filename`` parameter is taken directly to the ``filename`` + attribute of the calling template. Therefore a custom + :class:`.TemplateCollection` subclass can place any string + identifier desired in the ``filename`` parameter of the + :class:`.Template` objects it constructs and have them come back + here. + + """ + return uri + + +class TemplateLookup(TemplateCollection): + + """Represent a collection of templates that locates template source files + from the local filesystem. + + The primary argument is the ``directories`` argument, the list of + directories to search: + + .. sourcecode:: python + + lookup = TemplateLookup(["/path/to/templates"]) + some_template = lookup.get_template("/index.html") + + The :class:`.TemplateLookup` can also be given :class:`.Template` objects + programatically using :meth:`.put_string` or :meth:`.put_template`: + + .. sourcecode:: python + + lookup = TemplateLookup() + lookup.put_string("base.html", ''' + <html><body>${self.next()}</body></html> + ''') + lookup.put_string("hello.html", ''' + <%include file='base.html'/> + + Hello, world ! + ''') + + + :param directories: A list of directory names which will be + searched for a particular template URI. The URI is appended + to each directory and the filesystem checked. + + :param collection_size: Approximate size of the collection used + to store templates. If left at its default of ``-1``, the size + is unbounded, and a plain Python dictionary is used to + relate URI strings to :class:`.Template` instances. + Otherwise, a least-recently-used cache object is used which + will maintain the size of the collection approximately to + the number given. + + :param filesystem_checks: When at its default value of ``True``, + each call to :meth:`.TemplateLookup.get_template()` will + compare the filesystem last modified time to the time in + which an existing :class:`.Template` object was created. + This allows the :class:`.TemplateLookup` to regenerate a + new :class:`.Template` whenever the original source has + been updated. Set this to ``False`` for a very minor + performance increase. + + :param modulename_callable: A callable which, when present, + is passed the path of the source file as well as the + requested URI, and then returns the full path of the + generated Python module file. This is used to inject + alternate schemes for Python module location. If left at + its default of ``None``, the built in system of generation + based on ``module_directory`` plus ``uri`` is used. + + All other keyword parameters available for + :class:`.Template` are mirrored here. When new + :class:`.Template` objects are created, the keywords + established with this :class:`.TemplateLookup` are passed on + to each new :class:`.Template`. + + """ + + def __init__( + self, + directories=None, + module_directory=None, + filesystem_checks=True, + collection_size=-1, + format_exceptions=False, + error_handler=None, + output_encoding=None, + encoding_errors="strict", + cache_args=None, + cache_impl="beaker", + cache_enabled=True, + cache_type=None, + cache_dir=None, + cache_url=None, + modulename_callable=None, + module_writer=None, + default_filters=None, + buffer_filters=(), + strict_undefined=False, + imports=None, + future_imports=None, + enable_loop=True, + input_encoding=None, + preprocessor=None, + lexer_cls=None, + include_error_handler=None, + ): + self.directories = [ + posixpath.normpath(d) for d in util.to_list(directories, ()) + ] + self.module_directory = module_directory + self.modulename_callable = modulename_callable + self.filesystem_checks = filesystem_checks + self.collection_size = collection_size + + if cache_args is None: + cache_args = {} + # transfer deprecated cache_* args + if cache_dir: + cache_args.setdefault("dir", cache_dir) + if cache_url: + cache_args.setdefault("url", cache_url) + if cache_type: + cache_args.setdefault("type", cache_type) + + self.template_args = { + "format_exceptions": format_exceptions, + "error_handler": error_handler, + "include_error_handler": include_error_handler, + "output_encoding": output_encoding, + "cache_impl": cache_impl, + "encoding_errors": encoding_errors, + "input_encoding": input_encoding, + "module_directory": module_directory, + "module_writer": module_writer, + "cache_args": cache_args, + "cache_enabled": cache_enabled, + "default_filters": default_filters, + "buffer_filters": buffer_filters, + "strict_undefined": strict_undefined, + "imports": imports, + "future_imports": future_imports, + "enable_loop": enable_loop, + "preprocessor": preprocessor, + "lexer_cls": lexer_cls, + } + + if collection_size == -1: + self._collection = {} + self._uri_cache = {} + else: + self._collection = util.LRUCache(collection_size) + self._uri_cache = util.LRUCache(collection_size) + self._mutex = threading.Lock() + + def get_template(self, uri): + """Return a :class:`.Template` object corresponding to the given + ``uri``. + + .. note:: The ``relativeto`` argument is not supported here at + the moment. + + """ + + try: + if self.filesystem_checks: + return self._check(uri, self._collection[uri]) + else: + return self._collection[uri] + except KeyError as e: + u = re.sub(r"^\/+", "", uri) + for dir_ in self.directories: + # make sure the path seperators are posix - os.altsep is empty + # on POSIX and cannot be used. + dir_ = dir_.replace(os.path.sep, posixpath.sep) + srcfile = posixpath.normpath(posixpath.join(dir_, u)) + if os.path.isfile(srcfile): + return self._load(srcfile, uri) + else: + raise exceptions.TopLevelLookupException( + "Can't locate template for uri %r" % uri + ) from e + + def adjust_uri(self, uri, relativeto): + """Adjust the given ``uri`` based on the given relative URI.""" + + key = (uri, relativeto) + if key in self._uri_cache: + return self._uri_cache[key] + + if uri[0] == "/": + v = self._uri_cache[key] = uri + elif relativeto is not None: + v = self._uri_cache[key] = posixpath.join( + posixpath.dirname(relativeto), uri + ) + else: + v = self._uri_cache[key] = "/" + uri + return v + + def filename_to_uri(self, filename): + """Convert the given ``filename`` to a URI relative to + this :class:`.TemplateCollection`.""" + + try: + return self._uri_cache[filename] + except KeyError: + value = self._relativeize(filename) + self._uri_cache[filename] = value + return value + + def _relativeize(self, filename): + """Return the portion of a filename that is 'relative' + to the directories in this lookup. + + """ + + filename = posixpath.normpath(filename) + for dir_ in self.directories: + if filename[0 : len(dir_)] == dir_: + return filename[len(dir_) :] + else: + return None + + def _load(self, filename, uri): + self._mutex.acquire() + try: + try: + # try returning from collection one + # more time in case concurrent thread already loaded + return self._collection[uri] + except KeyError: + pass + try: + if self.modulename_callable is not None: + module_filename = self.modulename_callable(filename, uri) + else: + module_filename = None + self._collection[uri] = template = Template( + uri=uri, + filename=posixpath.normpath(filename), + lookup=self, + module_filename=module_filename, + **self.template_args, + ) + return template + except: + # if compilation fails etc, ensure + # template is removed from collection, + # re-raise + self._collection.pop(uri, None) + raise + finally: + self._mutex.release() + + def _check(self, uri, template): + if template.filename is None: + return template + + try: + template_stat = os.stat(template.filename) + if template.module._modified_time >= template_stat[stat.ST_MTIME]: + return template + self._collection.pop(uri, None) + return self._load(template.filename, uri) + except OSError as e: + self._collection.pop(uri, None) + raise exceptions.TemplateLookupException( + "Can't locate template for uri %r" % uri + ) from e + + def put_string(self, uri, text): + """Place a new :class:`.Template` object into this + :class:`.TemplateLookup`, based on the given string of + ``text``. + + """ + self._collection[uri] = Template( + text, lookup=self, uri=uri, **self.template_args + ) + + def put_template(self, uri, template): + """Place a new :class:`.Template` object into this + :class:`.TemplateLookup`, based on the given + :class:`.Template` object. + + """ + self._collection[uri] = template diff --git a/mako/parsetree.py b/mako/parsetree.py new file mode 100644 index 0000000..3d550b2 --- /dev/null +++ b/mako/parsetree.py @@ -0,0 +1,656 @@ +# mako/parsetree.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""defines the parse tree components for Mako templates.""" + +import re + +from mako import ast +from mako import exceptions +from mako import filters +from mako import util + + +class Node: + + """base class for a Node in the parse tree.""" + + def __init__(self, source, lineno, pos, filename): + self.source = source + self.lineno = lineno + self.pos = pos + self.filename = filename + + @property + def exception_kwargs(self): + return { + "source": self.source, + "lineno": self.lineno, + "pos": self.pos, + "filename": self.filename, + } + + def get_children(self): + return [] + + def accept_visitor(self, visitor): + def traverse(node): + for n in node.get_children(): + n.accept_visitor(visitor) + + method = getattr(visitor, "visit" + self.__class__.__name__, traverse) + method(self) + + +class TemplateNode(Node): + + """a 'container' node that stores the overall collection of nodes.""" + + def __init__(self, filename): + super().__init__("", 0, 0, filename) + self.nodes = [] + self.page_attributes = {} + + def get_children(self): + return self.nodes + + def __repr__(self): + return "TemplateNode(%s, %r)" % ( + util.sorted_dict_repr(self.page_attributes), + self.nodes, + ) + + +class ControlLine(Node): + + """defines a control line, a line-oriented python line or end tag. + + e.g.:: + + % if foo: + (markup) + % endif + + """ + + has_loop_context = False + + def __init__(self, keyword, isend, text, **kwargs): + super().__init__(**kwargs) + self.text = text + self.keyword = keyword + self.isend = isend + self.is_primary = keyword in ["for", "if", "while", "try", "with"] + self.nodes = [] + if self.isend: + self._declared_identifiers = [] + self._undeclared_identifiers = [] + else: + code = ast.PythonFragment(text, **self.exception_kwargs) + self._declared_identifiers = code.declared_identifiers + self._undeclared_identifiers = code.undeclared_identifiers + + def get_children(self): + return self.nodes + + def declared_identifiers(self): + return self._declared_identifiers + + def undeclared_identifiers(self): + return self._undeclared_identifiers + + def is_ternary(self, keyword): + """return true if the given keyword is a ternary keyword + for this ControlLine""" + + cases = { + "if": {"else", "elif"}, + "try": {"except", "finally"}, + "for": {"else"}, + } + + return keyword in cases.get(self.keyword, set()) + + def __repr__(self): + return "ControlLine(%r, %r, %r, %r)" % ( + self.keyword, + self.text, + self.isend, + (self.lineno, self.pos), + ) + + +class Text(Node): + """defines plain text in the template.""" + + def __init__(self, content, **kwargs): + super().__init__(**kwargs) + self.content = content + + def __repr__(self): + return "Text(%r, %r)" % (self.content, (self.lineno, self.pos)) + + +class Code(Node): + """defines a Python code block, either inline or module level. + + e.g.:: + + inline: + <% + x = 12 + %> + + module level: + <%! + import logger + %> + + """ + + def __init__(self, text, ismodule, **kwargs): + super().__init__(**kwargs) + self.text = text + self.ismodule = ismodule + self.code = ast.PythonCode(text, **self.exception_kwargs) + + def declared_identifiers(self): + return self.code.declared_identifiers + + def undeclared_identifiers(self): + return self.code.undeclared_identifiers + + def __repr__(self): + return "Code(%r, %r, %r)" % ( + self.text, + self.ismodule, + (self.lineno, self.pos), + ) + + +class Comment(Node): + """defines a comment line. + + # this is a comment + + """ + + def __init__(self, text, **kwargs): + super().__init__(**kwargs) + self.text = text + + def __repr__(self): + return "Comment(%r, %r)" % (self.text, (self.lineno, self.pos)) + + +class Expression(Node): + """defines an inline expression. + + ${x+y} + + """ + + def __init__(self, text, escapes, **kwargs): + super().__init__(**kwargs) + self.text = text + self.escapes = escapes + self.escapes_code = ast.ArgumentList(escapes, **self.exception_kwargs) + self.code = ast.PythonCode(text, **self.exception_kwargs) + + def declared_identifiers(self): + return [] + + def undeclared_identifiers(self): + # TODO: make the "filter" shortcut list configurable at parse/gen time + return self.code.undeclared_identifiers.union( + self.escapes_code.undeclared_identifiers.difference( + filters.DEFAULT_ESCAPES + ) + ).difference(self.code.declared_identifiers) + + def __repr__(self): + return "Expression(%r, %r, %r)" % ( + self.text, + self.escapes_code.args, + (self.lineno, self.pos), + ) + + +class _TagMeta(type): + """metaclass to allow Tag to produce a subclass according to + its keyword""" + + _classmap = {} + + def __init__(cls, clsname, bases, dict_): + if getattr(cls, "__keyword__", None) is not None: + cls._classmap[cls.__keyword__] = cls + super().__init__(clsname, bases, dict_) + + def __call__(cls, keyword, attributes, **kwargs): + if ":" in keyword: + ns, defname = keyword.split(":") + return type.__call__( + CallNamespaceTag, ns, defname, attributes, **kwargs + ) + + try: + cls = _TagMeta._classmap[keyword] + except KeyError: + raise exceptions.CompileException( + "No such tag: '%s'" % keyword, + source=kwargs["source"], + lineno=kwargs["lineno"], + pos=kwargs["pos"], + filename=kwargs["filename"], + ) + return type.__call__(cls, keyword, attributes, **kwargs) + + +class Tag(Node, metaclass=_TagMeta): + """abstract base class for tags. + + e.g.:: + + <%sometag/> + + <%someothertag> + stuff + </%someothertag> + + """ + + __keyword__ = None + + def __init__( + self, + keyword, + attributes, + expressions, + nonexpressions, + required, + **kwargs, + ): + r"""construct a new Tag instance. + + this constructor not called directly, and is only called + by subclasses. + + :param keyword: the tag keyword + + :param attributes: raw dictionary of attribute key/value pairs + + :param expressions: a set of identifiers that are legal attributes, + which can also contain embedded expressions + + :param nonexpressions: a set of identifiers that are legal + attributes, which cannot contain embedded expressions + + :param \**kwargs: + other arguments passed to the Node superclass (lineno, pos) + + """ + super().__init__(**kwargs) + self.keyword = keyword + self.attributes = attributes + self._parse_attributes(expressions, nonexpressions) + missing = [r for r in required if r not in self.parsed_attributes] + if len(missing): + raise exceptions.CompileException( + ( + "Missing attribute(s): %s" + % ",".join(repr(m) for m in missing) + ), + **self.exception_kwargs, + ) + + self.parent = None + self.nodes = [] + + def is_root(self): + return self.parent is None + + def get_children(self): + return self.nodes + + def _parse_attributes(self, expressions, nonexpressions): + undeclared_identifiers = set() + self.parsed_attributes = {} + for key in self.attributes: + if key in expressions: + expr = [] + for x in re.compile(r"(\${.+?})", re.S).split( + self.attributes[key] + ): + m = re.compile(r"^\${(.+?)}$", re.S).match(x) + if m: + code = ast.PythonCode( + m.group(1).rstrip(), **self.exception_kwargs + ) + # we aren't discarding "declared_identifiers" here, + # which we do so that list comprehension-declared + # variables aren't counted. As yet can't find a + # condition that requires it here. + undeclared_identifiers = undeclared_identifiers.union( + code.undeclared_identifiers + ) + expr.append("(%s)" % m.group(1)) + elif x: + expr.append(repr(x)) + self.parsed_attributes[key] = " + ".join(expr) or repr("") + elif key in nonexpressions: + if re.search(r"\${.+?}", self.attributes[key]): + raise exceptions.CompileException( + "Attribute '%s' in tag '%s' does not allow embedded " + "expressions" % (key, self.keyword), + **self.exception_kwargs, + ) + self.parsed_attributes[key] = repr(self.attributes[key]) + else: + raise exceptions.CompileException( + "Invalid attribute for tag '%s': '%s'" + % (self.keyword, key), + **self.exception_kwargs, + ) + self.expression_undeclared_identifiers = undeclared_identifiers + + def declared_identifiers(self): + return [] + + def undeclared_identifiers(self): + return self.expression_undeclared_identifiers + + def __repr__(self): + return "%s(%r, %s, %r, %r)" % ( + self.__class__.__name__, + self.keyword, + util.sorted_dict_repr(self.attributes), + (self.lineno, self.pos), + self.nodes, + ) + + +class IncludeTag(Tag): + __keyword__ = "include" + + def __init__(self, keyword, attributes, **kwargs): + super().__init__( + keyword, + attributes, + ("file", "import", "args"), + (), + ("file",), + **kwargs, + ) + self.page_args = ast.PythonCode( + "__DUMMY(%s)" % attributes.get("args", ""), **self.exception_kwargs + ) + + def declared_identifiers(self): + return [] + + def undeclared_identifiers(self): + identifiers = self.page_args.undeclared_identifiers.difference( + {"__DUMMY"} + ).difference(self.page_args.declared_identifiers) + return identifiers.union(super().undeclared_identifiers()) + + +class NamespaceTag(Tag): + __keyword__ = "namespace" + + def __init__(self, keyword, attributes, **kwargs): + super().__init__( + keyword, + attributes, + ("file",), + ("name", "inheritable", "import", "module"), + (), + **kwargs, + ) + + self.name = attributes.get("name", "__anon_%s" % hex(abs(id(self)))) + if "name" not in attributes and "import" not in attributes: + raise exceptions.CompileException( + "'name' and/or 'import' attributes are required " + "for <%namespace>", + **self.exception_kwargs, + ) + if "file" in attributes and "module" in attributes: + raise exceptions.CompileException( + "<%namespace> may only have one of 'file' or 'module'", + **self.exception_kwargs, + ) + + def declared_identifiers(self): + return [] + + +class TextTag(Tag): + __keyword__ = "text" + + def __init__(self, keyword, attributes, **kwargs): + super().__init__(keyword, attributes, (), ("filter"), (), **kwargs) + self.filter_args = ast.ArgumentList( + attributes.get("filter", ""), **self.exception_kwargs + ) + + def undeclared_identifiers(self): + return self.filter_args.undeclared_identifiers.difference( + filters.DEFAULT_ESCAPES.keys() + ).union(self.expression_undeclared_identifiers) + + +class DefTag(Tag): + __keyword__ = "def" + + def __init__(self, keyword, attributes, **kwargs): + expressions = ["buffered", "cached"] + [ + c for c in attributes if c.startswith("cache_") + ] + + super().__init__( + keyword, + attributes, + expressions, + ("name", "filter", "decorator"), + ("name",), + **kwargs, + ) + name = attributes["name"] + if re.match(r"^[\w_]+$", name): + raise exceptions.CompileException( + "Missing parenthesis in %def", **self.exception_kwargs + ) + self.function_decl = ast.FunctionDecl( + "def " + name + ":pass", **self.exception_kwargs + ) + self.name = self.function_decl.funcname + self.decorator = attributes.get("decorator", "") + self.filter_args = ast.ArgumentList( + attributes.get("filter", ""), **self.exception_kwargs + ) + + is_anonymous = False + is_block = False + + @property + def funcname(self): + return self.function_decl.funcname + + def get_argument_expressions(self, **kw): + return self.function_decl.get_argument_expressions(**kw) + + def declared_identifiers(self): + return self.function_decl.allargnames + + def undeclared_identifiers(self): + res = [] + for c in self.function_decl.defaults: + res += list( + ast.PythonCode( + c, **self.exception_kwargs + ).undeclared_identifiers + ) + return ( + set(res) + .union( + self.filter_args.undeclared_identifiers.difference( + filters.DEFAULT_ESCAPES.keys() + ) + ) + .union(self.expression_undeclared_identifiers) + .difference(self.function_decl.allargnames) + ) + + +class BlockTag(Tag): + __keyword__ = "block" + + def __init__(self, keyword, attributes, **kwargs): + expressions = ["buffered", "cached", "args"] + [ + c for c in attributes if c.startswith("cache_") + ] + + super().__init__( + keyword, + attributes, + expressions, + ("name", "filter", "decorator"), + (), + **kwargs, + ) + name = attributes.get("name") + if name and not re.match(r"^[\w_]+$", name): + raise exceptions.CompileException( + "%block may not specify an argument signature", + **self.exception_kwargs, + ) + if not name and attributes.get("args", None): + raise exceptions.CompileException( + "Only named %blocks may specify args", **self.exception_kwargs + ) + self.body_decl = ast.FunctionArgs( + attributes.get("args", ""), **self.exception_kwargs + ) + + self.name = name + self.decorator = attributes.get("decorator", "") + self.filter_args = ast.ArgumentList( + attributes.get("filter", ""), **self.exception_kwargs + ) + + is_block = True + + @property + def is_anonymous(self): + return self.name is None + + @property + def funcname(self): + return self.name or "__M_anon_%d" % (self.lineno,) + + def get_argument_expressions(self, **kw): + return self.body_decl.get_argument_expressions(**kw) + + def declared_identifiers(self): + return self.body_decl.allargnames + + def undeclared_identifiers(self): + return ( + self.filter_args.undeclared_identifiers.difference( + filters.DEFAULT_ESCAPES.keys() + ) + ).union(self.expression_undeclared_identifiers) + + +class CallTag(Tag): + __keyword__ = "call" + + def __init__(self, keyword, attributes, **kwargs): + super().__init__( + keyword, attributes, ("args"), ("expr",), ("expr",), **kwargs + ) + self.expression = attributes["expr"] + self.code = ast.PythonCode(self.expression, **self.exception_kwargs) + self.body_decl = ast.FunctionArgs( + attributes.get("args", ""), **self.exception_kwargs + ) + + def declared_identifiers(self): + return self.code.declared_identifiers.union(self.body_decl.allargnames) + + def undeclared_identifiers(self): + return self.code.undeclared_identifiers.difference( + self.code.declared_identifiers + ) + + +class CallNamespaceTag(Tag): + def __init__(self, namespace, defname, attributes, **kwargs): + super().__init__( + namespace + ":" + defname, + attributes, + tuple(attributes.keys()) + ("args",), + (), + (), + **kwargs, + ) + + self.expression = "%s.%s(%s)" % ( + namespace, + defname, + ",".join( + "%s=%s" % (k, v) + for k, v in self.parsed_attributes.items() + if k != "args" + ), + ) + + self.code = ast.PythonCode(self.expression, **self.exception_kwargs) + self.body_decl = ast.FunctionArgs( + attributes.get("args", ""), **self.exception_kwargs + ) + + def declared_identifiers(self): + return self.code.declared_identifiers.union(self.body_decl.allargnames) + + def undeclared_identifiers(self): + return self.code.undeclared_identifiers.difference( + self.code.declared_identifiers + ) + + +class InheritTag(Tag): + __keyword__ = "inherit" + + def __init__(self, keyword, attributes, **kwargs): + super().__init__( + keyword, attributes, ("file",), (), ("file",), **kwargs + ) + + +class PageTag(Tag): + __keyword__ = "page" + + def __init__(self, keyword, attributes, **kwargs): + expressions = [ + "cached", + "args", + "expression_filter", + "enable_loop", + ] + [c for c in attributes if c.startswith("cache_")] + + super().__init__(keyword, attributes, expressions, (), (), **kwargs) + self.body_decl = ast.FunctionArgs( + attributes.get("args", ""), **self.exception_kwargs + ) + self.filter_args = ast.ArgumentList( + attributes.get("expression_filter", ""), **self.exception_kwargs + ) + + def declared_identifiers(self): + return self.body_decl.allargnames diff --git a/mako/pygen.py b/mako/pygen.py new file mode 100644 index 0000000..baeb93a --- /dev/null +++ b/mako/pygen.py @@ -0,0 +1,309 @@ +# mako/pygen.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""utilities for generating and formatting literal Python code.""" + +import re + +from mako import exceptions + + +class PythonPrinter: + def __init__(self, stream): + # indentation counter + self.indent = 0 + + # a stack storing information about why we incremented + # the indentation counter, to help us determine if we + # should decrement it + self.indent_detail = [] + + # the string of whitespace multiplied by the indent + # counter to produce a line + self.indentstring = " " + + # the stream we are writing to + self.stream = stream + + # current line number + self.lineno = 1 + + # a list of lines that represents a buffered "block" of code, + # which can be later printed relative to an indent level + self.line_buffer = [] + + self.in_indent_lines = False + + self._reset_multi_line_flags() + + # mapping of generated python lines to template + # source lines + self.source_map = {} + + self._re_space_comment = re.compile(r"^\s*#") + self._re_space = re.compile(r"^\s*$") + self._re_indent = re.compile(r":[ \t]*(?:#.*)?$") + self._re_compound = re.compile(r"^\s*(if|try|elif|while|for|with)") + self._re_indent_keyword = re.compile( + r"^\s*(def|class|else|elif|except|finally)" + ) + self._re_unindentor = re.compile(r"^\s*(else|elif|except|finally).*\:") + + def _update_lineno(self, num): + self.lineno += num + + def start_source(self, lineno): + if self.lineno not in self.source_map: + self.source_map[self.lineno] = lineno + + def write_blanks(self, num): + self.stream.write("\n" * num) + self._update_lineno(num) + + def write_indented_block(self, block, starting_lineno=None): + """print a line or lines of python which already contain indentation. + + The indentation of the total block of lines will be adjusted to that of + the current indent level.""" + self.in_indent_lines = False + for i, l in enumerate(re.split(r"\r?\n", block)): + self.line_buffer.append(l) + if starting_lineno is not None: + self.start_source(starting_lineno + i) + self._update_lineno(1) + + def writelines(self, *lines): + """print a series of lines of python.""" + for line in lines: + self.writeline(line) + + def writeline(self, line): + """print a line of python, indenting it according to the current + indent level. + + this also adjusts the indentation counter according to the + content of the line. + + """ + + if not self.in_indent_lines: + self._flush_adjusted_lines() + self.in_indent_lines = True + + if ( + line is None + or self._re_space_comment.match(line) + or self._re_space.match(line) + ): + hastext = False + else: + hastext = True + + is_comment = line and len(line) and line[0] == "#" + + # see if this line should decrease the indentation level + if ( + not is_comment + and (not hastext or self._is_unindentor(line)) + and self.indent > 0 + ): + self.indent -= 1 + # if the indent_detail stack is empty, the user + # probably put extra closures - the resulting + # module wont compile. + if len(self.indent_detail) == 0: + # TODO: no coverage here + raise exceptions.MakoException("Too many whitespace closures") + self.indent_detail.pop() + + if line is None: + return + + # write the line + self.stream.write(self._indent_line(line) + "\n") + self._update_lineno(len(line.split("\n"))) + + # see if this line should increase the indentation level. + # note that a line can both decrase (before printing) and + # then increase (after printing) the indentation level. + + if self._re_indent.search(line): + # increment indentation count, and also + # keep track of what the keyword was that indented us, + # if it is a python compound statement keyword + # where we might have to look for an "unindent" keyword + match = self._re_compound.match(line) + if match: + # its a "compound" keyword, so we will check for "unindentors" + indentor = match.group(1) + self.indent += 1 + self.indent_detail.append(indentor) + else: + indentor = None + # its not a "compound" keyword. but lets also + # test for valid Python keywords that might be indenting us, + # else assume its a non-indenting line + m2 = self._re_indent_keyword.match(line) + if m2: + self.indent += 1 + self.indent_detail.append(indentor) + + def close(self): + """close this printer, flushing any remaining lines.""" + self._flush_adjusted_lines() + + def _is_unindentor(self, line): + """return true if the given line is an 'unindentor', + relative to the last 'indent' event received. + + """ + + # no indentation detail has been pushed on; return False + if len(self.indent_detail) == 0: + return False + + indentor = self.indent_detail[-1] + + # the last indent keyword we grabbed is not a + # compound statement keyword; return False + if indentor is None: + return False + + # if the current line doesnt have one of the "unindentor" keywords, + # return False + match = self._re_unindentor.match(line) + # if True, whitespace matches up, we have a compound indentor, + # and this line has an unindentor, this + # is probably good enough + return bool(match) + + # should we decide that its not good enough, heres + # more stuff to check. + # keyword = match.group(1) + + # match the original indent keyword + # for crit in [ + # (r'if|elif', r'else|elif'), + # (r'try', r'except|finally|else'), + # (r'while|for', r'else'), + # ]: + # if re.match(crit[0], indentor) and re.match(crit[1], keyword): + # return True + + # return False + + def _indent_line(self, line, stripspace=""): + """indent the given line according to the current indent level. + + stripspace is a string of space that will be truncated from the + start of the line before indenting.""" + if stripspace == "": + # Fast path optimization. + return self.indentstring * self.indent + line + + return re.sub( + r"^%s" % stripspace, self.indentstring * self.indent, line + ) + + def _reset_multi_line_flags(self): + """reset the flags which would indicate we are in a backslashed + or triple-quoted section.""" + + self.backslashed, self.triplequoted = False, False + + def _in_multi_line(self, line): + """return true if the given line is part of a multi-line block, + via backslash or triple-quote.""" + + # we are only looking for explicitly joined lines here, not + # implicit ones (i.e. brackets, braces etc.). this is just to + # guard against the possibility of modifying the space inside of + # a literal multiline string with unfortunately placed + # whitespace + + current_state = self.backslashed or self.triplequoted + + self.backslashed = bool(re.search(r"\\$", line)) + triples = len(re.findall(r"\"\"\"|\'\'\'", line)) + if triples == 1 or triples % 2 != 0: + self.triplequoted = not self.triplequoted + + return current_state + + def _flush_adjusted_lines(self): + stripspace = None + self._reset_multi_line_flags() + + for entry in self.line_buffer: + if self._in_multi_line(entry): + self.stream.write(entry + "\n") + else: + entry = entry.expandtabs() + if stripspace is None and re.search(r"^[ \t]*[^# \t]", entry): + stripspace = re.match(r"^([ \t]*)", entry).group(1) + self.stream.write(self._indent_line(entry, stripspace) + "\n") + + self.line_buffer = [] + self._reset_multi_line_flags() + + +def adjust_whitespace(text): + """remove the left-whitespace margin of a block of Python code.""" + + state = [False, False] + (backslashed, triplequoted) = (0, 1) + + def in_multi_line(line): + start_state = state[backslashed] or state[triplequoted] + + if re.search(r"\\$", line): + state[backslashed] = True + else: + state[backslashed] = False + + def match(reg, t): + m = re.match(reg, t) + if m: + return m, t[len(m.group(0)) :] + else: + return None, t + + while line: + if state[triplequoted]: + m, line = match(r"%s" % state[triplequoted], line) + if m: + state[triplequoted] = False + else: + m, line = match(r".*?(?=%s|$)" % state[triplequoted], line) + else: + m, line = match(r"#", line) + if m: + return start_state + + m, line = match(r"\"\"\"|\'\'\'", line) + if m: + state[triplequoted] = m.group(0) + continue + + m, line = match(r".*?(?=\"\"\"|\'\'\'|#|$)", line) + + return start_state + + def _indent_line(line, stripspace=""): + return re.sub(r"^%s" % stripspace, "", line) + + lines = [] + stripspace = None + + for line in re.split(r"\r?\n", text): + if in_multi_line(line): + lines.append(line) + else: + line = line.expandtabs() + if stripspace is None and re.search(r"^[ \t]*[^# \t]", line): + stripspace = re.match(r"^([ \t]*)", line).group(1) + lines.append(_indent_line(line, stripspace)) + return "\n".join(lines) diff --git a/mako/pyparser.py b/mako/pyparser.py new file mode 100644 index 0000000..68218a0 --- /dev/null +++ b/mako/pyparser.py @@ -0,0 +1,217 @@ +# mako/pyparser.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""Handles parsing of Python code. + +Parsing to AST is done via _ast on Python > 2.5, otherwise the compiler +module is used. +""" + +import operator + +import _ast + +from mako import _ast_util +from mako import compat +from mako import exceptions +from mako import util + +# words that cannot be assigned to (notably +# smaller than the total keys in __builtins__) +reserved = {"True", "False", "None", "print"} + +# the "id" attribute on a function node +arg_id = operator.attrgetter("arg") + +util.restore__ast(_ast) + + +def parse(code, mode="exec", **exception_kwargs): + """Parse an expression into AST""" + + try: + return _ast_util.parse(code, "<unknown>", mode) + except Exception as e: + raise exceptions.SyntaxException( + "(%s) %s (%r)" + % ( + compat.exception_as().__class__.__name__, + compat.exception_as(), + code[0:50], + ), + **exception_kwargs, + ) from e + + +class FindIdentifiers(_ast_util.NodeVisitor): + def __init__(self, listener, **exception_kwargs): + self.in_function = False + self.in_assign_targets = False + self.local_ident_stack = set() + self.listener = listener + self.exception_kwargs = exception_kwargs + + def _add_declared(self, name): + if not self.in_function: + self.listener.declared_identifiers.add(name) + else: + self.local_ident_stack.add(name) + + def visit_ClassDef(self, node): + self._add_declared(node.name) + + def visit_Assign(self, node): + # flip around the visiting of Assign so the expression gets + # evaluated first, in the case of a clause like "x=x+5" (x + # is undeclared) + + self.visit(node.value) + in_a = self.in_assign_targets + self.in_assign_targets = True + for n in node.targets: + self.visit(n) + self.in_assign_targets = in_a + + def visit_ExceptHandler(self, node): + if node.name is not None: + self._add_declared(node.name) + if node.type is not None: + self.visit(node.type) + for statement in node.body: + self.visit(statement) + + def visit_Lambda(self, node, *args): + self._visit_function(node, True) + + def visit_FunctionDef(self, node): + self._add_declared(node.name) + self._visit_function(node, False) + + def _expand_tuples(self, args): + for arg in args: + if isinstance(arg, _ast.Tuple): + yield from arg.elts + else: + yield arg + + def _visit_function(self, node, islambda): + # push function state onto stack. dont log any more + # identifiers as "declared" until outside of the function, + # but keep logging identifiers as "undeclared". track + # argument names in each function header so they arent + # counted as "undeclared" + + inf = self.in_function + self.in_function = True + + local_ident_stack = self.local_ident_stack + self.local_ident_stack = local_ident_stack.union( + [arg_id(arg) for arg in self._expand_tuples(node.args.args)] + ) + if islambda: + self.visit(node.body) + else: + for n in node.body: + self.visit(n) + self.in_function = inf + self.local_ident_stack = local_ident_stack + + def visit_For(self, node): + # flip around visit + + self.visit(node.iter) + self.visit(node.target) + for statement in node.body: + self.visit(statement) + for statement in node.orelse: + self.visit(statement) + + def visit_Name(self, node): + if isinstance(node.ctx, _ast.Store): + # this is eqiuvalent to visit_AssName in + # compiler + self._add_declared(node.id) + elif ( + node.id not in reserved + and node.id not in self.listener.declared_identifiers + and node.id not in self.local_ident_stack + ): + self.listener.undeclared_identifiers.add(node.id) + + def visit_Import(self, node): + for name in node.names: + if name.asname is not None: + self._add_declared(name.asname) + else: + self._add_declared(name.name.split(".")[0]) + + def visit_ImportFrom(self, node): + for name in node.names: + if name.asname is not None: + self._add_declared(name.asname) + elif name.name == "*": + raise exceptions.CompileException( + "'import *' is not supported, since all identifier " + "names must be explicitly declared. Please use the " + "form 'from <modulename> import <name1>, <name2>, " + "...' instead.", + **self.exception_kwargs, + ) + else: + self._add_declared(name.name) + + +class FindTuple(_ast_util.NodeVisitor): + def __init__(self, listener, code_factory, **exception_kwargs): + self.listener = listener + self.exception_kwargs = exception_kwargs + self.code_factory = code_factory + + def visit_Tuple(self, node): + for n in node.elts: + p = self.code_factory(n, **self.exception_kwargs) + self.listener.codeargs.append(p) + self.listener.args.append(ExpressionGenerator(n).value()) + ldi = self.listener.declared_identifiers + self.listener.declared_identifiers = ldi.union( + p.declared_identifiers + ) + lui = self.listener.undeclared_identifiers + self.listener.undeclared_identifiers = lui.union( + p.undeclared_identifiers + ) + + +class ParseFunc(_ast_util.NodeVisitor): + def __init__(self, listener, **exception_kwargs): + self.listener = listener + self.exception_kwargs = exception_kwargs + + def visit_FunctionDef(self, node): + self.listener.funcname = node.name + + argnames = [arg_id(arg) for arg in node.args.args] + if node.args.vararg: + argnames.append(node.args.vararg.arg) + + kwargnames = [arg_id(arg) for arg in node.args.kwonlyargs] + if node.args.kwarg: + kwargnames.append(node.args.kwarg.arg) + self.listener.argnames = argnames + self.listener.defaults = node.args.defaults # ast + self.listener.kwargnames = kwargnames + self.listener.kwdefaults = node.args.kw_defaults + self.listener.varargs = node.args.vararg + self.listener.kwargs = node.args.kwarg + + +class ExpressionGenerator: + def __init__(self, astnode): + self.generator = _ast_util.SourceGenerator(" " * 4) + self.generator.visit(astnode) + + def value(self): + return "".join(self.generator.result) diff --git a/mako/runtime.py b/mako/runtime.py new file mode 100644 index 0000000..23401b7 --- /dev/null +++ b/mako/runtime.py @@ -0,0 +1,968 @@ +# mako/runtime.py +# Copyright 2006-2020 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""provides runtime services for templates, including Context, +Namespace, and various helper functions.""" + +import builtins +import functools +import sys + +from mako import compat +from mako import exceptions +from mako import util + + +class Context: + + """Provides runtime namespace, output buffer, and various + callstacks for templates. + + See :ref:`runtime_toplevel` for detail on the usage of + :class:`.Context`. + + """ + + def __init__(self, buffer, **data): + self._buffer_stack = [buffer] + + self._data = data + + self._kwargs = data.copy() + self._with_template = None + self._outputting_as_unicode = None + self.namespaces = {} + + # "capture" function which proxies to the + # generic "capture" function + self._data["capture"] = functools.partial(capture, self) + + # "caller" stack used by def calls with content + self.caller_stack = self._data["caller"] = CallerStack() + + def _set_with_template(self, t): + self._with_template = t + illegal_names = t.reserved_names.intersection(self._data) + if illegal_names: + raise exceptions.NameConflictError( + "Reserved words passed to render(): %s" + % ", ".join(illegal_names) + ) + + @property + def lookup(self): + """Return the :class:`.TemplateLookup` associated + with this :class:`.Context`. + + """ + return self._with_template.lookup + + @property + def kwargs(self): + """Return the dictionary of top level keyword arguments associated + with this :class:`.Context`. + + This dictionary only includes the top-level arguments passed to + :meth:`.Template.render`. It does not include names produced within + the template execution such as local variable names or special names + such as ``self``, ``next``, etc. + + The purpose of this dictionary is primarily for the case that + a :class:`.Template` accepts arguments via its ``<%page>`` tag, + which are normally expected to be passed via :meth:`.Template.render`, + except the template is being called in an inheritance context, + using the ``body()`` method. :attr:`.Context.kwargs` can then be + used to propagate these arguments to the inheriting template:: + + ${next.body(**context.kwargs)} + + """ + return self._kwargs.copy() + + def push_caller(self, caller): + """Push a ``caller`` callable onto the callstack for + this :class:`.Context`.""" + + self.caller_stack.append(caller) + + def pop_caller(self): + """Pop a ``caller`` callable onto the callstack for this + :class:`.Context`.""" + + del self.caller_stack[-1] + + def keys(self): + """Return a list of all names established in this :class:`.Context`.""" + + return list(self._data.keys()) + + def __getitem__(self, key): + if key in self._data: + return self._data[key] + else: + return builtins.__dict__[key] + + def _push_writer(self): + """push a capturing buffer onto this Context and return + the new writer function.""" + + buf = util.FastEncodingBuffer() + self._buffer_stack.append(buf) + return buf.write + + def _pop_buffer_and_writer(self): + """pop the most recent capturing buffer from this Context + and return the current writer after the pop. + + """ + + buf = self._buffer_stack.pop() + return buf, self._buffer_stack[-1].write + + def _push_buffer(self): + """push a capturing buffer onto this Context.""" + + self._push_writer() + + def _pop_buffer(self): + """pop the most recent capturing buffer from this Context.""" + + return self._buffer_stack.pop() + + def get(self, key, default=None): + """Return a value from this :class:`.Context`.""" + + return self._data.get(key, builtins.__dict__.get(key, default)) + + def write(self, string): + """Write a string to this :class:`.Context` object's + underlying output buffer.""" + + self._buffer_stack[-1].write(string) + + def writer(self): + """Return the current writer function.""" + + return self._buffer_stack[-1].write + + def _copy(self): + c = Context.__new__(Context) + c._buffer_stack = self._buffer_stack + c._data = self._data.copy() + c._kwargs = self._kwargs + c._with_template = self._with_template + c._outputting_as_unicode = self._outputting_as_unicode + c.namespaces = self.namespaces + c.caller_stack = self.caller_stack + return c + + def _locals(self, d): + """Create a new :class:`.Context` with a copy of this + :class:`.Context`'s current state, + updated with the given dictionary. + + The :attr:`.Context.kwargs` collection remains + unaffected. + + + """ + + if not d: + return self + c = self._copy() + c._data.update(d) + return c + + def _clean_inheritance_tokens(self): + """create a new copy of this :class:`.Context`. with + tokens related to inheritance state removed.""" + + c = self._copy() + x = c._data + x.pop("self", None) + x.pop("parent", None) + x.pop("next", None) + return c + + +class CallerStack(list): + def __init__(self): + self.nextcaller = None + + def __nonzero__(self): + return self.__bool__() + + def __bool__(self): + return len(self) and self._get_caller() and True or False + + def _get_caller(self): + # this method can be removed once + # codegen MAGIC_NUMBER moves past 7 + return self[-1] + + def __getattr__(self, key): + return getattr(self._get_caller(), key) + + def _push_frame(self): + frame = self.nextcaller or None + self.append(frame) + self.nextcaller = None + return frame + + def _pop_frame(self): + self.nextcaller = self.pop() + + +class Undefined: + + """Represents an undefined value in a template. + + All template modules have a constant value + ``UNDEFINED`` present which is an instance of this + object. + + """ + + def __str__(self): + raise NameError("Undefined") + + def __nonzero__(self): + return self.__bool__() + + def __bool__(self): + return False + + +UNDEFINED = Undefined() +STOP_RENDERING = "" + + +class LoopStack: + + """a stack for LoopContexts that implements the context manager protocol + to automatically pop off the top of the stack on context exit + """ + + def __init__(self): + self.stack = [] + + def _enter(self, iterable): + self._push(iterable) + return self._top + + def _exit(self): + self._pop() + return self._top + + @property + def _top(self): + if self.stack: + return self.stack[-1] + else: + return self + + def _pop(self): + return self.stack.pop() + + def _push(self, iterable): + new = LoopContext(iterable) + if self.stack: + new.parent = self.stack[-1] + return self.stack.append(new) + + def __getattr__(self, key): + raise exceptions.RuntimeException("No loop context is established") + + def __iter__(self): + return iter(self._top) + + +class LoopContext: + + """A magic loop variable. + Automatically accessible in any ``% for`` block. + + See the section :ref:`loop_context` for usage + notes. + + :attr:`parent` -> :class:`.LoopContext` or ``None`` + The parent loop, if one exists. + :attr:`index` -> `int` + The 0-based iteration count. + :attr:`reverse_index` -> `int` + The number of iterations remaining. + :attr:`first` -> `bool` + ``True`` on the first iteration, ``False`` otherwise. + :attr:`last` -> `bool` + ``True`` on the last iteration, ``False`` otherwise. + :attr:`even` -> `bool` + ``True`` when ``index`` is even. + :attr:`odd` -> `bool` + ``True`` when ``index`` is odd. + """ + + def __init__(self, iterable): + self._iterable = iterable + self.index = 0 + self.parent = None + + def __iter__(self): + for i in self._iterable: + yield i + self.index += 1 + + @util.memoized_instancemethod + def __len__(self): + return len(self._iterable) + + @property + def reverse_index(self): + return len(self) - self.index - 1 + + @property + def first(self): + return self.index == 0 + + @property + def last(self): + return self.index == len(self) - 1 + + @property + def even(self): + return not self.odd + + @property + def odd(self): + return bool(self.index % 2) + + def cycle(self, *values): + """Cycle through values as the loop progresses.""" + if not values: + raise ValueError("You must provide values to cycle through") + return values[self.index % len(values)] + + +class _NSAttr: + def __init__(self, parent): + self.__parent = parent + + def __getattr__(self, key): + ns = self.__parent + while ns: + if hasattr(ns.module, key): + return getattr(ns.module, key) + else: + ns = ns.inherits + raise AttributeError(key) + + +class Namespace: + + """Provides access to collections of rendering methods, which + can be local, from other templates, or from imported modules. + + To access a particular rendering method referenced by a + :class:`.Namespace`, use plain attribute access: + + .. sourcecode:: mako + + ${some_namespace.foo(x, y, z)} + + :class:`.Namespace` also contains several built-in attributes + described here. + + """ + + def __init__( + self, + name, + context, + callables=None, + inherits=None, + populate_self=True, + calling_uri=None, + ): + self.name = name + self.context = context + self.inherits = inherits + if callables is not None: + self.callables = {c.__name__: c for c in callables} + + callables = () + + module = None + """The Python module referenced by this :class:`.Namespace`. + + If the namespace references a :class:`.Template`, then + this module is the equivalent of ``template.module``, + i.e. the generated module for the template. + + """ + + template = None + """The :class:`.Template` object referenced by this + :class:`.Namespace`, if any. + + """ + + context = None + """The :class:`.Context` object for this :class:`.Namespace`. + + Namespaces are often created with copies of contexts that + contain slightly different data, particularly in inheritance + scenarios. Using the :class:`.Context` off of a :class:`.Namespace` one + can traverse an entire chain of templates that inherit from + one-another. + + """ + + filename = None + """The path of the filesystem file used for this + :class:`.Namespace`'s module or template. + + If this is a pure module-based + :class:`.Namespace`, this evaluates to ``module.__file__``. If a + template-based namespace, it evaluates to the original + template file location. + + """ + + uri = None + """The URI for this :class:`.Namespace`'s template. + + I.e. whatever was sent to :meth:`.TemplateLookup.get_template()`. + + This is the equivalent of :attr:`.Template.uri`. + + """ + + _templateuri = None + + @util.memoized_property + def attr(self): + """Access module level attributes by name. + + This accessor allows templates to supply "scalar" + attributes which are particularly handy in inheritance + relationships. + + .. seealso:: + + :ref:`inheritance_attr` + + :ref:`namespace_attr_for_includes` + + """ + return _NSAttr(self) + + def get_namespace(self, uri): + """Return a :class:`.Namespace` corresponding to the given ``uri``. + + If the given ``uri`` is a relative URI (i.e. it does not + contain a leading slash ``/``), the ``uri`` is adjusted to + be relative to the ``uri`` of the namespace itself. This + method is therefore mostly useful off of the built-in + ``local`` namespace, described in :ref:`namespace_local`. + + In + most cases, a template wouldn't need this function, and + should instead use the ``<%namespace>`` tag to load + namespaces. However, since all ``<%namespace>`` tags are + evaluated before the body of a template ever runs, + this method can be used to locate namespaces using + expressions that were generated within the body code of + the template, or to conditionally use a particular + namespace. + + """ + key = (self, uri) + if key in self.context.namespaces: + return self.context.namespaces[key] + ns = TemplateNamespace( + uri, + self.context._copy(), + templateuri=uri, + calling_uri=self._templateuri, + ) + self.context.namespaces[key] = ns + return ns + + def get_template(self, uri): + """Return a :class:`.Template` from the given ``uri``. + + The ``uri`` resolution is relative to the ``uri`` of this + :class:`.Namespace` object's :class:`.Template`. + + """ + return _lookup_template(self.context, uri, self._templateuri) + + def get_cached(self, key, **kwargs): + """Return a value from the :class:`.Cache` referenced by this + :class:`.Namespace` object's :class:`.Template`. + + The advantage to this method versus direct access to the + :class:`.Cache` is that the configuration parameters + declared in ``<%page>`` take effect here, thereby calling + up the same configured backend as that configured + by ``<%page>``. + + """ + + return self.cache.get(key, **kwargs) + + @property + def cache(self): + """Return the :class:`.Cache` object referenced + by this :class:`.Namespace` object's + :class:`.Template`. + + """ + return self.template.cache + + def include_file(self, uri, **kwargs): + """Include a file at the given ``uri``.""" + + _include_file(self.context, uri, self._templateuri, **kwargs) + + def _populate(self, d, l): + for ident in l: + if ident == "*": + for k, v in self._get_star(): + d[k] = v + else: + d[ident] = getattr(self, ident) + + def _get_star(self): + if self.callables: + for key in self.callables: + yield (key, self.callables[key]) + + def __getattr__(self, key): + if key in self.callables: + val = self.callables[key] + elif self.inherits: + val = getattr(self.inherits, key) + else: + raise AttributeError( + "Namespace '%s' has no member '%s'" % (self.name, key) + ) + setattr(self, key, val) + return val + + +class TemplateNamespace(Namespace): + + """A :class:`.Namespace` specific to a :class:`.Template` instance.""" + + def __init__( + self, + name, + context, + template=None, + templateuri=None, + callables=None, + inherits=None, + populate_self=True, + calling_uri=None, + ): + self.name = name + self.context = context + self.inherits = inherits + if callables is not None: + self.callables = {c.__name__: c for c in callables} + + if templateuri is not None: + self.template = _lookup_template(context, templateuri, calling_uri) + self._templateuri = self.template.module._template_uri + elif template is not None: + self.template = template + self._templateuri = template.module._template_uri + else: + raise TypeError("'template' argument is required.") + + if populate_self: + lclcallable, lclcontext = _populate_self_namespace( + context, self.template, self_ns=self + ) + + @property + def module(self): + """The Python module referenced by this :class:`.Namespace`. + + If the namespace references a :class:`.Template`, then + this module is the equivalent of ``template.module``, + i.e. the generated module for the template. + + """ + return self.template.module + + @property + def filename(self): + """The path of the filesystem file used for this + :class:`.Namespace`'s module or template. + """ + return self.template.filename + + @property + def uri(self): + """The URI for this :class:`.Namespace`'s template. + + I.e. whatever was sent to :meth:`.TemplateLookup.get_template()`. + + This is the equivalent of :attr:`.Template.uri`. + + """ + return self.template.uri + + def _get_star(self): + if self.callables: + for key in self.callables: + yield (key, self.callables[key]) + + def get(key): + callable_ = self.template._get_def_callable(key) + return functools.partial(callable_, self.context) + + for k in self.template.module._exports: + yield (k, get(k)) + + def __getattr__(self, key): + if key in self.callables: + val = self.callables[key] + elif self.template.has_def(key): + callable_ = self.template._get_def_callable(key) + val = functools.partial(callable_, self.context) + elif self.inherits: + val = getattr(self.inherits, key) + + else: + raise AttributeError( + "Namespace '%s' has no member '%s'" % (self.name, key) + ) + setattr(self, key, val) + return val + + +class ModuleNamespace(Namespace): + + """A :class:`.Namespace` specific to a Python module instance.""" + + def __init__( + self, + name, + context, + module, + callables=None, + inherits=None, + populate_self=True, + calling_uri=None, + ): + self.name = name + self.context = context + self.inherits = inherits + if callables is not None: + self.callables = {c.__name__: c for c in callables} + + mod = __import__(module) + for token in module.split(".")[1:]: + mod = getattr(mod, token) + self.module = mod + + @property + def filename(self): + """The path of the filesystem file used for this + :class:`.Namespace`'s module or template. + """ + return self.module.__file__ + + def _get_star(self): + if self.callables: + for key in self.callables: + yield (key, self.callables[key]) + for key in dir(self.module): + if key[0] != "_": + callable_ = getattr(self.module, key) + if callable(callable_): + yield key, functools.partial(callable_, self.context) + + def __getattr__(self, key): + if key in self.callables: + val = self.callables[key] + elif hasattr(self.module, key): + callable_ = getattr(self.module, key) + val = functools.partial(callable_, self.context) + elif self.inherits: + val = getattr(self.inherits, key) + else: + raise AttributeError( + "Namespace '%s' has no member '%s'" % (self.name, key) + ) + setattr(self, key, val) + return val + + +def supports_caller(func): + """Apply a caller_stack compatibility decorator to a plain + Python function. + + See the example in :ref:`namespaces_python_modules`. + + """ + + def wrap_stackframe(context, *args, **kwargs): + context.caller_stack._push_frame() + try: + return func(context, *args, **kwargs) + finally: + context.caller_stack._pop_frame() + + return wrap_stackframe + + +def capture(context, callable_, *args, **kwargs): + """Execute the given template def, capturing the output into + a buffer. + + See the example in :ref:`namespaces_python_modules`. + + """ + + if not callable(callable_): + raise exceptions.RuntimeException( + "capture() function expects a callable as " + "its argument (i.e. capture(func, *args, **kwargs))" + ) + context._push_buffer() + try: + callable_(*args, **kwargs) + finally: + buf = context._pop_buffer() + return buf.getvalue() + + +def _decorate_toplevel(fn): + def decorate_render(render_fn): + def go(context, *args, **kw): + def y(*args, **kw): + return render_fn(context, *args, **kw) + + try: + y.__name__ = render_fn.__name__[7:] + except TypeError: + # < Python 2.4 + pass + return fn(y)(context, *args, **kw) + + return go + + return decorate_render + + +def _decorate_inline(context, fn): + def decorate_render(render_fn): + dec = fn(render_fn) + + def go(*args, **kw): + return dec(context, *args, **kw) + + return go + + return decorate_render + + +def _include_file(context, uri, calling_uri, **kwargs): + """locate the template from the given uri and include it in + the current output.""" + + template = _lookup_template(context, uri, calling_uri) + (callable_, ctx) = _populate_self_namespace( + context._clean_inheritance_tokens(), template + ) + kwargs = _kwargs_for_include(callable_, context._data, **kwargs) + if template.include_error_handler: + try: + callable_(ctx, **kwargs) + except Exception: + result = template.include_error_handler(ctx, compat.exception_as()) + if not result: + raise + else: + callable_(ctx, **kwargs) + + +def _inherit_from(context, uri, calling_uri): + """called by the _inherit method in template modules to set + up the inheritance chain at the start of a template's + execution.""" + + if uri is None: + return None + template = _lookup_template(context, uri, calling_uri) + self_ns = context["self"] + ih = self_ns + while ih.inherits is not None: + ih = ih.inherits + lclcontext = context._locals({"next": ih}) + ih.inherits = TemplateNamespace( + "self:%s" % template.uri, + lclcontext, + template=template, + populate_self=False, + ) + context._data["parent"] = lclcontext._data["local"] = ih.inherits + callable_ = getattr(template.module, "_mako_inherit", None) + if callable_ is not None: + ret = callable_(template, lclcontext) + if ret: + return ret + + gen_ns = getattr(template.module, "_mako_generate_namespaces", None) + if gen_ns is not None: + gen_ns(context) + return (template.callable_, lclcontext) + + +def _lookup_template(context, uri, relativeto): + lookup = context._with_template.lookup + if lookup is None: + raise exceptions.TemplateLookupException( + "Template '%s' has no TemplateLookup associated" + % context._with_template.uri + ) + uri = lookup.adjust_uri(uri, relativeto) + try: + return lookup.get_template(uri) + except exceptions.TopLevelLookupException as e: + raise exceptions.TemplateLookupException( + str(compat.exception_as()) + ) from e + + +def _populate_self_namespace(context, template, self_ns=None): + if self_ns is None: + self_ns = TemplateNamespace( + "self:%s" % template.uri, + context, + template=template, + populate_self=False, + ) + context._data["self"] = context._data["local"] = self_ns + if hasattr(template.module, "_mako_inherit"): + ret = template.module._mako_inherit(template, context) + if ret: + return ret + return (template.callable_, context) + + +def _render(template, callable_, args, data, as_unicode=False): + """create a Context and return the string + output of the given template and template callable.""" + + if as_unicode: + buf = util.FastEncodingBuffer() + else: + buf = util.FastEncodingBuffer( + encoding=template.output_encoding, errors=template.encoding_errors + ) + context = Context(buf, **data) + context._outputting_as_unicode = as_unicode + context._set_with_template(template) + + _render_context( + template, + callable_, + context, + *args, + **_kwargs_for_callable(callable_, data), + ) + return context._pop_buffer().getvalue() + + +def _kwargs_for_callable(callable_, data): + argspec = compat.inspect_getargspec(callable_) + # for normal pages, **pageargs is usually present + if argspec[2]: + return data + + # for rendering defs from the top level, figure out the args + namedargs = argspec[0] + [v for v in argspec[1:3] if v is not None] + kwargs = {} + for arg in namedargs: + if arg != "context" and arg in data and arg not in kwargs: + kwargs[arg] = data[arg] + return kwargs + + +def _kwargs_for_include(callable_, data, **kwargs): + argspec = compat.inspect_getargspec(callable_) + namedargs = argspec[0] + [v for v in argspec[1:3] if v is not None] + for arg in namedargs: + if arg != "context" and arg in data and arg not in kwargs: + kwargs[arg] = data[arg] + return kwargs + + +def _render_context(tmpl, callable_, context, *args, **kwargs): + import mako.template as template + + # create polymorphic 'self' namespace for this + # template with possibly updated context + if not isinstance(tmpl, template.DefTemplate): + # if main render method, call from the base of the inheritance stack + (inherit, lclcontext) = _populate_self_namespace(context, tmpl) + _exec_template(inherit, lclcontext, args=args, kwargs=kwargs) + else: + # otherwise, call the actual rendering method specified + (inherit, lclcontext) = _populate_self_namespace(context, tmpl.parent) + _exec_template(callable_, context, args=args, kwargs=kwargs) + + +def _exec_template(callable_, context, args=None, kwargs=None): + """execute a rendering callable given the callable, a + Context, and optional explicit arguments + + the contextual Template will be located if it exists, and + the error handling options specified on that Template will + be interpreted here. + """ + template = context._with_template + if template is not None and ( + template.format_exceptions or template.error_handler + ): + try: + callable_(context, *args, **kwargs) + except Exception: + _render_error(template, context, compat.exception_as()) + except: + e = sys.exc_info()[0] + _render_error(template, context, e) + else: + callable_(context, *args, **kwargs) + + +def _render_error(template, context, error): + if template.error_handler: + result = template.error_handler(context, error) + if not result: + tp, value, tb = sys.exc_info() + if value and tb: + raise value.with_traceback(tb) + else: + raise error + else: + error_template = exceptions.html_error_template() + if context._outputting_as_unicode: + context._buffer_stack[:] = [util.FastEncodingBuffer()] + else: + context._buffer_stack[:] = [ + util.FastEncodingBuffer( + error_template.output_encoding, + error_template.encoding_errors, + ) + ] + + context._set_with_template(error_template) + error_template.render_context(context, error=error) diff --git a/mako/template.py b/mako/template.py new file mode 100644 index 0000000..e72915b --- /dev/null +++ b/mako/template.py @@ -0,0 +1,715 @@ +# mako/template.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""Provides the Template class, a facade for parsing, generating and executing +template strings, as well as template runtime operations.""" + +import json +import os +import re +import shutil +import stat +import tempfile +import types +import weakref + +from mako import cache +from mako import codegen +from mako import compat +from mako import exceptions +from mako import runtime +from mako import util +from mako.lexer import Lexer + + +class Template: + r"""Represents a compiled template. + + :class:`.Template` includes a reference to the original + template source (via the :attr:`.source` attribute) + as well as the source code of the + generated Python module (i.e. the :attr:`.code` attribute), + as well as a reference to an actual Python module. + + :class:`.Template` is constructed using either a literal string + representing the template text, or a filename representing a filesystem + path to a source file. + + :param text: textual template source. This argument is mutually + exclusive versus the ``filename`` parameter. + + :param filename: filename of the source template. This argument is + mutually exclusive versus the ``text`` parameter. + + :param buffer_filters: string list of filters to be applied + to the output of ``%def``\ s which are buffered, cached, or otherwise + filtered, after all filters + defined with the ``%def`` itself have been applied. Allows the + creation of default expression filters that let the output + of return-valued ``%def``\ s "opt out" of that filtering via + passing special attributes or objects. + + :param cache_args: Dictionary of cache configuration arguments that + will be passed to the :class:`.CacheImpl`. See :ref:`caching_toplevel`. + + :param cache_dir: + + .. deprecated:: 0.6 + Use the ``'dir'`` argument in the ``cache_args`` dictionary. + See :ref:`caching_toplevel`. + + :param cache_enabled: Boolean flag which enables caching of this + template. See :ref:`caching_toplevel`. + + :param cache_impl: String name of a :class:`.CacheImpl` caching + implementation to use. Defaults to ``'beaker'``. + + :param cache_type: + + .. deprecated:: 0.6 + Use the ``'type'`` argument in the ``cache_args`` dictionary. + See :ref:`caching_toplevel`. + + :param cache_url: + + .. deprecated:: 0.6 + Use the ``'url'`` argument in the ``cache_args`` dictionary. + See :ref:`caching_toplevel`. + + :param default_filters: List of string filter names that will + be applied to all expressions. See :ref:`filtering_default_filters`. + + :param enable_loop: When ``True``, enable the ``loop`` context variable. + This can be set to ``False`` to support templates that may + be making usage of the name "``loop``". Individual templates can + re-enable the "loop" context by placing the directive + ``enable_loop="True"`` inside the ``<%page>`` tag -- see + :ref:`migrating_loop`. + + :param encoding_errors: Error parameter passed to ``encode()`` when + string encoding is performed. See :ref:`usage_unicode`. + + :param error_handler: Python callable which is called whenever + compile or runtime exceptions occur. The callable is passed + the current context as well as the exception. If the + callable returns ``True``, the exception is considered to + be handled, else it is re-raised after the function + completes. Is used to provide custom error-rendering + functions. + + .. seealso:: + + :paramref:`.Template.include_error_handler` - include-specific + error handler function + + :param format_exceptions: if ``True``, exceptions which occur during + the render phase of this template will be caught and + formatted into an HTML error page, which then becomes the + rendered result of the :meth:`.render` call. Otherwise, + runtime exceptions are propagated outwards. + + :param imports: String list of Python statements, typically individual + "import" lines, which will be placed into the module level + preamble of all generated Python modules. See the example + in :ref:`filtering_default_filters`. + + :param future_imports: String list of names to import from `__future__`. + These will be concatenated into a comma-separated string and inserted + into the beginning of the template, e.g. ``futures_imports=['FOO', + 'BAR']`` results in ``from __future__ import FOO, BAR``. If you're + interested in using features like the new division operator, you must + use future_imports to convey that to the renderer, as otherwise the + import will not appear as the first executed statement in the generated + code and will therefore not have the desired effect. + + :param include_error_handler: An error handler that runs when this template + is included within another one via the ``<%include>`` tag, and raises an + error. Compare to the :paramref:`.Template.error_handler` option. + + .. versionadded:: 1.0.6 + + .. seealso:: + + :paramref:`.Template.error_handler` - top-level error handler function + + :param input_encoding: Encoding of the template's source code. Can + be used in lieu of the coding comment. See + :ref:`usage_unicode` as well as :ref:`unicode_toplevel` for + details on source encoding. + + :param lookup: a :class:`.TemplateLookup` instance that will be used + for all file lookups via the ``<%namespace>``, + ``<%include>``, and ``<%inherit>`` tags. See + :ref:`usage_templatelookup`. + + :param module_directory: Filesystem location where generated + Python module files will be placed. + + :param module_filename: Overrides the filename of the generated + Python module file. For advanced usage only. + + :param module_writer: A callable which overrides how the Python + module is written entirely. The callable is passed the + encoded source content of the module and the destination + path to be written to. The default behavior of module writing + uses a tempfile in conjunction with a file move in order + to make the operation atomic. So a user-defined module + writing function that mimics the default behavior would be: + + .. sourcecode:: python + + import tempfile + import os + import shutil + + def module_writer(source, outputpath): + (dest, name) = \\ + tempfile.mkstemp( + dir=os.path.dirname(outputpath) + ) + + os.write(dest, source) + os.close(dest) + shutil.move(name, outputpath) + + from mako.template import Template + mytemplate = Template( + filename="index.html", + module_directory="/path/to/modules", + module_writer=module_writer + ) + + The function is provided for unusual configurations where + certain platform-specific permissions or other special + steps are needed. + + :param output_encoding: The encoding to use when :meth:`.render` + is called. + See :ref:`usage_unicode` as well as :ref:`unicode_toplevel`. + + :param preprocessor: Python callable which will be passed + the full template source before it is parsed. The return + result of the callable will be used as the template source + code. + + :param lexer_cls: A :class:`.Lexer` class used to parse + the template. The :class:`.Lexer` class is used by + default. + + .. versionadded:: 0.7.4 + + :param strict_undefined: Replaces the automatic usage of + ``UNDEFINED`` for any undeclared variables not located in + the :class:`.Context` with an immediate raise of + ``NameError``. The advantage is immediate reporting of + missing variables which include the name. + + .. versionadded:: 0.3.6 + + :param uri: string URI or other identifier for this template. + If not provided, the ``uri`` is generated from the filesystem + path, or from the in-memory identity of a non-file-based + template. The primary usage of the ``uri`` is to provide a key + within :class:`.TemplateLookup`, as well as to generate the + file path of the generated Python module file, if + ``module_directory`` is specified. + + """ + + lexer_cls = Lexer + + def __init__( + self, + text=None, + filename=None, + uri=None, + format_exceptions=False, + error_handler=None, + lookup=None, + output_encoding=None, + encoding_errors="strict", + module_directory=None, + cache_args=None, + cache_impl="beaker", + cache_enabled=True, + cache_type=None, + cache_dir=None, + cache_url=None, + module_filename=None, + input_encoding=None, + module_writer=None, + default_filters=None, + buffer_filters=(), + strict_undefined=False, + imports=None, + future_imports=None, + enable_loop=True, + preprocessor=None, + lexer_cls=None, + include_error_handler=None, + ): + if uri: + self.module_id = re.sub(r"\W", "_", uri) + self.uri = uri + elif filename: + self.module_id = re.sub(r"\W", "_", filename) + drive, path = os.path.splitdrive(filename) + path = os.path.normpath(path).replace(os.path.sep, "/") + self.uri = path + else: + self.module_id = "memory:" + hex(id(self)) + self.uri = self.module_id + + u_norm = self.uri + if u_norm.startswith("/"): + u_norm = u_norm[1:] + u_norm = os.path.normpath(u_norm) + if u_norm.startswith(".."): + raise exceptions.TemplateLookupException( + 'Template uri "%s" is invalid - ' + "it cannot be relative outside " + "of the root path." % self.uri + ) + + self.input_encoding = input_encoding + self.output_encoding = output_encoding + self.encoding_errors = encoding_errors + self.enable_loop = enable_loop + self.strict_undefined = strict_undefined + self.module_writer = module_writer + + if default_filters is None: + self.default_filters = ["str"] + else: + self.default_filters = default_filters + self.buffer_filters = buffer_filters + + self.imports = imports + self.future_imports = future_imports + self.preprocessor = preprocessor + + if lexer_cls is not None: + self.lexer_cls = lexer_cls + + # if plain text, compile code in memory only + if text is not None: + (code, module) = _compile_text(self, text, filename) + self._code = code + self._source = text + ModuleInfo(module, None, self, filename, code, text, uri) + elif filename is not None: + # if template filename and a module directory, load + # a filesystem-based module file, generating if needed + if module_filename is not None: + path = module_filename + elif module_directory is not None: + path = os.path.abspath( + os.path.join( + os.path.normpath(module_directory), u_norm + ".py" + ) + ) + else: + path = None + module = self._compile_from_file(path, filename) + else: + raise exceptions.RuntimeException( + "Template requires text or filename" + ) + + self.module = module + self.filename = filename + self.callable_ = self.module.render_body + self.format_exceptions = format_exceptions + self.error_handler = error_handler + self.include_error_handler = include_error_handler + self.lookup = lookup + + self.module_directory = module_directory + + self._setup_cache_args( + cache_impl, + cache_enabled, + cache_args, + cache_type, + cache_dir, + cache_url, + ) + + @util.memoized_property + def reserved_names(self): + if self.enable_loop: + return codegen.RESERVED_NAMES + else: + return codegen.RESERVED_NAMES.difference(["loop"]) + + def _setup_cache_args( + self, + cache_impl, + cache_enabled, + cache_args, + cache_type, + cache_dir, + cache_url, + ): + self.cache_impl = cache_impl + self.cache_enabled = cache_enabled + self.cache_args = cache_args or {} + # transfer deprecated cache_* args + if cache_type: + self.cache_args["type"] = cache_type + if cache_dir: + self.cache_args["dir"] = cache_dir + if cache_url: + self.cache_args["url"] = cache_url + + def _compile_from_file(self, path, filename): + if path is not None: + util.verify_directory(os.path.dirname(path)) + filemtime = os.stat(filename)[stat.ST_MTIME] + if ( + not os.path.exists(path) + or os.stat(path)[stat.ST_MTIME] < filemtime + ): + data = util.read_file(filename) + _compile_module_file( + self, data, filename, path, self.module_writer + ) + module = compat.load_module(self.module_id, path) + if module._magic_number != codegen.MAGIC_NUMBER: + data = util.read_file(filename) + _compile_module_file( + self, data, filename, path, self.module_writer + ) + module = compat.load_module(self.module_id, path) + ModuleInfo(module, path, self, filename, None, None, None) + else: + # template filename and no module directory, compile code + # in memory + data = util.read_file(filename) + code, module = _compile_text(self, data, filename) + self._source = None + self._code = code + ModuleInfo(module, None, self, filename, code, None, None) + return module + + @property + def source(self): + """Return the template source code for this :class:`.Template`.""" + + return _get_module_info_from_callable(self.callable_).source + + @property + def code(self): + """Return the module source code for this :class:`.Template`.""" + + return _get_module_info_from_callable(self.callable_).code + + @util.memoized_property + def cache(self): + return cache.Cache(self) + + @property + def cache_dir(self): + return self.cache_args["dir"] + + @property + def cache_url(self): + return self.cache_args["url"] + + @property + def cache_type(self): + return self.cache_args["type"] + + def render(self, *args, **data): + """Render the output of this template as a string. + + If the template specifies an output encoding, the string + will be encoded accordingly, else the output is raw (raw + output uses `StringIO` and can't handle multibyte + characters). A :class:`.Context` object is created corresponding + to the given data. Arguments that are explicitly declared + by this template's internal rendering method are also + pulled from the given ``*args``, ``**data`` members. + + """ + return runtime._render(self, self.callable_, args, data) + + def render_unicode(self, *args, **data): + """Render the output of this template as a unicode object.""" + + return runtime._render( + self, self.callable_, args, data, as_unicode=True + ) + + def render_context(self, context, *args, **kwargs): + """Render this :class:`.Template` with the given context. + + The data is written to the context's buffer. + + """ + if getattr(context, "_with_template", None) is None: + context._set_with_template(self) + runtime._render_context(self, self.callable_, context, *args, **kwargs) + + def has_def(self, name): + return hasattr(self.module, "render_%s" % name) + + def get_def(self, name): + """Return a def of this template as a :class:`.DefTemplate`.""" + + return DefTemplate(self, getattr(self.module, "render_%s" % name)) + + def list_defs(self): + """return a list of defs in the template. + + .. versionadded:: 1.0.4 + + """ + return [i[7:] for i in dir(self.module) if i[:7] == "render_"] + + def _get_def_callable(self, name): + return getattr(self.module, "render_%s" % name) + + @property + def last_modified(self): + return self.module._modified_time + + +class ModuleTemplate(Template): + + """A Template which is constructed given an existing Python module. + + e.g.:: + + t = Template("this is a template") + f = file("mymodule.py", "w") + f.write(t.code) + f.close() + + import mymodule + + t = ModuleTemplate(mymodule) + print(t.render()) + + """ + + def __init__( + self, + module, + module_filename=None, + template=None, + template_filename=None, + module_source=None, + template_source=None, + output_encoding=None, + encoding_errors="strict", + format_exceptions=False, + error_handler=None, + lookup=None, + cache_args=None, + cache_impl="beaker", + cache_enabled=True, + cache_type=None, + cache_dir=None, + cache_url=None, + include_error_handler=None, + ): + self.module_id = re.sub(r"\W", "_", module._template_uri) + self.uri = module._template_uri + self.input_encoding = module._source_encoding + self.output_encoding = output_encoding + self.encoding_errors = encoding_errors + self.enable_loop = module._enable_loop + + self.module = module + self.filename = template_filename + ModuleInfo( + module, + module_filename, + self, + template_filename, + module_source, + template_source, + module._template_uri, + ) + + self.callable_ = self.module.render_body + self.format_exceptions = format_exceptions + self.error_handler = error_handler + self.include_error_handler = include_error_handler + self.lookup = lookup + self._setup_cache_args( + cache_impl, + cache_enabled, + cache_args, + cache_type, + cache_dir, + cache_url, + ) + + +class DefTemplate(Template): + + """A :class:`.Template` which represents a callable def in a parent + template.""" + + def __init__(self, parent, callable_): + self.parent = parent + self.callable_ = callable_ + self.output_encoding = parent.output_encoding + self.module = parent.module + self.encoding_errors = parent.encoding_errors + self.format_exceptions = parent.format_exceptions + self.error_handler = parent.error_handler + self.include_error_handler = parent.include_error_handler + self.enable_loop = parent.enable_loop + self.lookup = parent.lookup + + def get_def(self, name): + return self.parent.get_def(name) + + +class ModuleInfo: + + """Stores information about a module currently loaded into + memory, provides reverse lookups of template source, module + source code based on a module's identifier. + + """ + + _modules = weakref.WeakValueDictionary() + + def __init__( + self, + module, + module_filename, + template, + template_filename, + module_source, + template_source, + template_uri, + ): + self.module = module + self.module_filename = module_filename + self.template_filename = template_filename + self.module_source = module_source + self.template_source = template_source + self.template_uri = template_uri + self._modules[module.__name__] = template._mmarker = self + if module_filename: + self._modules[module_filename] = self + + @classmethod + def get_module_source_metadata(cls, module_source, full_line_map=False): + source_map = re.search( + r"__M_BEGIN_METADATA(.+?)__M_END_METADATA", module_source, re.S + ).group(1) + source_map = json.loads(source_map) + source_map["line_map"] = { + int(k): int(v) for k, v in source_map["line_map"].items() + } + if full_line_map: + f_line_map = source_map["full_line_map"] = [] + line_map = source_map["line_map"] + + curr_templ_line = 1 + for mod_line in range(1, max(line_map)): + if mod_line in line_map: + curr_templ_line = line_map[mod_line] + f_line_map.append(curr_templ_line) + return source_map + + @property + def code(self): + if self.module_source is not None: + return self.module_source + else: + return util.read_python_file(self.module_filename) + + @property + def source(self): + if self.template_source is None: + data = util.read_file(self.template_filename) + if self.module._source_encoding: + return data.decode(self.module._source_encoding) + else: + return data + + elif self.module._source_encoding and not isinstance( + self.template_source, str + ): + return self.template_source.decode(self.module._source_encoding) + else: + return self.template_source + + +def _compile(template, text, filename, generate_magic_comment): + lexer = template.lexer_cls( + text, + filename, + input_encoding=template.input_encoding, + preprocessor=template.preprocessor, + ) + node = lexer.parse() + source = codegen.compile( + node, + template.uri, + filename, + default_filters=template.default_filters, + buffer_filters=template.buffer_filters, + imports=template.imports, + future_imports=template.future_imports, + source_encoding=lexer.encoding, + generate_magic_comment=generate_magic_comment, + strict_undefined=template.strict_undefined, + enable_loop=template.enable_loop, + reserved_names=template.reserved_names, + ) + return source, lexer + + +def _compile_text(template, text, filename): + identifier = template.module_id + source, lexer = _compile( + template, text, filename, generate_magic_comment=False + ) + + cid = identifier + module = types.ModuleType(cid) + code = compile(source, cid, "exec") + + # this exec() works for 2.4->3.3. + exec(code, module.__dict__, module.__dict__) + return (source, module) + + +def _compile_module_file(template, text, filename, outputpath, module_writer): + source, lexer = _compile( + template, text, filename, generate_magic_comment=True + ) + + if isinstance(source, str): + source = source.encode(lexer.encoding or "ascii") + + if module_writer: + module_writer(source, outputpath) + else: + # make tempfiles in the same location as the ultimate + # location. this ensures they're on the same filesystem, + # avoiding synchronization issues. + (dest, name) = tempfile.mkstemp(dir=os.path.dirname(outputpath)) + + os.write(dest, source) + os.close(dest) + shutil.move(name, outputpath) + + +def _get_module_info_from_callable(callable_): + return _get_module_info(callable_.__globals__["__name__"]) + + +def _get_module_info(filename): + return ModuleInfo._modules[filename] diff --git a/mako/testing/__init__.py b/mako/testing/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/mako/testing/__init__.py diff --git a/mako/testing/_config.py b/mako/testing/_config.py new file mode 100644 index 0000000..4ee3d0a --- /dev/null +++ b/mako/testing/_config.py @@ -0,0 +1,128 @@ +import configparser +import dataclasses +from dataclasses import dataclass +from pathlib import Path +from typing import Callable +from typing import ClassVar +from typing import Optional +from typing import Union + +from .helpers import make_path + + +class ConfigError(BaseException): + pass + + +class MissingConfig(ConfigError): + pass + + +class MissingConfigSection(ConfigError): + pass + + +class MissingConfigItem(ConfigError): + pass + + +class ConfigValueTypeError(ConfigError): + pass + + +class _GetterDispatch: + def __init__(self, initialdata, default_getter: Callable): + self.default_getter = default_getter + self.data = initialdata + + def get_fn_for_type(self, type_): + return self.data.get(type_, self.default_getter) + + def get_typed_value(self, type_, name): + get_fn = self.get_fn_for_type(type_) + return get_fn(name) + + +def _parse_cfg_file(filespec: Union[Path, str]): + cfg = configparser.ConfigParser() + try: + filepath = make_path(filespec, check_exists=True) + except FileNotFoundError as e: + raise MissingConfig(f"No config file found at {filespec}") from e + else: + with open(filepath, encoding="utf-8") as f: + cfg.read_file(f) + return cfg + + +def _build_getter(cfg_obj, cfg_section, method, converter=None): + def caller(option, **kwargs): + try: + rv = getattr(cfg_obj, method)(cfg_section, option, **kwargs) + except configparser.NoSectionError as nse: + raise MissingConfigSection( + f"No config section named {cfg_section}" + ) from nse + except configparser.NoOptionError as noe: + raise MissingConfigItem(f"No config item for {option}") from noe + except ValueError as ve: + # ConfigParser.getboolean, .getint, .getfloat raise ValueError + # on bad types + raise ConfigValueTypeError( + f"Wrong value type for {option}" + ) from ve + else: + if converter: + try: + rv = converter(rv) + except Exception as e: + raise ConfigValueTypeError( + f"Wrong value type for {option}" + ) from e + return rv + + return caller + + +def _build_getter_dispatch(cfg_obj, cfg_section, converters=None): + converters = converters or {} + + default_getter = _build_getter(cfg_obj, cfg_section, "get") + + # support ConfigParser builtins + getters = { + int: _build_getter(cfg_obj, cfg_section, "getint"), + bool: _build_getter(cfg_obj, cfg_section, "getboolean"), + float: _build_getter(cfg_obj, cfg_section, "getfloat"), + str: default_getter, + } + + # use ConfigParser.get and convert value + getters.update( + { + type_: _build_getter( + cfg_obj, cfg_section, "get", converter=converter_fn + ) + for type_, converter_fn in converters.items() + } + ) + + return _GetterDispatch(getters, default_getter) + + +@dataclass +class ReadsCfg: + section_header: ClassVar[str] + converters: ClassVar[Optional[dict]] = None + + @classmethod + def from_cfg_file(cls, filespec: Union[Path, str]): + cfg = _parse_cfg_file(filespec) + dispatch = _build_getter_dispatch( + cfg, cls.section_header, converters=cls.converters + ) + kwargs = { + field.name: dispatch.get_typed_value(field.type, field.name) + for field in dataclasses.fields(cls) + } + return cls(**kwargs) diff --git a/mako/testing/assertions.py b/mako/testing/assertions.py new file mode 100644 index 0000000..22221cd --- /dev/null +++ b/mako/testing/assertions.py @@ -0,0 +1,166 @@ +import contextlib +import re +import sys + + +def eq_(a, b, msg=None): + """Assert a == b, with repr messaging on failure.""" + assert a == b, msg or "%r != %r" % (a, b) + + +def ne_(a, b, msg=None): + """Assert a != b, with repr messaging on failure.""" + assert a != b, msg or "%r == %r" % (a, b) + + +def in_(a, b, msg=None): + """Assert a in b, with repr messaging on failure.""" + assert a in b, msg or "%r not in %r" % (a, b) + + +def not_in(a, b, msg=None): + """Assert a in not b, with repr messaging on failure.""" + assert a not in b, msg or "%r is in %r" % (a, b) + + +def _assert_proper_exception_context(exception): + """assert that any exception we're catching does not have a __context__ + without a __cause__, and that __suppress_context__ is never set. + + Python 3 will report nested as exceptions as "during the handling of + error X, error Y occurred". That's not what we want to do. We want + these exceptions in a cause chain. + + """ + + if ( + exception.__context__ is not exception.__cause__ + and not exception.__suppress_context__ + ): + assert False, ( + "Exception %r was correctly raised but did not set a cause, " + "within context %r as its cause." + % (exception, exception.__context__) + ) + + +def _assert_proper_cause_cls(exception, cause_cls): + """assert that any exception we're catching does not have a __context__ + without a __cause__, and that __suppress_context__ is never set. + + Python 3 will report nested as exceptions as "during the handling of + error X, error Y occurred". That's not what we want to do. We want + these exceptions in a cause chain. + + """ + assert isinstance(exception.__cause__, cause_cls), ( + "Exception %r was correctly raised but has cause %r, which does not " + "have the expected cause type %r." + % (exception, exception.__cause__, cause_cls) + ) + + +def assert_raises(except_cls, callable_, *args, **kw): + return _assert_raises(except_cls, callable_, args, kw) + + +def assert_raises_with_proper_context(except_cls, callable_, *args, **kw): + return _assert_raises(except_cls, callable_, args, kw, check_context=True) + + +def assert_raises_with_given_cause( + except_cls, cause_cls, callable_, *args, **kw +): + return _assert_raises(except_cls, callable_, args, kw, cause_cls=cause_cls) + + +def assert_raises_message(except_cls, msg, callable_, *args, **kwargs): + return _assert_raises(except_cls, callable_, args, kwargs, msg=msg) + + +def assert_raises_message_with_proper_context( + except_cls, msg, callable_, *args, **kwargs +): + return _assert_raises( + except_cls, callable_, args, kwargs, msg=msg, check_context=True + ) + + +def assert_raises_message_with_given_cause( + except_cls, msg, cause_cls, callable_, *args, **kwargs +): + return _assert_raises( + except_cls, callable_, args, kwargs, msg=msg, cause_cls=cause_cls + ) + + +def _assert_raises( + except_cls, + callable_, + args, + kwargs, + msg=None, + check_context=False, + cause_cls=None, +): + with _expect_raises(except_cls, msg, check_context, cause_cls) as ec: + callable_(*args, **kwargs) + return ec.error + + +class _ErrorContainer: + error = None + + +@contextlib.contextmanager +def _expect_raises(except_cls, msg=None, check_context=False, cause_cls=None): + ec = _ErrorContainer() + if check_context: + are_we_already_in_a_traceback = sys.exc_info()[0] + try: + yield ec + success = False + except except_cls as err: + ec.error = err + success = True + if msg is not None: + # I'm often pdbing here, and "err" above isn't + # in scope, so assign the string explicitly + error_as_string = str(err) + assert re.search(msg, error_as_string, re.UNICODE), "%r !~ %s" % ( + msg, + error_as_string, + ) + if cause_cls is not None: + _assert_proper_cause_cls(err, cause_cls) + if check_context and not are_we_already_in_a_traceback: + _assert_proper_exception_context(err) + print(str(err).encode("utf-8")) + + # it's generally a good idea to not carry traceback objects outside + # of the except: block, but in this case especially we seem to have + # hit some bug in either python 3.10.0b2 or greenlet or both which + # this seems to fix: + # https://github.com/python-greenlet/greenlet/issues/242 + del ec + + # assert outside the block so it works for AssertionError too ! + assert success, "Callable did not raise an exception" + + +def expect_raises(except_cls, check_context=False): + return _expect_raises(except_cls, check_context=check_context) + + +def expect_raises_message(except_cls, msg, check_context=False): + return _expect_raises(except_cls, msg=msg, check_context=check_context) + + +def expect_raises_with_proper_context(except_cls, check_context=True): + return _expect_raises(except_cls, check_context=check_context) + + +def expect_raises_message_with_proper_context( + except_cls, msg, check_context=True +): + return _expect_raises(except_cls, msg=msg, check_context=check_context) diff --git a/mako/testing/config.py b/mako/testing/config.py new file mode 100644 index 0000000..b77d0c0 --- /dev/null +++ b/mako/testing/config.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass +from pathlib import Path + +from ._config import ReadsCfg +from .helpers import make_path + + +@dataclass +class Config(ReadsCfg): + module_base: Path + template_base: Path + + section_header = "mako_testing" + converters = {Path: make_path} + + +config = Config.from_cfg_file("./setup.cfg") diff --git a/mako/testing/exclusions.py b/mako/testing/exclusions.py new file mode 100644 index 0000000..37b2d14 --- /dev/null +++ b/mako/testing/exclusions.py @@ -0,0 +1,80 @@ +import pytest + +from mako.ext.beaker_cache import has_beaker +from mako.util import update_wrapper + + +try: + import babel.messages.extract as babel +except ImportError: + babel = None + + +try: + import lingua +except ImportError: + lingua = None + + +try: + import dogpile.cache # noqa +except ImportError: + has_dogpile_cache = False +else: + has_dogpile_cache = True + + +requires_beaker = pytest.mark.skipif( + not has_beaker, reason="Beaker is required for these tests." +) + + +requires_babel = pytest.mark.skipif( + babel is None, reason="babel not installed: skipping babelplugin test" +) + + +requires_lingua = pytest.mark.skipif( + lingua is None, reason="lingua not installed: skipping linguaplugin test" +) + + +requires_dogpile_cache = pytest.mark.skipif( + not has_dogpile_cache, + reason="dogpile.cache is required to run these tests", +) + + +def _pygments_version(): + try: + import pygments + + version = pygments.__version__ + except: + version = "0" + return version + + +requires_pygments_14 = pytest.mark.skipif( + _pygments_version() < "1.4", reason="Requires pygments 1.4 or greater" +) + + +# def requires_pygments_14(fn): + +# return skip_if( +# lambda: version < "1.4", "Requires pygments 1.4 or greater" +# )(fn) + + +def requires_no_pygments_exceptions(fn): + def go(*arg, **kw): + from mako import exceptions + + exceptions._install_fallback() + try: + return fn(*arg, **kw) + finally: + exceptions._install_highlighting() + + return update_wrapper(go, fn) diff --git a/mako/testing/fixtures.py b/mako/testing/fixtures.py new file mode 100644 index 0000000..01e9961 --- /dev/null +++ b/mako/testing/fixtures.py @@ -0,0 +1,119 @@ +import os + +from mako.cache import CacheImpl +from mako.cache import register_plugin +from mako.template import Template +from .assertions import eq_ +from .config import config + + +class TemplateTest: + def _file_template(self, filename, **kw): + filepath = self._file_path(filename) + return Template( + uri=filename, + filename=filepath, + module_directory=config.module_base, + **kw, + ) + + def _file_path(self, filename): + name, ext = os.path.splitext(filename) + py3k_path = os.path.join(config.template_base, name + "_py3k" + ext) + if os.path.exists(py3k_path): + return py3k_path + + return os.path.join(config.template_base, filename) + + def _do_file_test( + self, + filename, + expected, + filters=None, + unicode_=True, + template_args=None, + **kw, + ): + t1 = self._file_template(filename, **kw) + self._do_test( + t1, + expected, + filters=filters, + unicode_=unicode_, + template_args=template_args, + ) + + def _do_memory_test( + self, + source, + expected, + filters=None, + unicode_=True, + template_args=None, + **kw, + ): + t1 = Template(text=source, **kw) + self._do_test( + t1, + expected, + filters=filters, + unicode_=unicode_, + template_args=template_args, + ) + + def _do_test( + self, + template, + expected, + filters=None, + template_args=None, + unicode_=True, + ): + if template_args is None: + template_args = {} + if unicode_: + output = template.render_unicode(**template_args) + else: + output = template.render(**template_args) + + if filters: + output = filters(output) + eq_(output, expected) + + def indicates_unbound_local_error(self, rendered_output, unbound_var): + var = f"'{unbound_var}'" + error_msgs = ( + # < 3.11 + f"local variable {var} referenced before assignment", + # >= 3.11 + f"cannot access local variable {var} where it is not associated", + ) + return any((msg in rendered_output) for msg in error_msgs) + + +class PlainCacheImpl(CacheImpl): + """Simple memory cache impl so that tests which + use caching can run without beaker.""" + + def __init__(self, cache): + self.cache = cache + self.data = {} + + def get_or_create(self, key, creation_function, **kw): + if key in self.data: + return self.data[key] + else: + self.data[key] = data = creation_function(**kw) + return data + + def put(self, key, value, **kw): + self.data[key] = value + + def get(self, key, **kw): + return self.data[key] + + def invalidate(self, key, **kw): + del self.data[key] + + +register_plugin("plain", __name__, "PlainCacheImpl") diff --git a/mako/testing/helpers.py b/mako/testing/helpers.py new file mode 100644 index 0000000..77cca36 --- /dev/null +++ b/mako/testing/helpers.py @@ -0,0 +1,67 @@ +import contextlib +import pathlib +from pathlib import Path +import re +import time +from typing import Union +from unittest import mock + + +def flatten_result(result): + return re.sub(r"[\s\r\n]+", " ", result).strip() + + +def result_lines(result): + return [ + x.strip() + for x in re.split(r"\r?\n", re.sub(r" +", " ", result)) + if x.strip() != "" + ] + + +def make_path( + filespec: Union[Path, str], + make_absolute: bool = True, + check_exists: bool = False, +) -> Path: + path = Path(filespec) + if make_absolute: + path = path.resolve(strict=check_exists) + if check_exists and (not path.exists()): + raise FileNotFoundError(f"No file or directory at {filespec}") + return path + + +def _unlink_path(path, missing_ok=False): + # Replicate 3.8+ functionality in 3.7 + cm = contextlib.nullcontext() + if missing_ok: + cm = contextlib.suppress(FileNotFoundError) + + with cm: + path.unlink() + + +def replace_file_with_dir(pathspec): + path = pathlib.Path(pathspec) + _unlink_path(path, missing_ok=True) + path.mkdir(exist_ok=True) + return path + + +def file_with_template_code(filespec): + with open(filespec, "w") as f: + f.write( + """ +i am an artificial template just for you +""" + ) + return filespec + + +@contextlib.contextmanager +def rewind_compile_time(hours=1): + rewound = time.time() - (hours * 3_600) + with mock.patch("mako.codegen.time") as codegen_time: + codegen_time.time.return_value = rewound + yield diff --git a/mako/util.py b/mako/util.py new file mode 100644 index 0000000..991235b --- /dev/null +++ b/mako/util.py @@ -0,0 +1,388 @@ +# mako/util.py +# Copyright 2006-2023 the Mako authors and contributors <see AUTHORS file> +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php +from ast import parse +import codecs +import collections +import operator +import os +import re +import timeit + +from .compat import importlib_metadata_get + + +def update_wrapper(decorated, fn): + decorated.__wrapped__ = fn + decorated.__name__ = fn.__name__ + return decorated + + +class PluginLoader: + def __init__(self, group): + self.group = group + self.impls = {} + + def load(self, name): + if name in self.impls: + return self.impls[name]() + + for impl in importlib_metadata_get(self.group): + if impl.name == name: + self.impls[name] = impl.load + return impl.load() + + from mako import exceptions + + raise exceptions.RuntimeException( + "Can't load plugin %s %s" % (self.group, name) + ) + + def register(self, name, modulepath, objname): + def load(): + mod = __import__(modulepath) + for token in modulepath.split(".")[1:]: + mod = getattr(mod, token) + return getattr(mod, objname) + + self.impls[name] = load + + +def verify_directory(dir_): + """create and/or verify a filesystem directory.""" + + tries = 0 + + while not os.path.exists(dir_): + try: + tries += 1 + os.makedirs(dir_, 0o755) + except: + if tries > 5: + raise + + +def to_list(x, default=None): + if x is None: + return default + if not isinstance(x, (list, tuple)): + return [x] + else: + return x + + +class memoized_property: + + """A read-only @property that is only evaluated once.""" + + def __init__(self, fget, doc=None): + self.fget = fget + self.__doc__ = doc or fget.__doc__ + self.__name__ = fget.__name__ + + def __get__(self, obj, cls): + if obj is None: + return self + obj.__dict__[self.__name__] = result = self.fget(obj) + return result + + +class memoized_instancemethod: + + """Decorate a method memoize its return value. + + Best applied to no-arg methods: memoization is not sensitive to + argument values, and will always return the same value even when + called with different arguments. + + """ + + def __init__(self, fget, doc=None): + self.fget = fget + self.__doc__ = doc or fget.__doc__ + self.__name__ = fget.__name__ + + def __get__(self, obj, cls): + if obj is None: + return self + + def oneshot(*args, **kw): + result = self.fget(obj, *args, **kw) + + def memo(*a, **kw): + return result + + memo.__name__ = self.__name__ + memo.__doc__ = self.__doc__ + obj.__dict__[self.__name__] = memo + return result + + oneshot.__name__ = self.__name__ + oneshot.__doc__ = self.__doc__ + return oneshot + + +class SetLikeDict(dict): + + """a dictionary that has some setlike methods on it""" + + def union(self, other): + """produce a 'union' of this dict and another (at the key level). + + values in the second dict take precedence over that of the first""" + x = SetLikeDict(**self) + x.update(other) + return x + + +class FastEncodingBuffer: + + """a very rudimentary buffer that is faster than StringIO, + and supports unicode data.""" + + def __init__(self, encoding=None, errors="strict"): + self.data = collections.deque() + self.encoding = encoding + self.delim = "" + self.errors = errors + self.write = self.data.append + + def truncate(self): + self.data = collections.deque() + self.write = self.data.append + + def getvalue(self): + if self.encoding: + return self.delim.join(self.data).encode( + self.encoding, self.errors + ) + else: + return self.delim.join(self.data) + + +class LRUCache(dict): + + """A dictionary-like object that stores a limited number of items, + discarding lesser used items periodically. + + this is a rewrite of LRUCache from Myghty to use a periodic timestamp-based + paradigm so that synchronization is not really needed. the size management + is inexact. + """ + + class _Item: + def __init__(self, key, value): + self.key = key + self.value = value + self.timestamp = timeit.default_timer() + + def __repr__(self): + return repr(self.value) + + def __init__(self, capacity, threshold=0.5): + self.capacity = capacity + self.threshold = threshold + + def __getitem__(self, key): + item = dict.__getitem__(self, key) + item.timestamp = timeit.default_timer() + return item.value + + def values(self): + return [i.value for i in dict.values(self)] + + def setdefault(self, key, value): + if key in self: + return self[key] + self[key] = value + return value + + def __setitem__(self, key, value): + item = dict.get(self, key) + if item is None: + item = self._Item(key, value) + dict.__setitem__(self, key, item) + else: + item.value = value + self._manage_size() + + def _manage_size(self): + while len(self) > self.capacity + self.capacity * self.threshold: + bytime = sorted( + dict.values(self), + key=operator.attrgetter("timestamp"), + reverse=True, + ) + for item in bytime[self.capacity :]: + try: + del self[item.key] + except KeyError: + # if we couldn't find a key, most likely some other thread + # broke in on us. loop around and try again + break + + +# Regexp to match python magic encoding line +_PYTHON_MAGIC_COMMENT_re = re.compile( + r"[ \t\f]* \# .* coding[=:][ \t]*([-\w.]+)", re.VERBOSE +) + + +def parse_encoding(fp): + """Deduce the encoding of a Python source file (binary mode) from magic + comment. + + It does this in the same way as the `Python interpreter`__ + + .. __: http://docs.python.org/ref/encodings.html + + The ``fp`` argument should be a seekable file object in binary mode. + """ + pos = fp.tell() + fp.seek(0) + try: + line1 = fp.readline() + has_bom = line1.startswith(codecs.BOM_UTF8) + if has_bom: + line1 = line1[len(codecs.BOM_UTF8) :] + + m = _PYTHON_MAGIC_COMMENT_re.match(line1.decode("ascii", "ignore")) + if not m: + try: + parse(line1.decode("ascii", "ignore")) + except (ImportError, SyntaxError): + # Either it's a real syntax error, in which case the source + # is not valid python source, or line2 is a continuation of + # line1, in which case we don't want to scan line2 for a magic + # comment. + pass + else: + line2 = fp.readline() + m = _PYTHON_MAGIC_COMMENT_re.match( + line2.decode("ascii", "ignore") + ) + + if has_bom: + if m: + raise SyntaxError( + "python refuses to compile code with both a UTF8" + " byte-order-mark and a magic encoding comment" + ) + return "utf_8" + elif m: + return m.group(1) + else: + return None + finally: + fp.seek(pos) + + +def sorted_dict_repr(d): + """repr() a dictionary with the keys in order. + + Used by the lexer unit test to compare parse trees based on strings. + + """ + keys = list(d.keys()) + keys.sort() + return "{" + ", ".join("%r: %r" % (k, d[k]) for k in keys) + "}" + + +def restore__ast(_ast): + """Attempt to restore the required classes to the _ast module if it + appears to be missing them + """ + if hasattr(_ast, "AST"): + return + _ast.PyCF_ONLY_AST = 2 << 9 + m = compile( + """\ +def foo(): pass +class Bar: pass +if False: pass +baz = 'mako' +1 + 2 - 3 * 4 / 5 +6 // 7 % 8 << 9 >> 10 +11 & 12 ^ 13 | 14 +15 and 16 or 17 +-baz + (not +18) - ~17 +baz and 'foo' or 'bar' +(mako is baz == baz) is not baz != mako +mako > baz < mako >= baz <= mako +mako in baz not in mako""", + "<unknown>", + "exec", + _ast.PyCF_ONLY_AST, + ) + _ast.Module = type(m) + + for cls in _ast.Module.__mro__: + if cls.__name__ == "mod": + _ast.mod = cls + elif cls.__name__ == "AST": + _ast.AST = cls + + _ast.FunctionDef = type(m.body[0]) + _ast.ClassDef = type(m.body[1]) + _ast.If = type(m.body[2]) + + _ast.Name = type(m.body[3].targets[0]) + _ast.Store = type(m.body[3].targets[0].ctx) + _ast.Str = type(m.body[3].value) + + _ast.Sub = type(m.body[4].value.op) + _ast.Add = type(m.body[4].value.left.op) + _ast.Div = type(m.body[4].value.right.op) + _ast.Mult = type(m.body[4].value.right.left.op) + + _ast.RShift = type(m.body[5].value.op) + _ast.LShift = type(m.body[5].value.left.op) + _ast.Mod = type(m.body[5].value.left.left.op) + _ast.FloorDiv = type(m.body[5].value.left.left.left.op) + + _ast.BitOr = type(m.body[6].value.op) + _ast.BitXor = type(m.body[6].value.left.op) + _ast.BitAnd = type(m.body[6].value.left.left.op) + + _ast.Or = type(m.body[7].value.op) + _ast.And = type(m.body[7].value.values[0].op) + + _ast.Invert = type(m.body[8].value.right.op) + _ast.Not = type(m.body[8].value.left.right.op) + _ast.UAdd = type(m.body[8].value.left.right.operand.op) + _ast.USub = type(m.body[8].value.left.left.op) + + _ast.Or = type(m.body[9].value.op) + _ast.And = type(m.body[9].value.values[0].op) + + _ast.IsNot = type(m.body[10].value.ops[0]) + _ast.NotEq = type(m.body[10].value.ops[1]) + _ast.Is = type(m.body[10].value.left.ops[0]) + _ast.Eq = type(m.body[10].value.left.ops[1]) + + _ast.Gt = type(m.body[11].value.ops[0]) + _ast.Lt = type(m.body[11].value.ops[1]) + _ast.GtE = type(m.body[11].value.ops[2]) + _ast.LtE = type(m.body[11].value.ops[3]) + + _ast.In = type(m.body[12].value.ops[0]) + _ast.NotIn = type(m.body[12].value.ops[1]) + + +def read_file(path, mode="rb"): + with open(path, mode) as fp: + return fp.read() + + +def read_python_file(path): + fp = open(path, "rb") + try: + encoding = parse_encoding(fp) + data = fp.read() + if encoding: + data = data.decode(encoding) + return data + finally: + fp.close() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..320d94a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +build-backend = 'setuptools.build_meta' +requires = ['setuptools >= 47', 'wheel'] + +[tool.black] +line-length = 79 +target-version = ['py38'] diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..f643d01 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,104 @@ +[metadata] +name = Mako +version = attr: mako.__version__ +description = A super-fast templating language that borrows the best ideas from the existing templating languages. +long_description = file: README.rst +long_description_content_type = text/x-rst +url = https://www.makotemplates.org/ +author = Mike Bayer +author_email = mike@zzzcomputing.com +license = MIT +license_files = LICENSE +classifiers = + Development Status :: 5 - Production/Stable + License :: OSI Approved :: MIT License + Environment :: Web Environment + Intended Audience :: Developers + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 + Programming Language :: Python :: Implementation :: CPython + Programming Language :: Python :: Implementation :: PyPy + Topic :: Internet :: WWW/HTTP :: Dynamic Content +project_urls = + Documentation=https://docs.makotemplates.org + Issue Tracker=https://github.com/sqlalchemy/mako + +[options] +packages = find: +python_requires = >=3.8 +zip_safe = false + +install_requires = + MarkupSafe >= 0.9.2 + +[options.packages.find] +exclude = + test* + examples* + +[options.extras_require] +testing = + pytest +babel = + Babel +lingua = + lingua + +[options.entry_points] +python.templating.engines = + mako = mako.ext.turbogears:TGPlugin + +pygments.lexers = + mako = mako.ext.pygmentplugin:MakoLexer + html+mako = mako.ext.pygmentplugin:MakoHtmlLexer + xml+mako = mako.ext.pygmentplugin:MakoXmlLexer + js+mako = mako.ext.pygmentplugin:MakoJavascriptLexer + css+mako = mako.ext.pygmentplugin:MakoCssLexer + +babel.extractors = + mako = mako.ext.babelplugin:extract [babel] + +lingua.extractors= + mako = mako.ext.linguaplugin:LinguaMakoExtractor [lingua] + +console_scripts= + mako-render = mako.cmd:cmdline + +[egg_info] +tag_build = dev + +[tool:pytest] +addopts= --tb native -v -r fxX -p warnings +python_files=test/*test_*.py +python_classes=*Test Test* +filterwarnings = + error::DeprecationWarning:test + error::DeprecationWarning:mako + +[upload] +sign = 1 +identity = 4BFDF51E + +[flake8] +show-source = true +enable-extensions = G +# E203 is due to https://github.com/PyCQA/pycodestyle/issues/373 +ignore = + A003, + D, + E203,E305,E711,E712,E721,E722,E741, + N801,N802,N806, + RST304,RST303,RST299,RST399, + W503,W504 +exclude = .venv,.git,.tox,dist,docs/*,*egg,build +import-order-style = google +application-import-names = mako,test + +[mako_testing] +module_base = ./test/templates/modules +template_base = ./test/templates/ diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6068493 --- /dev/null +++ b/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup() diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/__init__.py diff --git a/test/ext/__init__.py b/test/ext/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/ext/__init__.py diff --git a/test/ext/test_babelplugin.py b/test/ext/test_babelplugin.py new file mode 100644 index 0000000..cfe79b6 --- /dev/null +++ b/test/ext/test_babelplugin.py @@ -0,0 +1,112 @@ +import io +import os + +import pytest + +from mako.testing.assertions import eq_ +from mako.testing.config import config +from mako.testing.exclusions import requires_babel +from mako.testing.fixtures import TemplateTest + + +class UsesExtract: + @pytest.fixture(scope="class") + def extract(self): + from mako.ext.babelplugin import extract + + return extract + + +@requires_babel +class PluginExtractTest(UsesExtract): + def test_parse_python_expression(self, extract): + input_ = io.BytesIO(b'<p>${_("Message")}</p>') + messages = list(extract(input_, ["_"], [], {})) + eq_(messages, [(1, "_", ("Message"), [])]) + + def test_python_gettext_call(self, extract): + input_ = io.BytesIO(b'<p>${_("Message")}</p>') + messages = list(extract(input_, ["_"], [], {})) + eq_(messages, [(1, "_", ("Message"), [])]) + + def test_translator_comment(self, extract): + input_ = io.BytesIO( + b""" + <p> + ## TRANSLATORS: This is a comment. + ${_("Message")} + </p>""" + ) + messages = list(extract(input_, ["_"], ["TRANSLATORS:"], {})) + eq_( + messages, + [ + ( + 4, + "_", + ("Message"), + [("TRANSLATORS: This is a comment.")], + ) + ], + ) + + +@requires_babel +class MakoExtractTest(UsesExtract, TemplateTest): + def test_extract(self, extract): + with open( + os.path.join(config.template_base, "gettext.mako") + ) as mako_tmpl: + messages = list( + extract( + mako_tmpl, + {"_": None, "gettext": None, "ungettext": (1, 2)}, + ["TRANSLATOR:"], + {}, + ) + ) + expected = [ + (1, "_", "Page arg 1", []), + (1, "_", "Page arg 2", []), + (10, "gettext", "Begin", []), + (14, "_", "Hi there!", ["TRANSLATOR: Hi there!"]), + (19, "_", "Hello", []), + (22, "_", "Welcome", []), + (25, "_", "Yo", []), + (36, "_", "The", ["TRANSLATOR: Ensure so and", "so, thanks"]), + (36, "ungettext", ("bunny", "bunnies", None), []), + (41, "_", "Goodbye", ["TRANSLATOR: Good bye"]), + (44, "_", "Babel", []), + (45, "ungettext", ("hella", "hellas", None), []), + (62, "_", "The", ["TRANSLATOR: Ensure so and", "so, thanks"]), + (62, "ungettext", ("bunny", "bunnies", None), []), + (68, "_", "Goodbye, really!", ["TRANSLATOR: HTML comment"]), + (71, "_", "P.S. byebye", []), + (77, "_", "Top", []), + (83, "_", "foo", []), + (83, "_", "hoho", []), + (85, "_", "bar", []), + (92, "_", "Inside a p tag", ["TRANSLATOR: <p> tag is ok?"]), + (95, "_", "Later in a p tag", ["TRANSLATOR: also this"]), + (99, "_", "No action at a distance.", []), + ] + eq_(expected, messages) + + def test_extract_utf8(self, extract): + with open( + os.path.join(config.template_base, "gettext_utf8.mako"), "rb" + ) as mako_tmpl: + message = next( + extract(mako_tmpl, {"_", None}, [], {"encoding": "utf-8"}) + ) + assert message == (1, "_", "K\xf6ln", []) + + def test_extract_cp1251(self, extract): + with open( + os.path.join(config.template_base, "gettext_cp1251.mako"), "rb" + ) as mako_tmpl: + message = next( + extract(mako_tmpl, {"_", None}, [], {"encoding": "cp1251"}) + ) + # "test" in Rusian. File encoding is cp1251 (aka "windows-1251") + assert message == (1, "_", "\u0442\u0435\u0441\u0442", []) diff --git a/test/ext/test_linguaplugin.py b/test/ext/test_linguaplugin.py new file mode 100644 index 0000000..6e2faa8 --- /dev/null +++ b/test/ext/test_linguaplugin.py @@ -0,0 +1,63 @@ +import os + +import pytest + +from mako.testing.assertions import eq_ +from mako.testing.config import config +from mako.testing.exclusions import requires_lingua +from mako.testing.fixtures import TemplateTest + + +class MockOptions: + keywords = [] + domain = None + comment_tag = True + + +@requires_lingua +class MakoExtractTest(TemplateTest): + @pytest.fixture(autouse=True) + def register_lingua_extractors(self): + from lingua.extractors import register_extractors + + register_extractors() + + def test_extract(self): + from mako.ext.linguaplugin import LinguaMakoExtractor + + plugin = LinguaMakoExtractor({"comment-tags": "TRANSLATOR"}) + messages = list( + plugin( + os.path.join(config.template_base, "gettext.mako"), + MockOptions(), + ) + ) + msgids = [(m.msgid, m.msgid_plural) for m in messages] + eq_( + msgids, + [ + ("Page arg 1", None), + ("Page arg 2", None), + ("Begin", None), + ("Hi there!", None), + ("Hello", None), + ("Welcome", None), + ("Yo", None), + ("The", None), + ("bunny", "bunnies"), + ("Goodbye", None), + ("Babel", None), + ("hella", "hellas"), + ("The", None), + ("bunny", "bunnies"), + ("Goodbye, really!", None), + ("P.S. byebye", None), + ("Top", None), + ("foo", None), + ("hoho", None), + ("bar", None), + ("Inside a p tag", None), + ("Later in a p tag", None), + ("No action at a distance.", None), + ], + ) diff --git a/test/foo/__init__.py b/test/foo/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/foo/__init__.py diff --git a/test/foo/mod_no_encoding.py b/test/foo/mod_no_encoding.py new file mode 100644 index 0000000..004cc44 --- /dev/null +++ b/test/foo/mod_no_encoding.py @@ -0,0 +1,7 @@ +from mako.lookup import TemplateLookup + +template_lookup = TemplateLookup() + + +def run(): + tpl = template_lookup.get_template("not_found.html") diff --git a/test/foo/test_ns.py b/test/foo/test_ns.py new file mode 100644 index 0000000..f67e22e --- /dev/null +++ b/test/foo/test_ns.py @@ -0,0 +1,11 @@ +def foo1(context): + context.write("this is foo1.") + return "" + + +def foo2(context, x): + context.write("this is foo2, x is " + x) + return "" + + +foo3 = "I'm not a callable" diff --git a/test/module_to_import.py b/test/module_to_import.py new file mode 100644 index 0000000..11ffb98 --- /dev/null +++ b/test/module_to_import.py @@ -0,0 +1,2 @@ +def some_function(): + pass diff --git a/test/sample_module_namespace.py b/test/sample_module_namespace.py new file mode 100644 index 0000000..886e8dd --- /dev/null +++ b/test/sample_module_namespace.py @@ -0,0 +1,8 @@ +def foo1(context): + context.write("this is foo1.") + return "" + + +def foo2(context, x): + context.write("this is foo2, x is " + x) + return "" diff --git a/test/templates/badbom.html b/test/templates/badbom.html new file mode 100644 index 0000000..2af085b --- /dev/null +++ b/test/templates/badbom.html @@ -0,0 +1,2 @@ +## -*- coding: ascii -*- +Alors vous imaginez ma surprise, au lever du jour, quand une drôle de petite voix m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! »
\ No newline at end of file diff --git a/test/templates/bom.html b/test/templates/bom.html new file mode 100644 index 0000000..1259946 --- /dev/null +++ b/test/templates/bom.html @@ -0,0 +1 @@ +Alors vous imaginez ma surprise, au lever du jour, quand une drôle de petite voix m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! »
\ No newline at end of file diff --git a/test/templates/bommagic.html b/test/templates/bommagic.html new file mode 100644 index 0000000..0e4b587 --- /dev/null +++ b/test/templates/bommagic.html @@ -0,0 +1,2 @@ +## -*- coding: utf-8 -*- +Alors vous imaginez ma surprise, au lever du jour, quand une drôle de petite voix m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! »
\ No newline at end of file diff --git a/test/templates/chs_unicode_py3k.html b/test/templates/chs_unicode_py3k.html new file mode 100644 index 0000000..1ee49cc --- /dev/null +++ b/test/templates/chs_unicode_py3k.html @@ -0,0 +1,10 @@ +<% + msg = '新中国的主席' +%> + +<%def name="welcome(who, place='北京')"> +Welcome ${who} to ${place}. +</%def> + +${name} 是 ${msg}<br/> +${welcome('你')} diff --git a/test/templates/chs_utf8.html b/test/templates/chs_utf8.html new file mode 100644 index 0000000..50886be --- /dev/null +++ b/test/templates/chs_utf8.html @@ -0,0 +1,16 @@ +<% + msg = '新中国的主席' +%> + +<%def name="welcome(who, place='北京')"> +Welcome ${who} to ${place}. +</%def> + +<%def name="welcome_buffered(who, place='北京')" buffered="True"> +Welcome ${who} to ${place}. +</%def> + +${name} 是 ${msg}<br/> +${welcome('你')} +${welcome_buffered('你')} + diff --git a/test/templates/cmd_good.mako b/test/templates/cmd_good.mako new file mode 100644 index 0000000..68ebec4 --- /dev/null +++ b/test/templates/cmd_good.mako @@ -0,0 +1 @@ +hello world ${x}
\ No newline at end of file diff --git a/test/templates/cmd_runtime.mako b/test/templates/cmd_runtime.mako new file mode 100644 index 0000000..6c2675b --- /dev/null +++ b/test/templates/cmd_runtime.mako @@ -0,0 +1 @@ +${q}
\ No newline at end of file diff --git a/test/templates/cmd_syntax.mako b/test/templates/cmd_syntax.mako new file mode 100644 index 0000000..d2117db --- /dev/null +++ b/test/templates/cmd_syntax.mako @@ -0,0 +1 @@ +${x
\ No newline at end of file diff --git a/test/templates/crlf.html b/test/templates/crlf.html new file mode 100644 index 0000000..d2620db --- /dev/null +++ b/test/templates/crlf.html @@ -0,0 +1,19 @@ +<html>
+
+<%page args="a=['foo',
+ 'bar']"/>
+
+like the name says.
+
+ % for x in [1,2,3]:
+ ${x}\
+ % endfor
+
+${trumpeter == 'Miles' and trumpeter or \
+ 'Dizzy'}
+
+<%def name="hi()">
+ hi!
+</%def>
+
+</html>
diff --git a/test/templates/foo/modtest.html.py b/test/templates/foo/modtest.html.py new file mode 100644 index 0000000..c35420f --- /dev/null +++ b/test/templates/foo/modtest.html.py @@ -0,0 +1,25 @@ +from mako import cache +from mako import runtime + +UNDEFINED = runtime.UNDEFINED +__M_dict_builtin = dict +__M_locals_builtin = locals +_magic_number = 5 +_modified_time = 1267565427.7968459 +_template_filename = "/Users/classic/dev/mako/test/templates/modtest.html" +_template_uri = "/modtest.html" +_template_cache = cache.Cache(__name__, _modified_time) +_source_encoding = None +_exports = [] + + +def render_body(context, **pageargs): + context.caller_stack._push_frame() + try: + __M_locals = __M_dict_builtin(pageargs=pageargs) + __M_writer = context.writer() + # SOURCE LINE 1 + __M_writer("this is a test") + return "" + finally: + context.caller_stack._pop_frame() diff --git a/test/templates/gettext.mako b/test/templates/gettext.mako new file mode 100644 index 0000000..45b8262 --- /dev/null +++ b/test/templates/gettext.mako @@ -0,0 +1,130 @@ +<%page args="x, y=_('Page arg 1'), z=_('Page arg 2')"/> +<%! +import random +def gettext(message): return message +_ = gettext +def ungettext(s, p, c): + if c == 1: + return s + return p +top = gettext('Begin') +%> +<% + # TRANSLATOR: Hi there! + hithere = _('Hi there!') + + # TRANSLATOR: you should not be seeing this in the .po + rows = [[v for v in range(0,10)] for row in range(0,10)] + + hello = _('Hello') +%> +<div id="header"> + ${_('Welcome')} +</div> +<table> + % for row in (hithere, hello, _('Yo')): + ${makerow(row)} + % endfor + ${makerow(count=2)} +</table> + + +<div id="main"> + +## TRANSLATOR: Ensure so and +## so, thanks + ${_('The')} fuzzy ${ungettext('bunny', 'bunnies', random.randint(1, 2))} +</div> + +<div id="footer"> + ## TRANSLATOR: Good bye + ${_('Goodbye')} +</div> + +<%def name="makerow(row=_('Babel'), count=1)"> + <!-- ${ungettext('hella', 'hellas', count)} --> + % for i in range(count): + <tr> + % for name in row: + <td>${name}</td>\ + % endfor + </tr> + % endfor +</%def> + +<%def name="comment()"> + <!-- ${caller.body()} --> +</%def> + +<%block name="foo"> + ## TRANSLATOR: Ensure so and + ## so, thanks + ${_('The')} fuzzy ${ungettext('bunny', 'bunnies', random.randint(1, 2))} +</%block> + +<%call expr="comment"> + P.S. + ## TRANSLATOR: HTML comment + ${_('Goodbye, really!')} +</%call> + +<!-- ${_('P.S. byebye')} --> + +<div id="end"> + <a href="#top"> + ## TRANSLATOR: you won't see this either + + ${_('Top')} + </a> +</div> + +<%def name="panel()"> + +${_(u'foo')} <%self:block_tpl title="#123", name="_('baz')" value="${_('hoho')}" something="hi'there" somethingelse='hi"there'> + +${_(u'bar')} + +</%self:block_tpl> + +</%def> + +## TRANSLATOR: <p> tag is ok? +<p>${_("Inside a p tag")}</p> + +## TRANSLATOR: also this +<p>${even_with_other_code_first()} - ${_("Later in a p tag")}</p> + +## TRANSLATOR: we still ignore comments too far from the string + +<p>${_("No action at a distance.")}</p> + +## TRANSLATOR: nothing to extract from these blocks + +% if 1==1: +<p>One is one!</p> +% elif 1==2: +<p>One is two!</p> +% else: +<p>How much is one?</p> +% endif + +% for i in range(10): +<p>${i} squared is ${i*i}</p> +% else: +<p>Done with squares!</p> +% endfor + +% while random.randint(1,6) != 6: +<p>Not 6!</p> +% endwhile + +## TRANSLATOR: for now, try/except blocks are ignored + +% try: +<% 1/0 %> +% except: +<p>Failed!</p> +% endtry + +## TRANSLATOR: this should not cause a parse error +${ 1 } diff --git a/test/templates/gettext_cp1251.mako b/test/templates/gettext_cp1251.mako new file mode 100644 index 0000000..9341d93 --- /dev/null +++ b/test/templates/gettext_cp1251.mako @@ -0,0 +1 @@ +${_("")} diff --git a/test/templates/gettext_utf8.mako b/test/templates/gettext_utf8.mako new file mode 100644 index 0000000..761f946 --- /dev/null +++ b/test/templates/gettext_utf8.mako @@ -0,0 +1 @@ +${_("Köln")} diff --git a/test/templates/index.html b/test/templates/index.html new file mode 100644 index 0000000..591e380 --- /dev/null +++ b/test/templates/index.html @@ -0,0 +1 @@ +this is index
\ No newline at end of file diff --git a/test/templates/internationalization.html b/test/templates/internationalization.html new file mode 100644 index 0000000..da5b61c --- /dev/null +++ b/test/templates/internationalization.html @@ -0,0 +1,920 @@ +<div class="rst-docs"> + + <h1 class="pudge-member-page-heading">Internationalization, Localization and Unicode</h1> + + <table rules="none" frame="void" class="docinfo"> +<col class="docinfo-name"></col> +<col class="docinfo-content"></col> +<tbody valign="top"> +<tr><th class="docinfo-name">Author:</th> +<td>James Gardner</td></tr> +<tr class="field"><th class="docinfo-name">updated:</th><td class="field-body">2006-12-11</td> +</tr> +</tbody> +</table> + + <div class="note"> +<p class="first admonition-title">Note</p> +<p>This is a work in progress. We hope the internationalization, localization +and Unicode support in Pylons is now robust and flexible but we would +appreciate hearing about any issues we have. Just drop a line to the +pylons-discuss mailing list on Google Groups.</p> +<p class="last">This is the first draft of the full document including Unicode. Expect +some typos and spelling mistakes!</p> +</div> +<div class="contents topic"> +<p class="topic-title first"><a id="table-of-contents" name="table-of-contents">Table of Contents</a></p> +<ul class="auto-toc simple"> +<li><a href="#understanding-unicode" id="id1" name="id1" class="reference">1 Understanding Unicode</a><ul class="auto-toc"> +<li><a href="#what-is-unicode" id="id2" name="id2" class="reference">1.1 What is Unicode?</a></li> +<li><a href="#unicode-in-python" id="id3" name="id3" class="reference">1.2 Unicode in Python</a></li> +<li><a href="#unicode-literals-in-python-source-code" id="id4" name="id4" class="reference">1.3 Unicode Literals in Python Source Code</a></li> +<li><a href="#input-and-output" id="id5" name="id5" class="reference">1.4 Input and Output</a></li> +<li><a href="#unicode-filenames" id="id6" name="id6" class="reference">1.5 Unicode Filenames</a></li> +</ul> +</li> +<li><a href="#applying-this-to-web-programming" id="id7" name="id7" class="reference">2 Applying this to Web Programming</a><ul class="auto-toc"> +<li><a href="#request-parameters" id="id8" name="id8" class="reference">2.1 Request Parameters</a></li> +<li><a href="#templating" id="id9" name="id9" class="reference">2.2 Templating</a></li> +<li><a href="#output-encoding" id="id10" name="id10" class="reference">2.3 Output Encoding</a></li> +<li><a href="#databases" id="id11" name="id11" class="reference">2.4 Databases</a></li> +</ul> +</li> +<li><a href="#internationalization-and-localization" id="id12" name="id12" class="reference">3 Internationalization and Localization</a><ul class="auto-toc"> +<li><a href="#getting-started" id="id13" name="id13" class="reference">3.1 Getting Started</a></li> +<li><a href="#testing-the-application" id="id14" name="id14" class="reference">3.2 Testing the Application</a></li> +<li><a href="#missing-translations" id="id15" name="id15" class="reference">3.3 Missing Translations</a></li> +<li><a href="#translations-within-templates" id="id16" name="id16" class="reference">3.4 Translations Within Templates</a></li> +<li><a href="#producing-a-python-egg" id="id17" name="id17" class="reference">3.5 Producing a Python Egg</a></li> +<li><a href="#plural-forms" id="id18" name="id18" class="reference">3.6 Plural Forms</a></li> +</ul> +</li> +<li><a href="#summary" id="id19" name="id19" class="reference">4 Summary</a></li> +<li><a href="#further-reading" id="id20" name="id20" class="reference">5 Further Reading</a></li> +</ul> +</div> +<p>Internationalization and localization are means of adapting software for +non-native environments, especially for other nations and cultures.</p> +<p>Parts of an application which might need to be localized might include:</p> +<blockquote> +<ul class="simple"> +<li>Language</li> +<li>Date/time format</li> +<li>Formatting of numbers e.g. decimal points, positioning of separators, +character used as separator</li> +<li>Time zones (UTC in internationalized environments)</li> +<li>Currency</li> +<li>Weights and measures</li> +</ul> +</blockquote> +<p>The distinction between internationalization and localization is subtle but +important. Internationalization is the adaptation of products for potential use +virtually everywhere, while localization is the addition of special features +for use in a specific locale.</p> +<p>For example, in terms of language used in software, internationalization is the +process of marking up all strings that might need to be translated whilst +localization is the process of producing translations for a particular locale.</p> +<p>Pylons provides built-in support to enable you to internationalize language but +leaves you to handle any other aspects of internationalization which might be +appropriate to your application.</p> +<div class="note"> +<p class="first admonition-title">Note</p> +<p class="last">Internationalization is often abbreviated as I18N (or i18n or I18n) where the +number 18 refers to the number of letters omitted. +Localization is often abbreviated L10n or l10n in the same manner. These +abbreviations also avoid picking one spelling (internationalisation vs. +internationalization, etc.) over the other.</p> +</div> +<p>In order to represent characters from multiple languages, you will need to use +Unicode so this documentation will start with a description of why Unicode is +useful, its history and how to use Unicode in Python.</p> +<div class="section"> +<h1><a href="#id1" id="understanding-unicode" name="understanding-unicode" class="toc-backref">1 Understanding Unicode</a></h1> +<p>If you've ever come across text in a foreign language that contains lots of +<tt class="docutils literal"><span class="pre">????</span></tt> characters or have written some Python code and received a message +such as <tt class="docutils literal"><span class="pre">UnicodeDecodeError:</span> <span class="pre">'ascii'</span> <span class="pre">codec</span> <span class="pre">can't</span> <span class="pre">decode</span> <span class="pre">byte</span> <span class="pre">0xff</span> <span class="pre">in</span> <span class="pre">position</span> +<span class="pre">6:</span> <span class="pre">ordinal</span> <span class="pre">not</span> <span class="pre">in</span> <span class="pre">range(128)</span></tt> then you have run into a problem with character +sets, encodings, Unicode and the like.</p> +<p>The truth is that many developers are put off by Unicode because most of the +time it is possible to muddle through rather than take the time to learn the +basics. To make the problem worse if you have a system that manages to fudge +the issues and just about work and then start trying to do things properly with +Unicode it often highlights problems in other parts of your code.</p> +<p>The good news is that Python has great Unicode support, so the rest of +this article will show you how to correctly use Unicode in Pylons to avoid +unwanted <tt class="docutils literal"><span class="pre">?</span></tt> characters and <tt class="docutils literal"><span class="pre">UnicodeDecodeErrors</span></tt>.</p> +<div class="section"> +<h2><a href="#id2" id="what-is-unicode" name="what-is-unicode" class="toc-backref">1.1 What is Unicode?</a></h2> +<p>When computers were first being used the characters that were most important +were unaccented English letters. Each of these letters could be represented by +a number between 32 and 127 and thus was born ASCII, a character set where +space was 32, the letter "A" was 65 and everything could be stored in 7 bits.</p> +<p>Most computers in those days were using 8-bit bytes so people quickly realized +that they could use the codes 128-255 for their own purposes. Different people +used the codes 128-255 to represent different characters and before long these +different sets of characters were also standardized into <em>code pages</em>. This +meant that if you needed some non-ASCII characters in a document you could also +specify a codepage which would define which extra characters were available. +For example Israel DOS used a code page called 862, while Greek users used 737. +This just about worked for Western languages provided you didn't want to write +an Israeli document with Greek characters but it didn't work at all for Asian +languages where there are many more characters than can be represented in 8 +bits.</p> +<p>Unicode is a character set that solves these problems by uniquely defining +<em>every</em> character that is used anywhere in the world. Rather than defining a +character as a particular combination of bits in the way ASCII does, each +character is assigned a <em>code point</em>. For example the word <tt class="docutils literal"><span class="pre">hello</span></tt> is made +from code points <tt class="docutils literal"><span class="pre">U+0048</span> <span class="pre">U+0065</span> <span class="pre">U+006C</span> <span class="pre">U+006C</span> <span class="pre">U+006F</span></tt>. The full list of code +points can be found at <a href="http://www.unicode.org/charts/" class="reference">http://www.unicode.org/charts/</a>.</p> +<p>There are lots of different ways of encoding Unicode code points into bits but +the most popular encoding is UTF-8. Using UTF-8, every code point from 0-127 is +stored in a single byte. Only code points 128 and above are stored using 2, 3, +in fact, up to 6 bytes. This has the useful side effect that English text looks +exactly the same in UTF-8 as it did in ASCII, because for every +ASCII character with hexadecimal value 0xXY, the corresponding Unicode +code point is U+00XY. This backwards compatibility is why if you are developing +an application that is only used by English speakers you can often get away +without handling characters properly and still expect things to work most of +the time. Of course, if you use a different encoding such as UTF-16 this +doesn't apply since none of the code points are encoded to 8 bits.</p> +<p>The important things to note from the discussion so far are that:</p> +<ul> +<li><p class="first">Unicode can represent pretty much any character in any writing system in +widespread use today</p> +</li> +<li><p class="first">Unicode uses code points to represent characters and the way these map to bits +in memory depends on the encoding</p> +</li> +<li><dl class="first docutils"> +<dt>The most popular encoding is UTF-8 which has several convenient properties:</dt> +<dd><ol class="first last arabic simple"> +<li>It can handle any Unicode code point</li> +<li>A Unicode string is turned into a string of bytes containing no embedded +zero bytes. This avoids byte-ordering issues, and means UTF-8 strings can be +processed by C functions such as strcpy() and sent through protocols that can't +handle zero bytes</li> +<li>A string of ASCII text is also valid UTF-8 text</li> +<li>UTF-8 is fairly compact; the majority of code points are turned into two +bytes, and values less than 128 occupy only a single byte.</li> +<li>If bytes are corrupted or lost, it's possible to determine the start of +the next UTF-8-encoded code point and resynchronize.</li> +</ol> +</dd> +</dl> +</li> +</ul> +<div class="note"> +<p class="first admonition-title">Note</p> +<p class="last">Since Unicode 3.1, some extensions have even been defined so that the +defined range is now U+000000 to U+10FFFF (21 bits), and formally, the +character set is defined as 31-bits to allow for future expansion. It is a myth +that there are 65,536 Unicode code points and that every Unicode letter can +really be squeezed into two bytes. It is also incorrect to think that UTF-8 can +represent less characters than UTF-16. UTF-8 simply uses a variable number of +bytes for a character, sometimes just one byte (8 bits).</p> +</div> +</div> +<div class="section"> +<h2><a href="#id3" id="unicode-in-python" name="unicode-in-python" class="toc-backref">1.2 Unicode in Python</a></h2> +<p>In Python Unicode strings are expressed as instances of the built-in +<tt class="docutils literal"><span class="pre">unicode</span></tt> type. Under the hood, Python represents Unicode strings as either +16 or 32 bit integers, depending on how the Python interpreter was compiled.</p> +<p>The <tt class="docutils literal"><span class="pre">unicode()</span></tt> constructor has the signature <tt class="docutils literal"><span class="pre">unicode(string[,</span> <span class="pre">encoding,</span> +<span class="pre">errors])</span></tt>. All of its arguments should be 8-bit strings. The first argument is +converted to Unicode using the specified encoding; if you leave off the +encoding argument, the ASCII encoding is used for the conversion, so characters +greater than 127 will be treated as errors:</p> +<pre class="literal-block"> +>>> unicode('hello') +u'hello' +>>> s = unicode('hello') +>>> type(s) +<type 'unicode'> +>>> unicode('hello' + chr(255)) +Traceback (most recent call last): + File "<stdin>", line 1, in ? +UnicodeDecodeError: 'ascii' codec can't decode byte 0xff in position 6: + ordinal not in range(128) +</pre> +<p>The <tt class="docutils literal"><span class="pre">errors</span></tt> argument specifies what to do if the string can't be decoded to +ascii. Legal values for this argument are <tt class="docutils literal"><span class="pre">'strict'</span></tt> (raise a +<tt class="docutils literal"><span class="pre">UnicodeDecodeError</span></tt> exception), <tt class="docutils literal"><span class="pre">'replace'</span></tt> (replace the character that +can't be decoded with another one), or <tt class="docutils literal"><span class="pre">'ignore'</span></tt> (just leave the character +out of the Unicode result).</p> +<blockquote> +<pre class="doctest-block"> +>>> unicode('\x80abc', errors='strict') +Traceback (most recent call last): + File "<stdin>", line 1, in ? +UnicodeDecodeError: 'ascii' codec can't decode byte 0x80 in position 0: + ordinal not in range(128) +>>> unicode('\x80abc', errors='replace') +u'\ufffdabc' +>>> unicode('\x80abc', errors='ignore') +u'abc' +</pre> +</blockquote> +<p>It is important to understand the difference between <em>encoding</em> and <em>decoding</em>. +Unicode strings are considered to be the Unicode code points but any +representation of the Unicode string has to be encoded to something else, for +example UTF-8 or ASCII. So when you are converting an ASCII or UTF-8 string to +Unicode you are <em>decoding</em> it and when you are converting from Unicode to UTF-8 +or ASCII you are <em>encoding</em> it. This is why the error in the example above says +that the ASCII codec cannot decode the byte <tt class="docutils literal"><span class="pre">0x80</span></tt> from ASCII to Unicode +because it is not in the range(128) or 0-127. In fact <tt class="docutils literal"><span class="pre">0x80</span></tt> is hex for 128 +which the first number outside the ASCII range. However if we tell Python that +the character <tt class="docutils literal"><span class="pre">0x80</span></tt> is encoded with the <tt class="docutils literal"><span class="pre">'latin-1'</span></tt>, <tt class="docutils literal"><span class="pre">'iso_8859_1'</span></tt> or +<tt class="docutils literal"><span class="pre">'8859'</span></tt> character sets (which incidentally are different names for the same +thing) we get the result we expected:</p> +<textarea name="code" class="python"> +>>> unicode('\x80', encoding='latin-1') +u'\x80' +</textarea><div class="note"> +<p class="first admonition-title">Note</p> +<p class="last">The character encodings Python supports are listed at +<a href="http://docs.python.org/lib/standard-encodings.html" class="reference">http://docs.python.org/lib/standard-encodings.html</a></p> +</div> +<p>Unicode objects in Python have most of the same methods that normal Python +strings provide. Python will try to use the <tt class="docutils literal"><span class="pre">'ascii'</span></tt> codec to convert +strings to Unicode if you do an operation on both types:</p> +<textarea name="code" class="python"> +>>> a = 'hello' +>>> b = unicode(' world!') +>>> print a + b +u'hello world!' +</textarea><p>You can encode a Unicode string using a particular encoding like this:</p> +<textarea name="code" class="python"> +>>> u'Hello World!'.encode('UTF-8') +'Hello World!' +</textarea></div> +<div class="section"> +<h2><a href="#id4" id="unicode-literals-in-python-source-code" name="unicode-literals-in-python-source-code" class="toc-backref">1.3 Unicode Literals in Python Source Code</a></h2> +<p>In Python source code, Unicode literals are written as strings prefixed with +the 'u' or 'U' character:</p> +<textarea name="code" class="python"> +>>> u'abcdefghijk' +>>> U'lmnopqrstuv' +</textarea><p>You can also use <tt class="docutils literal"><span class="pre">"</span></tt>, <tt class="docutils literal"><span class="pre">"""`</span></tt> or <tt class="docutils literal"><span class="pre">'''</span></tt> versions too. For example:</p> +<textarea name="code" class="python"> +>>> u"""This +... is a really long +... Unicode string""" +</textarea><p>Specific code points can be written using the <tt class="docutils literal"><span class="pre">\u</span></tt> escape sequence, which is +followed by four hex digits giving the code point. If you use <tt class="docutils literal"><span class="pre">\U</span></tt> instead +you specify 8 hex digits instead of 4. Unicode literals can also use the same +escape sequences as 8-bit strings, including <tt class="docutils literal"><span class="pre">\x</span></tt>, but <tt class="docutils literal"><span class="pre">\x</span></tt> only takes two +hex digits so it can't express all the available code points. You can add +characters to Unicode strings using the <tt class="docutils literal"><span class="pre">unichr()</span></tt> built-in function and find +out what the ordinal is with <tt class="docutils literal"><span class="pre">ord()</span></tt>.</p> +<p>Here is an example demonstrating the different alternatives:</p> +<textarea name="code" class="python"> +>>> s = u"\x66\u0072\u0061\U0000006e" + unichr(231) + u"ais" +>>> # ^^^^ two-digit hex escape +>>> # ^^^^^^ four-digit Unicode escape +>>> # ^^^^^^^^^^ eight-digit Unicode escape +>>> for c in s: print ord(c), +... +97 102 114 97 110 231 97 105 115 +>>> print s +franÁais +</textarea><p>Using escape sequences for code points greater than 127 is fine in small doses +but Python 2.4 and above support writing Unicode literals in any encoding as +long as you declare the encoding being used by including a special comment as +either the first or second line of the source file:</p> +<textarea name="code" class="python"> +#!/usr/bin/env python +# -*- coding: latin-1 -*- + +u = u'abcdÈ' +print ord(u[-1]) +</textarea><p>If you don't include such a comment, the default encoding used will be ASCII. +Versions of Python before 2.4 were Euro-centric and assumed Latin-1 as a +default encoding for string literals; in Python 2.4, characters greater than +127 still work but result in a warning. For example, the following program has +no encoding declaration:</p> +<textarea name="code" class="python"> +#!/usr/bin/env python +u = u'abcdÈ' +print ord(u[-1]) +</textarea><p>When you run it with Python 2.4, it will output the following warning:</p> +<pre class="literal-block"> +sys:1: DeprecationWarning: Non-ASCII character '\xe9' in file testas.py on line +2, but no encoding declared; see http://www.python.org/peps/pep-0263.html for de +tails +</pre> +<p>and then the following output:</p> +<pre class="literal-block"> +233 +</pre> +<p>For real world use it is recommended that you use the UTF-8 encoding for your +file but you must be sure that your text editor actually saves the file as +UTF-8 otherwise the Python interpreter will try to parse UTF-8 characters but +they will actually be stored as something else.</p> +<div class="note"> +<p class="first admonition-title">Note</p> +<p class="last">Windows users who use the <a href="http://www.scintilla.org/SciTE.html" class="reference">SciTE</a> +editor can specify the encoding of their file from the menu using the +<tt class="docutils literal"><span class="pre">File->Encoding</span></tt>.</p> +</div> +<div class="note"> +<p class="first admonition-title">Note</p> +<p class="last">If you are working with Unicode in detail you might also be interested in +the <tt class="docutils literal"><span class="pre">unicodedata</span></tt> module which can be used to find out Unicode properties +such as a character's name, category, numeric value and the like.</p> +</div> +</div> +<div class="section"> +<h2><a href="#id5" id="input-and-output" name="input-and-output" class="toc-backref">1.4 Input and Output</a></h2> +<p>We now know how to use Unicode in Python source code but input and output can +also be different using Unicode. Of course, some libraries natively support +Unicode and if these libraries return Unicode objects you will not have to do +anything special to support them. XML parsers and SQL databases frequently +support Unicode for example.</p> +<p>If you remember from the discussion earlier, Unicode data consists of code +points. In order to send Unicode data via a socket or write it to a file you +usually need to encode it to a series of bytes and then decode the data back to +Unicode when reading it. You can of course perform the encoding manually +reading a byte at the time but since encodings such as UTF-8 can have variable +numbers of bytes per character it is usually much easier to use Python's +built-in support in the form of the <tt class="docutils literal"><span class="pre">codecs</span></tt> module.</p> +<p>The codecs module includes a version of the <tt class="docutils literal"><span class="pre">open()</span></tt> function that +returns a file-like object that assumes the file's contents are in a specified +encoding and accepts Unicode parameters for methods such as <tt class="docutils literal"><span class="pre">.read()</span></tt> and +<tt class="docutils literal"><span class="pre">.write()</span></tt>.</p> +<p>The function's parameters are open(filename, mode='rb', encoding=None, +errors='strict', buffering=1). <tt class="docutils literal"><span class="pre">mode</span></tt> can be 'r', 'w', or 'a', just like the +corresponding parameter to the regular built-in <tt class="docutils literal"><span class="pre">open()</span></tt> function. You can +add a <tt class="docutils literal"><span class="pre">+</span></tt> character to update the file. <tt class="docutils literal"><span class="pre">buffering</span></tt> is similar to the +standard function's parameter. <tt class="docutils literal"><span class="pre">encoding</span></tt> is a string giving the encoding to +use, if not specified or specified as <tt class="docutils literal"><span class="pre">None</span></tt>, a regular Python file object +that accepts 8-bit strings is returned. Otherwise, a wrapper object is +returned, and data written to or read from the wrapper object will be converted +as needed. <tt class="docutils literal"><span class="pre">errors</span></tt> specifies the action for encoding errors and can be one +of the usual values of <tt class="docutils literal"><span class="pre">'strict'</span></tt>, <tt class="docutils literal"><span class="pre">'ignore'</span></tt>, or <tt class="docutils literal"><span class="pre">'replace'</span></tt> which we +saw right at the begining of this document when we were encoding strings in +Python source files.</p> +<p>Here is an example of how to read Unicode from a UTF-8 encoded file:</p> +<textarea name="code" class="python"> +import codecs +f = codecs.open('unicode.txt', encoding='utf-8') +for line in f: + print repr(line) +</textarea><p>It's also possible to open files in update mode, allowing both reading and writing:</p> +<textarea name="code" class="python"> +f = codecs.open('unicode.txt', encoding='utf-8', mode='w+') +f.write(u"\x66\u0072\u0061\U0000006e" + unichr(231) + u"ais") +f.seek(0) +print repr(f.readline()[:1]) +f.close() +</textarea><p>Notice that we used the <tt class="docutils literal"><span class="pre">repr()</span></tt> function to display the Unicode data. This +is very useful because if you tried to print the Unicode data directly, Python +would need to encode it before it could be sent the console and depending on +which characters were present and the character set used by the console, an +error might be raised. This is avoided if you use <tt class="docutils literal"><span class="pre">repr()</span></tt>.</p> +<p>The Unicode character <tt class="docutils literal"><span class="pre">U+FEFF</span></tt> is used as a byte-order mark or BOM, and is often +written as the first character of a file in order to assist with auto-detection +of the file's byte ordering. Some encodings, such as UTF-16, expect a BOM to be +present at the start of a file, but with others such as UTF-8 it isn't necessary.</p> +<p>When such an encoding is used, the BOM will be automatically written as the +first character and will be silently dropped when the file is read. There are +variants of these encodings, such as 'utf-16-le' and 'utf-16-be' for +little-endian and big-endian encodings, that specify one particular byte +ordering and don't skip the BOM.</p> +<div class="note"> +<p class="first admonition-title">Note</p> +<p class="last">Some editors including SciTE will put a byte order mark (BOM) in the text +file when saved as UTF-8, which is strange because UTF-8 doesn't need BOMs.</p> +</div> +</div> +<div class="section"> +<h2><a href="#id6" id="unicode-filenames" name="unicode-filenames" class="toc-backref">1.5 Unicode Filenames</a></h2> +<p>Most modern operating systems support the use of Unicode filenames. The +filenames are transparently converted to the underlying filesystem encoding. +The type of encoding depends on the operating system.</p> +<p>On Windows 9x, the encoding is <tt class="docutils literal"><span class="pre">mbcs</span></tt>.</p> +<p>On Mac OS X, the encoding is <tt class="docutils literal"><span class="pre">utf-8</span></tt>.</p> +<p>On Unix, the encoding is the user's preference according to the +result of nl_langinfo(CODESET), or None if the nl_langinfo(CODESET) failed.</p> +<p>On Windows NT+, file names are Unicode natively, so no conversion is performed. +getfilesystemencoding still returns <tt class="docutils literal"><span class="pre">mbcs</span></tt>, as this is the encoding that +applications should use when they explicitly want to convert Unicode strings to +byte strings that are equivalent when used as file names.</p> +<p><tt class="docutils literal"><span class="pre">mbcs</span></tt> is a special encoding for Windows that effectively means "use +whichever encoding is appropriate". In Python 2.3 and above you can find out +the system encoding with <tt class="docutils literal"><span class="pre">sys.getfilesystemencoding()</span></tt>.</p> +<p>Most file and directory functions and methods support Unicode. For example:</p> +<textarea name="code" class="python"> +filename = u"\x66\u0072\u0061\U0000006e" + unichr(231) + u"ais" +f = open(filename, 'w') +f.write('Some data\n') +f.close() +</textarea><p>Other functions such as <tt class="docutils literal"><span class="pre">os.listdir()</span></tt> will return Unicode if you pass a +Unicode argument and will try to return strings if you pass an ordinary 8 bit +string. For example running this example as <tt class="docutils literal"><span class="pre">test.py</span></tt>:</p> +<textarea name="code" class="python"> +filename = u"Sample " + unichar(5000) +f = open(filename, 'w') +f.close() + +import os +print os.listdir('.') +print os.listdir(u'.') +</textarea><p>will produce the following output:</p> +<blockquote> +['Sample?', 'test.py'] +[u'Sampleu1388', u'test.py']</blockquote> +</div> +</div> +<div class="section"> +<h1><a href="#id7" id="applying-this-to-web-programming" name="applying-this-to-web-programming" class="toc-backref">2 Applying this to Web Programming</a></h1> +<p>So far we've seen how to use encoding in source files and seen how to decode +text to Unicode and encode it back to text. We've also seen that Unicode +objects can be manipulated in similar ways to strings and we've seen how to +perform input and output operations on files. Next we are going to look at how +best to use Unicode in a web app.</p> +<p>The main rule is this:</p> +<pre class="literal-block"> +Your application should use Unicode for all strings internally, decoding +any input to Unicode as soon as it enters the application and encoding the +Unicode to UTF-8 or another encoding only on output. +</pre> +<p>If you fail to do this you will find that <tt class="docutils literal"><span class="pre">UnicodeDecodeError</span></tt> s will start +popping up in unexpected places when Unicode strings are used with normal 8-bit +strings because Python's default encoding is ASCII and it will try to decode +the text to ASCII and fail. It is always better to do any encoding or decoding +at the edges of your application otherwise you will end up patching lots of +different parts of your application unnecessarily as and when errors pop up.</p> +<p>Unless you have a very good reason not to it is wise to use UTF-8 as the +default encoding since it is so widely supported.</p> +<p>The second rule is:</p> +<pre class="literal-block"> +Always test your application with characters above 127 and above 255 +wherever possible. +</pre> +<p>If you fail to do this you might think your application is working fine, but as +soon as your users do put in non-ASCII characters you will have problems. +Using arabic is always a good test and www.google.ae is a good source of sample +text.</p> +<p>The third rule is:</p> +<pre class="literal-block"> +Always do any checking of a string for illegal characters once it's in the +form that will be used or stored, otherwise the illegal characters might be +disguised. +</pre> +<p>For example, let's say you have a content management system that takes a +Unicode filename, and you want to disallow paths with a '/' character. You +might write this code:</p> +<textarea name="code" class="python"> +def read_file(filename, encoding): + if '/' in filename: + raise ValueError("'/' not allowed in filenames") + unicode_name = filename.decode(encoding) + f = open(unicode_name, 'r') + # ... return contents of file ... +</textarea><p>This is INCORRECT. If an attacker could specify the 'base64' encoding, they +could pass <tt class="docutils literal"><span class="pre">L2V0Yy9wYXNzd2Q=</span></tt> which is the base-64 encoded form of the string +<tt class="docutils literal"><span class="pre">'/etc/passwd'</span></tt> which is a file you clearly don't want an attacker to get +hold of. The above code looks for <tt class="docutils literal"><span class="pre">/</span></tt> characters in the encoded form and +misses the dangerous character in the resulting decoded form.</p> +<p>Those are the three basic rules so now we will look at some of the places you +might want to perform Unicode decoding in a Pylons application.</p> +<div class="section"> +<h2><a href="#id8" id="request-parameters" name="request-parameters" class="toc-backref">2.1 Request Parameters</a></h2> +<p>Currently the Pylons input values come from <tt class="docutils literal"><span class="pre">request.params</span></tt> but these are +not decoded to Unicode by default because not all input should be assumed to be +Unicode data.</p> +<p>If you would like However you can use the two functions below:</p> +<textarea name="code" class="python"> +def decode_multi_dict(md, encoding="UTF-8", errors="strict"): + """Given a MultiDict, decode all its parts from the given encoding. + + This modifies the MultiDict in place. + + encoding, strict + These are passed to the decode function. + + """ + items = md.items() + md.clear() + for (k, v) in items: + md.add(k.decode(encoding, errors), + v.decode(encoding, errors)) + + +def decode_request(request, encoding="UTF-8", errors="strict"): + """Given a request object, decode GET and POST in place. + + This implicitly takes care of params as well. + + """ + decode_multi_dict(request.GET, encoding, errors) + decode_multi_dict(request.POST, encoding, errors) +</textarea><p>These can then be used as follows:</p> +<textarea name="code" class="python"> +unicode_params = decode_request(request.params) +</textarea><p>This code is discussed in <a href="http://pylonshq.com/project/pylonshq/ticket/135" class="reference">ticket 135</a> but shouldn't be used with +file uploads since these shouldn't ordinarily be decoded to Unicode.</p> +</div> +<div class="section"> +<h2><a href="#id9" id="templating" name="templating" class="toc-backref">2.2 Templating</a></h2> +<p>Pylons uses Myghty as its default templating language and Myghty 1.1 and above +fully support Unicode. The Myghty documentation explains how to use Unicode and +you at <a href="http://www.myghty.org/docs/unicode.myt" class="reference">http://www.myghty.org/docs/unicode.myt</a> but the important idea is that +you can Unicode literals pretty much anywhere you can use normal 8-bit strings +including in <tt class="docutils literal"><span class="pre">m.write()</span></tt> and <tt class="docutils literal"><span class="pre">m.comp()</span></tt>. You can also pass Unicode data to +Pylons' <tt class="docutils literal"><span class="pre">render_response()</span></tt> and <tt class="docutils literal"><span class="pre">Response()</span></tt> callables.</p> +<p>Any Unicode data output by Myghty is automatically decoded to whichever +encoding you have chosen. The default is UTF-8 but you can choose which +encoding to use by editing your project's <tt class="docutils literal"><span class="pre">config/environment.py</span></tt> file and +adding an option like this:</p> +<textarea name="code" class="python"> +# Add your own Myghty config options here, note that all config options will override +# any Pylons config options + +myghty['output_encoding'] = 'UTF-8' +</textarea><p>replacing <tt class="docutils literal"><span class="pre">UTF-8</span></tt> with the encoding you wish to use.</p> +<p>If you need to disable Unicode support altogether you can set this:</p> +<textarea name="code" class="python"> +myghty['disable_unicode'] = True +</textarea><p>but again, you would have to have a good reason to want to do this.</p> +</div> +<div class="section"> +<h2><a href="#id10" id="output-encoding" name="output-encoding" class="toc-backref">2.3 Output Encoding</a></h2> +<p>Web pages should be generated with a specific encoding, most likely UTF-8. At +the very least, that means you should specify the following in the <tt class="docutils literal"><span class="pre"><head></span></tt> +section:</p> +<pre class="literal-block"> +<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> +</pre> +<p>You should also set the charset in the <tt class="docutils literal"><span class="pre">Content-Type</span></tt> header:</p> +<textarea name="code" class="python"> +respones = Response(...) +response.headers['Content-type'] = 'text/html; charset=utf-8' +</textarea><p>If you specify that your output is UTF-8, generally the web browser will +give you UTF-8. If you want the browser to submit data using a different +character set, you can set the encoding by adding the <tt class="docutils literal"><span class="pre">accept-encoding</span></tt> +tag to your form. Here is an example:</p> +<pre class="literal-block"> +<form accept-encoding="US-ASCII" ...> +</pre> +<p>However, be forewarned that if the user tries to give you non-ASCII +text, then:</p> +<blockquote> +<ul class="simple"> +<li>Firefox will translate the non-ASCII text into HTML entities.</li> +<li>IE will ignore your suggested encoding and give you UTF-8 anyway.</li> +</ul> +</blockquote> +<p>The lesson to be learned is that if you output UTF-8, you had better be +prepared to accept UTF-8 by decoding the data in <tt class="docutils literal"><span class="pre">request.params</span></tt> as +described in the section above entitled "Request Parameters".</p> +<p>Another technique which is sometimes used to determine the character set is to +use an algorithm to analyse the input and guess the encoding based on +probabilities.</p> +<p>For instance, if you get a file, and you don't know what encoding it is encoded +in, you can often rename the file with a .txt extension and then try to open it +in Firefox. Then you can use the "View->Character Encoding" menu to try to +auto-detect the encoding.</p> +</div> +<div class="section"> +<h2><a href="#id11" id="databases" name="databases" class="toc-backref">2.4 Databases</a></h2> +<p>Your database driver should automatically convert from Unicode objects to a +particular charset when writing and back again when reading. Again it is normal +to use UTF-8 which is well supported.</p> +<p>You should check your database's documentation for information on how it handles +Unicode.</p> +<p>For example MySQL's Unicode documentation is here +<a href="http://dev.mysql.com/doc/refman/5.0/en/charset-unicode.html" class="reference">http://dev.mysql.com/doc/refman/5.0/en/charset-unicode.html</a></p> +<p>Also note that you need to consider both the encoding of the database +and the encoding used by the database driver.</p> +<p>If you're using MySQL together with SQLAlchemy, see the following, as +there are some bugs in MySQLdb that you'll need to work around:</p> +<p><a href="http://www.mail-archive.com/sqlalchemy@googlegroups.com/msg00366.html" class="reference">http://www.mail-archive.com/sqlalchemy@googlegroups.com/msg00366.html</a></p> +</div> +</div> +<div class="section"> +<h1><a href="#id12" id="internationalization-and-localization" name="internationalization-and-localization" class="toc-backref">3 Internationalization and Localization</a></h1> +<p>By now you should have a good idea of what Unicode is, how to use it in Python +and which areas of you application need to pay specific attention to decoding and +encoding Unicode data.</p> +<p>This final section will look at the issue of making your application work with +multiple languages.</p> +<div class="section"> +<h2><a href="#id13" id="getting-started" name="getting-started" class="toc-backref">3.1 Getting Started</a></h2> +<p>Everywhere in your code where you want strings to be available in different +languages you wrap them in the <tt class="docutils literal"><span class="pre">_()</span></tt> function. There +are also a number of other translation functions which are documented in the API reference at +<a href="http://pylonshq.com/docs/module-pylons.i18n.translation.html" class="reference">http://pylonshq.com/docs/module-pylons.i18n.translation.html</a></p> +<div class="note"> +<p class="first admonition-title">Note</p> +<p class="last">The <tt class="docutils literal"><span class="pre">_()</span></tt> function is a reference to the <tt class="docutils literal"><span class="pre">ugettext()</span></tt> function. +<tt class="docutils literal"><span class="pre">_()</span></tt> is a convention for marking text to be translated and saves on keystrokes. +<tt class="docutils literal"><span class="pre">ugettext()</span></tt> is the Unicode version of <tt class="docutils literal"><span class="pre">gettext()</span></tt>.</p> +</div> +<p>In our example we want the string <tt class="docutils literal"><span class="pre">'Hello'</span></tt> to appear in three different +languages: English, French and Spanish. We also want to display the word +<tt class="docutils literal"><span class="pre">'Hello'</span></tt> in the default language. We'll then go on to use some pural words +too.</p> +<p>Lets call our project <tt class="docutils literal"><span class="pre">translate_demo</span></tt>:</p> +<pre class="literal-block"> +paster create --template=pylons translate_demo +</pre> +<p>Now lets add a friendly controller that says hello:</p> +<pre class="literal-block"> +cd translate_demo +paster controller hello +</pre> +<p>Edit <tt class="docutils literal"><span class="pre">controllers/hello.py</span></tt> controller to look like this making use of the +<tt class="docutils literal"><span class="pre">_()</span></tt> function everywhere where the string <tt class="docutils literal"><span class="pre">Hello</span></tt> appears:</p> +<textarea name="code" class="python"> +from translate_demo.lib.base import * + +class HelloController(BaseController): + + def index(self): + resp = Response() + resp.write('Default: %s<br />' % _('Hello')) + for lang in ['fr','en','es']: + h.set_lang(lang) + resp.write("%s: %s<br />" % (h.get_lang(), _('Hello'))) + return resp +</textarea><p>When writing your controllers it is important not to piece sentences together manually because +certain languages might need to invert the grammars. As an example this is bad:</p> +<textarea name="code" class="python"> +# BAD! +msg = _("He told her ") +msg += _("not to go outside.") +</textarea><p>but this is perfectly acceptable:</p> +<textarea name="code" class="python"> +# GOOD +msg = _("He told her not to go outside") +</textarea><p>The controller has now been internationalized but it will raise a <tt class="docutils literal"><span class="pre">LanguageError</span></tt> +until we have specified the alternative languages.</p> +<p>Pylons uses <a href="http://www.gnu.org/software/gettext/" class="reference">GNU gettext</a> to handle +internationalization. GNU gettext use three types of files in the +translation framework.</p> +<p>POT (Portable Object Template) files</p> +<blockquote> +The first step in the localization process. A program is used to search through +your project's source code and pick out every string passed to one of the +translation functions, such as <tt class="docutils literal"><span class="pre">_()</span></tt>. This list is put together in a +specially-formatted template file that will form the basis of all +translations. This is the <tt class="docutils literal"><span class="pre">.pot</span></tt> file.</blockquote> +<p>PO (Portable Object) files</p> +<blockquote> +The second step in the localization process. Using the POT file as a template, +the list of messages are translated and saved as a <tt class="docutils literal"><span class="pre">.po</span></tt> file.</blockquote> +<p>MO (Machine Object) files</p> +<blockquote> +The final step in the localization process. The PO file is run through a +program that turns it into an optimized machine-readable binary file, which is +the <tt class="docutils literal"><span class="pre">.mo</span></tt> file. Compiling the translations to machine code makes the +localized program much faster in retrieving the translations while it is +running.</blockquote> +<p>Versions of Pylons prior to 0.9.4 came with a setuptools extension to help with +the extraction of strings and production of a <tt class="docutils literal"><span class="pre">.mo</span></tt> file. The implementation +did not support Unicode nor the ungettext function and was therfore dropped in +Python 0.9.4.</p> +<p>You will therefore need to use an external program to perform these tasks. You +may use whichever you prefer but <tt class="docutils literal"><span class="pre">xgettext</span></tt> is highly recommended. Python's +gettext utility has some bugs, especially regarding plurals.</p> +<p>Here are some compatible tools and projects:</p> +<p>The Rosetta Project (<a href="https://launchpad.ubuntu.com/rosetta/" class="reference">https://launchpad.ubuntu.com/rosetta/</a>)</p> +<blockquote> +The Ubuntu Linux project has a web site that allows you to translate +messages without even looking at a PO or POT file, and export directly to a MO.</blockquote> +<p>poEdit (<a href="http://www.poedit.org/" class="reference">http://www.poedit.org/</a>)</p> +<blockquote> +An open source program for Windows and UNIX/Linux which provides an easy-to-use +GUI for editing PO files and generating MO files.</blockquote> +<p>KBabel (<a href="http://i18n.kde.org/tools/kbabel/" class="reference">http://i18n.kde.org/tools/kbabel/</a>)</p> +<blockquote> +Another open source PO editing program for KDE.</blockquote> +<p>GNU Gettext (<a href="http://www.gnu.org/software/gettext/" class="reference">http://www.gnu.org/software/gettext/</a>)</p> +<blockquote> +The official Gettext tools package contains command-line tools for creating +POTs, manipulating POs, and generating MOs. For those comfortable with a +command shell.</blockquote> +<p>As an example we will quickly discuss the use of poEdit which is cross platform +and has a GUI which makes it easier to get started with.</p> +<p>To use poEdit with the <tt class="docutils literal"><span class="pre">translate_demo</span></tt> you would do the following:</p> +<ol class="arabic simple"> +<li>Download and install poEdit.</li> +<li>A dialog pops up. Fill in <em>all</em> the fields you can on the <tt class="docutils literal"><span class="pre">Project</span> <span class="pre">Info</span></tt> tab, enter the path to your project on the <tt class="docutils literal"><span class="pre">Paths</span></tt> tab (ie <tt class="docutils literal"><span class="pre">/path/to/translate_demo</span></tt>) and enter the following keywords on separate lines on the <tt class="docutils literal"><span class="pre">keywords</span></tt> tab: <tt class="docutils literal"><span class="pre">_</span></tt>, <tt class="docutils literal"><span class="pre">N_</span></tt>, <tt class="docutils literal"><span class="pre">ugettext</span></tt>, <tt class="docutils literal"><span class="pre">gettext</span></tt>, <tt class="docutils literal"><span class="pre">ngettext</span></tt>, <tt class="docutils literal"><span class="pre">ungettext</span></tt>.</li> +<li>Click OK</li> +</ol> +<p>poEdit will search your source tree and find all the strings you have marked +up. You can then enter your translations in whatever charset you chose in +the project info tab. UTF-8 is a good choice.</p> +<p>Finally, after entering your translations you then save the catalog and rename +the <tt class="docutils literal"><span class="pre">.mo</span></tt> file produced to <tt class="docutils literal"><span class="pre">translate_demo.mo</span></tt> and put it in the +<tt class="docutils literal"><span class="pre">translate_demo/i18n/es/LC_MESSAGES</span></tt> directory or whatever is appropriate for +your translation.</p> +<p>You will need to repeat the process of creating a <tt class="docutils literal"><span class="pre">.mo</span></tt> file for the <tt class="docutils literal"><span class="pre">fr</span></tt>, +<tt class="docutils literal"><span class="pre">es</span></tt> and <tt class="docutils literal"><span class="pre">en</span></tt> translations.</p> +<p>The relevant lines from <tt class="docutils literal"><span class="pre">i18n/en/LC_MESSAGES/translate_demo.po</span></tt> look like this:</p> +<pre class="literal-block"> +#: translate_demo\controllers\hello.py:6 translate_demo\controllers\hello.py:9 +msgid "Hello" +msgstr "Hello" +</pre> +<p>The relevant lines from <tt class="docutils literal"><span class="pre">i18n/es/LC_MESSAGES/translate_demo.po</span></tt> look like this:</p> +<pre class="literal-block"> +#: translate_demo\controllers\hello.py:6 translate_demo\controllers\hello.py:9 +msgid "Hello" +msgstr "°Hola!" +</pre> +<p>The relevant lines from <tt class="docutils literal"><span class="pre">i18n/fr/LC_MESSAGES/translate_demo.po</span></tt> look like this:</p> +<pre class="literal-block"> +#: translate_demo\controllers\hello.py:6 translate_demo\controllers\hello.py:9 +msgid "Hello" +msgstr "Bonjour" +</pre> +<p>Whichever tools you use you should end up with an <tt class="docutils literal"><span class="pre">i18n</span></tt> directory that looks +like this when you have finished:</p> +<pre class="literal-block"> +i18n/en/LC_MESSAGES/translate_demo.po +i18n/en/LC_MESSAGES/translate_demo.mo +i18n/es/LC_MESSAGES/translate_demo.po +i18n/es/LC_MESSAGES/translate_demo.mo +i18n/fr/LC_MESSAGES/translate_demo.po +i18n/fr/LC_MESSAGES/translate_demo.mo +</pre> +</div> +<div class="section"> +<h2><a href="#id14" id="testing-the-application" name="testing-the-application" class="toc-backref">3.2 Testing the Application</a></h2> +<p>Start the server with the following command:</p> +<pre class="literal-block"> +paster serve --reload development.ini +</pre> +<p>Test your controller by visiting <a href="http://localhost:5000/hello" class="reference">http://localhost:5000/hello</a>. You should see +the following output:</p> +<pre class="literal-block"> +Default: Hello +fr: Bonjour +en: Hello +es: °Hola! +</pre> +<p>You can now set the language used in a controller on the fly.</p> +<p>For example this could be used to allow a user to set which language they +wanted your application to work in. You could save the value to the session +object:</p> +<textarea name="code" class="python"> +session['lang'] = 'en' +</textarea><p>then on each controller call the language to be used could be read from the +session and set in your controller's <tt class="docutils literal"><span class="pre">__before__()</span></tt> method so that the pages +remained in the same language that was previously set:</p> +<textarea name="code" class="python"> +def __before__(self, action): + if session.has_key('lang'): + h.set_lang(session['lang']) +</textarea><p>One more useful thing to be able to do is to set the default language to be +used in the configuration file. Just add a <tt class="docutils literal"><span class="pre">lang</span></tt> variable together with the +code of the language you wanted to use in your <tt class="docutils literal"><span class="pre">development.ini</span></tt> file. For +example to set the default language to Spanish you would add <tt class="docutils literal"><span class="pre">lang</span> <span class="pre">=</span> <span class="pre">es</span></tt> to +your <tt class="docutils literal"><span class="pre">development.ini</span></tt>. The relevant part from the file might look something +like this:</p> +<textarea name="code" class="pasteini"> +[app:main] +use = egg:translate_demo +lang = es +</textarea><p>If you are running the server with the <tt class="docutils literal"><span class="pre">--reload</span></tt> option the server will +automatically restart if you change the <tt class="docutils literal"><span class="pre">development.ini</span></tt> file. Otherwise +restart the server manually and the output would this time be as follows:</p> +<pre class="literal-block"> +Default: °Hola! +fr: Bonjour +en: Hello +es: °Hola! +</pre> +</div> +<div class="section"> +<h2><a href="#id15" id="missing-translations" name="missing-translations" class="toc-backref">3.3 Missing Translations</a></h2> +<p>If your code calls <tt class="docutils literal"><span class="pre">_()</span></tt> with a string that doesn't exist in your language +catalogue, the string passed to <tt class="docutils literal"><span class="pre">_()</span></tt> is returned instead.</p> +<p>Modify the last line of the hello controller to look like this:</p> +<textarea name="code" class="python"> +resp.write("%s: %s %s<br />" % (h.get_lang(), _('Hello'), _('World!'))) +</textarea><div class="warning"> +<p class="first admonition-title">Warning</p> +<p class="last">Of course, in real life breaking up sentences in this way is very dangerous because some +grammars might require the order of the words to be different.</p> +</div> +<p>If you run the example again the output will be:</p> +<pre class="literal-block"> +Default: °Hola! +fr: Bonjour World! +en: Hello World! +es: °Hola! World! +</pre> +<p>This is because we never provided a translation for the string <tt class="docutils literal"><span class="pre">'World!'</span></tt> so +the string itself is used.</p> +</div> +<div class="section"> +<h2><a href="#id16" id="translations-within-templates" name="translations-within-templates" class="toc-backref">3.4 Translations Within Templates</a></h2> +<p>You can also use the <tt class="docutils literal"><span class="pre">_()</span></tt> function within templates in exactly the same way +you do in code. For example:</p> +<textarea name="code" class="html"> +<% _('Hello') %> +</textarea><p>would produce the string <tt class="docutils literal"><span class="pre">'Hello'</span></tt> in the language you had set.</p> +<p>There is one complication though. gettext's <tt class="docutils literal"><span class="pre">xgettext</span></tt> command can only extract +strings that need translating from Python code in <tt class="docutils literal"><span class="pre">.py</span></tt> files. This means +that if you write <tt class="docutils literal"><span class="pre">_('Hello')</span></tt> in a template such as a Myghty template, +<tt class="docutils literal"><span class="pre">xgettext</span></tt> will not find the string <tt class="docutils literal"><span class="pre">'Hello'</span></tt> as one which needs +translating.</p> +<p>As long as <tt class="docutils literal"><span class="pre">xgettext</span></tt> can find a string marked for translation with one +of the translation functions and defined in Python code in your project +filesystem it will manage the translation when the same string is defined in a +Myghty template and marked for translation.</p> +<p>One solution to ensure all strings are picked up for translation is to create a +file in <tt class="docutils literal"><span class="pre">lib</span></tt> with an appropriate filename, <tt class="docutils literal"><span class="pre">i18n.py</span></tt> for example, and then +add a list of all the strings which appear in your templates so that your +translation tool can then extract the strings in <tt class="docutils literal"><span class="pre">lib/i18n.py</span></tt> for +translation and use the translated versions in your templates as well.</p> +<p>For example if you wanted to ensure the translated string <tt class="docutils literal"><span class="pre">'Good</span> <span class="pre">Morning'</span></tt> +was available in all templates you could create a <tt class="docutils literal"><span class="pre">lib/i18n.py</span></tt> file that +looked something like this:</p> +<textarea name="code" class="python"> +from base import _ +_('Good Morning') +</textarea><p>This approach requires quite a lot of work and is rather fragile. The best +solution if you are using a templating system such as Myghty or Cheetah which +uses compiled Python files is to use a Makefile to ensure that every template +is compiled to Python before running the extraction tool to make sure that +every template is scanned.</p> +<p>Of course, if your cache directory is in the default location or elsewhere +within your project's filesystem, you will probably find that all templates +have been compiled as Python files during the course of the development process. +This means that your tool's extraction command will successfully pick up +strings to translate from the cached files anyway.</p> +<p>You may also find that your extraction tool is capable of extracting the +strings correctly from the template anyway, particularly if the templating +langauge is quite similar to Python. It is best not to rely on this though.</p> +</div> +<div class="section"> +<h2><a href="#id17" id="producing-a-python-egg" name="producing-a-python-egg" class="toc-backref">3.5 Producing a Python Egg</a></h2> +<p>Finally you can produce an egg of your project which includes the translation +files like this:</p> +<pre class="literal-block"> +python setup.py bdist_egg +</pre> +<p>The <tt class="docutils literal"><span class="pre">setup.py</span></tt> automatically includes the <tt class="docutils literal"><span class="pre">.mo</span></tt> language catalogs your +application needs so that your application can be distributed as an egg. This +is done with the following line in your <tt class="docutils literal"><span class="pre">setup.py</span></tt> file:</p> +<pre class="literal-block"> +package_data={'translate_demo': ['i18n/*/LC_MESSAGES/*.mo']}, +</pre> +<p>Internationalization support is zip safe so your application can be run +directly from the egg without the need for <tt class="docutils literal"><span class="pre">easy_install</span></tt> to extract it.</p> +</div> +<div class="section"> +<h2><a href="#id18" id="plural-forms" name="plural-forms" class="toc-backref">3.6 Plural Forms</a></h2> +<p>Pylons also defines <tt class="docutils literal"><span class="pre">ungettext()</span></tt> and <tt class="docutils literal"><span class="pre">ngettext()</span></tt> functions which can be imported +from <tt class="docutils literal"><span class="pre">pylons.i18n</span></tt>. They are designed for internationalizing plural words and can be +used as follows:</p> +<textarea name="code" class="python"> +from pylons.i18n import ungettext + +ungettext( + 'There is %(num)d file here', + 'There are %(num)d files here', + n +) % {'num': n} +</textarea><p>If you wish to use plural forms in your application you need to add the appropriate +headers to the <tt class="docutils literal"><span class="pre">.po</span></tt> files for the language you are using. You can read more about +this at <a href="http://www.gnu.org/software/gettext/manual/html_chapter/gettext_10.html#SEC150" class="reference">http://www.gnu.org/software/gettext/manual/html_chapter/gettext_10.html#SEC150</a></p> +<p>One thing to keep in mind is that other languages don't have the same +plural forms as English. While English only has 2 pulral forms, singular and +plural, Slovenian has 4! That means that you must use gettext's +support for pluralization if you hope to get pluralization right. +Specifically, the following will not work:</p> +<textarea name="code" class="python"> +# BAD! + if n == 1: + msg = _("There was no dog.") + else: + msg = _("There were no dogs.") +</textarea></div> +</div> +<div class="section"> +<h1><a href="#id19" id="summary" name="summary" class="toc-backref">4 Summary</a></h1> +<p>Hopefully you now understand the history of Unicode, how to use it in Python +and where to apply Unicode encoding and decoding in a Pylons application. You +should also be able to use Unicode in your web app remembering the basic rule to +use UTF-8 to talk to the world, do the encode and decode at the edge of your +application.</p> +<p>You should also be able to internationalize and then localize your application +using Pylons' support for GNU gettext.</p> +</div> +<div class="section"> +<h1><a href="#id20" id="further-reading" name="further-reading" class="toc-backref">5 Further Reading</a></h1> +<p>This information is based partly on the following articles which can be +consulted for further information.:</p> +<p><a href="http://www.joelonsoftware.com/articles/Unicode.html" class="reference">http://www.joelonsoftware.com/articles/Unicode.html</a></p> +<p><a href="http://www.amk.ca/python/howto/unicode" class="reference">http://www.amk.ca/python/howto/unicode</a></p> +<p><a href="http://en.wikipedia.org/wiki/Internationalization" class="reference">http://en.wikipedia.org/wiki/Internationalization</a></p> +<p>Please feel free to report any mistakes to the Pylons mailing list or to the +author. Any corrections or clarifications would be gratefully received.</p> +</div> + +</div>
\ No newline at end of file diff --git a/test/templates/modtest.html b/test/templates/modtest.html new file mode 100644 index 0000000..a8a9406 --- /dev/null +++ b/test/templates/modtest.html @@ -0,0 +1 @@ +this is a test
\ No newline at end of file diff --git a/test/templates/othersubdir/foo.html b/test/templates/othersubdir/foo.html new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/templates/othersubdir/foo.html diff --git a/test/templates/read_unicode_py3k.html b/test/templates/read_unicode_py3k.html new file mode 100644 index 0000000..c94399e --- /dev/null +++ b/test/templates/read_unicode_py3k.html @@ -0,0 +1,10 @@ +<% +try: + file_content = open(path, encoding='utf-8', errors='ignore') +except: + raise "Should never execute here" +doc_content = ''.join(file_content.readlines()) +file_content.close() +%> + +${bytes(doc_content, encoding='utf-8')} diff --git a/test/templates/runtimeerr_py3k.html b/test/templates/runtimeerr_py3k.html new file mode 100644 index 0000000..d2569e9 --- /dev/null +++ b/test/templates/runtimeerr_py3k.html @@ -0,0 +1,4 @@ +<% + print(y) + y = 10 +%>
\ No newline at end of file diff --git a/test/templates/subdir/foo/modtest.html.py b/test/templates/subdir/foo/modtest.html.py new file mode 100644 index 0000000..9df72e0 --- /dev/null +++ b/test/templates/subdir/foo/modtest.html.py @@ -0,0 +1,27 @@ +from mako import cache +from mako import runtime + +UNDEFINED = runtime.UNDEFINED +__M_dict_builtin = dict +__M_locals_builtin = locals +_magic_number = 5 +_modified_time = 1267565427.799504 +_template_filename = ( + "/Users/classic/dev/mako/test/templates/subdir/modtest.html" +) +_template_uri = "/subdir/modtest.html" +_template_cache = cache.Cache(__name__, _modified_time) +_source_encoding = None +_exports = [] + + +def render_body(context, **pageargs): + context.caller_stack._push_frame() + try: + __M_locals = __M_dict_builtin(pageargs=pageargs) + __M_writer = context.writer() + # SOURCE LINE 1 + __M_writer("this is a test") + return "" + finally: + context.caller_stack._pop_frame() diff --git a/test/templates/subdir/incl.html b/test/templates/subdir/incl.html new file mode 100644 index 0000000..6505b7c --- /dev/null +++ b/test/templates/subdir/incl.html @@ -0,0 +1,2 @@ + + this is include 2 diff --git a/test/templates/subdir/index.html b/test/templates/subdir/index.html new file mode 100644 index 0000000..5b878b8 --- /dev/null +++ b/test/templates/subdir/index.html @@ -0,0 +1,3 @@ + + this is sub index + <%include file="incl.html"/> diff --git a/test/templates/subdir/modtest.html b/test/templates/subdir/modtest.html new file mode 100644 index 0000000..a8a9406 --- /dev/null +++ b/test/templates/subdir/modtest.html @@ -0,0 +1 @@ +this is a test
\ No newline at end of file diff --git a/test/templates/unicode.html b/test/templates/unicode.html new file mode 100644 index 0000000..8713f7f --- /dev/null +++ b/test/templates/unicode.html @@ -0,0 +1,2 @@ +## -*- coding: utf-8 -*- +Alors vous imaginez ma surprise, au lever du jour, quand une drôle de petite voix m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! »
\ No newline at end of file diff --git a/test/templates/unicode_arguments_py3k.html b/test/templates/unicode_arguments_py3k.html new file mode 100644 index 0000000..871517b --- /dev/null +++ b/test/templates/unicode_arguments_py3k.html @@ -0,0 +1,9 @@ + +<%def name="my_def(x)"> + x is: ${x} +</%def> + +${my_def('drôle de petite voix m’a réveillé')} +<%self:my_def x='drôle de petite voix m’a réveillé'/> +<%self:my_def x="${'drôle de petite voix m’a réveillé'}"/> +<%call expr="my_def('drôle de petite voix m’a réveillé')"/> diff --git a/test/templates/unicode_code_py3k.html b/test/templates/unicode_code_py3k.html new file mode 100644 index 0000000..8835b25 --- /dev/null +++ b/test/templates/unicode_code_py3k.html @@ -0,0 +1,7 @@ +## -*- coding: utf-8 -*- +<% + x = "drôle de petite voix m’a réveillé." +%> +% if x=="drôle de petite voix m’a réveillé.": + hi, ${x} +% endif diff --git a/test/templates/unicode_expr_py3k.html b/test/templates/unicode_expr_py3k.html new file mode 100644 index 0000000..f9b292d --- /dev/null +++ b/test/templates/unicode_expr_py3k.html @@ -0,0 +1,2 @@ +## -*- coding: utf-8 -*- +${"Alors vous imaginez ma surprise, au lever du jour, quand une drôle de petite voix m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! »"} diff --git a/test/templates/unicode_runtime_error.html b/test/templates/unicode_runtime_error.html new file mode 100644 index 0000000..dda7f62 --- /dev/null +++ b/test/templates/unicode_runtime_error.html @@ -0,0 +1,2 @@ +## -*- coding: utf-8 -*- +<% x = 'Alors vous imaginez ma surprise, au lever du jour, quand une drôle de petite voix m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! »' + int(5/0) %>
\ No newline at end of file diff --git a/test/templates/unicode_syntax_error.html b/test/templates/unicode_syntax_error.html new file mode 100644 index 0000000..aa53025 --- /dev/null +++ b/test/templates/unicode_syntax_error.html @@ -0,0 +1,2 @@ +## -*- coding: utf-8 -*- +<% x = 'Alors vous imaginez ma surprise, au lever du jour, quand une drôle de petite voix m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! » %>
\ No newline at end of file diff --git a/test/test_ast.py b/test/test_ast.py new file mode 100644 index 0000000..6b3a3e2 --- /dev/null +++ b/test/test_ast.py @@ -0,0 +1,350 @@ +from mako import ast +from mako import exceptions +from mako import pyparser +from mako.testing.assertions import assert_raises +from mako.testing.assertions import eq_ + +exception_kwargs = {"source": "", "lineno": 0, "pos": 0, "filename": ""} + + +class AstParseTest: + def test_locate_identifiers(self): + """test the location of identifiers in a python code string""" + code = """ +a = 10 +b = 5 +c = x * 5 + a + b + q +(g,h,i) = (1,2,3) +[u,k,j] = [4,5,6] +foo.hoho.lala.bar = 7 + gah.blah + u + blah +for lar in (1,2,3): + gh = 5 + x = 12 +("hello world, ", a, b) +("Another expr", c) +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_( + parsed.declared_identifiers, + {"a", "b", "c", "g", "h", "i", "u", "k", "j", "gh", "lar", "x"}, + ) + eq_( + parsed.undeclared_identifiers, + {"x", "q", "foo", "gah", "blah"}, + ) + + parsed = ast.PythonCode("x + 5 * (y-z)", **exception_kwargs) + assert parsed.undeclared_identifiers == {"x", "y", "z"} + assert parsed.declared_identifiers == set() + + def test_locate_identifiers_2(self): + code = """ +import foobar +from lala import hoho, yaya +import bleep as foo +result = [] +data = get_data() +for x in data: + result.append(x+7) +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.undeclared_identifiers, {"get_data"}) + eq_( + parsed.declared_identifiers, + {"result", "data", "x", "hoho", "foobar", "foo", "yaya"}, + ) + + def test_locate_identifiers_3(self): + """test that combination assignment/expressions + of the same identifier log the ident as 'undeclared'""" + code = """ +x = x + 5 +for y in range(1, y): + ("hi",) +[z for z in range(1, z)] +(q for q in range (1, q)) +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.undeclared_identifiers, {"x", "y", "z", "q", "range"}) + + def test_locate_identifiers_4(self): + code = """ +x = 5 +(y, ) +def mydef(mydefarg): + print("mda is", mydefarg) +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.undeclared_identifiers, {"y"}) + eq_(parsed.declared_identifiers, {"mydef", "x"}) + + def test_locate_identifiers_5(self): + code = """ +try: + print(x) +except: + print(y) +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.undeclared_identifiers, {"x", "y"}) + + def test_locate_identifiers_6(self): + code = """ +def foo(): + return bar() +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.undeclared_identifiers, {"bar"}) + + code = """ +def lala(x, y): + return x, y, z +print(x) +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.undeclared_identifiers, {"z", "x"}) + eq_(parsed.declared_identifiers, {"lala"}) + + code = """ +def lala(x, y): + def hoho(): + def bar(): + z = 7 +print(z) +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.undeclared_identifiers, {"z"}) + eq_(parsed.declared_identifiers, {"lala"}) + + def test_locate_identifiers_7(self): + code = """ +import foo.bar +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.declared_identifiers, {"foo"}) + eq_(parsed.undeclared_identifiers, set()) + + def test_locate_identifiers_8(self): + code = """ +class Hi: + foo = 7 + def hoho(self): + x = 5 +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.declared_identifiers, {"Hi"}) + eq_(parsed.undeclared_identifiers, set()) + + def test_locate_identifiers_9(self): + code = """ + ",".join([t for t in ("a", "b", "c")]) +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.declared_identifiers, {"t"}) + eq_(parsed.undeclared_identifiers, {"t"}) + + code = """ + [(val, name) for val, name in x] +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.declared_identifiers, {"val", "name"}) + eq_(parsed.undeclared_identifiers, {"val", "name", "x"}) + + def test_locate_identifiers_10(self): + code = """ +lambda q: q + 5 +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.declared_identifiers, set()) + eq_(parsed.undeclared_identifiers, set()) + + def test_locate_identifiers_11(self): + code = """ +def x(q): + return q + 5 +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.declared_identifiers, {"x"}) + eq_(parsed.undeclared_identifiers, set()) + + def test_locate_identifiers_12(self): + code = """ +def foo(): + s = 1 + def bar(): + t = s +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.declared_identifiers, {"foo"}) + eq_(parsed.undeclared_identifiers, set()) + + def test_locate_identifiers_13(self): + code = """ +def foo(): + class Bat: + pass + Bat +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.declared_identifiers, {"foo"}) + eq_(parsed.undeclared_identifiers, set()) + + def test_locate_identifiers_14(self): + code = """ +def foo(): + class Bat: + pass + Bat + +print(Bat) +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.declared_identifiers, {"foo"}) + eq_(parsed.undeclared_identifiers, {"Bat"}) + + def test_locate_identifiers_16(self): + code = """ +try: + print(x) +except Exception as e: + print(y) +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.undeclared_identifiers, {"x", "y", "Exception"}) + + def test_locate_identifiers_17(self): + code = """ +try: + print(x) +except (Foo, Bar) as e: + print(y) +""" + parsed = ast.PythonCode(code, **exception_kwargs) + eq_(parsed.undeclared_identifiers, {"x", "y", "Foo", "Bar"}) + + def test_no_global_imports(self): + code = """ +from foo import * +import x as bar +""" + assert_raises( + exceptions.CompileException, + ast.PythonCode, + code, + **exception_kwargs, + ) + + def test_python_fragment(self): + parsed = ast.PythonFragment("for x in foo:", **exception_kwargs) + eq_(parsed.declared_identifiers, {"x"}) + eq_(parsed.undeclared_identifiers, {"foo"}) + + parsed = ast.PythonFragment("try:", **exception_kwargs) + + parsed = ast.PythonFragment( + "except MyException as e:", **exception_kwargs + ) + eq_(parsed.declared_identifiers, {"e"}) + eq_(parsed.undeclared_identifiers, {"MyException"}) + + def test_argument_list(self): + parsed = ast.ArgumentList( + "3, 5, 'hi', x+5, " "context.get('lala')", **exception_kwargs + ) + eq_(parsed.undeclared_identifiers, {"x", "context"}) + eq_( + [x for x in parsed.args], + ["3", "5", "'hi'", "(x + 5)", "context.get('lala')"], + ) + + parsed = ast.ArgumentList("h", **exception_kwargs) + eq_(parsed.args, ["h"]) + + def test_function_decl(self): + """test getting the arguments from a function""" + code = "def foo(a, b, c=None, d='hi', e=x, f=y+7):pass" + parsed = ast.FunctionDecl(code, **exception_kwargs) + eq_(parsed.funcname, "foo") + eq_(parsed.argnames, ["a", "b", "c", "d", "e", "f"]) + eq_(parsed.kwargnames, []) + + def test_function_decl_2(self): + """test getting the arguments from a function""" + code = "def foo(a, b, c=None, *args, **kwargs):pass" + parsed = ast.FunctionDecl(code, **exception_kwargs) + eq_(parsed.funcname, "foo") + eq_(parsed.argnames, ["a", "b", "c", "args"]) + eq_(parsed.kwargnames, ["kwargs"]) + + def test_function_decl_3(self): + """test getting the arguments from a fancy py3k function""" + code = "def foo(a, b, *c, d, e, **f):pass" + parsed = ast.FunctionDecl(code, **exception_kwargs) + eq_(parsed.funcname, "foo") + eq_(parsed.argnames, ["a", "b", "c"]) + eq_(parsed.kwargnames, ["d", "e", "f"]) + + def test_expr_generate(self): + """test the round trip of expressions to AST back to python source""" + x = 1 + y = 2 + + class F: + def bar(self, a, b): + return a + b + + def lala(arg): + return "blah" + arg + + local_dict = dict(x=x, y=y, foo=F(), lala=lala) + + code = "str((x+7*y) / foo.bar(5,6)) + lala('ho')" + astnode = pyparser.parse(code) + newcode = pyparser.ExpressionGenerator(astnode).value() + eq_(eval(code, local_dict), eval(newcode, local_dict)) + + a = ["one", "two", "three"] + hoho = {"somevalue": "asdf"} + g = [1, 2, 3, 4, 5] + local_dict = dict(a=a, hoho=hoho, g=g) + code = ( + "a[2] + hoho['somevalue'] + " + "repr(g[3:5]) + repr(g[3:]) + repr(g[:5])" + ) + astnode = pyparser.parse(code) + newcode = pyparser.ExpressionGenerator(astnode).value() + eq_(eval(code, local_dict), eval(newcode, local_dict)) + + local_dict = {"f": lambda: 9, "x": 7} + code = "x+f()" + astnode = pyparser.parse(code) + newcode = pyparser.ExpressionGenerator(astnode).value() + eq_(eval(code, local_dict), eval(newcode, local_dict)) + + for code in [ + "repr({'x':7,'y':18})", + "repr([])", + "repr({})", + "repr([{3:[]}])", + "repr({'x':37*2 + len([6,7,8])})", + "repr([1, 2, {}, {'x':'7'}])", + "repr({'x':-1})", + "repr(((1,2,3), (4,5,6)))", + "repr(1 and 2 and 3 and 4)", + "repr(True and False or 55)", + "repr(lambda x, y: (x + y))", + "repr(lambda *arg, **kw: arg, kw)", + "repr(1 & 2 | 3)", + "repr(3//5)", + "repr(3^5)", + "repr([q.endswith('e') for q in " "['one', 'two', 'three']])", + "repr([x for x in (5,6,7) if x == 6])", + "repr(not False)", + ]: + local_dict = {} + astnode = pyparser.parse(code) + newcode = pyparser.ExpressionGenerator(astnode).value() + if "lambda" in code: + eq_(code, newcode) + else: + eq_(eval(code, local_dict), eval(newcode, local_dict)) diff --git a/test/test_block.py b/test/test_block.py new file mode 100644 index 0000000..be2fbf7 --- /dev/null +++ b/test/test_block.py @@ -0,0 +1,648 @@ +from mako import exceptions +from mako.lookup import TemplateLookup +from mako.template import Template +from mako.testing.assertions import assert_raises_message +from mako.testing.fixtures import TemplateTest +from mako.testing.helpers import result_lines + + +class BlockTest(TemplateTest): + def test_anonymous_block_namespace_raises(self): + assert_raises_message( + exceptions.CompileException, + "Can't put anonymous blocks inside <%namespace>", + Template, + """ + <%namespace name="foo"> + <%block> + block + </%block> + </%namespace> + """, + ) + + def test_anonymous_block_in_call(self): + template = Template( + """ + + <%self:foo x="5"> + <%block> + this is the block x + </%block> + </%self:foo> + + <%def name="foo(x)"> + foo: + ${caller.body()} + </%def> + """ + ) + self._do_test( + template, ["foo:", "this is the block x"], filters=result_lines + ) + + def test_named_block_in_call(self): + assert_raises_message( + exceptions.CompileException, + "Named block 'y' not allowed inside of <%call> tag", + Template, + """ + + <%self:foo x="5"> + <%block name="y"> + this is the block + </%block> + </%self:foo> + + <%def name="foo(x)"> + foo: + ${caller.body()} + ${caller.y()} + </%def> + """, + ) + + def test_name_collision_blocks_toplevel(self): + assert_raises_message( + exceptions.CompileException, + "%def or %block named 'x' already exists in this template", + Template, + """ + <%block name="x"> + block + </%block> + + foob + + <%block name="x"> + block + </%block> + """, + ) + + def test_name_collision_blocks_nested_block(self): + assert_raises_message( + exceptions.CompileException, + "%def or %block named 'x' already exists in this template", + Template, + """ + <%block> + <%block name="x"> + block + </%block> + + foob + + <%block name="x"> + block + </%block> + </%block> + """, + ) + + def test_name_collision_blocks_nested_def(self): + assert_raises_message( + exceptions.CompileException, + "Named block 'x' not allowed inside of def 'foo'", + Template, + """ + <%def name="foo()"> + <%block name="x"> + block + </%block> + + foob + + <%block name="x"> + block + </%block> + </%def> + """, + ) + + def test_name_collision_block_def_toplevel(self): + assert_raises_message( + exceptions.CompileException, + "%def or %block named 'x' already exists in this template", + Template, + """ + <%block name="x"> + block + </%block> + + foob + + <%def name="x()"> + block + </%def> + """, + ) + + def test_name_collision_def_block_toplevel(self): + assert_raises_message( + exceptions.CompileException, + "%def or %block named 'x' already exists in this template", + Template, + """ + <%def name="x()"> + block + </%def> + + foob + + <%block name="x"> + block + </%block> + + """, + ) + + def test_named_block_renders(self): + template = Template( + """ + above + <%block name="header"> + the header + </%block> + below + """ + ) + self._do_test( + template, ["above", "the header", "below"], filters=result_lines + ) + + def test_inherited_block_no_render(self): + l = TemplateLookup() + l.put_string( + "index", + """ + <%inherit file="base"/> + <%block name="header"> + index header + </%block> + """, + ) + l.put_string( + "base", + """ + above + <%block name="header"> + the header + </%block> + + ${next.body()} + below + """, + ) + self._do_test( + l.get_template("index"), + ["above", "index header", "below"], + filters=result_lines, + ) + + def test_no_named_in_def(self): + assert_raises_message( + exceptions.CompileException, + "Named block 'y' not allowed inside of def 'q'", + Template, + """ + <%def name="q()"> + <%block name="y"> + </%block> + </%def> + """, + ) + + def test_inherited_block_nested_both(self): + l = TemplateLookup() + l.put_string( + "index", + """ + <%inherit file="base"/> + <%block name="title"> + index title + </%block> + + <%block name="header"> + index header + ${parent.header()} + </%block> + """, + ) + l.put_string( + "base", + """ + above + <%block name="header"> + base header + <%block name="title"> + the title + </%block> + </%block> + + ${next.body()} + below + """, + ) + self._do_test( + l.get_template("index"), + ["above", "index header", "base header", "index title", "below"], + filters=result_lines, + ) + + def test_inherited_block_nested_inner_only(self): + l = TemplateLookup() + l.put_string( + "index", + """ + <%inherit file="base"/> + <%block name="title"> + index title + </%block> + + """, + ) + l.put_string( + "base", + """ + above + <%block name="header"> + base header + <%block name="title"> + the title + </%block> + </%block> + + ${next.body()} + below + """, + ) + self._do_test( + l.get_template("index"), + ["above", "base header", "index title", "below"], + filters=result_lines, + ) + + def test_noninherited_block_no_render(self): + l = TemplateLookup() + l.put_string( + "index", + """ + <%inherit file="base"/> + <%block name="some_thing"> + some thing + </%block> + """, + ) + l.put_string( + "base", + """ + above + <%block name="header"> + the header + </%block> + + ${next.body()} + below + """, + ) + self._do_test( + l.get_template("index"), + ["above", "the header", "some thing", "below"], + filters=result_lines, + ) + + def test_no_conflict_nested_one(self): + l = TemplateLookup() + l.put_string( + "index", + """ + <%inherit file="base"/> + <%block> + <%block name="header"> + inner header + </%block> + </%block> + """, + ) + l.put_string( + "base", + """ + above + <%block name="header"> + the header + </%block> + + ${next.body()} + below + """, + ) + self._do_test( + l.get_template("index"), + ["above", "inner header", "below"], + filters=result_lines, + ) + + def test_nested_dupe_names_raise(self): + assert_raises_message( + exceptions.CompileException, + "%def or %block named 'header' already exists in this template.", + Template, + """ + <%inherit file="base"/> + <%block name="header"> + <%block name="header"> + inner header + </%block> + </%block> + """, + ) + + def test_two_levels_one(self): + l = TemplateLookup() + l.put_string( + "index", + """ + <%inherit file="middle"/> + <%block name="header"> + index header + </%block> + <%block> + index anon + </%block> + """, + ) + l.put_string( + "middle", + """ + <%inherit file="base"/> + <%block> + middle anon + </%block> + ${next.body()} + """, + ) + l.put_string( + "base", + """ + above + <%block name="header"> + the header + </%block> + + ${next.body()} + below + """, + ) + self._do_test( + l.get_template("index"), + ["above", "index header", "middle anon", "index anon", "below"], + filters=result_lines, + ) + + def test_filter(self): + template = Template( + """ + <%block filter="h"> + <html> + </%block> + """ + ) + self._do_test(template, ["<html>"], filters=result_lines) + + def test_anon_in_named(self): + template = Template( + """ + <%block name="x"> + outer above + <%block> + inner + </%block> + outer below + </%block> + """ + ) + self._test_block_in_block(template) + + def test_named_in_anon(self): + template = Template( + """ + <%block> + outer above + <%block name="x"> + inner + </%block> + outer below + </%block> + """ + ) + self._test_block_in_block(template) + + def test_anon_in_anon(self): + template = Template( + """ + <%block> + outer above + <%block> + inner + </%block> + outer below + </%block> + """ + ) + self._test_block_in_block(template) + + def test_named_in_named(self): + template = Template( + """ + <%block name="x"> + outer above + <%block name="y"> + inner + </%block> + outer below + </%block> + """ + ) + self._test_block_in_block(template) + + def _test_block_in_block(self, template): + self._do_test( + template, + ["outer above", "inner", "outer below"], + filters=result_lines, + ) + + def test_iteration(self): + t = Template( + """ + % for i in (1, 2, 3): + <%block>${i}</%block> + % endfor + """ + ) + self._do_test(t, ["1", "2", "3"], filters=result_lines) + + def test_conditional(self): + t = Template( + """ + % if True: + <%block>true</%block> + % endif + + % if False: + <%block>false</%block> + % endif + """ + ) + self._do_test(t, ["true"], filters=result_lines) + + def test_block_overridden_by_def(self): + l = TemplateLookup() + l.put_string( + "index", + """ + <%inherit file="base"/> + <%def name="header()"> + inner header + </%def> + """, + ) + l.put_string( + "base", + """ + above + <%block name="header"> + the header + </%block> + + ${next.body()} + below + """, + ) + self._do_test( + l.get_template("index"), + ["above", "inner header", "below"], + filters=result_lines, + ) + + def test_def_overridden_by_block(self): + l = TemplateLookup() + l.put_string( + "index", + """ + <%inherit file="base"/> + <%block name="header"> + inner header + </%block> + """, + ) + l.put_string( + "base", + """ + above + ${self.header()} + <%def name="header()"> + the header + </%def> + + ${next.body()} + below + """, + ) + self._do_test( + l.get_template("index"), + ["above", "inner header", "below"], + filters=result_lines, + ) + + def test_block_args(self): + l = TemplateLookup() + l.put_string( + "caller", + """ + + <%include file="callee" args="val1='3', val2='4'"/> + + """, + ) + l.put_string( + "callee", + """ + <%page args="val1, val2"/> + <%block name="foob" args="val1, val2"> + foob, ${val1}, ${val2} + </%block> + """, + ) + self._do_test( + l.get_template("caller"), ["foob, 3, 4"], filters=result_lines + ) + + def test_block_variables_contextual(self): + t = Template( + """ + <%block name="foob" > + foob, ${val1}, ${val2} + </%block> + """ + ) + self._do_test( + t, + ["foob, 3, 4"], + template_args={"val1": 3, "val2": 4}, + filters=result_lines, + ) + + def test_block_args_contextual(self): + t = Template( + """ + <%page args="val1"/> + <%block name="foob" args="val1"> + foob, ${val1}, ${val2} + </%block> + """ + ) + self._do_test( + t, + ["foob, 3, 4"], + template_args={"val1": 3, "val2": 4}, + filters=result_lines, + ) + + def test_block_pageargs_contextual(self): + t = Template( + """ + <%block name="foob"> + foob, ${pageargs['val1']}, ${pageargs['val2']} + </%block> + """ + ) + self._do_test( + t, + ["foob, 3, 4"], + template_args={"val1": 3, "val2": 4}, + filters=result_lines, + ) + + def test_block_pageargs(self): + l = TemplateLookup() + l.put_string( + "caller", + """ + + <%include file="callee" args="val1='3', val2='4'"/> + + """, + ) + l.put_string( + "callee", + """ + <%block name="foob"> + foob, ${pageargs['val1']}, ${pageargs['val2']} + </%block> + """, + ) + self._do_test( + l.get_template("caller"), ["foob, 3, 4"], filters=result_lines + ) diff --git a/test/test_cache.py b/test/test_cache.py new file mode 100644 index 0000000..9e0d559 --- /dev/null +++ b/test/test_cache.py @@ -0,0 +1,687 @@ +import time + +from mako import lookup +from mako.cache import CacheImpl +from mako.cache import register_plugin +from mako.lookup import TemplateLookup +from mako.template import Template +from mako.testing.assertions import eq_ +from mako.testing.config import config +from mako.testing.exclusions import requires_beaker +from mako.testing.exclusions import requires_dogpile_cache +from mako.testing.helpers import result_lines + + +module_base = str(config.module_base) + + +class SimpleBackend: + def __init__(self): + self.cache = {} + + def get(self, key, **kw): + return self.cache[key] + + def invalidate(self, key, **kw): + self.cache.pop(key, None) + + def put(self, key, value, **kw): + self.cache[key] = value + + def get_or_create(self, key, creation_function, **kw): + if key in self.cache: + return self.cache[key] + + self.cache[key] = value = creation_function() + return value + + +class MockCacheImpl(CacheImpl): + realcacheimpl = None + + def __init__(self, cache): + self.cache = cache + + def set_backend(self, cache, backend): + if backend == "simple": + self.realcacheimpl = SimpleBackend() + else: + self.realcacheimpl = cache._load_impl(backend) + + def _setup_kwargs(self, kw): + self.kwargs = kw.copy() + self.kwargs.pop("regions", None) + self.kwargs.pop("manager", None) + if self.kwargs.get("region") != "myregion": + self.kwargs.pop("region", None) + + def get_or_create(self, key, creation_function, **kw): + self.key = key + self._setup_kwargs(kw) + return self.realcacheimpl.get_or_create(key, creation_function, **kw) + + def put(self, key, value, **kw): + self.key = key + self._setup_kwargs(kw) + self.realcacheimpl.put(key, value, **kw) + + def get(self, key, **kw): + self.key = key + self._setup_kwargs(kw) + return self.realcacheimpl.get(key, **kw) + + def invalidate(self, key, **kw): + self.key = key + self._setup_kwargs(kw) + self.realcacheimpl.invalidate(key, **kw) + + +register_plugin("mock", __name__, "MockCacheImpl") + + +class CacheTest: + real_backend = "simple" + + def _install_mock_cache(self, template, implname=None): + template.cache_impl = "mock" + impl = template.cache.impl + impl.set_backend(template.cache, implname or self.real_backend) + return impl + + def test_def(self): + t = Template( + """ + <%! + callcount = [0] + %> + <%def name="foo()" cached="True"> + this is foo + <% + callcount[0] += 1 + %> + </%def> + + ${foo()} + ${foo()} + ${foo()} + callcount: ${callcount} +""" + ) + m = self._install_mock_cache(t) + assert result_lines(t.render()) == [ + "this is foo", + "this is foo", + "this is foo", + "callcount: [1]", + ] + assert m.kwargs == {} + + def test_cache_enable(self): + t = Template( + """ + <%! + callcount = [0] + %> + <%def name="foo()" cached="True"> + <% callcount[0] += 1 %> + </%def> + ${foo()} + ${foo()} + callcount: ${callcount} + """, + cache_enabled=False, + ) + self._install_mock_cache(t) + + eq_(t.render().strip(), "callcount: [2]") + + def test_nested_def(self): + t = Template( + """ + <%! + callcount = [0] + %> + <%def name="foo()"> + <%def name="bar()" cached="True"> + this is foo + <% + callcount[0] += 1 + %> + </%def> + ${bar()} + </%def> + + ${foo()} + ${foo()} + ${foo()} + callcount: ${callcount} +""" + ) + m = self._install_mock_cache(t) + assert result_lines(t.render()) == [ + "this is foo", + "this is foo", + "this is foo", + "callcount: [1]", + ] + assert m.kwargs == {} + + def test_page(self): + t = Template( + """ + <%! + callcount = [0] + %> + <%page cached="True"/> + this is foo + <% + callcount[0] += 1 + %> + callcount: ${callcount} +""" + ) + m = self._install_mock_cache(t) + t.render() + t.render() + assert result_lines(t.render()) == ["this is foo", "callcount: [1]"] + assert m.kwargs == {} + + def test_dynamic_key_with_context(self): + t = Template( + """ + <%block name="foo" cached="True" cache_key="${mykey}"> + some block + </%block> + """ + ) + m = self._install_mock_cache(t) + t.render(mykey="thekey") + t.render(mykey="thekey") + eq_(result_lines(t.render(mykey="thekey")), ["some block"]) + eq_(m.key, "thekey") + + t = Template( + """ + <%def name="foo()" cached="True" cache_key="${mykey}"> + some def + </%def> + ${foo()} + """ + ) + m = self._install_mock_cache(t) + t.render(mykey="thekey") + t.render(mykey="thekey") + eq_(result_lines(t.render(mykey="thekey")), ["some def"]) + eq_(m.key, "thekey") + + def test_dynamic_key_with_funcargs(self): + t = Template( + """ + <%def name="foo(num=5)" cached="True" cache_key="foo_${str(num)}"> + hi + </%def> + + ${foo()} + """ + ) + m = self._install_mock_cache(t) + t.render() + t.render() + assert result_lines(t.render()) == ["hi"] + assert m.key == "foo_5" + + t = Template( + """ + <%def name="foo(*args, **kwargs)" cached="True" + cache_key="foo_${kwargs['bar']}"> + hi + </%def> + + ${foo(1, 2, bar='lala')} + """ + ) + m = self._install_mock_cache(t) + t.render() + assert result_lines(t.render()) == ["hi"] + assert m.key == "foo_lala" + + t = Template( + """ + <%page args="bar='hi'" cache_key="foo_${bar}" cached="True"/> + hi + """ + ) + m = self._install_mock_cache(t) + t.render() + assert result_lines(t.render()) == ["hi"] + assert m.key == "foo_hi" + + def test_dynamic_key_with_imports(self): + lookup = TemplateLookup() + lookup.put_string( + "foo.html", + """ + <%! + callcount = [0] + %> + <%namespace file="ns.html" import="*"/> + <%page cached="True" cache_key="${foo}"/> + this is foo + <% + callcount[0] += 1 + %> + callcount: ${callcount} +""", + ) + lookup.put_string("ns.html", """""") + t = lookup.get_template("foo.html") + m = self._install_mock_cache(t) + t.render(foo="somekey") + t.render(foo="somekey") + assert result_lines(t.render(foo="somekey")) == [ + "this is foo", + "callcount: [1]", + ] + assert m.kwargs == {} + + def test_fileargs_implicit(self): + l = lookup.TemplateLookup(module_directory=module_base) + l.put_string( + "test", + """ + <%! + callcount = [0] + %> + <%def name="foo()" cached="True" cache_type='dbm'> + this is foo + <% + callcount[0] += 1 + %> + </%def> + + ${foo()} + ${foo()} + ${foo()} + callcount: ${callcount} + """, + ) + + m = self._install_mock_cache(l.get_template("test")) + assert result_lines(l.get_template("test").render()) == [ + "this is foo", + "this is foo", + "this is foo", + "callcount: [1]", + ] + eq_(m.kwargs, {"type": "dbm"}) + + def test_fileargs_deftag(self): + t = Template( + """ + <%%! + callcount = [0] + %%> + <%%def name="foo()" cached="True" cache_type='file' cache_dir='%s'> + this is foo + <%% + callcount[0] += 1 + %%> + </%%def> + + ${foo()} + ${foo()} + ${foo()} + callcount: ${callcount} +""" + % module_base + ) + m = self._install_mock_cache(t) + assert result_lines(t.render()) == [ + "this is foo", + "this is foo", + "this is foo", + "callcount: [1]", + ] + assert m.kwargs == {"type": "file", "dir": module_base} + + def test_fileargs_pagetag(self): + t = Template( + """ + <%%page cache_dir='%s' cache_type='dbm'/> + <%%! + callcount = [0] + %%> + <%%def name="foo()" cached="True"> + this is foo + <%% + callcount[0] += 1 + %%> + </%%def> + + ${foo()} + ${foo()} + ${foo()} + callcount: ${callcount} +""" + % module_base + ) + m = self._install_mock_cache(t) + assert result_lines(t.render()) == [ + "this is foo", + "this is foo", + "this is foo", + "callcount: [1]", + ] + eq_(m.kwargs, {"dir": module_base, "type": "dbm"}) + + def test_args_complete(self): + t = Template( + """ + <%%def name="foo()" cached="True" cache_timeout="30" cache_dir="%s" + cache_type="file" cache_key='somekey'> + this is foo + </%%def> + + ${foo()} +""" + % module_base + ) + m = self._install_mock_cache(t) + t.render() + eq_(m.kwargs, {"dir": module_base, "type": "file", "timeout": 30}) + + t2 = Template( + """ + <%%page cached="True" cache_timeout="30" cache_dir="%s" + cache_type="file" cache_key='somekey'/> + hi + """ + % module_base + ) + m = self._install_mock_cache(t2) + t2.render() + eq_(m.kwargs, {"dir": module_base, "type": "file", "timeout": 30}) + + def test_fileargs_lookup(self): + l = lookup.TemplateLookup(cache_dir=module_base, cache_type="file") + l.put_string( + "test", + """ + <%! + callcount = [0] + %> + <%def name="foo()" cached="True"> + this is foo + <% + callcount[0] += 1 + %> + </%def> + + ${foo()} + ${foo()} + ${foo()} + callcount: ${callcount} + """, + ) + + t = l.get_template("test") + m = self._install_mock_cache(t) + assert result_lines(l.get_template("test").render()) == [ + "this is foo", + "this is foo", + "this is foo", + "callcount: [1]", + ] + eq_(m.kwargs, {"dir": module_base, "type": "file"}) + + def test_buffered(self): + t = Template( + """ + <%! + def a(text): + return "this is a " + text.strip() + %> + ${foo()} + ${foo()} + <%def name="foo()" cached="True" buffered="True"> + this is a test + </%def> + """, + buffer_filters=["a"], + ) + self._install_mock_cache(t) + eq_( + result_lines(t.render()), + ["this is a this is a test", "this is a this is a test"], + ) + + def test_load_from_expired(self): + """test that the cache callable can be called safely after the + originating template has completed rendering. + + """ + t = Template( + """ + ${foo()} + <%def name="foo()" cached="True" cache_timeout="1"> + foo + </%def> + """ + ) + self._install_mock_cache(t) + + x1 = t.render() + time.sleep(1.2) + x2 = t.render() + assert x1.strip() == x2.strip() == "foo" + + def test_namespace_access(self): + t = Template( + """ + <%def name="foo(x)" cached="True"> + foo: ${x} + </%def> + + <% + foo(1) + foo(2) + local.cache.invalidate_def('foo') + foo(3) + foo(4) + %> + """ + ) + self._install_mock_cache(t) + eq_(result_lines(t.render()), ["foo: 1", "foo: 1", "foo: 3", "foo: 3"]) + + def test_lookup(self): + l = TemplateLookup(cache_impl="mock") + l.put_string( + "x", + """ + <%page cached="True" /> + ${y} + """, + ) + t = l.get_template("x") + self._install_mock_cache(t) + assert result_lines(t.render(y=5)) == ["5"] + assert result_lines(t.render(y=7)) == ["5"] + assert isinstance(t.cache.impl, MockCacheImpl) + + def test_invalidate(self): + t = Template( + """ + <%%def name="foo()" cached="True"> + foo: ${x} + </%%def> + + <%%def name="bar()" cached="True" cache_type='dbm' cache_dir='%s'> + bar: ${x} + </%%def> + ${foo()} ${bar()} + """ + % module_base + ) + self._install_mock_cache(t) + assert result_lines(t.render(x=1)) == ["foo: 1", "bar: 1"] + assert result_lines(t.render(x=2)) == ["foo: 1", "bar: 1"] + t.cache.invalidate_def("foo") + assert result_lines(t.render(x=3)) == ["foo: 3", "bar: 1"] + t.cache.invalidate_def("bar") + assert result_lines(t.render(x=4)) == ["foo: 3", "bar: 4"] + + t = Template( + """ + <%%page cached="True" cache_type="dbm" cache_dir="%s"/> + + page: ${x} + """ + % module_base + ) + self._install_mock_cache(t) + assert result_lines(t.render(x=1)) == ["page: 1"] + assert result_lines(t.render(x=2)) == ["page: 1"] + t.cache.invalidate_body() + assert result_lines(t.render(x=3)) == ["page: 3"] + assert result_lines(t.render(x=4)) == ["page: 3"] + + def test_custom_args_def(self): + t = Template( + """ + <%def name="foo()" cached="True" cache_region="myregion" + cache_timeout="50" cache_foo="foob"> + </%def> + ${foo()} + """ + ) + m = self._install_mock_cache(t, "simple") + t.render() + eq_(m.kwargs, {"region": "myregion", "timeout": 50, "foo": "foob"}) + + def test_custom_args_block(self): + t = Template( + """ + <%block name="foo" cached="True" cache_region="myregion" + cache_timeout="50" cache_foo="foob"> + </%block> + """ + ) + m = self._install_mock_cache(t, "simple") + t.render() + eq_(m.kwargs, {"region": "myregion", "timeout": 50, "foo": "foob"}) + + def test_custom_args_page(self): + t = Template( + """ + <%page cached="True" cache_region="myregion" + cache_timeout="50" cache_foo="foob"/> + """ + ) + m = self._install_mock_cache(t, "simple") + t.render() + eq_(m.kwargs, {"region": "myregion", "timeout": 50, "foo": "foob"}) + + def test_pass_context(self): + t = Template( + """ + <%page cached="True"/> + """ + ) + m = self._install_mock_cache(t) + t.render() + assert "context" not in m.kwargs + + m.pass_context = True + t.render(x="bar") + assert "context" in m.kwargs + assert m.kwargs["context"].get("x") == "bar" + + +class RealBackendMixin: + def test_cache_uses_current_context(self): + t = Template( + """ + ${foo()} + <%def name="foo()" cached="True" cache_timeout="1"> + foo: ${x} + </%def> + """ + ) + self._install_mock_cache(t) + + x1 = t.render(x=1) + time.sleep(1.2) + x2 = t.render(x=2) + eq_(x1.strip(), "foo: 1") + eq_(x2.strip(), "foo: 2") + + def test_region(self): + t = Template( + """ + <%block name="foo" cached="True" cache_region="short"> + short term ${x} + </%block> + <%block name="bar" cached="True" cache_region="long"> + long term ${x} + </%block> + <%block name="lala"> + none ${x} + </%block> + """ + ) + + self._install_mock_cache(t) + r1 = result_lines(t.render(x=5)) + time.sleep(1.2) + r2 = result_lines(t.render(x=6)) + r3 = result_lines(t.render(x=7)) + eq_(r1, ["short term 5", "long term 5", "none 5"]) + eq_(r2, ["short term 6", "long term 5", "none 6"]) + eq_(r3, ["short term 6", "long term 5", "none 7"]) + + +@requires_beaker +class BeakerCacheTest(RealBackendMixin, CacheTest): + real_backend = "beaker" + + def _install_mock_cache(self, template, implname=None): + template.cache_args["manager"] = self._regions() + return super()._install_mock_cache(template, implname) + + def _regions(self): + import beaker + + return beaker.cache.CacheManager( + cache_regions={ + "short": {"expire": 1, "type": "memory"}, + "long": {"expire": 60, "type": "memory"}, + } + ) + + +@requires_dogpile_cache +class DogpileCacheTest(RealBackendMixin, CacheTest): + real_backend = "dogpile.cache" + + def _install_mock_cache(self, template, implname=None): + template.cache_args["regions"] = self._regions() + template.cache_args.setdefault("region", "short") + return super()._install_mock_cache(template, implname) + + def _regions(self): + from dogpile.cache import make_region + + my_regions = { + "short": make_region().configure( + "dogpile.cache.memory", expiration_time=1 + ), + "long": make_region().configure( + "dogpile.cache.memory", expiration_time=60 + ), + "myregion": make_region().configure( + "dogpile.cache.memory", expiration_time=60 + ), + } + + return my_regions diff --git a/test/test_call.py b/test/test_call.py new file mode 100644 index 0000000..4dea2b3 --- /dev/null +++ b/test/test_call.py @@ -0,0 +1,573 @@ +from mako.template import Template +from mako.testing.assertions import eq_ +from mako.testing.fixtures import TemplateTest +from mako.testing.helpers import flatten_result +from mako.testing.helpers import result_lines + + +class CallTest(TemplateTest): + def test_call(self): + t = Template( + """ + <%def name="foo()"> + hi im foo ${caller.body(y=5)} + </%def> + + <%call expr="foo()" args="y, **kwargs"> + this is the body, y is ${y} + </%call> +""" + ) + assert result_lines(t.render()) == [ + "hi im foo", + "this is the body, y is 5", + ] + + def test_compound_call(self): + t = Template( + """ + + <%def name="bar()"> + this is bar + </%def> + + <%def name="comp1()"> + this comp1 should not be called + </%def> + + <%def name="foo()"> + foo calling comp1: ${caller.comp1(x=5)} + foo calling body: ${caller.body()} + </%def> + + <%call expr="foo()"> + <%def name="comp1(x)"> + this is comp1, ${x} + </%def> + this is the body, ${comp1(6)} + </%call> + ${bar()} + +""" + ) + assert result_lines(t.render()) == [ + "foo calling comp1:", + "this is comp1, 5", + "foo calling body:", + "this is the body,", + "this is comp1, 6", + "this is bar", + ] + + def test_new_syntax(self): + """test foo:bar syntax, including multiline args and expression + eval.""" + + # note the trailing whitespace in the bottom ${} expr, need to strip + # that off < python 2.7 + + t = Template( + """ + <%def name="foo(x, y, q, z)"> + ${x} + ${y} + ${q} + ${",".join("%s->%s" % (a, b) for a, b in z)} + </%def> + + <%self:foo x="this is x" y="${'some ' + 'y'}" q=" + this + is + q" + + z="${[ + (1, 2), + (3, 4), + (5, 6) + ] + + }"/> + """ + ) + + eq_( + result_lines(t.render()), + ["this is x", "some y", "this", "is", "q", "1->2,3->4,5->6"], + ) + + def test_ccall_caller(self): + t = Template( + """ + <%def name="outer_func()"> + OUTER BEGIN + <%call expr="caller.inner_func()"> + INNER CALL + </%call> + OUTER END + </%def> + + <%call expr="outer_func()"> + <%def name="inner_func()"> + INNER BEGIN + ${caller.body()} + INNER END + </%def> + </%call> + + """ + ) + # print t.code + assert result_lines(t.render()) == [ + "OUTER BEGIN", + "INNER BEGIN", + "INNER CALL", + "INNER END", + "OUTER END", + ] + + def test_stack_pop(self): + t = Template( + """ + <%def name="links()" buffered="True"> + Some links + </%def> + + <%def name="wrapper(links)"> + <h1>${caller.body()}</h1> + ${links} + </%def> + + ## links() pushes a stack frame on. when complete, + ## 'nextcaller' must be restored + <%call expr="wrapper(links())"> + Some title + </%call> + + """ + ) + + assert result_lines(t.render()) == [ + "<h1>", + "Some title", + "</h1>", + "Some links", + ] + + def test_conditional_call(self): + """test that 'caller' is non-None only if the immediate <%def> was + called via <%call>""" + + t = Template( + """ + <%def name="a()"> + % if caller: + ${ caller.body() } \\ + % endif + AAA + ${ b() } + </%def> + + <%def name="b()"> + % if caller: + ${ caller.body() } \\ + % endif + BBB + ${ c() } + </%def> + + <%def name="c()"> + % if caller: + ${ caller.body() } \\ + % endif + CCC + </%def> + + <%call expr="a()"> + CALL + </%call> + + """ + ) + assert result_lines(t.render()) == ["CALL", "AAA", "BBB", "CCC"] + + def test_chained_call(self): + """test %calls that are chained through their targets""" + t = Template( + """ + <%def name="a()"> + this is a. + <%call expr="b()"> + this is a's ccall. heres my body: ${caller.body()} + </%call> + </%def> + <%def name="b()"> + this is b. heres my body: ${caller.body()} + whats in the body's caller's body ? + ${context.caller_stack[-2].body()} + </%def> + + <%call expr="a()"> + heres the main templ call + </%call> + +""" + ) + assert result_lines(t.render()) == [ + "this is a.", + "this is b. heres my body:", + "this is a's ccall. heres my body:", + "heres the main templ call", + "whats in the body's caller's body ?", + "heres the main templ call", + ] + + def test_nested_call(self): + """test %calls that are nested inside each other""" + t = Template( + """ + <%def name="foo()"> + ${caller.body(x=10)} + </%def> + + x is ${x} + <%def name="bar()"> + bar: ${caller.body()} + </%def> + + <%call expr="foo()" args="x"> + this is foo body: ${x} + + <%call expr="bar()"> + this is bar body: ${x} + </%call> + </%call> +""" + ) + assert result_lines(t.render(x=5)) == [ + "x is 5", + "this is foo body: 10", + "bar:", + "this is bar body: 10", + ] + + def test_nested_call_2(self): + t = Template( + """ + x is ${x} + <%def name="foo()"> + ${caller.foosub(x=10)} + </%def> + + <%def name="bar()"> + bar: ${caller.barsub()} + </%def> + + <%call expr="foo()"> + <%def name="foosub(x)"> + this is foo body: ${x} + + <%call expr="bar()"> + <%def name="barsub()"> + this is bar body: ${x} + </%def> + </%call> + + </%def> + + </%call> +""" + ) + assert result_lines(t.render(x=5)) == [ + "x is 5", + "this is foo body: 10", + "bar:", + "this is bar body: 10", + ] + + def test_nested_call_3(self): + template = Template( + """\ + <%def name="A()"> + ${caller.body()} + </%def> + + <%def name="B()"> + ${caller.foo()} + </%def> + + <%call expr="A()"> + <%call expr="B()"> + <%def name="foo()"> + foo + </%def> + </%call> + </%call> + + """ + ) + assert flatten_result(template.render()) == "foo" + + def test_nested_call_4(self): + base = """ + <%def name="A()"> + A_def + ${caller.body()} + </%def> + + <%def name="B()"> + B_def + ${caller.body()} + </%def> + """ + + template = Template( + base + + """ + <%def name="C()"> + C_def + <%self:B> + <%self:A> + A_body + </%self:A> + B_body + ${caller.body()} + </%self:B> + </%def> + + <%self:C> + C_body + </%self:C> + """ + ) + + eq_( + flatten_result(template.render()), + "C_def B_def A_def A_body B_body C_body", + ) + + template = Template( + base + + """ + <%def name="C()"> + C_def + <%self:B> + B_body + ${caller.body()} + <%self:A> + A_body + </%self:A> + </%self:B> + </%def> + + <%self:C> + C_body + </%self:C> + """ + ) + + eq_( + flatten_result(template.render()), + "C_def B_def B_body C_body A_def A_body", + ) + + def test_chained_call_in_nested(self): + t = Template( + """ + <%def name="embedded()"> + <%def name="a()"> + this is a. + <%call expr="b()"> + this is a's ccall. heres my body: ${caller.body()} + </%call> + </%def> + <%def name="b()"> + this is b. heres my body: ${caller.body()} + whats in the body's caller's body ? """ + """${context.caller_stack[-2].body()} + </%def> + + <%call expr="a()"> + heres the main templ call + </%call> + </%def> + ${embedded()} +""" + ) + # print t.code + # print result_lines(t.render()) + assert result_lines(t.render()) == [ + "this is a.", + "this is b. heres my body:", + "this is a's ccall. heres my body:", + "heres the main templ call", + "whats in the body's caller's body ?", + "heres the main templ call", + ] + + def test_call_in_nested(self): + t = Template( + """ + <%def name="a()"> + this is a ${b()} + <%def name="b()"> + this is b + <%call expr="c()"> + this is the body in b's call + </%call> + </%def> + <%def name="c()"> + this is c: ${caller.body()} + </%def> + </%def> + ${a()} +""" + ) + assert result_lines(t.render()) == [ + "this is a", + "this is b", + "this is c:", + "this is the body in b's call", + ] + + def test_composed_def(self): + t = Template( + """ + <%def name="f()"><f>${caller.body()}</f></%def> + <%def name="g()"><g>${caller.body()}</g></%def> + <%def name="fg()"> + <%self:f><%self:g>${caller.body()}</%self:g></%self:f> + </%def> + <%self:fg>fgbody</%self:fg> + """ + ) + assert result_lines(t.render()) == ["<f><g>fgbody</g></f>"] + + def test_regular_defs(self): + t = Template( + """ + <%! + @runtime.supports_caller + def a(context): + context.write("this is a") + if context['caller']: + context['caller'].body() + context.write("a is done") + return '' + %> + + <%def name="b()"> + this is b + our body: ${caller.body()} + ${a(context)} + </%def> + test 1 + <%call expr="a(context)"> + this is the body + </%call> + test 2 + <%call expr="b()"> + this is the body + </%call> + test 3 + <%call expr="b()"> + this is the body + <%call expr="b()"> + this is the nested body + </%call> + </%call> + + + """ + ) + assert result_lines(t.render()) == [ + "test 1", + "this is a", + "this is the body", + "a is done", + "test 2", + "this is b", + "our body:", + "this is the body", + "this is aa is done", + "test 3", + "this is b", + "our body:", + "this is the body", + "this is b", + "our body:", + "this is the nested body", + "this is aa is done", + "this is aa is done", + ] + + def test_call_in_nested_2(self): + t = Template( + """ + <%def name="a()"> + <%def name="d()"> + not this d + </%def> + this is a ${b()} + <%def name="b()"> + <%def name="d()"> + not this d either + </%def> + this is b + <%call expr="c()"> + <%def name="d()"> + this is d + </%def> + this is the body in b's call + </%call> + </%def> + <%def name="c()"> + this is c: ${caller.body()} + the embedded "d" is: ${caller.d()} + </%def> + </%def> + ${a()} +""" + ) + assert result_lines(t.render()) == [ + "this is a", + "this is b", + "this is c:", + "this is the body in b's call", + 'the embedded "d" is:', + "this is d", + ] + + +class SelfCacheTest(TemplateTest): + """this test uses a now non-public API.""" + + def test_basic(self): + t = Template( + """ + <%! + cached = None + %> + <%def name="foo()"> + <% + global cached + if cached: + return "cached: " + cached + __M_writer = context._push_writer() + %> + this is foo + <% + buf, __M_writer = context._pop_buffer_and_writer() + cached = buf.getvalue() + return cached + %> + </%def> + + ${foo()} + ${foo()} +""" + ) + assert result_lines(t.render()) == [ + "this is foo", + "cached:", + "this is foo", + ] diff --git a/test/test_cmd.py b/test/test_cmd.py new file mode 100644 index 0000000..785c652 --- /dev/null +++ b/test/test_cmd.py @@ -0,0 +1,97 @@ +from contextlib import contextmanager +import os +from unittest import mock + +from mako.cmd import cmdline +from mako.testing.assertions import eq_ +from mako.testing.assertions import expect_raises +from mako.testing.assertions import expect_raises_message +from mako.testing.config import config +from mako.testing.fixtures import TemplateTest + + +class CmdTest(TemplateTest): + @contextmanager + def _capture_output_fixture(self, stream="stdout"): + with mock.patch("sys.%s" % stream) as stdout: + yield stdout + + def test_stdin_success(self): + with self._capture_output_fixture() as stdout: + with mock.patch( + "sys.stdin", + mock.Mock(read=mock.Mock(return_value="hello world ${x}")), + ): + cmdline(["--var", "x=5", "-"]) + + eq_(stdout.write.mock_calls[0][1][0], "hello world 5") + + def test_stdin_syntax_err(self): + with mock.patch( + "sys.stdin", mock.Mock(read=mock.Mock(return_value="${x")) + ): + with self._capture_output_fixture("stderr") as stderr: + with expect_raises(SystemExit): + cmdline(["--var", "x=5", "-"]) + + assert ( + "SyntaxException: Expected" in stderr.write.mock_calls[0][1][0] + ) + assert "Traceback" in stderr.write.mock_calls[0][1][0] + + def test_stdin_rt_err(self): + with mock.patch( + "sys.stdin", mock.Mock(read=mock.Mock(return_value="${q}")) + ): + with self._capture_output_fixture("stderr") as stderr: + with expect_raises(SystemExit): + cmdline(["--var", "x=5", "-"]) + + assert "NameError: Undefined" in stderr.write.mock_calls[0][1][0] + assert "Traceback" in stderr.write.mock_calls[0][1][0] + + def test_file_success(self): + with self._capture_output_fixture() as stdout: + cmdline( + [ + "--var", + "x=5", + os.path.join(config.template_base, "cmd_good.mako"), + ] + ) + + eq_(stdout.write.mock_calls[0][1][0], "hello world 5") + + def test_file_syntax_err(self): + with self._capture_output_fixture("stderr") as stderr: + with expect_raises(SystemExit): + cmdline( + [ + "--var", + "x=5", + os.path.join(config.template_base, "cmd_syntax.mako"), + ] + ) + + assert "SyntaxException: Expected" in stderr.write.mock_calls[0][1][0] + assert "Traceback" in stderr.write.mock_calls[0][1][0] + + def test_file_rt_err(self): + with self._capture_output_fixture("stderr") as stderr: + with expect_raises(SystemExit): + cmdline( + [ + "--var", + "x=5", + os.path.join(config.template_base, "cmd_runtime.mako"), + ] + ) + + assert "NameError: Undefined" in stderr.write.mock_calls[0][1][0] + assert "Traceback" in stderr.write.mock_calls[0][1][0] + + def test_file_notfound(self): + with expect_raises_message( + SystemExit, "error: can't find fake.lalala" + ): + cmdline(["--var", "x=5", "fake.lalala"]) diff --git a/test/test_decorators.py b/test/test_decorators.py new file mode 100644 index 0000000..68ea903 --- /dev/null +++ b/test/test_decorators.py @@ -0,0 +1,125 @@ +from mako.template import Template +from mako.testing.helpers import flatten_result + + +class DecoratorTest: + def test_toplevel(self): + template = Template( + """ + <%! + def bar(fn): + def decorate(context, *args, **kw): + return "BAR" + runtime.capture""" + """(context, fn, *args, **kw) + "BAR" + return decorate + %> + + <%def name="foo(y, x)" decorator="bar"> + this is foo ${y} ${x} + </%def> + + ${foo(1, x=5)} + """ + ) + + assert flatten_result(template.render()) == "BAR this is foo 1 5 BAR" + + def test_toplevel_contextual(self): + template = Template( + """ + <%! + def bar(fn): + def decorate(context): + context.write("BAR") + fn() + context.write("BAR") + return '' + return decorate + %> + + <%def name="foo()" decorator="bar"> + this is foo + </%def> + + ${foo()} + """ + ) + + assert flatten_result(template.render()) == "BAR this is foo BAR" + + assert ( + flatten_result(template.get_def("foo").render()) + == "BAR this is foo BAR" + ) + + def test_nested(self): + template = Template( + """ + <%! + def bat(fn): + def decorate(context): + return "BAT" + runtime.capture(context, fn) + "BAT" + return decorate + %> + + <%def name="foo()"> + + <%def name="bar()" decorator="bat"> + this is bar + </%def> + ${bar()} + </%def> + + ${foo()} + """ + ) + + assert flatten_result(template.render()) == "BAT this is bar BAT" + + def test_toplevel_decorated_name(self): + template = Template( + """ + <%! + def bar(fn): + def decorate(context, *args, **kw): + return "function " + fn.__name__ + """ + """" " + runtime.capture(context, fn, *args, **kw) + return decorate + %> + + <%def name="foo(y, x)" decorator="bar"> + this is foo ${y} ${x} + </%def> + + ${foo(1, x=5)} + """ + ) + + assert ( + flatten_result(template.render()) == "function foo this is foo 1 5" + ) + + def test_nested_decorated_name(self): + template = Template( + """ + <%! + def bat(fn): + def decorate(context): + return "function " + fn.__name__ + " " + """ + """runtime.capture(context, fn) + return decorate + %> + + <%def name="foo()"> + + <%def name="bar()" decorator="bat"> + this is bar + </%def> + ${bar()} + </%def> + + ${foo()} + """ + ) + + assert flatten_result(template.render()) == "function bar this is bar" diff --git a/test/test_def.py b/test/test_def.py new file mode 100644 index 0000000..fd96433 --- /dev/null +++ b/test/test_def.py @@ -0,0 +1,755 @@ +from mako import lookup +from mako.template import Template +from mako.testing.assertions import assert_raises +from mako.testing.assertions import eq_ +from mako.testing.fixtures import TemplateTest +from mako.testing.helpers import flatten_result +from mako.testing.helpers import result_lines + + +class DefTest(TemplateTest): + def test_def_noargs(self): + template = Template( + """ + + ${mycomp()} + + <%def name="mycomp()"> + hello mycomp ${variable} + </%def> + + """ + ) + eq_(template.render(variable="hi").strip(), """hello mycomp hi""") + + def test_def_blankargs(self): + template = Template( + """ + <%def name="mycomp()"> + hello mycomp ${variable} + </%def> + + ${mycomp()}""" + ) + eq_(template.render(variable="hi").strip(), "hello mycomp hi") + + def test_def_args(self): + template = Template( + """ + <%def name="mycomp(a, b)"> + hello mycomp ${variable}, ${a}, ${b} + </%def> + + ${mycomp(5, 6)}""" + ) + eq_( + template.render(variable="hi", a=5, b=6).strip(), + """hello mycomp hi, 5, 6""", + ) + + def test_def_py3k_args(self): + template = Template( + """ + <%def name="kwonly(one, two, *three, four, five=5, **six)"> + look at all these args: ${one} ${two} ${three[0]} """ + """${four} ${five} ${six['seven']} + </%def> + + ${kwonly('one', 'two', 'three', four='four', seven='seven')}""" + ) + eq_( + template.render(one=1, two=2, three=(3,), six=6).strip(), + """look at all these args: one two three four 5 seven""", + ) + + def test_inter_def(self): + """test defs calling each other""" + template = Template( + """ + ${b()} + + <%def name="a()">\ + im a + </%def> + + <%def name="b()"> + im b + and heres a: ${a()} + </%def> + + <%def name="c()"> + im c + </%def> +""" + ) + # check that "a" is declared in "b", but not in "c" + assert "a" not in template.module.render_c.__code__.co_varnames + assert "a" in template.module.render_b.__code__.co_varnames + + # then test output + eq_(flatten_result(template.render()), "im b and heres a: im a") + + def test_toplevel(self): + """test calling a def from the top level""" + + template = Template( + """ + + this is the body + + <%def name="a()"> + this is a + </%def> + + <%def name="b(x, y)"> + this is b, ${x} ${y} + </%def> + + """ + ) + + self._do_test( + template.get_def("a"), "this is a", filters=flatten_result + ) + self._do_test( + template.get_def("b"), + "this is b, 10 15", + template_args={"x": 10, "y": 15}, + filters=flatten_result, + ) + self._do_test( + template.get_def("body"), + "this is the body", + filters=flatten_result, + ) + + # test that args outside of the dict can be used + self._do_test( + template.get_def("a"), + "this is a", + filters=flatten_result, + template_args={"q": 5, "zq": "test"}, + ) + + def test_def_operations(self): + """test get/list/has def""" + + template = Template( + """ + + this is the body + + <%def name="a()"> + this is a + </%def> + + <%def name="b(x, y)"> + this is b, ${x} ${y} + </%def> + + """ + ) + + assert template.get_def("a") + assert template.get_def("b") + assert_raises(AttributeError, template.get_def, ("c")) + + assert template.has_def("a") + assert template.has_def("b") + assert not template.has_def("c") + + defs = template.list_defs() + assert "a" in defs + assert "b" in defs + assert "body" in defs + assert "c" not in defs + + +class ScopeTest(TemplateTest): + """test scoping rules. The key is, enclosing + scope always takes precedence over contextual scope.""" + + def test_scope_one(self): + self._do_memory_test( + """ + <%def name="a()"> + this is a, and y is ${y} + </%def> + + ${a()} + + <% + y = 7 + %> + + ${a()} + +""", + "this is a, and y is None this is a, and y is 7", + filters=flatten_result, + template_args={"y": None}, + ) + + def test_scope_two(self): + t = Template( + """ + y is ${y} + + <% + y = 7 + %> + + y is ${y} +""" + ) + try: + t.render(y=None) + assert False + except UnboundLocalError: + assert True + + def test_scope_four(self): + """test that variables are pulled + from 'enclosing' scope before context.""" + t = Template( + """ + <% + x = 5 + %> + <%def name="a()"> + this is a. x is ${x}. + </%def> + + <%def name="b()"> + <% + x = 9 + %> + this is b. x is ${x}. + calling a. ${a()} + </%def> + + ${b()} +""" + ) + eq_( + flatten_result(t.render()), + "this is b. x is 9. calling a. this is a. x is 5.", + ) + + def test_scope_five(self): + """test that variables are pulled from + 'enclosing' scope before context.""" + # same as test four, but adds a scope around it. + t = Template( + """ + <%def name="enclosing()"> + <% + x = 5 + %> + <%def name="a()"> + this is a. x is ${x}. + </%def> + + <%def name="b()"> + <% + x = 9 + %> + this is b. x is ${x}. + calling a. ${a()} + </%def> + + ${b()} + </%def> + ${enclosing()} +""" + ) + eq_( + flatten_result(t.render()), + "this is b. x is 9. calling a. this is a. x is 5.", + ) + + def test_scope_six(self): + """test that the initial context counts + as 'enclosing' scope, for plain defs""" + t = Template( + """ + + <%def name="a()"> + a: x is ${x} + </%def> + + <%def name="b()"> + <% + x = 10 + %> + b. x is ${x}. ${a()} + </%def> + + ${b()} + """ + ) + eq_(flatten_result(t.render(x=5)), "b. x is 10. a: x is 5") + + def test_scope_seven(self): + """test that the initial context counts + as 'enclosing' scope, for nested defs""" + t = Template( + """ + <%def name="enclosing()"> + <%def name="a()"> + a: x is ${x} + </%def> + + <%def name="b()"> + <% + x = 10 + %> + b. x is ${x}. ${a()} + </%def> + + ${b()} + </%def> + ${enclosing()} + """ + ) + eq_(flatten_result(t.render(x=5)), "b. x is 10. a: x is 5") + + def test_scope_eight(self): + """test that the initial context counts + as 'enclosing' scope, for nested defs""" + t = Template( + """ + <%def name="enclosing()"> + <%def name="a()"> + a: x is ${x} + </%def> + + <%def name="b()"> + <% + x = 10 + %> + + b. x is ${x}. ${a()} + </%def> + + ${b()} + </%def> + ${enclosing()} + """ + ) + eq_(flatten_result(t.render(x=5)), "b. x is 10. a: x is 5") + + def test_scope_nine(self): + """test that 'enclosing scope' doesnt + get exported to other templates""" + + l = lookup.TemplateLookup() + l.put_string( + "main", + """ + <% + x = 5 + %> + this is main. <%include file="secondary"/> +""", + ) + + l.put_string( + "secondary", + """ + this is secondary. x is ${x} +""", + ) + + eq_( + flatten_result(l.get_template("main").render(x=2)), + "this is main. this is secondary. x is 2", + ) + + def test_scope_ten(self): + t = Template( + """ + <%def name="a()"> + <%def name="b()"> + <% + y = 19 + %> + b/c: ${c()} + b/y: ${y} + </%def> + <%def name="c()"> + c/y: ${y} + </%def> + + <% + # we assign to "y". but the 'enclosing + # scope' of "b" and "c" is from + # the "y" on the outside + y = 10 + %> + a/y: ${y} + a/b: ${b()} + </%def> + + <% + y = 7 + %> + main/a: ${a()} + main/y: ${y} + """ + ) + eq_( + flatten_result(t.render()), + "main/a: a/y: 10 a/b: b/c: c/y: 10 b/y: 19 main/y: 7", + ) + + def test_scope_eleven(self): + t = Template( + """ + x is ${x} + <%def name="a(x)"> + this is a, ${b()} + <%def name="b()"> + this is b, x is ${x} + </%def> + </%def> + + ${a(x=5)} +""" + ) + eq_( + result_lines(t.render(x=10)), + ["x is 10", "this is a,", "this is b, x is 5"], + ) + + def test_unbound_scope(self): + t = Template( + """ + <% + y = 10 + %> + <%def name="a()"> + y is: ${y} + <% + # should raise error ? + y = 15 + %> + y is ${y} + </%def> + ${a()} +""" + ) + assert_raises(UnboundLocalError, t.render) + + def test_unbound_scope_two(self): + t = Template( + """ + <%def name="enclosing()"> + <% + y = 10 + %> + <%def name="a()"> + y is: ${y} + <% + # should raise error ? + y = 15 + %> + y is ${y} + </%def> + ${a()} + </%def> + ${enclosing()} +""" + ) + try: + print(t.render()) + assert False + except UnboundLocalError: + assert True + + def test_canget_kwargs(self): + """test that arguments passed to the body() + function are accessible by top-level defs""" + l = lookup.TemplateLookup() + l.put_string( + "base", + """ + + ${next.body(x=12)} + + """, + ) + + l.put_string( + "main", + """ + <%inherit file="base"/> + <%page args="x"/> + this is main. x is ${x} + + ${a()} + + <%def name="a(**args)"> + this is a, x is ${x} + </%def> + """, + ) + + # test via inheritance + eq_( + result_lines(l.get_template("main").render()), + ["this is main. x is 12", "this is a, x is 12"], + ) + + l.put_string( + "another", + """ + <%namespace name="ns" file="main"/> + + ${ns.body(x=15)} + """, + ) + # test via namespace + eq_( + result_lines(l.get_template("another").render()), + ["this is main. x is 15", "this is a, x is 15"], + ) + + def test_inline_expression_from_arg_one(self): + """test that cache_key=${foo} gets its value from + the 'foo' argument in the <%def> tag, + and strict_undefined doesn't complain. + + this is #191. + + """ + t = Template( + """ + <%def name="layout(foo)" cached="True" cache_key="${foo}"> + foo: ${foo} + </%def> + + ${layout(3)} + """, + strict_undefined=True, + cache_impl="plain", + ) + + eq_(result_lines(t.render()), ["foo: 3"]) + + def test_interpret_expression_from_arg_two(self): + """test that cache_key=${foo} gets its value from + the 'foo' argument regardless of it being passed + from the context. + + This is here testing that there's no change + to existing behavior before and after #191. + + """ + t = Template( + """ + <%def name="layout(foo)" cached="True" cache_key="${foo}"> + foo: ${value} + </%def> + + ${layout(3)} + """, + cache_impl="plain", + ) + + eq_(result_lines(t.render(foo="foo", value=1)), ["foo: 1"]) + eq_(result_lines(t.render(foo="bar", value=2)), ["foo: 1"]) + + +class NestedDefTest(TemplateTest): + def test_nested_def(self): + t = Template( + """ + + ${hi()} + + <%def name="hi()"> + hey, im hi. + and heres ${foo()}, ${bar()} + + <%def name="foo()"> + this is foo + </%def> + + <%def name="bar()"> + this is bar + </%def> + </%def> +""" + ) + eq_( + flatten_result(t.render()), + "hey, im hi. and heres this is foo , this is bar", + ) + + def test_nested_2(self): + t = Template( + """ + x is ${x} + <%def name="a()"> + this is a, x is ${x} + ${b()} + <%def name="b()"> + this is b: ${x} + </%def> + </%def> + ${a()} +""" + ) + + eq_( + flatten_result(t.render(x=10)), + "x is 10 this is a, x is 10 this is b: 10", + ) + + def test_nested_with_args(self): + t = Template( + """ + ${a()} + <%def name="a()"> + <%def name="b(x, y=2)"> + b x is ${x} y is ${y} + </%def> + a ${b(5)} + </%def> +""" + ) + eq_(flatten_result(t.render()), "a b x is 5 y is 2") + + def test_nested_def_2(self): + template = Template( + """ + ${a()} + <%def name="a()"> + <%def name="b()"> + <%def name="c()"> + comp c + </%def> + ${c()} + </%def> + ${b()} + </%def> +""" + ) + eq_(flatten_result(template.render()), "comp c") + + def test_nested_nested_def(self): + t = Template( + """ + + ${a()} + <%def name="a()"> + a + <%def name="b1()"> + a_b1 + </%def> + <%def name="b2()"> + a_b2 ${c1()} + <%def name="c1()"> + a_b2_c1 + </%def> + </%def> + <%def name="b3()"> + a_b3 ${c1()} + <%def name="c1()"> + a_b3_c1 heres x: ${x} + <% + y = 7 + %> + y is ${y} + </%def> + <%def name="c2()"> + a_b3_c2 + y is ${y} + c1 is ${c1()} + </%def> + ${c2()} + </%def> + + ${b1()} ${b2()} ${b3()} + </%def> +""" + ) + eq_( + flatten_result(t.render(x=5, y=None)), + "a a_b1 a_b2 a_b2_c1 a_b3 a_b3_c1 " + "heres x: 5 y is 7 a_b3_c2 y is " + "None c1 is a_b3_c1 heres x: 5 y is 7", + ) + + def test_nested_nested_def_2(self): + t = Template( + """ + <%def name="a()"> + this is a ${b()} + <%def name="b()"> + this is b + ${c()} + </%def> + + <%def name="c()"> + this is c + </%def> + </%def> + ${a()} +""" + ) + eq_(flatten_result(t.render()), "this is a this is b this is c") + + def test_outer_scope(self): + t = Template( + """ + <%def name="a()"> + a: x is ${x} + </%def> + + <%def name="b()"> + <%def name="c()"> + <% + x = 10 + %> + c. x is ${x}. ${a()} + </%def> + + b. ${c()} + </%def> + + ${b()} + + x is ${x} +""" + ) + eq_(flatten_result(t.render(x=5)), "b. c. x is 10. a: x is 5 x is 5") + + +class ExceptionTest(TemplateTest): + def test_raise(self): + template = Template( + """ + <% + raise Exception("this is a test") + %> + """, + format_exceptions=False, + ) + assert_raises(Exception, template.render) + + def test_handler(self): + def handle(context, error): + context.write("error message is " + str(error)) + return True + + template = Template( + """ + <% + raise Exception("this is a test") + %> + """, + error_handler=handle, + ) + eq_(template.render().strip(), "error message is this is a test") diff --git a/test/test_exceptions.py b/test/test_exceptions.py new file mode 100644 index 0000000..e1654ff --- /dev/null +++ b/test/test_exceptions.py @@ -0,0 +1,376 @@ +import sys + +from mako import exceptions +from mako.lookup import TemplateLookup +from mako.template import Template +from mako.testing.exclusions import requires_no_pygments_exceptions +from mako.testing.exclusions import requires_pygments_14 +from mako.testing.fixtures import TemplateTest +from mako.testing.helpers import result_lines + + +class ExceptionsTest(TemplateTest): + def test_html_error_template(self): + """test the html_error_template""" + code = """ +% i = 0 +""" + try: + template = Template(code) + template.render_unicode() + assert False + except exceptions.CompileException: + html_error = exceptions.html_error_template().render_unicode() + assert ( + "CompileException: Fragment 'i = 0' is not " + "a partial control statement at line: 2 char: 1" + ) in html_error + assert "<style>" in html_error + html_error_stripped = html_error.strip() + assert html_error_stripped.startswith("<html>") + assert html_error_stripped.endswith("</html>") + + not_full = exceptions.html_error_template().render_unicode( + full=False + ) + assert "<html>" not in not_full + assert "<style>" in not_full + + no_css = exceptions.html_error_template().render_unicode(css=False) + assert "<style>" not in no_css + else: + assert False, ( + "This function should trigger a CompileException, " + "but didn't" + ) + + def test_text_error_template(self): + code = """ +% i = 0 +""" + try: + template = Template(code) + template.render_unicode() + assert False + except exceptions.CompileException: + text_error = exceptions.text_error_template().render_unicode() + assert "Traceback (most recent call last):" in text_error + assert ( + "CompileException: Fragment 'i = 0' is not a partial " + "control statement" + ) in text_error + + @requires_pygments_14 + def test_utf8_html_error_template_pygments(self): + """test the html_error_template with a Template containing UTF-8 + chars""" + + code = """# -*- coding: utf-8 -*- +% if 2 == 2: /an error +${'привет'} +% endif +""" + try: + template = Template(code) + template.render_unicode() + except exceptions.CompileException: + html_error = exceptions.html_error_template().render() + assert ( + "CompileException: Fragment 'if 2 == 2: /an " + "error' is not a partial control statement " + "at line: 2 char: 1" + ).encode( + sys.getdefaultencoding(), "htmlentityreplace" + ) in html_error + + assert ( + "".encode(sys.getdefaultencoding(), "htmlentityreplace") + in html_error + ) + else: + assert False, ( + "This function should trigger a CompileException, " + "but didn't" + ) + + @requires_no_pygments_exceptions + def test_utf8_html_error_template_no_pygments(self): + """test the html_error_template with a Template containing UTF-8 + chars""" + + code = """# -*- coding: utf-8 -*- +% if 2 == 2: /an error +${'привет'} +% endif +""" + try: + template = Template(code) + template.render_unicode() + except exceptions.CompileException: + html_error = exceptions.html_error_template().render() + assert ( + "CompileException: Fragment 'if 2 == 2: /an " + "error' is not a partial control statement " + "at line: 2 char: 1" + ).encode( + sys.getdefaultencoding(), "htmlentityreplace" + ) in html_error + assert ( + "${'привет'}".encode( + sys.getdefaultencoding(), "htmlentityreplace" + ) + in html_error + ) + else: + assert False, ( + "This function should trigger a CompileException, " + "but didn't" + ) + + def test_format_closures(self): + try: + exec("def foo():" " raise RuntimeError('test')", locals()) + foo() # noqa + except: + html_error = exceptions.html_error_template().render() + assert "RuntimeError: test" in str(html_error) + + def test_py_utf8_html_error_template(self): + try: + foo = "日本" # noqa + raise RuntimeError("test") + except: + html_error = exceptions.html_error_template().render() + assert "RuntimeError: test" in html_error.decode("utf-8") + assert "foo = "日本"" in html_error.decode( + "utf-8" + ) or "foo = "日本"" in html_error.decode("utf-8") + + def test_py_unicode_error_html_error_template(self): + try: + raise RuntimeError("日本") + except: + html_error = exceptions.html_error_template().render() + assert "RuntimeError: 日本".encode("ascii", "ignore") in html_error + + @requires_pygments_14 + def test_format_exceptions_pygments(self): + l = TemplateLookup(format_exceptions=True) + + l.put_string( + "foo.html", + """ +<%inherit file="base.html"/> +${foobar} + """, + ) + + l.put_string( + "base.html", + """ + ${self.body()} + """, + ) + + assert ( + '<table class="syntax-highlightedtable">' + in l.get_template("foo.html").render_unicode() + ) + + @requires_no_pygments_exceptions + def test_format_exceptions_no_pygments(self): + l = TemplateLookup(format_exceptions=True) + + l.put_string( + "foo.html", + """ +<%inherit file="base.html"/> +${foobar} + """, + ) + + l.put_string( + "base.html", + """ + ${self.body()} + """, + ) + + assert '<div class="sourceline">${foobar}</div>' in result_lines( + l.get_template("foo.html").render_unicode() + ) + + @requires_pygments_14 + def test_utf8_format_exceptions_pygments(self): + """test that htmlentityreplace formatting is applied to + exceptions reported with format_exceptions=True""" + + l = TemplateLookup(format_exceptions=True) + l.put_string( + "foo.html", """# -*- coding: utf-8 -*-\n${'привет' + foobar}""" + ) + + assert "'привет'</span>" in l.get_template( + "foo.html" + ).render().decode("utf-8") + + @requires_no_pygments_exceptions + def test_utf8_format_exceptions_no_pygments(self): + """test that htmlentityreplace formatting is applied to + exceptions reported with format_exceptions=True""" + + l = TemplateLookup(format_exceptions=True) + l.put_string( + "foo.html", """# -*- coding: utf-8 -*-\n${'привет' + foobar}""" + ) + + assert ( + '<div class="sourceline">${'привет' + foobar}</div>' + in result_lines( + l.get_template("foo.html").render().decode("utf-8") + ) + ) + + def test_mod_no_encoding(self): + mod = __import__("test.foo.mod_no_encoding").foo.mod_no_encoding + try: + mod.run() + except: + t, v, tback = sys.exc_info() + exceptions.html_error_template().render_unicode( + error=v, traceback=tback + ) + + def test_custom_tback(self): + try: + raise RuntimeError("error 1") + foo("bar") # noqa + except: + t, v, tback = sys.exc_info() + + try: + raise RuntimeError("error 2") + except: + html_error = exceptions.html_error_template().render_unicode( + error=v, traceback=tback + ) + + # obfuscate the text so that this text + # isn't in the 'wrong' exception + assert ( + "".join(reversed(");touq&rab;touq&(oof")) in html_error + or "".join(reversed(");43#&rab;43#&(oof")) in html_error + ) + + def test_tback_no_trace_from_py_file(self): + try: + t = self._file_template("runtimeerr.html") + t.render() + except: + t, v, tback = sys.exc_info() + + # and don't even send what we have. + html_error = exceptions.html_error_template().render_unicode( + error=v, traceback=None + ) + + assert self.indicates_unbound_local_error(html_error, "y") + + def test_tback_trace_from_py_file(self): + t = self._file_template("runtimeerr.html") + try: + t.render() + assert False + except: + html_error = exceptions.html_error_template().render_unicode() + + assert self.indicates_unbound_local_error(html_error, "y") + + def test_code_block_line_number(self): + l = TemplateLookup() + l.put_string( + "foo.html", + """ +<% +msg = "Something went wrong." +raise RuntimeError(msg) # This is the line. +%> + """, + ) + t = l.get_template("foo.html") + try: + t.render() + except: + text_error = exceptions.text_error_template().render_unicode() + assert 'File "foo.html", line 4, in render_body' in text_error + assert "raise RuntimeError(msg) # This is the line." in text_error + else: + assert False + + def test_module_block_line_number(self): + l = TemplateLookup() + l.put_string( + "foo.html", + """ +<%! +def foo(): + msg = "Something went wrong." + raise RuntimeError(msg) # This is the line. +%> +${foo()} + """, + ) + t = l.get_template("foo.html") + try: + t.render() + except: + text_error = exceptions.text_error_template().render_unicode() + assert 'File "foo.html", line 7, in render_body' in text_error + assert 'File "foo.html", line 5, in foo' in text_error + assert "raise RuntimeError(msg) # This is the line." in text_error + else: + assert False + + def test_alternating_file_names(self): + l = TemplateLookup() + l.put_string( + "base.html", + """ +<%! +def broken(): + raise RuntimeError("Something went wrong.") +%> body starts here +<%block name="foo"> + ${broken()} +</%block> + """, + ) + l.put_string( + "foo.html", + """ +<%inherit file="base.html"/> +<%block name="foo"> + ${parent.foo()} +</%block> + """, + ) + t = l.get_template("foo.html") + try: + t.render() + except: + text_error = exceptions.text_error_template().render_unicode() + assert ( + """ + File "base.html", line 5, in render_body + %> body starts here + File "foo.html", line 4, in render_foo + ${parent.foo()} + File "base.html", line 7, in render_foo + ${broken()} + File "base.html", line 4, in broken + raise RuntimeError("Something went wrong.") +""" + in text_error + ) + else: + assert False diff --git a/test/test_filters.py b/test/test_filters.py new file mode 100644 index 0000000..726f5d7 --- /dev/null +++ b/test/test_filters.py @@ -0,0 +1,455 @@ +from mako.template import Template +from mako.testing.assertions import eq_ +from mako.testing.fixtures import TemplateTest +from mako.testing.helpers import flatten_result +from mako.testing.helpers import result_lines + + +class FilterTest(TemplateTest): + def test_basic(self): + t = Template( + """ + ${x | myfilter} +""" + ) + assert ( + flatten_result( + t.render( + x="this is x", + myfilter=lambda t: "MYFILTER->%s<-MYFILTER" % t, + ) + ) + == "MYFILTER->this is x<-MYFILTER" + ) + + def test_expr(self): + """test filters that are themselves expressions""" + t = Template( + """ + ${x | myfilter(y)} +""" + ) + + def myfilter(y): + return lambda x: "MYFILTER->%s<-%s" % (x, y) + + assert ( + flatten_result( + t.render(x="this is x", myfilter=myfilter, y="this is y") + ) + == "MYFILTER->this is x<-this is y" + ) + + def test_convert_str(self): + """test that string conversion happens in expressions before + sending to filters""" + t = Template( + """ + ${x | trim} + """ + ) + assert flatten_result(t.render(x=5)) == "5" + + def test_quoting(self): + t = Template( + """ + foo ${bar | h} + """ + ) + + eq_( + flatten_result(t.render(bar="<'some bar'>")), + "foo <'some bar'>", + ) + + def test_url_escaping(self): + t = Template( + """ + http://example.com/?bar=${bar | u}&v=1 + """ + ) + + eq_( + flatten_result(t.render(bar="酒吧bar")), + "http://example.com/?bar=%E9%85%92%E5%90%A7bar&v=1", + ) + + def test_entity(self): + t = Template("foo ${bar | entity}") + eq_( + flatten_result(t.render(bar="<'some bar'>")), + "foo <'some bar'>", + ) + + def test_def(self): + t = Template( + """ + <%def name="foo()" filter="myfilter"> + this is foo + </%def> + ${foo()} +""" + ) + + eq_( + flatten_result( + t.render( + x="this is x", + myfilter=lambda t: "MYFILTER->%s<-MYFILTER" % t, + ) + ), + "MYFILTER-> this is foo <-MYFILTER", + ) + + def test_import(self): + t = Template( + """ + <%! + from mako import filters + %>\ + trim this string: """ + """${" some string to trim " | filters.trim} continue\ + """ + ) + + assert ( + t.render().strip() + == "trim this string: some string to trim continue" + ) + + def test_import_2(self): + t = Template( + """ + trim this string: """ + """${" some string to trim " | filters.trim} continue\ + """, + imports=["from mako import filters"], + ) + # print t.code + assert ( + t.render().strip() + == "trim this string: some string to trim continue" + ) + + def test_encode_filter(self): + t = Template( + """# coding: utf-8 + some stuff.... ${x} + """, + default_filters=["decode.utf8"], + ) + eq_( + t.render_unicode(x="voix m’a réveillé").strip(), + "some stuff.... voix m’a réveillé", + ) + + def test_encode_filter_non_str(self): + t = Template( + """# coding: utf-8 + some stuff.... ${x} + """, + default_filters=["decode.utf8"], + ) + eq_(t.render_unicode(x=3).strip(), "some stuff.... 3") + + def test_custom_default(self): + t = Template( + """ + <%! + def myfilter(x): + return "->" + x + "<-" + %> + + hi ${'there'} + """, + default_filters=["myfilter"], + ) + assert t.render().strip() == "hi ->there<-" + + def test_global(self): + t = Template( + """ + <%page expression_filter="h"/> + ${"<tag>this is html</tag>"} + """ + ) + assert t.render().strip() == "<tag>this is html</tag>" + + def test_block_via_context(self): + t = Template( + """ + <%block name="foo" filter="myfilter"> + some text + </%block> + """ + ) + + def myfilter(text): + return "MYTEXT" + text + + eq_(result_lines(t.render(myfilter=myfilter)), ["MYTEXT", "some text"]) + + def test_def_via_context(self): + t = Template( + """ + <%def name="foo()" filter="myfilter"> + some text + </%def> + ${foo()} + """ + ) + + def myfilter(text): + return "MYTEXT" + text + + eq_(result_lines(t.render(myfilter=myfilter)), ["MYTEXT", "some text"]) + + def test_text_via_context(self): + t = Template( + """ + <%text filter="myfilter"> + some text + </%text> + """ + ) + + def myfilter(text): + return "MYTEXT" + text + + eq_(result_lines(t.render(myfilter=myfilter)), ["MYTEXT", "some text"]) + + def test_nflag(self): + t = Template( + """ + ${"<tag>this is html</tag>" | n} + """, + default_filters=["h", "unicode"], + ) + assert t.render().strip() == "<tag>this is html</tag>" + + t = Template( + """ + <%page expression_filter="h"/> + ${"<tag>this is html</tag>" | n} + """ + ) + assert t.render().strip() == "<tag>this is html</tag>" + + t = Template( + """ + <%page expression_filter="h"/> + ${"<tag>this is html</tag>" | n, h} + """ + ) + assert t.render().strip() == "<tag>this is html</tag>" + + def test_global_json(self): + t = Template( + """ +<%! +import json +%><%page expression_filter="n, json.dumps"/> +data = {a: ${123}, b: ${"123"}}; + """ + ) + assert t.render().strip() == """data = {a: 123, b: "123"};""" + + def test_non_expression(self): + t = Template( + """ + <%! + def a(text): + return "this is a" + def b(text): + return "this is b" + %> + + ${foo()} + <%def name="foo()" buffered="True"> + this is text + </%def> + """, + buffer_filters=["a"], + ) + assert t.render().strip() == "this is a" + + t = Template( + """ + <%! + def a(text): + return "this is a" + def b(text): + return "this is b" + %> + + ${'hi'} + ${foo()} + <%def name="foo()" buffered="True"> + this is text + </%def> + """, + buffer_filters=["a"], + default_filters=["b"], + ) + assert flatten_result(t.render()) == "this is b this is b" + + t = Template( + """ + <%! + class Foo: + foo = True + def __str__(self): + return "this is a" + def a(text): + return Foo() + def b(text): + if hasattr(text, 'foo'): + return str(text) + else: + return "this is b" + %> + + ${'hi'} + ${foo()} + <%def name="foo()" buffered="True"> + this is text + </%def> + """, + buffer_filters=["a"], + default_filters=["b"], + ) + assert flatten_result(t.render()) == "this is b this is a" + + t = Template( + """ + <%! + def a(text): + return "this is a" + def b(text): + return "this is b" + %> + + ${foo()} + ${bar()} + <%def name="foo()" filter="b"> + this is text + </%def> + <%def name="bar()" filter="b" buffered="True"> + this is text + </%def> + """, + buffer_filters=["a"], + ) + assert flatten_result(t.render()) == "this is b this is a" + + def test_builtins(self): + t = Template( + """ + ${"this is <text>" | h} +""" + ) + assert flatten_result(t.render()) == "this is <text>" + + t = Template( + """ + http://foo.com/arg1=${"hi! this is a string." | u} +""" + ) + assert ( + flatten_result(t.render()) + == "http://foo.com/arg1=hi%21+this+is+a+string." + ) + + +class BufferTest: + def test_buffered_def(self): + t = Template( + """ + <%def name="foo()" buffered="True"> + this is foo + </%def> + ${"hi->" + foo() + "<-hi"} +""" + ) + assert flatten_result(t.render()) == "hi-> this is foo <-hi" + + def test_unbuffered_def(self): + t = Template( + """ + <%def name="foo()" buffered="False"> + this is foo + </%def> + ${"hi->" + foo() + "<-hi"} +""" + ) + assert flatten_result(t.render()) == "this is foo hi-><-hi" + + def test_capture(self): + t = Template( + """ + <%def name="foo()" buffered="False"> + this is foo + </%def> + ${"hi->" + capture(foo) + "<-hi"} +""" + ) + assert flatten_result(t.render()) == "hi-> this is foo <-hi" + + def test_capture_exception(self): + template = Template( + """ + <%def name="a()"> + this is a + <% + raise TypeError("hi") + %> + </%def> + <% + c = capture(a) + %> + a->${c}<-a + """ + ) + try: + template.render() + assert False + except TypeError: + assert True + + def test_buffered_exception(self): + template = Template( + """ + <%def name="a()" buffered="True"> + <% + raise TypeError("hi") + %> + </%def> + + ${a()} + +""" + ) + try: + print(template.render()) + assert False + except TypeError: + assert True + + def test_capture_ccall(self): + t = Template( + """ + <%def name="foo()"> + <% + x = capture(caller.body) + %> + this is foo. body: ${x} + </%def> + + <%call expr="foo()"> + ccall body + </%call> +""" + ) + + # print t.render() + assert flatten_result(t.render()) == "this is foo. body: ccall body" diff --git a/test/test_inheritance.py b/test/test_inheritance.py new file mode 100644 index 0000000..15bd54b --- /dev/null +++ b/test/test_inheritance.py @@ -0,0 +1,428 @@ +from mako import lookup +from mako.testing.helpers import result_lines + + +class InheritanceTest: + def test_basic(self): + collection = lookup.TemplateLookup() + + collection.put_string( + "main", + """ +<%inherit file="base"/> + +<%def name="header()"> + main header. +</%def> + +this is the content. +""", + ) + + collection.put_string( + "base", + """ +This is base. + +header: ${self.header()} + +body: ${self.body()} + +footer: ${self.footer()} + +<%def name="footer()"> + this is the footer. header again ${next.header()} +</%def> +""", + ) + + assert result_lines(collection.get_template("main").render()) == [ + "This is base.", + "header:", + "main header.", + "body:", + "this is the content.", + "footer:", + "this is the footer. header again", + "main header.", + ] + + def test_multilevel_nesting(self): + collection = lookup.TemplateLookup() + + collection.put_string( + "main", + """ +<%inherit file="layout"/> +<%def name="d()">main_d</%def> +main_body ${parent.d()} +full stack from the top: + ${self.name} ${parent.name} ${parent.context['parent'].name} """ + """${parent.context['parent'].context['parent'].name} +""", + ) + + collection.put_string( + "layout", + """ +<%inherit file="general"/> +<%def name="d()">layout_d</%def> +layout_body +parent name: ${parent.name} +${parent.d()} +${parent.context['parent'].d()} +${next.body()} +""", + ) + + collection.put_string( + "general", + """ +<%inherit file="base"/> +<%def name="d()">general_d</%def> +general_body +${next.d()} +${next.context['next'].d()} +${next.body()} +""", + ) + collection.put_string( + "base", + """ +base_body +full stack from the base: + ${self.name} ${self.context['parent'].name} """ + """${self.context['parent'].context['parent'].name} """ + """${self.context['parent'].context['parent'].context['parent'].name} +${next.body()} +<%def name="d()">base_d</%def> +""", + ) + + assert result_lines(collection.get_template("main").render()) == [ + "base_body", + "full stack from the base:", + "self:main self:layout self:general self:base", + "general_body", + "layout_d", + "main_d", + "layout_body", + "parent name: self:general", + "general_d", + "base_d", + "main_body layout_d", + "full stack from the top:", + "self:main self:layout self:general self:base", + ] + + def test_includes(self): + """test that an included template also has its full hierarchy + invoked.""" + collection = lookup.TemplateLookup() + + collection.put_string( + "base", + """ + <%def name="a()">base_a</%def> + This is the base. + ${next.body()} + End base. +""", + ) + + collection.put_string( + "index", + """ + <%inherit file="base"/> + this is index. + a is: ${self.a()} + <%include file="secondary"/> +""", + ) + + collection.put_string( + "secondary", + """ + <%inherit file="base"/> + this is secondary. + a is: ${self.a()} +""", + ) + + assert result_lines(collection.get_template("index").render()) == [ + "This is the base.", + "this is index.", + "a is: base_a", + "This is the base.", + "this is secondary.", + "a is: base_a", + "End base.", + "End base.", + ] + + def test_namespaces(self): + """test that templates used via <%namespace> have access to an + inheriting 'self', and that the full 'self' is also exported.""" + collection = lookup.TemplateLookup() + + collection.put_string( + "base", + """ + <%def name="a()">base_a</%def> + <%def name="b()">base_b</%def> + This is the base. + ${next.body()} +""", + ) + + collection.put_string( + "layout", + """ + <%inherit file="base"/> + <%def name="a()">layout_a</%def> + This is the layout.. + ${next.body()} +""", + ) + + collection.put_string( + "index", + """ + <%inherit file="base"/> + <%namespace name="sc" file="secondary"/> + this is index. + a is: ${self.a()} + sc.a is: ${sc.a()} + sc.b is: ${sc.b()} + sc.c is: ${sc.c()} + sc.body is: ${sc.body()} +""", + ) + + collection.put_string( + "secondary", + """ + <%inherit file="layout"/> + <%def name="c()">secondary_c. a is ${self.a()} b is ${self.b()} """ + """d is ${self.d()}</%def> + <%def name="d()">secondary_d.</%def> + this is secondary. + a is: ${self.a()} + c is: ${self.c()} +""", + ) + + assert result_lines(collection.get_template("index").render()) == [ + "This is the base.", + "this is index.", + "a is: base_a", + "sc.a is: layout_a", + "sc.b is: base_b", + "sc.c is: secondary_c. a is layout_a b is base_b d is " + "secondary_d.", + "sc.body is:", + "this is secondary.", + "a is: layout_a", + "c is: secondary_c. a is layout_a b is base_b d is secondary_d.", + ] + + def test_pageargs(self): + collection = lookup.TemplateLookup() + collection.put_string( + "base", + """ + this is the base. + + <% + sorted_ = pageargs.items() + sorted_ = sorted(sorted_) + %> + pageargs: (type: ${type(pageargs)}) ${sorted_} + <%def name="foo()"> + ${next.body(**context.kwargs)} + </%def> + + ${foo()} + """, + ) + collection.put_string( + "index", + """ + <%inherit file="base"/> + <%page args="x, y, z=7"/> + print ${x}, ${y}, ${z} + """, + ) + + assert result_lines( + collection.get_template("index").render_unicode(x=5, y=10) + ) == [ + "this is the base.", + "pageargs: (type: <class 'dict'>) [('x', 5), ('y', 10)]", + "print 5, 10, 7", + ] + + def test_pageargs_2(self): + collection = lookup.TemplateLookup() + collection.put_string( + "base", + """ + this is the base. + + ${next.body(**context.kwargs)} + + <%def name="foo(**kwargs)"> + ${next.body(**kwargs)} + </%def> + + <%def name="bar(**otherargs)"> + ${next.body(z=16, **context.kwargs)} + </%def> + + ${foo(x=12, y=15, z=8)} + ${bar(x=19, y=17)} + """, + ) + collection.put_string( + "index", + """ + <%inherit file="base"/> + <%page args="x, y, z=7"/> + pageargs: ${x}, ${y}, ${z} + """, + ) + assert result_lines( + collection.get_template("index").render(x=5, y=10) + ) == [ + "this is the base.", + "pageargs: 5, 10, 7", + "pageargs: 12, 15, 8", + "pageargs: 5, 10, 16", + ] + + def test_pageargs_err(self): + collection = lookup.TemplateLookup() + collection.put_string( + "base", + """ + this is the base. + ${next.body()} + """, + ) + collection.put_string( + "index", + """ + <%inherit file="base"/> + <%page args="x, y, z=7"/> + print ${x}, ${y}, ${z} + """, + ) + try: + print(collection.get_template("index").render(x=5, y=10)) + assert False + except TypeError: + assert True + + def test_toplevel(self): + collection = lookup.TemplateLookup() + collection.put_string( + "base", + """ + this is the base. + ${next.body()} + """, + ) + collection.put_string( + "index", + """ + <%inherit file="base"/> + this is the body + """, + ) + assert result_lines(collection.get_template("index").render()) == [ + "this is the base.", + "this is the body", + ] + assert result_lines( + collection.get_template("index").get_def("body").render() + ) == ["this is the body"] + + def test_dynamic(self): + collection = lookup.TemplateLookup() + collection.put_string( + "base", + """ + this is the base. + ${next.body()} + """, + ) + collection.put_string( + "index", + """ + <%! + def dyn(context): + if context.get('base', None) is not None: + return 'base' + else: + return None + %> + <%inherit file="${dyn(context)}"/> + this is index. + """, + ) + assert result_lines(collection.get_template("index").render()) == [ + "this is index." + ] + assert result_lines( + collection.get_template("index").render(base=True) + ) == ["this is the base.", "this is index."] + + def test_in_call(self): + collection = lookup.TemplateLookup() + collection.put_string( + "/layout.html", + """ + Super layout! + <%call expr="self.grid()"> + ${next.body()} + </%call> + Oh yea! + + <%def name="grid()"> + Parent grid + ${caller.body()} + End Parent + </%def> + """, + ) + + collection.put_string( + "/subdir/layout.html", + """ + ${next.body()} + <%def name="grid()"> + Subdir grid + ${caller.body()} + End subdir + </%def> + <%inherit file="/layout.html"/> + """, + ) + + collection.put_string( + "/subdir/renderedtemplate.html", + """ + Holy smokes! + <%inherit file="/subdir/layout.html"/> + """, + ) + + assert result_lines( + collection.get_template("/subdir/renderedtemplate.html").render() + ) == [ + "Super layout!", + "Subdir grid", + "Holy smokes!", + "End subdir", + "Oh yea!", + ] diff --git a/test/test_lexer.py b/test/test_lexer.py new file mode 100644 index 0000000..f4983a3 --- /dev/null +++ b/test/test_lexer.py @@ -0,0 +1,1232 @@ +import re + +import pytest + +from mako import compat +from mako import exceptions +from mako import parsetree +from mako import util +from mako.lexer import Lexer +from mako.template import Template +from mako.testing.assertions import assert_raises +from mako.testing.assertions import assert_raises_message +from mako.testing.assertions import eq_ +from mako.testing.fixtures import TemplateTest +from mako.testing.helpers import flatten_result + +# create fake parsetree classes which are constructed +# exactly as the repr() of a real parsetree object. +# this allows us to use a Python construct as the source +# of a comparable repr(), which is also hit by the 2to3 tool. + + +def repr_arg(x): + if isinstance(x, dict): + return util.sorted_dict_repr(x) + else: + return repr(x) + + +def _as_unicode(arg): + if isinstance(arg, dict): + return {k: _as_unicode(v) for k, v in arg.items()} + else: + return arg + + +Node = None +TemplateNode = None +ControlLine = None +Text = None +Code = None +Comment = None +Expression = None +_TagMeta = None +Tag = None +IncludeTag = None +NamespaceTag = None +TextTag = None +DefTag = None +BlockTag = None +CallTag = None +CallNamespaceTag = None +InheritTag = None +PageTag = None + +# go through all the elements in parsetree and build out +# mocks of them +for cls in list(parsetree.__dict__.values()): + if isinstance(cls, type) and issubclass(cls, parsetree.Node): + clsname = cls.__name__ + exec( + ( + """ +class %s: + def __init__(self, *args): + self.args = [_as_unicode(arg) for arg in args] + def __repr__(self): + return "%%s(%%s)" %% ( + self.__class__.__name__, + ", ".join(repr_arg(x) for x in self.args) + ) +""" + % clsname + ), + locals(), + ) + +# NOTE: most assertion expressions were generated, then formatted +# by PyTidy, hence the dense formatting. + + +class LexerTest(TemplateTest): + def _compare(self, node, expected): + eq_(repr(node), repr(expected)) + + def test_text_and_tag(self): + template = """ +<b>Hello world</b> + <%def name="foo()"> + this is a def. + </%def> + + and some more text. +""" + node = Lexer(template).parse() + self._compare( + node, + TemplateNode( + {}, + [ + Text("""\n<b>Hello world</b>\n """, (1, 1)), + DefTag( + "def", + {"name": "foo()"}, + (3, 9), + [ + Text( + "\n this is a def.\n ", + (3, 28), + ) + ], + ), + Text("""\n\n and some more text.\n""", (5, 16)), + ], + ), + ) + + def test_unclosed_tag(self): + template = """ + + <%def name="foo()"> + other text + """ + try: + Lexer(template).parse() + assert False + except exceptions.SyntaxException: + eq_( + str(compat.exception_as()), + "Unclosed tag: <%def> at line: 5 char: 9", + ) + + def test_onlyclosed_tag(self): + template = """ + <%def name="foo()"> + foo + </%def> + + </%namespace> + + hi. + """ + assert_raises(exceptions.SyntaxException, Lexer(template).parse) + + def test_noexpr_allowed(self): + template = """ + <%namespace name="${foo}"/> + """ + assert_raises(exceptions.CompileException, Lexer(template).parse) + + def test_closing_tag_many_spaces(self): + """test #367""" + template = '<%def name="foo()"> this is a def. </%' + " " * 10000 + assert_raises(exceptions.SyntaxException, Lexer(template).parse) + + def test_opening_tag_many_quotes(self): + """test #366""" + template = "<%0" + '"' * 3000 + assert_raises(exceptions.SyntaxException, Lexer(template).parse) + + def test_unmatched_tag(self): + template = """ + <%namespace name="bar"> + <%def name="foo()"> + foo + </%namespace> + </%def> + + + hi. +""" + assert_raises(exceptions.SyntaxException, Lexer(template).parse) + + def test_nonexistent_tag(self): + template = """ + <%lala x="5"/> + """ + assert_raises(exceptions.CompileException, Lexer(template).parse) + + def test_wrongcase_tag(self): + template = """ + <%DEF name="foo()"> + </%def> + + """ + assert_raises(exceptions.CompileException, Lexer(template).parse) + + def test_percent_escape(self): + template = """ + +%% some whatever. + + %% more some whatever + % if foo: + % endif + """ + node = Lexer(template).parse() + self._compare( + node, + TemplateNode( + {}, + [ + Text("""\n\n""", (1, 1)), + Text("""% some whatever.\n\n""", (3, 2)), + Text(" %% more some whatever\n", (5, 2)), + ControlLine("if", "if foo:", False, (6, 1)), + ControlLine("if", "endif", True, (7, 1)), + Text(" ", (8, 1)), + ], + ), + ) + + def test_old_multiline_comment(self): + template = """#*""" + node = Lexer(template).parse() + self._compare(node, TemplateNode({}, [Text("""#*""", (1, 1))])) + + def test_text_tag(self): + template = """ + ## comment + % if foo: + hi + % endif + <%text> + # more code + + % more code + <%illegal compionent>/></> + <%def name="laal()">def</%def> + + + </%text> + + <%def name="foo()">this is foo</%def> + + % if bar: + code + % endif + """ + node = Lexer(template).parse() + self._compare( + node, + TemplateNode( + {}, + [ + Text("\n", (1, 1)), + Comment("comment", (2, 1)), + ControlLine("if", "if foo:", False, (3, 1)), + Text(" hi\n", (4, 1)), + ControlLine("if", "endif", True, (5, 1)), + Text(" ", (6, 1)), + TextTag( + "text", + {}, + (6, 9), + [ + Text( + "\n # more code\n\n " + " % more code\n " + "<%illegal compionent>/></>\n" + ' <%def name="laal()">def</%def>' + "\n\n\n ", + (6, 16), + ) + ], + ), + Text("\n\n ", (14, 17)), + DefTag( + "def", + {"name": "foo()"}, + (16, 9), + [Text("this is foo", (16, 28))], + ), + Text("\n\n", (16, 46)), + ControlLine("if", "if bar:", False, (18, 1)), + Text(" code\n", (19, 1)), + ControlLine("if", "endif", True, (20, 1)), + Text(" ", (21, 1)), + ], + ), + ) + + def test_def_syntax(self): + template = """ + <%def lala> + hi + </%def> +""" + assert_raises(exceptions.CompileException, Lexer(template).parse) + + def test_def_syntax_2(self): + template = """ + <%def name="lala"> + hi + </%def> + """ + assert_raises(exceptions.CompileException, Lexer(template).parse) + + def test_whitespace_equals(self): + template = """ + <%def name = "adef()" > + adef + </%def> + """ + node = Lexer(template).parse() + self._compare( + node, + TemplateNode( + {}, + [ + Text("\n ", (1, 1)), + DefTag( + "def", + {"name": "adef()"}, + (2, 13), + [ + Text( + """\n adef\n """, + (2, 36), + ) + ], + ), + Text("\n ", (4, 20)), + ], + ), + ) + + def test_ns_tag_closed(self): + template = """ + + <%self:go x="1" y="2" z="${'hi' + ' ' + 'there'}"/> + """ + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text( + """ + + """, + (1, 1), + ), + CallNamespaceTag( + "self:go", + {"x": "1", "y": "2", "z": "${'hi' + ' ' + 'there'}"}, + (3, 13), + [], + ), + Text("\n ", (3, 64)), + ], + ), + ) + + def test_ns_tag_empty(self): + template = """ + <%form:option value=""></%form:option> + """ + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text("\n ", (1, 1)), + CallNamespaceTag( + "form:option", {"value": ""}, (2, 13), [] + ), + Text("\n ", (2, 51)), + ], + ), + ) + + def test_ns_tag_open(self): + template = """ + + <%self:go x="1" y="${process()}"> + this is the body + </%self:go> + """ + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text( + """ + + """, + (1, 1), + ), + CallNamespaceTag( + "self:go", + {"x": "1", "y": "${process()}"}, + (3, 13), + [ + Text( + """ + this is the body + """, + (3, 46), + ) + ], + ), + Text("\n ", (5, 24)), + ], + ), + ) + + def test_expr_in_attribute(self): + """test some slightly trickier expressions. + + you can still trip up the expression parsing, though, unless we + integrated really deeply somehow with AST.""" + + template = """ + <%call expr="foo>bar and 'lala' or 'hoho'"/> + <%call expr='foo<bar and hoho>lala and "x" + "y"'/> + """ + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text("\n ", (1, 1)), + CallTag( + "call", + {"expr": "foo>bar and 'lala' or 'hoho'"}, + (2, 13), + [], + ), + Text("\n ", (2, 57)), + CallTag( + "call", + {"expr": 'foo<bar and hoho>lala and "x" + "y"'}, + (3, 13), + [], + ), + Text("\n ", (3, 64)), + ], + ), + ) + + @pytest.mark.parametrize("comma,numchars", [(",", 48), ("", 47)]) + def test_pagetag(self, comma, numchars): + # note that the comma here looks like: + # <%page cached="True", args="a, b"/> + # that's what this test has looked like for decades, however, the + # comma there is not actually the right syntax. When issue #366 + # was fixed, the reg was altered to accommodate for this comma to allow + # backwards compat + template = f""" + <%page cached="True"{comma} args="a, b"/> + + some template + """ + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text("\n ", (1, 1)), + PageTag( + "page", {"args": "a, b", "cached": "True"}, (2, 13), [] + ), + Text( + """ + + some template + """, + (2, numchars), + ), + ], + ), + ) + + def test_nesting(self): + template = """ + + <%namespace name="ns"> + <%def name="lala(hi, there)"> + <%call expr="something()"/> + </%def> + </%namespace> + + """ + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text( + """ + + """, + (1, 1), + ), + NamespaceTag( + "namespace", + {"name": "ns"}, + (3, 9), + [ + Text("\n ", (3, 31)), + DefTag( + "def", + {"name": "lala(hi, there)"}, + (4, 13), + [ + Text("\n ", (4, 42)), + CallTag( + "call", + {"expr": "something()"}, + (5, 17), + [], + ), + Text("\n ", (5, 44)), + ], + ), + Text("\n ", (6, 20)), + ], + ), + Text( + """ + + """, + (7, 22), + ), + ], + ), + ) + + def test_code(self): + template = """text + <% + print("hi") + for x in range(1,5): + print(x) + %> +more text + <%! + import foo + %> +""" + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text("text\n ", (1, 1)), + Code( + '\nprint("hi")\nfor x in range(1,5):\n ' + "print(x)\n \n", + False, + (2, 5), + ), + Text("\nmore text\n ", (6, 7)), + Code("\nimport foo\n \n", True, (8, 5)), + Text("\n", (10, 7)), + ], + ), + ) + + def test_code_and_tags(self): + template = """ +<%namespace name="foo"> + <%def name="x()"> + this is x + </%def> + <%def name="y()"> + this is y + </%def> +</%namespace> + +<% + result = [] + data = get_data() + for x in data: + result.append(x+7) +%> + + result: <%call expr="foo.x(result)"/> +""" + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text("\n", (1, 1)), + NamespaceTag( + "namespace", + {"name": "foo"}, + (2, 1), + [ + Text("\n ", (2, 24)), + DefTag( + "def", + {"name": "x()"}, + (3, 5), + [ + Text( + """\n this is x\n """, + (3, 22), + ) + ], + ), + Text("\n ", (5, 12)), + DefTag( + "def", + {"name": "y()"}, + (6, 5), + [ + Text( + """\n this is y\n """, + (6, 22), + ) + ], + ), + Text("\n", (8, 12)), + ], + ), + Text("""\n\n""", (9, 14)), + Code( + """\nresult = []\ndata = get_data()\n""" + """for x in data:\n result.append(x+7)\n\n""", + False, + (11, 1), + ), + Text("""\n\n result: """, (16, 3)), + CallTag("call", {"expr": "foo.x(result)"}, (18, 13), []), + Text("\n", (18, 42)), + ], + ), + ) + + def test_expression(self): + template = """ + this is some ${text} and this is ${textwith | escapes, moreescapes} + <%def name="hi()"> + give me ${foo()} and ${bar()} + </%def> + ${hi()} +""" + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text("\n this is some ", (1, 1)), + Expression("text", [], (2, 22)), + Text(" and this is ", (2, 29)), + Expression( + "textwith ", ["escapes", "moreescapes"], (2, 42) + ), + Text("\n ", (2, 76)), + DefTag( + "def", + {"name": "hi()"}, + (3, 9), + [ + Text("\n give me ", (3, 27)), + Expression("foo()", [], (4, 21)), + Text(" and ", (4, 29)), + Expression("bar()", [], (4, 34)), + Text("\n ", (4, 42)), + ], + ), + Text("\n ", (5, 16)), + Expression("hi()", [], (6, 9)), + Text("\n", (6, 16)), + ], + ), + ) + + def test_tricky_expression(self): + template = """ + + ${x and "|" or "hi"} + """ + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text("\n\n ", (1, 1)), + Expression('x and "|" or "hi"', [], (3, 13)), + Text("\n ", (3, 33)), + ], + ), + ) + + template = r""" + + ${hello + '''heres '{|}' text | | }''' | escape1} + ${'Tricky string: ' + '\\\"\\\'|\\'} + """ + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text("\n\n ", (1, 1)), + Expression( + "hello + '''heres '{|}' text | | }''' ", + ["escape1"], + (3, 13), + ), + Text("\n ", (3, 62)), + Expression( + r"""'Tricky string: ' + '\\\"\\\'|\\'""", [], (4, 13) + ), + Text("\n ", (4, 49)), + ], + ), + ) + + def test_tricky_code(self): + template = """<% print('hi %>') %>""" + nodes = Lexer(template).parse() + self._compare( + nodes, TemplateNode({}, [Code("print('hi %>') \n", False, (1, 1))]) + ) + + def test_tricky_code_2(self): + template = """<% + # someone's comment +%> + """ + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Code( + """ + # someone's comment + +""", + False, + (1, 1), + ), + Text("\n ", (3, 3)), + ], + ), + ) + + def test_tricky_code_3(self): + template = """<% + print('hi') + # this is a comment + # another comment + x = 7 # someone's '''comment + print(''' + there + ''') + # someone else's comment +%> '''and now some text '''""" + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Code( + """ +print('hi') +# this is a comment +# another comment +x = 7 # someone's '''comment +print(''' + there + ''') +# someone else's comment + +""", + False, + (1, 1), + ), + Text(" '''and now some text '''", (10, 3)), + ], + ), + ) + + def test_tricky_code_4(self): + template = """<% foo = "\\"\\\\" %>""" + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode({}, [Code("""foo = "\\"\\\\" \n""", False, (1, 1))]), + ) + + def test_tricky_code_5(self): + template = """before ${ {'key': 'value'} } after""" + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text("before ", (1, 1)), + Expression(" {'key': 'value'} ", [], (1, 8)), + Text(" after", (1, 29)), + ], + ), + ) + + def test_tricky_code_6(self): + template = """before ${ (0x5302 | 0x0400) } after""" + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text("before ", (1, 1)), + Expression(" (0x5302 | 0x0400) ", [], (1, 8)), + Text(" after", (1, 30)), + ], + ), + ) + + def test_control_lines(self): + template = """ +text text la la +% if foo(): + mroe text la la blah blah +% endif + + and osme more stuff + % for l in range(1,5): + tex tesl asdl l is ${l} kfmas d + % endfor + tetx text + +""" + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text("""\ntext text la la\n""", (1, 1)), + ControlLine("if", "if foo():", False, (3, 1)), + Text(" mroe text la la blah blah\n", (4, 1)), + ControlLine("if", "endif", True, (5, 1)), + Text("""\n and osme more stuff\n""", (6, 1)), + ControlLine("for", "for l in range(1,5):", False, (8, 1)), + Text(" tex tesl asdl l is ", (9, 1)), + Expression("l", [], (9, 24)), + Text(" kfmas d\n", (9, 28)), + ControlLine("for", "endfor", True, (10, 1)), + Text(""" tetx text\n\n""", (11, 1)), + ], + ), + ) + + def test_control_lines_2(self): + template = """% for file in requestattr['toc'].filenames: + x +% endfor +""" + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + ControlLine( + "for", + "for file in requestattr['toc'].filenames:", + False, + (1, 1), + ), + Text(" x\n", (2, 1)), + ControlLine("for", "endfor", True, (3, 1)), + ], + ), + ) + + def test_long_control_lines(self): + template = """ + % for file in \\ + requestattr['toc'].filenames: + x + % endfor + """ + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text("\n", (1, 1)), + ControlLine( + "for", + "for file in \\\n " + "requestattr['toc'].filenames:", + False, + (2, 1), + ), + Text(" x\n", (4, 1)), + ControlLine("for", "endfor", True, (5, 1)), + Text(" ", (6, 1)), + ], + ), + ) + + def test_unmatched_control(self): + template = """ + + % if foo: + % for x in range(1,5): + % endif +""" + assert_raises_message( + exceptions.SyntaxException, + "Keyword 'endif' doesn't match keyword 'for' at line: 5 char: 1", + Lexer(template).parse, + ) + + def test_unmatched_control_2(self): + template = """ + + % if foo: + % for x in range(1,5): + % endfor +""" + + assert_raises_message( + exceptions.SyntaxException, + "Unterminated control keyword: 'if' at line: 3 char: 1", + Lexer(template).parse, + ) + + def test_unmatched_control_3(self): + template = """ + + % if foo: + % for x in range(1,5): + % endlala + % endif +""" + assert_raises_message( + exceptions.SyntaxException, + "Keyword 'endlala' doesn't match keyword 'for' at line: 5 char: 1", + Lexer(template).parse, + ) + + def test_ternary_control(self): + template = """ + % if x: + hi + % elif y+7==10: + there + % elif lala: + lala + % else: + hi + % endif +""" + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text("\n", (1, 1)), + ControlLine("if", "if x:", False, (2, 1)), + Text(" hi\n", (3, 1)), + ControlLine("elif", "elif y+7==10:", False, (4, 1)), + Text(" there\n", (5, 1)), + ControlLine("elif", "elif lala:", False, (6, 1)), + Text(" lala\n", (7, 1)), + ControlLine("else", "else:", False, (8, 1)), + Text(" hi\n", (9, 1)), + ControlLine("if", "endif", True, (10, 1)), + ], + ), + ) + + def test_integration(self): + template = """<%namespace name="foo" file="somefile.html"/> + ## inherit from foobar.html +<%inherit file="foobar.html"/> + +<%def name="header()"> + <div>header</div> +</%def> +<%def name="footer()"> + <div> footer</div> +</%def> + +<table> + % for j in data(): + <tr> + % for x in j: + <td>Hello ${x| h}</td> + % endfor + </tr> + % endfor +</table> +""" + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + NamespaceTag( + "namespace", + {"file": "somefile.html", "name": "foo"}, + (1, 1), + [], + ), + Text("\n", (1, 46)), + Comment("inherit from foobar.html", (2, 1)), + InheritTag("inherit", {"file": "foobar.html"}, (3, 1), []), + Text("""\n\n""", (3, 31)), + DefTag( + "def", + {"name": "header()"}, + (5, 1), + [Text("""\n <div>header</div>\n""", (5, 23))], + ), + Text("\n", (7, 8)), + DefTag( + "def", + {"name": "footer()"}, + (8, 1), + [Text("""\n <div> footer</div>\n""", (8, 23))], + ), + Text("""\n\n<table>\n""", (10, 8)), + ControlLine("for", "for j in data():", False, (13, 1)), + Text(" <tr>\n", (14, 1)), + ControlLine("for", "for x in j:", False, (15, 1)), + Text(" <td>Hello ", (16, 1)), + Expression("x", ["h"], (16, 23)), + Text("</td>\n", (16, 30)), + ControlLine("for", "endfor", True, (17, 1)), + Text(" </tr>\n", (18, 1)), + ControlLine("for", "endfor", True, (19, 1)), + Text("</table>\n", (20, 1)), + ], + ), + ) + + def test_comment_after_statement(self): + template = """ + % if x: #comment + hi + % else: #next + hi + % endif #end +""" + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text("\n", (1, 1)), + ControlLine("if", "if x: #comment", False, (2, 1)), + Text(" hi\n", (3, 1)), + ControlLine("else", "else: #next", False, (4, 1)), + Text(" hi\n", (5, 1)), + ControlLine("if", "endif #end", True, (6, 1)), + ], + ), + ) + + def test_crlf(self): + template = util.read_file(self._file_path("crlf.html")) + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text("<html>\r\n\r\n", (1, 1)), + PageTag( + "page", + {"args": "a=['foo',\n 'bar']"}, + (3, 1), + [], + ), + Text("\r\n\r\nlike the name says.\r\n\r\n", (4, 26)), + ControlLine("for", "for x in [1,2,3]:", False, (8, 1)), + Text(" ", (9, 1)), + Expression("x", [], (9, 9)), + ControlLine("for", "endfor", True, (10, 1)), + Text("\r\n", (11, 1)), + Expression( + "trumpeter == 'Miles' and " + "trumpeter or \\\n 'Dizzy'", + [], + (12, 1), + ), + Text("\r\n\r\n", (13, 15)), + DefTag( + "def", + {"name": "hi()"}, + (15, 1), + [Text("\r\n hi!\r\n", (15, 19))], + ), + Text("\r\n\r\n</html>\r\n", (17, 8)), + ], + ), + ) + assert ( + flatten_result(Template(template).render()) + == """<html> like the name says. 1 2 3 Dizzy </html>""" + ) + + def test_comments(self): + template = """ +<style> + #someselector + # other non comment stuff +</style> +## a comment + +# also not a comment + + ## this is a comment + +this is ## not a comment + +<%doc> multiline +comment +</%doc> + +hi +""" + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text( + """\n<style>\n #someselector\n # """ + """other non comment stuff\n</style>\n""", + (1, 1), + ), + Comment("a comment", (6, 1)), + Text("""\n# also not a comment\n\n""", (7, 1)), + Comment("this is a comment", (10, 1)), + Text("""\nthis is ## not a comment\n\n""", (11, 1)), + Comment(""" multiline\ncomment\n""", (14, 1)), + Text( + """ + +hi +""", + (16, 8), + ), + ], + ), + ) + + def test_docs(self): + template = """ + <%doc> + this is a comment + </%doc> + <%def name="foo()"> + <%doc> + this is the foo func + </%doc> + </%def> + """ + nodes = Lexer(template).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text("\n ", (1, 1)), + Comment( + """\n this is a comment\n """, (2, 9) + ), + Text("\n ", (4, 16)), + DefTag( + "def", + {"name": "foo()"}, + (5, 9), + [ + Text("\n ", (5, 28)), + Comment( + """\n this is the foo func\n""" + """ """, + (6, 13), + ), + Text("\n ", (8, 20)), + ], + ), + Text("\n ", (9, 16)), + ], + ), + ) + + def test_preprocess(self): + def preproc(text): + return re.sub(r"(?<=\n)\s*#[^#]", "##", text) + + template = """ + hi + # old style comment +# another comment +""" + nodes = Lexer(template, preprocessor=preproc).parse() + self._compare( + nodes, + TemplateNode( + {}, + [ + Text("""\n hi\n""", (1, 1)), + Comment("old style comment", (3, 1)), + Comment("another comment", (4, 1)), + ], + ), + ) diff --git a/test/test_lookup.py b/test/test_lookup.py new file mode 100644 index 0000000..6a797d7 --- /dev/null +++ b/test/test_lookup.py @@ -0,0 +1,150 @@ +import os +import tempfile + +from mako import exceptions +from mako import lookup +from mako import runtime +from mako.template import Template +from mako.testing.assertions import assert_raises_message +from mako.testing.assertions import assert_raises_with_given_cause +from mako.testing.config import config +from mako.testing.helpers import file_with_template_code +from mako.testing.helpers import replace_file_with_dir +from mako.testing.helpers import result_lines +from mako.testing.helpers import rewind_compile_time +from mako.util import FastEncodingBuffer + +tl = lookup.TemplateLookup(directories=[config.template_base]) + + +class LookupTest: + def test_basic(self): + t = tl.get_template("index.html") + assert result_lines(t.render()) == ["this is index"] + + def test_subdir(self): + t = tl.get_template("/subdir/index.html") + assert result_lines(t.render()) == [ + "this is sub index", + "this is include 2", + ] + + assert ( + tl.get_template("/subdir/index.html").module_id + == "_subdir_index_html" + ) + + def test_updir(self): + t = tl.get_template("/subdir/foo/../bar/../index.html") + assert result_lines(t.render()) == [ + "this is sub index", + "this is include 2", + ] + + def test_directory_lookup(self): + """test that hitting an existent directory still raises + LookupError.""" + + assert_raises_with_given_cause( + exceptions.TopLevelLookupException, + KeyError, + tl.get_template, + "/subdir", + ) + + def test_no_lookup(self): + t = Template("hi <%include file='foo.html'/>") + + assert_raises_message( + exceptions.TemplateLookupException, + "Template 'memory:%s' has no TemplateLookup associated" + % hex(id(t)), + t.render, + ) + + def test_uri_adjust(self): + tl = lookup.TemplateLookup(directories=["/foo/bar"]) + assert ( + tl.filename_to_uri("/foo/bar/etc/lala/index.html") + == "/etc/lala/index.html" + ) + + tl = lookup.TemplateLookup(directories=["./foo/bar"]) + assert ( + tl.filename_to_uri("./foo/bar/etc/index.html") == "/etc/index.html" + ) + + def test_uri_cache(self): + """test that the _uri_cache dictionary is available""" + tl._uri_cache[("foo", "bar")] = "/some/path" + assert tl._uri_cache[("foo", "bar")] == "/some/path" + + def test_check_not_found(self): + tl = lookup.TemplateLookup() + tl.put_string("foo", "this is a template") + f = tl.get_template("foo") + assert f.uri in tl._collection + f.filename = "nonexistent" + assert_raises_with_given_cause( + exceptions.TemplateLookupException, + FileNotFoundError, + tl.get_template, + "foo", + ) + assert f.uri not in tl._collection + + def test_dont_accept_relative_outside_of_root(self): + """test the mechanics of an include where + the include goes outside of the path""" + tl = lookup.TemplateLookup( + directories=[os.path.join(config.template_base, "subdir")] + ) + index = tl.get_template("index.html") + + ctx = runtime.Context(FastEncodingBuffer()) + ctx._with_template = index + + assert_raises_message( + exceptions.TemplateLookupException, + 'Template uri "../index.html" is invalid - it ' + "cannot be relative outside of the root path", + runtime._lookup_template, + ctx, + "../index.html", + index.uri, + ) + + assert_raises_message( + exceptions.TemplateLookupException, + 'Template uri "../othersubdir/foo.html" is invalid - it ' + "cannot be relative outside of the root path", + runtime._lookup_template, + ctx, + "../othersubdir/foo.html", + index.uri, + ) + + # this is OK since the .. cancels out + runtime._lookup_template(ctx, "foo/../index.html", index.uri) + + def test_checking_against_bad_filetype(self): + with tempfile.TemporaryDirectory() as tempdir: + tl = lookup.TemplateLookup(directories=[tempdir]) + index_file = file_with_template_code( + os.path.join(tempdir, "index.html") + ) + + with rewind_compile_time(): + tmpl = Template(filename=index_file) + + tl.put_template("index.html", tmpl) + + replace_file_with_dir(index_file) + + assert_raises_with_given_cause( + exceptions.TemplateLookupException, + OSError, + tl._check, + "index.html", + tl._collection["index.html"], + ) diff --git a/test/test_loop.py b/test/test_loop.py new file mode 100644 index 0000000..2c11000 --- /dev/null +++ b/test/test_loop.py @@ -0,0 +1,336 @@ +import re +import unittest + +from mako import exceptions +from mako.codegen import _FOR_LOOP +from mako.lookup import TemplateLookup +from mako.runtime import LoopContext +from mako.runtime import LoopStack +from mako.template import Template +from mako.testing.assertions import assert_raises_message +from mako.testing.fixtures import TemplateTest +from mako.testing.helpers import flatten_result + + +class TestLoop(unittest.TestCase): + def test__FOR_LOOP(self): + for statement, target_list, expression_list in ( + ("for x in y:", "x", "y"), + ("for x, y in z:", "x, y", "z"), + ("for (x,y) in z:", "(x,y)", "z"), + ("for ( x, y, z) in a:", "( x, y, z)", "a"), + ("for x in [1, 2, 3]:", "x", "[1, 2, 3]"), + ('for x in "spam":', "x", '"spam"'), + ( + "for k,v in dict(a=1,b=2).items():", + "k,v", + "dict(a=1,b=2).items()", + ), + ( + "for x in [y+1 for y in [1, 2, 3]]:", + "x", + "[y+1 for y in [1, 2, 3]]", + ), + ( + "for ((key1, val1), (key2, val2)) in pairwise(dict.items()):", + "((key1, val1), (key2, val2))", + "pairwise(dict.items())", + ), + ( + "for (key1, val1), (key2, val2) in pairwise(dict.items()):", + "(key1, val1), (key2, val2)", + "pairwise(dict.items())", + ), + ): + match = _FOR_LOOP.match(statement) + assert match and match.groups() == (target_list, expression_list) + + def test_no_loop(self): + template = Template( + """% for x in 'spam': +${x} +% endfor""" + ) + code = template.code + assert not re.match(r"loop = __M_loop._enter\(:", code), ( + "No need to " + "generate a loop context if the loop variable wasn't accessed" + ) + print(template.render()) + + def test_loop_demo(self): + template = Template( + """x|index|reverse_index|first|last|cycle|even|odd +% for x in 'ham': +${x}|${loop.index}|${loop.reverse_index}|${loop.first}|""" + """${loop.last}|${loop.cycle('even', 'odd')}|""" + """${loop.even}|${loop.odd} +% endfor""" + ) + expected = [ + "x|index|reverse_index|first|last|cycle|even|odd", + "h|0|2|True|False|even|True|False", + "a|1|1|False|False|odd|False|True", + "m|2|0|False|True|even|True|False", + ] + code = template.code + assert "loop = __M_loop._enter(" in code, ( + "Generated a loop context since " "the loop variable was accessed" + ) + rendered = template.render() + print(rendered) + for line in expected: + assert line in rendered, ( + "Loop variables give information about " + "the progress of the loop" + ) + + def test_nested_loops(self): + template = Template( + """% for x in 'ab': +${x} ${loop.index} <- start in outer loop +% for y in [0, 1]: +${y} ${loop.index} <- go to inner loop +% endfor +${x} ${loop.index} <- back to outer loop +% endfor""" + ) + rendered = template.render() + expected = [ + "a 0 <- start in outer loop", + "0 0 <- go to inner loop", + "1 1 <- go to inner loop", + "a 0 <- back to outer loop", + "b 1 <- start in outer loop", + "0 0 <- go to inner loop", + "1 1 <- go to inner loop", + "b 1 <- back to outer loop", + ] + for line in expected: + assert line in rendered, ( + "The LoopStack allows you to take " + "advantage of the loop variable even in embedded loops" + ) + + def test_parent_loops(self): + template = Template( + """% for x in 'ab': +${x} ${loop.index} <- outer loop +% for y in [0, 1]: +${y} ${loop.index} <- inner loop +${x} ${loop.parent.index} <- parent loop +% endfor +${x} ${loop.index} <- outer loop +% endfor""" + ) + code = template.code + rendered = template.render() + expected = [ + "a 0 <- outer loop", + "a 0 <- parent loop", + "b 1 <- outer loop", + "b 1 <- parent loop", + ] + for line in expected: + print(code) + assert line in rendered, ( + "The parent attribute of a loop gives " + "you the previous loop context in the stack" + ) + + def test_out_of_context_access(self): + template = Template("""${loop.index}""") + assert_raises_message( + exceptions.RuntimeException, + "No loop context is established", + template.render, + ) + + +class TestLoopStack(unittest.TestCase): + def setUp(self): + self.stack = LoopStack() + self.bottom = "spam" + self.stack.stack = [self.bottom] + + def test_enter(self): + iterable = "ham" + s = self.stack._enter(iterable) + assert s is self.stack.stack[-1], ( + "Calling the stack with an iterable returns " "the stack" + ) + assert iterable == self.stack.stack[-1]._iterable, ( + "and pushes the " "iterable on the top of the stack" + ) + + def test__top(self): + assert self.bottom == self.stack._top, ( + "_top returns the last item " "on the stack" + ) + + def test__pop(self): + assert len(self.stack.stack) == 1 + top = self.stack._pop() + assert top == self.bottom + assert len(self.stack.stack) == 0 + + def test__push(self): + assert len(self.stack.stack) == 1 + iterable = "ham" + self.stack._push(iterable) + assert len(self.stack.stack) == 2 + assert iterable is self.stack._top._iterable + + def test_exit(self): + iterable = "ham" + self.stack._enter(iterable) + before = len(self.stack.stack) + self.stack._exit() + after = len(self.stack.stack) + assert before == (after + 1), "Exiting a context pops the stack" + + +class TestLoopContext(unittest.TestCase): + def setUp(self): + self.iterable = [1, 2, 3] + self.ctx = LoopContext(self.iterable) + + def test___len__(self): + assert len(self.iterable) == len(self.ctx), ( + "The LoopContext is the " "same length as the iterable" + ) + + def test_index(self): + expected = tuple(range(len(self.iterable))) + actual = tuple(self.ctx.index for i in self.ctx) + assert expected == actual, ( + "The index is consistent with the current " "iteration count" + ) + + def test_reverse_index(self): + length = len(self.iterable) + expected = tuple(length - i - 1 for i in range(length)) + actual = tuple(self.ctx.reverse_index for i in self.ctx) + print(expected, actual) + assert expected == actual, ( + "The reverse_index is the number of " "iterations until the end" + ) + + def test_first(self): + expected = (True, False, False) + actual = tuple(self.ctx.first for i in self.ctx) + assert expected == actual, "first is only true on the first iteration" + + def test_last(self): + expected = (False, False, True) + actual = tuple(self.ctx.last for i in self.ctx) + assert expected == actual, "last is only true on the last iteration" + + def test_even(self): + expected = (True, False, True) + actual = tuple(self.ctx.even for i in self.ctx) + assert expected == actual, "even is true on even iterations" + + def test_odd(self): + expected = (False, True, False) + actual = tuple(self.ctx.odd for i in self.ctx) + assert expected == actual, "odd is true on odd iterations" + + def test_cycle(self): + expected = ("a", "b", "a") + actual = tuple(self.ctx.cycle("a", "b") for i in self.ctx) + assert expected == actual, "cycle endlessly cycles through the values" + + +class TestLoopFlags(TemplateTest): + def test_loop_disabled_template(self): + self._do_memory_test( + """ + the loop: ${loop} + """, + "the loop: hi", + template_args=dict(loop="hi"), + filters=flatten_result, + enable_loop=False, + ) + + def test_loop_disabled_lookup(self): + l = TemplateLookup(enable_loop=False) + l.put_string( + "x", + """ + the loop: ${loop} + """, + ) + + self._do_test( + l.get_template("x"), + "the loop: hi", + template_args=dict(loop="hi"), + filters=flatten_result, + ) + + def test_loop_disabled_override_template(self): + self._do_memory_test( + """ + <%page enable_loop="True" /> + % for i in (1, 2, 3): + ${i} ${loop.index} + % endfor + """, + "1 0 2 1 3 2", + template_args=dict(loop="hi"), + filters=flatten_result, + enable_loop=False, + ) + + def test_loop_disabled_override_lookup(self): + l = TemplateLookup(enable_loop=False) + l.put_string( + "x", + """ + <%page enable_loop="True" /> + % for i in (1, 2, 3): + ${i} ${loop.index} + % endfor + """, + ) + + self._do_test( + l.get_template("x"), + "1 0 2 1 3 2", + template_args=dict(loop="hi"), + filters=flatten_result, + ) + + def test_loop_enabled_override_template(self): + self._do_memory_test( + """ + <%page enable_loop="True" /> + % for i in (1, 2, 3): + ${i} ${loop.index} + % endfor + """, + "1 0 2 1 3 2", + template_args=dict(), + filters=flatten_result, + ) + + def test_loop_enabled_override_lookup(self): + l = TemplateLookup() + l.put_string( + "x", + """ + <%page enable_loop="True" /> + % for i in (1, 2, 3): + ${i} ${loop.index} + % endfor + """, + ) + + self._do_test( + l.get_template("x"), + "1 0 2 1 3 2", + template_args=dict(), + filters=flatten_result, + ) diff --git a/test/test_lru.py b/test/test_lru.py new file mode 100644 index 0000000..f54bd15 --- /dev/null +++ b/test/test_lru.py @@ -0,0 +1,39 @@ +from mako.util import LRUCache + + +class item: + def __init__(self, id_): + self.id = id_ + + def __str__(self): + return "item id %d" % self.id + + +class LRUTest: + def testlru(self): + l = LRUCache(10, threshold=0.2) + + for id_ in range(1, 20): + l[id_] = item(id_) + + # first couple of items should be gone + assert 1 not in l + assert 2 not in l + + # next batch over the threshold of 10 should be present + for id_ in range(11, 20): + assert id_ in l + + l[12] + l[15] + l[23] = item(23) + l[24] = item(24) + l[25] = item(25) + l[26] = item(26) + l[27] = item(27) + + assert 11 not in l + assert 13 not in l + + for id_ in (25, 24, 23, 14, 12, 19, 18, 17, 16, 15): + assert id_ in l diff --git a/test/test_namespace.py b/test/test_namespace.py new file mode 100644 index 0000000..b6b0544 --- /dev/null +++ b/test/test_namespace.py @@ -0,0 +1,1031 @@ +from mako import exceptions +from mako import lookup +from mako.template import Template +from mako.testing.assertions import assert_raises +from mako.testing.assertions import assert_raises_message_with_given_cause +from mako.testing.assertions import eq_ +from mako.testing.fixtures import TemplateTest +from mako.testing.helpers import flatten_result +from mako.testing.helpers import result_lines + + +class NamespaceTest(TemplateTest): + def test_inline_crossreference(self): + self._do_memory_test( + """ + <%namespace name="x"> + <%def name="a()"> + this is x a + </%def> + <%def name="b()"> + this is x b, and heres ${a()} + </%def> + </%namespace> + + ${x.a()} + + ${x.b()} + """, + "this is x a this is x b, and heres this is x a", + filters=flatten_result, + ) + + def test_inline_assignment(self): + self._do_memory_test( + """ + <%namespace name="x"> + <%def name="a()"> + <% + x = 5 + %> + this is x: ${x} + </%def> + </%namespace> + + ${x.a()} + + """, + "this is x: 5", + filters=flatten_result, + ) + + def test_inline_arguments(self): + self._do_memory_test( + """ + <%namespace name="x"> + <%def name="a(x, y)"> + <% + result = x * y + %> + result: ${result} + </%def> + </%namespace> + + ${x.a(5, 10)} + + """, + "result: 50", + filters=flatten_result, + ) + + def test_inline_not_duped(self): + self._do_memory_test( + """ + <%namespace name="x"> + <%def name="a()"> + foo + </%def> + </%namespace> + + <% + assert x.a is not UNDEFINED, "namespace x.a wasn't defined" + assert a is UNDEFINED, "name 'a' is in the body locals" + %> + + """, + "", + filters=flatten_result, + ) + + def test_dynamic(self): + collection = lookup.TemplateLookup() + + collection.put_string( + "a", + """ + <%namespace name="b" file="${context['b_def']}"/> + + a. b: ${b.body()} +""", + ) + + collection.put_string( + "b", + """ + b. +""", + ) + + eq_( + flatten_result(collection.get_template("a").render(b_def="b")), + "a. b: b.", + ) + + def test_template(self): + collection = lookup.TemplateLookup() + + collection.put_string( + "main.html", + """ + <%namespace name="comp" file="defs.html"/> + + this is main. ${comp.def1("hi")} + ${comp.def2("there")} +""", + ) + + collection.put_string( + "defs.html", + """ + <%def name="def1(s)"> + def1: ${s} + </%def> + + <%def name="def2(x)"> + def2: ${x} + </%def> +""", + ) + + assert ( + flatten_result(collection.get_template("main.html").render()) + == "this is main. def1: hi def2: there" + ) + + def test_module(self): + collection = lookup.TemplateLookup() + + collection.put_string( + "main.html", + """ + <%namespace name="comp" module="test.sample_module_namespace"/> + + this is main. ${comp.foo1()} + ${comp.foo2("hi")} +""", + ) + + assert ( + flatten_result(collection.get_template("main.html").render()) + == "this is main. this is foo1. this is foo2, x is hi" + ) + + def test_module_2(self): + collection = lookup.TemplateLookup() + + collection.put_string( + "main.html", + """ + <%namespace name="comp" module="test.foo.test_ns"/> + + this is main. ${comp.foo1()} + ${comp.foo2("hi")} +""", + ) + + assert ( + flatten_result(collection.get_template("main.html").render()) + == "this is main. this is foo1. this is foo2, x is hi" + ) + + def test_module_imports(self): + collection = lookup.TemplateLookup() + + collection.put_string( + "main.html", + """ + <%namespace import="*" module="test.foo.test_ns"/> + + this is main. ${foo1()} + ${foo2("hi")} +""", + ) + + assert ( + flatten_result(collection.get_template("main.html").render()) + == "this is main. this is foo1. this is foo2, x is hi" + ) + + def test_module_imports_2(self): + collection = lookup.TemplateLookup() + + collection.put_string( + "main.html", + """ + <%namespace import="foo1, foo2" module="test.foo.test_ns"/> + + this is main. ${foo1()} + ${foo2("hi")} +""", + ) + + assert ( + flatten_result(collection.get_template("main.html").render()) + == "this is main. this is foo1. this is foo2, x is hi" + ) + + def test_context(self): + """test that namespace callables get access to the current context""" + collection = lookup.TemplateLookup() + + collection.put_string( + "main.html", + """ + <%namespace name="comp" file="defs.html"/> + + this is main. ${comp.def1()} + ${comp.def2("there")} +""", + ) + + collection.put_string( + "defs.html", + """ + <%def name="def1()"> + def1: x is ${x} + </%def> + + <%def name="def2(x)"> + def2: x is ${x} + </%def> +""", + ) + + assert ( + flatten_result( + collection.get_template("main.html").render(x="context x") + ) + == "this is main. def1: x is context x def2: x is there" + ) + + def test_overload(self): + collection = lookup.TemplateLookup() + + collection.put_string( + "main.html", + """ + <%namespace name="comp" file="defs.html"> + <%def name="def1(x, y)"> + overridden def1 ${x}, ${y} + </%def> + </%namespace> + + this is main. ${comp.def1("hi", "there")} + ${comp.def2("there")} + """, + ) + + collection.put_string( + "defs.html", + """ + <%def name="def1(s)"> + def1: ${s} + </%def> + + <%def name="def2(x)"> + def2: ${x} + </%def> + """, + ) + + assert ( + flatten_result(collection.get_template("main.html").render()) + == "this is main. overridden def1 hi, there def2: there" + ) + + def test_getattr(self): + collection = lookup.TemplateLookup() + collection.put_string( + "main.html", + """ + <%namespace name="foo" file="ns.html"/> + <% + if hasattr(foo, 'lala'): + foo.lala() + if not hasattr(foo, 'hoho'): + context.write('foo has no hoho.') + %> + """, + ) + collection.put_string( + "ns.html", + """ + <%def name="lala()">this is lala.</%def> + """, + ) + assert ( + flatten_result(collection.get_template("main.html").render()) + == "this is lala.foo has no hoho." + ) + + def test_in_def(self): + collection = lookup.TemplateLookup() + collection.put_string( + "main.html", + """ + <%namespace name="foo" file="ns.html"/> + + this is main. ${bar()} + <%def name="bar()"> + this is bar, foo is ${foo.bar()} + </%def> + """, + ) + + collection.put_string( + "ns.html", + """ + <%def name="bar()"> + this is ns.html->bar + </%def> + """, + ) + + assert result_lines(collection.get_template("main.html").render()) == [ + "this is main.", + "this is bar, foo is", + "this is ns.html->bar", + ] + + def test_in_remote_def(self): + collection = lookup.TemplateLookup() + collection.put_string( + "main.html", + """ + <%namespace name="foo" file="ns.html"/> + + this is main. ${bar()} + <%def name="bar()"> + this is bar, foo is ${foo.bar()} + </%def> + """, + ) + + collection.put_string( + "ns.html", + """ + <%def name="bar()"> + this is ns.html->bar + </%def> + """, + ) + + collection.put_string( + "index.html", + """ + <%namespace name="main" file="main.html"/> + + this is index + ${main.bar()} + """, + ) + + assert result_lines( + collection.get_template("index.html").render() + ) == ["this is index", "this is bar, foo is", "this is ns.html->bar"] + + def test_dont_pollute_self(self): + # test that get_namespace() doesn't modify the original context + # incompatibly + + collection = lookup.TemplateLookup() + collection.put_string( + "base.html", + """ + + <%def name="foo()"> + <% + foo = local.get_namespace("foo.html") + %> + </%def> + + name: ${self.name} + name via bar: ${bar()} + + ${next.body()} + + name: ${self.name} + name via bar: ${bar()} + <%def name="bar()"> + ${self.name} + </%def> + + + """, + ) + + collection.put_string( + "page.html", + """ + <%inherit file="base.html"/> + + ${self.foo()} + + hello world + + """, + ) + + collection.put_string("foo.html", """<%inherit file="base.html"/>""") + assert result_lines(collection.get_template("page.html").render()) == [ + "name: self:page.html", + "name via bar:", + "self:page.html", + "hello world", + "name: self:page.html", + "name via bar:", + "self:page.html", + ] + + def test_inheritance(self): + """test namespace initialization in a base inherited template that + doesnt otherwise access the namespace""" + collection = lookup.TemplateLookup() + collection.put_string( + "base.html", + """ + <%namespace name="foo" file="ns.html" inheritable="True"/> + + ${next.body()} +""", + ) + collection.put_string( + "ns.html", + """ + <%def name="bar()"> + this is ns.html->bar + </%def> + """, + ) + + collection.put_string( + "index.html", + """ + <%inherit file="base.html"/> + + this is index + ${self.foo.bar()} + """, + ) + + assert result_lines( + collection.get_template("index.html").render() + ) == ["this is index", "this is ns.html->bar"] + + def test_inheritance_two(self): + collection = lookup.TemplateLookup() + collection.put_string( + "base.html", + """ + <%def name="foo()"> + base.foo + </%def> + + <%def name="bat()"> + base.bat + </%def> +""", + ) + collection.put_string( + "lib.html", + """ + <%inherit file="base.html"/> + <%def name="bar()"> + lib.bar + ${parent.foo()} + ${self.foo()} + ${parent.bat()} + ${self.bat()} + </%def> + + <%def name="foo()"> + lib.foo + </%def> + + """, + ) + + collection.put_string( + "front.html", + """ + <%namespace name="lib" file="lib.html"/> + ${lib.bar()} + """, + ) + + assert result_lines( + collection.get_template("front.html").render() + ) == ["lib.bar", "base.foo", "lib.foo", "base.bat", "base.bat"] + + def test_attr(self): + l = lookup.TemplateLookup() + + l.put_string( + "foo.html", + """ + <%! + foofoo = "foo foo" + onlyfoo = "only foo" + %> + <%inherit file="base.html"/> + <%def name="setup()"> + <% + self.attr.foolala = "foo lala" + %> + </%def> + ${self.attr.basefoo} + ${self.attr.foofoo} + ${self.attr.onlyfoo} + ${self.attr.lala} + ${self.attr.foolala} + """, + ) + + l.put_string( + "base.html", + """ + <%! + basefoo = "base foo 1" + foofoo = "base foo 2" + %> + <% + self.attr.lala = "base lala" + %> + + ${self.attr.basefoo} + ${self.attr.foofoo} + ${self.attr.onlyfoo} + ${self.attr.lala} + ${self.setup()} + ${self.attr.foolala} + body + ${self.body()} + """, + ) + + assert result_lines(l.get_template("foo.html").render()) == [ + "base foo 1", + "foo foo", + "only foo", + "base lala", + "foo lala", + "body", + "base foo 1", + "foo foo", + "only foo", + "base lala", + "foo lala", + ] + + def test_attr_raise(self): + l = lookup.TemplateLookup() + + l.put_string( + "foo.html", + """ + <%def name="foo()"> + </%def> + """, + ) + + l.put_string( + "bar.html", + """ + <%namespace name="foo" file="foo.html"/> + + ${foo.notfoo()} + """, + ) + + assert_raises(AttributeError, l.get_template("bar.html").render) + + def test_custom_tag_1(self): + template = Template( + """ + + <%def name="foo(x, y)"> + foo: ${x} ${y} + </%def> + + <%self:foo x="5" y="${7+8}"/> + """ + ) + assert result_lines(template.render()) == ["foo: 5 15"] + + def test_custom_tag_2(self): + collection = lookup.TemplateLookup() + collection.put_string( + "base.html", + """ + <%def name="foo(x, y)"> + foo: ${x} ${y} + </%def> + + <%def name="bat(g)"><% + return "the bat! %s" % g + %></%def> + + <%def name="bar(x)"> + ${caller.body(z=x)} + </%def> + """, + ) + + collection.put_string( + "index.html", + """ + <%namespace name="myns" file="base.html"/> + + <%myns:foo x="${'some x'}" y="some y"/> + + <%myns:bar x="${myns.bat(10)}" args="z"> + record: ${z} + </%myns:bar> + + """, + ) + + assert result_lines( + collection.get_template("index.html").render() + ) == ["foo: some x some y", "record: the bat! 10"] + + def test_custom_tag_3(self): + collection = lookup.TemplateLookup() + collection.put_string( + "base.html", + """ + <%namespace name="foo" file="ns.html" inheritable="True"/> + + ${next.body()} + """, + ) + collection.put_string( + "ns.html", + """ + <%def name="bar()"> + this is ns.html->bar + caller body: ${caller.body()} + </%def> + """, + ) + + collection.put_string( + "index.html", + """ + <%inherit file="base.html"/> + + this is index + <%self.foo:bar> + call body + </%self.foo:bar> + """, + ) + + assert result_lines( + collection.get_template("index.html").render() + ) == [ + "this is index", + "this is ns.html->bar", + "caller body:", + "call body", + ] + + def test_custom_tag_case_sensitive(self): + t = Template( + """ + <%def name="renderPanel()"> + panel ${caller.body()} + </%def> + + <%def name="renderTablePanel()"> + <%self:renderPanel> + hi + </%self:renderPanel> + </%def> + + <%self:renderTablePanel/> + """ + ) + assert result_lines(t.render()) == ["panel", "hi"] + + def test_expr_grouping(self): + """test that parenthesis are placed around string-embedded + expressions.""" + + template = Template( + """ + <%def name="bar(x, y)"> + ${x} + ${y} + </%def> + + <%self:bar x=" ${foo} " y="x${g and '1' or '2'}y"/> + """, + input_encoding="utf-8", + ) + + # the concat has to come out as "x + (g and '1' or '2') + y" + assert result_lines(template.render(foo="this is foo", g=False)) == [ + "this is foo", + "x2y", + ] + + def test_ccall(self): + collection = lookup.TemplateLookup() + collection.put_string( + "base.html", + """ + <%namespace name="foo" file="ns.html" inheritable="True"/> + + ${next.body()} + """, + ) + collection.put_string( + "ns.html", + """ + <%def name="bar()"> + this is ns.html->bar + caller body: ${caller.body()} + </%def> + """, + ) + + collection.put_string( + "index.html", + """ + <%inherit file="base.html"/> + + this is index + <%call expr="self.foo.bar()"> + call body + </%call> + """, + ) + + assert result_lines( + collection.get_template("index.html").render() + ) == [ + "this is index", + "this is ns.html->bar", + "caller body:", + "call body", + ] + + def test_ccall_2(self): + collection = lookup.TemplateLookup() + collection.put_string( + "base.html", + """ + <%namespace name="foo" file="ns1.html" inheritable="True"/> + + ${next.body()} + """, + ) + collection.put_string( + "ns1.html", + """ + <%namespace name="foo2" file="ns2.html"/> + <%def name="bar()"> + <%call expr="foo2.ns2_bar()"> + this is ns1.html->bar + caller body: ${caller.body()} + </%call> + </%def> + """, + ) + + collection.put_string( + "ns2.html", + """ + <%def name="ns2_bar()"> + this is ns2.html->bar + caller body: ${caller.body()} + </%def> + """, + ) + + collection.put_string( + "index.html", + """ + <%inherit file="base.html"/> + + this is index + <%call expr="self.foo.bar()"> + call body + </%call> + """, + ) + + assert result_lines( + collection.get_template("index.html").render() + ) == [ + "this is index", + "this is ns2.html->bar", + "caller body:", + "this is ns1.html->bar", + "caller body:", + "call body", + ] + + def test_import(self): + collection = lookup.TemplateLookup() + collection.put_string( + "functions.html", + """ + <%def name="foo()"> + this is foo + </%def> + + <%def name="bar()"> + this is bar + </%def> + + <%def name="lala()"> + this is lala + </%def> + """, + ) + + collection.put_string( + "func2.html", + """ + <%def name="a()"> + this is a + </%def> + <%def name="b()"> + this is b + </%def> + """, + ) + collection.put_string( + "index.html", + """ + <%namespace file="functions.html" import="*"/> + <%namespace file="func2.html" import="a, b"/> + ${foo()} + ${bar()} + ${lala()} + ${a()} + ${b()} + ${x} + """, + ) + + assert result_lines( + collection.get_template("index.html").render( + bar="this is bar", x="this is x" + ) + ) == [ + "this is foo", + "this is bar", + "this is lala", + "this is a", + "this is b", + "this is x", + ] + + def test_import_calledfromdef(self): + l = lookup.TemplateLookup() + l.put_string( + "a", + """ + <%def name="table()"> + im table + </%def> + """, + ) + + l.put_string( + "b", + """ + <%namespace file="a" import="table"/> + + <% + def table2(): + table() + return "" + %> + + ${table2()} + """, + ) + + t = l.get_template("b") + assert flatten_result(t.render()) == "im table" + + def test_closure_import(self): + collection = lookup.TemplateLookup() + collection.put_string( + "functions.html", + """ + <%def name="foo()"> + this is foo + </%def> + + <%def name="bar()"> + this is bar + </%def> + """, + ) + + collection.put_string( + "index.html", + """ + <%namespace file="functions.html" import="*"/> + <%def name="cl1()"> + ${foo()} + </%def> + + <%def name="cl2()"> + ${bar()} + </%def> + + ${cl1()} + ${cl2()} + """, + ) + assert result_lines( + collection.get_template("index.html").render( + bar="this is bar", x="this is x" + ) + ) == ["this is foo", "this is bar"] + + def test_import_local(self): + t = Template( + """ + <%namespace import="*"> + <%def name="foo()"> + this is foo + </%def> + </%namespace> + + ${foo()} + + """ + ) + assert flatten_result(t.render()) == "this is foo" + + def test_ccall_import(self): + collection = lookup.TemplateLookup() + collection.put_string( + "functions.html", + """ + <%def name="foo()"> + this is foo + </%def> + + <%def name="bar()"> + this is bar. + ${caller.body()} + ${caller.lala()} + </%def> + """, + ) + + collection.put_string( + "index.html", + """ + <%namespace name="func" file="functions.html" import="*"/> + <%call expr="bar()"> + this is index embedded + foo is ${foo()} + <%def name="lala()"> + this is lala ${foo()} + </%def> + </%call> + """, + ) + # print collection.get_template("index.html").code + # print collection.get_template("functions.html").code + assert result_lines( + collection.get_template("index.html").render() + ) == [ + "this is bar.", + "this is index embedded", + "foo is", + "this is foo", + "this is lala", + "this is foo", + ] + + def test_nonexistent_namespace_uri(self): + collection = lookup.TemplateLookup() + collection.put_string( + "main.html", + """ + <%namespace name="defs" file="eefs.html"/> + + this is main. ${defs.def1("hi")} + ${defs.def2("there")} +""", + ) + + collection.put_string( + "defs.html", + """ + <%def name="def1(s)"> + def1: ${s} + </%def> + + <%def name="def2(x)"> + def2: ${x} + </%def> +""", + ) + + assert_raises_message_with_given_cause( + exceptions.TemplateLookupException, + "Can't locate template for uri 'eefs.html", + exceptions.TopLevelLookupException, + collection.get_template("main.html").render, + ) diff --git a/test/test_pygen.py b/test/test_pygen.py new file mode 100644 index 0000000..8adc142 --- /dev/null +++ b/test/test_pygen.py @@ -0,0 +1,277 @@ +from io import StringIO + +from mako.pygen import adjust_whitespace +from mako.pygen import PythonPrinter +from mako.testing.assertions import eq_ + + +class GeneratePythonTest: + def test_generate_normal(self): + stream = StringIO() + printer = PythonPrinter(stream) + printer.writeline("import lala") + printer.writeline("for x in foo:") + printer.writeline("print x") + printer.writeline(None) + printer.writeline("print y") + assert ( + stream.getvalue() + == """import lala +for x in foo: + print x +print y +""" + ) + + def test_generate_adjusted(self): + block = """ + x = 5 +6 + if x > 7: + for y in range(1,5): + print "<td>%s</td>" % y +""" + stream = StringIO() + printer = PythonPrinter(stream) + printer.write_indented_block(block) + printer.close() + # print stream.getvalue() + assert ( + stream.getvalue() + == """ +x = 5 +6 +if x > 7: + for y in range(1,5): + print "<td>%s</td>" % y + +""" + ) + + def test_generate_combo(self): + block = """ + x = 5 +6 + if x > 7: + for y in range(1,5): + print "<td>%s</td>" % y + print "hi" + print "there" + foo(lala) +""" + stream = StringIO() + printer = PythonPrinter(stream) + printer.writeline("import lala") + printer.writeline("for x in foo:") + printer.writeline("print x") + printer.write_indented_block(block) + printer.writeline(None) + printer.writeline("print y") + printer.close() + # print "->" + stream.getvalue().replace(' ', '#') + "<-" + eq_( + stream.getvalue(), + """import lala +for x in foo: + print x + + x = 5 +6 + if x > 7: + for y in range(1,5): + print "<td>%s</td>" % y + print "hi" + print "there" + foo(lala) + +print y +""", + ) + + def test_multi_line(self): + block = """ + if test: + print ''' this is a block of stuff. +this is more stuff in the block. +and more block. +''' + do_more_stuff(g) +""" + stream = StringIO() + printer = PythonPrinter(stream) + printer.write_indented_block(block) + printer.close() + # print stream.getvalue() + assert ( + stream.getvalue() + == """ +if test: + print ''' this is a block of stuff. +this is more stuff in the block. +and more block. +''' + do_more_stuff(g) + +""" + ) + + def test_false_unindentor(self): + stream = StringIO() + printer = PythonPrinter(stream) + for line in [ + "try:", + "elsemyvar = 12", + "if True:", + "print 'hi'", + None, + "finally:", + "dosomething", + None, + ]: + printer.writeline(line) + + assert ( + stream.getvalue() + == """try: + elsemyvar = 12 + if True: + print 'hi' +finally: + dosomething +""" + ), stream.getvalue() + + def test_backslash_line(self): + block = """ + # comment + if test: + if (lala + hoho) + \\ +(foobar + blat) == 5: + print "hi" + print "more indent" +""" + stream = StringIO() + printer = PythonPrinter(stream) + printer.write_indented_block(block) + printer.close() + assert ( + stream.getvalue() + == """ + # comment +if test: + if (lala + hoho) + \\ +(foobar + blat) == 5: + print "hi" +print "more indent" + +""" + ) + + +class WhitespaceTest: + def test_basic(self): + text = """ + for x in range(0,15): + print x + print "hi" + """ + assert ( + adjust_whitespace(text) + == """ +for x in range(0,15): + print x +print "hi" +""" + ) + + def test_blank_lines(self): + text = """ + print "hi" # a comment + + # more comments + + print g +""" + assert ( + adjust_whitespace(text) + == """ +print "hi" # a comment + +# more comments + +print g +""" + ) + + def test_open_quotes_with_pound(self): + text = ''' + print """ this is text + # and this is text + # and this is too """ +''' + assert ( + adjust_whitespace(text) + == ''' +print """ this is text + # and this is text + # and this is too """ +''' + ) + + def test_quote_with_comments(self): + text = """ + print 'hi' + # this is a comment + # another comment + x = 7 # someone's '''comment + print ''' + there + ''' + # someone else's comment +""" + + assert ( + adjust_whitespace(text) + == """ +print 'hi' +# this is a comment +# another comment +x = 7 # someone's '''comment +print ''' + there + ''' +# someone else's comment +""" + ) + + def test_quotes_with_pound(self): + text = ''' + if True: + """#""" + elif False: + "bar" +''' + assert ( + adjust_whitespace(text) + == ''' +if True: + """#""" +elif False: + "bar" +''' + ) + + def test_quotes(self): + text = """ + print ''' aslkjfnas kjdfn +askdjfnaskfd fkasnf dknf sadkfjn asdkfjna sdakjn +asdkfjnads kfajns ''' + if x: + print y +""" + assert ( + adjust_whitespace(text) + == """ +print ''' aslkjfnas kjdfn +askdjfnaskfd fkasnf dknf sadkfjn asdkfjna sdakjn +asdkfjnads kfajns ''' +if x: + print y +""" + ) diff --git a/test/test_runtime.py b/test/test_runtime.py new file mode 100644 index 0000000..0d6fce3 --- /dev/null +++ b/test/test_runtime.py @@ -0,0 +1,19 @@ +"""Assorted runtime unit tests +""" +from mako import runtime +from mako.testing.assertions import eq_ + + +class ContextTest: + def test_locals_kwargs(self): + c = runtime.Context(None, foo="bar") + eq_(c.kwargs, {"foo": "bar"}) + + d = c._locals({"zig": "zag"}) + + # kwargs is the original args sent to the Context, + # it's intentionally kept separate from _data + eq_(c.kwargs, {"foo": "bar"}) + eq_(d.kwargs, {"foo": "bar"}) + + eq_(d._data["zig"], "zag") diff --git a/test/test_template.py b/test/test_template.py new file mode 100644 index 0000000..62fd21d --- /dev/null +++ b/test/test_template.py @@ -0,0 +1,1669 @@ +import os + +from mako import exceptions +from mako import runtime +from mako import util +from mako.ext.preprocessors import convert_comments +from mako.lookup import TemplateLookup +from mako.template import ModuleInfo +from mako.template import ModuleTemplate +from mako.template import Template +from mako.testing.assertions import assert_raises +from mako.testing.assertions import assert_raises_message +from mako.testing.assertions import eq_ +from mako.testing.config import config +from mako.testing.fixtures import TemplateTest +from mako.testing.helpers import flatten_result +from mako.testing.helpers import result_lines + + +class ctx: + def __init__(self, a, b): + pass + + def __enter__(self): + return self + + def __exit__(self, *arg): + pass + + +class MiscTest(TemplateTest): + def test_crlf_linebreaks(self): + crlf = r""" +<% + foo = True + bar = True +%> +% if foo and \ + bar: + foo and bar +%endif +""" + crlf = crlf.replace("\n", "\r\n") + self._do_test(Template(crlf), "\r\n\r\n foo and bar\r\n") + + +class EncodingTest(TemplateTest): + def test_escapes_html_tags(self): + from mako.exceptions import html_error_template + + x = Template( + """ + X: + <% raise Exception('<span style="color:red">Foobar</span>') %> + """ + ) + + try: + x.render() + except: + # <h3>Exception: <span style="color:red">Foobar</span></h3> + markup = html_error_template().render(full=False, css=False) + assert ( + '<span style="color:red">Foobar</span></h3>'.encode("ascii") + not in markup + ) + assert ( + "<span style="color:red"" + ">Foobar</span>".encode("ascii") in markup + ) + + def test_unicode(self): + self._do_memory_test( + ( + "Alors vous imaginez ma surprise, au lever du jour, quand " + "une drôle de petite voix m’a réveillé. Elle disait: " + "« S’il vous plaît… dessine-moi un mouton! »" + ), + ( + "Alors vous imaginez ma surprise, au lever du jour, quand " + "une drôle de petite voix m’a réveillé. Elle disait: " + "« S’il vous plaît… dessine-moi un mouton! »" + ), + ) + + def test_encoding_doesnt_conflict(self): + self._do_memory_test( + ( + "Alors vous imaginez ma surprise, au lever du jour, quand " + "une drôle de petite voix m’a réveillé. Elle disait: " + "« S’il vous plaît… dessine-moi un mouton! »" + ), + ( + "Alors vous imaginez ma surprise, au lever du jour, quand " + "une drôle de petite voix m’a réveillé. Elle disait: " + "« S’il vous plaît… dessine-moi un mouton! »" + ), + output_encoding="utf-8", + ) + + def test_unicode_arg(self): + val = ( + "Alors vous imaginez ma surprise, au lever du jour, quand " + "une drôle de petite voix m’a réveillé. Elle disait: " + "« S’il vous plaît… dessine-moi un mouton! »" + ) + self._do_memory_test( + "${val}", + ( + "Alors vous imaginez ma surprise, au lever du jour, quand " + "une drôle de petite voix m’a réveillé. Elle disait: " + "« S’il vous plaît… dessine-moi un mouton! »" + ), + template_args={"val": val}, + ) + + def test_unicode_file(self): + self._do_file_test( + "unicode.html", + ( + "Alors vous imaginez ma surprise, au lever du jour, quand " + "une drôle de petite voix m’a réveillé. Elle disait: " + "« S’il vous plaît… dessine-moi un mouton! »" + ), + ) + + def test_unicode_file_code(self): + self._do_file_test( + "unicode_code.html", + ("""hi, drôle de petite voix m’a réveillé."""), + filters=flatten_result, + ) + + def test_unicode_file_lookup(self): + lookup = TemplateLookup( + directories=[config.template_base], + output_encoding="utf-8", + default_filters=["decode.utf8"], + ) + template = lookup.get_template("/chs_unicode_py3k.html") + eq_( + flatten_result(template.render_unicode(name="毛泽东")), + ("毛泽东 是 新中国的主席<br/> Welcome 你 to 北京."), + ) + + def test_unicode_bom(self): + self._do_file_test( + "bom.html", + ( + "Alors vous imaginez ma surprise, au lever du jour, quand " + "une drôle de petite voix m’a réveillé. Elle disait: " + "« S’il vous plaît… dessine-moi un mouton! »" + ), + ) + + self._do_file_test( + "bommagic.html", + ( + "Alors vous imaginez ma surprise, au lever du jour, quand " + "une drôle de petite voix m’a réveillé. Elle disait: " + "« S’il vous plaît… dessine-moi un mouton! »" + ), + ) + + assert_raises( + exceptions.CompileException, + Template, + filename=self._file_path("badbom.html"), + module_directory=config.module_base, + ) + + def test_unicode_memory(self): + val = ( + "Alors vous imaginez ma surprise, au lever du jour, quand " + "une drôle de petite voix m’a réveillé. Elle disait: " + "« S’il vous plaît… dessine-moi un mouton! »" + ) + self._do_memory_test( + ("## -*- coding: utf-8 -*-\n" + val).encode("utf-8"), + ( + "Alors vous imaginez ma surprise, au lever du jour, quand " + "une drôle de petite voix m’a réveillé. Elle disait: " + "« S’il vous plaît… dessine-moi un mouton! »" + ), + ) + + def test_unicode_text(self): + val = ( + "<%text>Alors vous imaginez ma surprise, au lever du jour, quand " + "une drôle de petite voix m’a réveillé. Elle disait: " + "« S’il vous plaît… dessine-moi un mouton! »</%text>" + ) + self._do_memory_test( + ("## -*- coding: utf-8 -*-\n" + val).encode("utf-8"), + ( + "Alors vous imaginez ma surprise, au lever du jour, quand " + "une drôle de petite voix m’a réveillé. Elle disait: " + "« S’il vous plaît… dessine-moi un mouton! »" + ), + ) + + def test_unicode_text_ccall(self): + val = """ + <%def name="foo()"> + ${capture(caller.body)} + </%def> + <%call expr="foo()"> + <%text>Alors vous imaginez ma surprise, au lever du jour, +quand une drôle de petite voix m’a réveillé. Elle disait: +« S’il vous plaît… dessine-moi un mouton! »</%text> + </%call>""" + self._do_memory_test( + ("## -*- coding: utf-8 -*-\n" + val).encode("utf-8"), + ( + "Alors vous imaginez ma surprise, au lever du jour, quand " + "une drôle de petite voix m’a réveillé. Elle disait: " + "« S’il vous plaît… dessine-moi un mouton! »" + ), + filters=flatten_result, + ) + + def test_unicode_literal_in_expr(self): + self._do_memory_test( + ( + "## -*- coding: utf-8 -*-\n" + '${"Alors vous imaginez ma surprise, au lever du jour, ' + "quand une drôle de petite voix m’a réveillé. " + "Elle disait: " + '« S’il vous plaît… dessine-moi un mouton! »"}\n' + ).encode("utf-8"), + ( + "Alors vous imaginez ma surprise, au lever du jour, " + "quand une drôle de petite voix m’a réveillé. " + "Elle disait: « S’il vous plaît… dessine-moi un mouton! »" + ), + filters=lambda s: s.strip(), + ) + + def test_unicode_literal_in_expr_file(self): + self._do_file_test( + "unicode_expr.html", + ( + "Alors vous imaginez ma surprise, au lever du jour, " + "quand une drôle de petite voix m’a réveillé. " + "Elle disait: « S’il vous plaît… dessine-moi un mouton! »" + ), + lambda t: t.strip(), + ) + + def test_unicode_literal_in_code(self): + self._do_memory_test( + ( + """## -*- coding: utf-8 -*- + <% + context.write("Alors vous imaginez ma surprise, au """ + """lever du jour, quand une drôle de petite voix m’a """ + """réveillé. Elle disait: """ + """« S’il vous plaît… dessine-moi un mouton! »") + %> + """ + ).encode("utf-8"), + ( + "Alors vous imaginez ma surprise, au lever du jour, " + "quand une drôle de petite voix m’a réveillé. " + "Elle disait: « S’il vous plaît… dessine-moi un mouton! »" + ), + filters=lambda s: s.strip(), + ) + + def test_unicode_literal_in_controlline(self): + self._do_memory_test( + ( + """## -*- coding: utf-8 -*- + <% + x = "drôle de petite voix m’a réveillé." + %> + % if x=="drôle de petite voix m’a réveillé.": + hi, ${x} + % endif + """ + ).encode("utf-8"), + ("""hi, drôle de petite voix m’a réveillé."""), + filters=lambda s: s.strip(), + ) + + def test_unicode_literal_in_tag(self): + self._do_file_test( + "unicode_arguments.html", + [ + ("x is: drôle de petite voix m’a réveillé"), + ("x is: drôle de petite voix m’a réveillé"), + ("x is: drôle de petite voix m’a réveillé"), + ("x is: drôle de petite voix m’a réveillé"), + ], + filters=result_lines, + ) + + self._do_memory_test( + util.read_file(self._file_path("unicode_arguments.html")), + [ + ("x is: drôle de petite voix m’a réveillé"), + ("x is: drôle de petite voix m’a réveillé"), + ("x is: drôle de petite voix m’a réveillé"), + ("x is: drôle de petite voix m’a réveillé"), + ], + filters=result_lines, + ) + + def test_unicode_literal_in_def(self): + self._do_memory_test( + ( + """## -*- coding: utf-8 -*- + <%def name="bello(foo, bar)"> + Foo: ${ foo } + Bar: ${ bar } + </%def> + <%call expr="bello(foo='árvíztűrő tükörfúrógép', """ + """bar='ÁRVÍZTŰRŐ TÜKÖRFÚRÓGÉP')"> + </%call>""" + ).encode("utf-8"), + ( + """Foo: árvíztűrő tükörfúrógép """ + """Bar: ÁRVÍZTŰRŐ TÜKÖRFÚRÓGÉP""" + ), + filters=flatten_result, + ) + + self._do_memory_test( + ( + "## -*- coding: utf-8 -*-\n" + """<%def name="hello(foo='árvíztűrő tükörfúrógép', """ + """bar='ÁRVÍZTŰRŐ TÜKÖRFÚRÓGÉP')">\n""" + "Foo: ${ foo }\n" + "Bar: ${ bar }\n" + "</%def>\n" + "${ hello() }" + ).encode("utf-8"), + ( + """Foo: árvíztűrő tükörfúrógép Bar: """ + """ÁRVÍZTŰRŐ TÜKÖRFÚRÓGÉP""" + ), + filters=flatten_result, + ) + + def test_input_encoding(self): + """test the 'input_encoding' flag on Template, and that unicode + objects arent double-decoded""" + + self._do_memory_test( + ("hello ${f('śląsk')}"), + ("hello śląsk"), + input_encoding="utf-8", + template_args={"f": lambda x: x}, + ) + + self._do_memory_test( + ("## -*- coding: utf-8 -*-\nhello ${f('śląsk')}"), + ("hello śląsk"), + template_args={"f": lambda x: x}, + ) + + def test_encoding(self): + self._do_memory_test( + ( + "Alors vous imaginez ma surprise, au lever du jour, quand " + "une drôle de petite voix m’a réveillé. Elle disait: " + "« S’il vous plaît… dessine-moi un mouton! »" + ), + ( + "Alors vous imaginez ma surprise, au lever du jour, quand " + "une drôle de petite voix m’a réveillé. Elle disait: " + "« S’il vous plaît… dessine-moi un mouton! »" + ).encode("utf-8"), + output_encoding="utf-8", + unicode_=False, + ) + + def test_encoding_errors(self): + self._do_memory_test( + ( + """KGB (transliteration of "КГБ") is the Russian-language """ + """abbreviation for Committee for State Security, """ + """(Russian: Комит́ет Госуд́арственной Безоп́асности """ + """(help·info); Komitet Gosudarstvennoy Bezopasnosti)""" + ), + ( + """KGB (transliteration of "КГБ") is the Russian-language """ + """abbreviation for Committee for State Security, """ + """(Russian: Комит́ет Госуд́арственной Безоп́асности """ + """(help·info); Komitet Gosudarstvennoy Bezopasnosti)""" + ).encode("iso-8859-1", "replace"), + output_encoding="iso-8859-1", + encoding_errors="replace", + unicode_=False, + ) + + def test_read_unicode(self): + lookup = TemplateLookup( + directories=[config.template_base], + filesystem_checks=True, + output_encoding="utf-8", + ) + template = lookup.get_template("/read_unicode_py3k.html") + # TODO: I've no idea what encoding this file is, Python 3.1.2 + # won't read the file even with open(...encoding='utf-8') unless + # errors is specified. or if there's some quirk in 3.1.2 + # since I'm pretty sure this test worked with py3k when I wrote it. + template.render(path=self._file_path("internationalization.html")) + + +class PageArgsTest(TemplateTest): + def test_basic(self): + template = Template( + """ + <%page args="x, y, z=7"/> + + this is page, ${x}, ${y}, ${z} +""" + ) + + assert ( + flatten_result(template.render(x=5, y=10)) + == "this is page, 5, 10, 7" + ) + assert ( + flatten_result(template.render(x=5, y=10, z=32)) + == "this is page, 5, 10, 32" + ) + assert_raises(TypeError, template.render, y=10) + + def test_inherits(self): + lookup = TemplateLookup() + lookup.put_string( + "base.tmpl", + """ + <%page args="bar" /> + ${bar} + ${pageargs['foo']} + ${self.body(**pageargs)} + """, + ) + lookup.put_string( + "index.tmpl", + """ + <%inherit file="base.tmpl" /> + <%page args="variable" /> + ${variable} + """, + ) + + self._do_test( + lookup.get_template("index.tmpl"), + "bar foo var", + filters=flatten_result, + template_args={"variable": "var", "bar": "bar", "foo": "foo"}, + ) + + def test_includes(self): + lookup = TemplateLookup() + lookup.put_string( + "incl1.tmpl", + """ + <%page args="bar" /> + ${bar} + ${pageargs['foo']} + """, + ) + lookup.put_string( + "incl2.tmpl", + """ + ${pageargs} + """, + ) + lookup.put_string( + "index.tmpl", + """ + <%include file="incl1.tmpl" args="**pageargs"/> + <%page args="variable" /> + ${variable} + <%include file="incl2.tmpl" /> + """, + ) + + self._do_test( + lookup.get_template("index.tmpl"), + "bar foo var {}", + filters=flatten_result, + template_args={"variable": "var", "bar": "bar", "foo": "foo"}, + ) + + def test_context_small(self): + ctx = runtime.Context([].append, x=5, y=4) + eq_(sorted(ctx.keys()), ["caller", "capture", "x", "y"]) + + def test_with_context(self): + template = Template( + """ + <%page args="x, y, z=7"/> + + this is page, ${x}, ${y}, ${z}, ${w} +""" + ) + # print template.code + assert ( + flatten_result(template.render(x=5, y=10, w=17)) + == "this is page, 5, 10, 7, 17" + ) + + def test_overrides_builtins(self): + template = Template( + """ + <%page args="id"/> + + this is page, id is ${id} + """ + ) + + assert ( + flatten_result(template.render(id="im the id")) + == "this is page, id is im the id" + ) + + def test_canuse_builtin_names(self): + template = Template( + """ + exception: ${Exception} + id: ${id} + """ + ) + assert ( + flatten_result( + template.render(id="some id", Exception="some exception") + ) + == "exception: some exception id: some id" + ) + + def test_builtin_names_dont_clobber_defaults_in_includes(self): + lookup = TemplateLookup() + lookup.put_string( + "test.mako", + """ + <%include file="test1.mako"/> + + """, + ) + + lookup.put_string( + "test1.mako", + """ + <%page args="id='foo'"/> + + ${id} + """, + ) + + for template in ("test.mako", "test1.mako"): + assert ( + flatten_result(lookup.get_template(template).render()) == "foo" + ) + assert ( + flatten_result(lookup.get_template(template).render(id=5)) + == "5" + ) + assert ( + flatten_result(lookup.get_template(template).render(id=id)) + == "<built-in function id>" + ) + + def test_dict_locals(self): + template = Template( + """ + <% + dict = "this is dict" + locals = "this is locals" + %> + dict: ${dict} + locals: ${locals} + """ + ) + assert ( + flatten_result(template.render()) + == "dict: this is dict locals: this is locals" + ) + + +class IncludeTest(TemplateTest): + def test_basic(self): + lookup = TemplateLookup() + lookup.put_string( + "a", + """ + this is a + <%include file="b" args="a=3,b=4,c=5"/> + """, + ) + lookup.put_string( + "b", + """ + <%page args="a,b,c"/> + this is b. ${a}, ${b}, ${c} + """, + ) + assert ( + flatten_result(lookup.get_template("a").render()) + == "this is a this is b. 3, 4, 5" + ) + + def test_localargs(self): + lookup = TemplateLookup() + lookup.put_string( + "a", + """ + this is a + <%include file="b" args="a=a,b=b,c=5"/> + """, + ) + lookup.put_string( + "b", + """ + <%page args="a,b,c"/> + this is b. ${a}, ${b}, ${c} + """, + ) + assert ( + flatten_result(lookup.get_template("a").render(a=7, b=8)) + == "this is a this is b. 7, 8, 5" + ) + + def test_viakwargs(self): + lookup = TemplateLookup() + lookup.put_string( + "a", + """ + this is a + <%include file="b" args="c=5, **context.kwargs"/> + """, + ) + lookup.put_string( + "b", + """ + <%page args="a,b,c"/> + this is b. ${a}, ${b}, ${c} + """, + ) + # print lookup.get_template("a").code + assert ( + flatten_result(lookup.get_template("a").render(a=7, b=8)) + == "this is a this is b. 7, 8, 5" + ) + + def test_include_withargs(self): + lookup = TemplateLookup() + lookup.put_string( + "a", + """ + this is a + <%include file="${i}" args="c=5, **context.kwargs"/> + """, + ) + lookup.put_string( + "b", + """ + <%page args="a,b,c"/> + this is b. ${a}, ${b}, ${c} + """, + ) + assert ( + flatten_result(lookup.get_template("a").render(a=7, b=8, i="b")) + == "this is a this is b. 7, 8, 5" + ) + + def test_within_ccall(self): + lookup = TemplateLookup() + lookup.put_string("a", """this is a""") + lookup.put_string( + "b", + """ + <%def name="bar()"> + bar: ${caller.body()} + <%include file="a"/> + </%def> + """, + ) + lookup.put_string( + "c", + """ + <%namespace name="b" file="b"/> + <%b:bar> + calling bar + </%b:bar> + """, + ) + assert ( + flatten_result(lookup.get_template("c").render()) + == "bar: calling bar this is a" + ) + + def test_include_error_handler(self): + def handle(context, error): + context.write("include error") + return True + + lookup = TemplateLookup(include_error_handler=handle) + lookup.put_string( + "a", + """ + this is a. + <%include file="b"/> + """, + ) + lookup.put_string( + "b", + """ + this is b ${1/0} end. + """, + ) + assert ( + flatten_result(lookup.get_template("a").render()) + == "this is a. this is b include error" + ) + + +class UndefinedVarsTest(TemplateTest): + def test_undefined(self): + t = Template( + """ + % if x is UNDEFINED: + undefined + % else: + x: ${x} + % endif + """ + ) + + assert result_lines(t.render(x=12)) == ["x: 12"] + assert result_lines(t.render(y=12)) == ["undefined"] + + def test_strict(self): + t = Template( + """ + % if x is UNDEFINED: + undefined + % else: + x: ${x} + % endif + """, + strict_undefined=True, + ) + + assert result_lines(t.render(x=12)) == ["x: 12"] + + assert_raises(NameError, t.render, y=12) + + l = TemplateLookup(strict_undefined=True) + l.put_string("a", "some template") + l.put_string( + "b", + """ + <%namespace name='a' file='a' import='*'/> + % if x is UNDEFINED: + undefined + % else: + x: ${x} + % endif + """, + ) + + assert result_lines(t.render(x=12)) == ["x: 12"] + + assert_raises(NameError, t.render, y=12) + + def test_expression_declared(self): + t = Template( + """ + ${",".join([t for t in ("a", "b", "c")])} + """, + strict_undefined=True, + ) + + eq_(result_lines(t.render()), ["a,b,c"]) + + t = Template( + """ + <%self:foo value="${[(val, n) for val, n in [(1, 2)]]}"/> + + <%def name="foo(value)"> + ${value} + </%def> + + """, + strict_undefined=True, + ) + + eq_(result_lines(t.render()), ["[(1, 2)]"]) + + t = Template( + """ + <%call expr="foo(value=[(val, n) for val, n in [(1, 2)]])" /> + + <%def name="foo(value)"> + ${value} + </%def> + + """, + strict_undefined=True, + ) + + eq_(result_lines(t.render()), ["[(1, 2)]"]) + + l = TemplateLookup(strict_undefined=True) + l.put_string("i", "hi, ${pageargs['y']}") + l.put_string( + "t", + """ + <%include file="i" args="y=[x for x in range(3)]" /> + """, + ) + eq_(result_lines(l.get_template("t").render()), ["hi, [0, 1, 2]"]) + + l.put_string( + "q", + """ + <%namespace name="i" file="${(str([x for x in range(3)][2]) + """ + """'i')[-1]}" /> + ${i.body(y='x')} + """, + ) + eq_(result_lines(l.get_template("q").render()), ["hi, x"]) + + t = Template( + """ + <% + y = lambda q: str(q) + %> + ${y('hi')} + """, + strict_undefined=True, + ) + eq_(result_lines(t.render()), ["hi"]) + + def test_list_comprehensions_plus_undeclared_nonstrict(self): + # traditional behavior. variable inside a list comprehension + # is treated as an "undefined", so is pulled from the context. + t = Template( + """ + t is: ${t} + + ${",".join([t for t in ("a", "b", "c")])} + """ + ) + + eq_(result_lines(t.render(t="T")), ["t is: T", "a,b,c"]) + + def test_traditional_assignment_plus_undeclared(self): + t = Template( + """ + t is: ${t} + + <% + t = 12 + %> + """ + ) + assert_raises(UnboundLocalError, t.render, t="T") + + def test_list_comprehensions_plus_undeclared_strict(self): + # with strict, a list comprehension now behaves + # like the undeclared case above. + t = Template( + """ + t is: ${t} + + ${",".join([t for t in ("a", "b", "c")])} + """, + strict_undefined=True, + ) + + eq_(result_lines(t.render(t="T")), ["t is: T", "a,b,c"]) + + +class StopRenderingTest(TemplateTest): + def test_return_in_template(self): + t = Template( + """ + Line one + <% return STOP_RENDERING %> + Line Three + """, + strict_undefined=True, + ) + + eq_(result_lines(t.render()), ["Line one"]) + + +class ReservedNameTest(TemplateTest): + def test_names_on_context(self): + for name in ("context", "loop", "UNDEFINED", "STOP_RENDERING"): + assert_raises_message( + exceptions.NameConflictError, + r"Reserved words passed to render\(\): %s" % name, + Template("x").render, + **{name: "foo"}, + ) + + def test_names_in_template(self): + for name in ("context", "loop", "UNDEFINED", "STOP_RENDERING"): + assert_raises_message( + exceptions.NameConflictError, + r"Reserved words declared in template: %s" % name, + Template, + "<%% %s = 5 %%>" % name, + ) + + def test_exclude_loop_context(self): + self._do_memory_test( + "loop is ${loop}", + "loop is 5", + template_args=dict(loop=5), + enable_loop=False, + ) + + def test_exclude_loop_template(self): + self._do_memory_test( + "<% loop = 12 %>loop is ${loop}", "loop is 12", enable_loop=False + ) + + +class ControlTest(TemplateTest): + def test_control(self): + t = Template( + """ + ## this is a template. + % for x in y: + % if 'test' in x: + yes x has test + % else: + no x does not have test + %endif + %endfor +""" + ) + assert result_lines( + t.render( + y=[ + {"test": "one"}, + {"foo": "bar"}, + {"foo": "bar", "test": "two"}, + ] + ) + ) == ["yes x has test", "no x does not have test", "yes x has test"] + + def test_blank_control_1(self): + self._do_memory_test( + """ + % if True: + % endif + """, + "", + filters=lambda s: s.strip(), + ) + + def test_blank_control_2(self): + self._do_memory_test( + """ + % if True: + % elif True: + % endif + """, + "", + filters=lambda s: s.strip(), + ) + + def test_blank_control_3(self): + self._do_memory_test( + """ + % if True: + % else: + % endif + """, + "", + filters=lambda s: s.strip(), + ) + + def test_blank_control_4(self): + self._do_memory_test( + """ + % if True: + % elif True: + % else: + % endif + """, + "", + filters=lambda s: s.strip(), + ) + + def test_blank_control_5(self): + self._do_memory_test( + """ + % for x in range(10): + % endfor + """, + "", + filters=lambda s: s.strip(), + ) + + def test_blank_control_6(self): + self._do_memory_test( + """ + % while False: + % endwhile + """, + "", + filters=lambda s: s.strip(), + ) + + def test_blank_control_7(self): + self._do_memory_test( + """ + % try: + % except: + % endtry + """, + "", + filters=lambda s: s.strip(), + ) + + def test_blank_control_8(self): + self._do_memory_test( + """ + % with ctx('x', 'w') as fp: + % endwith + """, + "", + filters=lambda s: s.strip(), + template_args={"ctx": ctx}, + ) + + def test_commented_blank_control_1(self): + self._do_memory_test( + """ + % if True: + ## comment + % endif + """, + "", + filters=lambda s: s.strip(), + ) + + def test_commented_blank_control_2(self): + self._do_memory_test( + """ + % if True: + ## comment + % elif True: + ## comment + % endif + """, + "", + filters=lambda s: s.strip(), + ) + + def test_commented_blank_control_3(self): + self._do_memory_test( + """ + % if True: + ## comment + % else: + ## comment + % endif + """, + "", + filters=lambda s: s.strip(), + ) + + def test_commented_blank_control_4(self): + self._do_memory_test( + """ + % if True: + ## comment + % elif True: + ## comment + % else: + ## comment + % endif + """, + "", + filters=lambda s: s.strip(), + ) + + def test_commented_blank_control_5(self): + self._do_memory_test( + """ + % for x in range(10): + ## comment + % endfor + """, + "", + filters=lambda s: s.strip(), + ) + + def test_commented_blank_control_6(self): + self._do_memory_test( + """ + % while False: + ## comment + % endwhile + """, + "", + filters=lambda s: s.strip(), + ) + + def test_commented_blank_control_7(self): + self._do_memory_test( + """ + % try: + ## comment + % except: + ## comment + % endtry + """, + "", + filters=lambda s: s.strip(), + ) + + def test_commented_blank_control_8(self): + self._do_memory_test( + """ + % with ctx('x', 'w') as fp: + ## comment + % endwith + """, + "", + filters=lambda s: s.strip(), + template_args={"ctx": ctx}, + ) + + def test_multiline_control(self): + t = Template( + """ + % for x in \\ + [y for y in [1,2,3]]: + ${x} + % endfor +""" + ) + # print t.code + assert flatten_result(t.render()) == "1 2 3" + + +class GlobalsTest(TemplateTest): + def test_globals(self): + self._do_memory_test( + """ + <%! + y = "hi" + %> + y is ${y} + """, + "y is hi", + filters=lambda t: t.strip(), + ) + + +class RichTracebackTest(TemplateTest): + def _do_test_traceback(self, utf8, memory, syntax): + if memory: + if syntax: + source = ( + '## coding: utf-8\n<% print "m’a réveillé. ' + "Elle disait: « S’il vous plaît… dessine-moi " + "un mouton! » %>" + ) + else: + source = ( + '## coding: utf-8\n<% print u"m’a réveillé. ' + "Elle disait: « S’il vous plaît… dessine-moi un " + 'mouton! »" + str(5/0) %>' + ) + if utf8: + source = source.encode("utf-8") + else: + source = source + templateargs = {"text": source} + else: + if syntax: + filename = "unicode_syntax_error.html" + else: + filename = "unicode_runtime_error.html" + source = util.read_file(self._file_path(filename), "rb") + if not utf8: + source = source.decode("utf-8") + templateargs = {"filename": self._file_path(filename)} + try: + template = Template(**templateargs) + if not syntax: + template.render_unicode() + assert False + except Exception: + tback = exceptions.RichTraceback() + if utf8: + assert tback.source == source.decode("utf-8") + else: + assert tback.source == source + + +for utf8 in (True, False): + for memory in (True, False): + for syntax in (True, False): + + def _do_test(self): + self._do_test_traceback(utf8, memory, syntax) + + name = "test_%s_%s_%s" % ( + utf8 and "utf8" or "unicode", + memory and "memory" or "file", + syntax and "syntax" or "runtime", + ) + _do_test.__name__ = name + setattr(RichTracebackTest, name, _do_test) + del _do_test + + +class ModuleDirTest(TemplateTest): + def teardown_method(self): + import shutil + + shutil.rmtree(config.module_base, True) + + def test_basic(self): + t = self._file_template("modtest.html") + t2 = self._file_template("subdir/modtest.html") + + eq_( + t.module.__file__, + os.path.join(config.module_base, "modtest.html.py"), + ) + eq_( + t2.module.__file__, + os.path.join(config.module_base, "subdir", "modtest.html.py"), + ) + + def test_callable(self): + def get_modname(filename, uri): + return os.path.join( + config.module_base, + os.path.dirname(uri)[1:], + "foo", + os.path.basename(filename) + ".py", + ) + + lookup = TemplateLookup( + config.template_base, modulename_callable=get_modname + ) + t = lookup.get_template("/modtest.html") + t2 = lookup.get_template("/subdir/modtest.html") + eq_( + t.module.__file__, + os.path.join(config.module_base, "foo", "modtest.html.py"), + ) + eq_( + t2.module.__file__, + os.path.join( + config.module_base, "subdir", "foo", "modtest.html.py" + ), + ) + + def test_custom_writer(self): + canary = [] + + def write_module(source, outputpath): + f = open(outputpath, "wb") + canary.append(outputpath) + f.write(source) + f.close() + + lookup = TemplateLookup( + config.template_base, + module_writer=write_module, + module_directory=config.module_base, + ) + lookup.get_template("/modtest.html") + lookup.get_template("/subdir/modtest.html") + eq_( + canary, + [ + os.path.join(config.module_base, "modtest.html.py"), + os.path.join(config.module_base, "subdir", "modtest.html.py"), + ], + ) + + +class FilenameToURITest(TemplateTest): + def test_windows_paths(self): + """test that windows filenames are handled appropriately by + Template.""" + + current_path = os.path + import ntpath + + os.path = ntpath + try: + + class NoCompileTemplate(Template): + def _compile_from_file(self, path, filename): + self.path = path + return Template("foo bar").module + + t1 = NoCompileTemplate( + filename="c:\\foo\\template.html", + module_directory="c:\\modules\\", + ) + + eq_(t1.uri, "/foo/template.html") + eq_(t1.path, "c:\\modules\\foo\\template.html.py") + + t1 = NoCompileTemplate( + filename="c:\\path\\to\\templates\\template.html", + uri="/bar/template.html", + module_directory="c:\\modules\\", + ) + + eq_(t1.uri, "/bar/template.html") + eq_(t1.path, "c:\\modules\\bar\\template.html.py") + + finally: + os.path = current_path + + def test_posix_paths(self): + """test that posixs filenames are handled appropriately by Template.""" + + current_path = os.path + import posixpath + + os.path = posixpath + try: + + class NoCompileTemplate(Template): + def _compile_from_file(self, path, filename): + self.path = path + return Template("foo bar").module + + t1 = NoCompileTemplate( + filename="/var/www/htdocs/includes/template.html", + module_directory="/var/lib/modules", + ) + + eq_(t1.uri, "/var/www/htdocs/includes/template.html") + eq_( + t1.path, + "/var/lib/modules/var/www/htdocs/includes/template.html.py", + ) + + t1 = NoCompileTemplate( + filename="/var/www/htdocs/includes/template.html", + uri="/bar/template.html", + module_directory="/var/lib/modules", + ) + + eq_(t1.uri, "/bar/template.html") + eq_(t1.path, "/var/lib/modules/bar/template.html.py") + + finally: + os.path = current_path + + def test_dont_accept_relative_outside_of_root(self): + assert_raises_message( + exceptions.TemplateLookupException, + 'Template uri "../../foo.html" is invalid - it ' + "cannot be relative outside of the root path", + Template, + "test", + uri="../../foo.html", + ) + + assert_raises_message( + exceptions.TemplateLookupException, + 'Template uri "/../../foo.html" is invalid - it ' + "cannot be relative outside of the root path", + Template, + "test", + uri="/../../foo.html", + ) + + # normalizes in the root is OK + t = Template("test", uri="foo/bar/../../foo.html") + eq_(t.uri, "foo/bar/../../foo.html") + + +class ModuleTemplateTest(TemplateTest): + def test_module_roundtrip(self): + lookup = TemplateLookup() + + template = Template( + """ + <%inherit file="base.html"/> + + % for x in range(5): + ${x} + % endfor +""", + lookup=lookup, + ) + + base = Template( + """ + This is base. + ${self.body()} +""", + lookup=lookup, + ) + + lookup.put_template("base.html", base) + lookup.put_template("template.html", template) + + assert result_lines(template.render()) == [ + "This is base.", + "0", + "1", + "2", + "3", + "4", + ] + + lookup = TemplateLookup() + template = ModuleTemplate(template.module, lookup=lookup) + base = ModuleTemplate(base.module, lookup=lookup) + + lookup.put_template("base.html", base) + lookup.put_template("template.html", template) + + assert result_lines(template.render()) == [ + "This is base.", + "0", + "1", + "2", + "3", + "4", + ] + + +class TestTemplateAPI: + def test_metadata(self): + t = Template( + """ +Text +Text +% if bar: + ${expression} +% endif + +<%include file='bar'/> + +""", + uri="/some/template", + ) + eq_( + ModuleInfo.get_module_source_metadata(t.code, full_line_map=True), + { + "full_line_map": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 4, + 5, + 5, + 5, + 7, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + ], + "source_encoding": "utf-8", + "filename": None, + "line_map": { + 35: 29, + 15: 0, + 22: 1, + 23: 4, + 24: 5, + 25: 5, + 26: 5, + 27: 7, + 28: 8, + 29: 8, + }, + "uri": "/some/template", + }, + ) + + def test_metadata_two(self): + t = Template( + """ +Text +Text +% if bar: + ${expression} +% endif + + <%block name="foo"> + hi block + </%block> + + +""", + uri="/some/template", + ) + eq_( + ModuleInfo.get_module_source_metadata(t.code, full_line_map=True), + { + "full_line_map": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 4, + 5, + 5, + 5, + 7, + 7, + 7, + 7, + 7, + 10, + 10, + 10, + 10, + 10, + 10, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + ], + "source_encoding": "utf-8", + "filename": None, + "line_map": { + 34: 10, + 40: 8, + 46: 8, + 15: 0, + 52: 46, + 24: 1, + 25: 4, + 26: 5, + 27: 5, + 28: 5, + 29: 7, + }, + "uri": "/some/template", + }, + ) + + +class PreprocessTest(TemplateTest): + def test_old_comments(self): + t = Template( + """ + im a template +# old style comment + # more old style comment + + ## new style comment + - # not a comment + - ## not a comment +""", + preprocessor=convert_comments, + ) + + assert ( + flatten_result(t.render()) + == "im a template - # not a comment - ## not a comment" + ) + + +class LexerTest(TemplateTest): + def _fixture(self): + from mako.parsetree import TemplateNode, Text + + class MyLexer: + encoding = "ascii" + + def __init__(self, *arg, **kw): + pass + + def parse(self): + t = TemplateNode("foo") + t.nodes.append( + Text( + "hello world", + source="foo", + lineno=0, + pos=0, + filename=None, + ) + ) + return t + + return MyLexer + + def _test_custom_lexer(self, template): + eq_(result_lines(template.render()), ["hello world"]) + + def test_via_template(self): + t = Template("foo", lexer_cls=self._fixture()) + self._test_custom_lexer(t) + + def test_via_lookup(self): + tl = TemplateLookup(lexer_cls=self._fixture()) + tl.put_string("foo", "foo") + t = tl.get_template("foo") + self._test_custom_lexer(t) + + +class FuturesTest(TemplateTest): + def test_future_import(self): + t = Template("${ x / y }", future_imports=["division"]) + assert result_lines(t.render(x=12, y=5)) == ["2.4"] diff --git a/test/test_tgplugin.py b/test/test_tgplugin.py new file mode 100644 index 0000000..38998c2 --- /dev/null +++ b/test/test_tgplugin.py @@ -0,0 +1,53 @@ +from mako.ext.turbogears import TGPlugin +from mako.testing.config import config +from mako.testing.fixtures import TemplateTest +from mako.testing.helpers import result_lines + +tl = TGPlugin( + options=dict(directories=[config.template_base]), extension="html" +) + + +class TestTGPlugin(TemplateTest): + def test_basic(self): + t = tl.load_template("/index.html") + assert result_lines(t.render()) == ["this is index"] + + def test_subdir(self): + t = tl.load_template("/subdir/index.html") + assert result_lines(t.render()) == [ + "this is sub index", + "this is include 2", + ] + + assert ( + tl.load_template("/subdir/index.html").module_id + == "_subdir_index_html" + ) + + def test_basic_dot(self): + t = tl.load_template("index") + assert result_lines(t.render()) == ["this is index"] + + def test_subdir_dot(self): + t = tl.load_template("subdir.index") + assert result_lines(t.render()) == [ + "this is sub index", + "this is include 2", + ] + + assert ( + tl.load_template("subdir.index").module_id == "_subdir_index_html" + ) + + def test_string(self): + t = tl.load_template("foo", "hello world") + assert t.render() == "hello world" + + def test_render(self): + assert result_lines(tl.render({}, template="/index.html")) == [ + "this is index" + ] + assert result_lines(tl.render({}, template=("/index.html"))) == [ + "this is index" + ] diff --git a/test/test_util.py b/test/test_util.py new file mode 100644 index 0000000..95c1cb4 --- /dev/null +++ b/test/test_util.py @@ -0,0 +1,62 @@ +import os +import sys + +import pytest + +from mako import compat +from mako import exceptions +from mako import util +from mako.testing.assertions import assert_raises_message +from mako.testing.assertions import eq_ +from mako.testing.assertions import in_ +from mako.testing.assertions import ne_ +from mako.testing.assertions import not_in + + +class UtilTest: + def test_fast_buffer_write(self): + buf = util.FastEncodingBuffer() + buf.write("string a ") + buf.write("string b") + eq_(buf.getvalue(), "string a string b") + + def test_fast_buffer_truncate(self): + buf = util.FastEncodingBuffer() + buf.write("string a ") + buf.write("string b") + buf.truncate() + buf.write("string c ") + buf.write("string d") + eq_(buf.getvalue(), "string c string d") + + def test_fast_buffer_encoded(self): + s = "drôl m’a rée « S’il" + buf = util.FastEncodingBuffer(encoding="utf-8") + buf.write(s[0:10]) + buf.write(s[10:]) + eq_(buf.getvalue(), s.encode("utf-8")) + + def test_read_file(self): + fn = os.path.join(os.path.dirname(__file__), "test_util.py") + data = util.read_file(fn, "rb") + assert b"test_util" in data + + @pytest.mark.skipif(compat.pypy, reason="Pypy does this differently") + def test_load_module(self): + path = os.path.join(os.path.dirname(__file__), "module_to_import.py") + some_module = compat.load_module("test.module_to_import", path) + + not_in("test.module_to_import", sys.modules) + in_("some_function", dir(some_module)) + import test.module_to_import + + ne_(some_module, test.module_to_import) + + def test_load_plugin_failure(self): + loader = util.PluginLoader("fakegroup") + assert_raises_message( + exceptions.RuntimeException, + "Can't load plugin fakegroup fake", + loader.load, + "fake", + ) diff --git a/test/testing/dummy.cfg b/test/testing/dummy.cfg new file mode 100644 index 0000000..39644a3 --- /dev/null +++ b/test/testing/dummy.cfg @@ -0,0 +1,25 @@ +[boolean_values] +yes = yes +one = 1 +true = true +on = on +no = no +zero = 0 +false = false +off = off + +[additional_types] +decimal_value = 100001.01 +datetime_value = 2021-12-04 00:05:23.283 + +[type_mismatch] +int_value = foo + +[missing_item] +present_item = HERE + +[basic_values] +int_value = 15421 +bool_value = true +float_value = 14.01 +str_value = Ceci n'est pas une chaîne diff --git a/test/testing/test_config.py b/test/testing/test_config.py new file mode 100644 index 0000000..680d7a4 --- /dev/null +++ b/test/testing/test_config.py @@ -0,0 +1,176 @@ +import configparser +from dataclasses import dataclass +from datetime import datetime +from decimal import Decimal +from pathlib import Path + +import pytest + +from mako.testing._config import ConfigValueTypeError +from mako.testing._config import MissingConfig +from mako.testing._config import MissingConfigItem +from mako.testing._config import MissingConfigSection +from mako.testing._config import ReadsCfg +from mako.testing.assertions import assert_raises_message_with_given_cause +from mako.testing.assertions import assert_raises_with_given_cause + +PATH_TO_TEST_CONFIG = Path(__file__).parent / "dummy.cfg" + + +@dataclass +class BasicConfig(ReadsCfg): + int_value: int + bool_value: bool + float_value: float + str_value: str + + section_header = "basic_values" + + +@dataclass +class BooleanConfig(ReadsCfg): + yes: bool + one: bool + true: bool + on: bool + no: bool + zero: bool + false: bool + off: bool + + section_header = "boolean_values" + + +@dataclass +class UnsupportedTypesConfig(ReadsCfg): + decimal_value: Decimal + datetime_value: datetime + + section_header = "additional_types" + + +@dataclass +class SupportedTypesConfig(ReadsCfg): + decimal_value: Decimal + datetime_value: datetime + + section_header = "additional_types" + converters = { + Decimal: lambda v: Decimal(str(v)), + datetime: lambda v: datetime.fromisoformat(v), + } + + +@dataclass +class NonexistentSectionConfig(ReadsCfg): + some_value: str + another_value: str + + section_header = "i_dont_exist" + + +@dataclass +class TypeMismatchConfig(ReadsCfg): + int_value: int + + section_header = "type_mismatch" + + +@dataclass +class MissingItemConfig(ReadsCfg): + present_item: str + missing_item: str + + section_header = "missing_item" + + +class BasicConfigTest: + @pytest.fixture(scope="class") + def config(self): + return BasicConfig.from_cfg_file(PATH_TO_TEST_CONFIG) + + def test_coercions(self, config): + assert isinstance(config.int_value, int) + assert isinstance(config.bool_value, bool) + assert isinstance(config.float_value, float) + assert isinstance(config.str_value, str) + + def test_values(self, config): + assert config.int_value == 15421 + assert config.bool_value == True + assert config.float_value == 14.01 + assert config.str_value == "Ceci n'est pas une chaîne" + + def test_error_on_loading_from_nonexistent_file(self): + assert_raises_with_given_cause( + MissingConfig, + FileNotFoundError, + BasicConfig.from_cfg_file, + "./n/o/f/i/l/e/h.ere", + ) + + def test_error_on_loading_from_nonexistent_section(self): + assert_raises_with_given_cause( + MissingConfigSection, + configparser.NoSectionError, + NonexistentSectionConfig.from_cfg_file, + PATH_TO_TEST_CONFIG, + ) + + +class BooleanConfigTest: + @pytest.fixture(scope="class") + def config(self): + return BooleanConfig.from_cfg_file(PATH_TO_TEST_CONFIG) + + def test_values(self, config): + assert config.yes is True + assert config.one is True + assert config.true is True + assert config.on is True + assert config.no is False + assert config.zero is False + assert config.false is False + assert config.off is False + + +class UnsupportedTypesConfigTest: + @pytest.fixture(scope="class") + def config(self): + return UnsupportedTypesConfig.from_cfg_file(PATH_TO_TEST_CONFIG) + + def test_values(self, config): + assert config.decimal_value == "100001.01" + assert config.datetime_value == "2021-12-04 00:05:23.283" + + +class SupportedTypesConfigTest: + @pytest.fixture(scope="class") + def config(self): + return SupportedTypesConfig.from_cfg_file(PATH_TO_TEST_CONFIG) + + def test_values(self, config): + assert config.decimal_value == Decimal("100001.01") + assert config.datetime_value == datetime(2021, 12, 4, 0, 5, 23, 283000) + + +class TypeMismatchConfigTest: + def test_error_on_load(self): + assert_raises_message_with_given_cause( + ConfigValueTypeError, + "Wrong value type for int_value", + ValueError, + TypeMismatchConfig.from_cfg_file, + PATH_TO_TEST_CONFIG, + ) + + +class MissingItemConfigTest: + def test_error_on_load(self): + assert_raises_message_with_given_cause( + MissingConfigItem, + "No config item for missing_item", + configparser.NoOptionError, + MissingItemConfig.from_cfg_file, + PATH_TO_TEST_CONFIG, + ) @@ -0,0 +1,36 @@ +[tox] +envlist = py + +[testenv] +cov_args=--cov=mako --cov-report term --cov-report xml + +deps=pytest>=3.1.0 + beaker + markupsafe + pygments + babel + dogpile.cache + lingua<4 + cov: pytest-cov + +setenv= + cov: COVERAGE={[testenv]cov_args} + +commands=pytest {env:COVERAGE:} {posargs} + + +[testenv:pep8] +basepython = python3 +deps= + flake8 + flake8-import-order + flake8-builtins + flake8-docstrings + flake8-rst-docstrings + pydocstyle + # used by flake8-rst-docstrings + pygments + black==23.9.1 +commands = + flake8 ./mako/ ./test/ setup.py --exclude test/templates,test/foo {posargs} + black --check . |