diff options
| author | Iris Yang <irisykyang@google.com> | 2020-07-15 10:11:33 +0800 |
|---|---|---|
| committer | IRIS YANG <irisykyang@google.com> | 2020-07-17 17:07:40 +0000 |
| commit | 7dfef6a6d6dd4cc081e86839e562440c0873d998 (patch) | |
| tree | 2046b95d53a74b793dd54b7ea6e1b86724b93435 | |
| parent | 285303dd8e585e8e397974cc5f199e0adab6874c (diff) | |
| parent | 03c719f3ec4db9660aa4dea30bef9ac8e47aa933 (diff) | |
| download | jinja-7dfef6a6d6dd4cc081e86839e562440c0873d998.tar.gz | |
Import external/python/jinja into master
Merge branch 'upstream-master' of sso://googleplex-android/platform/external/python/jinja into master
Third-party review: ag/11821018
Bug:156559156
Change-Id: I233b6667e9093a0785d665945450d1cac38cfafe
106 files changed, 25728 insertions, 0 deletions
diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..e32c8029 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,13 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +insert_final_newline = true +trim_trailing_whitespace = true +end_of_line = lf +charset = utf-8 +max_line_length = 88 + +[*.{yml,yaml,json,js,css,html}] +indent_size = 2 diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 00000000..4273496d --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,33 @@ +The issue tracker is a tool to address bugs in Jinja itself. +Please use the #pocoo IRC channel on freenode or Stack Overflow for general +questions about using Jinja or issues not related to Jinja. + +If you'd like to report a bug in Jinja, fill out the template below and provide +any extra information that may be useful / related to your problem. +Ideally, you create an [MCVE](http://stackoverflow.com/help/mcve) reproducing +the problem before opening an issue to ensure it's not caused by something in +your code. + +--- + +## Expected Behavior +Tell us what should happen + +## Actual Behavior +Tell us what happens instead + +## Template Code +```jinja +Paste the template code (ideally a minimal example) that causes the issue + +``` + +## Full Traceback +```pytb +Paste the full traceback in case there is an exception + +``` + +## Your Environment +* Python version: +* Jinja version: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 00000000..a826fda6 --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,52 @@ +name: Tests +on: + push: + branches: + - master + - '*.x' + pull_request: + branches: + - master + - '*.x' +jobs: + tests: + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - {name: Linux, python: '3.8', os: ubuntu-latest, tox: py38} + - {name: '3.7', python: '3.7', os: ubuntu-latest, tox: py37} + - {name: '3.6', python: '3.6', os: ubuntu-latest, tox: py36} + - {name: 'PyPy', python: pypy3, os: ubuntu-latest, tox: pypy3} + - {name: Style, python: '3.8', os: ubuntu-latest, tox: style} + - {name: Docs, python: '3.8', os: ubuntu-latest, tox: docs} + - {name: Windows, python: '3.8', os: windows-latest, tox: py38} + - {name: Mac, python: '3.8', os: macos-latest, tox: py38} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: update pip + run: | + pip install -U wheel + pip install -U setuptools + python -m pip install -U pip + - name: get pip cache dir + id: pip-cache + run: echo "::set-output name=dir::$(pip cache dir)" + - name: cache pip + uses: actions/cache@v1 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: pip|${{ runner.os }}|${{ matrix.python }}|${{ hashFiles('setup.py') }}|${{ hashFiles('requirements/*.txt') }} + - name: cache pre-commit + uses: actions/cache@v1 + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ matrix.python }}|${{ hashFiles('.pre-commit-config.yaml') }} + if: matrix.tox == 'style' + - run: pip install tox + - run: tox -e ${{ matrix.tox }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..81752e0e --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +*.so +docs/_build/ +*.pyc +*.pyo +*.egg-info/ +*.egg +build/ +dist/ +.DS_Store +.tox/ +.cache/ +.idea/ +env/ +venv/ +venv-*/ +.coverage +.coverage.* +htmlcov +.pytest_cache/ +/.vscode/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..a982b5f0 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,26 @@ +repos: + - repo: https://github.com/asottile/pyupgrade + rev: v1.26.2 + hooks: + - id: pyupgrade + args: ["--py36-plus"] + - repo: https://github.com/asottile/reorder_python_imports + rev: v1.9.0 + hooks: + - id: reorder-python-imports + args: ["--application-directories", "src"] + - repo: https://github.com/ambv/black + rev: 19.10b0 + hooks: + - id: black + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.7.9 + hooks: + - id: flake8 + additional_dependencies: [flake8-bugbear] + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.4.0 + hooks: + - id: check-byte-order-marker + - id: trailing-whitespace + - id: end-of-file-fixer diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..19069520 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,8 @@ +version: 2 +python: + install: + - requirements: requirements/docs.txt + - method: pip + path: . +sphinx: + builder: dirhtml diff --git a/CHANGES.rst b/CHANGES.rst new file mode 100644 index 00000000..298cf0ab --- /dev/null +++ b/CHANGES.rst @@ -0,0 +1,789 @@ +.. currentmodule:: jinja2 + +Version 3.0.0 +------------- + +Unreleased + +- Drop support for Python 2.7 and 3.5. +- Bump MarkupSafe dependency to >=1.1. +- Bump Babel optional dependency to >=2.1. +- Remove code that was marked deprecated. +- Use :pep:`451` API to load templates with + :class:`~loaders.PackageLoader`. :issue:`1168` +- Fix a bug that caused imported macros to not have access to the + current template's globals. :issue:`688` +- Add ability to ignore ``trim_blocks`` using ``+%}``. :issue:`1036` + +Version 2.11.2 +-------------- + +Released 2020-04-13 + +- Fix a bug that caused callable objects with ``__getattr__``, like + :class:`~unittest.mock.Mock` to be treated as a + :func:`contextfunction`. :issue:`1145` +- Update ``wordcount`` filter to trigger :class:`Undefined` methods + by wrapping the input in :func:`soft_str`. :pr:`1160` +- Fix a hang when displaying tracebacks on Python 32-bit. + :issue:`1162` +- Showing an undefined error for an object that raises + ``AttributeError`` on access doesn't cause a recursion error. + :issue:`1177` +- Revert changes to :class:`~loaders.PackageLoader` from 2.10 which + removed the dependency on setuptools and pkg_resources, and added + limited support for namespace packages. The changes caused issues + when using Pytest. Due to the difficulty in supporting Python 2 and + :pep:`451` simultaneously, the changes are reverted until 3.0. + :pr:`1182` +- Fix line numbers in error messages when newlines are stripped. + :pr:`1178` +- The special ``namespace()`` assignment object in templates works in + async environments. :issue:`1180` +- Fix whitespace being removed before tags in the middle of lines when + ``lstrip_blocks`` is enabled. :issue:`1138` +- :class:`~nativetypes.NativeEnvironment` doesn't evaluate + intermediate strings during rendering. This prevents early + evaluation which could change the value of an expression. + :issue:`1186` + + +Version 2.11.1 +-------------- + +Released 2020-01-30 + +- Fix a bug that prevented looking up a key after an attribute + (``{{ data.items[1:] }}``) in an async template. :issue:`1141` + + +Version 2.11.0 +-------------- + +Released 2020-01-27 + +- Drop support for Python 2.6, 3.3, and 3.4. This will be the last + version to support Python 2.7 and 3.5. +- Added a new ``ChainableUndefined`` class to support getitem and + getattr on an undefined object. :issue:`977` +- Allow ``{%+`` syntax (with NOP behavior) when ``lstrip_blocks`` is + disabled. :issue:`748` +- Added a ``default`` parameter for the ``map`` filter. :issue:`557` +- Exclude environment globals from + :func:`meta.find_undeclared_variables`. :issue:`931` +- Float literals can be written with scientific notation, like + 2.56e-3. :issue:`912`, :pr:`922` +- Int and float literals can be written with the '_' separator for + legibility, like 12_345. :pr:`923` +- Fix a bug causing deadlocks in ``LRUCache.setdefault``. :pr:`1000` +- The ``trim`` filter takes an optional string of characters to trim. + :pr:`828` +- A new ``jinja2.ext.debug`` extension adds a ``{% debug %}`` tag to + quickly dump the current context and available filters and tests. + :issue:`174`, :pr:`798, 983` +- Lexing templates with large amounts of whitespace is much faster. + :issue:`857`, :pr:`858` +- Parentheses around comparisons are preserved, so + ``{{ 2 * (3 < 5) }}`` outputs "2" instead of "False". + :issue:`755`, :pr:`938` +- Add new ``boolean``, ``false``, ``true``, ``integer`` and ``float`` + tests. :pr:`824` +- The environment's ``finalize`` function is only applied to the + output of expressions (constant or not), not static template data. + :issue:`63` +- When providing multiple paths to ``FileSystemLoader``, a template + can have the same name as a directory. :issue:`821` +- Always return :class:`Undefined` when omitting the ``else`` clause + in a ``{{ 'foo' if bar }}`` expression, regardless of the + environment's ``undefined`` class. Omitting the ``else`` clause is a + valid shortcut and should not raise an error when using + :class:`StrictUndefined`. :issue:`710`, :pr:`1079` +- Fix behavior of ``loop`` control variables such as ``length`` and + ``revindex0`` when looping over a generator. :issue:`459, 751, 794`, + :pr:`993` +- Async support is only loaded the first time an environment enables + it, in order to avoid a slow initial import. :issue:`765` +- In async environments, the ``|map`` filter will await the filter + call if needed. :pr:`913` +- In for loops that access ``loop`` attributes, the iterator is not + advanced ahead of the current iteration unless ``length``, + ``revindex``, ``nextitem``, or ``last`` are accessed. This makes it + less likely to break ``groupby`` results. :issue:`555`, :pr:`1101` +- In async environments, the ``loop`` attributes ``length`` and + ``revindex`` work for async iterators. :pr:`1101` +- In async environments, values from attribute/property access will + be awaited if needed. :pr:`1101` +- :class:`~loader.PackageLoader` doesn't depend on setuptools or + pkg_resources. :issue:`970` +- ``PackageLoader`` has limited support for :pep:`420` namespace + packages. :issue:`1097` +- Support :class:`os.PathLike` objects in + :class:`~loader.FileSystemLoader` and :class:`~loader.ModuleLoader`. + :issue:`870` +- :class:`~nativetypes.NativeTemplate` correctly handles quotes + between expressions. ``"'{{ a }}', '{{ b }}'"`` renders as the tuple + ``('1', '2')`` rather than the string ``'1, 2'``. :issue:`1020` +- Creating a :class:`~nativetypes.NativeTemplate` directly creates a + :class:`~nativetypes.NativeEnvironment` instead of a default + :class:`Environment`. :issue:`1091` +- After calling ``LRUCache.copy()``, the copy's queue methods point to + the correct queue. :issue:`843` +- Compiling templates always writes UTF-8 instead of defaulting to the + system encoding. :issue:`889` +- ``|wordwrap`` filter treats existing newlines as separate paragraphs + to be wrapped individually, rather than creating short intermediate + lines. :issue:`175` +- Add ``break_on_hyphens`` parameter to ``|wordwrap`` filter. + :issue:`550` +- Cython compiled functions decorated as context functions will be + passed the context. :pr:`1108` +- When chained comparisons of constants are evaluated at compile time, + the result follows Python's behavior of returning ``False`` if any + comparison returns ``False``, rather than only the last one. + :issue:`1102` +- Tracebacks for exceptions in templates show the correct line numbers + and source for Python >= 3.7. :issue:`1104` +- Tracebacks for template syntax errors in Python 3 no longer show + internal compiler frames. :issue:`763` +- Add a ``DerivedContextReference`` node that can be used by + extensions to get the current context and local variables such as + ``loop``. :issue:`860` +- Constant folding during compilation is applied to some node types + that were previously overlooked. :issue:`733` +- ``TemplateSyntaxError.source`` is not empty when raised from an + included template. :issue:`457` +- Passing an ``Undefined`` value to ``get_template`` (such as through + ``extends``, ``import``, or ``include``), raises an + ``UndefinedError`` consistently. ``select_template`` will show the + undefined message in the list of attempts rather than the empty + string. :issue:`1037` +- ``TemplateSyntaxError`` can be pickled. :pr:`1117` + + +Version 2.10.3 +-------------- + +Released 2019-10-04 + +- Fix a typo in Babel entry point in ``setup.py`` that was preventing + installation. + + +Version 2.10.2 +-------------- + +Released 2019-10-04 + +- Fix Python 3.7 deprecation warnings. +- Using ``range`` in the sandboxed environment uses ``xrange`` on + Python 2 to avoid memory use. :issue:`933` +- Use Python 3.7's better traceback support to avoid a core dump when + using debug builds of Python 3.7. :issue:`1050` + + +Version 2.10.1 +-------------- + +Released 2019-04-06 + +- ``SandboxedEnvironment`` securely handles ``str.format_map`` in + order to prevent code execution through untrusted format strings. + The sandbox already handled ``str.format``. + + +Version 2.10 +------------ + +Released 2017-11-08 + +- Added a new extension node called ``OverlayScope`` which can be used + to create an unoptimized scope that will look up all variables from + a derived context. +- Added an ``in`` test that works like the in operator. This can be + used in combination with ``reject`` and ``select``. +- Added ``previtem`` and ``nextitem`` to loop contexts, providing + access to the previous/next item in the loop. If such an item does + not exist, the value is undefined. +- Added ``changed(*values)`` to loop contexts, providing an easy way + of checking whether a value has changed since the last iteration (or + rather since the last call of the method) +- Added a ``namespace`` function that creates a special object which + allows attribute assignment using the ``set`` tag. This can be used + to carry data across scopes, e.g. from a loop body to code that + comes after the loop. +- Added a ``trimmed`` modifier to ``{% trans %}`` to strip linebreaks + and surrounding whitespace. Also added a new policy to enable this + for all ``trans`` blocks. +- The ``random`` filter is no longer incorrectly constant folded and + will produce a new random choice each time the template is rendered. + :pr:`478` +- Added a ``unique`` filter. :pr:`469` +- Added ``min`` and ``max`` filters. :pr:`475` +- Added tests for all comparison operators: ``eq``, ``ne``, ``lt``, + ``le``, ``gt``, ``ge``. :pr:`665` +- ``import`` statement cannot end with a trailing comma. :pr:`617`, + :pr:`618` +- ``indent`` filter will not indent blank lines by default. :pr:`685` +- Add ``reverse`` argument for ``dictsort`` filter. :pr:`692` +- Add a ``NativeEnvironment`` that renders templates to native Python + types instead of strings. :pr:`708` +- Added filter support to the block ``set`` tag. :pr:`489` +- ``tojson`` filter marks output as safe to match documented behavior. + :pr:`718` +- Resolved a bug where getting debug locals for tracebacks could + modify template context. +- Fixed a bug where having many ``{% elif ... %}`` blocks resulted in + a "too many levels of indentation" error. These blocks now compile + to native ``elif ..:`` instead of ``else: if ..:`` :issue:`759` + + +Version 2.9.6 +------------- + +Released 2017-04-03 + +- Fixed custom context behavior in fast resolve mode :issue:`675` + + +Version 2.9.5 +------------- + +Released 2017-01-28 + +- Restored the original repr of the internal ``_GroupTuple`` because + this caused issues with ansible and it was an unintended change. + :issue:`654` +- Added back support for custom contexts that override the old + ``resolve`` method since it was hard for people to spot that this + could cause a regression. +- Correctly use the buffer for the else block of for loops. This + caused invalid syntax errors to be caused on 2.x and completely + wrong behavior on Python 3 :issue:`669` +- Resolve an issue where the ``{% extends %}`` tag could not be used + with async environments. :issue:`668` +- Reduce memory footprint slightly by reducing our unicode database + dump we use for identifier matching on Python 3 :issue:`666` +- Fixed autoescaping not working for macros in async compilation mode. + :issue:`671` + + +Version 2.9.4 +------------- + +Released 2017-01-10 + +- Solved some warnings for string literals. :issue:`646` +- Increment the bytecode cache version which was not done due to an + oversight before. +- Corrected bad code generation and scoping for filtered loops. + :issue:`649` +- Resolved an issue where top-level output silencing after known + extend blocks could generate invalid code when blocks where + contained in if statements. :issue:`651` +- Made the ``truncate.leeway`` default configurable to improve + compatibility with older templates. + + +Version 2.9.3 +------------- + +Released 2017-01-08 + +- Restored the use of blocks in macros to the extend that was possible + before. On Python 3 it would render a generator repr instead of the + block contents. :issue:`645` +- Set a consistent behavior for assigning of variables in inner scopes + when the variable is also read from an outer scope. This now sets + the intended behavior in all situations however it does not restore + the old behavior where limited assignments to outer scopes was + possible. For more information and a discussion see :issue:`641` +- Resolved an issue where ``block scoped`` would not take advantage of + the new scoping rules. In some more exotic cases a variable + overriden in a local scope would not make it into a block. +- Change the code generation of the ``with`` statement to be in line + with the new scoping rules. This resolves some unlikely bugs in edge + cases. This also introduces a new internal ``With`` node that can be + used by extensions. + + +Version 2.9.2 +------------- + +Released 2017-01-08 + +- Fixed a regression that caused for loops to not be able to use the + same variable for the target as well as source iterator. + :issue:`640` +- Add support for a previously unknown behavior of macros. It used to + be possible in some circumstances to explicitly provide a caller + argument to macros. While badly buggy and unintended it turns out + that this is a common case that gets copy pasted around. To not + completely break backwards compatibility with the most common cases + it's now possible to provide an explicit keyword argument for caller + if it's given an explicit default. :issue:`642` + + +Version 2.9.1 +------------- + +Released 2017-01-07 + +- Resolved a regression with call block scoping for macros. Nested + caller blocks that used the same identifiers as outer macros could + refer to the wrong variable incorrectly. + + +Version 2.9 +----------- + +Released 2017-01-07, codename Derivation + +- Change cache key definition in environment. This fixes a performance + regression introduced in 2.8. +- Added support for ``generator_stop`` on supported Python versions + (Python 3.5 and later) +- Corrected a long standing issue with operator precedence of math + operations not being what was expected. +- Added support for Python 3.6 async iterators through a new async + mode. +- Added policies for filter defaults and similar things. +- Urlize now sets "rel noopener" by default. +- Support attribute fallback for old-style classes in 2.x. +- Support toplevel set statements in extend situations. +- Restored behavior of Cycler for Python 3 users. +- Subtraction now follows the same behavior as other operators on + undefined values. +- ``map`` and friends will now give better error messages if you + forgot to quote the parameter. +- Depend on MarkupSafe 0.23 or higher. +- Improved the ``truncate`` filter to support better truncation in + case the string is barely truncated at all. +- Change the logic for macro autoescaping to be based on the runtime + autoescaping information at call time instead of macro define time. +- Ported a modified version of the ``tojson`` filter from Flask to + Jinja and hooked it up with the new policy framework. +- Block sets are now marked ``safe`` by default. +- On Python 2 the asciification of ASCII strings can now be disabled + with the ``compiler.ascii_str`` policy. +- Tests now no longer accept an arbitrary expression as first argument + but a restricted one. This means that you can now properly use + multiple tests in one expression without extra parentheses. In + particular you can now write ``foo is divisibleby 2 or foo is + divisibleby 3`` as you would expect. +- Greatly changed the scoping system to be more consistent with what + template designers and developers expect. There is now no more magic + difference between the different include and import constructs. + Context is now always propagated the same way. The only remaining + differences is the defaults for ``with context`` and ``without + context``. +- The ``with`` and ``autoescape`` tags are now built-in. +- Added the new ``select_autoescape`` function which helps configuring + better autoescaping easier. +- Fixed a runtime error in the sandbox when attributes of async + generators were accessed. + + +Version 2.8.1 +------------- + +Released 2016-12-29 + +- Fixed the ``for_qs`` flag for ``urlencode``. +- Fixed regression when applying ``int`` to non-string values. +- SECURITY: if the sandbox mode is used format expressions are now + sandboxed with the same rules as in Jinja. This solves various + information leakage problems that can occur with format strings. + + +Version 2.8 +----------- + +Released 2015-07-26, codename Replacement + +- Added ``target`` parameter to urlize function. +- Added support for ``followsymlinks`` to the file system loader. +- The truncate filter now counts the length. +- Added equalto filter that helps with select filters. +- Changed cache keys to use absolute file names if available instead + of load names. +- Fixed loop length calculation for some iterators. +- Changed how Jinja enforces strings to be native strings in Python 2 + to work when people break their default encoding. +- Added ``make_logging_undefined`` which returns an undefined + object that logs failures into a logger. +- If unmarshalling of cached data fails the template will be reloaded + now. +- Implemented a block ``set`` tag. +- Default cache size was increased to 400 from a low 50. +- Fixed ``is number`` test to accept long integers in all Python + versions. +- Changed ``is number`` to accept Decimal as a number. +- Added a check for default arguments followed by non-default + arguments. This change makes ``{% macro m(x, y=1, z) %}`` a syntax + error. The previous behavior for this code was broken anyway + (resulting in the default value being applied to ``y``). +- Add ability to use custom subclasses of + ``jinja2.compiler.CodeGenerator`` and ``jinja2.runtime.Context`` by + adding two new attributes to the environment + (``code_generator_class`` and ``context_class``). :pr:`404` +- Added support for context/environment/evalctx decorator functions on + the finalize callback of the environment. +- Escape query strings for urlencode properly. Previously slashes were + not escaped in that place. +- Add 'base' parameter to 'int' filter. + + +Version 2.7.3 +------------- + +Released 2014-06-06 + +- Security issue: Corrected the security fix for the cache folder. + This fix was provided by RedHat. + + +Version 2.7.2 +------------- + +Released 2014-01-10 + +- Prefix loader was not forwarding the locals properly to inner + loaders. This is now fixed. +- Security issue: Changed the default folder for the filesystem cache + to be user specific and read and write protected on UNIX systems. + See `Debian bug 734747`_ for more information. + +.. _Debian bug 734747: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=734747 + + +Version 2.7.1 +------------- + +Released 2013-08-07 + +- Fixed a bug with ``call_filter`` not working properly on environment + and context filters. +- Fixed lack of Python 3 support for bytecode caches. +- Reverted support for defining blocks in included templates as this + broke existing templates for users. +- Fixed some warnings with hashing of undefineds and nodes if Python + is run with warnings for Python 3. +- Added support for properly hashing undefined objects. +- Fixed a bug with the title filter not working on already uppercase + strings. + + +Version 2.7 +----------- + +Released 2013-05-20, codename Translation + +- Choice and prefix loaders now dispatch source and template lookup + separately in order to work in combination with module loaders as + advertised. +- Fixed filesizeformat. +- Added a non-silent option for babel extraction. +- Added ``urlencode`` filter that automatically quotes values for URL + safe usage with utf-8 as only supported encoding. If applications + want to change this encoding they can override the filter. +- Added ``keep-trailing-newline`` configuration to environments and + templates to optionally preserve the final trailing newline. +- Accessing ``last`` on the loop context no longer causes the iterator + to be consumed into a list. +- Python requirement changed: 2.6, 2.7 or >= 3.3 are required now, + supported by same source code, using the "six" compatibility + library. +- Allow ``contextfunction`` and other decorators to be applied to + ``__call__``. +- Added support for changing from newline to different signs in the + ``wordwrap`` filter. +- Added support for ignoring memcache errors silently. +- Added support for keeping the trailing newline in templates. +- Added finer grained support for stripping whitespace on the left + side of blocks. +- Added ``map``, ``select``, ``reject``, ``selectattr`` and + ``rejectattr`` filters. +- Added support for ``loop.depth`` to figure out how deep inside a + recursive loop the code is. +- Disabled py_compile for pypy and python 3. + + +Version 2.6 +----------- + +Released 2011-07-24, codename Convolution + +- Internal attributes now raise an internal attribute error now + instead of returning an undefined. This fixes problems when passing + undefined objects to Python semantics expecting APIs. +- Traceback support now works properly for PyPy. (Tested with 1.4) +- Implemented operator intercepting for sandboxed environments. This + allows application developers to disable builtin operators for + better security. (For instance limit the mathematical operators to + actual integers instead of longs) +- Groupby filter now supports dotted notation for grouping by + attributes of attributes. +- Scoped blocks now properly treat toplevel assignments and imports. + Previously an import suddenly "disappeared" in a scoped block. +- Automatically detect newer Python interpreter versions before + loading code from bytecode caches to prevent segfaults on invalid + opcodes. The segfault in earlier Jinja versions here was not a + Jinja bug but a limitation in the underlying Python interpreter. If + you notice Jinja segfaulting in earlier versions after an upgrade + of the Python interpreter you don't have to upgrade, it's enough to + flush the bytecode cache. This just no longer makes this necessary, + Jinja will automatically detect these cases now. +- The sum filter can now sum up values by attribute. This is a + backwards incompatible change. The argument to the filter previously + was the optional starting index which defaults to zero. This now + became the second argument to the function because it's rarely used. +- Like sum, sort now also makes it possible to order items by + attribute. +- Like sum and sort, join now also is able to join attributes of + objects as string. +- The internal eval context now has a reference to the environment. +- Added a mapping test to see if an object is a dict or an object with + a similar interface. + + +Version 2.5.5 +------------- + +Released 2010-10-18 + +- Built documentation is no longer part of release. + + +Version 2.5.4 +------------- + +Released 2010-10-17 + +- Fixed extensions not loading properly with overlays. +- Work around a bug in cpython for the debugger that causes segfaults + on 64bit big-endian architectures. + + +Version 2.5.3 +------------- + +Released 2010-10-17 + +- Fixed an operator precedence error introduced in 2.5.2. Statements + like "-foo.bar" had their implicit parentheses applied around the + first part of the expression ("(-foo).bar") instead of the more + correct "-(foo.bar)". + + +Version 2.5.2 +------------- + +Released 2010-08-18 + +- Improved setup.py script to better work with assumptions people + might still have from it (``--with-speedups``). +- Fixed a packaging error that excluded the new debug support. + + +Version 2.5.1 +------------- + +Released 2010-08-17 + +- StopIteration exceptions raised by functions called from templates + are now intercepted and converted to undefineds. This solves a lot + of debugging grief. (StopIteration is used internally to abort + template execution) +- Improved performance of macro calls slightly. +- Babel extraction can now properly extract newstyle gettext calls. +- Using the variable ``num`` in newstyle gettext for something else + than the pluralize count will no longer raise a :exc:`KeyError`. +- Removed builtin markup class and switched to markupsafe. For + backwards compatibility the pure Python implementation still exists + but is pulled from markupsafe by the Jinja developers. The debug + support went into a separate feature called "debugsupport" and is + disabled by default because it is only relevant for Python 2.4 +- Fixed an issue with unary operators having the wrong precedence. + + +Version 2.5 +----------- + +Released 2010-05-29, codename Incoherence + +- Improved the sort filter (should have worked like this for a long + time) by adding support for case insensitive searches. +- Fixed a bug for getattribute constant folding. +- Support for newstyle gettext translations which result in a nicer + in-template user interface and more consistent catalogs. +- It's now possible to register extensions after an environment was + created. + + +Version 2.4.1 +------------- + +Released 2010-04-20 + +- Fixed an error reporting bug for undefined. + + +Version 2.4 +----------- + +Released 2010-04-13, codename Correlation + +- The environment template loading functions now transparently pass + through a template object if it was passed to it. This makes it + possible to import or extend from a template object that was passed + to the template. +- Added a ``ModuleLoader`` that can load templates from + precompiled sources. The environment now features a method to + compile the templates from a configured loader into a zip file or + folder. +- The _speedups C extension now supports Python 3. +- Added support for autoescaping toggling sections and support for + evaluation contexts. +- Extensions have a priority now. + + +Version 2.3.1 +------------- + +Released 2010-02-19 + +- Fixed an error reporting bug on all python versions +- Fixed an error reporting bug on Python 2.4 + + +Version 2.3 +----------- + +Released 2010-02-10, codename 3000 Pythons + +- Fixes issue with code generator that causes unbound variables to be + generated if set was used in if-blocks and other small identifier + problems. +- Include tags are now able to select between multiple templates and + take the first that exists, if a list of templates is given. +- Fixed a problem with having call blocks in outer scopes that have an + argument that is also used as local variable in an inner frame + :issue:`360`. +- Greatly improved error message reporting :pr:`339` +- Implicit tuple expressions can no longer be totally empty. This + change makes ``{% if %}`` a syntax error now. :issue:`364` +- Added support for translator comments if extracted via babel. +- Added with-statement extension. +- Experimental Python 3 support. + + +Version 2.2.1 +------------- + +Released 2009-09-14 + +- Fixes some smaller problems for Jinja on Jython. + + +Version 2.2 +----------- + +Released 2009-09-13, codename Kong + +- Include statements can now be marked with ``ignore missing`` to skip + non existing templates. +- Priority of ``not`` raised. It's now possible to write ``not foo in + bar`` as an alias to ``foo not in bar`` like in python. Previously + the grammar required parentheses (``not (foo in bar)``) which was + odd. +- Fixed a bug that caused syntax errors when defining macros or using + the ``{% call %}`` tag inside loops. +- Fixed a bug in the parser that made ``{{ foo[1, 2] }}`` impossible. +- Made it possible to refer to names from outer scopes in included + templates that were unused in the callers frame :issue:`327` +- Fixed a bug that caused internal errors if names where used as + iteration variable and regular variable *after* the loop if that + variable was unused *before* the loop. :pr:`331` +- Added support for optional ``scoped`` modifier to blocks. +- Added support for line-comments. +- Added the ``meta`` module. +- Renamed (undocumented) attribute "overlay" to "overlayed" on the + environment because it was clashing with a method of the same name. +- Speedup extension is now disabled by default. + + +Version 2.1.1 +------------- + +Released 2008-12-25 + +- Fixed a translation error caused by looping over empty recursive + loops. + + +Version 2.1 +----------- + +Released 2008-11-23, codename Yasuzō + +- Fixed a bug with nested loops and the special loop variable. Before + the change an inner loop overwrote the loop variable from the outer + one after iteration. +- Fixed a bug with the i18n extension that caused the explicit + pluralization block to look up the wrong variable. +- Fixed a limitation in the lexer that made ``{{ foo.0.0 }}`` + impossible. +- Index based subscribing of variables with a constant value returns + an undefined object now instead of raising an index error. This was + a bug caused by eager optimizing. +- The i18n extension looks up ``foo.ugettext`` now followed by + ``foo.gettext`` if an translations object is installed. This makes + dealing with custom translations classes easier. +- Fixed a confusing behavior with conditional extending. loops were + partially executed under some conditions even though they were not + part of a visible area. +- Added ``sort`` filter that works like ``dictsort`` but for arbitrary + sequences. +- Fixed a bug with empty statements in macros. +- Implemented a bytecode cache system. +- The template context is now weakref-able +- Inclusions and imports "with context" forward all variables now, not + only the initial context. +- Added a cycle helper called ``cycler``. +- Added a joining helper called ``joiner``. +- Added a ``compile_expression`` method to the environment that allows + compiling of Jinja expressions into callable Python objects. +- Fixed an escaping bug in urlize + + +Version 2.0 +----------- + +Released 2008-07-17, codename Jinjavitus + +- The subscribing of objects (looking up attributes and items) changed + from slightly. It's now possible to give attributes or items a + higher priority by either using dot-notation lookup or the bracket + syntax. This also changed the AST slightly. ``Subscript`` is gone + and was replaced with ``Getitem`` and ``Getattr``. +- Added support for preprocessing and token stream filtering for + extensions. This would allow extensions to allow simplified gettext + calls in template data and something similar. +- Added ``TemplateStream.dump``. +- Added missing support for implicit string literal concatenation. + ``{{ "foo" "bar" }}`` is equivalent to ``{{ "foobar" }}`` +- ``else`` is optional for conditional expressions. If not given it + evaluates to ``false``. +- Improved error reporting for undefined values by providing a + position. +- ``filesizeformat`` filter uses decimal prefixes now per default and + can be set to binary mode with the second parameter. +- Fixed bug in finalizer + + +Version 2.0rc1 +-------------- + +Released 2008-06-09 + +- First release of Jinja 2. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..f4ba197d --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at report@palletsprojects.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 00000000..7eda0f39 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,215 @@ +How to contribute to Jinja +========================== + +Thank you for considering contributing to Jinja! + + +Support questions +----------------- + +Please, don't use the issue tracker for this. The issue tracker is a +tool to address bugs and feature requests in Jinja itself. Use one of +the following resources for questions about using Jinja or issues with +your own code: + +- The ``#get-help`` channel on our Discord chat: + https://discord.gg/t6rrQZH +- The mailing list flask@python.org for long term discussion or larger + issues. +- Ask on `Stack Overflow`_. Search with Google first using: + ``site:stackoverflow.com jinja {search term, exception message, etc.}`` + +.. _Stack Overflow: https://stackoverflow.com/questions/tagged/jinja?sort=linked + + +Reporting issues +---------------- + +Include the following information in your post: + +- Describe what you expected to happen. +- If possible, include a `minimal reproducible example`_ to help us + identify the issue. This also helps check that the issue is not with + your own code. +- Describe what actually happened. Include the full traceback if there + was an exception. +- List your Python and Jinja versions. If possible, check if this + issue is already fixed in the latest releases or the latest code in + the repository. + +.. _minimal reproducible example: https://stackoverflow.com/help/minimal-reproducible-example + + +Submitting patches +------------------ + +If there is not an open issue for what you want to submit, prefer +opening one for discussion before working on a PR. You can work on any +issue that doesn't have an open PR linked to it or a maintainer assigned +to it. These show up in the sidebar. No need to ask if you can work on +an issue that interests you. + +Include the following in your patch: + +- Use `Black`_ to format your code. This and other tools will run + automatically if you install `pre-commit`_ using the instructions + below. +- Include tests if your patch adds or changes code. Make sure the test + fails without your patch. +- Update any relevant docs pages and docstrings. Docs pages and + docstrings should be wrapped at 72 characters. +- Add an entry in ``CHANGES.rst``. Use the same style as other + entries. Also include ``.. versionchanged::`` inline changelogs in + relevant docstrings. + +.. _Black: https://black.readthedocs.io +.. _pre-commit: https://pre-commit.com + + +First time setup +~~~~~~~~~~~~~~~~ + +- Download and install the `latest version of git`_. +- Configure git with your `username`_ and `email`_. + + .. code-block:: text + + $ git config --global user.name 'your name' + $ git config --global user.email 'your email' + +- Make sure you have a `GitHub account`_. +- Fork Jinja to your GitHub account by clicking the `Fork`_ button. +- `Clone`_ the main repository locally. + + .. code-block:: text + + $ git clone https://github.com/pallets/jinja + $ cd jinja + +- Add your fork as a remote to push your work to. Replace + ``{username}`` with your username. This names the remote "fork", the + default Pallets remote is "origin". + + .. code-block:: text + + git remote add fork https://github.com/{username}/jinja + +- Create a virtualenv. + + .. code-block:: text + + $ python3 -m venv env + $ . env/bin/activate + + On Windows, activating is different. + + .. code-block:: text + + > env\Scripts\activate + +- Install Jinja in editable mode with development dependencies. + + .. code-block:: text + + $ pip install -e . -r requirements/dev.txt + +- Install the pre-commit hooks. + + .. code-block:: text + + $ pre-commit install + +.. _latest version of git: https://git-scm.com/downloads +.. _username: https://help.github.com/en/articles/setting-your-username-in-git +.. _email: https://help.github.com/en/articles/setting-your-commit-email-address-in-git +.. _GitHub account: https://github.com/join +.. _Fork: https://github.com/pallets/jinja/fork +.. _Clone: https://help.github.com/en/articles/fork-a-repo#step-2-create-a-local-clone-of-your-fork + + +Start coding +~~~~~~~~~~~~ + +- Create a branch to identify the issue you would like to work on. If + you're submitting a bug or documentation fix, branch off of the + latest ".x" branch. + + .. code-block:: text + + $ git fetch origin + $ git checkout -b your-branch-name origin/1.1.x + + If you're submitting a feature addition or change, branch off of the + "master" branch. + + .. code-block:: text + + $ git fetch origin + $ git checkout -b your-branch-name origin/master + +- Using your favorite editor, make your changes, + `committing as you go`_. +- Include tests that cover any code changes you make. Make sure the + test fails without your patch. Run the tests as described below. +- Push your commits to your fork on GitHub and + `create a pull request`_. Link to the issue being addressed with + ``fixes #123`` in the pull request. + + .. code-block:: text + + $ git push --set-upstream fork your-branch-name + +.. _committing as you go: https://dont-be-afraid-to-commit.readthedocs.io/en/latest/git/commandlinegit.html#commit-your-changes +.. _create a pull request: https://help.github.com/en/articles/creating-a-pull-request + + +Running the tests +~~~~~~~~~~~~~~~~~ + +Run the basic test suite with pytest. + +.. code-block:: text + + $ pytest + +This runs the tests for the current environment, which is usually +sufficient. CI will run the full suite when you submit your pull +request. You can run the full test suite with tox if you don't want to +wait. + +.. code-block:: text + + $ tox + + +Running test coverage +~~~~~~~~~~~~~~~~~~~~~ + +Generating a report of lines that do not have test coverage can indicate +where to start contributing. Run ``pytest`` using ``coverage`` and +generate a report. + +.. code-block:: text + + $ pip install coverage + $ coverage run -m pytest + $ coverage html + +Open ``htmlcov/index.html`` in your browser to explore the report. + +Read more about `coverage <https://coverage.readthedocs.io>`__. + + +Building the docs +~~~~~~~~~~~~~~~~~ + +Build the docs in the ``docs`` directory using Sphinx. + +.. code-block:: text + + $ cd docs + $ make html + +Open ``_build/html/index.html`` in your browser to view the docs. + +Read more about `Sphinx <https://www.sphinx-doc.org/en/stable/>`__. diff --git a/LICENSE.rst b/LICENSE.rst new file mode 100644 index 00000000..c37cae49 --- /dev/null +++ b/LICENSE.rst @@ -0,0 +1,28 @@ +Copyright 2007 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..8690e355 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,9 @@ +include CHANGES.rst +include tox.ini +include requirements/*.txt +graft artwork +graft docs +prune docs/_build +graft examples +graft tests +global-exclude *.pyc diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..060b19ef --- /dev/null +++ b/README.rst @@ -0,0 +1,66 @@ +Jinja +===== + +Jinja is a fast, expressive, extensible templating engine. Special +placeholders in the template allow writing code similar to Python +syntax. Then the template is passed data to render the final document. + +It includes: + +- Template inheritance and inclusion. +- Define and import macros within templates. +- HTML templates can use autoescaping to prevent XSS from untrusted + user input. +- A sandboxed environment can safely render untrusted templates. +- AsyncIO support for generating templates and calling async + functions. +- I18N support with Babel. +- Templates are compiled to optimized Python code just-in-time and + cached, or can be compiled ahead-of-time. +- Exceptions point to the correct line in templates to make debugging + easier. +- Extensible filters, tests, functions, and even syntax. + +Jinja's philosophy is that while application logic belongs in Python if +possible, it shouldn't make the template designer's job difficult by +restricting functionality too much. + + +Installing +---------- + +Install and update using `pip`_: + +.. code-block:: text + + $ pip install -U Jinja2 + +.. _pip: https://pip.pypa.io/en/stable/quickstart/ + + +In A Nutshell +------------- + +.. code-block:: jinja + + {% extends "base.html" %} + {% block title %}Members{% endblock %} + {% block content %} + <ul> + {% for user in users %} + <li><a href="{{ user.url }}">{{ user.username }}</a></li> + {% endfor %} + </ul> + {% endblock %} + + +Links +----- + +- Website: https://palletsprojects.com/p/jinja/ +- Documentation: https://jinja.palletsprojects.com/ +- Releases: https://pypi.org/project/Jinja2/ +- Code: https://github.com/pallets/jinja +- Issue tracker: https://github.com/pallets/jinja/issues +- Test status: https://dev.azure.com/pallets/jinja/_build +- Official chat: https://discord.gg/t6rrQZH diff --git a/artwork/jinjalogo.svg b/artwork/jinjalogo.svg new file mode 100644 index 00000000..0bc9ea4e --- /dev/null +++ b/artwork/jinjalogo.svg @@ -0,0 +1,132 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://web.resource.org/cc/" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="300" + height="120" + id="svg2" + sodipodi:version="0.32" + inkscape:version="0.45.1" + version="1.0" + sodipodi:docbase="/Users/mitsuhiko/Development/jinja2/artwork" + sodipodi:docname="jinjalogo.svg" + inkscape:export-filename="/Users/mitsuhiko/Development/jinja2/docs/_static/jinjabanner.png" + inkscape:export-xdpi="60" + inkscape:export-ydpi="60" + inkscape:output_extension="org.inkscape.output.svg.inkscape"> + <defs + id="defs4"> + <linearGradient + id="linearGradient6558"> + <stop + style="stop-color:#575757;stop-opacity:1;" + offset="0" + id="stop6560" /> + <stop + style="stop-color:#2f2f2f;stop-opacity:1;" + offset="1" + id="stop6562" /> + </linearGradient> + <radialGradient + inkscape:collect="always" + xlink:href="#linearGradient6558" + id="radialGradient6564" + cx="61.297766" + cy="60.910986" + fx="61.297766" + fy="60.910986" + r="44.688254" + gradientTransform="matrix(1,0,0,0.945104,0,3.343747)" + gradientUnits="userSpaceOnUse" /> + <radialGradient + inkscape:collect="always" + xlink:href="#linearGradient6558" + id="radialGradient6580" + gradientUnits="userSpaceOnUse" + gradientTransform="matrix(1,0,0,0.945104,0.355158,3.334402)" + cx="61.297766" + cy="60.910986" + fx="61.297766" + fy="60.910986" + r="44.688254" /> + <linearGradient + inkscape:collect="always" + xlink:href="#linearGradient6558" + id="linearGradient4173" + x1="255.15521" + y1="32.347946" + x2="279.8912" + y2="32.347946" + gradientUnits="userSpaceOnUse" + gradientTransform="matrix(0.8073249,0,0,0.8073249,57.960878,7.4036303)" /> + <linearGradient + inkscape:collect="always" + xlink:href="#linearGradient6558" + id="linearGradient5145" + gradientUnits="userSpaceOnUse" + gradientTransform="matrix(0.7902775,0,0,0.82474,60.019977,8.0684132)" + x1="255.15521" + y1="32.347946" + x2="279.8912" + y2="32.347946" /> + </defs> + <sodipodi:namedview + id="base" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + gridtolerance="10000" + guidetolerance="10" + objecttolerance="10" + inkscape:pageopacity="0.0" + inkscape:pageshadow="2" + inkscape:zoom="2.8" + inkscape:cx="137.4752" + inkscape:cy="57.574575" + inkscape:document-units="px" + inkscape:current-layer="layer1" + width="300px" + height="120px" + showguides="true" + inkscape:guide-bbox="true" + inkscape:window-width="1396" + inkscape:window-height="900" + inkscape:window-x="0" + inkscape:window-y="22" /> + <metadata + id="metadata7"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + </cc:Work> + </rdf:RDF> + </metadata> + <g + inkscape:label="Layer 1" + inkscape:groupmode="layer" + id="layer1"> + <path + style="font-size:12px;font-style:normal;font-weight:normal;fill:#f4f4f4;fill-opacity:1;stroke:#e7e7e7;stroke-width:0.8;stroke-linecap:butt;stroke-linejoin:round;stroke-opacity:1;font-family:Bitstream Vera Sans;stroke-miterlimit:4;stroke-dasharray:none" + d="M 165.36463,80.874808 L 165.36463,80.874808 L 153.32556,80.874808 L 153.32556,81.8344 L 147.64994,81.8344 L 147.64994,36.035583 L 165.36463,36.035583 L 165.36463,20.333129 C 170.58154,21.031083 173.07533,22.077914 172.84609,23.473621 C 172.78871,24.055258 172.21545,24.549594 171.12624,24.956624 L 171.12624,36.035583 L 189.09895,36.035583 L 189.09895,82.532286 L 183.33733,82.532286 L 183.33733,80.874808 L 171.12624,80.874808 L 171.12624,102.94548 L 165.36463,102.94548 L 165.36463,80.874808 M 153.32556,55.489173 L 153.32556,55.489173 L 165.36463,55.489173 L 165.36463,41.793146 L 153.32556,41.793146 L 153.32556,55.489173 M 171.12624,55.489173 L 171.12624,55.489173 L 183.33733,55.489173 L 183.33733,41.793146 L 171.12624,41.793146 L 171.12624,55.489173 M 183.33733,61.333977 L 183.33733,61.333977 L 171.12624,61.333977 L 171.12624,75.030006 L 183.33733,75.030006 L 183.33733,61.333977 M 165.36463,61.333977 L 165.36463,61.333977 L 153.32556,61.333977 L 153.32556,75.030006 L 165.36463,75.030006 L 165.36463,61.333977 M 132.85897,59.414792 C 137.33069,63.136883 140.99969,67.934848 143.86618,73.808701 L 139.13654,77.385372 C 137.24467,72.965445 134.6362,69.12707 131.31114,65.87024 L 131.31114,102.94548 L 125.63554,102.94548 L 125.63554,68.57455 C 122.31042,71.947693 118.52671,74.913707 114.28436,77.47261 L 109.64069,73.372526 C 121.50782,67.091566 130.62312,55.489212 136.98668,38.565417 L 116.26221,38.565417 L 116.26221,32.720615 L 125.80754,32.720615 L 125.80754,20.333129 C 130.85245,21.031083 133.31761,22.048838 133.20299,23.386383 C 133.14561,24.026183 132.57235,24.549594 131.48307,24.956624 L 131.48307,32.720615 L 140.77043,32.720615 L 143.60824,36.733469 C 140.68444,45.51526 137.10137,53.075692 132.85897,59.414792 M 254.11016,49.469901 L 254.11016,49.469901 L 254.11016,20.333129 C 259.21243,21.031083 261.67755,22.048838 261.50562,23.386383 C 261.44823,23.909869 261.04699,24.346044 260.30172,24.694917 C 260.30164,24.694986 260.30164,24.694986 260.30172,24.694917 L 260.30172,24.694917 L 259.78578,24.956624 L 259.78578,49.469901 L 277.15652,49.469901 L 277.15652,55.227471 L 259.78578,55.227471 L 259.78578,93.785712 L 281.45616,93.785712 L 281.45616,99.63051 L 232.35378,99.63051 L 232.35378,93.785712 L 254.11016,93.785712 L 254.11016,55.227471 L 236.22346,55.227471 L 236.22346,49.469901 L 254.11016,49.469901 M 225.5603,59.327554 C 231.12111,63.107798 235.62145,67.876693 239.06127,73.634235 L 234.76157,77.647079 C 231.60845,72.180322 227.82475,67.934848 223.41044,64.910648 L 223.41044,102.94548 L 217.73484,102.94548 L 217.73484,67.44049 C 212.91919,71.627831 207.70222,75.030021 202.084,77.647079 L 197.87027,73.198053 C 212.66118,66.917101 224.01239,55.372897 231.92377,38.565417 L 205.35172,38.565417 L 205.35172,32.720615 L 217.99283,32.720615 L 217.99283,20.333129 C 223.03774,21.031083 225.50291,22.048838 225.38829,23.386383 C 225.33089,24.026183 224.75765,24.549594 223.66837,24.956624 L 223.66837,32.720615 L 236.22346,32.720615 L 238.80326,36.733469 C 235.13421,45.51526 230.71987,53.046611 225.5603,59.327554" + id="text4761" /> + <path + style="font-size:44.09793472px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;writing-mode:lr-tb;text-anchor:start;fill:#b41717;fill-opacity:1;stroke:#7f2828;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;font-family:Candara;stroke-miterlimit:4;stroke-dasharray:none" + d="M 149.14708,37.774469 C 148.97807,41.117899 148.84526,44.824225 148.74871,48.893456 C 148.67626,52.962754 148.3818,70.641328 148.38184,75.524422 C 148.3818,79.065795 148.05588,81.991266 147.40406,84.300835 C 146.75219,86.610422 145.72612,88.557071 144.32585,90.140779 C 142.94969,91.724494 141.17522,92.901283 139.00239,93.671139 C 136.82953,94.440996 134.22211,94.825935 131.18014,94.825935 C 128.83828,94.825935 126.73787,94.59498 124.87889,94.133049 L 125.4221,89.31593 C 127.13623,90.0418 128.92278,90.404734 130.78177,90.404733 C 132.85805,90.404734 134.66875,90.140782 136.2139,89.612876 C 137.78315,89.062981 139.02651,88.216133 139.94396,87.072335 C 140.8855,85.928548 141.54942,84.520804 141.93572,82.8491 C 142.34613,81.177412 142.55134,78.988811 142.55136,76.283285 C 142.55134,66.297119 142.62852,44.659257 142.26641,37.774469 L 149.14708,37.774469 M 166.38498,80.732697 L 159.83024,80.732697 C 160.16821,76.333498 160.33723,71.307412 160.33723,65.654424 C 160.33723,59.2976 159.91471,53.963567 159.06973,49.652319 L 166.31257,48.761483 C 166.02284,53.358679 165.87799,58.98965 165.87799,65.654424 C 165.87799,70.933479 166.04699,75.959565 166.38498,80.732697 M 167.90601,39.490159 C 167.90598,40.611994 167.5076,41.590815 166.7109,42.42662 C 165.91418,43.240515 164.79155,43.647442 163.343,43.647399 C 162.11172,43.647442 161.146,43.295504 160.44588,42.591595 C 159.76988,41.865769 159.43188,40.996927 159.43188,39.98507 C 159.43188,38.885304 159.84231,37.928485 160.66315,37.114591 C 161.48399,36.30078 162.61869,35.893853 164.06727,35.893811 C 165.25023,35.893853 166.17975,36.256783 166.85575,36.982609 C 167.55588,37.686526 167.90598,38.522373 167.90601,39.490159 M 206.72748,80.732697 L 200.13651,80.732697 C 200.66763,74.947749 200.93319,68.634899 200.9332,61.794122 C 200.93319,58.406756 200.1727,56.097177 198.65174,54.865371 C 197.15487,53.61163 195.00619,52.984747 192.20564,52.984714 C 188.77731,52.984747 185.61465,54.117535 182.71753,56.383099 C 182.71753,63.883761 182.76583,72.000287 182.86238,80.732697 L 176.27142,80.732697 C 176.68182,73.254058 176.88707,67.843042 176.88707,64.499632 C 176.88707,59.352589 176.3559,54.359493 175.29363,49.520339 L 181.66734,48.695493 L 182.35539,52.720761 L 182.64511,52.720761 C 186.21823,49.773323 190.04483,48.299592 194.12499,48.299567 C 198.13265,48.299592 201.23499,49.113454 203.43201,50.741118 C 205.62895,52.346863 206.72747,55.217334 206.72748,59.352563 C 206.72747,59.770507 206.70331,60.595362 206.65507,61.827118 C 206.60675,63.058915 206.5826,63.883761 206.58262,64.30167 C 206.5826,67.975018 206.63088,73.452022 206.72748,80.732697 M 222.69791,48.695493 C 222.28747,55.514282 222.08225,62.355041 222.08225,69.21778 C 222.08225,71.043461 222.14262,73.463019 222.26332,76.476468 C 222.40822,79.467925 222.4806,81.502559 222.48063,82.580363 C 222.4806,89.685068 219.51105,93.996287 213.57195,95.514024 L 211.76124,93.006484 C 213.90995,91.356766 215.2378,89.597085 215.74478,87.727431 C 216.49321,85.043912 216.86743,79.324953 216.86743,70.570535 C 216.86743,61.178248 216.3846,54.16153 215.41887,49.520339 L 222.69791,48.695493 M 224.2551,39.490159 C 224.2551,40.611994 223.85673,41.590815 223.06006,42.42662 C 222.26332,43.240515 221.14069,43.647442 219.69213,43.647399 C 218.46084,43.647442 217.49515,43.295504 216.795,42.591595 C 216.119,41.865769 215.781,40.996927 215.781,39.98507 C 215.781,38.885304 216.19144,37.928485 217.01231,37.114591 C 217.83316,36.30078 218.96785,35.893853 220.4164,35.893811 C 221.5994,35.893853 222.52889,36.256783 223.20492,36.982609 C 223.90503,37.686526 224.2551,38.522373 224.2551,39.490159 M 259.60008,80.732697 L 253.91446,80.930661 C 253.62473,79.852857 253.47987,78.830045 253.4799,77.862216 L 253.11774,77.862216 C 250.14817,80.325772 246.10427,81.557546 240.98606,81.557547 C 238.20962,81.557546 235.8195,80.820682 233.81563,79.346948 C 231.81178,77.851221 230.80988,75.728607 230.80988,72.979099 C 230.80988,69.591724 232.37914,66.875216 235.51769,64.829574 C 238.65625,62.761967 244.48667,61.67316 253.00913,61.563165 C 253.08155,61.035275 253.11772,60.430386 253.11774,59.748497 C 253.11772,57.043003 252.32104,55.239336 250.72765,54.337474 C 249.15832,53.435661 246.76819,52.984747 243.55721,52.984714 C 239.76681,52.984747 236.03678,53.413668 232.3671,54.271484 L 232.9827,49.718301 C 236.60411,48.77251 240.76873,48.299592 245.47658,48.299567 C 249.77395,48.299592 253.09359,49.113454 255.43545,50.741118 C 257.77728,52.346863 258.94819,55.096363 258.94824,58.989625 C 258.94819,60.023469 258.88785,61.904117 258.76715,64.631608 C 258.67054,67.337133 258.62228,69.140806 258.6223,70.042632 C 258.62228,74.045913 258.94819,77.609265 259.60008,80.732697 M 253.19019,74.331856 C 253.06945,70.988469 253.00909,67.986016 253.00913,65.324484 C 248.47027,65.324498 245.01786,65.632443 242.65187,66.248318 C 238.69248,67.348131 236.71278,69.448748 236.71278,72.550177 C 236.71278,75.541643 239.03044,77.037371 243.66588,77.037366 C 247.64942,77.037371 250.82416,76.135534 253.19019,74.331856" + id="text3736" + sodipodi:nodetypes="ccsscssccscccsccccsccsccsscsssccccscscccsccccscsssccscscccscccsscsssccccccscsccscsccscscsccccssc" /> + <path + style="fill:url(#radialGradient6564);fill-opacity:1.0;fill-rule:evenodd;stroke:#323232;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:none" + d="M 105.45673,18.675923 C 105.45673,18.675923 88.211949,26.918461 74.172834,28.737898 C 60.133727,30.557333 33.360434,32.377571 28.045622,31.093256 C 22.730818,29.808941 18.915645,28.309196 18.915645,28.309196 L 20.021441,32.056583 L 16.609513,35.052471 L 17.2144,36.121726 L 18.61792,36.22764 L 22.92773,36.762252 L 23.532621,38.688909 L 25.937975,38.905784 L 27.143021,42.970927 C 27.143021,42.970927 32.254764,43.399628 33.758953,43.399628 C 35.263142,43.399628 38.271966,43.187802 38.271966,43.187802 L 38.371202,44.791657 L 39.477002,45.003495 L 39.477002,46.824227 L 37.066917,48.967759 L 37.671807,49.073671 L 37.671807,49.820127 C 37.671807,49.820127 32.255457,50.252157 30.049301,49.93109 C 27.843157,49.610006 27.440747,49.608286 27.440747,49.608286 L 27.242258,49.820127 L 27.143021,50.783455 L 27.643946,50.783455 L 27.84242,54.959544 L 38.976091,54.530844 L 38.172728,68.980747 L 38.073481,70.796442 L 28.645781,70.261816 L 28.546544,66.408513 L 30.649462,66.408513 L 30.852673,64.910557 L 32.757107,64.481857 L 33.059555,64.058192 L 25.937975,62.343374 L 20.522364,63.947229 L 21.42496,64.698732 L 22.327572,64.698732 L 22.426809,65.984848 L 24.331254,66.09076 L 24.331254,69.838147 L 22.228335,70.372777 L 22.630009,71.225146 L 23.130934,71.547931 L 23.130934,74.437917 L 24.435218,74.437917 L 24.435218,87.813529 L 22.327572,88.13632 L 22.630009,91.989617 L 23.929569,92.206492 L 23.731093,100.98236 L 29.449141,101.08826 L 28.244105,92.418334 L 36.868446,92.206492 L 36.268285,96.912181 L 35.464925,100.23086 L 44.188501,100.33677 L 44.287739,91.777793 L 50.303506,91.243181 L 50.005786,96.700351 L 49.802585,99.90807 L 54.920484,99.90807 L 54.717274,91.132217 L 55.421397,91.243181 L 55.619882,87.067076 L 54.816521,87.067076 L 54.518798,85.352258 L 54.017874,80.429702 L 54.216359,74.760706 L 55.31743,74.760706 L 55.31743,71.336105 L 53.913913,71.442015 L 54.117112,67.402096 L 55.747469,67.240708 L 55.823083,65.929374 L 56.749319,65.793192 L 57.699176,65.071956 L 51.985842,63.896802 L 46.31977,65.15265 L 46.872668,66.060507 L 47.47283,66.010066 L 48.172228,65.984848 L 48.299828,67.639144 L 49.878196,67.563497 L 49.906548,71.144447 L 43.111042,70.988097 L 43.337879,67.160002 L 43.559978,63.679927 L 43.559978,59.105378 L 43.763188,54.288748 L 57.373101,53.592733 L 73.567955,52.659674 L 73.71917,55.736265 L 73.142647,63.120082 L 72.892183,69.9945 L 66.928387,69.888585 L 66.900039,65.071956 L 69.106918,64.991267 L 69.206169,63.629486 L 70.108765,63.493308 L 70.061506,63.226006 L 70.964116,63.175568 L 71.465028,62.504773 L 64.721507,60.926122 L 58.001612,62.368592 L 58.4789,63.200785 L 59.230285,63.1453 L 59.230285,63.523577 L 60.156518,63.523577 L 60.156518,65.046738 L 62.136575,65.071956 L 62.112937,69.298485 L 60.109259,69.298485 L 60.080907,70.261816 L 60.785031,70.342507 L 60.70942,74.009202 L 62.188552,74.089909 L 62.013701,88.620507 L 60.057282,89.018952 L 60.080907,89.714967 L 60.761406,89.714967 L 60.761406,93.437137 L 61.886113,93.437137 L 61.588391,98.52109 L 61.210343,102.95945 L 68.331912,103.14605 L 68.105084,99.29275 L 67.580538,96.085028 L 67.476575,93.300955 L 73.520696,93.195041 L 73.345845,97.502272 L 73.317494,102.05159 L 76.729426,102.3189 L 81.3653,102.1323 L 82.820807,101.70358 L 82.017437,99.26753 L 81.818959,95.439438 L 81.440912,92.710853 L 87.206218,92.499027 L 86.955759,95.842931 L 86.932133,101.08826 L 89.238253,101.30009 L 91.520751,101.24965 L 92.621828,100.90165 L 91.969693,95.923633 L 91.747577,92.176239 L 92.725793,92.070324 L 92.749427,88.726422 L 93.02352,88.670945 L 92.976244,87.949712 L 91.846823,87.949712 L 91.619996,85.488427 L 91.520751,74.811143 L 92.371377,74.785924 L 92.371377,71.280616 L 92.725793,71.336105 L 92.725793,70.640088 L 91.468773,70.529127 L 91.497126,66.463987 L 93.600043,66.277382 L 93.477182,64.910557 L 94.403419,64.829863 L 94.351424,64.562549 L 95.580099,63.947229 L 89.337489,62.69138 L 82.995657,63.977495 L 83.39733,64.723951 L 84.375543,64.643256 L 84.427528,64.966046 L 85.254515,64.966046 L 85.301775,66.569901 L 87.357445,66.544681 L 87.532293,70.478688 L 80.264217,70.423216 L 79.413593,64.512124 L 78.733106,61.380041 L 78.184923,55.761484 L 78.510996,52.473053 L 92.999878,51.373557 L 93.047136,46.476221 L 93.774891,46.289613 L 93.727651,45.543159 L 93.174743,45.220372 C 93.174629,45.220372 85.252181,46.395266 82.745197,46.66284 C 82.0389,46.738209 82.09239,46.733258 81.516524,46.79397 L 81.440912,45.886118 L 78.444837,44.317564 L 78.482644,42.491786 L 79.512842,42.461518 L 79.588444,39.949808 C 79.588444,39.949808 85.728225,39.546834 88.009582,39.0117 C 90.290937,38.476559 93.524432,37.942456 93.524432,37.942456 L 95.055545,33.79662 L 98.089437,32.913987 L 98.339888,32.217972 L 105.20628,30.316548 L 105.98602,29.676006 L 103.37744,23.976741 L 103.62792,22.690624 L 104.95584,21.994611 L 105.91041,19.079404 L 105.45673,18.675923 z M 72.466874,40.403728 L 72.429067,42.476654 L 73.983813,42.542211 L 73.884576,44.509221 L 70.836515,46.506487 L 70.647496,47.081457 L 71.876167,47.091543 L 71.866712,47.575729 L 62.552432,48.029652 L 62.613863,46.652742 L 63.039175,45.966809 L 63.067524,45.528025 L 63.07698,44.579832 L 63.341609,43.949374 L 63.440849,43.439982 L 63.440849,43.076841 L 63.842533,41.47297 L 72.466874,40.403728 z M 52.987688,42.168984 L 52.760853,43.561027 L 53.488599,44.418431 L 53.441349,45.916386 L 54.117112,46.960408 L 53.942262,48.191039 L 54.443185,48.912273 L 44.939872,49.2855 L 44.916247,48.967759 L 46.017333,48.831579 L 46.069307,48.428097 L 43.66394,47.121797 L 43.536351,45.03375 L 44.689411,44.978276 L 44.788661,42.72883 L 52.987688,42.168984 z M 67.051262,74.276518 L 72.81657,74.649742 L 72.618099,82.411833 L 73.36947,88.776857 L 67.254465,88.565018 L 67.051262,74.276518 z M 28.44258,74.599304 L 37.671807,75.078442 L 36.868446,80.429702 L 36.868446,84.928593 L 37.520583,87.440302 L 28.494569,87.869006 L 28.44258,74.599304 z M 87.508658,74.649742 L 87.508658,87.924488 L 81.644113,88.353194 L 81.440912,81.342592 L 80.788764,74.811143 L 87.508658,74.649742 z M 43.087416,74.947312 L 49.906548,74.972531 L 49.977434,87.278902 L 43.611966,87.389863 L 43.285891,83.400379 L 43.262266,79.441156 L 43.087416,74.947312 z " + id="path4735" /> + </g> +</svg> diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..51285967 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,19 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/jinja-logo-sidebar.png b/docs/_static/jinja-logo-sidebar.png Binary files differnew file mode 100644 index 00000000..455b4c3c --- /dev/null +++ b/docs/_static/jinja-logo-sidebar.png diff --git a/docs/_static/jinja-logo.png b/docs/_static/jinja-logo.png Binary files differnew file mode 100644 index 00000000..7f8ca5bb --- /dev/null +++ b/docs/_static/jinja-logo.png diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 00000000..ec083a8a --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,881 @@ +API +=== + +.. module:: jinja2 + :noindex: + :synopsis: public Jinja API + +This document describes the API to Jinja and not the template language +(for that, see :doc:`/templates`). It will be most useful as reference +to those implementing the template interface to the application and not +those who are creating Jinja templates. + +Basics +------ + +Jinja uses a central object called the template :class:`Environment`. +Instances of this class are used to store the configuration and global objects, +and are used to load templates from the file system or other locations. +Even if you are creating templates from strings by using the constructor of +:class:`Template` class, an environment is created automatically for you, +albeit a shared one. + +Most applications will create one :class:`Environment` object on application +initialization and use that to load templates. In some cases however, it's +useful to have multiple environments side by side, if different configurations +are in use. + +The simplest way to configure Jinja to load templates for your +application is to use :class:`~loaders.PackageLoader`. + +.. code-block:: python + + from jinja2 import Environment, PackageLoader, select_autoescape + env = Environment( + loader=PackageLoader("yourapp"), + autoescape=select_autoescape() + ) + +This will create a template environment with a loader that looks up +templates in the ``templates`` folder inside the ``yourapp`` Python +package (or next to the ``yourapp.py`` Python module). It also enables +autoescaping for HTML files. This loader only requires that ``yourapp`` +is importable, it figures out the absolute path to the folder for you. + +Different loaders are available to load templates in other ways or from +other locations. They're listed in the `Loaders`_ section below. You can +also write your own if you want to load templates from a source that's +more specialized to your project. + +To load a template from this environment, call the :meth:`get_template` +method, which returns the loaded :class:`Template`. + +.. code-block:: python + + template = env.get_template("mytemplate.html") + +To render it with some variables, call the :meth:`render` method. + +.. code-block:: python + + print(template.render(the="variables", go="here")) + +Using a template loader rather than passing strings to :class:`Template` +or :meth:`Environment.from_string` has multiple advantages. Besides being +a lot easier to use it also enables template inheritance. + +.. admonition:: Notes on Autoescaping + + In future versions of Jinja we might enable autoescaping by default + for security reasons. As such you are encouraged to explicitly + configure autoescaping now instead of relying on the default. + + +High Level API +-------------- + +The high-level API is the API you will use in the application to load and +render Jinja templates. The :ref:`low-level-api` on the other side is only +useful if you want to dig deeper into Jinja or :ref:`develop extensions +<jinja-extensions>`. + +.. autoclass:: Environment([options]) + :members: from_string, get_template, select_template, + get_or_select_template, join_path, extend, compile_expression, + compile_templates, list_templates, add_extension + + .. attribute:: shared + + If a template was created by using the :class:`Template` constructor + an environment is created automatically. These environments are + created as shared environments which means that multiple templates + may have the same anonymous environment. For all shared environments + this attribute is `True`, else `False`. + + .. attribute:: sandboxed + + If the environment is sandboxed this attribute is `True`. For the + sandbox mode have a look at the documentation for the + :class:`~jinja2.sandbox.SandboxedEnvironment`. + + .. attribute:: filters + + A dict of filters for this environment. As long as no template was + loaded it's safe to add new filters or remove old. For custom filters + see :ref:`writing-filters`. For valid filter names have a look at + :ref:`identifier-naming`. + + .. attribute:: tests + + A dict of test functions for this environment. As long as no + template was loaded it's safe to modify this dict. For custom tests + see :ref:`writing-tests`. For valid test names have a look at + :ref:`identifier-naming`. + + .. attribute:: globals + + A dict of global variables. These variables are always available + in a template. As long as no template was loaded it's safe + to modify this dict. For more details see :ref:`global-namespace`. + For valid object names have a look at :ref:`identifier-naming`. + + .. attribute:: policies + + A dictionary with :ref:`policies`. These can be reconfigured to + change the runtime behavior or certain template features. Usually + these are security related. + + .. attribute:: code_generator_class + + The class used for code generation. This should not be changed + in most cases, unless you need to modify the Python code a + template compiles to. + + .. attribute:: context_class + + The context used for templates. This should not be changed + in most cases, unless you need to modify internals of how + template variables are handled. For details, see + :class:`~jinja2.runtime.Context`. + + .. automethod:: overlay([options]) + + .. method:: undefined([hint, obj, name, exc]) + + Creates a new :class:`Undefined` object for `name`. This is useful + for filters or functions that may return undefined objects for + some operations. All parameters except of `hint` should be provided + as keyword parameters for better readability. The `hint` is used as + error message for the exception if provided, otherwise the error + message will be generated from `obj` and `name` automatically. The exception + provided as `exc` is raised if something with the generated undefined + object is done that the undefined object does not allow. The default + exception is :exc:`UndefinedError`. If a `hint` is provided the + `name` may be omitted. + + The most common way to create an undefined object is by providing + a name only:: + + return environment.undefined(name='some_name') + + This means that the name `some_name` is not defined. If the name + was from an attribute of an object it makes sense to tell the + undefined object the holder object to improve the error message:: + + if not hasattr(obj, 'attr'): + return environment.undefined(obj=obj, name='attr') + + For a more complex example you can provide a hint. For example + the :func:`first` filter creates an undefined object that way:: + + return environment.undefined('no first item, sequence was empty') + + If it the `name` or `obj` is known (for example because an attribute + was accessed) it should be passed to the undefined object, even if + a custom `hint` is provided. This gives undefined objects the + possibility to enhance the error message. + +.. autoclass:: Template + :members: module, make_module + + .. attribute:: globals + + The dict with the globals of that template. It's unsafe to modify + this dict as it may be shared with other templates or the environment + that loaded the template. + + .. attribute:: name + + The loading name of the template. If the template was loaded from a + string this is `None`. + + .. attribute:: filename + + The filename of the template on the file system if it was loaded from + there. Otherwise this is `None`. + + .. automethod:: render([context]) + + .. automethod:: generate([context]) + + .. automethod:: stream([context]) + + .. automethod:: render_async([context]) + + .. automethod:: generate_async([context]) + + +.. autoclass:: jinja2.environment.TemplateStream() + :members: disable_buffering, enable_buffering, dump + + +Autoescaping +------------ + +.. versionchanged:: 2.4 + +Jinja now comes with autoescaping support. As of Jinja 2.9 the +autoescape extension is removed and built-in. However autoescaping is +not yet enabled by default though this will most likely change in the +future. It's recommended to configure a sensible default for +autoescaping. This makes it possible to enable and disable autoescaping +on a per-template basis (HTML versus text for instance). + +.. autofunction:: jinja2.select_autoescape + +Here a recommended setup that enables autoescaping for templates ending +in ``'.html'``, ``'.htm'`` and ``'.xml'`` and disabling it by default +for all other extensions. You can use the :func:`~jinja2.select_autoescape` +function for this:: + + from jinja2 import Environment, select_autoescape + env = Environment(autoescape=select_autoescape(['html', 'htm', 'xml']), + loader=PackageLoader('mypackage')) + +The :func:`~jinja.select_autoescape` function returns a function that +works roughly like this:: + + def autoescape(template_name): + if template_name is None: + return False + if template_name.endswith(('.html', '.htm', '.xml')) + +When implementing a guessing autoescape function, make sure you also +accept `None` as valid template name. This will be passed when generating +templates from strings. You should always configure autoescaping as +defaults in the future might change. + +Inside the templates the behaviour can be temporarily changed by using +the `autoescape` block (see :ref:`autoescape-overrides`). + + +.. _identifier-naming: + +Notes on Identifiers +-------------------- + +Jinja uses Python naming rules. Valid identifiers can be any combination +of characters accepted by Python. + +Filters and tests are looked up in separate namespaces and have slightly +modified identifier syntax. Filters and tests may contain dots to group +filters and tests by topic. For example it's perfectly valid to add a +function into the filter dict and call it `to.str`. The regular +expression for filter and test identifiers is +``[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*```. + + +Undefined Types +--------------- + +These classes can be used as undefined types. The :class:`Environment` +constructor takes an `undefined` parameter that can be one of those classes +or a custom subclass of :class:`Undefined`. Whenever the template engine is +unable to look up a name or access an attribute one of those objects is +created and returned. Some operations on undefined values are then allowed, +others fail. + +The closest to regular Python behavior is the :class:`StrictUndefined` which +disallows all operations beside testing if it's an undefined object. + +.. autoclass:: jinja2.Undefined() + + .. attribute:: _undefined_hint + + Either `None` or a string with the error message for the + undefined object. + + .. attribute:: _undefined_obj + + Either `None` or the owner object that caused the undefined object + to be created (for example because an attribute does not exist). + + .. attribute:: _undefined_name + + The name for the undefined variable / attribute or just `None` + if no such information exists. + + .. attribute:: _undefined_exception + + The exception that the undefined object wants to raise. This + is usually one of :exc:`UndefinedError` or :exc:`SecurityError`. + + .. method:: _fail_with_undefined_error(\*args, \**kwargs) + + When called with any arguments this method raises + :attr:`_undefined_exception` with an error message generated + from the undefined hints stored on the undefined object. + +.. autoclass:: jinja2.ChainableUndefined() + +.. autoclass:: jinja2.DebugUndefined() + +.. autoclass:: jinja2.StrictUndefined() + +There is also a factory function that can decorate undefined objects to +implement logging on failures: + +.. autofunction:: jinja2.make_logging_undefined + +Undefined objects are created by calling :attr:`undefined`. + +.. admonition:: Implementation + + :class:`Undefined` is implemented by overriding the special + ``__underscore__`` methods. For example the default + :class:`Undefined` class implements ``__str__`` to returns an empty + string, while ``__int__`` and others fail with an exception. To + allow conversion to int by returning ``0`` you can implement your + own subclass. + + .. code-block:: python + + class NullUndefined(Undefined): + def __int__(self): + return 0 + + def __float__(self): + return 0.0 + + To disallow a method, override it and raise + :attr:`~Undefined._undefined_exception`. Because this is very + common there is the helper method + :meth:`~Undefined._fail_with_undefined_error` that raises the error + with the correct information. Here's a class that works like the + regular :class:`Undefined` but fails on iteration:: + + class NonIterableUndefined(Undefined): + def __iter__(self): + self._fail_with_undefined_error() + + +The Context +----------- + +.. autoclass:: jinja2.runtime.Context() + :members: resolve, get_exported, get_all + + .. attribute:: parent + + A dict of read only, global variables the template looks up. These + can either come from another :class:`Context`, from the + :attr:`Environment.globals` or :attr:`Template.globals` or points + to a dict created by combining the globals with the variables + passed to the render function. It must not be altered. + + .. attribute:: vars + + The template local variables. This list contains environment and + context functions from the :attr:`parent` scope as well as local + modifications and exported variables from the template. The template + will modify this dict during template evaluation but filters and + context functions are not allowed to modify it. + + .. attribute:: environment + + The environment that loaded the template. + + .. attribute:: exported_vars + + This set contains all the names the template exports. The values for + the names are in the :attr:`vars` dict. In order to get a copy of the + exported variables as dict, :meth:`get_exported` can be used. + + .. attribute:: name + + The load name of the template owning this context. + + .. attribute:: blocks + + A dict with the current mapping of blocks in the template. The keys + in this dict are the names of the blocks, and the values a list of + blocks registered. The last item in each list is the current active + block (latest in the inheritance chain). + + .. attribute:: eval_ctx + + The current :ref:`eval-context`. + + .. automethod:: jinja2.runtime.Context.call(callable, \*args, \**kwargs) + + +.. admonition:: Implementation + + Context is immutable for the same reason Python's frame locals are + immutable inside functions. Both Jinja and Python are not using the + context / frame locals as data storage for variables but only as primary + data source. + + When a template accesses a variable the template does not define, Jinja + looks up the variable in the context, after that the variable is treated + as if it was defined in the template. + + +.. _loaders: + +Loaders +------- + +Loaders are responsible for loading templates from a resource such as the +file system. The environment will keep the compiled modules in memory like +Python's `sys.modules`. Unlike `sys.modules` however this cache is limited in +size by default and templates are automatically reloaded. +All loaders are subclasses of :class:`BaseLoader`. If you want to create your +own loader, subclass :class:`BaseLoader` and override `get_source`. + +.. autoclass:: jinja2.BaseLoader + :members: get_source, load + +Here a list of the builtin loaders Jinja provides: + +.. autoclass:: jinja2.FileSystemLoader + +.. autoclass:: jinja2.PackageLoader + +.. autoclass:: jinja2.DictLoader + +.. autoclass:: jinja2.FunctionLoader + +.. autoclass:: jinja2.PrefixLoader + +.. autoclass:: jinja2.ChoiceLoader + +.. autoclass:: jinja2.ModuleLoader + + +.. _bytecode-cache: + +Bytecode Cache +-------------- + +Jinja 2.1 and higher support external bytecode caching. Bytecode caches make +it possible to store the generated bytecode on the file system or a different +location to avoid parsing the templates on first use. + +This is especially useful if you have a web application that is initialized on +the first request and Jinja compiles many templates at once which slows down +the application. + +To use a bytecode cache, instantiate it and pass it to the :class:`Environment`. + +.. autoclass:: jinja2.BytecodeCache + :members: load_bytecode, dump_bytecode, clear + +.. autoclass:: jinja2.bccache.Bucket + :members: write_bytecode, load_bytecode, bytecode_from_string, + bytecode_to_string, reset + + .. attribute:: environment + + The :class:`Environment` that created the bucket. + + .. attribute:: key + + The unique cache key for this bucket + + .. attribute:: code + + The bytecode if it's loaded, otherwise `None`. + + +Builtin bytecode caches: + +.. autoclass:: jinja2.FileSystemBytecodeCache + +.. autoclass:: jinja2.MemcachedBytecodeCache + + +Async Support +------------- + +.. versionadded:: 2.9 + +Jinja supports the Python ``async`` and ``await`` syntax. For the +template designer, this support (when enabled) is entirely transparent, +templates continue to look exactly the same. However, developers should +be aware of the implementation as it affects what types of APIs you can +use. + +By default, async support is disabled. Enabling it will cause the +environment to compile different code behind the scenes in order to +handle async and sync code in an asyncio event loop. This has the +following implications: + +- Template rendering requires an event loop to be available to the + current thread. :func:`asyncio.get_event_loop` must return an event + loop. +- The compiled code uses ``await`` for functions and attributes, and + uses ``async for`` loops. In order to support using both async and + sync functions in this context, a small wrapper is placed around + all calls and access, which add overhead compared to purely async + code. +- Sync methods and filters become wrappers around their corresponding + async implementations where needed. For example, ``render`` invokes + ``async_render``, and ``|map`` supports async iterables. + +Awaitable objects can be returned from functions in templates and any +function call in a template will automatically await the result. The +``await`` you would normally add in Python is implied. For example, you +can provide a method that asynchronously loads data from a database, and +from the template designer's point of view it can be called like any +other function. + + +.. _policies: + +Policies +-------- + +Starting with Jinja 2.9 policies can be configured on the environment +which can slightly influence how filters and other template constructs +behave. They can be configured with the +:attr:`~jinja2.Environment.policies` attribute. + +Example:: + + env.policies['urlize.rel'] = 'nofollow noopener' + +``truncate.leeway``: + Configures the leeway default for the `truncate` filter. Leeway as + introduced in 2.9 but to restore compatibility with older templates + it can be configured to `0` to get the old behavior back. The default + is `5`. + +``urlize.rel``: + A string that defines the items for the `rel` attribute of generated + links with the `urlize` filter. These items are always added. The + default is `noopener`. + +``urlize.target``: + The default target that is issued for links from the `urlize` filter + if no other target is defined by the call explicitly. + +``json.dumps_function``: + If this is set to a value other than `None` then the `tojson` filter + will dump with this function instead of the default one. Note that + this function should accept arbitrary extra arguments which might be + passed in the future from the filter. Currently the only argument + that might be passed is `indent`. The default dump function is + ``json.dumps``. + +``json.dumps_kwargs``: + Keyword arguments to be passed to the dump function. The default is + ``{'sort_keys': True}``. + +.. _ext-i18n-trimmed: + +``ext.i18n.trimmed``: + If this is set to `True`, ``{% trans %}`` blocks of the + :ref:`i18n-extension` will always unify linebreaks and surrounding + whitespace as if the `trimmed` modifier was used. + + +Utilities +--------- + +These helper functions and classes are useful if you add custom filters or +functions to a Jinja environment. + +.. autofunction:: jinja2.environmentfilter + +.. autofunction:: jinja2.contextfilter + +.. autofunction:: jinja2.evalcontextfilter + +.. autofunction:: jinja2.environmentfunction + +.. autofunction:: jinja2.contextfunction + +.. autofunction:: jinja2.evalcontextfunction + +.. function:: escape(s) + + Convert the characters ``&``, ``<``, ``>``, ``'``, and ``"`` in string `s` + to HTML-safe sequences. Use this if you need to display text that might + contain such characters in HTML. This function will not escaped objects + that do have an HTML representation such as already escaped data. + + The return value is a :class:`Markup` string. + +.. autofunction:: jinja2.clear_caches + +.. autofunction:: jinja2.is_undefined + +.. autoclass:: jinja2.Markup([string]) + :members: escape, unescape, striptags + +.. admonition:: Note + + The Jinja :class:`Markup` class is compatible with at least Pylons and + Genshi. It's expected that more template engines and framework will pick + up the `__html__` concept soon. + + +Exceptions +---------- + +.. autoexception:: jinja2.TemplateError + +.. autoexception:: jinja2.UndefinedError + +.. autoexception:: jinja2.TemplateNotFound + +.. autoexception:: jinja2.TemplatesNotFound + +.. autoexception:: jinja2.TemplateSyntaxError + + .. attribute:: message + + The error message. + + .. attribute:: lineno + + The line number where the error occurred. + + .. attribute:: name + + The load name for the template. + + .. attribute:: filename + + The filename that loaded the template in the encoding of the + file system (most likely utf-8, or mbcs on Windows systems). + +.. autoexception:: jinja2.TemplateRuntimeError + +.. autoexception:: jinja2.TemplateAssertionError + + +.. _writing-filters: + +Custom Filters +-------------- + +Custom filters are just regular Python functions that take the left side of +the filter as first argument and the arguments passed to the filter as +extra arguments or keyword arguments. + +For example in the filter ``{{ 42|myfilter(23) }}`` the function would be +called with ``myfilter(42, 23)``. Here for example a simple filter that can +be applied to datetime objects to format them:: + + def datetimeformat(value, format='%H:%M / %d-%m-%Y'): + return value.strftime(format) + +You can register it on the template environment by updating the +:attr:`~Environment.filters` dict on the environment:: + + environment.filters['datetimeformat'] = datetimeformat + +Inside the template it can then be used as follows: + +.. sourcecode:: jinja + + written on: {{ article.pub_date|datetimeformat }} + publication date: {{ article.pub_date|datetimeformat('%d-%m-%Y') }} + +Filters can also be passed the current template context or environment. This +is useful if a filter wants to return an undefined value or check the current +:attr:`~Environment.autoescape` setting. For this purpose three decorators +exist: :func:`environmentfilter`, :func:`contextfilter` and +:func:`evalcontextfilter`. + +Here a small example filter that breaks a text into HTML line breaks and +paragraphs and marks the return value as safe HTML string if autoescaping is +enabled:: + + import re + from jinja2 import evalcontextfilter, Markup, escape + + _paragraph_re = re.compile(r"(?:\r\n|\r(?!\n)|\n){2,}") + + @evalcontextfilter + def nl2br(eval_ctx, value): + result = "\n\n".join( + f"<p>{p.replace('\n', Markup('<br>\n'))}</p>" + for p in _paragraph_re.split(escape(value)) + ) + if eval_ctx.autoescape: + result = Markup(result) + return result + +Context filters work the same just that the first argument is the current +active :class:`Context` rather than the environment. + + +.. _eval-context: + +Evaluation Context +------------------ + +The evaluation context (short eval context or eval ctx) is a new object +introduced in Jinja 2.4 that makes it possible to activate and deactivate +compiled features at runtime. + +Currently it is only used to enable and disable the automatic escaping but +can be used for extensions as well. + +In previous Jinja versions filters and functions were marked as +environment callables in order to check for the autoescape status from the +environment. In new versions it's encouraged to check the setting from the +evaluation context instead. + +Previous versions:: + + @environmentfilter + def filter(env, value): + result = do_something(value) + if env.autoescape: + result = Markup(result) + return result + +In new versions you can either use a :func:`contextfilter` and access the +evaluation context from the actual context, or use a +:func:`evalcontextfilter` which directly passes the evaluation context to +the function:: + + @contextfilter + def filter(context, value): + result = do_something(value) + if context.eval_ctx.autoescape: + result = Markup(result) + return result + + @evalcontextfilter + def filter(eval_ctx, value): + result = do_something(value) + if eval_ctx.autoescape: + result = Markup(result) + return result + +The evaluation context must not be modified at runtime. Modifications +must only happen with a :class:`nodes.EvalContextModifier` and +:class:`nodes.ScopedEvalContextModifier` from an extension, not on the +eval context object itself. + +.. autoclass:: jinja2.nodes.EvalContext + + .. attribute:: autoescape + + `True` or `False` depending on if autoescaping is active or not. + + .. attribute:: volatile + + `True` if the compiler cannot evaluate some expressions at compile + time. At runtime this should always be `False`. + + +.. _writing-tests: + +Custom Tests +------------ + +Tests work like filters just that there is no way for a test to get access +to the environment or context and that they can't be chained. The return +value of a test should be `True` or `False`. The purpose of a test is to +give the template designers the possibility to perform type and conformability +checks. + +Here a simple test that checks if a variable is a prime number:: + + import math + + def is_prime(n): + if n == 2: + return True + for i in range(2, int(math.ceil(math.sqrt(n))) + 1): + if n % i == 0: + return False + return True + + +You can register it on the template environment by updating the +:attr:`~Environment.tests` dict on the environment:: + + environment.tests['prime'] = is_prime + +A template designer can then use the test like this: + +.. sourcecode:: jinja + + {% if 42 is prime %} + 42 is a prime number + {% else %} + 42 is not a prime number + {% endif %} + + +.. _global-namespace: + +The Global Namespace +-------------------- + +Variables stored in the :attr:`Environment.globals` dict are special as they +are available for imported templates too, even if they are imported without +context. This is the place where you can put variables and functions +that should be available all the time. Additionally :attr:`Template.globals` +exist that are variables available to a specific template that are available +to all :meth:`~Template.render` calls. + + +.. _low-level-api: + +Low Level API +------------- + +The low level API exposes functionality that can be useful to understand some +implementation details, debugging purposes or advanced :ref:`extension +<jinja-extensions>` techniques. Unless you know exactly what you are doing we +don't recommend using any of those. + +.. automethod:: Environment.lex + +.. automethod:: Environment.parse + +.. automethod:: Environment.preprocess + +.. automethod:: Template.new_context + +.. method:: Template.root_render_func(context) + + This is the low level render function. It's passed a :class:`Context` + that has to be created by :meth:`new_context` of the same template or + a compatible template. This render function is generated by the + compiler from the template code and returns a generator that yields + strings. + + If an exception in the template code happens the template engine will + not rewrite the exception but pass through the original one. As a + matter of fact this function should only be called from within a + :meth:`render` / :meth:`generate` / :meth:`stream` call. + +.. attribute:: Template.blocks + + A dict of block render functions. Each of these functions works exactly + like the :meth:`root_render_func` with the same limitations. + +.. attribute:: Template.is_up_to_date + + This attribute is `False` if there is a newer version of the template + available, otherwise `True`. + +.. admonition:: Note + + The low-level API is fragile. Future Jinja versions will try not to + change it in a backwards incompatible way but modifications in the Jinja + core may shine through. For example if Jinja introduces a new AST node + in later versions that may be returned by :meth:`~Environment.parse`. + +The Meta API +------------ + +.. versionadded:: 2.2 + +The meta API returns some information about abstract syntax trees that +could help applications to implement more advanced template concepts. All +the functions of the meta API operate on an abstract syntax tree as +returned by the :meth:`Environment.parse` method. + +.. autofunction:: jinja2.meta.find_undeclared_variables + +.. autofunction:: jinja2.meta.find_referenced_templates diff --git a/docs/changelog.rst b/docs/changelog.rst new file mode 100644 index 00000000..218fe333 --- /dev/null +++ b/docs/changelog.rst @@ -0,0 +1,4 @@ +Changelog +========= + +.. include:: ../CHANGES.rst diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..783bae20 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,50 @@ +from pallets_sphinx_themes import get_version +from pallets_sphinx_themes import ProjectLink + +# Project -------------------------------------------------------------- + +project = "Jinja" +copyright = "2007 Pallets" +author = "Pallets" +release, version = get_version("Jinja2") + +# General -------------------------------------------------------------- + +master_doc = "index" +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "pallets_sphinx_themes", + "sphinxcontrib.log_cabinet", + "sphinx_issues", +] +intersphinx_mapping = {"python": ("https://docs.python.org/3/", None)} +issues_github_path = "pallets/jinja" + +# HTML ----------------------------------------------------------------- + +html_theme = "jinja" +html_theme_options = {"index_sidebar_logo": False} +html_context = { + "project_links": [ + ProjectLink("Donate to Pallets", "https://palletsprojects.com/donate"), + ProjectLink("Jinja Website", "https://palletsprojects.com/p/jinja/"), + ProjectLink("PyPI releases", "https://pypi.org/project/Jinja2/"), + ProjectLink("Source Code", "https://github.com/pallets/jinja/"), + ProjectLink("Issue Tracker", "https://github.com/pallets/jinja/issues/"), + ] +} +html_sidebars = { + "index": ["project.html", "localtoc.html", "searchbox.html"], + "**": ["localtoc.html", "relations.html", "searchbox.html"], +} +singlehtml_sidebars = {"index": ["project.html", "localtoc.html"]} +html_static_path = ["_static"] +html_favicon = "_static/jinja-logo-sidebar.png" +html_logo = "_static/jinja-logo-sidebar.png" +html_title = f"Jinja Documentation ({version})" +html_show_sourcelink = False + +# LaTeX ---------------------------------------------------------------- + +latex_documents = [(master_doc, f"Jinja-{version}.tex", html_title, author, "manual")] diff --git a/docs/examples/cache_extension.py b/docs/examples/cache_extension.py new file mode 100644 index 00000000..46af67ce --- /dev/null +++ b/docs/examples/cache_extension.py @@ -0,0 +1,54 @@ +from jinja2 import nodes +from jinja2.ext import Extension + + +class FragmentCacheExtension(Extension): + # a set of names that trigger the extension. + tags = {"cache"} + + def __init__(self, environment): + super().__init__(environment) + + # add the defaults to the environment + environment.extend(fragment_cache_prefix="", fragment_cache=None) + + def parse(self, parser): + # the first token is the token that started the tag. In our case + # we only listen to ``'cache'`` so this will be a name token with + # `cache` as value. We get the line number so that we can give + # that line number to the nodes we create by hand. + lineno = next(parser.stream).lineno + + # now we parse a single expression that is used as cache key. + args = [parser.parse_expression()] + + # if there is a comma, the user provided a timeout. If not use + # None as second parameter. + if parser.stream.skip_if("comma"): + args.append(parser.parse_expression()) + else: + args.append(nodes.Const(None)) + + # now we parse the body of the cache block up to `endcache` and + # drop the needle (which would always be `endcache` in that case) + body = parser.parse_statements(["name:endcache"], drop_needle=True) + + # now return a `CallBlock` node that calls our _cache_support + # helper method on this extension. + return nodes.CallBlock( + self.call_method("_cache_support", args), [], [], body + ).set_lineno(lineno) + + def _cache_support(self, name, timeout, caller): + """Helper callback.""" + key = self.environment.fragment_cache_prefix + name + + # try to load the block from the cache + # if there is no fragment in the cache, render it and store + # it in the cache. + rv = self.environment.fragment_cache.get(key) + if rv is not None: + return rv + rv = caller() + self.environment.fragment_cache.add(key, rv, timeout) + return rv diff --git a/docs/examples/inline_gettext_extension.py b/docs/examples/inline_gettext_extension.py new file mode 100644 index 00000000..d75119cf --- /dev/null +++ b/docs/examples/inline_gettext_extension.py @@ -0,0 +1,72 @@ +import re + +from jinja2.exceptions import TemplateSyntaxError +from jinja2.ext import Extension +from jinja2.lexer import count_newlines +from jinja2.lexer import Token + + +_outside_re = re.compile(r"\\?(gettext|_)\(") +_inside_re = re.compile(r"\\?[()]") + + +class InlineGettext(Extension): + """This extension implements support for inline gettext blocks:: + + <h1>_(Welcome)</h1> + <p>_(This is a paragraph)</p> + + Requires the i18n extension to be loaded and configured. + """ + + def filter_stream(self, stream): + paren_stack = 0 + + for token in stream: + if token.type != "data": + yield token + continue + + pos = 0 + lineno = token.lineno + + while 1: + if not paren_stack: + match = _outside_re.search(token.value, pos) + else: + match = _inside_re.search(token.value, pos) + if match is None: + break + new_pos = match.start() + if new_pos > pos: + preval = token.value[pos:new_pos] + yield Token(lineno, "data", preval) + lineno += count_newlines(preval) + gtok = match.group() + if gtok[0] == "\\": + yield Token(lineno, "data", gtok[1:]) + elif not paren_stack: + yield Token(lineno, "block_begin", None) + yield Token(lineno, "name", "trans") + yield Token(lineno, "block_end", None) + paren_stack = 1 + else: + if gtok == "(" or paren_stack > 1: + yield Token(lineno, "data", gtok) + paren_stack += -1 if gtok == ")" else 1 + if not paren_stack: + yield Token(lineno, "block_begin", None) + yield Token(lineno, "name", "endtrans") + yield Token(lineno, "block_end", None) + pos = match.end() + + if pos < len(token.value): + yield Token(lineno, "data", token.value[pos:]) + + if paren_stack: + raise TemplateSyntaxError( + "unclosed gettext expression", + token.lineno, + stream.name, + stream.filename, + ) diff --git a/docs/extensions.rst b/docs/extensions.rst new file mode 100644 index 00000000..bb81f217 --- /dev/null +++ b/docs/extensions.rst @@ -0,0 +1,402 @@ +.. _jinja-extensions: + +Extensions +========== + +Jinja supports extensions that can add extra filters, tests, globals or even +extend the parser. The main motivation of extensions is to move often used +code into a reusable class like adding support for internationalization. + + +Adding Extensions +----------------- + +Extensions are added to the Jinja environment at creation time. Once the +environment is created additional extensions cannot be added. To add an +extension pass a list of extension classes or import paths to the +``extensions`` parameter of the :class:`~jinja2.Environment` constructor. The following +example creates a Jinja environment with the i18n extension loaded:: + + jinja_env = Environment(extensions=['jinja2.ext.i18n']) + + +.. _i18n-extension: + +i18n Extension +-------------- + +**Import name:** ``jinja2.ext.i18n`` + +The i18n extension can be used in combination with `gettext`_ or +`Babel`_. When it's enabled, Jinja provides a ``trans`` statement that +marks a block as translatable and calls ``gettext``. + +After enabling, an application has to provide ``gettext`` and +``ngettext`` functions, either globally or when rendering. A ``_()`` +function is added as an alias to the ``gettext`` function. + +Environment Methods +~~~~~~~~~~~~~~~~~~~ + +After enabling the extension, the environment provides the following +additional methods: + +.. method:: jinja2.Environment.install_gettext_translations(translations, newstyle=False) + + Installs a translation globally for the environment. The + ``translations`` object must implement ``gettext`` and ``ngettext``. + :class:`gettext.NullTranslations`, :class:`gettext.GNUTranslations`, + and `Babel`_\s ``Translations`` are supported. + + .. versionchanged:: 2.5 Added new-style gettext support. + +.. method:: jinja2.Environment.install_null_translations(newstyle=False) + + Install no-op gettext functions. This is useful if you want to + prepare the application for internationalization but don't want to + implement the full system yet. + + .. versionchanged:: 2.5 Added new-style gettext support. + +.. method:: jinja2.Environment.install_gettext_callables(gettext, ngettext, newstyle=False) + + Install the given ``gettext`` and ``ngettext`` callables into the + environment. They should behave exactly like + :func:`gettext.gettext` and :func:`gettext.ngettext`. + + If ``newstyle`` is activated, the callables are wrapped to work like + newstyle callables. See :ref:`newstyle-gettext` for more information. + + .. versionadded:: 2.5 Added new-style gettext support. + +.. method:: jinja2.Environment.uninstall_gettext_translations() + + Uninstall the environment's globally installed translation. + +.. method:: jinja2.Environment.extract_translations(source) + + Extract localizable strings from the given template node or source. + + For every string found this function yields a ``(lineno, function, + message)`` tuple, where: + + - ``lineno`` is the number of the line on which the string was + found. + - ``function`` is the name of the ``gettext`` function used (if + the string was extracted from embedded Python code). + - ``message`` is the string itself, or a tuple of strings for + functions with multiple arguments. + + If `Babel`_ is installed, see :ref:`babel-integration` to extract + the strings. + +For a web application that is available in multiple languages but gives +all the users the same language (for example, multilingual forum +software installed for a French community), the translation may be +installed when the environment is created. + +.. code-block:: python + + translations = get_gettext_translations() + env = Environment(extensions=["jinja2.ext.i18n"]) + env.install_gettext_translations(translations) + +The ``get_gettext_translations`` function would return the translator +for the current configuration, for example by using ``gettext.find``. + +The usage of the ``i18n`` extension for template designers is covered in +:ref:`the template documentation <i18n-in-templates>`. + +.. _gettext: https://docs.python.org/3/library/gettext.html +.. _Babel: http://babel.pocoo.org/ + + +Whitespace Trimming +~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 2.10 + +Within ``{% trans %}`` blocks, it can be useful to trim line breaks and +whitespace so that the block of text looks like a simple string with +single spaces in the translation file. + +Linebreaks and surrounding whitespace can be automatically trimmed by +enabling the ``ext.i18n.trimmed`` :ref:`policy <ext-i18n-trimmed>`. + + +.. _newstyle-gettext: + +New Style Gettext +~~~~~~~~~~~~~~~~~ + +.. versionadded:: 2.5 + +New style gettext calls are less to type, less error prone, and support +autoescaping better. + +You can use "new style" gettext calls by setting +``env.newstyle_gettext = True`` or passing ``newstyle=True`` to +``env.install_translations``. They are fully supported by the Babel +extraction tool, but might not work as expected with other extraction +tools. + +With standard ``gettext`` calls, string formatting is a separate step +done with the ``|format`` filter. This requires duplicating work for +``ngettext`` calls. + +.. sourcecode:: jinja + + {{ gettext("Hello, World!") }} + {{ gettext("Hello, %(name)s!")|format(name=name) }} + {{ ngettext( + "%(num)d apple", "%(num)d apples", apples|count + )|format(num=apples|count) }} + +New style ``gettext`` make formatting part of the call, and behind the +scenes enforce more consistency. + +.. sourcecode:: jinja + + {{ gettext("Hello, World!") }} + {{ gettext("Hello, %(name)s!", name=name) }} + {{ ngettext("%(num)d apple", "%(num)d apples", apples|count) }} + +The advantages of newstyle gettext are: + +- There's no separate formatting step, you don't have to remember to + use the ``|format`` filter. +- Only named placeholders are allowed. This solves a common problem + translators face because positional placeholders can't switch + positions meaningfully. Named placeholders always carry semantic + information about what value goes where. +- String formatting is used even if no placeholders are used, which + makes all strings use a consistent format. Remember to escape any + raw percent signs as ``%%``, such as ``100%%``. +- The translated string is marked safe, formatting performs escaping + as needed. Mark a parameter as ``|safe`` if it has already been + escaped. + + +Expression Statement +-------------------- + +**Import name:** ``jinja2.ext.do`` + +The "do" aka expression-statement extension adds a simple ``do`` tag to the +template engine that works like a variable expression but ignores the +return value. + +.. _loopcontrols-extension: + +Loop Controls +------------- + +**Import name:** ``jinja2.ext.loopcontrols`` + +This extension adds support for ``break`` and ``continue`` in loops. After +enabling, Jinja provides those two keywords which work exactly like in +Python. + +.. _with-extension: + +With Statement +-------------- + +**Import name:** ``jinja2.ext.with_`` + +.. versionchanged:: 2.9 + + This extension is now built-in and no longer does anything. + +.. _autoescape-extension: + +Autoescape Extension +-------------------- + +**Import name:** ``jinja2.ext.autoescape`` + +.. versionchanged:: 2.9 + + This extension was removed and is now built-in. Enabling the + extension no longer does anything. + + +.. _debug-extension: + +Debug Extension +--------------- + +**Import name:** ``jinja2.ext.debug`` + +Adds a ``{% debug %}`` tag to dump the current context as well as the +available filters and tests. This is useful to see what's available to +use in the template without setting up a debugger. + + +.. _writing-extensions: + +Writing Extensions +------------------ + +.. module:: jinja2.ext + +By writing extensions you can add custom tags to Jinja. This is a non-trivial +task and usually not needed as the default tags and expressions cover all +common use cases. The i18n extension is a good example of why extensions are +useful. Another one would be fragment caching. + +When writing extensions you have to keep in mind that you are working with the +Jinja template compiler which does not validate the node tree you are passing +to it. If the AST is malformed you will get all kinds of compiler or runtime +errors that are horrible to debug. Always make sure you are using the nodes +you create correctly. The API documentation below shows which nodes exist and +how to use them. + + +Example Extensions +------------------ + +Cache +~~~~~ + +The following example implements a ``cache`` tag for Jinja by using the +`cachelib`_ library: + +.. literalinclude:: examples/cache_extension.py + :language: python + +And here is how you use it in an environment:: + + from jinja2 import Environment + from cachelib import SimpleCache + + env = Environment(extensions=[FragmentCacheExtension]) + env.fragment_cache = SimpleCache() + +Inside the template it's then possible to mark blocks as cacheable. The +following example caches a sidebar for 300 seconds: + +.. sourcecode:: html+jinja + + {% cache 'sidebar', 300 %} + <div class="sidebar"> + ... + </div> + {% endcache %} + +.. _cachelib: https://github.com/pallets/cachelib + + +Inline ``gettext`` +~~~~~~~~~~~~~~~~~~ + +The following example demonstrates using :meth:`Extension.filter_stream` +to parse calls to the ``_()`` gettext function inline with static data +without needing Jinja blocks. + +.. code-block:: html + + <h1>_(Welcome)</h1> + <p>_(This is a paragraph)</p> + +It requires the i18n extension to be loaded and configured. + +.. literalinclude:: examples/inline_gettext_extension.py + :language: python + + +Extension API +------------- + +Extension +~~~~~~~~~ + +Extensions always have to extend the :class:`jinja2.ext.Extension` class: + +.. autoclass:: Extension + :members: preprocess, filter_stream, parse, attr, call_method + + .. attribute:: identifier + + The identifier of the extension. This is always the true import name + of the extension class and must not be changed. + + .. attribute:: tags + + If the extension implements custom tags this is a set of tag names + the extension is listening for. + + +Parser +~~~~~~ + +The parser passed to :meth:`Extension.parse` provides ways to parse +expressions of different types. The following methods may be used by +extensions: + +.. autoclass:: jinja2.parser.Parser + :members: parse_expression, parse_tuple, parse_assign_target, + parse_statements, free_identifier, fail + + .. attribute:: filename + + The filename of the template the parser processes. This is **not** + the load name of the template. For the load name see :attr:`name`. + For templates that were not loaded form the file system this is + ``None``. + + .. attribute:: name + + The load name of the template. + + .. attribute:: stream + + The current :class:`~jinja2.lexer.TokenStream` + +.. autoclass:: jinja2.lexer.TokenStream + :members: push, look, eos, skip, __next__, next_if, skip_if, expect + + .. attribute:: current + + The current :class:`~jinja2.lexer.Token`. + +.. autoclass:: jinja2.lexer.Token + :members: test, test_any + + .. attribute:: lineno + + The line number of the token + + .. attribute:: type + + The type of the token. This string is interned so you may compare + it with arbitrary strings using the ``is`` operator. + + .. attribute:: value + + The value of the token. + +There is also a utility function in the lexer module that can count newline +characters in strings: + +.. autofunction:: jinja2.lexer.count_newlines + + +AST +~~~ + +The AST (Abstract Syntax Tree) is used to represent a template after parsing. +It's build of nodes that the compiler then converts into executable Python +code objects. Extensions that provide custom statements can return nodes to +execute custom Python code. + +The list below describes all nodes that are currently available. The AST may +change between Jinja versions but will stay backwards compatible. + +For more information have a look at the repr of :meth:`jinja2.Environment.parse`. + +.. module:: jinja2.nodes + +.. jinja:nodes:: jinja2.nodes.Node + +.. autoexception:: Impossible diff --git a/docs/faq.rst b/docs/faq.rst new file mode 100644 index 00000000..1e29e12f --- /dev/null +++ b/docs/faq.rst @@ -0,0 +1,175 @@ +Frequently Asked Questions +========================== + +This page answers some of the often asked questions about Jinja. + +.. highlight:: html+jinja + +Why is it called Jinja? +----------------------- + +The name Jinja was chosen because it's the name of a Japanese temple and +temple and template share a similar pronunciation. It is not named after +the city in Uganda. + +How fast is it? +--------------- + +We really hate benchmarks especially since they don't reflect much. The +performance of a template depends on many factors and you would have to +benchmark different engines in different situations. The benchmarks from the +testsuite show that Jinja has a similar performance to `Mako`_ and is between +10 and 20 times faster than Django's template engine or Genshi. These numbers +should be taken with tons of salt as the benchmarks that took these numbers +only test a few performance related situations such as looping. Generally +speaking the performance of a template engine doesn't matter much as the +usual bottleneck in a web application is either the database or the application +code. + +.. _Mako: https://www.makotemplates.org/ + +How Compatible is Jinja with Django? +------------------------------------ + +The default syntax of Jinja matches Django syntax in many ways. However +this similarity doesn't mean that you can use a Django template unmodified +in Jinja. For example filter arguments use a function call syntax rather +than a colon to separate filter name and arguments. Additionally the +extension interface in Jinja is fundamentally different from the Django one +which means that your custom tags won't work any longer. + +Generally speaking you will use much less custom extensions as the Jinja +template system allows you to use a certain subset of Python expressions +which can replace most Django extensions. For example instead of using +something like this:: + + {% load comments %} + {% get_latest_comments 10 as latest_comments %} + {% for comment in latest_comments %} + ... + {% endfor %} + +You will most likely provide an object with attributes to retrieve +comments from the database:: + + {% for comment in models.comments.latest(10) %} + ... + {% endfor %} + +Or directly provide the model for quick testing:: + + {% for comment in Comment.objects.order_by('-pub_date')[:10] %} + ... + {% endfor %} + +Please keep in mind that even though you may put such things into templates +it still isn't a good idea. Queries should go into the view code and not +the template! + +Isn't it a terrible idea to put Logic into Templates? +----------------------------------------------------- + +Without a doubt you should try to remove as much logic from templates as +possible. But templates without any logic mean that you have to do all +the processing in the code which is boring and stupid. A template engine +that does that is shipped with Python and called `string.Template`. Comes +without loops and if conditions and is by far the fastest template engine +you can get for Python. + +So some amount of logic is required in templates to keep everyone happy. +And Jinja leaves it pretty much to you how much logic you want to put into +templates. There are some restrictions in what you can do and what not. + +Jinja neither allows you to put arbitrary Python code into templates nor +does it allow all Python expressions. The operators are limited to the +most common ones and more advanced expressions such as list comprehensions +and generator expressions are not supported. This keeps the template engine +easier to maintain and templates more readable. + +Why is Autoescaping not the Default? +------------------------------------ + +There are multiple reasons why automatic escaping is not the default mode +and also not the recommended one. While automatic escaping of variables +means that you will less likely have an XSS problem it also causes a huge +amount of extra processing in the template engine which can cause serious +performance problems. As Python doesn't provide a way to mark strings as +unsafe Jinja has to hack around that limitation by providing a custom +string class (the :class:`Markup` string) that safely interacts with safe +and unsafe strings. + +With explicit escaping however the template engine doesn't have to perform +any safety checks on variables. Also a human knows not to escape integers +or strings that may never contain characters one has to escape or already +HTML markup. For example when iterating over a list over a table of +integers and floats for a table of statistics the template designer can +omit the escaping because he knows that integers or floats don't contain +any unsafe parameters. + +Additionally Jinja is a general purpose template engine and not only used +for HTML/XML generation. For example you may generate LaTeX, emails, +CSS, JavaScript, or configuration files. + +Why is the Context immutable? +----------------------------- + +When writing a :func:`contextfunction` or something similar you may have +noticed that the context tries to stop you from modifying it. If you have +managed to modify the context by using an internal context API you may +have noticed that changes in the context don't seem to be visible in the +template. The reason for this is that Jinja uses the context only as +primary data source for template variables for performance reasons. + +If you want to modify the context write a function that returns a variable +instead that one can assign to a variable by using set:: + + {% set comments = get_latest_comments() %} + +My tracebacks look weird. What's happening? +------------------------------------------- + +Jinja can rewrite tracebacks so they show the template lines numbers and +source rather than the underlying compiled code, but this requires +special Python support. CPython <3.7 requires ``ctypes``, and PyPy +requires transparent proxy support. + +If you are using Google App Engine, ``ctypes`` is not available. You can +make it available in development, but not in production. + +.. code-block:: python + + import os + if os.environ.get('SERVER_SOFTWARE', '').startswith('Dev'): + from google.appengine.tools.devappserver2.python import sandbox + sandbox._WHITE_LIST_C_MODULES += ['_ctypes', 'gestalt'] + +Credit for this snippet goes to `Thomas Johansson +<https://stackoverflow.com/questions/3086091/debug-jinja2-in-google-app-engine/3694434#3694434>`_ + +My Macros are overridden by something +------------------------------------- + +In some situations the Jinja scoping appears arbitrary: + +layout.tmpl: + +.. sourcecode:: jinja + + {% macro foo() %}LAYOUT{% endmacro %} + {% block body %}{% endblock %} + +child.tmpl: + +.. sourcecode:: jinja + + {% extends 'layout.tmpl' %} + {% macro foo() %}CHILD{% endmacro %} + {% block body %}{{ foo() }}{% endblock %} + +This will print ``LAYOUT`` in Jinja. This is a side effect of having +the parent template evaluated after the child one. This allows child +templates passing information to the parent template. To avoid this +issue rename the macro or variable in the parent template to have an +uncommon prefix. + +.. _Jinja 1: https://pypi.org/project/Jinja/ diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..dcaa9ffd --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,28 @@ +.. rst-class:: hide-header + +Jinja +===== + +.. image:: _static/jinja-logo.png + :align: center + :target: https://palletsprojects.com/p/jinja/ + +Jinja is a fast, expressive, extensible templating engine. Special +placeholders in the template allow writing code similar to Python +syntax. Then the template is passed data to render the final document. + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + intro + api + sandbox + nativetypes + templates + extensions + integration + switching + tricks + faq + changelog diff --git a/docs/integration.rst b/docs/integration.rst new file mode 100644 index 00000000..633e80d4 --- /dev/null +++ b/docs/integration.rst @@ -0,0 +1,75 @@ +Integration +=========== + +.. _babel-integration: + +Babel +----- + +Jinja provides support for extracting gettext messages from templates +via a `Babel`_ extractor entry point called +``jinja2.ext.babel_extract``. The support is implemented as part of the +:ref:`i18n-extension` extension. + +Gettext messages are extracted from both ``trans`` tags and code +expressions. + +To extract gettext messages from templates, the project needs a Jinja +section in its Babel extraction method `mapping file`_: + +.. sourcecode:: ini + + [jinja2: **/templates/**.html] + encoding = utf-8 + +The syntax related options of the :class:`Environment` are also +available as configuration values in the mapping file. For example, to +tell the extractor that templates use ``%`` as +``line_statement_prefix`` you can use this code: + +.. sourcecode:: ini + + [jinja2: **/templates/**.html] + encoding = utf-8 + line_statement_prefix = % + +:ref:`jinja-extensions` may also be defined by passing a comma separated +list of import paths as the ``extensions`` value. The i18n extension is +added automatically. + +Template syntax errors are ignored by default. The assumption is that +tests will catch syntax errors in templates. If you don't want to ignore +errors, add ``silent = false`` to the settings. + +.. _Babel: https://babel.readthedocs.io/ +.. _mapping file: https://babel.readthedocs.io/en/latest/messages.html#extraction-method-mapping-and-configuration + + +Pylons +------ + +It's easy to integrate Jinja into a `Pylons`_ application. + +The template engine is configured in ``config/environment.py``. The +configuration for Jinja looks something like this: + +.. code-block:: python + + from jinja2 import Environment, PackageLoader + config['pylons.app_globals'].jinja_env = Environment( + loader=PackageLoader('yourapplication', 'templates') + ) + +After that you can render Jinja templates by using the ``render_jinja`` +function from the ``pylons.templating`` module. + +Additionally it's a good idea to set the Pylons ``c`` object to strict +mode. By default attribute access on missing attributes on the ``c`` +object returns an empty string and not an undefined object. To change +this add this to ``config/environment.py``: + +.. code-block:: python + + config['pylons.strict_c'] = True + +.. _Pylons: https://pylonshq.com/ diff --git a/docs/intro.rst b/docs/intro.rst new file mode 100644 index 00000000..25c2b580 --- /dev/null +++ b/docs/intro.rst @@ -0,0 +1,63 @@ +Introduction +============ + +Jinja is a fast, expressive, extensible templating engine. Special +placeholders in the template allow writing code similar to Python +syntax. Then the template is passed data to render the final document. + +It includes: + +- Template inheritance and inclusion. +- Define and import macros within templates. +- HTML templates can use autoescaping to prevent XSS from untrusted + user input. +- A sandboxed environment can safely render untrusted templates. +- AsyncIO support for generating templates and calling async + functions. +- I18N support with Babel. +- Templates are compiled to optimized Python code just-in-time and + cached, or can be compiled ahead-of-time. +- Exceptions point to the correct line in templates to make debugging + easier. +- Extensible filters, tests, functions, and even syntax. + +Jinja's philosophy is that while application logic belongs in Python if +possible, it shouldn't make the template designer's job difficult by +restricting functionality too much. + + +Installation +------------ + +We recommend using the latest version of Python. Jinja supports Python +3.6 and newer. We also recommend using a `virtual environment`_ in order +to isolate your project dependencies from other projects and the system. + +.. _virtual environment: https://packaging.python.org/tutorials/installing-packages/#creating-virtual-environments + +Install the most recent Jinja version using pip: + +.. code-block:: text + + $ pip install Jinja2 + + +Dependencies +~~~~~~~~~~~~ + +These will be installed automatically when installing Jinja. + +- `MarkupSafe`_ escapes untrusted input when rendering templates to + avoid injection attacks. + +.. _MarkupSafe: https://markupsafe.palletsprojects.com/ + + +Optional Dependencies +~~~~~~~~~~~~~~~~~~~~~ + +These distributions will not be installed automatically. + +- `Babel`_ provides translation support in templates. + +.. _Babel: http://babel.pocoo.org/ diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..7893348a --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/nativetypes.rst b/docs/nativetypes.rst new file mode 100644 index 00000000..1a08700b --- /dev/null +++ b/docs/nativetypes.rst @@ -0,0 +1,64 @@ +.. module:: jinja2.nativetypes + +.. _nativetypes: + +Native Python Types +=================== + +The default :class:`~jinja2.Environment` renders templates to strings. With +:class:`NativeEnvironment`, rendering a template produces a native Python type. +This is useful if you are using Jinja outside the context of creating text +files. For example, your code may have an intermediate step where users may use +templates to define values that will then be passed to a traditional string +environment. + +Examples +-------- + +Adding two values results in an integer, not a string with a number: + +>>> env = NativeEnvironment() +>>> t = env.from_string('{{ x + y }}') +>>> result = t.render(x=4, y=2) +>>> print(result) +6 +>>> print(type(result)) +int + +Rendering list syntax produces a list: + +>>> t = env.from_string('[{% for item in data %}{{ item + 1 }},{% endfor %}]') +>>> result = t.render(data=range(5)) +>>> print(result) +[1, 2, 3, 4, 5] +>>> print(type(result)) +list + +Rendering something that doesn't look like a Python literal produces a string: + +>>> t = env.from_string('{{ x }} * {{ y }}') +>>> result = t.render(x=4, y=2) +>>> print(result) +4 * 2 +>>> print(type(result)) +str + +Rendering a Python object produces that object as long as it is the only node: + +>>> class Foo: +... def __init__(self, value): +... self.value = value +... +>>> result = env.from_string('{{ x }}').render(x=Foo(15)) +>>> print(type(result).__name__) +Foo +>>> print(result.value) +15 + +API +--- + +.. autoclass:: NativeEnvironment([options]) + +.. autoclass:: NativeTemplate([options]) + :members: render diff --git a/docs/sandbox.rst b/docs/sandbox.rst new file mode 100644 index 00000000..1222d025 --- /dev/null +++ b/docs/sandbox.rst @@ -0,0 +1,94 @@ +Sandbox +======= + +The Jinja sandbox can be used to evaluate untrusted code. Access to unsafe +attributes and methods is prohibited. + +Assuming `env` is a :class:`SandboxedEnvironment` in the default configuration +the following piece of code shows how it works: + +>>> env.from_string("{{ func.func_code }}").render(func=lambda:None) +u'' +>>> env.from_string("{{ func.func_code.do_something }}").render(func=lambda:None) +Traceback (most recent call last): + ... +SecurityError: access to attribute 'func_code' of 'function' object is unsafe. + +API +--- + +.. module:: jinja2.sandbox + +.. autoclass:: SandboxedEnvironment([options]) + :members: is_safe_attribute, is_safe_callable, default_binop_table, + default_unop_table, intercepted_binops, intercepted_unops, + call_binop, call_unop + +.. autoclass:: ImmutableSandboxedEnvironment([options]) + +.. autoexception:: SecurityError + +.. autofunction:: unsafe + +.. autofunction:: is_internal_attribute + +.. autofunction:: modifies_known_mutable + +.. admonition:: Note + + The Jinja sandbox alone is no solution for perfect security. Especially + for web applications you have to keep in mind that users may create + templates with arbitrary HTML in so it's crucial to ensure that (if you + are running multiple users on the same server) they can't harm each other + via JavaScript insertions and much more. + + Also the sandbox is only as good as the configuration. We strongly + recommend only passing non-shared resources to the template and use + some sort of whitelisting for attributes. + + Also keep in mind that templates may raise runtime or compile time errors, + so make sure to catch them. + +Operator Intercepting +--------------------- + +.. versionadded:: 2.6 + +For maximum performance Jinja will let operators call directly the type +specific callback methods. This means that it's not possible to have this +intercepted by overriding :meth:`Environment.call`. Furthermore a +conversion from operator to special method is not always directly possible +due to how operators work. For instance for divisions more than one +special method exist. + +With Jinja 2.6 there is now support for explicit operator intercepting. +This can be used to customize specific operators as necessary. In order +to intercept an operator one has to override the +:attr:`SandboxedEnvironment.intercepted_binops` attribute. Once the +operator that needs to be intercepted is added to that set Jinja will +generate bytecode that calls the :meth:`SandboxedEnvironment.call_binop` +function. For unary operators the `unary` attributes and methods have to +be used instead. + +The default implementation of :attr:`SandboxedEnvironment.call_binop` +will use the :attr:`SandboxedEnvironment.binop_table` to translate +operator symbols into callbacks performing the default operator behavior. + +This example shows how the power (``**``) operator can be disabled in +Jinja:: + + from jinja2.sandbox import SandboxedEnvironment + + + class MyEnvironment(SandboxedEnvironment): + intercepted_binops = frozenset(['**']) + + def call_binop(self, context, operator, left, right): + if operator == '**': + return self.undefined('the power operator is unavailable') + return SandboxedEnvironment.call_binop(self, context, + operator, left, right) + +Make sure to always call into the super method, even if you are not +intercepting the call. Jinja might internally call the method to +evaluate expressions. diff --git a/docs/switching.rst b/docs/switching.rst new file mode 100644 index 00000000..b9ff954c --- /dev/null +++ b/docs/switching.rst @@ -0,0 +1,226 @@ +Switching from other Template Engines +===================================== + +.. highlight:: html+jinja + +If you have used a different template engine in the past and want to switch +to Jinja here is a small guide that shows the basic syntactic and semantic +changes between some common, similar text template engines for Python. + +Jinja 1 +------- + +Jinja 2 is mostly compatible with Jinja 1 in terms of API usage and template +syntax. The differences between Jinja 1 and 2 are explained in the following +list. + +API +~~~ + +Loaders + Jinja 2 uses a different loader API. Because the internal representation + of templates changed there is no longer support for external caching + systems such as memcached. The memory consumed by templates is comparable + with regular Python modules now and external caching doesn't give any + advantage. If you have used a custom loader in the past have a look at + the new :ref:`loader API <loaders>`. + +Loading templates from strings + In the past it was possible to generate templates from a string with the + default environment configuration by using `jinja.from_string`. Jinja 2 + provides a :class:`Template` class that can be used to do the same, but + with optional additional configuration. + +Automatic unicode conversion + Jinja 1 performed automatic conversion of bytes in a given encoding + into unicode objects. This conversion is no longer implemented as it + was inconsistent as most libraries are using the regular Python + ASCII bytes to Unicode conversion. An application powered by Jinja 2 + *has to* use unicode internally everywhere or make sure that Jinja 2 + only gets unicode strings passed. + +i18n + Jinja 1 used custom translators for internationalization. i18n is now + available as Jinja 2 extension and uses a simpler, more gettext friendly + interface and has support for babel. For more details see + :ref:`i18n-extension`. + +Internal methods + Jinja 1 exposed a few internal methods on the environment object such + as `call_function`, `get_attribute` and others. While they were marked + as being an internal method it was possible to override them. Jinja 2 + doesn't have equivalent methods. + +Sandbox + Jinja 1 was running sandbox mode by default. Few applications actually + used that feature so it became optional in Jinja 2. For more details + about the sandboxed execution see :class:`SandboxedEnvironment`. + +Context + Jinja 1 had a stacked context as storage for variables passed to the + environment. In Jinja 2 a similar object exists but it doesn't allow + modifications nor is it a singleton. As inheritance is dynamic now + multiple context objects may exist during template evaluation. + +Filters and Tests + Filters and tests are regular functions now. It's no longer necessary + and allowed to use factory functions. + + +Templates +~~~~~~~~~ + +Jinja 2 has mostly the same syntax as Jinja 1. What's different is that +macros require parentheses around the argument list now. + +Additionally Jinja 2 allows dynamic inheritance now and dynamic includes. +The old helper function `rendertemplate` is gone now, `include` can be used +instead. Includes no longer import macros and variable assignments, for +that the new `import` tag is used. This concept is explained in the +:ref:`import` documentation. + +Another small change happened in the `for`-tag. The special loop variable +doesn't have a `parent` attribute, instead you have to alias the loop +yourself. See :ref:`accessing-the-parent-loop` for more details. + + +Django +------ + +If you have previously worked with Django templates, you should find +Jinja very familiar. In fact, most of the syntax elements look and +work the same. + +However, Jinja provides some more syntax elements covered in the +documentation and some work a bit different. + +This section covers the template changes. As the API is fundamentally +different we won't cover it here. + +Method Calls +~~~~~~~~~~~~ + +In Django method calls work implicitly, while Jinja requires the explicit +Python syntax. Thus this Django code:: + + {% for page in user.get_created_pages %} + ... + {% endfor %} + +...looks like this in Jinja:: + + {% for page in user.get_created_pages() %} + ... + {% endfor %} + +This allows you to pass variables to the method, which is not possible in +Django. This syntax is also used for macros. + +Filter Arguments +~~~~~~~~~~~~~~~~ + +Jinja provides more than one argument for filters. Also the syntax for +argument passing is different. A template that looks like this in Django:: + + {{ items|join:", " }} + +looks like this in Jinja:: + + {{ items|join(', ') }} + +It is a bit more verbose, but it allows different types of arguments - +including variables - and more than one of them. + +Tests +~~~~~ + +In addition to filters there also are tests you can perform using the is +operator. Here are some examples:: + + {% if user.user_id is odd %} + {{ user.username|e }} is odd + {% else %} + hmm. {{ user.username|e }} looks pretty normal + {% endif %} + +Loops +~~~~~ + +For loops work very similarly to Django, but notably the Jinja special +variable for the loop context is called `loop`, not `forloop` as in Django. + +In addition, the Django `empty` argument is called `else` in Jinja. For +example, the Django template:: + + {% for item in items %} + {{ item }} + {% empty %} + No items! + {% endfor %} + +...looks like this in Jinja:: + + {% for item in items %} + {{ item }} + {% else %} + No items! + {% endfor %} + +Cycle +~~~~~ + +The ``{% cycle %}`` tag does not exist in Jinja; however, you can achieve the +same output by using the `cycle` method on the loop context special variable. + +The following Django template:: + + {% for user in users %} + <li class="{% cycle 'odd' 'even' %}">{{ user }}</li> + {% endfor %} + +...looks like this in Jinja:: + + {% for user in users %} + <li class="{{ loop.cycle('odd', 'even') }}">{{ user }}</li> + {% endfor %} + +There is no equivalent of ``{% cycle ... as variable %}``. + + +Mako +---- + +.. highlight:: html+mako + +If you have used Mako so far and want to switch to Jinja you can configure +Jinja to look more like Mako: + +.. sourcecode:: python + + env = Environment('<%', '%>', '${', '}', '<%doc>', '</%doc>', '%', '##') + +With an environment configured like that, Jinja should be able to interpret +a small subset of Mako templates. Jinja does not support embedded Python +code, so you would have to move that out of the template. The syntax for defs +(which are called macros in Jinja) and template inheritance is different too. +The following Mako template:: + + <%inherit file="layout.html" /> + <%def name="title()">Page Title</%def> + <ul> + % for item in list: + <li>${item}</li> + % endfor + </ul> + +Looks like this in Jinja with the above configuration:: + + <% extends "layout.html" %> + <% block title %>Page Title<% endblock %> + <% block body %> + <ul> + % for item in list: + <li>${item}</li> + % endfor + </ul> + <% endblock %> diff --git a/docs/templates.rst b/docs/templates.rst new file mode 100644 index 00000000..3101d0ed --- /dev/null +++ b/docs/templates.rst @@ -0,0 +1,1828 @@ +Template Designer Documentation +=============================== + +.. highlight:: html+jinja + +This document describes the syntax and semantics of the template engine and +will be most useful as reference to those creating Jinja templates. As the +template engine is very flexible, the configuration from the application can +be slightly different from the code presented here in terms of delimiters and +behavior of undefined values. + + +Synopsis +-------- + +A Jinja template is simply a text file. Jinja can generate any text-based +format (HTML, XML, CSV, LaTeX, etc.). A Jinja template doesn't need to have a +specific extension: ``.html``, ``.xml``, or any other extension is just fine. + +A template contains **variables** and/or **expressions**, which get replaced +with values when a template is *rendered*; and **tags**, which control the +logic of the template. The template syntax is heavily inspired by Django and +Python. + +Below is a minimal template that illustrates a few basics using the default +Jinja configuration. We will cover the details later in this document:: + + <!DOCTYPE html> + <html lang="en"> + <head> + <title>My Webpage</title> + </head> + <body> + <ul id="navigation"> + {% for item in navigation %} + <li><a href="{{ item.href }}">{{ item.caption }}</a></li> + {% endfor %} + </ul> + + <h1>My Webpage</h1> + {{ a_variable }} + + {# a comment #} + </body> + </html> + +The following example shows the default configuration settings. An application +developer can change the syntax configuration from ``{% foo %}`` to ``<% foo +%>``, or something similar. + +There are a few kinds of delimiters. The default Jinja delimiters are +configured as follows: + +* ``{% ... %}`` for :ref:`Statements <list-of-control-structures>` +* ``{{ ... }}`` for :ref:`Expressions` to print to the template output +* ``{# ... #}`` for :ref:`Comments` not included in the template output +* ``# ... ##`` for :ref:`Line Statements <line-statements>` + + +Template File Extension +~~~~~~~~~~~~~~~~~~~~~~~ + +As stated above, any file can be loaded as a template, regardless of +file extension. Adding a ``.jinja`` extension, like ``user.html.jinja`` +may make it easier for some IDEs or editor plugins, but is not required. +Autoescaping, introduced later, can be applied based on file extension, +so you'll need to take the extra suffix into account in that case. + +Another good heuristic for identifying templates is that they are in a +``templates`` folder, regardless of extension. This is a common layout +for projects. + + +.. _variables: + +Variables +--------- + +Template variables are defined by the context dictionary passed to the +template. + +You can mess around with the variables in templates provided they are passed in +by the application. Variables may have attributes or elements on them you can +access too. What attributes a variable has depends heavily on the application +providing that variable. + +You can use a dot (``.``) to access attributes of a variable in addition +to the standard Python ``__getitem__`` "subscript" syntax (``[]``). + +The following lines do the same thing:: + + {{ foo.bar }} + {{ foo['bar'] }} + +It's important to know that the outer double-curly braces are *not* part of the +variable, but the print statement. If you access variables inside tags don't +put the braces around them. + +If a variable or attribute does not exist, you will get back an undefined +value. What you can do with that kind of value depends on the application +configuration: the default behavior is to evaluate to an empty string if +printed or iterated over, and to fail for every other operation. + +.. _notes-on-subscriptions: + +.. admonition:: Implementation + + For the sake of convenience, ``foo.bar`` in Jinja does the following + things on the Python layer: + + - check for an attribute called `bar` on `foo` + (``getattr(foo, 'bar')``) + - if there is not, check for an item ``'bar'`` in `foo` + (``foo.__getitem__('bar')``) + - if there is not, return an undefined object. + + ``foo['bar']`` works mostly the same with a small difference in sequence: + + - check for an item ``'bar'`` in `foo`. + (``foo.__getitem__('bar')``) + - if there is not, check for an attribute called `bar` on `foo`. + (``getattr(foo, 'bar')``) + - if there is not, return an undefined object. + + This is important if an object has an item and attribute with the same + name. Additionally, the :func:`attr` filter only looks up attributes. + +.. _filters: + +Filters +------- + +Variables can be modified by **filters**. Filters are separated from the +variable by a pipe symbol (``|``) and may have optional arguments in +parentheses. Multiple filters can be chained. The output of one filter is +applied to the next. + +For example, ``{{ name|striptags|title }}`` will remove all HTML Tags from +variable `name` and title-case the output (``title(striptags(name))``). + +Filters that accept arguments have parentheses around the arguments, just like +a function call. For example: ``{{ listx|join(', ') }}`` will join a list with +commas (``str.join(', ', listx)``). + +The :ref:`builtin-filters` below describes all the builtin filters. + +.. _tests: + +Tests +----- + +Beside filters, there are also so-called "tests" available. Tests can be used +to test a variable against a common expression. To test a variable or +expression, you add `is` plus the name of the test after the variable. For +example, to find out if a variable is defined, you can do ``name is defined``, +which will then return true or false depending on whether `name` is defined +in the current template context. + +Tests can accept arguments, too. If the test only takes one argument, you can +leave out the parentheses. For example, the following two +expressions do the same thing:: + + {% if loop.index is divisibleby 3 %} + {% if loop.index is divisibleby(3) %} + +The :ref:`builtin-tests` below describes all the builtin tests. + + +.. _comments: + +Comments +-------- + +To comment-out part of a line in a template, use the comment syntax which is +by default set to ``{# ... #}``. This is useful to comment out parts of the +template for debugging or to add information for other template designers or +yourself:: + + {# note: commented-out template because we no longer use this + {% for user in users %} + ... + {% endfor %} + #} + + +Whitespace Control +------------------ + +In the default configuration: + +* a single trailing newline is stripped if present +* other whitespace (spaces, tabs, newlines etc.) is returned unchanged + +If an application configures Jinja to `trim_blocks`, the first newline after a +template tag is removed automatically (like in PHP). The `lstrip_blocks` +option can also be set to strip tabs and spaces from the beginning of a +line to the start of a block. (Nothing will be stripped if there are +other characters before the start of the block.) + +With both `trim_blocks` and `lstrip_blocks` enabled, you can put block tags +on their own lines, and the entire block line will be removed when +rendered, preserving the whitespace of the contents. For example, +without the `trim_blocks` and `lstrip_blocks` options, this template:: + + <div> + {% if True %} + yay + {% endif %} + </div> + +gets rendered with blank lines inside the div:: + + <div> + + yay + + </div> + +But with both `trim_blocks` and `lstrip_blocks` enabled, the template block +lines are removed and other whitespace is preserved:: + + <div> + yay + </div> + +You can manually disable the `lstrip_blocks` behavior by putting a +plus sign (``+``) at the start of a block:: + + <div> + {%+ if something %}yay{% endif %} + </div> + +Similarly, you can manually disable the ``trim_blocks`` behavior by +putting a plus sign (``+``) at the end of a block:: + + <div> + {% if something +%} + yay + {% endif %} + </div> + +You can also strip whitespace in templates by hand. If you add a minus +sign (``-``) to the start or end of a block (e.g. a :ref:`for-loop` tag), a +comment, or a variable expression, the whitespaces before or after +that block will be removed:: + + {% for item in seq -%} + {{ item }} + {%- endfor %} + +This will yield all elements without whitespace between them. If `seq` was +a list of numbers from ``1`` to ``9``, the output would be ``123456789``. + +If :ref:`line-statements` are enabled, they strip leading whitespace +automatically up to the beginning of the line. + +By default, Jinja also removes trailing newlines. To keep single +trailing newlines, configure Jinja to `keep_trailing_newline`. + +.. admonition:: Note + + You must not add whitespace between the tag and the minus sign. + + **valid**:: + + {%- if foo -%}...{% endif %} + + **invalid**:: + + {% - if foo - %}...{% endif %} + + +Escaping +-------- + +It is sometimes desirable -- even necessary -- to have Jinja ignore parts +it would otherwise handle as variables or blocks. For example, if, with +the default syntax, you want to use ``{{`` as a raw string in a template and +not start a variable, you have to use a trick. + +The easiest way to output a literal variable delimiter (``{{``) is by using a +variable expression:: + + {{ '{{' }} + +For bigger sections, it makes sense to mark a block `raw`. For example, to +include example Jinja syntax in a template, you can use this snippet:: + + {% raw %} + <ul> + {% for item in seq %} + <li>{{ item }}</li> + {% endfor %} + </ul> + {% endraw %} + +.. admonition:: Note + + Minus sign at the end of ``{% raw -%}`` tag cleans all the spaces and newlines + preceding the first character of your raw data. + + +.. _line-statements: + +Line Statements +--------------- + +If line statements are enabled by the application, it's possible to mark a +line as a statement. For example, if the line statement prefix is configured +to ``#``, the following two examples are equivalent:: + + <ul> + # for item in seq + <li>{{ item }}</li> + # endfor + </ul> + + <ul> + {% for item in seq %} + <li>{{ item }}</li> + {% endfor %} + </ul> + +The line statement prefix can appear anywhere on the line as long as no text +precedes it. For better readability, statements that start a block (such as +`for`, `if`, `elif` etc.) may end with a colon:: + + # for item in seq: + ... + # endfor + + +.. admonition:: Note + + Line statements can span multiple lines if there are open parentheses, + braces or brackets:: + + <ul> + # for href, caption in [('index.html', 'Index'), + ('about.html', 'About')]: + <li><a href="{{ href }}">{{ caption }}</a></li> + # endfor + </ul> + +Since Jinja 2.2, line-based comments are available as well. For example, if +the line-comment prefix is configured to be ``##``, everything from ``##`` to +the end of the line is ignored (excluding the newline sign):: + + # for item in seq: + <li>{{ item }}</li> ## this comment is ignored + # endfor + + +.. _template-inheritance: + +Template Inheritance +-------------------- + +The most powerful part of Jinja is template inheritance. Template inheritance +allows you to build a base "skeleton" template that contains all the common +elements of your site and defines **blocks** that child templates can override. + +Sounds complicated but is very basic. It's easiest to understand it by starting +with an example. + + +Base Template +~~~~~~~~~~~~~ + +This template, which we'll call ``base.html``, defines a simple HTML skeleton +document that you might use for a simple two-column page. It's the job of +"child" templates to fill the empty blocks with content:: + + <!DOCTYPE html> + <html lang="en"> + <head> + {% block head %} + <link rel="stylesheet" href="style.css" /> + <title>{% block title %}{% endblock %} - My Webpage</title> + {% endblock %} + </head> + <body> + <div id="content">{% block content %}{% endblock %}</div> + <div id="footer"> + {% block footer %} + © Copyright 2008 by <a href="http://domain.invalid/">you</a>. + {% endblock %} + </div> + </body> + </html> + +In this example, the ``{% block %}`` tags define four blocks that child templates +can fill in. All the `block` tag does is tell the template engine that a +child template may override those placeholders in the template. + +``block`` tags can be inside other blocks such as ``if``, but they will +always be executed regardless of if the ``if`` block is actually +rendered. + +Child Template +~~~~~~~~~~~~~~ + +A child template might look like this:: + + {% extends "base.html" %} + {% block title %}Index{% endblock %} + {% block head %} + {{ super() }} + <style type="text/css"> + .important { color: #336699; } + </style> + {% endblock %} + {% block content %} + <h1>Index</h1> + <p class="important"> + Welcome to my awesome homepage. + </p> + {% endblock %} + +The ``{% extends %}`` tag is the key here. It tells the template engine that +this template "extends" another template. When the template system evaluates +this template, it first locates the parent. The extends tag should be the +first tag in the template. Everything before it is printed out normally and +may cause confusion. For details about this behavior and how to take +advantage of it, see :ref:`null-master-fallback`. Also a block will always be +filled in regardless of whether the surrounding condition is evaluated to be true +or false. + +The filename of the template depends on the template loader. For example, the +:class:`FileSystemLoader` allows you to access other templates by giving the +filename. You can access templates in subdirectories with a slash:: + + {% extends "layout/default.html" %} + +But this behavior can depend on the application embedding Jinja. Note that +since the child template doesn't define the ``footer`` block, the value from +the parent template is used instead. + +You can't define multiple ``{% block %}`` tags with the same name in the +same template. This limitation exists because a block tag works in "both" +directions. That is, a block tag doesn't just provide a placeholder to fill +- it also defines the content that fills the placeholder in the *parent*. +If there were two similarly-named ``{% block %}`` tags in a template, +that template's parent wouldn't know which one of the blocks' content to use. + +If you want to print a block multiple times, you can, however, use the special +`self` variable and call the block with that name:: + + <title>{% block title %}{% endblock %}</title> + <h1>{{ self.title() }}</h1> + {% block body %}{% endblock %} + + +Super Blocks +~~~~~~~~~~~~ + +It's possible to render the contents of the parent block by calling ``super()``. +This gives back the results of the parent block:: + + {% block sidebar %} + <h3>Table Of Contents</h3> + ... + {{ super() }} + {% endblock %} + + +Nesting extends +~~~~~~~~~~~~~~~ + +In the case of multiple levels of ``{% extends %}``, +``super`` references may be chained (as in ``super.super()``) +to skip levels in the inheritance tree. + +For example:: + + # parent.tmpl + body: {% block body %}Hi from parent.{% endblock %} + + # child.tmpl + {% extends "parent.tmpl" %} + {% block body %}Hi from child. {{ super() }}{% endblock %} + + # grandchild1.tmpl + {% extends "child.tmpl" %} + {% block body %}Hi from grandchild1.{% endblock %} + + # grandchild2.tmpl + {% extends "child.tmpl" %} + {% block body %}Hi from grandchild2. {{ super.super() }} {% endblock %} + + +Rendering ``child.tmpl`` will give +``body: Hi from child. Hi from parent.`` + +Rendering ``grandchild1.tmpl`` will give +``body: Hi from grandchild1.`` + +Rendering ``grandchild2.tmpl`` will give +``body: Hi from grandchild2. Hi from parent.`` + + +Named Block End-Tags +~~~~~~~~~~~~~~~~~~~~ + +Jinja allows you to put the name of the block after the end tag for better +readability:: + + {% block sidebar %} + {% block inner_sidebar %} + ... + {% endblock inner_sidebar %} + {% endblock sidebar %} + +However, the name after the `endblock` word must match the block name. + + +Block Nesting and Scope +~~~~~~~~~~~~~~~~~~~~~~~ + +Blocks can be nested for more complex layouts. However, per default blocks +may not access variables from outer scopes:: + + {% for item in seq %} + <li>{% block loop_item %}{{ item }}{% endblock %}</li> + {% endfor %} + +This example would output empty ``<li>`` items because `item` is unavailable +inside the block. The reason for this is that if the block is replaced by +a child template, a variable would appear that was not defined in the block or +passed to the context. + +Starting with Jinja 2.2, you can explicitly specify that variables are +available in a block by setting the block to "scoped" by adding the `scoped` +modifier to a block declaration:: + + {% for item in seq %} + <li>{% block loop_item scoped %}{{ item }}{% endblock %}</li> + {% endfor %} + +When overriding a block, the `scoped` modifier does not have to be provided. + + +Template Objects +~~~~~~~~~~~~~~~~ + +.. versionchanged:: 2.4 + +If a template object was passed in the template context, you can +extend from that object as well. Assuming the calling code passes +a layout template as `layout_template` to the environment, this +code works:: + + {% extends layout_template %} + +Previously, the `layout_template` variable had to be a string with +the layout template's filename for this to work. + + +HTML Escaping +------------- + +When generating HTML from templates, there's always a risk that a variable will +include characters that affect the resulting HTML. There are two approaches: + +a. manually escaping each variable; or +b. automatically escaping everything by default. + +Jinja supports both. What is used depends on the application configuration. +The default configuration is no automatic escaping; for various reasons: + +- Escaping everything except for safe values will also mean that Jinja is + escaping variables known to not include HTML (e.g. numbers, booleans) + which can be a huge performance hit. + +- The information about the safety of a variable is very fragile. It could + happen that by coercing safe and unsafe values, the return value is + double-escaped HTML. + +Working with Manual Escaping +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If manual escaping is enabled, it's **your** responsibility to escape +variables if needed. What to escape? If you have a variable that *may* +include any of the following chars (``>``, ``<``, ``&``, or ``"``) you +**SHOULD** escape it unless the variable contains well-formed and trusted +HTML. Escaping works by piping the variable through the ``|e`` filter:: + + {{ user.username|e }} + +Working with Automatic Escaping +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When automatic escaping is enabled, everything is escaped by default except +for values explicitly marked as safe. Variables and expressions +can be marked as safe either in: + +a. The context dictionary by the application with + :class:`markupsafe.Markup` +b. The template, with the ``|safe`` filter. + +If a string that you marked safe is passed through other Python code +that doesn't understand that mark, it may get lost. Be aware of when +your data is marked safe and how it is processed before arriving at the +template. + +If a value has been escaped but is not marked safe, auto-escaping will +still take place and result in double-escaped characters. If you know +you have data that is already safe but not marked, be sure to wrap it in +``Markup`` or use the ``|safe`` filter. + +Jinja functions (macros, `super`, `self.BLOCKNAME`) always return template +data that is marked as safe. + +String literals in templates with automatic escaping are considered +unsafe because native Python strings are not safe. + +.. _list-of-control-structures: + +List of Control Structures +-------------------------- + +A control structure refers to all those things that control the flow of a +program - conditionals (i.e. if/elif/else), for-loops, as well as things like +macros and blocks. With the default syntax, control structures appear inside +``{% ... %}`` blocks. + +.. _for-loop: + +For +~~~ + +Loop over each item in a sequence. For example, to display a list of users +provided in a variable called `users`:: + + <h1>Members</h1> + <ul> + {% for user in users %} + <li>{{ user.username|e }}</li> + {% endfor %} + </ul> + +As variables in templates retain their object properties, it is possible to +iterate over containers like `dict`:: + + <dl> + {% for key, value in my_dict.items() %} + <dt>{{ key|e }}</dt> + <dd>{{ value|e }}</dd> + {% endfor %} + </dl> + +Note, however, that **Python dicts are not ordered**; so you might want to +either pass a sorted ``list`` of ``tuple`` s -- or a +``collections.OrderedDict`` -- to the template, or use the `dictsort` filter. + +Inside of a for-loop block, you can access some special variables: + ++-----------------------+---------------------------------------------------+ +| Variable | Description | ++=======================+===================================================+ +| `loop.index` | The current iteration of the loop. (1 indexed) | ++-----------------------+---------------------------------------------------+ +| `loop.index0` | The current iteration of the loop. (0 indexed) | ++-----------------------+---------------------------------------------------+ +| `loop.revindex` | The number of iterations from the end of the loop | +| | (1 indexed) | ++-----------------------+---------------------------------------------------+ +| `loop.revindex0` | The number of iterations from the end of the loop | +| | (0 indexed) | ++-----------------------+---------------------------------------------------+ +| `loop.first` | True if first iteration. | ++-----------------------+---------------------------------------------------+ +| `loop.last` | True if last iteration. | ++-----------------------+---------------------------------------------------+ +| `loop.length` | The number of items in the sequence. | ++-----------------------+---------------------------------------------------+ +| `loop.cycle` | A helper function to cycle between a list of | +| | sequences. See the explanation below. | ++-----------------------+---------------------------------------------------+ +| `loop.depth` | Indicates how deep in a recursive loop | +| | the rendering currently is. Starts at level 1 | ++-----------------------+---------------------------------------------------+ +| `loop.depth0` | Indicates how deep in a recursive loop | +| | the rendering currently is. Starts at level 0 | ++-----------------------+---------------------------------------------------+ +| `loop.previtem` | The item from the previous iteration of the loop. | +| | Undefined during the first iteration. | ++-----------------------+---------------------------------------------------+ +| `loop.nextitem` | The item from the following iteration of the loop.| +| | Undefined during the last iteration. | ++-----------------------+---------------------------------------------------+ +| `loop.changed(*val)` | True if previously called with a different value | +| | (or not called at all). | ++-----------------------+---------------------------------------------------+ + +Within a for-loop, it's possible to cycle among a list of strings/variables +each time through the loop by using the special `loop.cycle` helper:: + + {% for row in rows %} + <li class="{{ loop.cycle('odd', 'even') }}">{{ row }}</li> + {% endfor %} + +Since Jinja 2.1, an extra `cycle` helper exists that allows loop-unbound +cycling. For more information, have a look at the :ref:`builtin-globals`. + +.. _loop-filtering: + +Unlike in Python, it's not possible to `break` or `continue` in a loop. You +can, however, filter the sequence during iteration, which allows you to skip +items. The following example skips all the users which are hidden:: + + {% for user in users if not user.hidden %} + <li>{{ user.username|e }}</li> + {% endfor %} + +The advantage is that the special `loop` variable will count correctly; thus +not counting the users not iterated over. + +If no iteration took place because the sequence was empty or the filtering +removed all the items from the sequence, you can render a default block +by using `else`:: + + <ul> + {% for user in users %} + <li>{{ user.username|e }}</li> + {% else %} + <li><em>no users found</em></li> + {% endfor %} + </ul> + +Note that, in Python, `else` blocks are executed whenever the corresponding +loop **did not** `break`. Since Jinja loops cannot `break` anyway, +a slightly different behavior of the `else` keyword was chosen. + +It is also possible to use loops recursively. This is useful if you are +dealing with recursive data such as sitemaps or RDFa. +To use loops recursively, you basically have to add the `recursive` modifier +to the loop definition and call the `loop` variable with the new iterable +where you want to recurse. + +The following example implements a sitemap with recursive loops:: + + <ul class="sitemap"> + {%- for item in sitemap recursive %} + <li><a href="{{ item.href|e }}">{{ item.title }}</a> + {%- if item.children -%} + <ul class="submenu">{{ loop(item.children) }}</ul> + {%- endif %}</li> + {%- endfor %} + </ul> + +The `loop` variable always refers to the closest (innermost) loop. If we +have more than one level of loops, we can rebind the variable `loop` by +writing `{% set outer_loop = loop %}` after the loop that we want to +use recursively. Then, we can call it using `{{ outer_loop(...) }}` + +Please note that assignments in loops will be cleared at the end of the +iteration and cannot outlive the loop scope. Older versions of Jinja had +a bug where in some circumstances it appeared that assignments would work. +This is not supported. See :ref:`assignments` for more information about +how to deal with this. + +If all you want to do is check whether some value has changed since the +last iteration or will change in the next iteration, you can use `previtem` +and `nextitem`:: + + {% for value in values %} + {% if loop.previtem is defined and value > loop.previtem %} + The value just increased! + {% endif %} + {{ value }} + {% if loop.nextitem is defined and loop.nextitem > value %} + The value will increase even more! + {% endif %} + {% endfor %} + +If you only care whether the value changed at all, using `changed` is even +easier:: + + {% for entry in entries %} + {% if loop.changed(entry.category) %} + <h2>{{ entry.category }}</h2> + {% endif %} + <p>{{ entry.message }}</p> + {% endfor %} + +.. _if: + +If +~~ + +The `if` statement in Jinja is comparable with the Python if statement. +In the simplest form, you can use it to test if a variable is defined, not +empty and not false:: + + {% if users %} + <ul> + {% for user in users %} + <li>{{ user.username|e }}</li> + {% endfor %} + </ul> + {% endif %} + +For multiple branches, `elif` and `else` can be used like in Python. You can +use more complex :ref:`expressions` there, too:: + + {% if kenny.sick %} + Kenny is sick. + {% elif kenny.dead %} + You killed Kenny! You bastard!!! + {% else %} + Kenny looks okay --- so far + {% endif %} + +If can also be used as an :ref:`inline expression <if-expression>` and for +:ref:`loop filtering <loop-filtering>`. + +.. _macros: + +Macros +~~~~~~ + +Macros are comparable with functions in regular programming languages. They +are useful to put often used idioms into reusable functions to not repeat +yourself ("DRY"). + +Here's a small example of a macro that renders a form element:: + + {% macro input(name, value='', type='text', size=20) -%} + <input type="{{ type }}" name="{{ name }}" value="{{ + value|e }}" size="{{ size }}"> + {%- endmacro %} + +The macro can then be called like a function in the namespace:: + + <p>{{ input('username') }}</p> + <p>{{ input('password', type='password') }}</p> + +If the macro was defined in a different template, you have to +:ref:`import <import>` it first. + +Inside macros, you have access to three special variables: + +`varargs` + If more positional arguments are passed to the macro than accepted by the + macro, they end up in the special `varargs` variable as a list of values. + +`kwargs` + Like `varargs` but for keyword arguments. All unconsumed keyword + arguments are stored in this special variable. + +`caller` + If the macro was called from a :ref:`call<call>` tag, the caller is stored + in this variable as a callable macro. + +Macros also expose some of their internal details. The following attributes +are available on a macro object: + +`name` + The name of the macro. ``{{ input.name }}`` will print ``input``. + +`arguments` + A tuple of the names of arguments the macro accepts. + +`defaults` + A tuple of default values. + +`catch_kwargs` + This is `true` if the macro accepts extra keyword arguments (i.e.: accesses + the special `kwargs` variable). + +`catch_varargs` + This is `true` if the macro accepts extra positional arguments (i.e.: + accesses the special `varargs` variable). + +`caller` + This is `true` if the macro accesses the special `caller` variable and may + be called from a :ref:`call<call>` tag. + +If a macro name starts with an underscore, it's not exported and can't +be imported. + + +.. _call: + +Call +~~~~ + +In some cases it can be useful to pass a macro to another macro. For this +purpose, you can use the special `call` block. The following example shows +a macro that takes advantage of the call functionality and how it can be +used:: + + {% macro render_dialog(title, class='dialog') -%} + <div class="{{ class }}"> + <h2>{{ title }}</h2> + <div class="contents"> + {{ caller() }} + </div> + </div> + {%- endmacro %} + + {% call render_dialog('Hello World') %} + This is a simple dialog rendered by using a macro and + a call block. + {% endcall %} + +It's also possible to pass arguments back to the call block. This makes it +useful as a replacement for loops. Generally speaking, a call block works +exactly like a macro without a name. + +Here's an example of how a call block can be used with arguments:: + + {% macro dump_users(users) -%} + <ul> + {%- for user in users %} + <li><p>{{ user.username|e }}</p>{{ caller(user) }}</li> + {%- endfor %} + </ul> + {%- endmacro %} + + {% call(user) dump_users(list_of_user) %} + <dl> + <dt>Realname</dt> + <dd>{{ user.realname|e }}</dd> + <dt>Description</dt> + <dd>{{ user.description }}</dd> + </dl> + {% endcall %} + + +Filters +~~~~~~~ + +Filter sections allow you to apply regular Jinja filters on a block of +template data. Just wrap the code in the special `filter` section:: + + {% filter upper %} + This text becomes uppercase + {% endfilter %} + + +.. _assignments: + +Assignments +~~~~~~~~~~~ + +Inside code blocks, you can also assign values to variables. Assignments at +top level (outside of blocks, macros or loops) are exported from the template +like top level macros and can be imported by other templates. + +Assignments use the `set` tag and can have multiple targets:: + + {% set navigation = [('index.html', 'Index'), ('about.html', 'About')] %} + {% set key, value = call_something() %} + +.. admonition:: Scoping Behavior + + Please keep in mind that it is not possible to set variables inside a + block and have them show up outside of it. This also applies to + loops. The only exception to that rule are if statements which do not + introduce a scope. As a result the following template is not going + to do what you might expect:: + + {% set iterated = false %} + {% for item in seq %} + {{ item }} + {% set iterated = true %} + {% endfor %} + {% if not iterated %} did not iterate {% endif %} + + It is not possible with Jinja syntax to do this. Instead use + alternative constructs like the loop else block or the special `loop` + variable:: + + {% for item in seq %} + {{ item }} + {% else %} + did not iterate + {% endfor %} + + As of version 2.10 more complex use cases can be handled using namespace + objects which allow propagating of changes across scopes:: + + {% set ns = namespace(found=false) %} + {% for item in items %} + {% if item.check_something() %} + {% set ns.found = true %} + {% endif %} + * {{ item.title }} + {% endfor %} + Found item having something: {{ ns.found }} + + Note that the ``obj.attr`` notation in the `set` tag is only allowed for + namespace objects; attempting to assign an attribute on any other object + will raise an exception. + + .. versionadded:: 2.10 Added support for namespace objects + + +Block Assignments +~~~~~~~~~~~~~~~~~ + +.. versionadded:: 2.8 + +Starting with Jinja 2.8, it's possible to also use block assignments to +capture the contents of a block into a variable name. This can be useful +in some situations as an alternative for macros. In that case, instead of +using an equals sign and a value, you just write the variable name and then +everything until ``{% endset %}`` is captured. + +Example:: + + {% set navigation %} + <li><a href="/">Index</a> + <li><a href="/downloads">Downloads</a> + {% endset %} + +The `navigation` variable then contains the navigation HTML source. + +.. versionchanged:: 2.10 + +Starting with Jinja 2.10, the block assignment supports filters. + +Example:: + + {% set reply | wordwrap %} + You wrote: + {{ message }} + {% endset %} + + +.. _extends: + +Extends +~~~~~~~ + +The `extends` tag can be used to extend one template from another. You can +have multiple `extends` tags in a file, but only one of them may be executed at +a time. + +See the section about :ref:`template-inheritance` above. + + +.. _blocks: + +Blocks +~~~~~~ + +Blocks are used for inheritance and act as both placeholders and replacements +at the same time. They are documented in detail in the +:ref:`template-inheritance` section. + + +Include +~~~~~~~ + +The `include` tag is useful to include a template and return the +rendered contents of that file into the current namespace:: + + {% include 'header.html' %} + Body + {% include 'footer.html' %} + +Included templates have access to the variables of the active context by +default. For more details about context behavior of imports and includes, +see :ref:`import-visibility`. + +From Jinja 2.2 onwards, you can mark an include with ``ignore missing``; in +which case Jinja will ignore the statement if the template to be included +does not exist. When combined with ``with`` or ``without context``, it must +be placed *before* the context visibility statement. Here are some valid +examples:: + + {% include "sidebar.html" ignore missing %} + {% include "sidebar.html" ignore missing with context %} + {% include "sidebar.html" ignore missing without context %} + +.. versionadded:: 2.2 + +You can also provide a list of templates that are checked for existence +before inclusion. The first template that exists will be included. If +`ignore missing` is given, it will fall back to rendering nothing if +none of the templates exist, otherwise it will raise an exception. + +Example:: + + {% include ['page_detailed.html', 'page.html'] %} + {% include ['special_sidebar.html', 'sidebar.html'] ignore missing %} + +.. versionchanged:: 2.4 + If a template object was passed to the template context, you can + include that object using `include`. + +.. _import: + +Import +~~~~~~ + +Jinja supports putting often used code into macros. These macros can go into +different templates and get imported from there. This works similarly to the +import statements in Python. It's important to know that imports are cached +and imported templates don't have access to the current template variables, +just the globals by default. For more details about context behavior of +imports and includes, see :ref:`import-visibility`. + +There are two ways to import templates. You can import a complete template +into a variable or request specific macros / exported variables from it. + +Imagine we have a helper module that renders forms (called `forms.html`):: + + {% macro input(name, value='', type='text') -%} + <input type="{{ type }}" value="{{ value|e }}" name="{{ name }}"> + {%- endmacro %} + + {%- macro textarea(name, value='', rows=10, cols=40) -%} + <textarea name="{{ name }}" rows="{{ rows }}" cols="{{ cols + }}">{{ value|e }}</textarea> + {%- endmacro %} + +The easiest and most flexible way to access a template's variables +and macros is to import the whole template module into a variable. +That way, you can access the attributes:: + + {% import 'forms.html' as forms %} + <dl> + <dt>Username</dt> + <dd>{{ forms.input('username') }}</dd> + <dt>Password</dt> + <dd>{{ forms.input('password', type='password') }}</dd> + </dl> + <p>{{ forms.textarea('comment') }}</p> + + +Alternatively, you can import specific names from a template into the current +namespace:: + + {% from 'forms.html' import input as input_field, textarea %} + <dl> + <dt>Username</dt> + <dd>{{ input_field('username') }}</dd> + <dt>Password</dt> + <dd>{{ input_field('password', type='password') }}</dd> + </dl> + <p>{{ textarea('comment') }}</p> + +Macros and variables starting with one or more underscores are private and +cannot be imported. + +.. versionchanged:: 2.4 + If a template object was passed to the template context, you can + import from that object. + + +.. _import-visibility: + +Import Context Behavior +----------------------- + +By default, included templates are passed the current context and imported +templates are not. The reason for this is that imports, unlike includes, +are cached; as imports are often used just as a module that holds macros. + +This behavior can be changed explicitly: by adding `with context` +or `without context` to the import/include directive, the current context +can be passed to the template and caching is disabled automatically. + +Here are two examples:: + + {% from 'forms.html' import input with context %} + {% include 'header.html' without context %} + +.. admonition:: Note + + In Jinja 2.0, the context that was passed to the included template + did not include variables defined in the template. As a matter of + fact, this did not work:: + + {% for box in boxes %} + {% include "render_box.html" %} + {% endfor %} + + The included template ``render_box.html`` is *not* able to access + `box` in Jinja 2.0. As of Jinja 2.1, ``render_box.html`` *is* able + to do so. + + +.. _expressions: + +Expressions +----------- + +Jinja allows basic expressions everywhere. These work very similarly to +regular Python; even if you're not working with Python +you should feel comfortable with it. + +Literals +~~~~~~~~ + +The simplest form of expressions are literals. Literals are representations +for Python objects such as strings and numbers. The following literals exist: + +``"Hello World"`` + Everything between two double or single quotes is a string. They are + useful whenever you need a string in the template (e.g. as + arguments to function calls and filters, or just to extend or include a + template). + +``42`` / ``123_456`` + Integers are whole numbers without a decimal part. The '_' character + can be used to separate groups for legibility. + +``42.23`` / ``42.1e2`` / ``123_456.789`` + Floating point numbers can be written using a '.' as a decimal mark. + They can also be written in scientific notation with an upper or + lower case 'e' to indicate the exponent part. The '_' character can + be used to separate groups for legibility, but cannot be used in the + exponent part. + +``['list', 'of', 'objects']`` + Everything between two brackets is a list. Lists are useful for storing + sequential data to be iterated over. For example, you can easily + create a list of links using lists and tuples for (and with) a for loop:: + + <ul> + {% for href, caption in [('index.html', 'Index'), ('about.html', 'About'), + ('downloads.html', 'Downloads')] %} + <li><a href="{{ href }}">{{ caption }}</a></li> + {% endfor %} + </ul> + +``('tuple', 'of', 'values')`` + Tuples are like lists that cannot be modified ("immutable"). If a tuple + only has one item, it must be followed by a comma (``('1-tuple',)``). + Tuples are usually used to represent items of two or more elements. + See the list example above for more details. + +``{'dict': 'of', 'key': 'and', 'value': 'pairs'}`` + A dict in Python is a structure that combines keys and values. Keys must + be unique and always have exactly one value. Dicts are rarely used in + templates; they are useful in some rare cases such as the :func:`xmlattr` + filter. + +``true`` / ``false`` + ``true`` is always true and ``false`` is always false. + +.. admonition:: Note + + The special constants `true`, `false`, and `none` are indeed lowercase. + Because that caused confusion in the past, (`True` used to expand + to an undefined variable that was considered false), + all three can now also be written in title case + (`True`, `False`, and `None`). + However, for consistency, (all Jinja identifiers are lowercase) + you should use the lowercase versions. + +Math +~~~~ + +Jinja allows you to calculate with values. This is rarely useful in templates +but exists for completeness' sake. The following operators are supported: + +``+`` + Adds two objects together. Usually the objects are numbers, but if both are + strings or lists, you can concatenate them this way. This, however, is not + the preferred way to concatenate strings! For string concatenation, have + a look-see at the ``~`` operator. ``{{ 1 + 1 }}`` is ``2``. + +``-`` + Subtract the second number from the first one. ``{{ 3 - 2 }}`` is ``1``. + +``/`` + Divide two numbers. The return value will be a floating point number. + ``{{ 1 / 2 }}`` is ``{{ 0.5 }}``. + +``//`` + Divide two numbers and return the truncated integer result. + ``{{ 20 // 7 }}`` is ``2``. + +``%`` + Calculate the remainder of an integer division. ``{{ 11 % 7 }}`` is ``4``. + +``*`` + Multiply the left operand with the right one. ``{{ 2 * 2 }}`` would + return ``4``. This can also be used to repeat a string multiple times. + ``{{ '=' * 80 }}`` would print a bar of 80 equal signs. + +``**`` + Raise the left operand to the power of the right operand. ``{{ 2**3 }}`` + would return ``8``. + +Comparisons +~~~~~~~~~~~ + +``==`` + Compares two objects for equality. + +``!=`` + Compares two objects for inequality. + +``>`` + ``true`` if the left hand side is greater than the right hand side. + +``>=`` + ``true`` if the left hand side is greater or equal to the right hand side. + +``<`` + ``true`` if the left hand side is lower than the right hand side. + +``<=`` + ``true`` if the left hand side is lower or equal to the right hand side. + +Logic +~~~~~ + +For ``if`` statements, ``for`` filtering, and ``if`` expressions, it can be useful to +combine multiple expressions: + +``and`` + Return true if the left and the right operand are true. + +``or`` + Return true if the left or the right operand are true. + +``not`` + negate a statement (see below). + +``(expr)`` + Parentheses group an expression. + +.. admonition:: Note + + The ``is`` and ``in`` operators support negation using an infix notation, + too: ``foo is not bar`` and ``foo not in bar`` instead of ``not foo is bar`` + and ``not foo in bar``. All other expressions require a prefix notation: + ``not (foo and bar).`` + + +Other Operators +~~~~~~~~~~~~~~~ + +The following operators are very useful but don't fit into any of the other +two categories: + +``in`` + Perform a sequence / mapping containment test. Returns true if the left + operand is contained in the right. ``{{ 1 in [1, 2, 3] }}`` would, for + example, return true. + +``is`` + Performs a :ref:`test <tests>`. + +``|`` + Applies a :ref:`filter <filters>`. + +``~`` + Converts all operands into strings and concatenates them. + + ``{{ "Hello " ~ name ~ "!" }}`` would return (assuming `name` is set + to ``'John'``) ``Hello John!``. + +``()`` + Call a callable: ``{{ post.render() }}``. Inside of the parentheses you + can use positional arguments and keyword arguments like in Python: + + ``{{ post.render(user, full=true) }}``. + +``.`` / ``[]`` + Get an attribute of an object. (See :ref:`variables`) + + +.. _if-expression: + +If Expression +~~~~~~~~~~~~~ + +It is also possible to use inline `if` expressions. These are useful in some +situations. For example, you can use this to extend from one template if a +variable is defined, otherwise from the default layout template:: + + {% extends layout_template if layout_template is defined else 'master.html' %} + +The general syntax is ``<do something> if <something is true> else <do +something else>``. + +The `else` part is optional. If not provided, the else block implicitly +evaluates into an :class:`Undefined` object (regardless of what ``undefined`` +in the environment is set to): + +.. sourcecode:: jinja + + {{ "[{}]".format(page.title) if page.title }} + + +.. _python-methods: + +Python Methods +~~~~~~~~~~~~~~ + +You can also use any of the methods of defined on a variable's type. +The value returned from the method invocation is used as the value of the expression. +Here is an example that uses methods defined on strings (where ``page.title`` is a string): + +.. code-block:: text + + {{ page.title.capitalize() }} + +This works for methods on user-defined types. For example, if variable +``f`` of type ``Foo`` has a method ``bar`` defined on it, you can do the +following: + +.. code-block:: text + + {{ f.bar(value) }} + +Operator methods also work as expected. For example, ``%`` implements +printf-style for strings: + +.. code-block:: text + + {{ "Hello, %s!" % name }} + +Although you should prefer the ``.format`` method for that case (which +is a bit contrived in the context of rendering a template): + +.. code-block:: text + + {{ "Hello, {}!".format(name) }} + + +.. _builtin-filters: + +List of Builtin Filters +----------------------- + +.. jinja:filters:: jinja2.defaults.DEFAULT_FILTERS + + +.. _builtin-tests: + +List of Builtin Tests +--------------------- + +.. jinja:tests:: jinja2.defaults.DEFAULT_TESTS + + +.. _builtin-globals: + +List of Global Functions +------------------------ + +The following functions are available in the global scope by default: + +.. function:: range([start,] stop[, step]) + + Return a list containing an arithmetic progression of integers. + ``range(i, j)`` returns ``[i, i+1, i+2, ..., j-1]``; + start (!) defaults to ``0``. + When step is given, it specifies the increment (or decrement). + For example, ``range(4)`` and ``range(0, 4, 1)`` return ``[0, 1, 2, 3]``. + The end point is omitted! + These are exactly the valid indices for a list of 4 elements. + + This is useful to repeat a template block multiple times, e.g. + to fill a list. Imagine you have 7 users in the list but you want to + render three empty items to enforce a height with CSS:: + + <ul> + {% for user in users %} + <li>{{ user.username }}</li> + {% endfor %} + {% for number in range(10 - users|count) %} + <li class="empty"><span>...</span></li> + {% endfor %} + </ul> + +.. function:: lipsum(n=5, html=True, min=20, max=100) + + Generates some lorem ipsum for the template. By default, five paragraphs + of HTML are generated with each paragraph between 20 and 100 words. + If html is False, regular text is returned. This is useful to generate simple + contents for layout testing. + +.. function:: dict(\**items) + + A convenient alternative to dict literals. ``{'foo': 'bar'}`` is the same + as ``dict(foo='bar')``. + +.. class:: cycler(\*items) + + Cycle through values by yielding them one at a time, then restarting + once the end is reached. + + Similar to ``loop.cycle``, but can be used outside loops or across + multiple loops. For example, render a list of folders and files in a + list, alternating giving them "odd" and "even" classes. + + .. code-block:: html+jinja + + {% set row_class = cycler("odd", "even") %} + <ul class="browser"> + {% for folder in folders %} + <li class="folder {{ row_class.next() }}">{{ folder }} + {% endfor %} + {% for file in files %} + <li class="file {{ row_class.next() }}">{{ file }} + {% endfor %} + </ul> + + :param items: Each positional argument will be yielded in the order + given for each cycle. + + .. versionadded:: 2.1 + + .. method:: current + :property: + + Return the current item. Equivalent to the item that will be + returned next time :meth:`next` is called. + + .. method:: next() + + Return the current item, then advance :attr:`current` to the + next item. + + .. method:: reset() + + Resets the current item to the first item. + +.. class:: joiner(sep=', ') + + A tiny helper that can be used to "join" multiple sections. A joiner is + passed a string and will return that string every time it's called, except + the first time (in which case it returns an empty string). You can + use this to join things:: + + {% set pipe = joiner("|") %} + {% if categories %} {{ pipe() }} + Categories: {{ categories|join(", ") }} + {% endif %} + {% if author %} {{ pipe() }} + Author: {{ author() }} + {% endif %} + {% if can_edit %} {{ pipe() }} + <a href="?action=edit">Edit</a> + {% endif %} + + .. versionadded:: 2.1 + +.. class:: namespace(...) + + Creates a new container that allows attribute assignment using the + ``{% set %}`` tag:: + + {% set ns = namespace() %} + {% set ns.foo = 'bar' %} + + The main purpose of this is to allow carrying a value from within a loop + body to an outer scope. Initial values can be provided as a dict, as + keyword arguments, or both (same behavior as Python's `dict` constructor):: + + {% set ns = namespace(found=false) %} + {% for item in items %} + {% if item.check_something() %} + {% set ns.found = true %} + {% endif %} + * {{ item.title }} + {% endfor %} + Found item having something: {{ ns.found }} + + .. versionadded:: 2.10 + + +Extensions +---------- + +The following sections cover the built-in Jinja extensions that may be +enabled by an application. An application could also provide further +extensions not covered by this documentation; in which case there should +be a separate document explaining said :ref:`extensions +<jinja-extensions>`. + + +.. _i18n-in-templates: + +i18n +~~~~ + +If the :ref:`i18n-extension` is enabled, it's possible to mark text in +the template as translatable. To mark a section as translatable, use a +``trans`` block: + +.. code-block:: jinja + + {% trans %}Hello, {{ user }}!{% endtrans %} + +Inside the block, no statements are allowed, only text and simple +variable tags. + +Variable tags can only be a name, not attribute access, filters, or +other expressions. To use an expression, bind it to a name in the +``trans`` tag for use in the block. + +.. code-block:: jinja + + {% trans user=user.username %}Hello, {{ user }}!{% endtrans %} + +To bind more than one expression, separate each with a comma (``,``). + +.. code-block:: jinja + + {% trans book_title=book.title, author=author.name %} + This is {{ book_title }} by {{ author }} + {% endtrans %} + +To pluralize, specify both the singular and plural forms separated by +the ``pluralize`` tag. + +.. code-block:: jinja + + {% trans count=list|length %} + There is {{ count }} {{ name }} object. + {% pluralize %} + There are {{ count }} {{ name }} objects. + {% endtrans %} + +By default, the first variable in a block is used to determine whether +to use singular or plural form. If that isn't correct, specify the +variable used for pluralizing as a parameter to ``pluralize``. + +.. code-block:: jinja + + {% trans ..., user_count=users|length %}... + {% pluralize user_count %}...{% endtrans %} + +When translating blocks of text, whitespace and linebreaks result in +hard to read and error-prone translation strings. To avoid this, a trans +block can be marked as trimmed, which will replace all linebreaks and +the whitespace surrounding them with a single space and remove leading +and trailing whitespace. + +.. code-block:: jinja + + {% trans trimmed book_title=book.title %} + This is {{ book_title }}. + You should read it! + {% endtrans %} + +This results in ``This is %(book_title)s. You should read it!`` in the +translation file. + +If trimming is enabled globally, the ``notrimmed`` modifier can be used +to disable it for a block. + +.. versionadded:: 2.10 + The ``trimmed`` and ``notrimmed`` modifiers have been added. + +It's possible to translate strings in expressions with these functions: + +- ``gettext``: translate a single string +- ``ngettext``: translate a pluralizable string +- ``_``: alias for ``gettext`` + +You can print a translated string like this: + +.. code-block:: jinja + + {{ _("Hello, World!") }} + +To use placeholders, use the ``format`` filter. + +.. code-block:: jinja + + {{ _("Hello, %(user)s!")|format(user=user.username) }} + +Always use keyword arguments to ``format``, as other languages may not +use the words in the same order. + +If :ref:`newstyle-gettext` calls are activated, using placeholders is +easier. Formatting is part of the ``gettext`` call instead of using the +``format`` filter. + +.. sourcecode:: jinja + + {{ gettext('Hello World!') }} + {{ gettext('Hello %(name)s!', name='World') }} + {{ ngettext('%(num)d apple', '%(num)d apples', apples|count) }} + +The ``ngettext`` function's format string automatically receives the +count as a ``num`` parameter in addition to the given parameters. + + +Expression Statement +~~~~~~~~~~~~~~~~~~~~ + +If the expression-statement extension is loaded, a tag called `do` is available +that works exactly like the regular variable expression (``{{ ... }}``); except +it doesn't print anything. This can be used to modify lists:: + + {% do navigation.append('a string') %} + + +Loop Controls +~~~~~~~~~~~~~ + +If the application enables the :ref:`loopcontrols-extension`, it's possible to +use `break` and `continue` in loops. When `break` is reached, the loop is +terminated; if `continue` is reached, the processing is stopped and continues +with the next iteration. + +Here's a loop that skips every second item:: + + {% for user in users %} + {%- if loop.index is even %}{% continue %}{% endif %} + ... + {% endfor %} + +Likewise, a loop that stops processing after the 10th iteration:: + + {% for user in users %} + {%- if loop.index >= 10 %}{% break %}{% endif %} + {%- endfor %} + +Note that ``loop.index`` starts with 1, and ``loop.index0`` starts with 0 +(See: :ref:`for-loop`). + + +Debug Statement +~~~~~~~~~~~~~~~ + +If the :ref:`debug-extension` is enabled, a ``{% debug %}`` tag will be +available to dump the current context as well as the available filters +and tests. This is useful to see what's available to use in the template +without setting up a debugger. + +.. code-block:: html+jinja + + <pre>{% debug %}</pre> + +.. code-block:: text + + {'context': {'cycler': <class 'jinja2.utils.Cycler'>, + ..., + 'namespace': <class 'jinja2.utils.Namespace'>}, + 'filters': ['abs', 'attr', 'batch', 'capitalize', 'center', 'count', 'd', + ..., 'urlencode', 'urlize', 'wordcount', 'wordwrap', 'xmlattr'], + 'tests': ['!=', '<', '<=', '==', '>', '>=', 'callable', 'defined', + ..., 'odd', 'sameas', 'sequence', 'string', 'undefined', 'upper']} + + +With Statement +~~~~~~~~~~~~~~ + +.. versionadded:: 2.3 + +The with statement makes it possible to create a new inner scope. +Variables set within this scope are not visible outside of the scope. + +With in a nutshell:: + + {% with %} + {% set foo = 42 %} + {{ foo }} foo is 42 here + {% endwith %} + foo is not visible here any longer + +Because it is common to set variables at the beginning of the scope, +you can do that within the `with` statement. The following two examples +are equivalent:: + + {% with foo = 42 %} + {{ foo }} + {% endwith %} + + {% with %} + {% set foo = 42 %} + {{ foo }} + {% endwith %} + +An important note on scoping here. In Jinja versions before 2.9 the +behavior of referencing one variable to another had some unintended +consequences. In particular one variable could refer to another defined +in the same with block's opening statement. This caused issues with the +cleaned up scoping behavior and has since been improved. In particular +in newer Jinja versions the following code always refers to the variable +`a` from outside the `with` block:: + + {% with a={}, b=a.attribute %}...{% endwith %} + +In earlier Jinja versions the `b` attribute would refer to the results of +the first attribute. If you depend on this behavior you can rewrite it to +use the ``set`` tag:: + + {% with a={} %} + {% set b = a.attribute %} + {% endwith %} + +.. admonition:: Extension + + In older versions of Jinja (before 2.9) it was required to enable this + feature with an extension. It's now enabled by default. + +.. _autoescape-overrides: + +Autoescape Overrides +-------------------- + +.. versionadded:: 2.4 + +If you want you can activate and deactivate the autoescaping from within +the templates. + +Example:: + + {% autoescape true %} + Autoescaping is active within this block + {% endautoescape %} + + {% autoescape false %} + Autoescaping is inactive within this block + {% endautoescape %} + +After an `endautoescape` the behavior is reverted to what it was before. + +.. admonition:: Extension + + In older versions of Jinja (before 2.9) it was required to enable this + feature with an extension. It's now enabled by default. diff --git a/docs/tricks.rst b/docs/tricks.rst new file mode 100644 index 00000000..78ac4086 --- /dev/null +++ b/docs/tricks.rst @@ -0,0 +1,100 @@ +Tips and Tricks +=============== + +.. highlight:: html+jinja + +This part of the documentation shows some tips and tricks for Jinja +templates. + + +.. _null-master-fallback: + +Null-Master Fallback +-------------------- + +Jinja supports dynamic inheritance and does not distinguish between parent +and child template as long as no `extends` tag is visited. While this leads +to the surprising behavior that everything before the first `extends` tag +including whitespace is printed out instead of being ignored, it can be used +for a neat trick. + +Usually child templates extend from one template that adds a basic HTML +skeleton. However it's possible to put the `extends` tag into an `if` tag to +only extend from the layout template if the `standalone` variable evaluates +to false which it does per default if it's not defined. Additionally a very +basic skeleton is added to the file so that if it's indeed rendered with +`standalone` set to `True` a very basic HTML skeleton is added:: + + {% if not standalone %}{% extends 'master.html' %}{% endif -%} + <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> + <title>{% block title %}The Page Title{% endblock %}</title> + <link rel="stylesheet" href="style.css" type="text/css"> + {% block body %} + <p>This is the page body.</p> + {% endblock %} + + +Alternating Rows +---------------- + +If you want to have different styles for each row of a table or +list you can use the `cycle` method on the `loop` object:: + + <ul> + {% for row in rows %} + <li class="{{ loop.cycle('odd', 'even') }}">{{ row }}</li> + {% endfor %} + </ul> + +`cycle` can take an unlimited amount of strings. Each time this +tag is encountered the next item from the list is rendered. + + +Highlighting Active Menu Items +------------------------------ + +Often you want to have a navigation bar with an active navigation +item. This is really simple to achieve. Because assignments outside +of `block`\s in child templates are global and executed before the layout +template is evaluated it's possible to define the active menu item in the +child template:: + + {% extends "layout.html" %} + {% set active_page = "index" %} + +The layout template can then access `active_page`. Additionally it makes +sense to define a default for that variable:: + + {% set navigation_bar = [ + ('/', 'index', 'Index'), + ('/downloads/', 'downloads', 'Downloads'), + ('/about/', 'about', 'About') + ] -%} + {% set active_page = active_page|default('index') -%} + ... + <ul id="navigation"> + {% for href, id, caption in navigation_bar %} + <li{% if id == active_page %} class="active"{% endif + %}><a href="{{ href|e }}">{{ caption|e }}</a></li> + {% endfor %} + </ul> + ... + +.. _accessing-the-parent-loop: + +Accessing the parent Loop +------------------------- + +The special `loop` variable always points to the innermost loop. If it's +desired to have access to an outer loop it's possible to alias it:: + + <table> + {% for row in table %} + <tr> + {% set rowloop = loop %} + {% for cell in row %} + <td id="cell-{{ rowloop.index }}-{{ loop.index }}">{{ cell }}</td> + {% endfor %} + </tr> + {% endfor %} + </table> diff --git a/examples/basic/cycle.py b/examples/basic/cycle.py new file mode 100644 index 00000000..1f97e379 --- /dev/null +++ b/examples/basic/cycle.py @@ -0,0 +1,16 @@ +from jinja2 import Environment + +env = Environment( + line_statement_prefix="#", variable_start_string="${", variable_end_string="}" +) +print( + env.from_string( + """\ +<ul> +# for item in range(10) + <li class="${loop.cycle('odd', 'even')}">${item}</li> +# endfor +</ul>\ +""" + ).render() +) diff --git a/examples/basic/debugger.py b/examples/basic/debugger.py new file mode 100644 index 00000000..f6a96270 --- /dev/null +++ b/examples/basic/debugger.py @@ -0,0 +1,6 @@ +from jinja2 import Environment +from jinja2.loaders import FileSystemLoader + +env = Environment(loader=FileSystemLoader("templates")) +tmpl = env.get_template("broken.html") +print(tmpl.render(seq=[3, 2, 4, 5, 3, 2, 0, 2, 1])) diff --git a/examples/basic/inheritance.py b/examples/basic/inheritance.py new file mode 100644 index 00000000..6d928df9 --- /dev/null +++ b/examples/basic/inheritance.py @@ -0,0 +1,13 @@ +from jinja2 import Environment +from jinja2.loaders import DictLoader + +env = Environment( + loader=DictLoader( + { + "a": "[A[{% block body %}{% endblock %}]]", + "b": "{% extends 'a' %}{% block body %}[B]{% endblock %}", + "c": "{% extends 'b' %}{% block body %}###{{ super() }}###{% endblock %}", + } + ) +) +print(env.get_template("c").render()) diff --git a/examples/basic/templates/broken.html b/examples/basic/templates/broken.html new file mode 100644 index 00000000..294d5c99 --- /dev/null +++ b/examples/basic/templates/broken.html @@ -0,0 +1,6 @@ +{% from 'subbroken.html' import may_break %} +<ul> +{% for item in seq %} + <li>{{ may_break(item) }}</li> +{% endfor %} +</ul> diff --git a/examples/basic/templates/subbroken.html b/examples/basic/templates/subbroken.html new file mode 100644 index 00000000..245eb7e6 --- /dev/null +++ b/examples/basic/templates/subbroken.html @@ -0,0 +1,3 @@ +{% macro may_break(item) -%} + [{{ item / 0 }}] +{%- endmacro %} diff --git a/examples/basic/test.py b/examples/basic/test.py new file mode 100644 index 00000000..d34b0fff --- /dev/null +++ b/examples/basic/test.py @@ -0,0 +1,29 @@ +from jinja2 import Environment +from jinja2.loaders import DictLoader + +env = Environment( + loader=DictLoader( + { + "child.html": """\ +{% extends master_layout or 'master.html' %} +{% include helpers = 'helpers.html' %} +{% macro get_the_answer() %}42{% endmacro %} +{% title = 'Hello World' %} +{% block body %} + {{ get_the_answer() }} + {{ helpers.conspirate() }} +{% endblock %} +""", + "master.html": """\ +<!doctype html> +<title>{{ title }}</title> +{% block body %}{% endblock %} +""", + "helpers.html": """\ +{% macro conspirate() %}23{% endmacro %} +""", + } + ) +) +tmpl = env.get_template("child.html") +print(tmpl.render()) diff --git a/examples/basic/test_filter_and_linestatements.py b/examples/basic/test_filter_and_linestatements.py new file mode 100644 index 00000000..9bbcbcaf --- /dev/null +++ b/examples/basic/test_filter_and_linestatements.py @@ -0,0 +1,27 @@ +from jinja2 import Environment + +env = Environment( + line_statement_prefix="%", variable_start_string="${", variable_end_string="}" +) +tmpl = env.from_string( + """\ +% macro foo() + ${caller(42)} +% endmacro +<ul> +% for item in seq + <li>${item}</li> +% endfor +</ul> +% call(var) foo() + [${var}] +% endcall +% filter escape + <hello world> + % for item in [1, 2, 3] + - ${item} + % endfor +% endfilter +""" +) +print(tmpl.render(seq=range(10))) diff --git a/examples/basic/test_loop_filter.py b/examples/basic/test_loop_filter.py new file mode 100644 index 00000000..6bd89fde --- /dev/null +++ b/examples/basic/test_loop_filter.py @@ -0,0 +1,13 @@ +from jinja2 import Environment + +tmpl = Environment().from_string( + """\ +<ul> +{%- for item in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] if item % 2 == 0 %} + <li>{{ loop.index }} / {{ loop.length }}: {{ item }}</li> +{%- endfor %} +</ul> +if condition: {{ 1 if foo else 0 }} +""" +) +print(tmpl.render(foo=True)) diff --git a/examples/basic/translate.py b/examples/basic/translate.py new file mode 100644 index 00000000..e6596817 --- /dev/null +++ b/examples/basic/translate.py @@ -0,0 +1,18 @@ +from jinja2 import Environment + +env = Environment(extensions=["jinja2.ext.i18n"]) +env.globals["gettext"] = {"Hello %(user)s!": "Hallo %(user)s!"}.__getitem__ +env.globals["ngettext"] = lambda s, p, n: { + "%(count)s user": "%(count)d Benutzer", + "%(count)s users": "%(count)d Benutzer", +}[s if n == 1 else p] +print( + env.from_string( + """\ +{% trans %}Hello {{ user }}!{% endtrans %} +{% trans count=users|count -%} +{{ count }} user{% pluralize %}{{ count }} users +{% endtrans %} +""" + ).render(user="someone", users=[1, 2, 3]) +) diff --git a/requirements/dev.in b/requirements/dev.in new file mode 100644 index 00000000..bcc48da1 --- /dev/null +++ b/requirements/dev.in @@ -0,0 +1,5 @@ +# -r docs.in # can't include due to Sphinx/Jinja mutual dependency +-r tests.in +pip-tools +pre-commit +tox diff --git a/requirements/dev.txt b/requirements/dev.txt new file mode 100644 index 00000000..7c408e28 --- /dev/null +++ b/requirements/dev.txt @@ -0,0 +1,31 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile requirements/dev.in +# +appdirs==1.4.4 # via virtualenv +attrs==19.3.0 # via pytest +cfgv==3.1.0 # via pre-commit +click==7.1.2 # via pip-tools +distlib==0.3.0 # via virtualenv +filelock==3.0.12 # via tox, virtualenv +identify==1.4.15 # via pre-commit +more-itertools==8.3.0 # via pytest +nodeenv==1.3.5 # via pre-commit +packaging==20.3 # via pytest, tox +pip-tools==5.2.1 # via -r requirements/dev.in +pluggy==0.13.1 # via pytest, tox +pre-commit==2.5.1 # via -r requirements/dev.in +py==1.8.1 # via pytest, tox +pyparsing==2.4.7 # via packaging +pytest==5.4.3 # via -r requirements/tests.in +pyyaml==5.3.1 # via pre-commit +six==1.14.0 # via packaging, pip-tools, tox, virtualenv +toml==0.10.1 # via pre-commit, tox +tox==3.15.2 # via -r requirements/dev.in +virtualenv==20.0.20 # via pre-commit, tox +wcwidth==0.1.9 # via pytest + +# The following packages are considered to be unsafe in a requirements file: +# pip diff --git a/requirements/docs.in b/requirements/docs.in new file mode 100644 index 00000000..42f16511 --- /dev/null +++ b/requirements/docs.in @@ -0,0 +1,4 @@ +Pallets-Sphinx-Themes +Sphinx<3 +sphinx-issues +sphinxcontrib-log-cabinet diff --git a/requirements/docs.txt b/requirements/docs.txt new file mode 100644 index 00000000..14a5ad6d --- /dev/null +++ b/requirements/docs.txt @@ -0,0 +1,36 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile requirements/docs.in +# +alabaster==0.7.12 # via sphinx +babel==2.8.0 # via sphinx +certifi==2020.4.5.1 # via requests +chardet==3.0.4 # via requests +docutils==0.16 # via sphinx +idna==2.9 # via requests +imagesize==1.2.0 # via sphinx +jinja2==2.11.2 # via sphinx +markupsafe==1.1.1 # via jinja2 +packaging==20.3 # via pallets-sphinx-themes, sphinx +pallets-sphinx-themes==1.2.3 # via -r requirements/docs.in +pygments==2.6.1 # via sphinx +pyparsing==2.4.7 # via packaging +pytz==2020.1 # via babel +requests==2.23.0 # via sphinx +six==1.14.0 # via packaging +snowballstemmer==2.0.0 # via sphinx +sphinx-issues==1.2.0 # via -r requirements/docs.in +sphinx==2.4.4 # via -r requirements/docs.in, pallets-sphinx-themes, sphinx-issues, sphinxcontrib-log-cabinet +sphinxcontrib-applehelp==1.0.2 # via sphinx +sphinxcontrib-devhelp==1.0.2 # via sphinx +sphinxcontrib-htmlhelp==1.0.3 # via sphinx +sphinxcontrib-jsmath==1.0.1 # via sphinx +sphinxcontrib-log-cabinet==1.0.1 # via -r requirements/docs.in +sphinxcontrib-qthelp==1.0.3 # via sphinx +sphinxcontrib-serializinghtml==1.1.4 # via sphinx +urllib3==1.25.9 # via requests + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/tests.in b/requirements/tests.in new file mode 100644 index 00000000..e079f8a6 --- /dev/null +++ b/requirements/tests.in @@ -0,0 +1 @@ +pytest diff --git a/requirements/tests.txt b/requirements/tests.txt new file mode 100644 index 00000000..9113b4e3 --- /dev/null +++ b/requirements/tests.txt @@ -0,0 +1,15 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile requirements/tests.in +# +attrs==19.3.0 # via pytest +more-itertools==8.3.0 # via pytest +packaging==20.3 # via pytest +pluggy==0.13.1 # via pytest +py==1.8.1 # via pytest +pyparsing==2.4.7 # via packaging +pytest==5.4.3 # via -r requirements/tests.in +six==1.14.0 # via packaging +wcwidth==0.1.9 # via pytest diff --git a/scripts/generate_identifier_pattern.py b/scripts/generate_identifier_pattern.py new file mode 100755 index 00000000..6b479535 --- /dev/null +++ b/scripts/generate_identifier_pattern.py @@ -0,0 +1,74 @@ +import itertools +import os +import re +import sys + + +def get_characters(): + """Find every Unicode character that is valid in a Python `identifier`_ but + is not matched by the regex ``\\w`` group. + + ``\\w`` matches some characters that aren't valid in identifiers, but + :meth:`str.isidentifier` will catch that later in lexing. + + All start characters are valid continue characters, so we only test for + continue characters. + + _identifier: https://docs.python.org/3/reference/lexical_analysis.html#identifiers + """ + for cp in range(sys.maxunicode + 1): + s = chr(cp) + + if ("a" + s).isidentifier() and not re.match(r"\w", s): + yield s + + +def collapse_ranges(data): + """Given a sorted list of unique characters, generate ranges representing + sequential code points. + + Source: https://stackoverflow.com/a/4629241/400617 + """ + for _, b in itertools.groupby(enumerate(data), lambda x: ord(x[1]) - x[0]): + b = list(b) + yield b[0][1], b[-1][1] + + +def build_pattern(ranges): + """Output the regex pattern for ranges of characters. + + One and two character ranges output the individual characters. + """ + out = [] + + for a, b in ranges: + if a == b: # single char + out.append(a) + elif ord(b) - ord(a) == 1: # two chars, range is redundant + out.append(a) + out.append(b) + else: + out.append(f"{a}-{b}") + + return "".join(out) + + +def main(): + """Build the regex pattern and write it to + ``jinja2/_identifier.py``. + """ + pattern = build_pattern(collapse_ranges(get_characters())) + filename = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "src", "jinja2", "_identifier.py") + ) + + with open(filename, "w", encoding="utf8") as f: + f.write("import re\n\n") + f.write("# generated by scripts/generate_identifier_pattern.py\n") + f.write("pattern = re.compile(\n") + f.write(f' r"[\\w{pattern}]+" # noqa: B950\n') + f.write(")\n") + + +if __name__ == "__main__": + main() diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..5d3d02ee --- /dev/null +++ b/setup.cfg @@ -0,0 +1,42 @@ +[metadata] +license_file = LICENSE.rst +long_description = file:README.rst +long_description_content_type = text/x-rst + +[tool:pytest] +testpaths = tests +filterwarnings = + error + +[coverage:run] +branch = True +source = + jinja2 + tests + +[coverage:paths] +source = + src + */site-packages + +[flake8] +# B = bugbear +# E = pycodestyle errors +# F = flake8 pyflakes +# W = pycodestyle warnings +# B9 = bugbear opinions +select = B, E, F, W, B9 +ignore = + # slice notation whitespace, invalid + E203 + # line length, handled by bugbear B950 + E501 + # bare except, handled by bugbear B001 + E722 + # bin op line break, invalid + W503 +# up to 88 allowed by bugbear B950 +max-line-length = 80 +per-file-ignores = + # __init__ module exports names + src/jinja2/__init__.py: F401 diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..f5ad968e --- /dev/null +++ b/setup.py @@ -0,0 +1,40 @@ +import re + +from setuptools import find_packages +from setuptools import setup + +with open("src/jinja2/__init__.py", "rt", encoding="utf8") as f: + version = re.search(r'__version__ = "(.*?)"', f.read(), re.M).group(1) + +setup( + name="Jinja2", + version=version, + url="https://palletsprojects.com/p/jinja/", + project_urls={ + "Documentation": "https://jinja.palletsprojects.com/", + "Code": "https://github.com/pallets/jinja", + "Issue tracker": "https://github.com/pallets/jinja/issues", + }, + license="BSD-3-Clause", + maintainer="Pallets", + maintainer_email="contact@palletsprojects.com", + description="A very fast and expressive template engine.", + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Topic :: Internet :: WWW/HTTP :: Dynamic Content", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Text Processing :: Markup :: HTML", + ], + packages=find_packages("src"), + package_dir={"": "src"}, + include_package_data=True, + python_requires=">=3.6", + install_requires=["MarkupSafe>=1.1"], + extras_require={"i18n": ["Babel>=2.1"]}, + entry_points={"babel.extractors": ["jinja2 = jinja2.ext:babel_extract[i18n]"]}, +) diff --git a/src/jinja2/__init__.py b/src/jinja2/__init__.py new file mode 100644 index 00000000..8fa05183 --- /dev/null +++ b/src/jinja2/__init__.py @@ -0,0 +1,43 @@ +"""Jinja is a template engine written in pure Python. It provides a +non-XML syntax that supports inline expressions and an optional +sandboxed environment. +""" +from markupsafe import escape +from markupsafe import Markup + +from .bccache import BytecodeCache +from .bccache import FileSystemBytecodeCache +from .bccache import MemcachedBytecodeCache +from .environment import Environment +from .environment import Template +from .exceptions import TemplateAssertionError +from .exceptions import TemplateError +from .exceptions import TemplateNotFound +from .exceptions import TemplateRuntimeError +from .exceptions import TemplatesNotFound +from .exceptions import TemplateSyntaxError +from .exceptions import UndefinedError +from .filters import contextfilter +from .filters import environmentfilter +from .filters import evalcontextfilter +from .loaders import BaseLoader +from .loaders import ChoiceLoader +from .loaders import DictLoader +from .loaders import FileSystemLoader +from .loaders import FunctionLoader +from .loaders import ModuleLoader +from .loaders import PackageLoader +from .loaders import PrefixLoader +from .runtime import ChainableUndefined +from .runtime import DebugUndefined +from .runtime import make_logging_undefined +from .runtime import StrictUndefined +from .runtime import Undefined +from .utils import clear_caches +from .utils import contextfunction +from .utils import environmentfunction +from .utils import evalcontextfunction +from .utils import is_undefined +from .utils import select_autoescape + +__version__ = "3.0.0a1" diff --git a/src/jinja2/_identifier.py b/src/jinja2/_identifier.py new file mode 100644 index 00000000..224d5449 --- /dev/null +++ b/src/jinja2/_identifier.py @@ -0,0 +1,6 @@ +import re + +# generated by scripts/generate_identifier_pattern.py +pattern = re.compile( + r"[\w·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣔ-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഁ-ഃാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳸᳹᷀-᷵᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑅳𑄴𑆀-𑆂𑆳-𑇊𑇀-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯]+" # noqa: B950 +) diff --git a/src/jinja2/asyncfilters.py b/src/jinja2/asyncfilters.py new file mode 100644 index 00000000..0aad12c8 --- /dev/null +++ b/src/jinja2/asyncfilters.py @@ -0,0 +1,157 @@ +from functools import wraps + +from . import filters +from .asyncsupport import auto_aiter +from .asyncsupport import auto_await + + +async def auto_to_seq(value): + seq = [] + if hasattr(value, "__aiter__"): + async for item in value: + seq.append(item) + else: + for item in value: + seq.append(item) + return seq + + +async def async_select_or_reject(args, kwargs, modfunc, lookup_attr): + seq, func = filters.prepare_select_or_reject(args, kwargs, modfunc, lookup_attr) + if seq: + async for item in auto_aiter(seq): + if func(item): + yield item + + +def dualfilter(normal_filter, async_filter): + wrap_evalctx = False + if getattr(normal_filter, "environmentfilter", False) is True: + + def is_async(args): + return args[0].is_async + + wrap_evalctx = False + else: + has_evalctxfilter = getattr(normal_filter, "evalcontextfilter", False) is True + has_ctxfilter = getattr(normal_filter, "contextfilter", False) is True + wrap_evalctx = not has_evalctxfilter and not has_ctxfilter + + def is_async(args): + return args[0].environment.is_async + + @wraps(normal_filter) + def wrapper(*args, **kwargs): + b = is_async(args) + if wrap_evalctx: + args = args[1:] + if b: + return async_filter(*args, **kwargs) + return normal_filter(*args, **kwargs) + + if wrap_evalctx: + wrapper.evalcontextfilter = True + + wrapper.asyncfiltervariant = True + + return wrapper + + +def asyncfiltervariant(original): + def decorator(f): + return dualfilter(original, f) + + return decorator + + +@asyncfiltervariant(filters.do_first) +async def do_first(environment, seq): + try: + return await auto_aiter(seq).__anext__() + except StopAsyncIteration: + return environment.undefined("No first item, sequence was empty.") + + +@asyncfiltervariant(filters.do_groupby) +async def do_groupby(environment, value, attribute): + expr = filters.make_attrgetter(environment, attribute) + return [ + filters._GroupTuple(key, await auto_to_seq(values)) + for key, values in filters.groupby( + sorted(await auto_to_seq(value), key=expr), expr + ) + ] + + +@asyncfiltervariant(filters.do_join) +async def do_join(eval_ctx, value, d="", attribute=None): + return filters.do_join(eval_ctx, await auto_to_seq(value), d, attribute) + + +@asyncfiltervariant(filters.do_list) +async def do_list(value): + return await auto_to_seq(value) + + +@asyncfiltervariant(filters.do_reject) +async def do_reject(*args, **kwargs): + return async_select_or_reject(args, kwargs, lambda x: not x, False) + + +@asyncfiltervariant(filters.do_rejectattr) +async def do_rejectattr(*args, **kwargs): + return async_select_or_reject(args, kwargs, lambda x: not x, True) + + +@asyncfiltervariant(filters.do_select) +async def do_select(*args, **kwargs): + return async_select_or_reject(args, kwargs, lambda x: x, False) + + +@asyncfiltervariant(filters.do_selectattr) +async def do_selectattr(*args, **kwargs): + return async_select_or_reject(args, kwargs, lambda x: x, True) + + +@asyncfiltervariant(filters.do_map) +async def do_map(*args, **kwargs): + seq, func = filters.prepare_map(args, kwargs) + if seq: + async for item in auto_aiter(seq): + yield await auto_await(func(item)) + + +@asyncfiltervariant(filters.do_sum) +async def do_sum(environment, iterable, attribute=None, start=0): + rv = start + if attribute is not None: + func = filters.make_attrgetter(environment, attribute) + else: + + def func(x): + return x + + async for item in auto_aiter(iterable): + rv += func(item) + return rv + + +@asyncfiltervariant(filters.do_slice) +async def do_slice(value, slices, fill_with=None): + return filters.do_slice(await auto_to_seq(value), slices, fill_with) + + +ASYNC_FILTERS = { + "first": do_first, + "groupby": do_groupby, + "join": do_join, + "list": do_list, + # we intentionally do not support do_last because it may not be safe in async + "reject": do_reject, + "rejectattr": do_rejectattr, + "map": do_map, + "select": do_select, + "selectattr": do_selectattr, + "sum": do_sum, + "slice": do_slice, +} diff --git a/src/jinja2/asyncsupport.py b/src/jinja2/asyncsupport.py new file mode 100644 index 00000000..e46a85a3 --- /dev/null +++ b/src/jinja2/asyncsupport.py @@ -0,0 +1,249 @@ +"""The code for async support. Importing this patches Jinja.""" +import asyncio +import inspect +from functools import update_wrapper + +from markupsafe import Markup + +from .environment import TemplateModule +from .runtime import LoopContext +from .utils import concat +from .utils import internalcode +from .utils import missing + + +async def concat_async(async_gen): + rv = [] + + async def collect(): + async for event in async_gen: + rv.append(event) + + await collect() + return concat(rv) + + +async def generate_async(self, *args, **kwargs): + vars = dict(*args, **kwargs) + try: + async for event in self.root_render_func(self.new_context(vars)): + yield event + except Exception: + yield self.environment.handle_exception() + + +def wrap_generate_func(original_generate): + def _convert_generator(self, loop, args, kwargs): + async_gen = self.generate_async(*args, **kwargs) + try: + while 1: + yield loop.run_until_complete(async_gen.__anext__()) + except StopAsyncIteration: + pass + + def generate(self, *args, **kwargs): + if not self.environment.is_async: + return original_generate(self, *args, **kwargs) + return _convert_generator(self, asyncio.get_event_loop(), args, kwargs) + + return update_wrapper(generate, original_generate) + + +async def render_async(self, *args, **kwargs): + if not self.environment.is_async: + raise RuntimeError("The environment was not created with async mode enabled.") + + vars = dict(*args, **kwargs) + ctx = self.new_context(vars) + + try: + return await concat_async(self.root_render_func(ctx)) + except Exception: + return self.environment.handle_exception() + + +def wrap_render_func(original_render): + def render(self, *args, **kwargs): + if not self.environment.is_async: + return original_render(self, *args, **kwargs) + loop = asyncio.get_event_loop() + return loop.run_until_complete(self.render_async(*args, **kwargs)) + + return update_wrapper(render, original_render) + + +def wrap_block_reference_call(original_call): + @internalcode + async def async_call(self): + rv = await concat_async(self._stack[self._depth](self._context)) + if self._context.eval_ctx.autoescape: + rv = Markup(rv) + return rv + + @internalcode + def __call__(self): + if not self._context.environment.is_async: + return original_call(self) + return async_call(self) + + return update_wrapper(__call__, original_call) + + +def wrap_macro_invoke(original_invoke): + @internalcode + async def async_invoke(self, arguments, autoescape): + rv = await self._func(*arguments) + if autoescape: + rv = Markup(rv) + return rv + + @internalcode + def _invoke(self, arguments, autoescape): + if not self._environment.is_async: + return original_invoke(self, arguments, autoescape) + return async_invoke(self, arguments, autoescape) + + return update_wrapper(_invoke, original_invoke) + + +@internalcode +async def get_default_module_async(self): + if self._module is not None: + return self._module + self._module = rv = await self.make_module_async() + return rv + + +def wrap_default_module(original_default_module): + @internalcode + def _get_default_module(self, ctx=None): + if self.environment.is_async: + raise RuntimeError("Template module attribute is unavailable in async mode") + return original_default_module(self, ctx) + + return _get_default_module + + +async def make_module_async(self, vars=None, shared=False, locals=None): + context = self.new_context(vars, shared, locals) + body_stream = [] + async for item in self.root_render_func(context): + body_stream.append(item) + return TemplateModule(self, context, body_stream) + + +def patch_template(): + from . import Template + + Template.generate = wrap_generate_func(Template.generate) + Template.generate_async = update_wrapper(generate_async, Template.generate_async) + Template.render_async = update_wrapper(render_async, Template.render_async) + Template.render = wrap_render_func(Template.render) + Template._get_default_module = wrap_default_module(Template._get_default_module) + Template._get_default_module_async = get_default_module_async + Template.make_module_async = update_wrapper( + make_module_async, Template.make_module_async + ) + + +def patch_runtime(): + from .runtime import BlockReference, Macro + + BlockReference.__call__ = wrap_block_reference_call(BlockReference.__call__) + Macro._invoke = wrap_macro_invoke(Macro._invoke) + + +def patch_filters(): + from .filters import FILTERS + from .asyncfilters import ASYNC_FILTERS + + FILTERS.update(ASYNC_FILTERS) + + +def patch_all(): + patch_template() + patch_runtime() + patch_filters() + + +async def auto_await(value): + if inspect.isawaitable(value): + return await value + return value + + +async def auto_aiter(iterable): + if hasattr(iterable, "__aiter__"): + async for item in iterable: + yield item + return + for item in iterable: + yield item + + +class AsyncLoopContext(LoopContext): + _to_iterator = staticmethod(auto_aiter) + + @property + async def length(self): + if self._length is not None: + return self._length + + try: + self._length = len(self._iterable) + except TypeError: + iterable = [x async for x in self._iterator] + self._iterator = self._to_iterator(iterable) + self._length = len(iterable) + self.index + (self._after is not missing) + + return self._length + + @property + async def revindex0(self): + return await self.length - self.index + + @property + async def revindex(self): + return await self.length - self.index0 + + async def _peek_next(self): + if self._after is not missing: + return self._after + + try: + self._after = await self._iterator.__anext__() + except StopAsyncIteration: + self._after = missing + + return self._after + + @property + async def last(self): + return await self._peek_next() is missing + + @property + async def nextitem(self): + rv = await self._peek_next() + + if rv is missing: + return self._undefined("there is no next item") + + return rv + + def __aiter__(self): + return self + + async def __anext__(self): + if self._after is not missing: + rv = self._after + self._after = missing + else: + rv = await self._iterator.__anext__() + + self.index0 += 1 + self._before = self._current + self._current = rv + return rv, self + + +patch_all() diff --git a/src/jinja2/bccache.py b/src/jinja2/bccache.py new file mode 100644 index 00000000..7ddcf405 --- /dev/null +++ b/src/jinja2/bccache.py @@ -0,0 +1,345 @@ +"""The optional bytecode cache system. This is useful if you have very +complex template situations and the compilation of all those templates +slows down your application too much. + +Situations where this is useful are often forking web applications that +are initialized on the first request. +""" +import errno +import fnmatch +import marshal +import os +import pickle +import stat +import sys +import tempfile +from hashlib import sha1 +from io import BytesIO + +from .utils import open_if_exists + +bc_version = 5 +# Magic bytes to identify Jinja bytecode cache files. Contains the +# Python major and minor version to avoid loading incompatible bytecode +# if a project upgrades its Python version. +bc_magic = ( + b"j2" + + pickle.dumps(bc_version, 2) + + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1], 2) +) + + +class Bucket: + """Buckets are used to store the bytecode for one template. It's created + and initialized by the bytecode cache and passed to the loading functions. + + The buckets get an internal checksum from the cache assigned and use this + to automatically reject outdated cache material. Individual bytecode + cache subclasses don't have to care about cache invalidation. + """ + + def __init__(self, environment, key, checksum): + self.environment = environment + self.key = key + self.checksum = checksum + self.reset() + + def reset(self): + """Resets the bucket (unloads the bytecode).""" + self.code = None + + def load_bytecode(self, f): + """Loads bytecode from a file or file like object.""" + # make sure the magic header is correct + magic = f.read(len(bc_magic)) + if magic != bc_magic: + self.reset() + return + # the source code of the file changed, we need to reload + checksum = pickle.load(f) + if self.checksum != checksum: + self.reset() + return + # if marshal_load fails then we need to reload + try: + self.code = marshal.load(f) + except (EOFError, ValueError, TypeError): + self.reset() + return + + def write_bytecode(self, f): + """Dump the bytecode into the file or file like object passed.""" + if self.code is None: + raise TypeError("can't write empty bucket") + f.write(bc_magic) + pickle.dump(self.checksum, f, 2) + marshal.dump(self.code, f) + + def bytecode_from_string(self, string): + """Load bytecode from a string.""" + self.load_bytecode(BytesIO(string)) + + def bytecode_to_string(self): + """Return the bytecode as string.""" + out = BytesIO() + self.write_bytecode(out) + return out.getvalue() + + +class BytecodeCache: + """To implement your own bytecode cache you have to subclass this class + and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of + these methods are passed a :class:`~jinja2.bccache.Bucket`. + + A very basic bytecode cache that saves the bytecode on the file system:: + + from os import path + + class MyCache(BytecodeCache): + + def __init__(self, directory): + self.directory = directory + + def load_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + if path.exists(filename): + with open(filename, 'rb') as f: + bucket.load_bytecode(f) + + def dump_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + with open(filename, 'wb') as f: + bucket.write_bytecode(f) + + A more advanced version of a filesystem based bytecode cache is part of + Jinja. + """ + + def load_bytecode(self, bucket): + """Subclasses have to override this method to load bytecode into a + bucket. If they are not able to find code in the cache for the + bucket, it must not do anything. + """ + raise NotImplementedError() + + def dump_bytecode(self, bucket): + """Subclasses have to override this method to write the bytecode + from a bucket back to the cache. If it unable to do so it must not + fail silently but raise an exception. + """ + raise NotImplementedError() + + def clear(self): + """Clears the cache. This method is not used by Jinja but should be + implemented to allow applications to clear the bytecode cache used + by a particular environment. + """ + + def get_cache_key(self, name, filename=None): + """Returns the unique hash key for this template name.""" + hash = sha1(name.encode("utf-8")) + if filename is not None: + filename = "|" + filename + if isinstance(filename, str): + filename = filename.encode("utf-8") + hash.update(filename) + return hash.hexdigest() + + def get_source_checksum(self, source): + """Returns a checksum for the source.""" + return sha1(source.encode("utf-8")).hexdigest() + + def get_bucket(self, environment, name, filename, source): + """Return a cache bucket for the given template. All arguments are + mandatory but filename may be `None`. + """ + key = self.get_cache_key(name, filename) + checksum = self.get_source_checksum(source) + bucket = Bucket(environment, key, checksum) + self.load_bytecode(bucket) + return bucket + + def set_bucket(self, bucket): + """Put the bucket into the cache.""" + self.dump_bytecode(bucket) + + +class FileSystemBytecodeCache(BytecodeCache): + """A bytecode cache that stores bytecode on the filesystem. It accepts + two arguments: The directory where the cache items are stored and a + pattern string that is used to build the filename. + + If no directory is specified a default cache directory is selected. On + Windows the user's temp directory is used, on UNIX systems a directory + is created for the user in the system temp directory. + + The pattern can be used to have multiple separate caches operate on the + same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s`` + is replaced with the cache key. + + >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache') + + This bytecode cache supports clearing of the cache using the clear method. + """ + + def __init__(self, directory=None, pattern="__jinja2_%s.cache"): + if directory is None: + directory = self._get_default_cache_dir() + self.directory = directory + self.pattern = pattern + + def _get_default_cache_dir(self): + def _unsafe_dir(): + raise RuntimeError( + "Cannot determine safe temp directory. You " + "need to explicitly provide one." + ) + + tmpdir = tempfile.gettempdir() + + # On windows the temporary directory is used specific unless + # explicitly forced otherwise. We can just use that. + if os.name == "nt": + return tmpdir + if not hasattr(os, "getuid"): + _unsafe_dir() + + dirname = f"_jinja2-cache-{os.getuid()}" + actual_dir = os.path.join(tmpdir, dirname) + + try: + os.mkdir(actual_dir, stat.S_IRWXU) + except OSError as e: + if e.errno != errno.EEXIST: + raise + try: + os.chmod(actual_dir, stat.S_IRWXU) + actual_dir_stat = os.lstat(actual_dir) + if ( + actual_dir_stat.st_uid != os.getuid() + or not stat.S_ISDIR(actual_dir_stat.st_mode) + or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU + ): + _unsafe_dir() + except OSError as e: + if e.errno != errno.EEXIST: + raise + + actual_dir_stat = os.lstat(actual_dir) + if ( + actual_dir_stat.st_uid != os.getuid() + or not stat.S_ISDIR(actual_dir_stat.st_mode) + or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU + ): + _unsafe_dir() + + return actual_dir + + def _get_cache_filename(self, bucket): + return os.path.join(self.directory, self.pattern % (bucket.key,)) + + def load_bytecode(self, bucket): + f = open_if_exists(self._get_cache_filename(bucket), "rb") + if f is not None: + try: + bucket.load_bytecode(f) + finally: + f.close() + + def dump_bytecode(self, bucket): + f = open(self._get_cache_filename(bucket), "wb") + try: + bucket.write_bytecode(f) + finally: + f.close() + + def clear(self): + # imported lazily here because google app-engine doesn't support + # write access on the file system and the function does not exist + # normally. + from os import remove + + files = fnmatch.filter(os.listdir(self.directory), self.pattern % ("*",)) + for filename in files: + try: + remove(os.path.join(self.directory, filename)) + except OSError: + pass + + +class MemcachedBytecodeCache(BytecodeCache): + """This class implements a bytecode cache that uses a memcache cache for + storing the information. It does not enforce a specific memcache library + (tummy's memcache or cmemcache) but will accept any class that provides + the minimal interface required. + + Libraries compatible with this class: + + - `cachelib <https://github.com/pallets/cachelib>`_ + - `python-memcached <https://pypi.org/project/python-memcached/>`_ + + (Unfortunately the django cache interface is not compatible because it + does not support storing binary data, only text. You can however pass + the underlying cache client to the bytecode cache which is available + as `django.core.cache.cache._client`.) + + The minimal interface for the client passed to the constructor is this: + + .. class:: MinimalClientInterface + + .. method:: set(key, value[, timeout]) + + Stores the bytecode in the cache. `value` is a string and + `timeout` the timeout of the key. If timeout is not provided + a default timeout or no timeout should be assumed, if it's + provided it's an integer with the number of seconds the cache + item should exist. + + .. method:: get(key) + + Returns the value for the cache key. If the item does not + exist in the cache the return value must be `None`. + + The other arguments to the constructor are the prefix for all keys that + is added before the actual cache key and the timeout for the bytecode in + the cache system. We recommend a high (or no) timeout. + + This bytecode cache does not support clearing of used items in the cache. + The clear method is a no-operation function. + + .. versionadded:: 2.7 + Added support for ignoring memcache errors through the + `ignore_memcache_errors` parameter. + """ + + def __init__( + self, + client, + prefix="jinja2/bytecode/", + timeout=None, + ignore_memcache_errors=True, + ): + self.client = client + self.prefix = prefix + self.timeout = timeout + self.ignore_memcache_errors = ignore_memcache_errors + + def load_bytecode(self, bucket): + try: + code = self.client.get(self.prefix + bucket.key) + except Exception: + if not self.ignore_memcache_errors: + raise + code = None + if code is not None: + bucket.bytecode_from_string(code) + + def dump_bytecode(self, bucket): + args = (self.prefix + bucket.key, bucket.bytecode_to_string()) + if self.timeout is not None: + args += (self.timeout,) + try: + self.client.set(*args) + except Exception: + if not self.ignore_memcache_errors: + raise diff --git a/src/jinja2/compiler.py b/src/jinja2/compiler.py new file mode 100644 index 00000000..abdbe6da --- /dev/null +++ b/src/jinja2/compiler.py @@ -0,0 +1,1754 @@ +"""Compiles nodes from the parser into Python code.""" +from collections import namedtuple +from functools import update_wrapper +from io import StringIO +from itertools import chain +from keyword import iskeyword as is_python_keyword + +from markupsafe import escape +from markupsafe import Markup + +from . import nodes +from .exceptions import TemplateAssertionError +from .idtracking import Symbols +from .idtracking import VAR_LOAD_ALIAS +from .idtracking import VAR_LOAD_PARAMETER +from .idtracking import VAR_LOAD_RESOLVE +from .idtracking import VAR_LOAD_UNDEFINED +from .nodes import EvalContext +from .optimizer import Optimizer +from .utils import concat +from .visitor import NodeVisitor + +operators = { + "eq": "==", + "ne": "!=", + "gt": ">", + "gteq": ">=", + "lt": "<", + "lteq": "<=", + "in": "in", + "notin": "not in", +} + + +def optimizeconst(f): + def new_func(self, node, frame, **kwargs): + # Only optimize if the frame is not volatile + if self.optimized and not frame.eval_ctx.volatile: + new_node = self.optimizer.visit(node, frame.eval_ctx) + if new_node != node: + return self.visit(new_node, frame) + return f(self, node, frame, **kwargs) + + return update_wrapper(new_func, f) + + +def generate( + node, environment, name, filename, stream=None, defer_init=False, optimized=True +): + """Generate the python source for a node tree.""" + if not isinstance(node, nodes.Template): + raise TypeError("Can't compile non template nodes") + generator = environment.code_generator_class( + environment, name, filename, stream, defer_init, optimized + ) + generator.visit(node) + if stream is None: + return generator.stream.getvalue() + + +def has_safe_repr(value): + """Does the node have a safe representation?""" + if value is None or value is NotImplemented or value is Ellipsis: + return True + + if type(value) in {bool, int, float, complex, range, str, Markup}: + return True + + if type(value) in {tuple, list, set, frozenset}: + return all(has_safe_repr(v) for v in value) + + if type(value) is dict: + return all(has_safe_repr(k) and has_safe_repr(v) for k, v in value.items()) + + return False + + +def find_undeclared(nodes, names): + """Check if the names passed are accessed undeclared. The return value + is a set of all the undeclared names from the sequence of names found. + """ + visitor = UndeclaredNameVisitor(names) + try: + for node in nodes: + visitor.visit(node) + except VisitorExit: + pass + return visitor.undeclared + + +class MacroRef: + def __init__(self, node): + self.node = node + self.accesses_caller = False + self.accesses_kwargs = False + self.accesses_varargs = False + + +class Frame: + """Holds compile time information for us.""" + + def __init__(self, eval_ctx, parent=None, level=None): + self.eval_ctx = eval_ctx + self.symbols = Symbols(parent.symbols if parent else None, level=level) + + # a toplevel frame is the root + soft frames such as if conditions. + self.toplevel = False + + # the root frame is basically just the outermost frame, so no if + # conditions. This information is used to optimize inheritance + # situations. + self.rootlevel = False + + # in some dynamic inheritance situations the compiler needs to add + # write tests around output statements. + self.require_output_check = parent and parent.require_output_check + + # inside some tags we are using a buffer rather than yield statements. + # this for example affects {% filter %} or {% macro %}. If a frame + # is buffered this variable points to the name of the list used as + # buffer. + self.buffer = None + + # the name of the block we're in, otherwise None. + self.block = parent.block if parent else None + + # the parent of this frame + self.parent = parent + + if parent is not None: + self.buffer = parent.buffer + + def copy(self): + """Create a copy of the current one.""" + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.symbols = self.symbols.copy() + return rv + + def inner(self, isolated=False): + """Return an inner frame.""" + if isolated: + return Frame(self.eval_ctx, level=self.symbols.level + 1) + return Frame(self.eval_ctx, self) + + def soft(self): + """Return a soft frame. A soft frame may not be modified as + standalone thing as it shares the resources with the frame it + was created of, but it's not a rootlevel frame any longer. + + This is only used to implement if-statements. + """ + rv = self.copy() + rv.rootlevel = False + return rv + + __copy__ = copy + + +class VisitorExit(RuntimeError): + """Exception used by the `UndeclaredNameVisitor` to signal a stop.""" + + +class DependencyFinderVisitor(NodeVisitor): + """A visitor that collects filter and test calls.""" + + def __init__(self): + self.filters = set() + self.tests = set() + + def visit_Filter(self, node): + self.generic_visit(node) + self.filters.add(node.name) + + def visit_Test(self, node): + self.generic_visit(node) + self.tests.add(node.name) + + def visit_Block(self, node): + """Stop visiting at blocks.""" + + +class UndeclaredNameVisitor(NodeVisitor): + """A visitor that checks if a name is accessed without being + declared. This is different from the frame visitor as it will + not stop at closure frames. + """ + + def __init__(self, names): + self.names = set(names) + self.undeclared = set() + + def visit_Name(self, node): + if node.ctx == "load" and node.name in self.names: + self.undeclared.add(node.name) + if self.undeclared == self.names: + raise VisitorExit() + else: + self.names.discard(node.name) + + def visit_Block(self, node): + """Stop visiting a blocks.""" + + +class CompilerExit(Exception): + """Raised if the compiler encountered a situation where it just + doesn't make sense to further process the code. Any block that + raises such an exception is not further processed. + """ + + +class CodeGenerator(NodeVisitor): + def __init__( + self, environment, name, filename, stream=None, defer_init=False, optimized=True + ): + if stream is None: + stream = StringIO() + self.environment = environment + self.name = name + self.filename = filename + self.stream = stream + self.created_block_context = False + self.defer_init = defer_init + self.optimized = optimized + if optimized: + self.optimizer = Optimizer(environment) + + # aliases for imports + self.import_aliases = {} + + # a registry for all blocks. Because blocks are moved out + # into the global python scope they are registered here + self.blocks = {} + + # the number of extends statements so far + self.extends_so_far = 0 + + # some templates have a rootlevel extends. In this case we + # can safely assume that we're a child template and do some + # more optimizations. + self.has_known_extends = False + + # the current line number + self.code_lineno = 1 + + # registry of all filters and tests (global, not block local) + self.tests = {} + self.filters = {} + + # the debug information + self.debug_info = [] + self._write_debug_info = None + + # the number of new lines before the next write() + self._new_lines = 0 + + # the line number of the last written statement + self._last_line = 0 + + # true if nothing was written so far. + self._first_write = True + + # used by the `temporary_identifier` method to get new + # unique, temporary identifier + self._last_identifier = 0 + + # the current indentation + self._indentation = 0 + + # Tracks toplevel assignments + self._assign_stack = [] + + # Tracks parameter definition blocks + self._param_def_block = [] + + # Tracks the current context. + self._context_reference_stack = ["context"] + + # -- Various compilation helpers + + def fail(self, msg, lineno): + """Fail with a :exc:`TemplateAssertionError`.""" + raise TemplateAssertionError(msg, lineno, self.name, self.filename) + + def temporary_identifier(self): + """Get a new unique identifier.""" + self._last_identifier += 1 + return f"t_{self._last_identifier}" + + def buffer(self, frame): + """Enable buffering for the frame from that point onwards.""" + frame.buffer = self.temporary_identifier() + self.writeline(f"{frame.buffer} = []") + + def return_buffer_contents(self, frame, force_unescaped=False): + """Return the buffer contents of the frame.""" + if not force_unescaped: + if frame.eval_ctx.volatile: + self.writeline("if context.eval_ctx.autoescape:") + self.indent() + self.writeline(f"return Markup(concat({frame.buffer}))") + self.outdent() + self.writeline("else:") + self.indent() + self.writeline(f"return concat({frame.buffer})") + self.outdent() + return + elif frame.eval_ctx.autoescape: + self.writeline(f"return Markup(concat({frame.buffer}))") + return + self.writeline(f"return concat({frame.buffer})") + + def indent(self): + """Indent by one.""" + self._indentation += 1 + + def outdent(self, step=1): + """Outdent by step.""" + self._indentation -= step + + def start_write(self, frame, node=None): + """Yield or write into the frame buffer.""" + if frame.buffer is None: + self.writeline("yield ", node) + else: + self.writeline(f"{frame.buffer}.append(", node) + + def end_write(self, frame): + """End the writing process started by `start_write`.""" + if frame.buffer is not None: + self.write(")") + + def simple_write(self, s, frame, node=None): + """Simple shortcut for start_write + write + end_write.""" + self.start_write(frame, node) + self.write(s) + self.end_write(frame) + + def blockvisit(self, nodes, frame): + """Visit a list of nodes as block in a frame. If the current frame + is no buffer a dummy ``if 0: yield None`` is written automatically. + """ + try: + self.writeline("pass") + for node in nodes: + self.visit(node, frame) + except CompilerExit: + pass + + def write(self, x): + """Write a string into the output stream.""" + if self._new_lines: + if not self._first_write: + self.stream.write("\n" * self._new_lines) + self.code_lineno += self._new_lines + if self._write_debug_info is not None: + self.debug_info.append((self._write_debug_info, self.code_lineno)) + self._write_debug_info = None + self._first_write = False + self.stream.write(" " * self._indentation) + self._new_lines = 0 + self.stream.write(x) + + def writeline(self, x, node=None, extra=0): + """Combination of newline and write.""" + self.newline(node, extra) + self.write(x) + + def newline(self, node=None, extra=0): + """Add one or more newlines before the next write.""" + self._new_lines = max(self._new_lines, 1 + extra) + if node is not None and node.lineno != self._last_line: + self._write_debug_info = node.lineno + self._last_line = node.lineno + + def signature(self, node, frame, extra_kwargs=None): + """Writes a function call to the stream for the current node. + A leading comma is added automatically. The extra keyword + arguments may not include python keywords otherwise a syntax + error could occur. The extra keyword arguments should be given + as python dict. + """ + # if any of the given keyword arguments is a python keyword + # we have to make sure that no invalid call is created. + kwarg_workaround = False + for kwarg in chain((x.key for x in node.kwargs), extra_kwargs or ()): + if is_python_keyword(kwarg): + kwarg_workaround = True + break + + for arg in node.args: + self.write(", ") + self.visit(arg, frame) + + if not kwarg_workaround: + for kwarg in node.kwargs: + self.write(", ") + self.visit(kwarg, frame) + if extra_kwargs is not None: + for key, value in extra_kwargs.items(): + self.write(f", {key}={value}") + if node.dyn_args: + self.write(", *") + self.visit(node.dyn_args, frame) + + if kwarg_workaround: + if node.dyn_kwargs is not None: + self.write(", **dict({") + else: + self.write(", **{") + for kwarg in node.kwargs: + self.write(f"{kwarg.key!r}: ") + self.visit(kwarg.value, frame) + self.write(", ") + if extra_kwargs is not None: + for key, value in extra_kwargs.items(): + self.write(f"{key!r}: {value}, ") + if node.dyn_kwargs is not None: + self.write("}, **") + self.visit(node.dyn_kwargs, frame) + self.write(")") + else: + self.write("}") + + elif node.dyn_kwargs is not None: + self.write(", **") + self.visit(node.dyn_kwargs, frame) + + def pull_dependencies(self, nodes): + """Pull all the dependencies.""" + visitor = DependencyFinderVisitor() + for node in nodes: + visitor.visit(node) + for dependency in "filters", "tests": + mapping = getattr(self, dependency) + for name in getattr(visitor, dependency): + if name not in mapping: + mapping[name] = self.temporary_identifier() + self.writeline(f"{mapping[name]} = environment.{dependency}[{name!r}]") + + def enter_frame(self, frame): + undefs = [] + for target, (action, param) in frame.symbols.loads.items(): + if action == VAR_LOAD_PARAMETER: + pass + elif action == VAR_LOAD_RESOLVE: + self.writeline(f"{target} = {self.get_resolve_func()}({param!r})") + elif action == VAR_LOAD_ALIAS: + self.writeline(f"{target} = {param}") + elif action == VAR_LOAD_UNDEFINED: + undefs.append(target) + else: + raise NotImplementedError("unknown load instruction") + if undefs: + self.writeline(f"{' = '.join(undefs)} = missing") + + def leave_frame(self, frame, with_python_scope=False): + if not with_python_scope: + undefs = [] + for target in frame.symbols.loads: + undefs.append(target) + if undefs: + self.writeline(f"{' = '.join(undefs)} = missing") + + def func(self, name): + if self.environment.is_async: + return f"async def {name}" + return f"def {name}" + + def macro_body(self, node, frame): + """Dump the function def of a macro or call block.""" + frame = frame.inner() + frame.symbols.analyze_node(node) + macro_ref = MacroRef(node) + + explicit_caller = None + skip_special_params = set() + args = [] + for idx, arg in enumerate(node.args): + if arg.name == "caller": + explicit_caller = idx + if arg.name in ("kwargs", "varargs"): + skip_special_params.add(arg.name) + args.append(frame.symbols.ref(arg.name)) + + undeclared = find_undeclared(node.body, ("caller", "kwargs", "varargs")) + + if "caller" in undeclared: + # In older Jinja versions there was a bug that allowed caller + # to retain the special behavior even if it was mentioned in + # the argument list. However thankfully this was only really + # working if it was the last argument. So we are explicitly + # checking this now and error out if it is anywhere else in + # the argument list. + if explicit_caller is not None: + try: + node.defaults[explicit_caller - len(node.args)] + except IndexError: + self.fail( + "When defining macros or call blocks the " + 'special "caller" argument must be omitted ' + "or be given a default.", + node.lineno, + ) + else: + args.append(frame.symbols.declare_parameter("caller")) + macro_ref.accesses_caller = True + if "kwargs" in undeclared and "kwargs" not in skip_special_params: + args.append(frame.symbols.declare_parameter("kwargs")) + macro_ref.accesses_kwargs = True + if "varargs" in undeclared and "varargs" not in skip_special_params: + args.append(frame.symbols.declare_parameter("varargs")) + macro_ref.accesses_varargs = True + + # macros are delayed, they never require output checks + frame.require_output_check = False + frame.symbols.analyze_node(node) + self.writeline(f"{self.func('macro')}({', '.join(args)}):", node) + self.indent() + + self.buffer(frame) + self.enter_frame(frame) + + self.push_parameter_definitions(frame) + for idx, arg in enumerate(node.args): + ref = frame.symbols.ref(arg.name) + self.writeline(f"if {ref} is missing:") + self.indent() + try: + default = node.defaults[idx - len(node.args)] + except IndexError: + self.writeline( + f'{ref} = undefined("parameter {arg.name!r} was not provided",' + f" name={arg.name!r})" + ) + else: + self.writeline(f"{ref} = ") + self.visit(default, frame) + self.mark_parameter_stored(ref) + self.outdent() + self.pop_parameter_definitions() + + self.blockvisit(node.body, frame) + self.return_buffer_contents(frame, force_unescaped=True) + self.leave_frame(frame, with_python_scope=True) + self.outdent() + + return frame, macro_ref + + def macro_def(self, macro_ref, frame): + """Dump the macro definition for the def created by macro_body.""" + arg_tuple = ", ".join(repr(x.name) for x in macro_ref.node.args) + name = getattr(macro_ref.node, "name", None) + if len(macro_ref.node.args) == 1: + arg_tuple += "," + self.write( + f"Macro(environment, macro, {name!r}, ({arg_tuple})," + f" {macro_ref.accesses_kwargs!r}, {macro_ref.accesses_varargs!r}," + f" {macro_ref.accesses_caller!r}, context.eval_ctx.autoescape)" + ) + + def position(self, node): + """Return a human readable position for the node.""" + rv = f"line {node.lineno}" + if self.name is not None: + rv = f"{rv} in {self.name!r}" + return rv + + def dump_local_context(self, frame): + items_kv = ", ".join( + f"{name!r}: {target}" + for name, target in frame.symbols.dump_stores().items() + ) + return f"{{{items_kv}}}" + + def write_commons(self): + """Writes a common preamble that is used by root and block functions. + Primarily this sets up common local helpers and enforces a generator + through a dead branch. + """ + self.writeline("resolve = context.resolve_or_missing") + self.writeline("undefined = environment.undefined") + # always use the standard Undefined class for the implicit else of + # conditional expressions + self.writeline("cond_expr_undefined = Undefined") + self.writeline("if 0: yield None") + + def push_parameter_definitions(self, frame): + """Pushes all parameter targets from the given frame into a local + stack that permits tracking of yet to be assigned parameters. In + particular this enables the optimization from `visit_Name` to skip + undefined expressions for parameters in macros as macros can reference + otherwise unbound parameters. + """ + self._param_def_block.append(frame.symbols.dump_param_targets()) + + def pop_parameter_definitions(self): + """Pops the current parameter definitions set.""" + self._param_def_block.pop() + + def mark_parameter_stored(self, target): + """Marks a parameter in the current parameter definitions as stored. + This will skip the enforced undefined checks. + """ + if self._param_def_block: + self._param_def_block[-1].discard(target) + + def push_context_reference(self, target): + self._context_reference_stack.append(target) + + def pop_context_reference(self): + self._context_reference_stack.pop() + + def get_context_ref(self): + return self._context_reference_stack[-1] + + def get_resolve_func(self): + target = self._context_reference_stack[-1] + if target == "context": + return "resolve" + return f"{target}.resolve" + + def derive_context(self, frame): + return f"{self.get_context_ref()}.derived({self.dump_local_context(frame)})" + + def parameter_is_undeclared(self, target): + """Checks if a given target is an undeclared parameter.""" + if not self._param_def_block: + return False + return target in self._param_def_block[-1] + + def push_assign_tracking(self): + """Pushes a new layer for assignment tracking.""" + self._assign_stack.append(set()) + + def pop_assign_tracking(self, frame): + """Pops the topmost level for assignment tracking and updates the + context variables if necessary. + """ + vars = self._assign_stack.pop() + if not frame.toplevel or not vars: + return + public_names = [x for x in vars if x[:1] != "_"] + if len(vars) == 1: + name = next(iter(vars)) + ref = frame.symbols.ref(name) + self.writeline(f"context.vars[{name!r}] = {ref}") + else: + self.writeline("context.vars.update({") + for idx, name in enumerate(vars): + if idx: + self.write(", ") + ref = frame.symbols.ref(name) + self.write(f"{name!r}: {ref}") + self.write("})") + if public_names: + if len(public_names) == 1: + self.writeline(f"context.exported_vars.add({public_names[0]!r})") + else: + names_str = ", ".join(map(repr, public_names)) + self.writeline(f"context.exported_vars.update(({names_str}))") + + # -- Statement Visitors + + def visit_Template(self, node, frame=None): + assert frame is None, "no root frame allowed" + eval_ctx = EvalContext(self.environment, self.name) + + from .runtime import exported + + self.writeline("from __future__ import generator_stop") # Python < 3.7 + self.writeline("from jinja2.runtime import " + ", ".join(exported)) + + if self.environment.is_async: + self.writeline( + "from jinja2.asyncsupport import auto_await, " + "auto_aiter, AsyncLoopContext" + ) + + # if we want a deferred initialization we cannot move the + # environment into a local name + envenv = "" if self.defer_init else ", environment=environment" + + # do we have an extends tag at all? If not, we can save some + # overhead by just not processing any inheritance code. + have_extends = node.find(nodes.Extends) is not None + + # find all blocks + for block in node.find_all(nodes.Block): + if block.name in self.blocks: + self.fail(f"block {block.name!r} defined twice", block.lineno) + self.blocks[block.name] = block + + # find all imports and import them + for import_ in node.find_all(nodes.ImportedName): + if import_.importname not in self.import_aliases: + imp = import_.importname + self.import_aliases[imp] = alias = self.temporary_identifier() + if "." in imp: + module, obj = imp.rsplit(".", 1) + self.writeline(f"from {module} import {obj} as {alias}") + else: + self.writeline(f"import {imp} as {alias}") + + # add the load name + self.writeline(f"name = {self.name!r}") + + # generate the root render function. + self.writeline( + f"{self.func('root')}(context, missing=missing{envenv}):", extra=1 + ) + self.indent() + self.write_commons() + + # process the root + frame = Frame(eval_ctx) + if "self" in find_undeclared(node.body, ("self",)): + ref = frame.symbols.declare_parameter("self") + self.writeline(f"{ref} = TemplateReference(context)") + frame.symbols.analyze_node(node) + frame.toplevel = frame.rootlevel = True + frame.require_output_check = have_extends and not self.has_known_extends + if have_extends: + self.writeline("parent_template = None") + self.enter_frame(frame) + self.pull_dependencies(node.body) + self.blockvisit(node.body, frame) + self.leave_frame(frame, with_python_scope=True) + self.outdent() + + # make sure that the parent root is called. + if have_extends: + if not self.has_known_extends: + self.indent() + self.writeline("if parent_template is not None:") + self.indent() + if not self.environment.is_async: + self.writeline("yield from parent_template.root_render_func(context)") + else: + loop = "async for" if self.environment.is_async else "for" + self.writeline( + f"{loop} event in parent_template.root_render_func(context):" + ) + self.indent() + self.writeline("yield event") + self.outdent() + self.outdent(1 + (not self.has_known_extends)) + + # at this point we now have the blocks collected and can visit them too. + for name, block in self.blocks.items(): + self.writeline( + f"{self.func('block_' + name)}(context, missing=missing{envenv}):", + block, + 1, + ) + self.indent() + self.write_commons() + # It's important that we do not make this frame a child of the + # toplevel template. This would cause a variety of + # interesting issues with identifier tracking. + block_frame = Frame(eval_ctx) + undeclared = find_undeclared(block.body, ("self", "super")) + if "self" in undeclared: + ref = block_frame.symbols.declare_parameter("self") + self.writeline(f"{ref} = TemplateReference(context)") + if "super" in undeclared: + ref = block_frame.symbols.declare_parameter("super") + self.writeline(f"{ref} = context.super({name!r}, block_{name})") + block_frame.symbols.analyze_node(block) + block_frame.block = name + self.enter_frame(block_frame) + self.pull_dependencies(block.body) + self.blockvisit(block.body, block_frame) + self.leave_frame(block_frame, with_python_scope=True) + self.outdent() + + blocks_kv_str = ", ".join(f"{x!r}: block_{x}" for x in self.blocks) + self.writeline(f"blocks = {{{blocks_kv_str}}}", extra=1) + debug_kv_str = "&".join(f"{k}={v}" for k, v in self.debug_info) + self.writeline(f"debug_info = {debug_kv_str!r}") + + def visit_Block(self, node, frame): + """Call a block and register it for the template.""" + level = 0 + if frame.toplevel: + # if we know that we are a child template, there is no need to + # check if we are one + if self.has_known_extends: + return + if self.extends_so_far > 0: + self.writeline("if parent_template is None:") + self.indent() + level += 1 + + if node.scoped: + context = self.derive_context(frame) + else: + context = self.get_context_ref() + + if not self.environment.is_async and frame.buffer is None: + self.writeline( + f"yield from context.blocks[{node.name!r}][0]({context})", node + ) + else: + loop = "async for" if self.environment.is_async else "for" + self.writeline( + f"{loop} event in context.blocks[{node.name!r}][0]({context}):", node + ) + self.indent() + self.simple_write("event", frame) + self.outdent() + + self.outdent(level) + + def visit_Extends(self, node, frame): + """Calls the extender.""" + if not frame.toplevel: + self.fail("cannot use extend from a non top-level scope", node.lineno) + + # if the number of extends statements in general is zero so + # far, we don't have to add a check if something extended + # the template before this one. + if self.extends_so_far > 0: + + # if we have a known extends we just add a template runtime + # error into the generated code. We could catch that at compile + # time too, but i welcome it not to confuse users by throwing the + # same error at different times just "because we can". + if not self.has_known_extends: + self.writeline("if parent_template is not None:") + self.indent() + self.writeline('raise TemplateRuntimeError("extended multiple times")') + + # if we have a known extends already we don't need that code here + # as we know that the template execution will end here. + if self.has_known_extends: + raise CompilerExit() + else: + self.outdent() + + self.writeline("parent_template = environment.get_template(", node) + self.visit(node.template, frame) + self.write(f", {self.name!r})") + self.writeline("for name, parent_block in parent_template.blocks.items():") + self.indent() + self.writeline("context.blocks.setdefault(name, []).append(parent_block)") + self.outdent() + + # if this extends statement was in the root level we can take + # advantage of that information and simplify the generated code + # in the top level from this point onwards + if frame.rootlevel: + self.has_known_extends = True + + # and now we have one more + self.extends_so_far += 1 + + def visit_Include(self, node, frame): + """Handles includes.""" + if node.ignore_missing: + self.writeline("try:") + self.indent() + + func_name = "get_or_select_template" + if isinstance(node.template, nodes.Const): + if isinstance(node.template.value, str): + func_name = "get_template" + elif isinstance(node.template.value, (tuple, list)): + func_name = "select_template" + elif isinstance(node.template, (nodes.Tuple, nodes.List)): + func_name = "select_template" + + self.writeline(f"template = environment.{func_name}(", node) + self.visit(node.template, frame) + self.write(f", {self.name!r})") + if node.ignore_missing: + self.outdent() + self.writeline("except TemplateNotFound:") + self.indent() + self.writeline("pass") + self.outdent() + self.writeline("else:") + self.indent() + + skip_event_yield = False + if node.with_context: + loop = "async for" if self.environment.is_async else "for" + self.writeline( + f"{loop} event in template.root_render_func(" + "template.new_context(context.get_all(), True," + f" {self.dump_local_context(frame)})):" + ) + elif self.environment.is_async: + self.writeline( + "for event in (await template._get_default_module_async())" + "._body_stream:" + ) + else: + self.writeline("yield from template._get_default_module()._body_stream") + skip_event_yield = True + + if not skip_event_yield: + self.indent() + self.simple_write("event", frame) + self.outdent() + + if node.ignore_missing: + self.outdent() + + def visit_Import(self, node, frame): + """Visit regular imports.""" + self.writeline(f"{frame.symbols.ref(node.target)} = ", node) + if frame.toplevel: + self.write(f"context.vars[{node.target!r}] = ") + if self.environment.is_async: + self.write("await ") + self.write("environment.get_template(") + self.visit(node.template, frame) + self.write(f", {self.name!r}).") + if node.with_context: + func = "make_module" + ("_async" if self.environment.is_async else "") + self.write( + f"{func}(context.get_all(), True, {self.dump_local_context(frame)})" + ) + elif self.environment.is_async: + self.write("_get_default_module_async()") + else: + self.write("_get_default_module(context)") + if frame.toplevel and not node.target.startswith("_"): + self.writeline(f"context.exported_vars.discard({node.target!r})") + + def visit_FromImport(self, node, frame): + """Visit named imports.""" + self.newline(node) + prefix = "await " if self.environment.is_async else "" + self.write(f"included_template = {prefix}environment.get_template(") + self.visit(node.template, frame) + self.write(f", {self.name!r}).") + if node.with_context: + func = "make_module" + ("_async" if self.environment.is_async else "") + self.write( + f"{func}(context.get_all(), True, {self.dump_local_context(frame)})" + ) + elif self.environment.is_async: + self.write("_get_default_module_async()") + else: + self.write("_get_default_module(context)") + + var_names = [] + discarded_names = [] + for name in node.names: + if isinstance(name, tuple): + name, alias = name + else: + alias = name + self.writeline( + f"{frame.symbols.ref(alias)} =" + f" getattr(included_template, {name!r}, missing)" + ) + self.writeline(f"if {frame.symbols.ref(alias)} is missing:") + self.indent() + message = ( + "the template {included_template.__name__!r}" + f" (imported on {self.position(node)})" + f" does not export the requested name {name!r}" + ) + self.writeline( + f"{frame.symbols.ref(alias)} = undefined(f{message!r}, name={name!r})" + ) + self.outdent() + if frame.toplevel: + var_names.append(alias) + if not alias.startswith("_"): + discarded_names.append(alias) + + if var_names: + if len(var_names) == 1: + name = var_names[0] + self.writeline(f"context.vars[{name!r}] = {frame.symbols.ref(name)}") + else: + names_kv = ", ".join( + f"{name!r}: {frame.symbols.ref(name)}" for name in var_names + ) + self.writeline(f"context.vars.update({{{names_kv}}})") + if discarded_names: + if len(discarded_names) == 1: + self.writeline(f"context.exported_vars.discard({discarded_names[0]!r})") + else: + names_str = ", ".join(map(repr, discarded_names)) + self.writeline( + f"context.exported_vars.difference_update(({names_str}))" + ) + + def visit_For(self, node, frame): + loop_frame = frame.inner() + test_frame = frame.inner() + else_frame = frame.inner() + + # try to figure out if we have an extended loop. An extended loop + # is necessary if the loop is in recursive mode if the special loop + # variable is accessed in the body. + extended_loop = node.recursive or "loop" in find_undeclared( + node.iter_child_nodes(only=("body",)), ("loop",) + ) + + loop_ref = None + if extended_loop: + loop_ref = loop_frame.symbols.declare_parameter("loop") + + loop_frame.symbols.analyze_node(node, for_branch="body") + if node.else_: + else_frame.symbols.analyze_node(node, for_branch="else") + + if node.test: + loop_filter_func = self.temporary_identifier() + test_frame.symbols.analyze_node(node, for_branch="test") + self.writeline(f"{self.func(loop_filter_func)}(fiter):", node.test) + self.indent() + self.enter_frame(test_frame) + self.writeline("async for " if self.environment.is_async else "for ") + self.visit(node.target, loop_frame) + self.write(" in ") + self.write("auto_aiter(fiter)" if self.environment.is_async else "fiter") + self.write(":") + self.indent() + self.writeline("if ", node.test) + self.visit(node.test, test_frame) + self.write(":") + self.indent() + self.writeline("yield ") + self.visit(node.target, loop_frame) + self.outdent(3) + self.leave_frame(test_frame, with_python_scope=True) + + # if we don't have an recursive loop we have to find the shadowed + # variables at that point. Because loops can be nested but the loop + # variable is a special one we have to enforce aliasing for it. + if node.recursive: + self.writeline( + f"{self.func('loop')}(reciter, loop_render_func, depth=0):", node + ) + self.indent() + self.buffer(loop_frame) + + # Use the same buffer for the else frame + else_frame.buffer = loop_frame.buffer + + # make sure the loop variable is a special one and raise a template + # assertion error if a loop tries to write to loop + if extended_loop: + self.writeline(f"{loop_ref} = missing") + + for name in node.find_all(nodes.Name): + if name.ctx == "store" and name.name == "loop": + self.fail( + "Can't assign to special loop variable in for-loop target", + name.lineno, + ) + + if node.else_: + iteration_indicator = self.temporary_identifier() + self.writeline(f"{iteration_indicator} = 1") + + self.writeline("async for " if self.environment.is_async else "for ", node) + self.visit(node.target, loop_frame) + if extended_loop: + prefix = "Async" if self.environment.is_async else "" + self.write(f", {loop_ref} in {prefix}LoopContext(") + else: + self.write(" in ") + + if node.test: + self.write(f"{loop_filter_func}(") + if node.recursive: + self.write("reciter") + else: + if self.environment.is_async and not extended_loop: + self.write("auto_aiter(") + self.visit(node.iter, frame) + if self.environment.is_async and not extended_loop: + self.write(")") + if node.test: + self.write(")") + + if node.recursive: + self.write(", undefined, loop_render_func, depth):") + else: + self.write(", undefined):" if extended_loop else ":") + + self.indent() + self.enter_frame(loop_frame) + + self.blockvisit(node.body, loop_frame) + if node.else_: + self.writeline(f"{iteration_indicator} = 0") + self.outdent() + self.leave_frame( + loop_frame, with_python_scope=node.recursive and not node.else_ + ) + + if node.else_: + self.writeline(f"if {iteration_indicator}:") + self.indent() + self.enter_frame(else_frame) + self.blockvisit(node.else_, else_frame) + self.leave_frame(else_frame) + self.outdent() + + # if the node was recursive we have to return the buffer contents + # and start the iteration code + if node.recursive: + self.return_buffer_contents(loop_frame) + self.outdent() + self.start_write(frame, node) + if self.environment.is_async: + self.write("await ") + self.write("loop(") + if self.environment.is_async: + self.write("auto_aiter(") + self.visit(node.iter, frame) + if self.environment.is_async: + self.write(")") + self.write(", loop)") + self.end_write(frame) + + def visit_If(self, node, frame): + if_frame = frame.soft() + self.writeline("if ", node) + self.visit(node.test, if_frame) + self.write(":") + self.indent() + self.blockvisit(node.body, if_frame) + self.outdent() + for elif_ in node.elif_: + self.writeline("elif ", elif_) + self.visit(elif_.test, if_frame) + self.write(":") + self.indent() + self.blockvisit(elif_.body, if_frame) + self.outdent() + if node.else_: + self.writeline("else:") + self.indent() + self.blockvisit(node.else_, if_frame) + self.outdent() + + def visit_Macro(self, node, frame): + macro_frame, macro_ref = self.macro_body(node, frame) + self.newline() + if frame.toplevel: + if not node.name.startswith("_"): + self.write(f"context.exported_vars.add({node.name!r})") + self.writeline(f"context.vars[{node.name!r}] = ") + self.write(f"{frame.symbols.ref(node.name)} = ") + self.macro_def(macro_ref, macro_frame) + + def visit_CallBlock(self, node, frame): + call_frame, macro_ref = self.macro_body(node, frame) + self.writeline("caller = ") + self.macro_def(macro_ref, call_frame) + self.start_write(frame, node) + self.visit_Call(node.call, frame, forward_caller=True) + self.end_write(frame) + + def visit_FilterBlock(self, node, frame): + filter_frame = frame.inner() + filter_frame.symbols.analyze_node(node) + self.enter_frame(filter_frame) + self.buffer(filter_frame) + self.blockvisit(node.body, filter_frame) + self.start_write(frame, node) + self.visit_Filter(node.filter, filter_frame) + self.end_write(frame) + self.leave_frame(filter_frame) + + def visit_With(self, node, frame): + with_frame = frame.inner() + with_frame.symbols.analyze_node(node) + self.enter_frame(with_frame) + for target, expr in zip(node.targets, node.values): + self.newline() + self.visit(target, with_frame) + self.write(" = ") + self.visit(expr, frame) + self.blockvisit(node.body, with_frame) + self.leave_frame(with_frame) + + def visit_ExprStmt(self, node, frame): + self.newline(node) + self.visit(node.node, frame) + + _FinalizeInfo = namedtuple("_FinalizeInfo", ("const", "src")) + #: The default finalize function if the environment isn't configured + #: with one. Or if the environment has one, this is called on that + #: function's output for constants. + _default_finalize = str + _finalize = None + + def _make_finalize(self): + """Build the finalize function to be used on constants and at + runtime. Cached so it's only created once for all output nodes. + + Returns a ``namedtuple`` with the following attributes: + + ``const`` + A function to finalize constant data at compile time. + + ``src`` + Source code to output around nodes to be evaluated at + runtime. + """ + if self._finalize is not None: + return self._finalize + + finalize = default = self._default_finalize + src = None + + if self.environment.finalize: + src = "environment.finalize(" + env_finalize = self.environment.finalize + + def finalize(value): + return default(env_finalize(value)) + + if getattr(env_finalize, "contextfunction", False) is True: + src += "context, " + finalize = None # noqa: F811 + elif getattr(env_finalize, "evalcontextfunction", False) is True: + src += "context.eval_ctx, " + finalize = None + elif getattr(env_finalize, "environmentfunction", False) is True: + src += "environment, " + + def finalize(value): + return default(env_finalize(self.environment, value)) + + self._finalize = self._FinalizeInfo(finalize, src) + return self._finalize + + def _output_const_repr(self, group): + """Given a group of constant values converted from ``Output`` + child nodes, produce a string to write to the template module + source. + """ + return repr(concat(group)) + + def _output_child_to_const(self, node, frame, finalize): + """Try to optimize a child of an ``Output`` node by trying to + convert it to constant, finalized data at compile time. + + If :exc:`Impossible` is raised, the node is not constant and + will be evaluated at runtime. Any other exception will also be + evaluated at runtime for easier debugging. + """ + const = node.as_const(frame.eval_ctx) + + if frame.eval_ctx.autoescape: + const = escape(const) + + # Template data doesn't go through finalize. + if isinstance(node, nodes.TemplateData): + return str(const) + + return finalize.const(const) + + def _output_child_pre(self, node, frame, finalize): + """Output extra source code before visiting a child of an + ``Output`` node. + """ + if frame.eval_ctx.volatile: + self.write("(escape if context.eval_ctx.autoescape else str)(") + elif frame.eval_ctx.autoescape: + self.write("escape(") + else: + self.write("str(") + + if finalize.src is not None: + self.write(finalize.src) + + def _output_child_post(self, node, frame, finalize): + """Output extra source code after visiting a child of an + ``Output`` node. + """ + self.write(")") + + if finalize.src is not None: + self.write(")") + + def visit_Output(self, node, frame): + # If an extends is active, don't render outside a block. + if frame.require_output_check: + # A top-level extends is known to exist at compile time. + if self.has_known_extends: + return + + self.writeline("if parent_template is None:") + self.indent() + + finalize = self._make_finalize() + body = [] + + # Evaluate constants at compile time if possible. Each item in + # body will be either a list of static data or a node to be + # evaluated at runtime. + for child in node.nodes: + try: + if not ( + # If the finalize function requires runtime context, + # constants can't be evaluated at compile time. + finalize.const + # Unless it's basic template data that won't be + # finalized anyway. + or isinstance(child, nodes.TemplateData) + ): + raise nodes.Impossible() + + const = self._output_child_to_const(child, frame, finalize) + except (nodes.Impossible, Exception): + # The node was not constant and needs to be evaluated at + # runtime. Or another error was raised, which is easier + # to debug at runtime. + body.append(child) + continue + + if body and isinstance(body[-1], list): + body[-1].append(const) + else: + body.append([const]) + + if frame.buffer is not None: + if len(body) == 1: + self.writeline(f"{frame.buffer}.append(") + else: + self.writeline(f"{frame.buffer}.extend((") + + self.indent() + + for item in body: + if isinstance(item, list): + # A group of constant data to join and output. + val = self._output_const_repr(item) + + if frame.buffer is None: + self.writeline("yield " + val) + else: + self.writeline(val + ",") + else: + if frame.buffer is None: + self.writeline("yield ", item) + else: + self.newline(item) + + # A node to be evaluated at runtime. + self._output_child_pre(item, frame, finalize) + self.visit(item, frame) + self._output_child_post(item, frame, finalize) + + if frame.buffer is not None: + self.write(",") + + if frame.buffer is not None: + self.outdent() + self.writeline(")" if len(body) == 1 else "))") + + if frame.require_output_check: + self.outdent() + + def visit_Assign(self, node, frame): + self.push_assign_tracking() + self.newline(node) + self.visit(node.target, frame) + self.write(" = ") + self.visit(node.node, frame) + self.pop_assign_tracking(frame) + + def visit_AssignBlock(self, node, frame): + self.push_assign_tracking() + block_frame = frame.inner() + # This is a special case. Since a set block always captures we + # will disable output checks. This way one can use set blocks + # toplevel even in extended templates. + block_frame.require_output_check = False + block_frame.symbols.analyze_node(node) + self.enter_frame(block_frame) + self.buffer(block_frame) + self.blockvisit(node.body, block_frame) + self.newline(node) + self.visit(node.target, frame) + self.write(" = (Markup if context.eval_ctx.autoescape else identity)(") + if node.filter is not None: + self.visit_Filter(node.filter, block_frame) + else: + self.write(f"concat({block_frame.buffer})") + self.write(")") + self.pop_assign_tracking(frame) + self.leave_frame(block_frame) + + # -- Expression Visitors + + def visit_Name(self, node, frame): + if node.ctx == "store" and frame.toplevel: + if self._assign_stack: + self._assign_stack[-1].add(node.name) + ref = frame.symbols.ref(node.name) + + # If we are looking up a variable we might have to deal with the + # case where it's undefined. We can skip that case if the load + # instruction indicates a parameter which are always defined. + if node.ctx == "load": + load = frame.symbols.find_load(ref) + if not ( + load is not None + and load[0] == VAR_LOAD_PARAMETER + and not self.parameter_is_undeclared(ref) + ): + self.write( + f"(undefined(name={node.name!r}) if {ref} is missing else {ref})" + ) + return + + self.write(ref) + + def visit_NSRef(self, node, frame): + # NSRefs can only be used to store values; since they use the normal + # `foo.bar` notation they will be parsed as a normal attribute access + # when used anywhere but in a `set` context + ref = frame.symbols.ref(node.name) + self.writeline(f"if not isinstance({ref}, Namespace):") + self.indent() + self.writeline( + "raise TemplateRuntimeError" + '("cannot assign attribute on non-namespace object")' + ) + self.outdent() + self.writeline(f"{ref}[{node.attr!r}]") + + def visit_Const(self, node, frame): + val = node.as_const(frame.eval_ctx) + if isinstance(val, float): + self.write(str(val)) + else: + self.write(repr(val)) + + def visit_TemplateData(self, node, frame): + try: + self.write(repr(node.as_const(frame.eval_ctx))) + except nodes.Impossible: + self.write( + f"(Markup if context.eval_ctx.autoescape else identity)({node.data!r})" + ) + + def visit_Tuple(self, node, frame): + self.write("(") + idx = -1 + for idx, item in enumerate(node.items): + if idx: + self.write(", ") + self.visit(item, frame) + self.write(",)" if idx == 0 else ")") + + def visit_List(self, node, frame): + self.write("[") + for idx, item in enumerate(node.items): + if idx: + self.write(", ") + self.visit(item, frame) + self.write("]") + + def visit_Dict(self, node, frame): + self.write("{") + for idx, item in enumerate(node.items): + if idx: + self.write(", ") + self.visit(item.key, frame) + self.write(": ") + self.visit(item.value, frame) + self.write("}") + + def binop(operator, interceptable=True): # noqa: B902 + @optimizeconst + def visitor(self, node, frame): + if ( + self.environment.sandboxed + and operator in self.environment.intercepted_binops + ): + self.write(f"environment.call_binop(context, {operator!r}, ") + self.visit(node.left, frame) + self.write(", ") + self.visit(node.right, frame) + else: + self.write("(") + self.visit(node.left, frame) + self.write(f" {operator} ") + self.visit(node.right, frame) + self.write(")") + + return visitor + + def uaop(operator, interceptable=True): # noqa: B902 + @optimizeconst + def visitor(self, node, frame): + if ( + self.environment.sandboxed + and operator in self.environment.intercepted_unops + ): + self.write(f"environment.call_unop(context, {operator!r}, ") + self.visit(node.node, frame) + else: + self.write("(" + operator) + self.visit(node.node, frame) + self.write(")") + + return visitor + + visit_Add = binop("+") + visit_Sub = binop("-") + visit_Mul = binop("*") + visit_Div = binop("/") + visit_FloorDiv = binop("//") + visit_Pow = binop("**") + visit_Mod = binop("%") + visit_And = binop("and", interceptable=False) + visit_Or = binop("or", interceptable=False) + visit_Pos = uaop("+") + visit_Neg = uaop("-") + visit_Not = uaop("not ", interceptable=False) + del binop, uaop + + @optimizeconst + def visit_Concat(self, node, frame): + if frame.eval_ctx.volatile: + func_name = "(markup_join if context.eval_ctx.volatile else str_join)" + elif frame.eval_ctx.autoescape: + func_name = "markup_join" + else: + func_name = "str_join" + self.write(f"{func_name}((") + for arg in node.nodes: + self.visit(arg, frame) + self.write(", ") + self.write("))") + + @optimizeconst + def visit_Compare(self, node, frame): + self.write("(") + self.visit(node.expr, frame) + for op in node.ops: + self.visit(op, frame) + self.write(")") + + def visit_Operand(self, node, frame): + self.write(f" {operators[node.op]} ") + self.visit(node.expr, frame) + + @optimizeconst + def visit_Getattr(self, node, frame): + if self.environment.is_async: + self.write("(await auto_await(") + + self.write("environment.getattr(") + self.visit(node.node, frame) + self.write(f", {node.attr!r})") + + if self.environment.is_async: + self.write("))") + + @optimizeconst + def visit_Getitem(self, node, frame): + # slices bypass the environment getitem method. + if isinstance(node.arg, nodes.Slice): + self.visit(node.node, frame) + self.write("[") + self.visit(node.arg, frame) + self.write("]") + else: + if self.environment.is_async: + self.write("(await auto_await(") + + self.write("environment.getitem(") + self.visit(node.node, frame) + self.write(", ") + self.visit(node.arg, frame) + self.write(")") + + if self.environment.is_async: + self.write("))") + + def visit_Slice(self, node, frame): + if node.start is not None: + self.visit(node.start, frame) + self.write(":") + if node.stop is not None: + self.visit(node.stop, frame) + if node.step is not None: + self.write(":") + self.visit(node.step, frame) + + @optimizeconst + def visit_Filter(self, node, frame): + if self.environment.is_async: + self.write("await auto_await(") + self.write(self.filters[node.name] + "(") + func = self.environment.filters.get(node.name) + if func is None: + self.fail(f"no filter named {node.name!r}", node.lineno) + if getattr(func, "contextfilter", False) is True: + self.write("context, ") + elif getattr(func, "evalcontextfilter", False) is True: + self.write("context.eval_ctx, ") + elif getattr(func, "environmentfilter", False) is True: + self.write("environment, ") + + # if the filter node is None we are inside a filter block + # and want to write to the current buffer + if node.node is not None: + self.visit(node.node, frame) + elif frame.eval_ctx.volatile: + self.write( + f"(Markup(concat({frame.buffer}))" + f" if context.eval_ctx.autoescape else concat({frame.buffer}))" + ) + elif frame.eval_ctx.autoescape: + self.write(f"Markup(concat({frame.buffer}))") + else: + self.write(f"concat({frame.buffer})") + self.signature(node, frame) + self.write(")") + if self.environment.is_async: + self.write(")") + + @optimizeconst + def visit_Test(self, node, frame): + self.write(self.tests[node.name] + "(") + if node.name not in self.environment.tests: + self.fail(f"no test named {node.name!r}", node.lineno) + self.visit(node.node, frame) + self.signature(node, frame) + self.write(")") + + @optimizeconst + def visit_CondExpr(self, node, frame): + def write_expr2(): + if node.expr2 is not None: + return self.visit(node.expr2, frame) + self.write( + f'cond_expr_undefined("the inline if-expression on' + f" {self.position(node)} evaluated to false and no else" + f' section was defined.")' + ) + + self.write("(") + self.visit(node.expr1, frame) + self.write(" if ") + self.visit(node.test, frame) + self.write(" else ") + write_expr2() + self.write(")") + + @optimizeconst + def visit_Call(self, node, frame, forward_caller=False): + if self.environment.is_async: + self.write("await auto_await(") + if self.environment.sandboxed: + self.write("environment.call(context, ") + else: + self.write("context.call(") + self.visit(node.node, frame) + extra_kwargs = {"caller": "caller"} if forward_caller else None + self.signature(node, frame, extra_kwargs) + self.write(")") + if self.environment.is_async: + self.write(")") + + def visit_Keyword(self, node, frame): + self.write(node.key + "=") + self.visit(node.value, frame) + + # -- Unused nodes for extensions + + def visit_MarkSafe(self, node, frame): + self.write("Markup(") + self.visit(node.expr, frame) + self.write(")") + + def visit_MarkSafeIfAutoescape(self, node, frame): + self.write("(Markup if context.eval_ctx.autoescape else identity)(") + self.visit(node.expr, frame) + self.write(")") + + def visit_EnvironmentAttribute(self, node, frame): + self.write("environment." + node.name) + + def visit_ExtensionAttribute(self, node, frame): + self.write(f"environment.extensions[{node.identifier!r}].{node.name}") + + def visit_ImportedName(self, node, frame): + self.write(self.import_aliases[node.importname]) + + def visit_InternalName(self, node, frame): + self.write(node.name) + + def visit_ContextReference(self, node, frame): + self.write("context") + + def visit_DerivedContextReference(self, node, frame): + self.write(self.derive_context(frame)) + + def visit_Continue(self, node, frame): + self.writeline("continue", node) + + def visit_Break(self, node, frame): + self.writeline("break", node) + + def visit_Scope(self, node, frame): + scope_frame = frame.inner() + scope_frame.symbols.analyze_node(node) + self.enter_frame(scope_frame) + self.blockvisit(node.body, scope_frame) + self.leave_frame(scope_frame) + + def visit_OverlayScope(self, node, frame): + ctx = self.temporary_identifier() + self.writeline(f"{ctx} = {self.derive_context(frame)}") + self.writeline(f"{ctx}.vars = ") + self.visit(node.context, frame) + self.push_context_reference(ctx) + + scope_frame = frame.inner(isolated=True) + scope_frame.symbols.analyze_node(node) + self.enter_frame(scope_frame) + self.blockvisit(node.body, scope_frame) + self.leave_frame(scope_frame) + self.pop_context_reference() + + def visit_EvalContextModifier(self, node, frame): + for keyword in node.options: + self.writeline(f"context.eval_ctx.{keyword.key} = ") + self.visit(keyword.value, frame) + try: + val = keyword.value.as_const(frame.eval_ctx) + except nodes.Impossible: + frame.eval_ctx.volatile = True + else: + setattr(frame.eval_ctx, keyword.key, val) + + def visit_ScopedEvalContextModifier(self, node, frame): + old_ctx_name = self.temporary_identifier() + saved_ctx = frame.eval_ctx.save() + self.writeline(f"{old_ctx_name} = context.eval_ctx.save()") + self.visit_EvalContextModifier(node, frame) + for child in node.body: + self.visit(child, frame) + frame.eval_ctx.revert(saved_ctx) + self.writeline(f"context.eval_ctx.revert({old_ctx_name})") diff --git a/src/jinja2/constants.py b/src/jinja2/constants.py new file mode 100644 index 00000000..41a1c23b --- /dev/null +++ b/src/jinja2/constants.py @@ -0,0 +1,20 @@ +#: list of lorem ipsum words used by the lipsum() helper function +LOREM_IPSUM_WORDS = """\ +a ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at +auctor augue bibendum blandit class commodo condimentum congue consectetuer +consequat conubia convallis cras cubilia cum curabitur curae cursus dapibus +diam dictum dictumst dignissim dis dolor donec dui duis egestas eget eleifend +elementum elit enim erat eros est et etiam eu euismod facilisi facilisis fames +faucibus felis fermentum feugiat fringilla fusce gravida habitant habitasse hac +hendrerit hymenaeos iaculis id imperdiet in inceptos integer interdum ipsum +justo lacinia lacus laoreet lectus leo libero ligula litora lobortis lorem +luctus maecenas magna magnis malesuada massa mattis mauris metus mi molestie +mollis montes morbi mus nam nascetur natoque nec neque netus nibh nisi nisl non +nonummy nostra nulla nullam nunc odio orci ornare parturient pede pellentesque +penatibus per pharetra phasellus placerat platea porta porttitor posuere +potenti praesent pretium primis proin pulvinar purus quam quis quisque rhoncus +ridiculus risus rutrum sagittis sapien scelerisque sed sem semper senectus sit +sociis sociosqu sodales sollicitudin suscipit suspendisse taciti tellus tempor +tempus tincidunt torquent tortor tristique turpis ullamcorper ultrices +ultricies urna ut varius vehicula vel velit venenatis vestibulum vitae vivamus +viverra volutpat vulputate""" diff --git a/src/jinja2/debug.py b/src/jinja2/debug.py new file mode 100644 index 00000000..5cac28ba --- /dev/null +++ b/src/jinja2/debug.py @@ -0,0 +1,261 @@ +import platform +import sys +from types import CodeType + +from . import TemplateSyntaxError +from .utils import internal_code +from .utils import missing + + +def rewrite_traceback_stack(source=None): + """Rewrite the current exception to replace any tracebacks from + within compiled template code with tracebacks that look like they + came from the template source. + + This must be called within an ``except`` block. + + :param source: For ``TemplateSyntaxError``, the original source if + known. + :return: The original exception with the rewritten traceback. + """ + _, exc_value, tb = sys.exc_info() + + if isinstance(exc_value, TemplateSyntaxError) and not exc_value.translated: + exc_value.translated = True + exc_value.source = source + # Remove the old traceback, otherwise the frames from the + # compiler still show up. + exc_value.with_traceback(None) + # Outside of runtime, so the frame isn't executing template + # code, but it still needs to point at the template. + tb = fake_traceback( + exc_value, None, exc_value.filename or "<unknown>", exc_value.lineno + ) + else: + # Skip the frame for the render function. + tb = tb.tb_next + + stack = [] + + # Build the stack of traceback object, replacing any in template + # code with the source file and line information. + while tb is not None: + # Skip frames decorated with @internalcode. These are internal + # calls that aren't useful in template debugging output. + if tb.tb_frame.f_code in internal_code: + tb = tb.tb_next + continue + + template = tb.tb_frame.f_globals.get("__jinja_template__") + + if template is not None: + lineno = template.get_corresponding_lineno(tb.tb_lineno) + fake_tb = fake_traceback(exc_value, tb, template.filename, lineno) + stack.append(fake_tb) + else: + stack.append(tb) + + tb = tb.tb_next + + tb_next = None + + # Assign tb_next in reverse to avoid circular references. + for tb in reversed(stack): + tb_next = tb_set_next(tb, tb_next) + + return exc_value.with_traceback(tb_next) + + +def fake_traceback(exc_value, tb, filename, lineno): + """Produce a new traceback object that looks like it came from the + template source instead of the compiled code. The filename, line + number, and location name will point to the template, and the local + variables will be the current template context. + + :param exc_value: The original exception to be re-raised to create + the new traceback. + :param tb: The original traceback to get the local variables and + code info from. + :param filename: The template filename. + :param lineno: The line number in the template source. + """ + if tb is not None: + # Replace the real locals with the context that would be + # available at that point in the template. + locals = get_template_locals(tb.tb_frame.f_locals) + locals.pop("__jinja_exception__", None) + else: + locals = {} + + globals = { + "__name__": filename, + "__file__": filename, + "__jinja_exception__": exc_value, + } + # Raise an exception at the correct line number. + code = compile("\n" * (lineno - 1) + "raise __jinja_exception__", filename, "exec") + + # Build a new code object that points to the template file and + # replaces the location with a block name. + try: + location = "template" + + if tb is not None: + function = tb.tb_frame.f_code.co_name + + if function == "root": + location = "top-level template code" + elif function.startswith("block_"): + location = f"block {function[6:]!r}" + + # Collect arguments for the new code object. CodeType only + # accepts positional arguments, and arguments were inserted in + # new Python versions. + code_args = [] + + for attr in ( + "argcount", + "posonlyargcount", # Python 3.8 + "kwonlyargcount", + "nlocals", + "stacksize", + "flags", + "code", # codestring + "consts", # constants + "names", + "varnames", + ("filename", filename), + ("name", location), + "firstlineno", + "lnotab", + "freevars", + "cellvars", + ): + if isinstance(attr, tuple): + # Replace with given value. + code_args.append(attr[1]) + continue + + try: + # Copy original value if it exists. + code_args.append(getattr(code, "co_" + attr)) + except AttributeError: + # Some arguments were added later. + continue + + code = CodeType(*code_args) + except Exception: + # Some environments such as Google App Engine don't support + # modifying code objects. + pass + + # Execute the new code, which is guaranteed to raise, and return + # the new traceback without this frame. + try: + exec(code, globals, locals) + except BaseException: + return sys.exc_info()[2].tb_next + + +def get_template_locals(real_locals): + """Based on the runtime locals, get the context that would be + available at that point in the template. + """ + # Start with the current template context. + ctx = real_locals.get("context") + + if ctx: + data = ctx.get_all().copy() + else: + data = {} + + # Might be in a derived context that only sets local variables + # rather than pushing a context. Local variables follow the scheme + # l_depth_name. Find the highest-depth local that has a value for + # each name. + local_overrides = {} + + for name, value in real_locals.items(): + if not name.startswith("l_") or value is missing: + # Not a template variable, or no longer relevant. + continue + + try: + _, depth, name = name.split("_", 2) + depth = int(depth) + except ValueError: + continue + + cur_depth = local_overrides.get(name, (-1,))[0] + + if cur_depth < depth: + local_overrides[name] = (depth, value) + + # Modify the context with any derived context. + for name, (_, value) in local_overrides.items(): + if value is missing: + data.pop(name, None) + else: + data[name] = value + + return data + + +if sys.version_info >= (3, 7): + # tb_next is directly assignable as of Python 3.7 + def tb_set_next(tb, tb_next): + tb.tb_next = tb_next + return tb + + +elif platform.python_implementation() == "PyPy": + # PyPy might have special support, and won't work with ctypes. + try: + import tputil + except ImportError: + # Without tproxy support, use the original traceback. + def tb_set_next(tb, tb_next): + return tb + + else: + # With tproxy support, create a proxy around the traceback that + # returns the new tb_next. + def tb_set_next(tb, tb_next): + def controller(op): + if op.opname == "__getattribute__" and op.args[0] == "tb_next": + return tb_next + + return op.delegate() + + return tputil.make_proxy(controller, obj=tb) + + +else: + # Use ctypes to assign tb_next at the C level since it's read-only + # from Python. + import ctypes + + class _CTraceback(ctypes.Structure): + _fields_ = [ + # Extra PyObject slots when compiled with Py_TRACE_REFS. + ("PyObject_HEAD", ctypes.c_byte * object().__sizeof__()), + # Only care about tb_next as an object, not a traceback. + ("tb_next", ctypes.py_object), + ] + + def tb_set_next(tb, tb_next): + c_tb = _CTraceback.from_address(id(tb)) + + # Clear out the old tb_next. + if tb.tb_next is not None: + c_tb_next = ctypes.py_object(tb.tb_next) + c_tb.tb_next = ctypes.py_object() + ctypes.pythonapi.Py_DecRef(c_tb_next) + + # Assign the new tb_next. + if tb_next is not None: + c_tb_next = ctypes.py_object(tb_next) + ctypes.pythonapi.Py_IncRef(c_tb_next) + c_tb.tb_next = c_tb_next + + return tb diff --git a/src/jinja2/defaults.py b/src/jinja2/defaults.py new file mode 100644 index 00000000..1f0b0ab0 --- /dev/null +++ b/src/jinja2/defaults.py @@ -0,0 +1,42 @@ +from .filters import FILTERS as DEFAULT_FILTERS # noqa: F401 +from .tests import TESTS as DEFAULT_TESTS # noqa: F401 +from .utils import Cycler +from .utils import generate_lorem_ipsum +from .utils import Joiner +from .utils import Namespace + +# defaults for the parser / lexer +BLOCK_START_STRING = "{%" +BLOCK_END_STRING = "%}" +VARIABLE_START_STRING = "{{" +VARIABLE_END_STRING = "}}" +COMMENT_START_STRING = "{#" +COMMENT_END_STRING = "#}" +LINE_STATEMENT_PREFIX = None +LINE_COMMENT_PREFIX = None +TRIM_BLOCKS = False +LSTRIP_BLOCKS = False +NEWLINE_SEQUENCE = "\n" +KEEP_TRAILING_NEWLINE = False + +# default filters, tests and namespace + +DEFAULT_NAMESPACE = { + "range": range, + "dict": dict, + "lipsum": generate_lorem_ipsum, + "cycler": Cycler, + "joiner": Joiner, + "namespace": Namespace, +} + +# default policies +DEFAULT_POLICIES = { + "compiler.ascii_str": True, + "urlize.rel": "noopener", + "urlize.target": None, + "truncate.leeway": 5, + "json.dumps_function": None, + "json.dumps_kwargs": {"sort_keys": True}, + "ext.i18n.trimmed": False, +} diff --git a/src/jinja2/environment.py b/src/jinja2/environment.py new file mode 100644 index 00000000..556f7255 --- /dev/null +++ b/src/jinja2/environment.py @@ -0,0 +1,1331 @@ +"""Classes for managing templates and their runtime and compile time +options. +""" +import os +import sys +import weakref +from functools import partial +from functools import reduce + +from markupsafe import Markup + +from . import nodes +from .compiler import CodeGenerator +from .compiler import generate +from .defaults import BLOCK_END_STRING +from .defaults import BLOCK_START_STRING +from .defaults import COMMENT_END_STRING +from .defaults import COMMENT_START_STRING +from .defaults import DEFAULT_FILTERS +from .defaults import DEFAULT_NAMESPACE +from .defaults import DEFAULT_POLICIES +from .defaults import DEFAULT_TESTS +from .defaults import KEEP_TRAILING_NEWLINE +from .defaults import LINE_COMMENT_PREFIX +from .defaults import LINE_STATEMENT_PREFIX +from .defaults import LSTRIP_BLOCKS +from .defaults import NEWLINE_SEQUENCE +from .defaults import TRIM_BLOCKS +from .defaults import VARIABLE_END_STRING +from .defaults import VARIABLE_START_STRING +from .exceptions import TemplateNotFound +from .exceptions import TemplateRuntimeError +from .exceptions import TemplatesNotFound +from .exceptions import TemplateSyntaxError +from .exceptions import UndefinedError +from .lexer import get_lexer +from .lexer import TokenStream +from .nodes import EvalContext +from .parser import Parser +from .runtime import Context +from .runtime import new_context +from .runtime import Undefined +from .utils import concat +from .utils import consume +from .utils import have_async_gen +from .utils import import_string +from .utils import internalcode +from .utils import LRUCache +from .utils import missing + +# for direct template usage we have up to ten living environments +_spontaneous_environments = LRUCache(10) + + +def get_spontaneous_environment(cls, *args): + """Return a new spontaneous environment. A spontaneous environment + is used for templates created directly rather than through an + existing environment. + + :param cls: Environment class to create. + :param args: Positional arguments passed to environment. + """ + key = (cls, args) + + try: + return _spontaneous_environments[key] + except KeyError: + _spontaneous_environments[key] = env = cls(*args) + env.shared = True + return env + + +def create_cache(size): + """Return the cache class for the given size.""" + if size == 0: + return None + if size < 0: + return {} + return LRUCache(size) + + +def copy_cache(cache): + """Create an empty copy of the given cache.""" + if cache is None: + return None + elif type(cache) is dict: + return {} + return LRUCache(cache.capacity) + + +def load_extensions(environment, extensions): + """Load the extensions from the list and bind it to the environment. + Returns a dict of instantiated environments. + """ + result = {} + for extension in extensions: + if isinstance(extension, str): + extension = import_string(extension) + result[extension.identifier] = extension(environment) + return result + + +def fail_for_missing_callable(thing, name): + msg = f"no {thing} named {name!r}" + + if isinstance(name, Undefined): + try: + name._fail_with_undefined_error() + except Exception as e: + msg = f"{msg} ({e}; did you forget to quote the callable name?)" + raise TemplateRuntimeError(msg) + + +def _environment_sanity_check(environment): + """Perform a sanity check on the environment.""" + assert issubclass( + environment.undefined, Undefined + ), "undefined must be a subclass of undefined because filters depend on it." + assert ( + environment.block_start_string + != environment.variable_start_string + != environment.comment_start_string + ), "block, variable and comment start strings must be different" + assert environment.newline_sequence in { + "\r", + "\r\n", + "\n", + }, "newline_sequence set to unknown line ending string." + return environment + + +class Environment: + r"""The core component of Jinja is the `Environment`. It contains + important shared variables like configuration, filters, tests, + globals and others. Instances of this class may be modified if + they are not shared and if no template was loaded so far. + Modifications on environments after the first template was loaded + will lead to surprising effects and undefined behavior. + + Here are the possible initialization parameters: + + `block_start_string` + The string marking the beginning of a block. Defaults to ``'{%'``. + + `block_end_string` + The string marking the end of a block. Defaults to ``'%}'``. + + `variable_start_string` + The string marking the beginning of a print statement. + Defaults to ``'{{'``. + + `variable_end_string` + The string marking the end of a print statement. Defaults to + ``'}}'``. + + `comment_start_string` + The string marking the beginning of a comment. Defaults to ``'{#'``. + + `comment_end_string` + The string marking the end of a comment. Defaults to ``'#}'``. + + `line_statement_prefix` + If given and a string, this will be used as prefix for line based + statements. See also :ref:`line-statements`. + + `line_comment_prefix` + If given and a string, this will be used as prefix for line based + comments. See also :ref:`line-statements`. + + .. versionadded:: 2.2 + + `trim_blocks` + If this is set to ``True`` the first newline after a block is + removed (block, not variable tag!). Defaults to `False`. + + `lstrip_blocks` + If this is set to ``True`` leading spaces and tabs are stripped + from the start of a line to a block. Defaults to `False`. + + `newline_sequence` + The sequence that starts a newline. Must be one of ``'\r'``, + ``'\n'`` or ``'\r\n'``. The default is ``'\n'`` which is a + useful default for Linux and OS X systems as well as web + applications. + + `keep_trailing_newline` + Preserve the trailing newline when rendering templates. + The default is ``False``, which causes a single newline, + if present, to be stripped from the end of the template. + + .. versionadded:: 2.7 + + `extensions` + List of Jinja extensions to use. This can either be import paths + as strings or extension classes. For more information have a + look at :ref:`the extensions documentation <jinja-extensions>`. + + `optimized` + should the optimizer be enabled? Default is ``True``. + + `undefined` + :class:`Undefined` or a subclass of it that is used to represent + undefined values in the template. + + `finalize` + A callable that can be used to process the result of a variable + expression before it is output. For example one can convert + ``None`` implicitly into an empty string here. + + `autoescape` + If set to ``True`` the XML/HTML autoescaping feature is enabled by + default. For more details about autoescaping see + :class:`~markupsafe.Markup`. As of Jinja 2.4 this can also + be a callable that is passed the template name and has to + return ``True`` or ``False`` depending on autoescape should be + enabled by default. + + .. versionchanged:: 2.4 + `autoescape` can now be a function + + `loader` + The template loader for this environment. + + `cache_size` + The size of the cache. Per default this is ``400`` which means + that if more than 400 templates are loaded the loader will clean + out the least recently used template. If the cache size is set to + ``0`` templates are recompiled all the time, if the cache size is + ``-1`` the cache will not be cleaned. + + .. versionchanged:: 2.8 + The cache size was increased to 400 from a low 50. + + `auto_reload` + Some loaders load templates from locations where the template + sources may change (ie: file system or database). If + ``auto_reload`` is set to ``True`` (default) every time a template is + requested the loader checks if the source changed and if yes, it + will reload the template. For higher performance it's possible to + disable that. + + `bytecode_cache` + If set to a bytecode cache object, this object will provide a + cache for the internal Jinja bytecode so that templates don't + have to be parsed if they were not changed. + + See :ref:`bytecode-cache` for more information. + + `enable_async` + If set to true this enables async template execution which + allows using async functions and generators. + """ + + #: if this environment is sandboxed. Modifying this variable won't make + #: the environment sandboxed though. For a real sandboxed environment + #: have a look at jinja2.sandbox. This flag alone controls the code + #: generation by the compiler. + sandboxed = False + + #: True if the environment is just an overlay + overlayed = False + + #: the environment this environment is linked to if it is an overlay + linked_to = None + + #: shared environments have this set to `True`. A shared environment + #: must not be modified + shared = False + + #: the class that is used for code generation. See + #: :class:`~jinja2.compiler.CodeGenerator` for more information. + code_generator_class = CodeGenerator + + #: the context class thatis used for templates. See + #: :class:`~jinja2.runtime.Context` for more information. + context_class = Context + + def __init__( + self, + block_start_string=BLOCK_START_STRING, + block_end_string=BLOCK_END_STRING, + variable_start_string=VARIABLE_START_STRING, + variable_end_string=VARIABLE_END_STRING, + comment_start_string=COMMENT_START_STRING, + comment_end_string=COMMENT_END_STRING, + line_statement_prefix=LINE_STATEMENT_PREFIX, + line_comment_prefix=LINE_COMMENT_PREFIX, + trim_blocks=TRIM_BLOCKS, + lstrip_blocks=LSTRIP_BLOCKS, + newline_sequence=NEWLINE_SEQUENCE, + keep_trailing_newline=KEEP_TRAILING_NEWLINE, + extensions=(), + optimized=True, + undefined=Undefined, + finalize=None, + autoescape=False, + loader=None, + cache_size=400, + auto_reload=True, + bytecode_cache=None, + enable_async=False, + ): + # !!Important notice!! + # The constructor accepts quite a few arguments that should be + # passed by keyword rather than position. However it's important to + # not change the order of arguments because it's used at least + # internally in those cases: + # - spontaneous environments (i18n extension and Template) + # - unittests + # If parameter changes are required only add parameters at the end + # and don't change the arguments (or the defaults!) of the arguments + # existing already. + + # lexer / parser information + self.block_start_string = block_start_string + self.block_end_string = block_end_string + self.variable_start_string = variable_start_string + self.variable_end_string = variable_end_string + self.comment_start_string = comment_start_string + self.comment_end_string = comment_end_string + self.line_statement_prefix = line_statement_prefix + self.line_comment_prefix = line_comment_prefix + self.trim_blocks = trim_blocks + self.lstrip_blocks = lstrip_blocks + self.newline_sequence = newline_sequence + self.keep_trailing_newline = keep_trailing_newline + + # runtime information + self.undefined = undefined + self.optimized = optimized + self.finalize = finalize + self.autoescape = autoescape + + # defaults + self.filters = DEFAULT_FILTERS.copy() + self.tests = DEFAULT_TESTS.copy() + self.globals = DEFAULT_NAMESPACE.copy() + + # set the loader provided + self.loader = loader + self.cache = create_cache(cache_size) + self.bytecode_cache = bytecode_cache + self.auto_reload = auto_reload + + # configurable policies + self.policies = DEFAULT_POLICIES.copy() + + # load extensions + self.extensions = load_extensions(self, extensions) + + self.enable_async = enable_async + self.is_async = self.enable_async and have_async_gen + if self.is_async: + # runs patch_all() to enable async support + from . import asyncsupport # noqa: F401 + + _environment_sanity_check(self) + + def add_extension(self, extension): + """Adds an extension after the environment was created. + + .. versionadded:: 2.5 + """ + self.extensions.update(load_extensions(self, [extension])) + + def extend(self, **attributes): + """Add the items to the instance of the environment if they do not exist + yet. This is used by :ref:`extensions <writing-extensions>` to register + callbacks and configuration values without breaking inheritance. + """ + for key, value in attributes.items(): + if not hasattr(self, key): + setattr(self, key, value) + + def overlay( + self, + block_start_string=missing, + block_end_string=missing, + variable_start_string=missing, + variable_end_string=missing, + comment_start_string=missing, + comment_end_string=missing, + line_statement_prefix=missing, + line_comment_prefix=missing, + trim_blocks=missing, + lstrip_blocks=missing, + extensions=missing, + optimized=missing, + undefined=missing, + finalize=missing, + autoescape=missing, + loader=missing, + cache_size=missing, + auto_reload=missing, + bytecode_cache=missing, + ): + """Create a new overlay environment that shares all the data with the + current environment except for cache and the overridden attributes. + Extensions cannot be removed for an overlayed environment. An overlayed + environment automatically gets all the extensions of the environment it + is linked to plus optional extra extensions. + + Creating overlays should happen after the initial environment was set + up completely. Not all attributes are truly linked, some are just + copied over so modifications on the original environment may not shine + through. + """ + args = dict(locals()) + del args["self"], args["cache_size"], args["extensions"] + + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.overlayed = True + rv.linked_to = self + + for key, value in args.items(): + if value is not missing: + setattr(rv, key, value) + + if cache_size is not missing: + rv.cache = create_cache(cache_size) + else: + rv.cache = copy_cache(self.cache) + + rv.extensions = {} + for key, value in self.extensions.items(): + rv.extensions[key] = value.bind(rv) + if extensions is not missing: + rv.extensions.update(load_extensions(rv, extensions)) + + return _environment_sanity_check(rv) + + lexer = property(get_lexer, doc="The lexer for this environment.") + + def iter_extensions(self): + """Iterates over the extensions by priority.""" + return iter(sorted(self.extensions.values(), key=lambda x: x.priority)) + + def getitem(self, obj, argument): + """Get an item or attribute of an object but prefer the item.""" + try: + return obj[argument] + except (AttributeError, TypeError, LookupError): + if isinstance(argument, str): + try: + attr = str(argument) + except Exception: + pass + else: + try: + return getattr(obj, attr) + except AttributeError: + pass + return self.undefined(obj=obj, name=argument) + + def getattr(self, obj, attribute): + """Get an item or attribute of an object but prefer the attribute. + Unlike :meth:`getitem` the attribute *must* be a bytestring. + """ + try: + return getattr(obj, attribute) + except AttributeError: + pass + try: + return obj[attribute] + except (TypeError, LookupError, AttributeError): + return self.undefined(obj=obj, name=attribute) + + def call_filter( + self, name, value, args=None, kwargs=None, context=None, eval_ctx=None + ): + """Invokes a filter on a value the same way the compiler does. + + This might return a coroutine if the filter is running from an + environment in async mode and the filter supports async + execution. It's your responsibility to await this if needed. + + .. versionadded:: 2.7 + """ + func = self.filters.get(name) + if func is None: + fail_for_missing_callable("filter", name) + args = [value] + list(args or ()) + if getattr(func, "contextfilter", False) is True: + if context is None: + raise TemplateRuntimeError( + "Attempted to invoke context filter without context" + ) + args.insert(0, context) + elif getattr(func, "evalcontextfilter", False) is True: + if eval_ctx is None: + if context is not None: + eval_ctx = context.eval_ctx + else: + eval_ctx = EvalContext(self) + args.insert(0, eval_ctx) + elif getattr(func, "environmentfilter", False) is True: + args.insert(0, self) + return func(*args, **(kwargs or {})) + + def call_test(self, name, value, args=None, kwargs=None): + """Invokes a test on a value the same way the compiler does it. + + .. versionadded:: 2.7 + """ + func = self.tests.get(name) + if func is None: + fail_for_missing_callable("test", name) + return func(value, *(args or ()), **(kwargs or {})) + + @internalcode + def parse(self, source, name=None, filename=None): + """Parse the sourcecode and return the abstract syntax tree. This + tree of nodes is used by the compiler to convert the template into + executable source- or bytecode. This is useful for debugging or to + extract information from templates. + + If you are :ref:`developing Jinja extensions <writing-extensions>` + this gives you a good overview of the node tree generated. + """ + try: + return self._parse(source, name, filename) + except TemplateSyntaxError: + self.handle_exception(source=source) + + def _parse(self, source, name, filename): + """Internal parsing function used by `parse` and `compile`.""" + return Parser(self, source, name, filename).parse() + + def lex(self, source, name=None, filename=None): + """Lex the given sourcecode and return a generator that yields + tokens as tuples in the form ``(lineno, token_type, value)``. + This can be useful for :ref:`extension development <writing-extensions>` + and debugging templates. + + This does not perform preprocessing. If you want the preprocessing + of the extensions to be applied you have to filter source through + the :meth:`preprocess` method. + """ + source = str(source) + try: + return self.lexer.tokeniter(source, name, filename) + except TemplateSyntaxError: + self.handle_exception(source=source) + + def preprocess(self, source, name=None, filename=None): + """Preprocesses the source with all extensions. This is automatically + called for all parsing and compiling methods but *not* for :meth:`lex` + because there you usually only want the actual source tokenized. + """ + return reduce( + lambda s, e: e.preprocess(s, name, filename), + self.iter_extensions(), + str(source), + ) + + def _tokenize(self, source, name, filename=None, state=None): + """Called by the parser to do the preprocessing and filtering + for all the extensions. Returns a :class:`~jinja2.lexer.TokenStream`. + """ + source = self.preprocess(source, name, filename) + stream = self.lexer.tokenize(source, name, filename, state) + for ext in self.iter_extensions(): + stream = ext.filter_stream(stream) + if not isinstance(stream, TokenStream): + stream = TokenStream(stream, name, filename) + return stream + + def _generate(self, source, name, filename, defer_init=False): + """Internal hook that can be overridden to hook a different generate + method in. + + .. versionadded:: 2.5 + """ + return generate( + source, + self, + name, + filename, + defer_init=defer_init, + optimized=self.optimized, + ) + + def _compile(self, source, filename): + """Internal hook that can be overridden to hook a different compile + method in. + + .. versionadded:: 2.5 + """ + return compile(source, filename, "exec") + + @internalcode + def compile(self, source, name=None, filename=None, raw=False, defer_init=False): + """Compile a node or template source code. The `name` parameter is + the load name of the template after it was joined using + :meth:`join_path` if necessary, not the filename on the file system. + the `filename` parameter is the estimated filename of the template on + the file system. If the template came from a database or memory this + can be omitted. + + The return value of this method is a python code object. If the `raw` + parameter is `True` the return value will be a string with python + code equivalent to the bytecode returned otherwise. This method is + mainly used internally. + + `defer_init` is use internally to aid the module code generator. This + causes the generated code to be able to import without the global + environment variable to be set. + + .. versionadded:: 2.4 + `defer_init` parameter added. + """ + source_hint = None + try: + if isinstance(source, str): + source_hint = source + source = self._parse(source, name, filename) + source = self._generate(source, name, filename, defer_init=defer_init) + if raw: + return source + if filename is None: + filename = "<template>" + return self._compile(source, filename) + except TemplateSyntaxError: + self.handle_exception(source=source_hint) + + def compile_expression(self, source, undefined_to_none=True): + """A handy helper method that returns a callable that accepts keyword + arguments that appear as variables in the expression. If called it + returns the result of the expression. + + This is useful if applications want to use the same rules as Jinja + in template "configuration files" or similar situations. + + Example usage: + + >>> env = Environment() + >>> expr = env.compile_expression('foo == 42') + >>> expr(foo=23) + False + >>> expr(foo=42) + True + + Per default the return value is converted to `None` if the + expression returns an undefined value. This can be changed + by setting `undefined_to_none` to `False`. + + >>> env.compile_expression('var')() is None + True + >>> env.compile_expression('var', undefined_to_none=False)() + Undefined + + .. versionadded:: 2.1 + """ + parser = Parser(self, source, state="variable") + try: + expr = parser.parse_expression() + if not parser.stream.eos: + raise TemplateSyntaxError( + "chunk after expression", parser.stream.current.lineno, None, None + ) + expr.set_environment(self) + except TemplateSyntaxError: + if sys.exc_info() is not None: + self.handle_exception(source=source) + + body = [nodes.Assign(nodes.Name("result", "store"), expr, lineno=1)] + template = self.from_string(nodes.Template(body, lineno=1)) + return TemplateExpression(template, undefined_to_none) + + def compile_templates( + self, + target, + extensions=None, + filter_func=None, + zip="deflated", + log_function=None, + ignore_errors=True, + ): + """Finds all the templates the loader can find, compiles them + and stores them in `target`. If `zip` is `None`, instead of in a + zipfile, the templates will be stored in a directory. + By default a deflate zip algorithm is used. To switch to + the stored algorithm, `zip` can be set to ``'stored'``. + + `extensions` and `filter_func` are passed to :meth:`list_templates`. + Each template returned will be compiled to the target folder or + zipfile. + + By default template compilation errors are ignored. In case a + log function is provided, errors are logged. If you want template + syntax errors to abort the compilation you can set `ignore_errors` + to `False` and you will get an exception on syntax errors. + + .. versionadded:: 2.4 + """ + from .loaders import ModuleLoader + + if log_function is None: + + def log_function(x): + pass + + def write_file(filename, data): + if zip: + info = ZipInfo(filename) + info.external_attr = 0o755 << 16 + zip_file.writestr(info, data) + else: + if isinstance(data, str): + data = data.encode("utf8") + + with open(os.path.join(target, filename), "wb") as f: + f.write(data) + + if zip is not None: + from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED, ZIP_STORED + + zip_file = ZipFile( + target, "w", dict(deflated=ZIP_DEFLATED, stored=ZIP_STORED)[zip] + ) + log_function(f"Compiling into Zip archive {target!r}") + else: + if not os.path.isdir(target): + os.makedirs(target) + log_function(f"Compiling into folder {target!r}") + + try: + for name in self.list_templates(extensions, filter_func): + source, filename, _ = self.loader.get_source(self, name) + try: + code = self.compile(source, name, filename, True, True) + except TemplateSyntaxError as e: + if not ignore_errors: + raise + log_function(f'Could not compile "{name}": {e}') + continue + + filename = ModuleLoader.get_module_filename(name) + + write_file(filename, code) + log_function(f'Compiled "{name}" as {filename}') + finally: + if zip: + zip_file.close() + + log_function("Finished compiling templates") + + def list_templates(self, extensions=None, filter_func=None): + """Returns a list of templates for this environment. This requires + that the loader supports the loader's + :meth:`~BaseLoader.list_templates` method. + + If there are other files in the template folder besides the + actual templates, the returned list can be filtered. There are two + ways: either `extensions` is set to a list of file extensions for + templates, or a `filter_func` can be provided which is a callable that + is passed a template name and should return `True` if it should end up + in the result list. + + If the loader does not support that, a :exc:`TypeError` is raised. + + .. versionadded:: 2.4 + """ + names = self.loader.list_templates() + + if extensions is not None: + if filter_func is not None: + raise TypeError( + "either extensions or filter_func can be passed, but not both" + ) + + def filter_func(x): + return "." in x and x.rsplit(".", 1)[1] in extensions + + if filter_func is not None: + names = [name for name in names if filter_func(name)] + + return names + + def handle_exception(self, source=None): + """Exception handling helper. This is used internally to either raise + rewritten exceptions or return a rendered traceback for the template. + """ + from .debug import rewrite_traceback_stack + + raise rewrite_traceback_stack(source=source) + + def join_path(self, template, parent): + """Join a template with the parent. By default all the lookups are + relative to the loader root so this method returns the `template` + parameter unchanged, but if the paths should be relative to the + parent template, this function can be used to calculate the real + template name. + + Subclasses may override this method and implement template path + joining here. + """ + return template + + @internalcode + def _load_template(self, name, globals): + if self.loader is None: + raise TypeError("no loader for this environment specified") + cache_key = (weakref.ref(self.loader), name) + if self.cache is not None: + template = self.cache.get(cache_key) + if template is not None and ( + not self.auto_reload or template.is_up_to_date + ): + return template + template = self.loader.load(self, name, globals) + if self.cache is not None: + self.cache[cache_key] = template + return template + + @internalcode + def get_template(self, name, parent=None, globals=None): + """Load a template from the loader. If a loader is configured this + method asks the loader for the template and returns a :class:`Template`. + If the `parent` parameter is not `None`, :meth:`join_path` is called + to get the real template name before loading. + + The `globals` parameter can be used to provide template wide globals. + These variables are available in the context at render time. + + If the template does not exist a :exc:`TemplateNotFound` exception is + raised. + + .. versionchanged:: 2.4 + If `name` is a :class:`Template` object it is returned from the + function unchanged. + """ + if isinstance(name, Template): + return name + if parent is not None: + name = self.join_path(name, parent) + return self._load_template(name, self.make_globals(globals)) + + @internalcode + def select_template(self, names, parent=None, globals=None): + """Works like :meth:`get_template` but tries a number of templates + before it fails. If it cannot find any of the templates, it will + raise a :exc:`TemplatesNotFound` exception. + + .. versionchanged:: 2.11 + If names is :class:`Undefined`, an :exc:`UndefinedError` is + raised instead. If no templates were found and names + contains :class:`Undefined`, the message is more helpful. + + .. versionchanged:: 2.4 + If `names` contains a :class:`Template` object it is returned + from the function unchanged. + + .. versionadded:: 2.3 + """ + if isinstance(names, Undefined): + names._fail_with_undefined_error() + + if not names: + raise TemplatesNotFound( + message="Tried to select from an empty list of templates." + ) + globals = self.make_globals(globals) + for name in names: + if isinstance(name, Template): + return name + if parent is not None: + name = self.join_path(name, parent) + try: + return self._load_template(name, globals) + except (TemplateNotFound, UndefinedError): + pass + raise TemplatesNotFound(names) + + @internalcode + def get_or_select_template(self, template_name_or_list, parent=None, globals=None): + """Does a typecheck and dispatches to :meth:`select_template` + if an iterable of template names is given, otherwise to + :meth:`get_template`. + + .. versionadded:: 2.3 + """ + if isinstance(template_name_or_list, (str, Undefined)): + return self.get_template(template_name_or_list, parent, globals) + elif isinstance(template_name_or_list, Template): + return template_name_or_list + return self.select_template(template_name_or_list, parent, globals) + + def from_string(self, source, globals=None, template_class=None): + """Load a template from a string. This parses the source given and + returns a :class:`Template` object. + """ + globals = self.make_globals(globals) + cls = template_class or self.template_class + return cls.from_code(self, self.compile(source), globals, None) + + def make_globals(self, d): + """Return a dict for the globals.""" + if not d: + return self.globals + return dict(self.globals, **d) + + +class Template: + """The central template object. This class represents a compiled template + and is used to evaluate it. + + Normally the template object is generated from an :class:`Environment` but + it also has a constructor that makes it possible to create a template + instance directly using the constructor. It takes the same arguments as + the environment constructor but it's not possible to specify a loader. + + Every template object has a few methods and members that are guaranteed + to exist. However it's important that a template object should be + considered immutable. Modifications on the object are not supported. + + Template objects created from the constructor rather than an environment + do have an `environment` attribute that points to a temporary environment + that is probably shared with other templates created with the constructor + and compatible settings. + + >>> template = Template('Hello {{ name }}!') + >>> template.render(name='John Doe') == u'Hello John Doe!' + True + >>> stream = template.stream(name='John Doe') + >>> next(stream) == u'Hello John Doe!' + True + >>> next(stream) + Traceback (most recent call last): + ... + StopIteration + """ + + #: Type of environment to create when creating a template directly + #: rather than through an existing environment. + environment_class = Environment + + def __new__( + cls, + source, + block_start_string=BLOCK_START_STRING, + block_end_string=BLOCK_END_STRING, + variable_start_string=VARIABLE_START_STRING, + variable_end_string=VARIABLE_END_STRING, + comment_start_string=COMMENT_START_STRING, + comment_end_string=COMMENT_END_STRING, + line_statement_prefix=LINE_STATEMENT_PREFIX, + line_comment_prefix=LINE_COMMENT_PREFIX, + trim_blocks=TRIM_BLOCKS, + lstrip_blocks=LSTRIP_BLOCKS, + newline_sequence=NEWLINE_SEQUENCE, + keep_trailing_newline=KEEP_TRAILING_NEWLINE, + extensions=(), + optimized=True, + undefined=Undefined, + finalize=None, + autoescape=False, + enable_async=False, + ): + env = get_spontaneous_environment( + cls.environment_class, + block_start_string, + block_end_string, + variable_start_string, + variable_end_string, + comment_start_string, + comment_end_string, + line_statement_prefix, + line_comment_prefix, + trim_blocks, + lstrip_blocks, + newline_sequence, + keep_trailing_newline, + frozenset(extensions), + optimized, + undefined, + finalize, + autoescape, + None, + 0, + False, + None, + enable_async, + ) + return env.from_string(source, template_class=cls) + + @classmethod + def from_code(cls, environment, code, globals, uptodate=None): + """Creates a template object from compiled code and the globals. This + is used by the loaders and environment to create a template object. + """ + namespace = {"environment": environment, "__file__": code.co_filename} + exec(code, namespace) + rv = cls._from_namespace(environment, namespace, globals) + rv._uptodate = uptodate + return rv + + @classmethod + def from_module_dict(cls, environment, module_dict, globals): + """Creates a template object from a module. This is used by the + module loader to create a template object. + + .. versionadded:: 2.4 + """ + return cls._from_namespace(environment, module_dict, globals) + + @classmethod + def _from_namespace(cls, environment, namespace, globals): + t = object.__new__(cls) + t.environment = environment + t.globals = globals + t.name = namespace["name"] + t.filename = namespace["__file__"] + t.blocks = namespace["blocks"] + + # render function and module + t.root_render_func = namespace["root"] + t._module = None + + # debug and loader helpers + t._debug_info = namespace["debug_info"] + t._uptodate = None + + # store the reference + namespace["environment"] = environment + namespace["__jinja_template__"] = t + + return t + + def render(self, *args, **kwargs): + """This method accepts the same arguments as the `dict` constructor: + A dict, a dict subclass or some keyword arguments. If no arguments + are given the context will be empty. These two calls do the same:: + + template.render(knights='that say nih') + template.render({'knights': 'that say nih'}) + + This will return the rendered template as a string. + """ + vars = dict(*args, **kwargs) + try: + return concat(self.root_render_func(self.new_context(vars))) + except Exception: + self.environment.handle_exception() + + def render_async(self, *args, **kwargs): + """This works similar to :meth:`render` but returns a coroutine + that when awaited returns the entire rendered template string. This + requires the async feature to be enabled. + + Example usage:: + + await template.render_async(knights='that say nih; asynchronously') + """ + # see asyncsupport for the actual implementation + raise NotImplementedError( + "This feature is not available for this version of Python" + ) + + def stream(self, *args, **kwargs): + """Works exactly like :meth:`generate` but returns a + :class:`TemplateStream`. + """ + return TemplateStream(self.generate(*args, **kwargs)) + + def generate(self, *args, **kwargs): + """For very large templates it can be useful to not render the whole + template at once but evaluate each statement after another and yield + piece for piece. This method basically does exactly that and returns + a generator that yields one item after another as strings. + + It accepts the same arguments as :meth:`render`. + """ + vars = dict(*args, **kwargs) + try: + yield from self.root_render_func(self.new_context(vars)) + except Exception: + yield self.environment.handle_exception() + + def generate_async(self, *args, **kwargs): + """An async version of :meth:`generate`. Works very similarly but + returns an async iterator instead. + """ + # see asyncsupport for the actual implementation + raise NotImplementedError( + "This feature is not available for this version of Python" + ) + + def new_context(self, vars=None, shared=False, locals=None): + """Create a new :class:`Context` for this template. The vars + provided will be passed to the template. Per default the globals + are added to the context. If shared is set to `True` the data + is passed as is to the context without adding the globals. + + `locals` can be a dict of local variables for internal usage. + """ + return new_context( + self.environment, self.name, self.blocks, vars, shared, self.globals, locals + ) + + def make_module(self, vars=None, shared=False, locals=None): + """This method works like the :attr:`module` attribute when called + without arguments but it will evaluate the template on every call + rather than caching it. It's also possible to provide + a dict which is then used as context. The arguments are the same + as for the :meth:`new_context` method. + """ + return TemplateModule(self, self.new_context(vars, shared, locals)) + + def make_module_async(self, vars=None, shared=False, locals=None): + """As template module creation can invoke template code for + asynchronous executions this method must be used instead of the + normal :meth:`make_module` one. Likewise the module attribute + becomes unavailable in async mode. + """ + # see asyncsupport for the actual implementation + raise NotImplementedError( + "This feature is not available for this version of Python" + ) + + @internalcode + def _get_default_module(self, ctx=None): + """If a context is passed in, this means that the template was + imported. Imported templates have access to the current template's + globals by default, but they can only be accessed via the context + during runtime. + + If there are new globals, we need to create a new + module because the cached module is already rendered and will not have + access to globals from the current context. This new module is not + cached as :attr:`_module` because the template can be imported elsewhere, + and it should have access to only the current template's globals. + """ + if ctx is not None: + globals = { + key: ctx.parent[key] for key in ctx.globals_keys - self.globals.keys() + } + if globals: + return self.make_module(globals) + if self._module is not None: + return self._module + self._module = rv = self.make_module() + return rv + + @property + def module(self): + """The template as module. This is used for imports in the + template runtime but is also useful if one wants to access + exported template variables from the Python layer: + + >>> t = Template('{% macro foo() %}42{% endmacro %}23') + >>> str(t.module) + '23' + >>> t.module.foo() == u'42' + True + + This attribute is not available if async mode is enabled. + """ + return self._get_default_module() + + def get_corresponding_lineno(self, lineno): + """Return the source line number of a line number in the + generated bytecode as they are not in sync. + """ + for template_line, code_line in reversed(self.debug_info): + if code_line <= lineno: + return template_line + return 1 + + @property + def is_up_to_date(self): + """If this variable is `False` there is a newer version available.""" + if self._uptodate is None: + return True + return self._uptodate() + + @property + def debug_info(self): + """The debug info mapping.""" + if self._debug_info: + return [tuple(map(int, x.split("="))) for x in self._debug_info.split("&")] + return [] + + def __repr__(self): + if self.name is None: + name = f"memory:{id(self):x}" + else: + name = repr(self.name) + return f"<{self.__class__.__name__} {name}>" + + +class TemplateModule: + """Represents an imported template. All the exported names of the + template are available as attributes on this object. Additionally + converting it into a string renders the contents. + """ + + def __init__(self, template, context, body_stream=None): + if body_stream is None: + if context.environment.is_async: + raise RuntimeError( + "Async mode requires a body stream " + "to be passed to a template module. Use " + "the async methods of the API you are " + "using." + ) + body_stream = list(template.root_render_func(context)) + self._body_stream = body_stream + self.__dict__.update(context.get_exported()) + self.__name__ = template.name + + def __html__(self): + return Markup(concat(self._body_stream)) + + def __str__(self): + return concat(self._body_stream) + + def __repr__(self): + if self.__name__ is None: + name = f"memory:{id(self):x}" + else: + name = repr(self.__name__) + return f"<{self.__class__.__name__} {name}>" + + +class TemplateExpression: + """The :meth:`jinja2.Environment.compile_expression` method returns an + instance of this object. It encapsulates the expression-like access + to the template with an expression it wraps. + """ + + def __init__(self, template, undefined_to_none): + self._template = template + self._undefined_to_none = undefined_to_none + + def __call__(self, *args, **kwargs): + context = self._template.new_context(dict(*args, **kwargs)) + consume(self._template.root_render_func(context)) + rv = context.vars["result"] + if self._undefined_to_none and isinstance(rv, Undefined): + rv = None + return rv + + +class TemplateStream: + """A template stream works pretty much like an ordinary python generator + but it can buffer multiple items to reduce the number of total iterations. + Per default the output is unbuffered which means that for every unbuffered + instruction in the template one string is yielded. + + If buffering is enabled with a buffer size of 5, five items are combined + into a new string. This is mainly useful if you are streaming + big templates to a client via WSGI which flushes after each iteration. + """ + + def __init__(self, gen): + self._gen = gen + self.disable_buffering() + + def dump(self, fp, encoding=None, errors="strict"): + """Dump the complete stream into a file or file-like object. + Per default strings are written, if you want to encode + before writing specify an `encoding`. + + Example usage:: + + Template('Hello {{ name }}!').stream(name='foo').dump('hello.html') + """ + close = False + if isinstance(fp, str): + if encoding is None: + encoding = "utf-8" + fp = open(fp, "wb") + close = True + try: + if encoding is not None: + iterable = (x.encode(encoding, errors) for x in self) + else: + iterable = self + if hasattr(fp, "writelines"): + fp.writelines(iterable) + else: + for item in iterable: + fp.write(item) + finally: + if close: + fp.close() + + def disable_buffering(self): + """Disable the output buffering.""" + self._next = partial(next, self._gen) + self.buffered = False + + def _buffered_generator(self, size): + buf = [] + c_size = 0 + push = buf.append + + while 1: + try: + while c_size < size: + c = next(self._gen) + push(c) + if c: + c_size += 1 + except StopIteration: + if not c_size: + return + yield concat(buf) + del buf[:] + c_size = 0 + + def enable_buffering(self, size=5): + """Enable buffering. Buffer `size` items before yielding them.""" + if size <= 1: + raise ValueError("buffer size too small") + + self.buffered = True + self._next = partial(next, self._buffered_generator(size)) + + def __iter__(self): + return self + + def __next__(self): + return self._next() + + +# hook in default template class. if anyone reads this comment: ignore that +# it's possible to use custom templates ;-) +Environment.template_class = Template diff --git a/src/jinja2/exceptions.py b/src/jinja2/exceptions.py new file mode 100644 index 00000000..07cfba26 --- /dev/null +++ b/src/jinja2/exceptions.py @@ -0,0 +1,147 @@ +class TemplateError(Exception): + """Baseclass for all template errors.""" + + def __init__(self, message=None): + super().__init__(message) + + @property + def message(self): + if self.args: + return self.args[0] + + +class TemplateNotFound(IOError, LookupError, TemplateError): + """Raised if a template does not exist. + + .. versionchanged:: 2.11 + If the given name is :class:`Undefined` and no message was + provided, an :exc:`UndefinedError` is raised. + """ + + # Silence the Python warning about message being deprecated since + # it's not valid here. + message = None + + def __init__(self, name, message=None): + IOError.__init__(self, name) + + if message is None: + from .runtime import Undefined + + if isinstance(name, Undefined): + name._fail_with_undefined_error() + + message = name + + self.message = message + self.name = name + self.templates = [name] + + def __str__(self): + return self.message + + +class TemplatesNotFound(TemplateNotFound): + """Like :class:`TemplateNotFound` but raised if multiple templates + are selected. This is a subclass of :class:`TemplateNotFound` + exception, so just catching the base exception will catch both. + + .. versionchanged:: 2.11 + If a name in the list of names is :class:`Undefined`, a message + about it being undefined is shown rather than the empty string. + + .. versionadded:: 2.2 + """ + + def __init__(self, names=(), message=None): + if message is None: + from .runtime import Undefined + + parts = [] + + for name in names: + if isinstance(name, Undefined): + parts.append(name._undefined_message) + else: + parts.append(name) + + message = "none of the templates given were found: " + ", ".join( + map(str, parts) + ) + TemplateNotFound.__init__(self, names[-1] if names else None, message) + self.templates = list(names) + + +class TemplateSyntaxError(TemplateError): + """Raised to tell the user that there is a problem with the template.""" + + def __init__(self, message, lineno, name=None, filename=None): + TemplateError.__init__(self, message) + self.lineno = lineno + self.name = name + self.filename = filename + self.source = None + + # this is set to True if the debug.translate_syntax_error + # function translated the syntax error into a new traceback + self.translated = False + + def __str__(self): + # for translated errors we only return the message + if self.translated: + return self.message + + # otherwise attach some stuff + location = f"line {self.lineno}" + name = self.filename or self.name + if name: + location = f'File "{name}", {location}' + lines = [self.message, " " + location] + + # if the source is set, add the line to the output + if self.source is not None: + try: + line = self.source.splitlines()[self.lineno - 1] + except IndexError: + line = None + if line: + lines.append(" " + line.strip()) + + return "\n".join(lines) + + def __reduce__(self): + # https://bugs.python.org/issue1692335 Exceptions that take + # multiple required arguments have problems with pickling. + # Without this, raises TypeError: __init__() missing 1 required + # positional argument: 'lineno' + return self.__class__, (self.message, self.lineno, self.name, self.filename) + + +class TemplateAssertionError(TemplateSyntaxError): + """Like a template syntax error, but covers cases where something in the + template caused an error at compile time that wasn't necessarily caused + by a syntax error. However it's a direct subclass of + :exc:`TemplateSyntaxError` and has the same attributes. + """ + + +class TemplateRuntimeError(TemplateError): + """A generic runtime error in the template engine. Under some situations + Jinja may raise this exception. + """ + + +class UndefinedError(TemplateRuntimeError): + """Raised if a template tries to operate on :class:`Undefined`.""" + + +class SecurityError(TemplateRuntimeError): + """Raised if a template tries to do something insecure if the + sandbox is enabled. + """ + + +class FilterArgumentError(TemplateRuntimeError): + """This error is raised if a filter was called with inappropriate + arguments + """ diff --git a/src/jinja2/ext.py b/src/jinja2/ext.py new file mode 100644 index 00000000..533ff179 --- /dev/null +++ b/src/jinja2/ext.py @@ -0,0 +1,700 @@ +"""Extension API for adding custom tags and behavior.""" +import pprint +import re +from sys import version_info + +from markupsafe import Markup + +from . import nodes +from .defaults import BLOCK_END_STRING +from .defaults import BLOCK_START_STRING +from .defaults import COMMENT_END_STRING +from .defaults import COMMENT_START_STRING +from .defaults import KEEP_TRAILING_NEWLINE +from .defaults import LINE_COMMENT_PREFIX +from .defaults import LINE_STATEMENT_PREFIX +from .defaults import LSTRIP_BLOCKS +from .defaults import NEWLINE_SEQUENCE +from .defaults import TRIM_BLOCKS +from .defaults import VARIABLE_END_STRING +from .defaults import VARIABLE_START_STRING +from .environment import Environment +from .exceptions import TemplateAssertionError +from .exceptions import TemplateSyntaxError +from .nodes import ContextReference +from .runtime import concat +from .utils import contextfunction +from .utils import import_string + +# I18N functions available in Jinja templates. If the I18N library +# provides ugettext, it will be assigned to gettext. +GETTEXT_FUNCTIONS = ("_", "gettext", "ngettext") +_ws_re = re.compile(r"\s*\n\s*") + + +class ExtensionRegistry(type): + """Gives the extension an unique identifier.""" + + def __new__(mcs, name, bases, d): + rv = type.__new__(mcs, name, bases, d) + rv.identifier = f"{rv.__module__}.{rv.__name__}" + return rv + + +class Extension(metaclass=ExtensionRegistry): + """Extensions can be used to add extra functionality to the Jinja template + system at the parser level. Custom extensions are bound to an environment + but may not store environment specific data on `self`. The reason for + this is that an extension can be bound to another environment (for + overlays) by creating a copy and reassigning the `environment` attribute. + + As extensions are created by the environment they cannot accept any + arguments for configuration. One may want to work around that by using + a factory function, but that is not possible as extensions are identified + by their import name. The correct way to configure the extension is + storing the configuration values on the environment. Because this way the + environment ends up acting as central configuration storage the + attributes may clash which is why extensions have to ensure that the names + they choose for configuration are not too generic. ``prefix`` for example + is a terrible name, ``fragment_cache_prefix`` on the other hand is a good + name as includes the name of the extension (fragment cache). + """ + + #: if this extension parses this is the list of tags it's listening to. + tags = set() + + #: the priority of that extension. This is especially useful for + #: extensions that preprocess values. A lower value means higher + #: priority. + #: + #: .. versionadded:: 2.4 + priority = 100 + + def __init__(self, environment): + self.environment = environment + + def bind(self, environment): + """Create a copy of this extension bound to another environment.""" + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.environment = environment + return rv + + def preprocess(self, source, name, filename=None): + """This method is called before the actual lexing and can be used to + preprocess the source. The `filename` is optional. The return value + must be the preprocessed source. + """ + return source + + def filter_stream(self, stream): + """It's passed a :class:`~jinja2.lexer.TokenStream` that can be used + to filter tokens returned. This method has to return an iterable of + :class:`~jinja2.lexer.Token`\\s, but it doesn't have to return a + :class:`~jinja2.lexer.TokenStream`. + """ + return stream + + def parse(self, parser): + """If any of the :attr:`tags` matched this method is called with the + parser as first argument. The token the parser stream is pointing at + is the name token that matched. This method has to return one or a + list of multiple nodes. + """ + raise NotImplementedError() + + def attr(self, name, lineno=None): + """Return an attribute node for the current extension. This is useful + to pass constants on extensions to generated template code. + + :: + + self.attr('_my_attribute', lineno=lineno) + """ + return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno) + + def call_method( + self, name, args=None, kwargs=None, dyn_args=None, dyn_kwargs=None, lineno=None + ): + """Call a method of the extension. This is a shortcut for + :meth:`attr` + :class:`jinja2.nodes.Call`. + """ + if args is None: + args = [] + if kwargs is None: + kwargs = [] + return nodes.Call( + self.attr(name, lineno=lineno), + args, + kwargs, + dyn_args, + dyn_kwargs, + lineno=lineno, + ) + + +@contextfunction +def _gettext_alias(__context, *args, **kwargs): + return __context.call(__context.resolve("gettext"), *args, **kwargs) + + +def _make_new_gettext(func): + @contextfunction + def gettext(__context, __string, **variables): + rv = __context.call(func, __string) + if __context.eval_ctx.autoescape: + rv = Markup(rv) + # Always treat as a format string, even if there are no + # variables. This makes translation strings more consistent + # and predictable. This requires escaping + return rv % variables + + return gettext + + +def _make_new_ngettext(func): + @contextfunction + def ngettext(__context, __singular, __plural, __num, **variables): + variables.setdefault("num", __num) + rv = __context.call(func, __singular, __plural, __num) + if __context.eval_ctx.autoescape: + rv = Markup(rv) + # Always treat as a format string, see gettext comment above. + return rv % variables + + return ngettext + + +class InternationalizationExtension(Extension): + """This extension adds gettext support to Jinja.""" + + tags = {"trans"} + + # TODO: the i18n extension is currently reevaluating values in a few + # situations. Take this example: + # {% trans count=something() %}{{ count }} foo{% pluralize + # %}{{ count }} fooss{% endtrans %} + # something is called twice here. One time for the gettext value and + # the other time for the n-parameter of the ngettext function. + + def __init__(self, environment): + Extension.__init__(self, environment) + environment.globals["_"] = _gettext_alias + environment.extend( + install_gettext_translations=self._install, + install_null_translations=self._install_null, + install_gettext_callables=self._install_callables, + uninstall_gettext_translations=self._uninstall, + extract_translations=self._extract, + newstyle_gettext=False, + ) + + def _install(self, translations, newstyle=None): + # ugettext and ungettext are preferred in case the I18N library + # is providing compatibility with older Python versions. + gettext = getattr(translations, "ugettext", None) + if gettext is None: + gettext = translations.gettext + ngettext = getattr(translations, "ungettext", None) + if ngettext is None: + ngettext = translations.ngettext + self._install_callables(gettext, ngettext, newstyle) + + def _install_null(self, newstyle=None): + self._install_callables( + lambda x: x, lambda s, p, n: s if n == 1 else p, newstyle + ) + + def _install_callables(self, gettext, ngettext, newstyle=None): + if newstyle is not None: + self.environment.newstyle_gettext = newstyle + if self.environment.newstyle_gettext: + gettext = _make_new_gettext(gettext) + ngettext = _make_new_ngettext(ngettext) + self.environment.globals.update(gettext=gettext, ngettext=ngettext) + + def _uninstall(self, translations): + for key in "gettext", "ngettext": + self.environment.globals.pop(key, None) + + def _extract(self, source, gettext_functions=GETTEXT_FUNCTIONS): + if isinstance(source, str): + source = self.environment.parse(source) + return extract_from_ast(source, gettext_functions) + + def parse(self, parser): + """Parse a translatable tag.""" + lineno = next(parser.stream).lineno + num_called_num = False + + # find all the variables referenced. Additionally a variable can be + # defined in the body of the trans block too, but this is checked at + # a later state. + plural_expr = None + plural_expr_assignment = None + variables = {} + trimmed = None + while parser.stream.current.type != "block_end": + if variables: + parser.stream.expect("comma") + + # skip colon for python compatibility + if parser.stream.skip_if("colon"): + break + + name = parser.stream.expect("name") + if name.value in variables: + parser.fail( + f"translatable variable {name.value!r} defined twice.", + name.lineno, + exc=TemplateAssertionError, + ) + + # expressions + if parser.stream.current.type == "assign": + next(parser.stream) + variables[name.value] = var = parser.parse_expression() + elif trimmed is None and name.value in ("trimmed", "notrimmed"): + trimmed = name.value == "trimmed" + continue + else: + variables[name.value] = var = nodes.Name(name.value, "load") + + if plural_expr is None: + if isinstance(var, nodes.Call): + plural_expr = nodes.Name("_trans", "load") + variables[name.value] = plural_expr + plural_expr_assignment = nodes.Assign( + nodes.Name("_trans", "store"), var + ) + else: + plural_expr = var + num_called_num = name.value == "num" + + parser.stream.expect("block_end") + + plural = None + have_plural = False + referenced = set() + + # now parse until endtrans or pluralize + singular_names, singular = self._parse_block(parser, True) + if singular_names: + referenced.update(singular_names) + if plural_expr is None: + plural_expr = nodes.Name(singular_names[0], "load") + num_called_num = singular_names[0] == "num" + + # if we have a pluralize block, we parse that too + if parser.stream.current.test("name:pluralize"): + have_plural = True + next(parser.stream) + if parser.stream.current.type != "block_end": + name = parser.stream.expect("name") + if name.value not in variables: + parser.fail( + f"unknown variable {name.value!r} for pluralization", + name.lineno, + exc=TemplateAssertionError, + ) + plural_expr = variables[name.value] + num_called_num = name.value == "num" + parser.stream.expect("block_end") + plural_names, plural = self._parse_block(parser, False) + next(parser.stream) + referenced.update(plural_names) + else: + next(parser.stream) + + # register free names as simple name expressions + for var in referenced: + if var not in variables: + variables[var] = nodes.Name(var, "load") + + if not have_plural: + plural_expr = None + elif plural_expr is None: + parser.fail("pluralize without variables", lineno) + + if trimmed is None: + trimmed = self.environment.policies["ext.i18n.trimmed"] + if trimmed: + singular = self._trim_whitespace(singular) + if plural: + plural = self._trim_whitespace(plural) + + node = self._make_node( + singular, + plural, + variables, + plural_expr, + bool(referenced), + num_called_num and have_plural, + ) + node.set_lineno(lineno) + if plural_expr_assignment is not None: + return [plural_expr_assignment, node] + else: + return node + + def _trim_whitespace(self, string, _ws_re=_ws_re): + return _ws_re.sub(" ", string.strip()) + + def _parse_block(self, parser, allow_pluralize): + """Parse until the next block tag with a given name.""" + referenced = [] + buf = [] + while 1: + if parser.stream.current.type == "data": + buf.append(parser.stream.current.value.replace("%", "%%")) + next(parser.stream) + elif parser.stream.current.type == "variable_begin": + next(parser.stream) + name = parser.stream.expect("name").value + referenced.append(name) + buf.append(f"%({name})s") + parser.stream.expect("variable_end") + elif parser.stream.current.type == "block_begin": + next(parser.stream) + if parser.stream.current.test("name:endtrans"): + break + elif parser.stream.current.test("name:pluralize"): + if allow_pluralize: + break + parser.fail( + "a translatable section can have only one pluralize section" + ) + parser.fail( + "control structures in translatable sections are not allowed" + ) + elif parser.stream.eos: + parser.fail("unclosed translation block") + else: + raise RuntimeError("internal parser error") + + return referenced, concat(buf) + + def _make_node( + self, singular, plural, variables, plural_expr, vars_referenced, num_called_num + ): + """Generates a useful node from the data provided.""" + # no variables referenced? no need to escape for old style + # gettext invocations only if there are vars. + if not vars_referenced and not self.environment.newstyle_gettext: + singular = singular.replace("%%", "%") + if plural: + plural = plural.replace("%%", "%") + + # singular only: + if plural_expr is None: + gettext = nodes.Name("gettext", "load") + node = nodes.Call(gettext, [nodes.Const(singular)], [], None, None) + + # singular and plural + else: + ngettext = nodes.Name("ngettext", "load") + node = nodes.Call( + ngettext, + [nodes.Const(singular), nodes.Const(plural), plural_expr], + [], + None, + None, + ) + + # in case newstyle gettext is used, the method is powerful + # enough to handle the variable expansion and autoescape + # handling itself + if self.environment.newstyle_gettext: + for key, value in variables.items(): + # the function adds that later anyways in case num was + # called num, so just skip it. + if num_called_num and key == "num": + continue + node.kwargs.append(nodes.Keyword(key, value)) + + # otherwise do that here + else: + # mark the return value as safe if we are in an + # environment with autoescaping turned on + node = nodes.MarkSafeIfAutoescape(node) + if variables: + node = nodes.Mod( + node, + nodes.Dict( + [ + nodes.Pair(nodes.Const(key), value) + for key, value in variables.items() + ] + ), + ) + return nodes.Output([node]) + + +class ExprStmtExtension(Extension): + """Adds a `do` tag to Jinja that works like the print statement just + that it doesn't print the return value. + """ + + tags = {"do"} + + def parse(self, parser): + node = nodes.ExprStmt(lineno=next(parser.stream).lineno) + node.node = parser.parse_tuple() + return node + + +class LoopControlExtension(Extension): + """Adds break and continue to the template engine.""" + + tags = {"break", "continue"} + + def parse(self, parser): + token = next(parser.stream) + if token.value == "break": + return nodes.Break(lineno=token.lineno) + return nodes.Continue(lineno=token.lineno) + + +class WithExtension(Extension): + pass + + +class AutoEscapeExtension(Extension): + pass + + +class DebugExtension(Extension): + """A ``{% debug %}`` tag that dumps the available variables, + filters, and tests. + + .. code-block:: html+jinja + + <pre>{% debug %}</pre> + + .. code-block:: text + + {'context': {'cycler': <class 'jinja2.utils.Cycler'>, + ..., + 'namespace': <class 'jinja2.utils.Namespace'>}, + 'filters': ['abs', 'attr', 'batch', 'capitalize', 'center', 'count', 'd', + ..., 'urlencode', 'urlize', 'wordcount', 'wordwrap', 'xmlattr'], + 'tests': ['!=', '<', '<=', '==', '>', '>=', 'callable', 'defined', + ..., 'odd', 'sameas', 'sequence', 'string', 'undefined', 'upper']} + + .. versionadded:: 2.11.0 + """ + + tags = {"debug"} + + def parse(self, parser): + lineno = parser.stream.expect("name:debug").lineno + context = ContextReference() + result = self.call_method("_render", [context], lineno=lineno) + return nodes.Output([result], lineno=lineno) + + def _render(self, context): + result = { + "context": context.get_all(), + "filters": sorted(self.environment.filters.keys()), + "tests": sorted(self.environment.tests.keys()), + } + + # Set the depth since the intent is to show the top few names. + if version_info[:2] >= (3, 4): + return pprint.pformat(result, depth=3, compact=True) + else: + return pprint.pformat(result, depth=3) + + +def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS, babel_style=True): + """Extract localizable strings from the given template node. Per + default this function returns matches in babel style that means non string + parameters as well as keyword arguments are returned as `None`. This + allows Babel to figure out what you really meant if you are using + gettext functions that allow keyword arguments for placeholder expansion. + If you don't want that behavior set the `babel_style` parameter to `False` + which causes only strings to be returned and parameters are always stored + in tuples. As a consequence invalid gettext calls (calls without a single + string parameter or string parameters after non-string parameters) are + skipped. + + This example explains the behavior: + + >>> from jinja2 import Environment + >>> env = Environment() + >>> node = env.parse('{{ (_("foo"), _(), ngettext("foo", "bar", 42)) }}') + >>> list(extract_from_ast(node)) + [(1, '_', 'foo'), (1, '_', ()), (1, 'ngettext', ('foo', 'bar', None))] + >>> list(extract_from_ast(node, babel_style=False)) + [(1, '_', ('foo',)), (1, 'ngettext', ('foo', 'bar'))] + + For every string found this function yields a ``(lineno, function, + message)`` tuple, where: + + * ``lineno`` is the number of the line on which the string was found, + * ``function`` is the name of the ``gettext`` function used (if the + string was extracted from embedded Python code), and + * ``message`` is the string, or a tuple of strings for functions + with multiple string arguments. + + This extraction function operates on the AST and is because of that unable + to extract any comments. For comment support you have to use the babel + extraction interface or extract comments yourself. + """ + for node in node.find_all(nodes.Call): + if ( + not isinstance(node.node, nodes.Name) + or node.node.name not in gettext_functions + ): + continue + + strings = [] + for arg in node.args: + if isinstance(arg, nodes.Const) and isinstance(arg.value, str): + strings.append(arg.value) + else: + strings.append(None) + + for _ in node.kwargs: + strings.append(None) + if node.dyn_args is not None: + strings.append(None) + if node.dyn_kwargs is not None: + strings.append(None) + + if not babel_style: + strings = tuple(x for x in strings if x is not None) + if not strings: + continue + else: + if len(strings) == 1: + strings = strings[0] + else: + strings = tuple(strings) + yield node.lineno, node.node.name, strings + + +class _CommentFinder: + """Helper class to find comments in a token stream. Can only + find comments for gettext calls forwards. Once the comment + from line 4 is found, a comment for line 1 will not return a + usable value. + """ + + def __init__(self, tokens, comment_tags): + self.tokens = tokens + self.comment_tags = comment_tags + self.offset = 0 + self.last_lineno = 0 + + def find_backwards(self, offset): + try: + for _, token_type, token_value in reversed( + self.tokens[self.offset : offset] + ): + if token_type in ("comment", "linecomment"): + try: + prefix, comment = token_value.split(None, 1) + except ValueError: + continue + if prefix in self.comment_tags: + return [comment.rstrip()] + return [] + finally: + self.offset = offset + + def find_comments(self, lineno): + if not self.comment_tags or self.last_lineno > lineno: + return [] + for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset :]): + if token_lineno > lineno: + return self.find_backwards(self.offset + idx) + return self.find_backwards(len(self.tokens)) + + +def babel_extract(fileobj, keywords, comment_tags, options): + """Babel extraction method for Jinja templates. + + .. versionchanged:: 2.3 + Basic support for translation comments was added. If `comment_tags` + is now set to a list of keywords for extraction, the extractor will + try to find the best preceding comment that begins with one of the + keywords. For best results, make sure to not have more than one + gettext call in one line of code and the matching comment in the + same line or the line before. + + .. versionchanged:: 2.5.1 + The `newstyle_gettext` flag can be set to `True` to enable newstyle + gettext calls. + + .. versionchanged:: 2.7 + A `silent` option can now be provided. If set to `False` template + syntax errors are propagated instead of being ignored. + + :param fileobj: the file-like object the messages should be extracted from + :param keywords: a list of keywords (i.e. function names) that should be + recognized as translation functions + :param comment_tags: a list of translator tags to search for and include + in the results. + :param options: a dictionary of additional options (optional) + :return: an iterator over ``(lineno, funcname, message, comments)`` tuples. + (comments will be empty currently) + """ + extensions = set() + for extension in options.get("extensions", "").split(","): + extension = extension.strip() + if not extension: + continue + extensions.add(import_string(extension)) + if InternationalizationExtension not in extensions: + extensions.add(InternationalizationExtension) + + def getbool(options, key, default=False): + return options.get(key, str(default)).lower() in ("1", "on", "yes", "true") + + silent = getbool(options, "silent", True) + environment = Environment( + options.get("block_start_string", BLOCK_START_STRING), + options.get("block_end_string", BLOCK_END_STRING), + options.get("variable_start_string", VARIABLE_START_STRING), + options.get("variable_end_string", VARIABLE_END_STRING), + options.get("comment_start_string", COMMENT_START_STRING), + options.get("comment_end_string", COMMENT_END_STRING), + options.get("line_statement_prefix") or LINE_STATEMENT_PREFIX, + options.get("line_comment_prefix") or LINE_COMMENT_PREFIX, + getbool(options, "trim_blocks", TRIM_BLOCKS), + getbool(options, "lstrip_blocks", LSTRIP_BLOCKS), + NEWLINE_SEQUENCE, + getbool(options, "keep_trailing_newline", KEEP_TRAILING_NEWLINE), + frozenset(extensions), + cache_size=0, + auto_reload=False, + ) + + if getbool(options, "trimmed"): + environment.policies["ext.i18n.trimmed"] = True + if getbool(options, "newstyle_gettext"): + environment.newstyle_gettext = True + + source = fileobj.read().decode(options.get("encoding", "utf-8")) + try: + node = environment.parse(source) + tokens = list(environment.lex(environment.preprocess(source))) + except TemplateSyntaxError: + if not silent: + raise + # skip templates with syntax errors + return + + finder = _CommentFinder(tokens, comment_tags) + for lineno, func, message in extract_from_ast(node, keywords): + yield lineno, func, message, finder.find_comments(lineno) + + +#: nicer import names +i18n = InternationalizationExtension +do = ExprStmtExtension +loopcontrols = LoopControlExtension +with_ = WithExtension +autoescape = AutoEscapeExtension +debug = DebugExtension diff --git a/src/jinja2/filters.py b/src/jinja2/filters.py new file mode 100644 index 00000000..c257d4c5 --- /dev/null +++ b/src/jinja2/filters.py @@ -0,0 +1,1361 @@ +"""Built-in template filters used with the ``|`` operator.""" +import math +import random +import re +from collections import abc +from collections import namedtuple +from itertools import chain +from itertools import groupby + +from markupsafe import escape +from markupsafe import Markup +from markupsafe import soft_str + +from .exceptions import FilterArgumentError +from .runtime import Undefined +from .utils import htmlsafe_json_dumps +from .utils import pformat +from .utils import url_quote +from .utils import urlize + +_word_re = re.compile(r"\w+") +_word_beginning_split_re = re.compile(r"([-\s({\[<]+)") + + +def contextfilter(f): + """Decorator for marking context dependent filters. The current + :class:`Context` will be passed as first argument. + """ + f.contextfilter = True + return f + + +def evalcontextfilter(f): + """Decorator for marking eval-context dependent filters. An eval + context object is passed as first argument. For more information + about the eval context, see :ref:`eval-context`. + + .. versionadded:: 2.4 + """ + f.evalcontextfilter = True + return f + + +def environmentfilter(f): + """Decorator for marking environment dependent filters. The current + :class:`Environment` is passed to the filter as first argument. + """ + f.environmentfilter = True + return f + + +def ignore_case(value): + """For use as a postprocessor for :func:`make_attrgetter`. Converts strings + to lowercase and returns other types as-is.""" + return value.lower() if isinstance(value, str) else value + + +def make_attrgetter(environment, attribute, postprocess=None, default=None): + """Returns a callable that looks up the given attribute from a + passed object with the rules of the environment. Dots are allowed + to access attributes of attributes. Integer parts in paths are + looked up as integers. + """ + attribute = _prepare_attribute_parts(attribute) + + def attrgetter(item): + for part in attribute: + item = environment.getitem(item, part) + + if default and isinstance(item, Undefined): + item = default + + if postprocess is not None: + item = postprocess(item) + + return item + + return attrgetter + + +def make_multi_attrgetter(environment, attribute, postprocess=None): + """Returns a callable that looks up the given comma separated + attributes from a passed object with the rules of the environment. + Dots are allowed to access attributes of each attribute. Integer + parts in paths are looked up as integers. + + The value returned by the returned callable is a list of extracted + attribute values. + + Examples of attribute: "attr1,attr2", "attr1.inner1.0,attr2.inner2.0", etc. + """ + attribute_parts = ( + attribute.split(",") if isinstance(attribute, str) else [attribute] + ) + attribute = [ + _prepare_attribute_parts(attribute_part) for attribute_part in attribute_parts + ] + + def attrgetter(item): + items = [None] * len(attribute) + for i, attribute_part in enumerate(attribute): + item_i = item + for part in attribute_part: + item_i = environment.getitem(item_i, part) + + if postprocess is not None: + item_i = postprocess(item_i) + + items[i] = item_i + return items + + return attrgetter + + +def _prepare_attribute_parts(attr): + if attr is None: + return [] + elif isinstance(attr, str): + return [int(x) if x.isdigit() else x for x in attr.split(".")] + else: + return [attr] + + +def do_forceescape(value): + """Enforce HTML escaping. This will probably double escape variables.""" + if hasattr(value, "__html__"): + value = value.__html__() + return escape(str(value)) + + +def do_urlencode(value): + """Quote data for use in a URL path or query using UTF-8. + + Basic wrapper around :func:`urllib.parse.quote` when given a + string, or :func:`urllib.parse.urlencode` for a dict or iterable. + + :param value: Data to quote. A string will be quoted directly. A + dict or iterable of ``(key, value)`` pairs will be joined as a + query string. + + When given a string, "/" is not quoted. HTTP servers treat "/" and + "%2F" equivalently in paths. If you need quoted slashes, use the + ``|replace("/", "%2F")`` filter. + + .. versionadded:: 2.7 + """ + if isinstance(value, str) or not isinstance(value, abc.Iterable): + return url_quote(value) + + if isinstance(value, dict): + items = value.items() + else: + items = iter(value) + + return "&".join( + f"{url_quote(k, for_qs=True)}={url_quote(v, for_qs=True)}" for k, v in items + ) + + +@evalcontextfilter +def do_replace(eval_ctx, s, old, new, count=None): + """Return a copy of the value with all occurrences of a substring + replaced with a new one. The first argument is the substring + that should be replaced, the second is the replacement string. + If the optional third argument ``count`` is given, only the first + ``count`` occurrences are replaced: + + .. sourcecode:: jinja + + {{ "Hello World"|replace("Hello", "Goodbye") }} + -> Goodbye World + + {{ "aaaaargh"|replace("a", "d'oh, ", 2) }} + -> d'oh, d'oh, aaargh + """ + if count is None: + count = -1 + if not eval_ctx.autoescape: + return str(s).replace(str(old), str(new), count) + if ( + hasattr(old, "__html__") + or hasattr(new, "__html__") + and not hasattr(s, "__html__") + ): + s = escape(s) + else: + s = soft_str(s) + return s.replace(soft_str(old), soft_str(new), count) + + +def do_upper(s): + """Convert a value to uppercase.""" + return soft_str(s).upper() + + +def do_lower(s): + """Convert a value to lowercase.""" + return soft_str(s).lower() + + +@evalcontextfilter +def do_xmlattr(_eval_ctx, d, autospace=True): + """Create an SGML/XML attribute string based on the items in a dict. + All values that are neither `none` nor `undefined` are automatically + escaped: + + .. sourcecode:: html+jinja + + <ul{{ {'class': 'my_list', 'missing': none, + 'id': 'list-%d'|format(variable)}|xmlattr }}> + ... + </ul> + + Results in something like this: + + .. sourcecode:: html + + <ul class="my_list" id="list-42"> + ... + </ul> + + As you can see it automatically prepends a space in front of the item + if the filter returned something unless the second parameter is false. + """ + rv = " ".join( + f'{escape(key)}="{escape(value)}"' + for key, value in d.items() + if value is not None and not isinstance(value, Undefined) + ) + if autospace and rv: + rv = " " + rv + if _eval_ctx.autoescape: + rv = Markup(rv) + return rv + + +def do_capitalize(s): + """Capitalize a value. The first character will be uppercase, all others + lowercase. + """ + return soft_str(s).capitalize() + + +def do_title(s): + """Return a titlecased version of the value. I.e. words will start with + uppercase letters, all remaining characters are lowercase. + """ + return "".join( + [ + item[0].upper() + item[1:].lower() + for item in _word_beginning_split_re.split(soft_str(s)) + if item + ] + ) + + +def do_dictsort(value, case_sensitive=False, by="key", reverse=False): + """Sort a dict and yield (key, value) pairs. Because python dicts are + unsorted you may want to use this function to order them by either + key or value: + + .. sourcecode:: jinja + + {% for item in mydict|dictsort %} + sort the dict by key, case insensitive + + {% for item in mydict|dictsort(reverse=true) %} + sort the dict by key, case insensitive, reverse order + + {% for item in mydict|dictsort(true) %} + sort the dict by key, case sensitive + + {% for item in mydict|dictsort(false, 'value') %} + sort the dict by value, case insensitive + """ + if by == "key": + pos = 0 + elif by == "value": + pos = 1 + else: + raise FilterArgumentError('You can only sort by either "key" or "value"') + + def sort_func(item): + value = item[pos] + + if not case_sensitive: + value = ignore_case(value) + + return value + + return sorted(value.items(), key=sort_func, reverse=reverse) + + +@environmentfilter +def do_sort(environment, value, reverse=False, case_sensitive=False, attribute=None): + """Sort an iterable using Python's :func:`sorted`. + + .. sourcecode:: jinja + + {% for city in cities|sort %} + ... + {% endfor %} + + :param reverse: Sort descending instead of ascending. + :param case_sensitive: When sorting strings, sort upper and lower + case separately. + :param attribute: When sorting objects or dicts, an attribute or + key to sort by. Can use dot notation like ``"address.city"``. + Can be a list of attributes like ``"age,name"``. + + The sort is stable, it does not change the relative order of + elements that compare equal. This makes it is possible to chain + sorts on different attributes and ordering. + + .. sourcecode:: jinja + + {% for user in users|sort(attribute="name") + |sort(reverse=true, attribute="age") %} + ... + {% endfor %} + + As a shortcut to chaining when the direction is the same for all + attributes, pass a comma separate list of attributes. + + .. sourcecode:: jinja + + {% for user users|sort(attribute="age,name") %} + ... + {% endfor %} + + .. versionchanged:: 2.11.0 + The ``attribute`` parameter can be a comma separated list of + attributes, e.g. ``"age,name"``. + + .. versionchanged:: 2.6 + The ``attribute`` parameter was added. + """ + key_func = make_multi_attrgetter( + environment, attribute, postprocess=ignore_case if not case_sensitive else None + ) + return sorted(value, key=key_func, reverse=reverse) + + +@environmentfilter +def do_unique(environment, value, case_sensitive=False, attribute=None): + """Returns a list of unique items from the given iterable. + + .. sourcecode:: jinja + + {{ ['foo', 'bar', 'foobar', 'FooBar']|unique|list }} + -> ['foo', 'bar', 'foobar'] + + The unique items are yielded in the same order as their first occurrence in + the iterable passed to the filter. + + :param case_sensitive: Treat upper and lower case strings as distinct. + :param attribute: Filter objects with unique values for this attribute. + """ + getter = make_attrgetter( + environment, attribute, postprocess=ignore_case if not case_sensitive else None + ) + seen = set() + + for item in value: + key = getter(item) + + if key not in seen: + seen.add(key) + yield item + + +def _min_or_max(environment, value, func, case_sensitive, attribute): + it = iter(value) + + try: + first = next(it) + except StopIteration: + return environment.undefined("No aggregated item, sequence was empty.") + + key_func = make_attrgetter( + environment, attribute, postprocess=ignore_case if not case_sensitive else None + ) + return func(chain([first], it), key=key_func) + + +@environmentfilter +def do_min(environment, value, case_sensitive=False, attribute=None): + """Return the smallest item from the sequence. + + .. sourcecode:: jinja + + {{ [1, 2, 3]|min }} + -> 1 + + :param case_sensitive: Treat upper and lower case strings as distinct. + :param attribute: Get the object with the min value of this attribute. + """ + return _min_or_max(environment, value, min, case_sensitive, attribute) + + +@environmentfilter +def do_max(environment, value, case_sensitive=False, attribute=None): + """Return the largest item from the sequence. + + .. sourcecode:: jinja + + {{ [1, 2, 3]|max }} + -> 3 + + :param case_sensitive: Treat upper and lower case strings as distinct. + :param attribute: Get the object with the max value of this attribute. + """ + return _min_or_max(environment, value, max, case_sensitive, attribute) + + +def do_default(value, default_value="", boolean=False): + """If the value is undefined it will return the passed default value, + otherwise the value of the variable: + + .. sourcecode:: jinja + + {{ my_variable|default('my_variable is not defined') }} + + This will output the value of ``my_variable`` if the variable was + defined, otherwise ``'my_variable is not defined'``. If you want + to use default with variables that evaluate to false you have to + set the second parameter to `true`: + + .. sourcecode:: jinja + + {{ ''|default('the string was empty', true) }} + + .. versionchanged:: 2.11 + It's now possible to configure the :class:`~jinja2.Environment` with + :class:`~jinja2.ChainableUndefined` to make the `default` filter work + on nested elements and attributes that may contain undefined values + in the chain without getting an :exc:`~jinja2.UndefinedError`. + """ + if isinstance(value, Undefined) or (boolean and not value): + return default_value + return value + + +@evalcontextfilter +def do_join(eval_ctx, value, d="", attribute=None): + """Return a string which is the concatenation of the strings in the + sequence. The separator between elements is an empty string per + default, you can define it with the optional parameter: + + .. sourcecode:: jinja + + {{ [1, 2, 3]|join('|') }} + -> 1|2|3 + + {{ [1, 2, 3]|join }} + -> 123 + + It is also possible to join certain attributes of an object: + + .. sourcecode:: jinja + + {{ users|join(', ', attribute='username') }} + + .. versionadded:: 2.6 + The `attribute` parameter was added. + """ + if attribute is not None: + value = map(make_attrgetter(eval_ctx.environment, attribute), value) + + # no automatic escaping? joining is a lot easier then + if not eval_ctx.autoescape: + return str(d).join(map(str, value)) + + # if the delimiter doesn't have an html representation we check + # if any of the items has. If yes we do a coercion to Markup + if not hasattr(d, "__html__"): + value = list(value) + do_escape = False + for idx, item in enumerate(value): + if hasattr(item, "__html__"): + do_escape = True + else: + value[idx] = str(item) + if do_escape: + d = escape(d) + else: + d = str(d) + return d.join(value) + + # no html involved, to normal joining + return soft_str(d).join(map(soft_str, value)) + + +def do_center(value, width=80): + """Centers the value in a field of a given width.""" + return str(value).center(width) + + +@environmentfilter +def do_first(environment, seq): + """Return the first item of a sequence.""" + try: + return next(iter(seq)) + except StopIteration: + return environment.undefined("No first item, sequence was empty.") + + +@environmentfilter +def do_last(environment, seq): + """ + Return the last item of a sequence. + + Note: Does not work with generators. You may want to explicitly + convert it to a list: + + .. sourcecode:: jinja + + {{ data | selectattr('name', '==', 'Jinja') | list | last }} + """ + try: + return next(iter(reversed(seq))) + except StopIteration: + return environment.undefined("No last item, sequence was empty.") + + +@contextfilter +def do_random(context, seq): + """Return a random item from the sequence.""" + try: + return random.choice(seq) + except IndexError: + return context.environment.undefined("No random item, sequence was empty.") + + +def do_filesizeformat(value, binary=False): + """Format the value like a 'human-readable' file size (i.e. 13 kB, + 4.1 MB, 102 Bytes, etc). Per default decimal prefixes are used (Mega, + Giga, etc.), if the second parameter is set to `True` the binary + prefixes are used (Mebi, Gibi). + """ + bytes = float(value) + base = 1024 if binary else 1000 + prefixes = [ + ("KiB" if binary else "kB"), + ("MiB" if binary else "MB"), + ("GiB" if binary else "GB"), + ("TiB" if binary else "TB"), + ("PiB" if binary else "PB"), + ("EiB" if binary else "EB"), + ("ZiB" if binary else "ZB"), + ("YiB" if binary else "YB"), + ] + if bytes == 1: + return "1 Byte" + elif bytes < base: + return f"{int(bytes)} Bytes" + else: + for i, prefix in enumerate(prefixes): + unit = base ** (i + 2) + if bytes < unit: + return f"{base * bytes / unit:.1f} {prefix}" + return f"{base * bytes / unit:.1f} {prefix}" + + +def do_pprint(value): + """Pretty print a variable. Useful for debugging.""" + return pformat(value) + + +@evalcontextfilter +def do_urlize( + eval_ctx, value, trim_url_limit=None, nofollow=False, target=None, rel=None +): + """Converts URLs in plain text into clickable links. + + If you pass the filter an additional integer it will shorten the urls + to that number. Also a third argument exists that makes the urls + "nofollow": + + .. sourcecode:: jinja + + {{ mytext|urlize(40, true) }} + links are shortened to 40 chars and defined with rel="nofollow" + + If *target* is specified, the ``target`` attribute will be added to the + ``<a>`` tag: + + .. sourcecode:: jinja + + {{ mytext|urlize(40, target='_blank') }} + + .. versionchanged:: 2.8 + The ``target`` parameter was added. + """ + policies = eval_ctx.environment.policies + rel = set((rel or "").split() or []) + if nofollow: + rel.add("nofollow") + rel.update((policies["urlize.rel"] or "").split()) + if target is None: + target = policies["urlize.target"] + rel = " ".join(sorted(rel)) or None + rv = urlize(value, trim_url_limit, rel=rel, target=target) + if eval_ctx.autoescape: + rv = Markup(rv) + return rv + + +def do_indent(s, width=4, first=False, blank=False): + """Return a copy of the string with each line indented by 4 spaces. The + first line and blank lines are not indented by default. + + :param width: Number of spaces to indent by. + :param first: Don't skip indenting the first line. + :param blank: Don't skip indenting empty lines. + + .. versionchanged:: 2.10 + Blank lines are not indented by default. + + Rename the ``indentfirst`` argument to ``first``. + """ + indention = " " * width + newline = "\n" + + if isinstance(s, Markup): + indention = Markup(indention) + newline = Markup(newline) + + s += newline # this quirk is necessary for splitlines method + + if blank: + rv = (newline + indention).join(s.splitlines()) + else: + lines = s.splitlines() + rv = lines.pop(0) + + if lines: + rv += newline + newline.join( + indention + line if line else line for line in lines + ) + + if first: + rv = indention + rv + + return rv + + +@environmentfilter +def do_truncate(env, s, length=255, killwords=False, end="...", leeway=None): + """Return a truncated copy of the string. The length is specified + with the first parameter which defaults to ``255``. If the second + parameter is ``true`` the filter will cut the text at length. Otherwise + it will discard the last word. If the text was in fact + truncated it will append an ellipsis sign (``"..."``). If you want a + different ellipsis sign than ``"..."`` you can specify it using the + third parameter. Strings that only exceed the length by the tolerance + margin given in the fourth parameter will not be truncated. + + .. sourcecode:: jinja + + {{ "foo bar baz qux"|truncate(9) }} + -> "foo..." + {{ "foo bar baz qux"|truncate(9, True) }} + -> "foo ba..." + {{ "foo bar baz qux"|truncate(11) }} + -> "foo bar baz qux" + {{ "foo bar baz qux"|truncate(11, False, '...', 0) }} + -> "foo bar..." + + The default leeway on newer Jinja versions is 5 and was 0 before but + can be reconfigured globally. + """ + if leeway is None: + leeway = env.policies["truncate.leeway"] + assert length >= len(end), f"expected length >= {len(end)}, got {length}" + assert leeway >= 0, f"expected leeway >= 0, got {leeway}" + if len(s) <= length + leeway: + return s + if killwords: + return s[: length - len(end)] + end + result = s[: length - len(end)].rsplit(" ", 1)[0] + return result + end + + +@environmentfilter +def do_wordwrap( + environment, + s, + width=79, + break_long_words=True, + wrapstring=None, + break_on_hyphens=True, +): + """Wrap a string to the given width. Existing newlines are treated + as paragraphs to be wrapped separately. + + :param s: Original text to wrap. + :param width: Maximum length of wrapped lines. + :param break_long_words: If a word is longer than ``width``, break + it across lines. + :param break_on_hyphens: If a word contains hyphens, it may be split + across lines. + :param wrapstring: String to join each wrapped line. Defaults to + :attr:`Environment.newline_sequence`. + + .. versionchanged:: 2.11 + Existing newlines are treated as paragraphs wrapped separately. + + .. versionchanged:: 2.11 + Added the ``break_on_hyphens`` parameter. + + .. versionchanged:: 2.7 + Added the ``wrapstring`` parameter. + """ + + import textwrap + + if not wrapstring: + wrapstring = environment.newline_sequence + + # textwrap.wrap doesn't consider existing newlines when wrapping. + # If the string has a newline before width, wrap will still insert + # a newline at width, resulting in a short line. Instead, split and + # wrap each paragraph individually. + return wrapstring.join( + [ + wrapstring.join( + textwrap.wrap( + line, + width=width, + expand_tabs=False, + replace_whitespace=False, + break_long_words=break_long_words, + break_on_hyphens=break_on_hyphens, + ) + ) + for line in s.splitlines() + ] + ) + + +def do_wordcount(s): + """Count the words in that string.""" + return len(_word_re.findall(soft_str(s))) + + +def do_int(value, default=0, base=10): + """Convert the value into an integer. If the + conversion doesn't work it will return ``0``. You can + override this default using the first parameter. You + can also override the default base (10) in the second + parameter, which handles input with prefixes such as + 0b, 0o and 0x for bases 2, 8 and 16 respectively. + The base is ignored for decimal numbers and non-string values. + """ + try: + if isinstance(value, str): + return int(value, base) + return int(value) + except (TypeError, ValueError): + # this quirk is necessary so that "42.23"|int gives 42. + try: + return int(float(value)) + except (TypeError, ValueError): + return default + + +def do_float(value, default=0.0): + """Convert the value into a floating point number. If the + conversion doesn't work it will return ``0.0``. You can + override this default using the first parameter. + """ + try: + return float(value) + except (TypeError, ValueError): + return default + + +def do_format(value, *args, **kwargs): + """Apply the given values to a `printf-style`_ format string, like + ``string % values``. + + .. sourcecode:: jinja + + {{ "%s, %s!"|format(greeting, name) }} + Hello, World! + + In most cases it should be more convenient and efficient to use the + ``%`` operator or :meth:`str.format`. + + .. code-block:: text + + {{ "%s, %s!" % (greeting, name) }} + {{ "{}, {}!".format(greeting, name) }} + + .. _printf-style: https://docs.python.org/library/stdtypes.html + #printf-style-string-formatting + """ + if args and kwargs: + raise FilterArgumentError( + "can't handle positional and keyword arguments at the same time" + ) + return soft_str(value) % (kwargs or args) + + +def do_trim(value, chars=None): + """Strip leading and trailing characters, by default whitespace.""" + return soft_str(value).strip(chars) + + +def do_striptags(value): + """Strip SGML/XML tags and replace adjacent whitespace by one space.""" + if hasattr(value, "__html__"): + value = value.__html__() + return Markup(str(value)).striptags() + + +def do_slice(value, slices, fill_with=None): + """Slice an iterator and return a list of lists containing + those items. Useful if you want to create a div containing + three ul tags that represent columns: + + .. sourcecode:: html+jinja + + <div class="columnwrapper"> + {%- for column in items|slice(3) %} + <ul class="column-{{ loop.index }}"> + {%- for item in column %} + <li>{{ item }}</li> + {%- endfor %} + </ul> + {%- endfor %} + </div> + + If you pass it a second argument it's used to fill missing + values on the last iteration. + """ + seq = list(value) + length = len(seq) + items_per_slice = length // slices + slices_with_extra = length % slices + offset = 0 + for slice_number in range(slices): + start = offset + slice_number * items_per_slice + if slice_number < slices_with_extra: + offset += 1 + end = offset + (slice_number + 1) * items_per_slice + tmp = seq[start:end] + if fill_with is not None and slice_number >= slices_with_extra: + tmp.append(fill_with) + yield tmp + + +def do_batch(value, linecount, fill_with=None): + """ + A filter that batches items. It works pretty much like `slice` + just the other way round. It returns a list of lists with the + given number of items. If you provide a second parameter this + is used to fill up missing items. See this example: + + .. sourcecode:: html+jinja + + <table> + {%- for row in items|batch(3, ' ') %} + <tr> + {%- for column in row %} + <td>{{ column }}</td> + {%- endfor %} + </tr> + {%- endfor %} + </table> + """ + tmp = [] + for item in value: + if len(tmp) == linecount: + yield tmp + tmp = [] + tmp.append(item) + if tmp: + if fill_with is not None and len(tmp) < linecount: + tmp += [fill_with] * (linecount - len(tmp)) + yield tmp + + +def do_round(value, precision=0, method="common"): + """Round the number to a given precision. The first + parameter specifies the precision (default is ``0``), the + second the rounding method: + + - ``'common'`` rounds either up or down + - ``'ceil'`` always rounds up + - ``'floor'`` always rounds down + + If you don't specify a method ``'common'`` is used. + + .. sourcecode:: jinja + + {{ 42.55|round }} + -> 43.0 + {{ 42.55|round(1, 'floor') }} + -> 42.5 + + Note that even if rounded to 0 precision, a float is returned. If + you need a real integer, pipe it through `int`: + + .. sourcecode:: jinja + + {{ 42.55|round|int }} + -> 43 + """ + if method not in {"common", "ceil", "floor"}: + raise FilterArgumentError("method must be common, ceil or floor") + if method == "common": + return round(value, precision) + func = getattr(math, method) + return func(value * (10 ** precision)) / (10 ** precision) + + +# Use a regular tuple repr here. This is what we did in the past and we +# really want to hide this custom type as much as possible. In particular +# we do not want to accidentally expose an auto generated repr in case +# people start to print this out in comments or something similar for +# debugging. +_GroupTuple = namedtuple("_GroupTuple", ["grouper", "list"]) +_GroupTuple.__repr__ = tuple.__repr__ +_GroupTuple.__str__ = tuple.__str__ + + +@environmentfilter +def do_groupby(environment, value, attribute): + """Group a sequence of objects by an attribute using Python's + :func:`itertools.groupby`. The attribute can use dot notation for + nested access, like ``"address.city"``. Unlike Python's ``groupby``, + the values are sorted first so only one group is returned for each + unique value. + + For example, a list of ``User`` objects with a ``city`` attribute + can be rendered in groups. In this example, ``grouper`` refers to + the ``city`` value of the group. + + .. sourcecode:: html+jinja + + <ul>{% for city, items in users|groupby("city") %} + <li>{{ city }} + <ul>{% for user in items %} + <li>{{ user.name }} + {% endfor %}</ul> + </li> + {% endfor %}</ul> + + ``groupby`` yields namedtuples of ``(grouper, list)``, which + can be used instead of the tuple unpacking above. ``grouper`` is the + value of the attribute, and ``list`` is the items with that value. + + .. sourcecode:: html+jinja + + <ul>{% for group in users|groupby("city") %} + <li>{{ group.grouper }}: {{ group.list|join(", ") }} + {% endfor %}</ul> + + .. versionchanged:: 2.6 + The attribute supports dot notation for nested access. + """ + expr = make_attrgetter(environment, attribute) + return [ + _GroupTuple(key, list(values)) + for key, values in groupby(sorted(value, key=expr), expr) + ] + + +@environmentfilter +def do_sum(environment, iterable, attribute=None, start=0): + """Returns the sum of a sequence of numbers plus the value of parameter + 'start' (which defaults to 0). When the sequence is empty it returns + start. + + It is also possible to sum up only certain attributes: + + .. sourcecode:: jinja + + Total: {{ items|sum(attribute='price') }} + + .. versionchanged:: 2.6 + The `attribute` parameter was added to allow suming up over + attributes. Also the `start` parameter was moved on to the right. + """ + if attribute is not None: + iterable = map(make_attrgetter(environment, attribute), iterable) + return sum(iterable, start) + + +def do_list(value): + """Convert the value into a list. If it was a string the returned list + will be a list of characters. + """ + return list(value) + + +def do_mark_safe(value): + """Mark the value as safe which means that in an environment with automatic + escaping enabled this variable will not be escaped. + """ + return Markup(value) + + +def do_mark_unsafe(value): + """Mark a value as unsafe. This is the reverse operation for :func:`safe`.""" + return str(value) + + +def do_reverse(value): + """Reverse the object or return an iterator that iterates over it the other + way round. + """ + if isinstance(value, str): + return value[::-1] + try: + return reversed(value) + except TypeError: + try: + rv = list(value) + rv.reverse() + return rv + except TypeError: + raise FilterArgumentError("argument must be iterable") + + +@environmentfilter +def do_attr(environment, obj, name): + """Get an attribute of an object. ``foo|attr("bar")`` works like + ``foo.bar`` just that always an attribute is returned and items are not + looked up. + + See :ref:`Notes on subscriptions <notes-on-subscriptions>` for more details. + """ + try: + name = str(name) + except UnicodeError: + pass + else: + try: + value = getattr(obj, name) + except AttributeError: + pass + else: + if environment.sandboxed and not environment.is_safe_attribute( + obj, name, value + ): + return environment.unsafe_undefined(obj, name) + return value + return environment.undefined(obj=obj, name=name) + + +@contextfilter +def do_map(*args, **kwargs): + """Applies a filter on a sequence of objects or looks up an attribute. + This is useful when dealing with lists of objects but you are really + only interested in a certain value of it. + + The basic usage is mapping on an attribute. Imagine you have a list + of users but you are only interested in a list of usernames: + + .. sourcecode:: jinja + + Users on this page: {{ users|map(attribute='username')|join(', ') }} + + You can specify a ``default`` value to use if an object in the list + does not have the given attribute. + + .. sourcecode:: jinja + + {{ users|map(attribute="username", default="Anonymous")|join(", ") }} + + Alternatively you can let it invoke a filter by passing the name of the + filter and the arguments afterwards. A good example would be applying a + text conversion filter on a sequence: + + .. sourcecode:: jinja + + Users on this page: {{ titles|map('lower')|join(', ') }} + + Similar to a generator comprehension such as: + + .. code-block:: python + + (u.username for u in users) + (u.username or "Anonymous" for u in users) + (do_lower(x) for x in titles) + + .. versionchanged:: 2.11.0 + Added the ``default`` parameter. + + .. versionadded:: 2.7 + """ + seq, func = prepare_map(args, kwargs) + if seq: + for item in seq: + yield func(item) + + +@contextfilter +def do_select(*args, **kwargs): + """Filters a sequence of objects by applying a test to each object, + and only selecting the objects with the test succeeding. + + If no test is specified, each object will be evaluated as a boolean. + + Example usage: + + .. sourcecode:: jinja + + {{ numbers|select("odd") }} + {{ numbers|select("odd") }} + {{ numbers|select("divisibleby", 3) }} + {{ numbers|select("lessthan", 42) }} + {{ strings|select("equalto", "mystring") }} + + Similar to a generator comprehension such as: + + .. code-block:: python + + (n for n in numbers if test_odd(n)) + (n for n in numbers if test_divisibleby(n, 3)) + + .. versionadded:: 2.7 + """ + return select_or_reject(args, kwargs, lambda x: x, False) + + +@contextfilter +def do_reject(*args, **kwargs): + """Filters a sequence of objects by applying a test to each object, + and rejecting the objects with the test succeeding. + + If no test is specified, each object will be evaluated as a boolean. + + Example usage: + + .. sourcecode:: jinja + + {{ numbers|reject("odd") }} + + Similar to a generator comprehension such as: + + .. code-block:: python + + (n for n in numbers if not test_odd(n)) + + .. versionadded:: 2.7 + """ + return select_or_reject(args, kwargs, lambda x: not x, False) + + +@contextfilter +def do_selectattr(*args, **kwargs): + """Filters a sequence of objects by applying a test to the specified + attribute of each object, and only selecting the objects with the + test succeeding. + + If no test is specified, the attribute's value will be evaluated as + a boolean. + + Example usage: + + .. sourcecode:: jinja + + {{ users|selectattr("is_active") }} + {{ users|selectattr("email", "none") }} + + Similar to a generator comprehension such as: + + .. code-block:: python + + (u for user in users if user.is_active) + (u for user in users if test_none(user.email)) + + .. versionadded:: 2.7 + """ + return select_or_reject(args, kwargs, lambda x: x, True) + + +@contextfilter +def do_rejectattr(*args, **kwargs): + """Filters a sequence of objects by applying a test to the specified + attribute of each object, and rejecting the objects with the test + succeeding. + + If no test is specified, the attribute's value will be evaluated as + a boolean. + + .. sourcecode:: jinja + + {{ users|rejectattr("is_active") }} + {{ users|rejectattr("email", "none") }} + + Similar to a generator comprehension such as: + + .. code-block:: python + + (u for user in users if not user.is_active) + (u for user in users if not test_none(user.email)) + + .. versionadded:: 2.7 + """ + return select_or_reject(args, kwargs, lambda x: not x, True) + + +@evalcontextfilter +def do_tojson(eval_ctx, value, indent=None): + """Dumps a structure to JSON so that it's safe to use in ``<script>`` + tags. It accepts the same arguments and returns a JSON string. Note that + this is available in templates through the ``|tojson`` filter which will + also mark the result as safe. Due to how this function escapes certain + characters this is safe even if used outside of ``<script>`` tags. + + The following characters are escaped in strings: + + - ``<`` + - ``>`` + - ``&`` + - ``'`` + + This makes it safe to embed such strings in any place in HTML with the + notable exception of double quoted attributes. In that case single + quote your attributes or HTML escape it in addition. + + The indent parameter can be used to enable pretty printing. Set it to + the number of spaces that the structures should be indented with. + + Note that this filter is for use in HTML contexts only. + + .. versionadded:: 2.9 + """ + policies = eval_ctx.environment.policies + dumper = policies["json.dumps_function"] + options = policies["json.dumps_kwargs"] + if indent is not None: + options = dict(options) + options["indent"] = indent + return htmlsafe_json_dumps(value, dumper=dumper, **options) + + +def prepare_map(args, kwargs): + context = args[0] + seq = args[1] + + if len(args) == 2 and "attribute" in kwargs: + attribute = kwargs.pop("attribute") + default = kwargs.pop("default", None) + if kwargs: + raise FilterArgumentError( + f"Unexpected keyword argument {next(iter(kwargs))!r}" + ) + func = make_attrgetter(context.environment, attribute, default=default) + else: + try: + name = args[2] + args = args[3:] + except LookupError: + raise FilterArgumentError("map requires a filter argument") + + def func(item): + return context.environment.call_filter( + name, item, args, kwargs, context=context + ) + + return seq, func + + +def prepare_select_or_reject(args, kwargs, modfunc, lookup_attr): + context = args[0] + seq = args[1] + if lookup_attr: + try: + attr = args[2] + except LookupError: + raise FilterArgumentError("Missing parameter for attribute name") + transfunc = make_attrgetter(context.environment, attr) + off = 1 + else: + off = 0 + + def transfunc(x): + return x + + try: + name = args[2 + off] + args = args[3 + off :] + + def func(item): + return context.environment.call_test(name, item, args, kwargs) + + except LookupError: + func = bool + + return seq, lambda item: modfunc(func(transfunc(item))) + + +def select_or_reject(args, kwargs, modfunc, lookup_attr): + seq, func = prepare_select_or_reject(args, kwargs, modfunc, lookup_attr) + if seq: + for item in seq: + if func(item): + yield item + + +FILTERS = { + "abs": abs, + "attr": do_attr, + "batch": do_batch, + "capitalize": do_capitalize, + "center": do_center, + "count": len, + "d": do_default, + "default": do_default, + "dictsort": do_dictsort, + "e": escape, + "escape": escape, + "filesizeformat": do_filesizeformat, + "first": do_first, + "float": do_float, + "forceescape": do_forceescape, + "format": do_format, + "groupby": do_groupby, + "indent": do_indent, + "int": do_int, + "join": do_join, + "last": do_last, + "length": len, + "list": do_list, + "lower": do_lower, + "map": do_map, + "min": do_min, + "max": do_max, + "pprint": do_pprint, + "random": do_random, + "reject": do_reject, + "rejectattr": do_rejectattr, + "replace": do_replace, + "reverse": do_reverse, + "round": do_round, + "safe": do_mark_safe, + "select": do_select, + "selectattr": do_selectattr, + "slice": do_slice, + "sort": do_sort, + "string": soft_str, + "striptags": do_striptags, + "sum": do_sum, + "title": do_title, + "trim": do_trim, + "truncate": do_truncate, + "unique": do_unique, + "upper": do_upper, + "urlencode": do_urlencode, + "urlize": do_urlize, + "wordcount": do_wordcount, + "wordwrap": do_wordwrap, + "xmlattr": do_xmlattr, + "tojson": do_tojson, +} diff --git a/src/jinja2/idtracking.py b/src/jinja2/idtracking.py new file mode 100644 index 00000000..78cad916 --- /dev/null +++ b/src/jinja2/idtracking.py @@ -0,0 +1,289 @@ +from .visitor import NodeVisitor + +VAR_LOAD_PARAMETER = "param" +VAR_LOAD_RESOLVE = "resolve" +VAR_LOAD_ALIAS = "alias" +VAR_LOAD_UNDEFINED = "undefined" + + +def find_symbols(nodes, parent_symbols=None): + sym = Symbols(parent=parent_symbols) + visitor = FrameSymbolVisitor(sym) + for node in nodes: + visitor.visit(node) + return sym + + +def symbols_for_node(node, parent_symbols=None): + sym = Symbols(parent=parent_symbols) + sym.analyze_node(node) + return sym + + +class Symbols: + def __init__(self, parent=None, level=None): + if level is None: + if parent is None: + level = 0 + else: + level = parent.level + 1 + self.level = level + self.parent = parent + self.refs = {} + self.loads = {} + self.stores = set() + + def analyze_node(self, node, **kwargs): + visitor = RootVisitor(self) + visitor.visit(node, **kwargs) + + def _define_ref(self, name, load=None): + ident = f"l_{self.level}_{name}" + self.refs[name] = ident + if load is not None: + self.loads[ident] = load + return ident + + def find_load(self, target): + if target in self.loads: + return self.loads[target] + if self.parent is not None: + return self.parent.find_load(target) + + def find_ref(self, name): + if name in self.refs: + return self.refs[name] + if self.parent is not None: + return self.parent.find_ref(name) + + def ref(self, name): + rv = self.find_ref(name) + if rv is None: + raise AssertionError( + "Tried to resolve a name to a reference that was" + f" unknown to the frame ({name!r})" + ) + return rv + + def copy(self): + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.refs = self.refs.copy() + rv.loads = self.loads.copy() + rv.stores = self.stores.copy() + return rv + + def store(self, name): + self.stores.add(name) + + # If we have not see the name referenced yet, we need to figure + # out what to set it to. + if name not in self.refs: + # If there is a parent scope we check if the name has a + # reference there. If it does it means we might have to alias + # to a variable there. + if self.parent is not None: + outer_ref = self.parent.find_ref(name) + if outer_ref is not None: + self._define_ref(name, load=(VAR_LOAD_ALIAS, outer_ref)) + return + + # Otherwise we can just set it to undefined. + self._define_ref(name, load=(VAR_LOAD_UNDEFINED, None)) + + def declare_parameter(self, name): + self.stores.add(name) + return self._define_ref(name, load=(VAR_LOAD_PARAMETER, None)) + + def load(self, name): + target = self.find_ref(name) + if target is None: + self._define_ref(name, load=(VAR_LOAD_RESOLVE, name)) + + def branch_update(self, branch_symbols): + stores = {} + for branch in branch_symbols: + for target in branch.stores: + if target in self.stores: + continue + stores[target] = stores.get(target, 0) + 1 + + for sym in branch_symbols: + self.refs.update(sym.refs) + self.loads.update(sym.loads) + self.stores.update(sym.stores) + + for name, branch_count in stores.items(): + if branch_count == len(branch_symbols): + continue + target = self.find_ref(name) + assert target is not None, "should not happen" + + if self.parent is not None: + outer_target = self.parent.find_ref(name) + if outer_target is not None: + self.loads[target] = (VAR_LOAD_ALIAS, outer_target) + continue + self.loads[target] = (VAR_LOAD_RESOLVE, name) + + def dump_stores(self): + rv = {} + node = self + while node is not None: + for name in node.stores: + if name not in rv: + rv[name] = self.find_ref(name) + node = node.parent + return rv + + def dump_param_targets(self): + rv = set() + node = self + while node is not None: + for target, (instr, _) in self.loads.items(): + if instr == VAR_LOAD_PARAMETER: + rv.add(target) + node = node.parent + return rv + + +class RootVisitor(NodeVisitor): + def __init__(self, symbols): + self.sym_visitor = FrameSymbolVisitor(symbols) + + def _simple_visit(self, node, **kwargs): + for child in node.iter_child_nodes(): + self.sym_visitor.visit(child) + + visit_Template = ( + visit_Block + ) = ( + visit_Macro + ) = ( + visit_FilterBlock + ) = visit_Scope = visit_If = visit_ScopedEvalContextModifier = _simple_visit + + def visit_AssignBlock(self, node, **kwargs): + for child in node.body: + self.sym_visitor.visit(child) + + def visit_CallBlock(self, node, **kwargs): + for child in node.iter_child_nodes(exclude=("call",)): + self.sym_visitor.visit(child) + + def visit_OverlayScope(self, node, **kwargs): + for child in node.body: + self.sym_visitor.visit(child) + + def visit_For(self, node, for_branch="body", **kwargs): + if for_branch == "body": + self.sym_visitor.visit(node.target, store_as_param=True) + branch = node.body + elif for_branch == "else": + branch = node.else_ + elif for_branch == "test": + self.sym_visitor.visit(node.target, store_as_param=True) + if node.test is not None: + self.sym_visitor.visit(node.test) + return + else: + raise RuntimeError("Unknown for branch") + for item in branch or (): + self.sym_visitor.visit(item) + + def visit_With(self, node, **kwargs): + for target in node.targets: + self.sym_visitor.visit(target) + for child in node.body: + self.sym_visitor.visit(child) + + def generic_visit(self, node, *args, **kwargs): + raise NotImplementedError( + f"Cannot find symbols for {node.__class__.__name__!r}" + ) + + +class FrameSymbolVisitor(NodeVisitor): + """A visitor for `Frame.inspect`.""" + + def __init__(self, symbols): + self.symbols = symbols + + def visit_Name(self, node, store_as_param=False, **kwargs): + """All assignments to names go through this function.""" + if store_as_param or node.ctx == "param": + self.symbols.declare_parameter(node.name) + elif node.ctx == "store": + self.symbols.store(node.name) + elif node.ctx == "load": + self.symbols.load(node.name) + + def visit_NSRef(self, node, **kwargs): + self.symbols.load(node.name) + + def visit_If(self, node, **kwargs): + self.visit(node.test, **kwargs) + + original_symbols = self.symbols + + def inner_visit(nodes): + self.symbols = rv = original_symbols.copy() + for subnode in nodes: + self.visit(subnode, **kwargs) + self.symbols = original_symbols + return rv + + body_symbols = inner_visit(node.body) + elif_symbols = inner_visit(node.elif_) + else_symbols = inner_visit(node.else_ or ()) + + self.symbols.branch_update([body_symbols, elif_symbols, else_symbols]) + + def visit_Macro(self, node, **kwargs): + self.symbols.store(node.name) + + def visit_Import(self, node, **kwargs): + self.generic_visit(node, **kwargs) + self.symbols.store(node.target) + + def visit_FromImport(self, node, **kwargs): + self.generic_visit(node, **kwargs) + for name in node.names: + if isinstance(name, tuple): + self.symbols.store(name[1]) + else: + self.symbols.store(name) + + def visit_Assign(self, node, **kwargs): + """Visit assignments in the correct order.""" + self.visit(node.node, **kwargs) + self.visit(node.target, **kwargs) + + def visit_For(self, node, **kwargs): + """Visiting stops at for blocks. However the block sequence + is visited as part of the outer scope. + """ + self.visit(node.iter, **kwargs) + + def visit_CallBlock(self, node, **kwargs): + self.visit(node.call, **kwargs) + + def visit_FilterBlock(self, node, **kwargs): + self.visit(node.filter, **kwargs) + + def visit_With(self, node, **kwargs): + for target in node.values: + self.visit(target) + + def visit_AssignBlock(self, node, **kwargs): + """Stop visiting at block assigns.""" + self.visit(node.target, **kwargs) + + def visit_Scope(self, node, **kwargs): + """Stop visiting at scopes.""" + + def visit_Block(self, node, **kwargs): + """Stop visiting at blocks.""" + + def visit_OverlayScope(self, node, **kwargs): + """Do not visit into overlay scopes.""" diff --git a/src/jinja2/lexer.py b/src/jinja2/lexer.py new file mode 100644 index 00000000..082a051d --- /dev/null +++ b/src/jinja2/lexer.py @@ -0,0 +1,801 @@ +"""Implements a Jinja / Python combination lexer. The ``Lexer`` class +is used to do some preprocessing. It filters out invalid operators like +the bitshift operators we don't allow in templates. It separates +template code and python code in expressions. +""" +import re +from ast import literal_eval +from collections import deque +from operator import itemgetter +from sys import intern + +from ._identifier import pattern as name_re +from .exceptions import TemplateSyntaxError +from .utils import LRUCache + +# cache for the lexers. Exists in order to be able to have multiple +# environments with the same lexer +_lexer_cache = LRUCache(50) + +# static regular expressions +whitespace_re = re.compile(r"\s+") +newline_re = re.compile(r"(\r\n|\r|\n)") +string_re = re.compile( + r"('([^'\\]*(?:\\.[^'\\]*)*)'" r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S +) +integer_re = re.compile(r"(\d+_)*\d+") +float_re = re.compile( + r""" + (?<!\.) # doesn't start with a . + (\d+_)*\d+ # digits, possibly _ separated + ( + (\.(\d+_)*\d+)? # optional fractional part + e[+\-]?(\d+_)*\d+ # exponent part + | + \.(\d+_)*\d+ # required fractional part + ) + """, + re.IGNORECASE | re.VERBOSE, +) + +# internal the tokens and keep references to them +TOKEN_ADD = intern("add") +TOKEN_ASSIGN = intern("assign") +TOKEN_COLON = intern("colon") +TOKEN_COMMA = intern("comma") +TOKEN_DIV = intern("div") +TOKEN_DOT = intern("dot") +TOKEN_EQ = intern("eq") +TOKEN_FLOORDIV = intern("floordiv") +TOKEN_GT = intern("gt") +TOKEN_GTEQ = intern("gteq") +TOKEN_LBRACE = intern("lbrace") +TOKEN_LBRACKET = intern("lbracket") +TOKEN_LPAREN = intern("lparen") +TOKEN_LT = intern("lt") +TOKEN_LTEQ = intern("lteq") +TOKEN_MOD = intern("mod") +TOKEN_MUL = intern("mul") +TOKEN_NE = intern("ne") +TOKEN_PIPE = intern("pipe") +TOKEN_POW = intern("pow") +TOKEN_RBRACE = intern("rbrace") +TOKEN_RBRACKET = intern("rbracket") +TOKEN_RPAREN = intern("rparen") +TOKEN_SEMICOLON = intern("semicolon") +TOKEN_SUB = intern("sub") +TOKEN_TILDE = intern("tilde") +TOKEN_WHITESPACE = intern("whitespace") +TOKEN_FLOAT = intern("float") +TOKEN_INTEGER = intern("integer") +TOKEN_NAME = intern("name") +TOKEN_STRING = intern("string") +TOKEN_OPERATOR = intern("operator") +TOKEN_BLOCK_BEGIN = intern("block_begin") +TOKEN_BLOCK_END = intern("block_end") +TOKEN_VARIABLE_BEGIN = intern("variable_begin") +TOKEN_VARIABLE_END = intern("variable_end") +TOKEN_RAW_BEGIN = intern("raw_begin") +TOKEN_RAW_END = intern("raw_end") +TOKEN_COMMENT_BEGIN = intern("comment_begin") +TOKEN_COMMENT_END = intern("comment_end") +TOKEN_COMMENT = intern("comment") +TOKEN_LINESTATEMENT_BEGIN = intern("linestatement_begin") +TOKEN_LINESTATEMENT_END = intern("linestatement_end") +TOKEN_LINECOMMENT_BEGIN = intern("linecomment_begin") +TOKEN_LINECOMMENT_END = intern("linecomment_end") +TOKEN_LINECOMMENT = intern("linecomment") +TOKEN_DATA = intern("data") +TOKEN_INITIAL = intern("initial") +TOKEN_EOF = intern("eof") + +# bind operators to token types +operators = { + "+": TOKEN_ADD, + "-": TOKEN_SUB, + "/": TOKEN_DIV, + "//": TOKEN_FLOORDIV, + "*": TOKEN_MUL, + "%": TOKEN_MOD, + "**": TOKEN_POW, + "~": TOKEN_TILDE, + "[": TOKEN_LBRACKET, + "]": TOKEN_RBRACKET, + "(": TOKEN_LPAREN, + ")": TOKEN_RPAREN, + "{": TOKEN_LBRACE, + "}": TOKEN_RBRACE, + "==": TOKEN_EQ, + "!=": TOKEN_NE, + ">": TOKEN_GT, + ">=": TOKEN_GTEQ, + "<": TOKEN_LT, + "<=": TOKEN_LTEQ, + "=": TOKEN_ASSIGN, + ".": TOKEN_DOT, + ":": TOKEN_COLON, + "|": TOKEN_PIPE, + ",": TOKEN_COMMA, + ";": TOKEN_SEMICOLON, +} + +reverse_operators = {v: k for k, v in operators.items()} +assert len(operators) == len(reverse_operators), "operators dropped" +operator_re = re.compile( + f"({'|'.join(re.escape(x) for x in sorted(operators, key=lambda x: -len(x)))})" +) + +ignored_tokens = frozenset( + [ + TOKEN_COMMENT_BEGIN, + TOKEN_COMMENT, + TOKEN_COMMENT_END, + TOKEN_WHITESPACE, + TOKEN_LINECOMMENT_BEGIN, + TOKEN_LINECOMMENT_END, + TOKEN_LINECOMMENT, + ] +) +ignore_if_empty = frozenset( + [TOKEN_WHITESPACE, TOKEN_DATA, TOKEN_COMMENT, TOKEN_LINECOMMENT] +) + + +def _describe_token_type(token_type): + if token_type in reverse_operators: + return reverse_operators[token_type] + return { + TOKEN_COMMENT_BEGIN: "begin of comment", + TOKEN_COMMENT_END: "end of comment", + TOKEN_COMMENT: "comment", + TOKEN_LINECOMMENT: "comment", + TOKEN_BLOCK_BEGIN: "begin of statement block", + TOKEN_BLOCK_END: "end of statement block", + TOKEN_VARIABLE_BEGIN: "begin of print statement", + TOKEN_VARIABLE_END: "end of print statement", + TOKEN_LINESTATEMENT_BEGIN: "begin of line statement", + TOKEN_LINESTATEMENT_END: "end of line statement", + TOKEN_DATA: "template data / text", + TOKEN_EOF: "end of template", + }.get(token_type, token_type) + + +def describe_token(token): + """Returns a description of the token.""" + if token.type == TOKEN_NAME: + return token.value + return _describe_token_type(token.type) + + +def describe_token_expr(expr): + """Like `describe_token` but for token expressions.""" + if ":" in expr: + type, value = expr.split(":", 1) + if type == TOKEN_NAME: + return value + else: + type = expr + return _describe_token_type(type) + + +def count_newlines(value): + """Count the number of newline characters in the string. This is + useful for extensions that filter a stream. + """ + return len(newline_re.findall(value)) + + +def compile_rules(environment): + """Compiles all the rules from the environment into a list of rules.""" + e = re.escape + rules = [ + ( + len(environment.comment_start_string), + TOKEN_COMMENT_BEGIN, + e(environment.comment_start_string), + ), + ( + len(environment.block_start_string), + TOKEN_BLOCK_BEGIN, + e(environment.block_start_string), + ), + ( + len(environment.variable_start_string), + TOKEN_VARIABLE_BEGIN, + e(environment.variable_start_string), + ), + ] + + if environment.line_statement_prefix is not None: + rules.append( + ( + len(environment.line_statement_prefix), + TOKEN_LINESTATEMENT_BEGIN, + r"^[ \t\v]*" + e(environment.line_statement_prefix), + ) + ) + if environment.line_comment_prefix is not None: + rules.append( + ( + len(environment.line_comment_prefix), + TOKEN_LINECOMMENT_BEGIN, + r"(?:^|(?<=\S))[^\S\r\n]*" + e(environment.line_comment_prefix), + ) + ) + + return [x[1:] for x in sorted(rules, reverse=True)] + + +class Failure: + """Class that raises a `TemplateSyntaxError` if called. + Used by the `Lexer` to specify known errors. + """ + + def __init__(self, message, cls=TemplateSyntaxError): + self.message = message + self.error_class = cls + + def __call__(self, lineno, filename): + raise self.error_class(self.message, lineno, filename) + + +class Token(tuple): + """Token class.""" + + __slots__ = () + lineno, type, value = (property(itemgetter(x)) for x in range(3)) + + def __new__(cls, lineno, type, value): + return tuple.__new__(cls, (lineno, intern(str(type)), value)) + + def __str__(self): + if self.type in reverse_operators: + return reverse_operators[self.type] + elif self.type == "name": + return self.value + return self.type + + def test(self, expr): + """Test a token against a token expression. This can either be a + token type or ``'token_type:token_value'``. This can only test + against string values and types. + """ + # here we do a regular string equality check as test_any is usually + # passed an iterable of not interned strings. + if self.type == expr: + return True + elif ":" in expr: + return expr.split(":", 1) == [self.type, self.value] + return False + + def test_any(self, *iterable): + """Test against multiple token expressions.""" + for expr in iterable: + if self.test(expr): + return True + return False + + def __repr__(self): + return f"Token({self.lineno!r}, {self.type!r}, {self.value!r})" + + +class TokenStreamIterator: + """The iterator for tokenstreams. Iterate over the stream + until the eof token is reached. + """ + + def __init__(self, stream): + self.stream = stream + + def __iter__(self): + return self + + def __next__(self): + token = self.stream.current + if token.type is TOKEN_EOF: + self.stream.close() + raise StopIteration() + next(self.stream) + return token + + +class TokenStream: + """A token stream is an iterable that yields :class:`Token`\\s. The + parser however does not iterate over it but calls :meth:`next` to go + one token ahead. The current active token is stored as :attr:`current`. + """ + + def __init__(self, generator, name, filename): + self._iter = iter(generator) + self._pushed = deque() + self.name = name + self.filename = filename + self.closed = False + self.current = Token(1, TOKEN_INITIAL, "") + next(self) + + def __iter__(self): + return TokenStreamIterator(self) + + def __bool__(self): + return bool(self._pushed) or self.current.type is not TOKEN_EOF + + __nonzero__ = __bool__ # py2 + + @property + def eos(self): + """Are we at the end of the stream?""" + return not self + + def push(self, token): + """Push a token back to the stream.""" + self._pushed.append(token) + + def look(self): + """Look at the next token.""" + old_token = next(self) + result = self.current + self.push(result) + self.current = old_token + return result + + def skip(self, n=1): + """Got n tokens ahead.""" + for _ in range(n): + next(self) + + def next_if(self, expr): + """Perform the token test and return the token if it matched. + Otherwise the return value is `None`. + """ + if self.current.test(expr): + return next(self) + + def skip_if(self, expr): + """Like :meth:`next_if` but only returns `True` or `False`.""" + return self.next_if(expr) is not None + + def __next__(self): + """Go one token ahead and return the old one. + + Use the built-in :func:`next` instead of calling this directly. + """ + rv = self.current + if self._pushed: + self.current = self._pushed.popleft() + elif self.current.type is not TOKEN_EOF: + try: + self.current = next(self._iter) + except StopIteration: + self.close() + return rv + + def close(self): + """Close the stream.""" + self.current = Token(self.current.lineno, TOKEN_EOF, "") + self._iter = None + self.closed = True + + def expect(self, expr): + """Expect a given token type and return it. This accepts the same + argument as :meth:`jinja2.lexer.Token.test`. + """ + if not self.current.test(expr): + expr = describe_token_expr(expr) + if self.current.type is TOKEN_EOF: + raise TemplateSyntaxError( + f"unexpected end of template, expected {expr!r}.", + self.current.lineno, + self.name, + self.filename, + ) + raise TemplateSyntaxError( + f"expected token {expr!r}, got {describe_token(self.current)!r}", + self.current.lineno, + self.name, + self.filename, + ) + try: + return self.current + finally: + next(self) + + +def get_lexer(environment): + """Return a lexer which is probably cached.""" + key = ( + environment.block_start_string, + environment.block_end_string, + environment.variable_start_string, + environment.variable_end_string, + environment.comment_start_string, + environment.comment_end_string, + environment.line_statement_prefix, + environment.line_comment_prefix, + environment.trim_blocks, + environment.lstrip_blocks, + environment.newline_sequence, + environment.keep_trailing_newline, + ) + lexer = _lexer_cache.get(key) + if lexer is None: + lexer = Lexer(environment) + _lexer_cache[key] = lexer + return lexer + + +class OptionalLStrip(tuple): + """A special tuple for marking a point in the state that can have + lstrip applied. + """ + + __slots__ = () + + # Even though it looks like a no-op, creating instances fails + # without this. + def __new__(cls, *members, **kwargs): + return super().__new__(cls, members) + + +class Lexer: + """Class that implements a lexer for a given environment. Automatically + created by the environment class, usually you don't have to do that. + + Note that the lexer is not automatically bound to an environment. + Multiple environments can share the same lexer. + """ + + def __init__(self, environment): + # shortcuts + e = re.escape + + def c(x): + return re.compile(x, re.M | re.S) + + # lexing rules for tags + tag_rules = [ + (whitespace_re, TOKEN_WHITESPACE, None), + (float_re, TOKEN_FLOAT, None), + (integer_re, TOKEN_INTEGER, None), + (name_re, TOKEN_NAME, None), + (string_re, TOKEN_STRING, None), + (operator_re, TOKEN_OPERATOR, None), + ] + + # assemble the root lexing rule. because "|" is ungreedy + # we have to sort by length so that the lexer continues working + # as expected when we have parsing rules like <% for block and + # <%= for variables. (if someone wants asp like syntax) + # variables are just part of the rules if variable processing + # is required. + root_tag_rules = compile_rules(environment) + + block_start_re = e(environment.block_start_string) + block_end_re = e(environment.block_end_string) + comment_end_re = e(environment.comment_end_string) + variable_end_re = e(environment.variable_end_string) + + # block suffix if trimming is enabled + block_suffix_re = "\\n?" if environment.trim_blocks else "" + + # If lstrip is enabled, it should not be applied if there is any + # non-whitespace between the newline and block. + self.lstrip_unless_re = c(r"[^ \t]") if environment.lstrip_blocks else None + + self.newline_sequence = environment.newline_sequence + self.keep_trailing_newline = environment.keep_trailing_newline + + root_raw_re = ( + fr"(?P<raw_begin>{block_start_re}(\-|\+|)\s*raw\s*" + fr"(?:\-{block_end_re}\s*|{block_end_re}))" + ) + root_parts_re = "|".join( + [root_raw_re] + [fr"(?P<{n}>{r}(\-|\+|))" for n, r in root_tag_rules] + ) + + # global lexing rules + self.rules = { + "root": [ + # directives + ( + c(fr"(.*?)(?:{root_parts_re})"), + OptionalLStrip(TOKEN_DATA, "#bygroup"), + "#bygroup", + ), + # data + (c(".+"), TOKEN_DATA, None), + ], + # comments + TOKEN_COMMENT_BEGIN: [ + ( + c( + fr"(.*?)((?:\+{comment_end_re}|\-{comment_end_re}\s*" + fr"|{comment_end_re}{block_suffix_re}))" + ), + (TOKEN_COMMENT, TOKEN_COMMENT_END), + "#pop", + ), + (c(r"(.)"), (Failure("Missing end of comment tag"),), None), + ], + # blocks + TOKEN_BLOCK_BEGIN: [ + ( + c( + fr"(?:\+{block_end_re}|\-{block_end_re}\s*" + fr"|{block_end_re}{block_suffix_re})" + ), + TOKEN_BLOCK_END, + "#pop", + ), + ] + + tag_rules, + # variables + TOKEN_VARIABLE_BEGIN: [ + ( + c(fr"\-{variable_end_re}\s*|{variable_end_re}"), + TOKEN_VARIABLE_END, + "#pop", + ) + ] + + tag_rules, + # raw block + TOKEN_RAW_BEGIN: [ + ( + c( + fr"(.*?)((?:{block_start_re}(\-|\+|))\s*endraw\s*" + fr"(?:\+{block_end_re}|\-{block_end_re}\s*" + fr"|{block_end_re}{block_suffix_re}))" + ), + OptionalLStrip(TOKEN_DATA, TOKEN_RAW_END), + "#pop", + ), + (c(r"(.)"), (Failure("Missing end of raw directive"),), None), + ], + # line statements + TOKEN_LINESTATEMENT_BEGIN: [ + (c(r"\s*(\n|$)"), TOKEN_LINESTATEMENT_END, "#pop") + ] + + tag_rules, + # line comments + TOKEN_LINECOMMENT_BEGIN: [ + ( + c(r"(.*?)()(?=\n|$)"), + (TOKEN_LINECOMMENT, TOKEN_LINECOMMENT_END), + "#pop", + ) + ], + } + + def _normalize_newlines(self, value): + """Replace all newlines with the configured sequence in strings + and template data. + """ + return newline_re.sub(self.newline_sequence, value) + + def tokenize(self, source, name=None, filename=None, state=None): + """Calls tokeniter + tokenize and wraps it in a token stream.""" + stream = self.tokeniter(source, name, filename, state) + return TokenStream(self.wrap(stream, name, filename), name, filename) + + def wrap(self, stream, name=None, filename=None): + """This is called with the stream as returned by `tokenize` and wraps + every token in a :class:`Token` and converts the value. + """ + for lineno, token, value in stream: + if token in ignored_tokens: + continue + elif token == TOKEN_LINESTATEMENT_BEGIN: + token = TOKEN_BLOCK_BEGIN + elif token == TOKEN_LINESTATEMENT_END: + token = TOKEN_BLOCK_END + # we are not interested in those tokens in the parser + elif token in (TOKEN_RAW_BEGIN, TOKEN_RAW_END): + continue + elif token == TOKEN_DATA: + value = self._normalize_newlines(value) + elif token == "keyword": + token = value + elif token == TOKEN_NAME: + value = str(value) + if not value.isidentifier(): + raise TemplateSyntaxError( + "Invalid character in identifier", lineno, name, filename + ) + elif token == TOKEN_STRING: + # try to unescape string + try: + value = ( + self._normalize_newlines(value[1:-1]) + .encode("ascii", "backslashreplace") + .decode("unicode-escape") + ) + except Exception as e: + msg = str(e).split(":")[-1].strip() + raise TemplateSyntaxError(msg, lineno, name, filename) + elif token == TOKEN_INTEGER: + value = int(value.replace("_", "")) + elif token == TOKEN_FLOAT: + # remove all "_" first to support more Python versions + value = literal_eval(value.replace("_", "")) + elif token == TOKEN_OPERATOR: + token = operators[value] + yield Token(lineno, token, value) + + def tokeniter(self, source, name, filename=None, state=None): + """This method tokenizes the text and returns the tokens in a + generator. Use this method if you just want to tokenize a template. + """ + lines = source.splitlines() + if self.keep_trailing_newline and source: + if source.endswith(("\r\n", "\r", "\n")): + lines.append("") + source = "\n".join(lines) + pos = 0 + lineno = 1 + stack = ["root"] + if state is not None and state != "root": + assert state in ("variable", "block"), "invalid state" + stack.append(state + "_begin") + statetokens = self.rules[stack[-1]] + source_length = len(source) + balancing_stack = [] + lstrip_unless_re = self.lstrip_unless_re + newlines_stripped = 0 + line_starting = True + + while 1: + # tokenizer loop + for regex, tokens, new_state in statetokens: + m = regex.match(source, pos) + # if no match we try again with the next rule + if m is None: + continue + + # we only match blocks and variables if braces / parentheses + # are balanced. continue parsing with the lower rule which + # is the operator rule. do this only if the end tags look + # like operators + if balancing_stack and tokens in ( + TOKEN_VARIABLE_END, + TOKEN_BLOCK_END, + TOKEN_LINESTATEMENT_END, + ): + continue + + # tuples support more options + if isinstance(tokens, tuple): + groups = m.groups() + + if isinstance(tokens, OptionalLStrip): + # Rule supports lstrip. Match will look like + # text, block type, whitespace control, type, control, ... + text = groups[0] + + # Skipping the text and first type, every other group is the + # whitespace control for each type. One of the groups will be + # -, +, or empty string instead of None. + strip_sign = next(g for g in groups[2::2] if g is not None) + + if strip_sign == "-": + # Strip all whitespace between the text and the tag. + stripped = text.rstrip() + newlines_stripped = text[len(stripped) :].count("\n") + groups = (stripped,) + groups[1:] + elif ( + # Not marked for preserving whitespace. + strip_sign != "+" + # lstrip is enabled. + and lstrip_unless_re is not None + # Not a variable expression. + and not m.groupdict().get(TOKEN_VARIABLE_BEGIN) + ): + # The start of text between the last newline and the tag. + l_pos = text.rfind("\n") + 1 + if l_pos > 0 or line_starting: + # If there's only whitespace between the newline and the + # tag, strip it. + if not lstrip_unless_re.search(text, l_pos): + groups = (text[:l_pos],) + groups[1:] + + for idx, token in enumerate(tokens): + # failure group + if token.__class__ is Failure: + raise token(lineno, filename) + # bygroup is a bit more complex, in that case we + # yield for the current token the first named + # group that matched + elif token == "#bygroup": + for key, value in m.groupdict().items(): + if value is not None: + yield lineno, key, value + lineno += value.count("\n") + break + else: + raise RuntimeError( + f"{regex!r} wanted to resolve the token dynamically" + " but no group matched" + ) + # normal group + else: + data = groups[idx] + if data or token not in ignore_if_empty: + yield lineno, token, data + lineno += data.count("\n") + newlines_stripped + newlines_stripped = 0 + + # strings as token just are yielded as it. + else: + data = m.group() + # update brace/parentheses balance + if tokens == TOKEN_OPERATOR: + if data == "{": + balancing_stack.append("}") + elif data == "(": + balancing_stack.append(")") + elif data == "[": + balancing_stack.append("]") + elif data in ("}", ")", "]"): + if not balancing_stack: + raise TemplateSyntaxError( + f"unexpected '{data}'", lineno, name, filename + ) + expected_op = balancing_stack.pop() + if expected_op != data: + raise TemplateSyntaxError( + f"unexpected '{data}', expected '{expected_op}'", + lineno, + name, + filename, + ) + # yield items + if data or tokens not in ignore_if_empty: + yield lineno, tokens, data + lineno += data.count("\n") + + line_starting = m.group()[-1:] == "\n" + + # fetch new position into new variable so that we can check + # if there is a internal parsing error which would result + # in an infinite loop + pos2 = m.end() + + # handle state changes + if new_state is not None: + # remove the uppermost state + if new_state == "#pop": + stack.pop() + # resolve the new state by group checking + elif new_state == "#bygroup": + for key, value in m.groupdict().items(): + if value is not None: + stack.append(key) + break + else: + raise RuntimeError( + f"{regex!r} wanted to resolve the new state dynamically" + f" but no group matched" + ) + # direct state name given + else: + stack.append(new_state) + statetokens = self.rules[stack[-1]] + # we are still at the same position and no stack change. + # this means a loop without break condition, avoid that and + # raise error + elif pos2 == pos: + raise RuntimeError( + f"{regex!r} yielded empty string without stack change" + ) + # publish new function and start again + pos = pos2 + break + # if loop terminated without break we haven't found a single match + # either we are at the end of the file or we have a problem + else: + # end of text + if pos >= source_length: + return + # something went wrong + raise TemplateSyntaxError( + f"unexpected char {source[pos]!r} at {pos}", lineno, name, filename + ) diff --git a/src/jinja2/loaders.py b/src/jinja2/loaders.py new file mode 100644 index 00000000..6b71b835 --- /dev/null +++ b/src/jinja2/loaders.py @@ -0,0 +1,566 @@ +"""API and implementations for loading templates from different data +sources. +""" +import importlib.util +import os +import sys +import weakref +import zipimport +from collections import abc +from hashlib import sha1 +from importlib import import_module +from types import ModuleType + +from .exceptions import TemplateNotFound +from .utils import internalcode +from .utils import open_if_exists + + +def split_template_path(template): + """Split a path into segments and perform a sanity check. If it detects + '..' in the path it will raise a `TemplateNotFound` error. + """ + pieces = [] + for piece in template.split("/"): + if ( + os.path.sep in piece + or (os.path.altsep and os.path.altsep in piece) + or piece == os.path.pardir + ): + raise TemplateNotFound(template) + elif piece and piece != ".": + pieces.append(piece) + return pieces + + +class BaseLoader: + """Baseclass for all loaders. Subclass this and override `get_source` to + implement a custom loading mechanism. The environment provides a + `get_template` method that calls the loader's `load` method to get the + :class:`Template` object. + + A very basic example for a loader that looks up templates on the file + system could look like this:: + + from jinja2 import BaseLoader, TemplateNotFound + from os.path import join, exists, getmtime + + class MyLoader(BaseLoader): + + def __init__(self, path): + self.path = path + + def get_source(self, environment, template): + path = join(self.path, template) + if not exists(path): + raise TemplateNotFound(template) + mtime = getmtime(path) + with open(path) as f: + source = f.read() + return source, path, lambda: mtime == getmtime(path) + """ + + #: if set to `False` it indicates that the loader cannot provide access + #: to the source of templates. + #: + #: .. versionadded:: 2.4 + has_source_access = True + + def get_source(self, environment, template): + """Get the template source, filename and reload helper for a template. + It's passed the environment and template name and has to return a + tuple in the form ``(source, filename, uptodate)`` or raise a + `TemplateNotFound` error if it can't locate the template. + + The source part of the returned tuple must be the source of the + template as a string. The filename should be the name of the + file on the filesystem if it was loaded from there, otherwise + ``None``. The filename is used by Python for the tracebacks + if no loader extension is used. + + The last item in the tuple is the `uptodate` function. If auto + reloading is enabled it's always called to check if the template + changed. No arguments are passed so the function must store the + old state somewhere (for example in a closure). If it returns `False` + the template will be reloaded. + """ + if not self.has_source_access: + raise RuntimeError( + f"{self.__class__.__name__} cannot provide access to the source" + ) + raise TemplateNotFound(template) + + def list_templates(self): + """Iterates over all templates. If the loader does not support that + it should raise a :exc:`TypeError` which is the default behavior. + """ + raise TypeError("this loader cannot iterate over all templates") + + @internalcode + def load(self, environment, name, globals=None): + """Loads a template. This method looks up the template in the cache + or loads one by calling :meth:`get_source`. Subclasses should not + override this method as loaders working on collections of other + loaders (such as :class:`PrefixLoader` or :class:`ChoiceLoader`) + will not call this method but `get_source` directly. + """ + code = None + if globals is None: + globals = {} + + # first we try to get the source for this template together + # with the filename and the uptodate function. + source, filename, uptodate = self.get_source(environment, name) + + # try to load the code from the bytecode cache if there is a + # bytecode cache configured. + bcc = environment.bytecode_cache + if bcc is not None: + bucket = bcc.get_bucket(environment, name, filename, source) + code = bucket.code + + # if we don't have code so far (not cached, no longer up to + # date) etc. we compile the template + if code is None: + code = environment.compile(source, name, filename) + + # if the bytecode cache is available and the bucket doesn't + # have a code so far, we give the bucket the new code and put + # it back to the bytecode cache. + if bcc is not None and bucket.code is None: + bucket.code = code + bcc.set_bucket(bucket) + + return environment.template_class.from_code( + environment, code, globals, uptodate + ) + + +class FileSystemLoader(BaseLoader): + """Load templates from a directory in the file system. + + The path can be relative or absolute. Relative paths are relative to + the current working directory. + + .. code-block:: python + + loader = FileSystemLoader("templates") + + A list of paths can be given. The directories will be searched in + order, stopping at the first matching template. + + .. code-block:: python + + loader = FileSystemLoader(["/override/templates", "/default/templates"]) + + :param searchpath: A path, or list of paths, to the directory that + contains the templates. + :param encoding: Use this encoding to read the text from template + files. + :param followlinks: Follow symbolic links in the path. + + .. versionchanged:: 2.8 + Added the ``followlinks`` parameter. + """ + + def __init__(self, searchpath, encoding="utf-8", followlinks=False): + if not isinstance(searchpath, abc.Iterable) or isinstance(searchpath, str): + searchpath = [searchpath] + + self.searchpath = list(searchpath) + self.encoding = encoding + self.followlinks = followlinks + + def get_source(self, environment, template): + pieces = split_template_path(template) + for searchpath in self.searchpath: + filename = os.path.join(searchpath, *pieces) + f = open_if_exists(filename) + if f is None: + continue + try: + contents = f.read().decode(self.encoding) + finally: + f.close() + + mtime = os.path.getmtime(filename) + + def uptodate(): + try: + return os.path.getmtime(filename) == mtime + except OSError: + return False + + return contents, filename, uptodate + raise TemplateNotFound(template) + + def list_templates(self): + found = set() + for searchpath in self.searchpath: + walk_dir = os.walk(searchpath, followlinks=self.followlinks) + for dirpath, _, filenames in walk_dir: + for filename in filenames: + template = ( + os.path.join(dirpath, filename)[len(searchpath) :] + .strip(os.path.sep) + .replace(os.path.sep, "/") + ) + if template[:2] == "./": + template = template[2:] + if template not in found: + found.add(template) + return sorted(found) + + +class PackageLoader(BaseLoader): + """Load templates from a directory in a Python package. + + :param package_name: Import name of the package that contains the + template directory. + :param package_path: Directory within the imported package that + contains the templates. + :param encoding: Encoding of template files. + + The following example looks up templates in the ``pages`` directory + within the ``project.ui`` package. + + .. code-block:: python + + loader = PackageLoader("project.ui", "pages") + + Only packages installed as directories (standard pip behavior) or + zip/egg files (less common) are supported. The Python API for + introspecting data in packages is too limited to support other + installation methods the way this loader requires. + + There is limited support for :pep:`420` namespace packages. The + template directory is assumed to only be in one namespace + contributor. Zip files contributing to a namespace are not + supported. + + .. versionchanged:: 3.0 + No longer uses ``setuptools`` as a dependency. + + .. versionchanged:: 3.0 + Limited PEP 420 namespace package support. + """ + + def __init__(self, package_name, package_path="templates", encoding="utf-8"): + if package_path == os.path.curdir: + package_path = "" + elif package_path[:2] == os.path.curdir + os.path.sep: + package_path = package_path[2:] + + package_path = os.path.normpath(package_path).rstrip(os.path.sep) + self.package_path = package_path + self.package_name = package_name + self.encoding = encoding + + # Make sure the package exists. This also makes namespace + # packages work, otherwise get_loader returns None. + import_module(package_name) + spec = importlib.util.find_spec(package_name) + self._loader = loader = spec.loader + self._archive = None + self._template_root = None + + if isinstance(loader, zipimport.zipimporter): + self._archive = loader.archive + pkgdir = next(iter(spec.submodule_search_locations)) + self._template_root = os.path.join(pkgdir, package_path) + elif spec.submodule_search_locations: + # This will be one element for regular packages and multiple + # for namespace packages. + for root in spec.submodule_search_locations: + root = os.path.join(root, package_path) + + if os.path.isdir(root): + self._template_root = root + break + + if self._template_root is None: + raise ValueError( + f"The {package_name!r} package was not installed in a" + " way that PackageLoader understands." + ) + + def get_source(self, environment, template): + p = os.path.join(self._template_root, *split_template_path(template)) + + if self._archive is None: + # Package is a directory. + if not os.path.isfile(p): + raise TemplateNotFound(template) + + with open(p, "rb") as f: + source = f.read() + + mtime = os.path.getmtime(p) + + def up_to_date(): + return os.path.isfile(p) and os.path.getmtime(p) == mtime + + else: + # Package is a zip file. + try: + source = self._loader.get_data(p) + except OSError: + raise TemplateNotFound(template) + + # Could use the zip's mtime for all template mtimes, but + # would need to safely reload the module if it's out of + # date, so just report it as always current. + up_to_date = None + + return source.decode(self.encoding), p, up_to_date + + def list_templates(self): + results = [] + + if self._archive is None: + # Package is a directory. + offset = len(self._template_root) + + for dirpath, _, filenames in os.walk(self._template_root): + dirpath = dirpath[offset:].lstrip(os.path.sep) + results.extend( + os.path.join(dirpath, name).replace(os.path.sep, "/") + for name in filenames + ) + else: + if not hasattr(self._loader, "_files"): + raise TypeError( + "This zip import does not have the required" + " metadata to list templates." + ) + + # Package is a zip file. + prefix = ( + self._template_root[len(self._archive) :].lstrip(os.path.sep) + + os.path.sep + ) + offset = len(prefix) + + for name in self._loader._files.keys(): + # Find names under the templates directory that aren't directories. + if name.startswith(prefix) and name[-1] != os.path.sep: + results.append(name[offset:].replace(os.path.sep, "/")) + + results.sort() + return results + + +class DictLoader(BaseLoader): + """Loads a template from a Python dict mapping template names to + template source. This loader is useful for unittesting: + + >>> loader = DictLoader({'index.html': 'source here'}) + + Because auto reloading is rarely useful this is disabled per default. + """ + + def __init__(self, mapping): + self.mapping = mapping + + def get_source(self, environment, template): + if template in self.mapping: + source = self.mapping[template] + return source, None, lambda: source == self.mapping.get(template) + raise TemplateNotFound(template) + + def list_templates(self): + return sorted(self.mapping) + + +class FunctionLoader(BaseLoader): + """A loader that is passed a function which does the loading. The + function receives the name of the template and has to return either + a string with the template source, a tuple in the form ``(source, + filename, uptodatefunc)`` or `None` if the template does not exist. + + >>> def load_template(name): + ... if name == 'index.html': + ... return '...' + ... + >>> loader = FunctionLoader(load_template) + + The `uptodatefunc` is a function that is called if autoreload is enabled + and has to return `True` if the template is still up to date. For more + details have a look at :meth:`BaseLoader.get_source` which has the same + return value. + """ + + def __init__(self, load_func): + self.load_func = load_func + + def get_source(self, environment, template): + rv = self.load_func(template) + if rv is None: + raise TemplateNotFound(template) + elif isinstance(rv, str): + return rv, None, None + return rv + + +class PrefixLoader(BaseLoader): + """A loader that is passed a dict of loaders where each loader is bound + to a prefix. The prefix is delimited from the template by a slash per + default, which can be changed by setting the `delimiter` argument to + something else:: + + loader = PrefixLoader({ + 'app1': PackageLoader('mypackage.app1'), + 'app2': PackageLoader('mypackage.app2') + }) + + By loading ``'app1/index.html'`` the file from the app1 package is loaded, + by loading ``'app2/index.html'`` the file from the second. + """ + + def __init__(self, mapping, delimiter="/"): + self.mapping = mapping + self.delimiter = delimiter + + def get_loader(self, template): + try: + prefix, name = template.split(self.delimiter, 1) + loader = self.mapping[prefix] + except (ValueError, KeyError): + raise TemplateNotFound(template) + return loader, name + + def get_source(self, environment, template): + loader, name = self.get_loader(template) + try: + return loader.get_source(environment, name) + except TemplateNotFound: + # re-raise the exception with the correct filename here. + # (the one that includes the prefix) + raise TemplateNotFound(template) + + @internalcode + def load(self, environment, name, globals=None): + loader, local_name = self.get_loader(name) + try: + return loader.load(environment, local_name, globals) + except TemplateNotFound: + # re-raise the exception with the correct filename here. + # (the one that includes the prefix) + raise TemplateNotFound(name) + + def list_templates(self): + result = [] + for prefix, loader in self.mapping.items(): + for template in loader.list_templates(): + result.append(prefix + self.delimiter + template) + return result + + +class ChoiceLoader(BaseLoader): + """This loader works like the `PrefixLoader` just that no prefix is + specified. If a template could not be found by one loader the next one + is tried. + + >>> loader = ChoiceLoader([ + ... FileSystemLoader('/path/to/user/templates'), + ... FileSystemLoader('/path/to/system/templates') + ... ]) + + This is useful if you want to allow users to override builtin templates + from a different location. + """ + + def __init__(self, loaders): + self.loaders = loaders + + def get_source(self, environment, template): + for loader in self.loaders: + try: + return loader.get_source(environment, template) + except TemplateNotFound: + pass + raise TemplateNotFound(template) + + @internalcode + def load(self, environment, name, globals=None): + for loader in self.loaders: + try: + return loader.load(environment, name, globals) + except TemplateNotFound: + pass + raise TemplateNotFound(name) + + def list_templates(self): + found = set() + for loader in self.loaders: + found.update(loader.list_templates()) + return sorted(found) + + +class _TemplateModule(ModuleType): + """Like a normal module but with support for weak references""" + + +class ModuleLoader(BaseLoader): + """This loader loads templates from precompiled templates. + + Example usage: + + >>> loader = ChoiceLoader([ + ... ModuleLoader('/path/to/compiled/templates'), + ... FileSystemLoader('/path/to/templates') + ... ]) + + Templates can be precompiled with :meth:`Environment.compile_templates`. + """ + + has_source_access = False + + def __init__(self, path): + package_name = f"_jinja2_module_templates_{id(self):x}" + + # create a fake module that looks for the templates in the + # path given. + mod = _TemplateModule(package_name) + + if not isinstance(path, abc.Iterable) or isinstance(path, str): + path = [path] + + mod.__path__ = [os.fspath(p) for p in path] + + sys.modules[package_name] = weakref.proxy( + mod, lambda x: sys.modules.pop(package_name, None) + ) + + # the only strong reference, the sys.modules entry is weak + # so that the garbage collector can remove it once the + # loader that created it goes out of business. + self.module = mod + self.package_name = package_name + + @staticmethod + def get_template_key(name): + return "tmpl_" + sha1(name.encode("utf-8")).hexdigest() + + @staticmethod + def get_module_filename(name): + return ModuleLoader.get_template_key(name) + ".py" + + @internalcode + def load(self, environment, name, globals=None): + key = self.get_template_key(name) + module = f"{self.package_name}.{key}" + mod = getattr(self.module, module, None) + if mod is None: + try: + mod = __import__(module, None, None, ["root"]) + except ImportError: + raise TemplateNotFound(name) + + # remove the entry from sys.modules, we only want the attribute + # on the module object we have stored on the loader. + sys.modules.pop(module, None) + + return environment.template_class.from_module_dict( + environment, mod.__dict__, globals + ) diff --git a/src/jinja2/meta.py b/src/jinja2/meta.py new file mode 100644 index 00000000..899e179a --- /dev/null +++ b/src/jinja2/meta.py @@ -0,0 +1,98 @@ +"""Functions that expose information about templates that might be +interesting for introspection. +""" +from . import nodes +from .compiler import CodeGenerator + + +class TrackingCodeGenerator(CodeGenerator): + """We abuse the code generator for introspection.""" + + def __init__(self, environment): + CodeGenerator.__init__(self, environment, "<introspection>", "<introspection>") + self.undeclared_identifiers = set() + + def write(self, x): + """Don't write.""" + + def enter_frame(self, frame): + """Remember all undeclared identifiers.""" + CodeGenerator.enter_frame(self, frame) + for _, (action, param) in frame.symbols.loads.items(): + if action == "resolve" and param not in self.environment.globals: + self.undeclared_identifiers.add(param) + + +def find_undeclared_variables(ast): + """Returns a set of all variables in the AST that will be looked up from + the context at runtime. Because at compile time it's not known which + variables will be used depending on the path the execution takes at + runtime, all variables are returned. + + >>> from jinja2 import Environment, meta + >>> env = Environment() + >>> ast = env.parse('{% set foo = 42 %}{{ bar + foo }}') + >>> meta.find_undeclared_variables(ast) == {'bar'} + True + + .. admonition:: Implementation + + Internally the code generator is used for finding undeclared variables. + This is good to know because the code generator might raise a + :exc:`TemplateAssertionError` during compilation and as a matter of + fact this function can currently raise that exception as well. + """ + codegen = TrackingCodeGenerator(ast.environment) + codegen.visit(ast) + return codegen.undeclared_identifiers + + +def find_referenced_templates(ast): + """Finds all the referenced templates from the AST. This will return an + iterator over all the hardcoded template extensions, inclusions and + imports. If dynamic inheritance or inclusion is used, `None` will be + yielded. + + >>> from jinja2 import Environment, meta + >>> env = Environment() + >>> ast = env.parse('{% extends "layout.html" %}{% include helper %}') + >>> list(meta.find_referenced_templates(ast)) + ['layout.html', None] + + This function is useful for dependency tracking. For example if you want + to rebuild parts of the website after a layout template has changed. + """ + for node in ast.find_all( + (nodes.Extends, nodes.FromImport, nodes.Import, nodes.Include) + ): + if not isinstance(node.template, nodes.Const): + # a tuple with some non consts in there + if isinstance(node.template, (nodes.Tuple, nodes.List)): + for template_name in node.template.items: + # something const, only yield the strings and ignore + # non-string consts that really just make no sense + if isinstance(template_name, nodes.Const): + if isinstance(template_name.value, str): + yield template_name.value + # something dynamic in there + else: + yield None + # something dynamic we don't know about here + else: + yield None + continue + # constant is a basestring, direct template name + if isinstance(node.template.value, str): + yield node.template.value + # a tuple or list (latter *should* not happen) made of consts, + # yield the consts that are strings. We could warn here for + # non string values + elif isinstance(node, nodes.Include) and isinstance( + node.template.value, (tuple, list) + ): + for template_name in node.template.value: + if isinstance(template_name, str): + yield template_name + # something else we don't care about, we could warn here + else: + yield None diff --git a/src/jinja2/nativetypes.py b/src/jinja2/nativetypes.py new file mode 100644 index 00000000..5ecf72b5 --- /dev/null +++ b/src/jinja2/nativetypes.py @@ -0,0 +1,93 @@ +from ast import literal_eval +from itertools import chain +from itertools import islice + +from . import nodes +from .compiler import CodeGenerator +from .compiler import has_safe_repr +from .environment import Environment +from .environment import Template + + +def native_concat(nodes): + """Return a native Python type from the list of compiled nodes. If + the result is a single node, its value is returned. Otherwise, the + nodes are concatenated as strings. If the result can be parsed with + :func:`ast.literal_eval`, the parsed value is returned. Otherwise, + the string is returned. + + :param nodes: Iterable of nodes to concatenate. + """ + head = list(islice(nodes, 2)) + + if not head: + return None + + if len(head) == 1: + raw = head[0] + else: + raw = "".join([str(v) for v in chain(head, nodes)]) + + try: + return literal_eval(raw) + except (ValueError, SyntaxError, MemoryError): + return raw + + +class NativeCodeGenerator(CodeGenerator): + """A code generator which renders Python types by not adding + ``str()`` around output nodes. + """ + + @staticmethod + def _default_finalize(value): + return value + + def _output_const_repr(self, group): + return repr("".join([str(v) for v in group])) + + def _output_child_to_const(self, node, frame, finalize): + const = node.as_const(frame.eval_ctx) + + if not has_safe_repr(const): + raise nodes.Impossible() + + if isinstance(node, nodes.TemplateData): + return const + + return finalize.const(const) + + def _output_child_pre(self, node, frame, finalize): + if finalize.src is not None: + self.write(finalize.src) + + def _output_child_post(self, node, frame, finalize): + if finalize.src is not None: + self.write(")") + + +class NativeEnvironment(Environment): + """An environment that renders templates to native Python types.""" + + code_generator_class = NativeCodeGenerator + + +class NativeTemplate(Template): + environment_class = NativeEnvironment + + def render(self, *args, **kwargs): + """Render the template to produce a native Python type. If the + result is a single node, its value is returned. Otherwise, the + nodes are concatenated as strings. If the result can be parsed + with :func:`ast.literal_eval`, the parsed value is returned. + Otherwise, the string is returned. + """ + vars = dict(*args, **kwargs) + + try: + return native_concat(self.root_render_func(self.new_context(vars))) + except Exception: + return self.environment.handle_exception() + + +NativeEnvironment.template_class = NativeTemplate diff --git a/src/jinja2/nodes.py b/src/jinja2/nodes.py new file mode 100644 index 00000000..d5133f75 --- /dev/null +++ b/src/jinja2/nodes.py @@ -0,0 +1,1052 @@ +"""AST nodes generated by the parser for the compiler. Also provides +some node tree helper functions used by the parser and compiler in order +to normalize nodes. +""" +import operator +from collections import deque + +from markupsafe import Markup + +_binop_to_func = { + "*": operator.mul, + "/": operator.truediv, + "//": operator.floordiv, + "**": operator.pow, + "%": operator.mod, + "+": operator.add, + "-": operator.sub, +} + +_uaop_to_func = { + "not": operator.not_, + "+": operator.pos, + "-": operator.neg, +} + +_cmpop_to_func = { + "eq": operator.eq, + "ne": operator.ne, + "gt": operator.gt, + "gteq": operator.ge, + "lt": operator.lt, + "lteq": operator.le, + "in": lambda a, b: a in b, + "notin": lambda a, b: a not in b, +} + + +class Impossible(Exception): + """Raised if the node could not perform a requested action.""" + + +class NodeType(type): + """A metaclass for nodes that handles the field and attribute + inheritance. fields and attributes from the parent class are + automatically forwarded to the child.""" + + def __new__(mcs, name, bases, d): + for attr in "fields", "attributes": + storage = [] + storage.extend(getattr(bases[0] if bases else object, attr, ())) + storage.extend(d.get(attr, ())) + assert len(bases) <= 1, "multiple inheritance not allowed" + assert len(storage) == len(set(storage)), "layout conflict" + d[attr] = tuple(storage) + d.setdefault("abstract", False) + return type.__new__(mcs, name, bases, d) + + +class EvalContext: + """Holds evaluation time information. Custom attributes can be attached + to it in extensions. + """ + + def __init__(self, environment, template_name=None): + self.environment = environment + if callable(environment.autoescape): + self.autoescape = environment.autoescape(template_name) + else: + self.autoescape = environment.autoescape + self.volatile = False + + def save(self): + return self.__dict__.copy() + + def revert(self, old): + self.__dict__.clear() + self.__dict__.update(old) + + +def get_eval_context(node, ctx): + if ctx is None: + if node.environment is None: + raise RuntimeError( + "if no eval context is passed, the node must have an" + " attached environment." + ) + return EvalContext(node.environment) + return ctx + + +class Node(metaclass=NodeType): + """Baseclass for all Jinja nodes. There are a number of nodes available + of different types. There are four major types: + + - :class:`Stmt`: statements + - :class:`Expr`: expressions + - :class:`Helper`: helper nodes + - :class:`Template`: the outermost wrapper node + + All nodes have fields and attributes. Fields may be other nodes, lists, + or arbitrary values. Fields are passed to the constructor as regular + positional arguments, attributes as keyword arguments. Each node has + two attributes: `lineno` (the line number of the node) and `environment`. + The `environment` attribute is set at the end of the parsing process for + all nodes automatically. + """ + + fields = () + attributes = ("lineno", "environment") + abstract = True + + def __init__(self, *fields, **attributes): + if self.abstract: + raise TypeError("abstract nodes are not instantiable") + if fields: + if len(fields) != len(self.fields): + if not self.fields: + raise TypeError(f"{self.__class__.__name__!r} takes 0 arguments") + raise TypeError( + f"{self.__class__.__name__!r} takes 0 or {len(self.fields)}" + f" argument{'s' if len(self.fields) != 1 else ''}" + ) + for name, arg in zip(self.fields, fields): + setattr(self, name, arg) + for attr in self.attributes: + setattr(self, attr, attributes.pop(attr, None)) + if attributes: + raise TypeError(f"unknown attribute {next(iter(attributes))!r}") + + def iter_fields(self, exclude=None, only=None): + """This method iterates over all fields that are defined and yields + ``(key, value)`` tuples. Per default all fields are returned, but + it's possible to limit that to some fields by providing the `only` + parameter or to exclude some using the `exclude` parameter. Both + should be sets or tuples of field names. + """ + for name in self.fields: + if ( + (exclude is only is None) + or (exclude is not None and name not in exclude) + or (only is not None and name in only) + ): + try: + yield name, getattr(self, name) + except AttributeError: + pass + + def iter_child_nodes(self, exclude=None, only=None): + """Iterates over all direct child nodes of the node. This iterates + over all fields and yields the values of they are nodes. If the value + of a field is a list all the nodes in that list are returned. + """ + for _, item in self.iter_fields(exclude, only): + if isinstance(item, list): + for n in item: + if isinstance(n, Node): + yield n + elif isinstance(item, Node): + yield item + + def find(self, node_type): + """Find the first node of a given type. If no such node exists the + return value is `None`. + """ + for result in self.find_all(node_type): + return result + + def find_all(self, node_type): + """Find all the nodes of a given type. If the type is a tuple, + the check is performed for any of the tuple items. + """ + for child in self.iter_child_nodes(): + if isinstance(child, node_type): + yield child + yield from child.find_all(node_type) + + def set_ctx(self, ctx): + """Reset the context of a node and all child nodes. Per default the + parser will all generate nodes that have a 'load' context as it's the + most common one. This method is used in the parser to set assignment + targets and other nodes to a store context. + """ + todo = deque([self]) + while todo: + node = todo.popleft() + if "ctx" in node.fields: + node.ctx = ctx + todo.extend(node.iter_child_nodes()) + return self + + def set_lineno(self, lineno, override=False): + """Set the line numbers of the node and children.""" + todo = deque([self]) + while todo: + node = todo.popleft() + if "lineno" in node.attributes: + if node.lineno is None or override: + node.lineno = lineno + todo.extend(node.iter_child_nodes()) + return self + + def set_environment(self, environment): + """Set the environment for all nodes.""" + todo = deque([self]) + while todo: + node = todo.popleft() + node.environment = environment + todo.extend(node.iter_child_nodes()) + return self + + def __eq__(self, other): + if type(self) is not type(other): + return NotImplemented + + return tuple(self.iter_fields()) == tuple(other.iter_fields()) + + def __hash__(self): + return hash(tuple(self.iter_fields())) + + def __repr__(self): + args_str = ", ".join(f"{a}={getattr(self, a, None)!r}" for a in self.fields) + return f"{self.__class__.__name__}({args_str})" + + def dump(self): + def _dump(node): + if not isinstance(node, Node): + buf.append(repr(node)) + return + + buf.append(f"nodes.{node.__class__.__name__}(") + if not node.fields: + buf.append(")") + return + for idx, field in enumerate(node.fields): + if idx: + buf.append(", ") + value = getattr(node, field) + if isinstance(value, list): + buf.append("[") + for idx, item in enumerate(value): + if idx: + buf.append(", ") + _dump(item) + buf.append("]") + else: + _dump(value) + buf.append(")") + + buf = [] + _dump(self) + return "".join(buf) + + +class Stmt(Node): + """Base node for all statements.""" + + abstract = True + + +class Helper(Node): + """Nodes that exist in a specific context only.""" + + abstract = True + + +class Template(Node): + """Node that represents a template. This must be the outermost node that + is passed to the compiler. + """ + + fields = ("body",) + + +class Output(Stmt): + """A node that holds multiple expressions which are then printed out. + This is used both for the `print` statement and the regular template data. + """ + + fields = ("nodes",) + + +class Extends(Stmt): + """Represents an extends statement.""" + + fields = ("template",) + + +class For(Stmt): + """The for loop. `target` is the target for the iteration (usually a + :class:`Name` or :class:`Tuple`), `iter` the iterable. `body` is a list + of nodes that are used as loop-body, and `else_` a list of nodes for the + `else` block. If no else node exists it has to be an empty list. + + For filtered nodes an expression can be stored as `test`, otherwise `None`. + """ + + fields = ("target", "iter", "body", "else_", "test", "recursive") + + +class If(Stmt): + """If `test` is true, `body` is rendered, else `else_`.""" + + fields = ("test", "body", "elif_", "else_") + + +class Macro(Stmt): + """A macro definition. `name` is the name of the macro, `args` a list of + arguments and `defaults` a list of defaults if there are any. `body` is + a list of nodes for the macro body. + """ + + fields = ("name", "args", "defaults", "body") + + +class CallBlock(Stmt): + """Like a macro without a name but a call instead. `call` is called with + the unnamed macro as `caller` argument this node holds. + """ + + fields = ("call", "args", "defaults", "body") + + +class FilterBlock(Stmt): + """Node for filter sections.""" + + fields = ("body", "filter") + + +class With(Stmt): + """Specific node for with statements. In older versions of Jinja the + with statement was implemented on the base of the `Scope` node instead. + + .. versionadded:: 2.9.3 + """ + + fields = ("targets", "values", "body") + + +class Block(Stmt): + """A node that represents a block.""" + + fields = ("name", "body", "scoped") + + +class Include(Stmt): + """A node that represents the include tag.""" + + fields = ("template", "with_context", "ignore_missing") + + +class Import(Stmt): + """A node that represents the import tag.""" + + fields = ("template", "target", "with_context") + + +class FromImport(Stmt): + """A node that represents the from import tag. It's important to not + pass unsafe names to the name attribute. The compiler translates the + attribute lookups directly into getattr calls and does *not* use the + subscript callback of the interface. As exported variables may not + start with double underscores (which the parser asserts) this is not a + problem for regular Jinja code, but if this node is used in an extension + extra care must be taken. + + The list of names may contain tuples if aliases are wanted. + """ + + fields = ("template", "names", "with_context") + + +class ExprStmt(Stmt): + """A statement that evaluates an expression and discards the result.""" + + fields = ("node",) + + +class Assign(Stmt): + """Assigns an expression to a target.""" + + fields = ("target", "node") + + +class AssignBlock(Stmt): + """Assigns a block to a target.""" + + fields = ("target", "filter", "body") + + +class Expr(Node): + """Baseclass for all expressions.""" + + abstract = True + + def as_const(self, eval_ctx=None): + """Return the value of the expression as constant or raise + :exc:`Impossible` if this was not possible. + + An :class:`EvalContext` can be provided, if none is given + a default context is created which requires the nodes to have + an attached environment. + + .. versionchanged:: 2.4 + the `eval_ctx` parameter was added. + """ + raise Impossible() + + def can_assign(self): + """Check if it's possible to assign something to this node.""" + return False + + +class BinExpr(Expr): + """Baseclass for all binary expressions.""" + + fields = ("left", "right") + operator = None + abstract = True + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + # intercepted operators cannot be folded at compile time + if ( + self.environment.sandboxed + and self.operator in self.environment.intercepted_binops + ): + raise Impossible() + f = _binop_to_func[self.operator] + try: + return f(self.left.as_const(eval_ctx), self.right.as_const(eval_ctx)) + except Exception: + raise Impossible() + + +class UnaryExpr(Expr): + """Baseclass for all unary expressions.""" + + fields = ("node",) + operator = None + abstract = True + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + # intercepted operators cannot be folded at compile time + if ( + self.environment.sandboxed + and self.operator in self.environment.intercepted_unops + ): + raise Impossible() + f = _uaop_to_func[self.operator] + try: + return f(self.node.as_const(eval_ctx)) + except Exception: + raise Impossible() + + +class Name(Expr): + """Looks up a name or stores a value in a name. + The `ctx` of the node can be one of the following values: + + - `store`: store a value in the name + - `load`: load that name + - `param`: like `store` but if the name was defined as function parameter. + """ + + fields = ("name", "ctx") + + def can_assign(self): + return self.name not in ("true", "false", "none", "True", "False", "None") + + +class NSRef(Expr): + """Reference to a namespace value assignment""" + + fields = ("name", "attr") + + def can_assign(self): + # We don't need any special checks here; NSRef assignments have a + # runtime check to ensure the target is a namespace object which will + # have been checked already as it is created using a normal assignment + # which goes through a `Name` node. + return True + + +class Literal(Expr): + """Baseclass for literals.""" + + abstract = True + + +class Const(Literal): + """All constant values. The parser will return this node for simple + constants such as ``42`` or ``"foo"`` but it can be used to store more + complex values such as lists too. Only constants with a safe + representation (objects where ``eval(repr(x)) == x`` is true). + """ + + fields = ("value",) + + def as_const(self, eval_ctx=None): + return self.value + + @classmethod + def from_untrusted(cls, value, lineno=None, environment=None): + """Return a const object if the value is representable as + constant value in the generated code, otherwise it will raise + an `Impossible` exception. + """ + from .compiler import has_safe_repr + + if not has_safe_repr(value): + raise Impossible() + return cls(value, lineno=lineno, environment=environment) + + +class TemplateData(Literal): + """A constant template string.""" + + fields = ("data",) + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + if eval_ctx.volatile: + raise Impossible() + if eval_ctx.autoescape: + return Markup(self.data) + return self.data + + +class Tuple(Literal): + """For loop unpacking and some other things like multiple arguments + for subscripts. Like for :class:`Name` `ctx` specifies if the tuple + is used for loading the names or storing. + """ + + fields = ("items", "ctx") + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return tuple(x.as_const(eval_ctx) for x in self.items) + + def can_assign(self): + for item in self.items: + if not item.can_assign(): + return False + return True + + +class List(Literal): + """Any list literal such as ``[1, 2, 3]``""" + + fields = ("items",) + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return [x.as_const(eval_ctx) for x in self.items] + + +class Dict(Literal): + """Any dict literal such as ``{1: 2, 3: 4}``. The items must be a list of + :class:`Pair` nodes. + """ + + fields = ("items",) + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return dict(x.as_const(eval_ctx) for x in self.items) + + +class Pair(Helper): + """A key, value pair for dicts.""" + + fields = ("key", "value") + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return self.key.as_const(eval_ctx), self.value.as_const(eval_ctx) + + +class Keyword(Helper): + """A key, value pair for keyword arguments where key is a string.""" + + fields = ("key", "value") + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return self.key, self.value.as_const(eval_ctx) + + +class CondExpr(Expr): + """A conditional expression (inline if expression). (``{{ + foo if bar else baz }}``) + """ + + fields = ("test", "expr1", "expr2") + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + if self.test.as_const(eval_ctx): + return self.expr1.as_const(eval_ctx) + + # if we evaluate to an undefined object, we better do that at runtime + if self.expr2 is None: + raise Impossible() + + return self.expr2.as_const(eval_ctx) + + +def args_as_const(node, eval_ctx): + args = [x.as_const(eval_ctx) for x in node.args] + kwargs = dict(x.as_const(eval_ctx) for x in node.kwargs) + + if node.dyn_args is not None: + try: + args.extend(node.dyn_args.as_const(eval_ctx)) + except Exception: + raise Impossible() + + if node.dyn_kwargs is not None: + try: + kwargs.update(node.dyn_kwargs.as_const(eval_ctx)) + except Exception: + raise Impossible() + + return args, kwargs + + +class Filter(Expr): + """This node applies a filter on an expression. `name` is the name of + the filter, the rest of the fields are the same as for :class:`Call`. + + If the `node` of a filter is `None` the contents of the last buffer are + filtered. Buffers are created by macros and filter blocks. + """ + + fields = ("node", "name", "args", "kwargs", "dyn_args", "dyn_kwargs") + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + + if eval_ctx.volatile or self.node is None: + raise Impossible() + + filter_ = self.environment.filters.get(self.name) + + if filter_ is None or getattr(filter_, "contextfilter", False) is True: + raise Impossible() + + # We cannot constant handle async filters, so we need to make sure + # to not go down this path. + if eval_ctx.environment.is_async and getattr( + filter_, "asyncfiltervariant", False + ): + raise Impossible() + + args, kwargs = args_as_const(self, eval_ctx) + args.insert(0, self.node.as_const(eval_ctx)) + + if getattr(filter_, "evalcontextfilter", False) is True: + args.insert(0, eval_ctx) + elif getattr(filter_, "environmentfilter", False) is True: + args.insert(0, self.environment) + + try: + return filter_(*args, **kwargs) + except Exception: + raise Impossible() + + +class Test(Expr): + """Applies a test on an expression. `name` is the name of the test, the + rest of the fields are the same as for :class:`Call`. + """ + + fields = ("node", "name", "args", "kwargs", "dyn_args", "dyn_kwargs") + + def as_const(self, eval_ctx=None): + test = self.environment.tests.get(self.name) + + if test is None: + raise Impossible() + + eval_ctx = get_eval_context(self, eval_ctx) + args, kwargs = args_as_const(self, eval_ctx) + args.insert(0, self.node.as_const(eval_ctx)) + + try: + return test(*args, **kwargs) + except Exception: + raise Impossible() + + +class Call(Expr): + """Calls an expression. `args` is a list of arguments, `kwargs` a list + of keyword arguments (list of :class:`Keyword` nodes), and `dyn_args` + and `dyn_kwargs` has to be either `None` or a node that is used as + node for dynamic positional (``*args``) or keyword (``**kwargs``) + arguments. + """ + + fields = ("node", "args", "kwargs", "dyn_args", "dyn_kwargs") + + +class Getitem(Expr): + """Get an attribute or item from an expression and prefer the item.""" + + fields = ("node", "arg", "ctx") + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + if self.ctx != "load": + raise Impossible() + try: + return self.environment.getitem( + self.node.as_const(eval_ctx), self.arg.as_const(eval_ctx) + ) + except Exception: + raise Impossible() + + def can_assign(self): + return False + + +class Getattr(Expr): + """Get an attribute or item from an expression that is a ascii-only + bytestring and prefer the attribute. + """ + + fields = ("node", "attr", "ctx") + + def as_const(self, eval_ctx=None): + if self.ctx != "load": + raise Impossible() + try: + eval_ctx = get_eval_context(self, eval_ctx) + return self.environment.getattr(self.node.as_const(eval_ctx), self.attr) + except Exception: + raise Impossible() + + def can_assign(self): + return False + + +class Slice(Expr): + """Represents a slice object. This must only be used as argument for + :class:`Subscript`. + """ + + fields = ("start", "stop", "step") + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + + def const(obj): + if obj is None: + return None + return obj.as_const(eval_ctx) + + return slice(const(self.start), const(self.stop), const(self.step)) + + +class Concat(Expr): + """Concatenates the list of expressions provided after converting + them to strings. + """ + + fields = ("nodes",) + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return "".join(str(x.as_const(eval_ctx)) for x in self.nodes) + + +class Compare(Expr): + """Compares an expression with some other expressions. `ops` must be a + list of :class:`Operand`\\s. + """ + + fields = ("expr", "ops") + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + result = value = self.expr.as_const(eval_ctx) + + try: + for op in self.ops: + new_value = op.expr.as_const(eval_ctx) + result = _cmpop_to_func[op.op](value, new_value) + + if not result: + return False + + value = new_value + except Exception: + raise Impossible() + + return result + + +class Operand(Helper): + """Holds an operator and an expression.""" + + fields = ("op", "expr") + + +class Mul(BinExpr): + """Multiplies the left with the right node.""" + + operator = "*" + + +class Div(BinExpr): + """Divides the left by the right node.""" + + operator = "/" + + +class FloorDiv(BinExpr): + """Divides the left by the right node and truncates conver the + result into an integer by truncating. + """ + + operator = "//" + + +class Add(BinExpr): + """Add the left to the right node.""" + + operator = "+" + + +class Sub(BinExpr): + """Subtract the right from the left node.""" + + operator = "-" + + +class Mod(BinExpr): + """Left modulo right.""" + + operator = "%" + + +class Pow(BinExpr): + """Left to the power of right.""" + + operator = "**" + + +class And(BinExpr): + """Short circuited AND.""" + + operator = "and" + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return self.left.as_const(eval_ctx) and self.right.as_const(eval_ctx) + + +class Or(BinExpr): + """Short circuited OR.""" + + operator = "or" + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return self.left.as_const(eval_ctx) or self.right.as_const(eval_ctx) + + +class Not(UnaryExpr): + """Negate the expression.""" + + operator = "not" + + +class Neg(UnaryExpr): + """Make the expression negative.""" + + operator = "-" + + +class Pos(UnaryExpr): + """Make the expression positive (noop for most expressions)""" + + operator = "+" + + +# Helpers for extensions + + +class EnvironmentAttribute(Expr): + """Loads an attribute from the environment object. This is useful for + extensions that want to call a callback stored on the environment. + """ + + fields = ("name",) + + +class ExtensionAttribute(Expr): + """Returns the attribute of an extension bound to the environment. + The identifier is the identifier of the :class:`Extension`. + + This node is usually constructed by calling the + :meth:`~jinja2.ext.Extension.attr` method on an extension. + """ + + fields = ("identifier", "name") + + +class ImportedName(Expr): + """If created with an import name the import name is returned on node + access. For example ``ImportedName('cgi.escape')`` returns the `escape` + function from the cgi module on evaluation. Imports are optimized by the + compiler so there is no need to assign them to local variables. + """ + + fields = ("importname",) + + +class InternalName(Expr): + """An internal name in the compiler. You cannot create these nodes + yourself but the parser provides a + :meth:`~jinja2.parser.Parser.free_identifier` method that creates + a new identifier for you. This identifier is not available from the + template and is not threated specially by the compiler. + """ + + fields = ("name",) + + def __init__(self): + raise TypeError( + "Can't create internal names. Use the " + "`free_identifier` method on a parser." + ) + + +class MarkSafe(Expr): + """Mark the wrapped expression as safe (wrap it as `Markup`).""" + + fields = ("expr",) + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + return Markup(self.expr.as_const(eval_ctx)) + + +class MarkSafeIfAutoescape(Expr): + """Mark the wrapped expression as safe (wrap it as `Markup`) but + only if autoescaping is active. + + .. versionadded:: 2.5 + """ + + fields = ("expr",) + + def as_const(self, eval_ctx=None): + eval_ctx = get_eval_context(self, eval_ctx) + if eval_ctx.volatile: + raise Impossible() + expr = self.expr.as_const(eval_ctx) + if eval_ctx.autoescape: + return Markup(expr) + return expr + + +class ContextReference(Expr): + """Returns the current template context. It can be used like a + :class:`Name` node, with a ``'load'`` ctx and will return the + current :class:`~jinja2.runtime.Context` object. + + Here an example that assigns the current template name to a + variable named `foo`:: + + Assign(Name('foo', ctx='store'), + Getattr(ContextReference(), 'name')) + + This is basically equivalent to using the + :func:`~jinja2.contextfunction` decorator when using the + high-level API, which causes a reference to the context to be passed + as the first argument to a function. + """ + + +class DerivedContextReference(Expr): + """Return the current template context including locals. Behaves + exactly like :class:`ContextReference`, but includes local + variables, such as from a ``for`` loop. + + .. versionadded:: 2.11 + """ + + +class Continue(Stmt): + """Continue a loop.""" + + +class Break(Stmt): + """Break a loop.""" + + +class Scope(Stmt): + """An artificial scope.""" + + fields = ("body",) + + +class OverlayScope(Stmt): + """An overlay scope for extensions. This is a largely unoptimized scope + that however can be used to introduce completely arbitrary variables into + a sub scope from a dictionary or dictionary like object. The `context` + field has to evaluate to a dictionary object. + + Example usage:: + + OverlayScope(context=self.call_method('get_context'), + body=[...]) + + .. versionadded:: 2.10 + """ + + fields = ("context", "body") + + +class EvalContextModifier(Stmt): + """Modifies the eval context. For each option that should be modified, + a :class:`Keyword` has to be added to the :attr:`options` list. + + Example to change the `autoescape` setting:: + + EvalContextModifier(options=[Keyword('autoescape', Const(True))]) + """ + + fields = ("options",) + + +class ScopedEvalContextModifier(EvalContextModifier): + """Modifies the eval context and reverts it later. Works exactly like + :class:`EvalContextModifier` but will only modify the + :class:`~jinja2.nodes.EvalContext` for nodes in the :attr:`body`. + """ + + fields = ("body",) + + +# make sure nobody creates custom nodes +def _failing_new(*args, **kwargs): + raise TypeError("can't create custom node types") + + +NodeType.__new__ = staticmethod(_failing_new) +del _failing_new diff --git a/src/jinja2/optimizer.py b/src/jinja2/optimizer.py new file mode 100644 index 00000000..39d059f1 --- /dev/null +++ b/src/jinja2/optimizer.py @@ -0,0 +1,40 @@ +"""The optimizer tries to constant fold expressions and modify the AST +in place so that it should be faster to evaluate. + +Because the AST does not contain all the scoping information and the +compiler has to find that out, we cannot do all the optimizations we +want. For example, loop unrolling doesn't work because unrolled loops +would have a different scope. The solution would be a second syntax tree +that stored the scoping rules. +""" +from . import nodes +from .visitor import NodeTransformer + + +def optimize(node, environment): + """The context hint can be used to perform an static optimization + based on the context given.""" + optimizer = Optimizer(environment) + return optimizer.visit(node) + + +class Optimizer(NodeTransformer): + def __init__(self, environment): + self.environment = environment + + def generic_visit(self, node, *args, **kwargs): + node = super().generic_visit(node, *args, **kwargs) + + # Do constant folding. Some other nodes besides Expr have + # as_const, but folding them causes errors later on. + if isinstance(node, nodes.Expr): + try: + return nodes.Const.from_untrusted( + node.as_const(args[0] if args else None), + lineno=node.lineno, + environment=self.environment, + ) + except nodes.Impossible: + pass + + return node diff --git a/src/jinja2/parser.py b/src/jinja2/parser.py new file mode 100644 index 00000000..eedea7a0 --- /dev/null +++ b/src/jinja2/parser.py @@ -0,0 +1,934 @@ +"""Parse tokens from the lexer into nodes for the compiler.""" +from . import nodes +from .exceptions import TemplateAssertionError +from .exceptions import TemplateSyntaxError +from .lexer import describe_token +from .lexer import describe_token_expr + +_statement_keywords = frozenset( + [ + "for", + "if", + "block", + "extends", + "print", + "macro", + "include", + "from", + "import", + "set", + "with", + "autoescape", + ] +) +_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"]) + +_math_nodes = { + "add": nodes.Add, + "sub": nodes.Sub, + "mul": nodes.Mul, + "div": nodes.Div, + "floordiv": nodes.FloorDiv, + "mod": nodes.Mod, +} + + +class Parser: + """This is the central parsing class Jinja uses. It's passed to + extensions and can be used to parse expressions or statements. + """ + + def __init__(self, environment, source, name=None, filename=None, state=None): + self.environment = environment + self.stream = environment._tokenize(source, name, filename, state) + self.name = name + self.filename = filename + self.closed = False + self.extensions = {} + for extension in environment.iter_extensions(): + for tag in extension.tags: + self.extensions[tag] = extension.parse + self._last_identifier = 0 + self._tag_stack = [] + self._end_token_stack = [] + + def fail(self, msg, lineno=None, exc=TemplateSyntaxError): + """Convenience method that raises `exc` with the message, passed + line number or last line number as well as the current name and + filename. + """ + if lineno is None: + lineno = self.stream.current.lineno + raise exc(msg, lineno, self.name, self.filename) + + def _fail_ut_eof(self, name, end_token_stack, lineno): + expected = [] + for exprs in end_token_stack: + expected.extend(map(describe_token_expr, exprs)) + if end_token_stack: + currently_looking = " or ".join( + map(repr, map(describe_token_expr, end_token_stack[-1])) + ) + else: + currently_looking = None + + if name is None: + message = ["Unexpected end of template."] + else: + message = [f"Encountered unknown tag {name!r}."] + + if currently_looking: + if name is not None and name in expected: + message.append( + "You probably made a nesting mistake. Jinja is expecting this tag," + f" but currently looking for {currently_looking}." + ) + else: + message.append( + f"Jinja was looking for the following tags: {currently_looking}." + ) + + if self._tag_stack: + message.append( + "The innermost block that needs to be closed is" + f" {self._tag_stack[-1]!r}." + ) + + self.fail(" ".join(message), lineno) + + def fail_unknown_tag(self, name, lineno=None): + """Called if the parser encounters an unknown tag. Tries to fail + with a human readable error message that could help to identify + the problem. + """ + return self._fail_ut_eof(name, self._end_token_stack, lineno) + + def fail_eof(self, end_tokens=None, lineno=None): + """Like fail_unknown_tag but for end of template situations.""" + stack = list(self._end_token_stack) + if end_tokens is not None: + stack.append(end_tokens) + return self._fail_ut_eof(None, stack, lineno) + + def is_tuple_end(self, extra_end_rules=None): + """Are we at the end of a tuple?""" + if self.stream.current.type in ("variable_end", "block_end", "rparen"): + return True + elif extra_end_rules is not None: + return self.stream.current.test_any(extra_end_rules) + return False + + def free_identifier(self, lineno=None): + """Return a new free identifier as :class:`~jinja2.nodes.InternalName`.""" + self._last_identifier += 1 + rv = object.__new__(nodes.InternalName) + nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno) + return rv + + def parse_statement(self): + """Parse a single statement.""" + token = self.stream.current + if token.type != "name": + self.fail("tag name expected", token.lineno) + self._tag_stack.append(token.value) + pop_tag = True + try: + if token.value in _statement_keywords: + return getattr(self, "parse_" + self.stream.current.value)() + if token.value == "call": + return self.parse_call_block() + if token.value == "filter": + return self.parse_filter_block() + ext = self.extensions.get(token.value) + if ext is not None: + return ext(self) + + # did not work out, remove the token we pushed by accident + # from the stack so that the unknown tag fail function can + # produce a proper error message. + self._tag_stack.pop() + pop_tag = False + self.fail_unknown_tag(token.value, token.lineno) + finally: + if pop_tag: + self._tag_stack.pop() + + def parse_statements(self, end_tokens, drop_needle=False): + """Parse multiple statements into a list until one of the end tokens + is reached. This is used to parse the body of statements as it also + parses template data if appropriate. The parser checks first if the + current token is a colon and skips it if there is one. Then it checks + for the block end and parses until if one of the `end_tokens` is + reached. Per default the active token in the stream at the end of + the call is the matched end token. If this is not wanted `drop_needle` + can be set to `True` and the end token is removed. + """ + # the first token may be a colon for python compatibility + self.stream.skip_if("colon") + + # in the future it would be possible to add whole code sections + # by adding some sort of end of statement token and parsing those here. + self.stream.expect("block_end") + result = self.subparse(end_tokens) + + # we reached the end of the template too early, the subparser + # does not check for this, so we do that now + if self.stream.current.type == "eof": + self.fail_eof(end_tokens) + + if drop_needle: + next(self.stream) + return result + + def parse_set(self): + """Parse an assign statement.""" + lineno = next(self.stream).lineno + target = self.parse_assign_target(with_namespace=True) + if self.stream.skip_if("assign"): + expr = self.parse_tuple() + return nodes.Assign(target, expr, lineno=lineno) + filter_node = self.parse_filter(None) + body = self.parse_statements(("name:endset",), drop_needle=True) + return nodes.AssignBlock(target, filter_node, body, lineno=lineno) + + def parse_for(self): + """Parse a for loop.""" + lineno = self.stream.expect("name:for").lineno + target = self.parse_assign_target(extra_end_rules=("name:in",)) + self.stream.expect("name:in") + iter = self.parse_tuple( + with_condexpr=False, extra_end_rules=("name:recursive",) + ) + test = None + if self.stream.skip_if("name:if"): + test = self.parse_expression() + recursive = self.stream.skip_if("name:recursive") + body = self.parse_statements(("name:endfor", "name:else")) + if next(self.stream).value == "endfor": + else_ = [] + else: + else_ = self.parse_statements(("name:endfor",), drop_needle=True) + return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno) + + def parse_if(self): + """Parse an if construct.""" + node = result = nodes.If(lineno=self.stream.expect("name:if").lineno) + while 1: + node.test = self.parse_tuple(with_condexpr=False) + node.body = self.parse_statements(("name:elif", "name:else", "name:endif")) + node.elif_ = [] + node.else_ = [] + token = next(self.stream) + if token.test("name:elif"): + node = nodes.If(lineno=self.stream.current.lineno) + result.elif_.append(node) + continue + elif token.test("name:else"): + result.else_ = self.parse_statements(("name:endif",), drop_needle=True) + break + return result + + def parse_with(self): + node = nodes.With(lineno=next(self.stream).lineno) + targets = [] + values = [] + while self.stream.current.type != "block_end": + if targets: + self.stream.expect("comma") + target = self.parse_assign_target() + target.set_ctx("param") + targets.append(target) + self.stream.expect("assign") + values.append(self.parse_expression()) + node.targets = targets + node.values = values + node.body = self.parse_statements(("name:endwith",), drop_needle=True) + return node + + def parse_autoescape(self): + node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno) + node.options = [nodes.Keyword("autoescape", self.parse_expression())] + node.body = self.parse_statements(("name:endautoescape",), drop_needle=True) + return nodes.Scope([node]) + + def parse_block(self): + node = nodes.Block(lineno=next(self.stream).lineno) + node.name = self.stream.expect("name").value + node.scoped = self.stream.skip_if("name:scoped") + + # common problem people encounter when switching from django + # to jinja. we do not support hyphens in block names, so let's + # raise a nicer error message in that case. + if self.stream.current.type == "sub": + self.fail( + "Block names in Jinja have to be valid Python identifiers and may not" + " contain hyphens, use an underscore instead." + ) + + node.body = self.parse_statements(("name:endblock",), drop_needle=True) + self.stream.skip_if("name:" + node.name) + return node + + def parse_extends(self): + node = nodes.Extends(lineno=next(self.stream).lineno) + node.template = self.parse_expression() + return node + + def parse_import_context(self, node, default): + if self.stream.current.test_any( + "name:with", "name:without" + ) and self.stream.look().test("name:context"): + node.with_context = next(self.stream).value == "with" + self.stream.skip() + else: + node.with_context = default + return node + + def parse_include(self): + node = nodes.Include(lineno=next(self.stream).lineno) + node.template = self.parse_expression() + if self.stream.current.test("name:ignore") and self.stream.look().test( + "name:missing" + ): + node.ignore_missing = True + self.stream.skip(2) + else: + node.ignore_missing = False + return self.parse_import_context(node, True) + + def parse_import(self): + node = nodes.Import(lineno=next(self.stream).lineno) + node.template = self.parse_expression() + self.stream.expect("name:as") + node.target = self.parse_assign_target(name_only=True).name + return self.parse_import_context(node, False) + + def parse_from(self): + node = nodes.FromImport(lineno=next(self.stream).lineno) + node.template = self.parse_expression() + self.stream.expect("name:import") + node.names = [] + + def parse_context(): + if self.stream.current.value in ( + "with", + "without", + ) and self.stream.look().test("name:context"): + node.with_context = next(self.stream).value == "with" + self.stream.skip() + return True + return False + + while 1: + if node.names: + self.stream.expect("comma") + if self.stream.current.type == "name": + if parse_context(): + break + target = self.parse_assign_target(name_only=True) + if target.name.startswith("_"): + self.fail( + "names starting with an underline can not be imported", + target.lineno, + exc=TemplateAssertionError, + ) + if self.stream.skip_if("name:as"): + alias = self.parse_assign_target(name_only=True) + node.names.append((target.name, alias.name)) + else: + node.names.append(target.name) + if parse_context() or self.stream.current.type != "comma": + break + else: + self.stream.expect("name") + if not hasattr(node, "with_context"): + node.with_context = False + return node + + def parse_signature(self, node): + node.args = args = [] + node.defaults = defaults = [] + self.stream.expect("lparen") + while self.stream.current.type != "rparen": + if args: + self.stream.expect("comma") + arg = self.parse_assign_target(name_only=True) + arg.set_ctx("param") + if self.stream.skip_if("assign"): + defaults.append(self.parse_expression()) + elif defaults: + self.fail("non-default argument follows default argument") + args.append(arg) + self.stream.expect("rparen") + + def parse_call_block(self): + node = nodes.CallBlock(lineno=next(self.stream).lineno) + if self.stream.current.type == "lparen": + self.parse_signature(node) + else: + node.args = [] + node.defaults = [] + + node.call = self.parse_expression() + if not isinstance(node.call, nodes.Call): + self.fail("expected call", node.lineno) + node.body = self.parse_statements(("name:endcall",), drop_needle=True) + return node + + def parse_filter_block(self): + node = nodes.FilterBlock(lineno=next(self.stream).lineno) + node.filter = self.parse_filter(None, start_inline=True) + node.body = self.parse_statements(("name:endfilter",), drop_needle=True) + return node + + def parse_macro(self): + node = nodes.Macro(lineno=next(self.stream).lineno) + node.name = self.parse_assign_target(name_only=True).name + self.parse_signature(node) + node.body = self.parse_statements(("name:endmacro",), drop_needle=True) + return node + + def parse_print(self): + node = nodes.Output(lineno=next(self.stream).lineno) + node.nodes = [] + while self.stream.current.type != "block_end": + if node.nodes: + self.stream.expect("comma") + node.nodes.append(self.parse_expression()) + return node + + def parse_assign_target( + self, + with_tuple=True, + name_only=False, + extra_end_rules=None, + with_namespace=False, + ): + """Parse an assignment target. As Jinja allows assignments to + tuples, this function can parse all allowed assignment targets. Per + default assignments to tuples are parsed, that can be disable however + by setting `with_tuple` to `False`. If only assignments to names are + wanted `name_only` can be set to `True`. The `extra_end_rules` + parameter is forwarded to the tuple parsing function. If + `with_namespace` is enabled, a namespace assignment may be parsed. + """ + if with_namespace and self.stream.look().type == "dot": + token = self.stream.expect("name") + next(self.stream) # dot + attr = self.stream.expect("name") + target = nodes.NSRef(token.value, attr.value, lineno=token.lineno) + elif name_only: + token = self.stream.expect("name") + target = nodes.Name(token.value, "store", lineno=token.lineno) + else: + if with_tuple: + target = self.parse_tuple( + simplified=True, extra_end_rules=extra_end_rules + ) + else: + target = self.parse_primary() + target.set_ctx("store") + if not target.can_assign(): + self.fail( + f"can't assign to {target.__class__.__name__.lower()!r}", target.lineno + ) + return target + + def parse_expression(self, with_condexpr=True): + """Parse an expression. Per default all expressions are parsed, if + the optional `with_condexpr` parameter is set to `False` conditional + expressions are not parsed. + """ + if with_condexpr: + return self.parse_condexpr() + return self.parse_or() + + def parse_condexpr(self): + lineno = self.stream.current.lineno + expr1 = self.parse_or() + while self.stream.skip_if("name:if"): + expr2 = self.parse_or() + if self.stream.skip_if("name:else"): + expr3 = self.parse_condexpr() + else: + expr3 = None + expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno) + lineno = self.stream.current.lineno + return expr1 + + def parse_or(self): + lineno = self.stream.current.lineno + left = self.parse_and() + while self.stream.skip_if("name:or"): + right = self.parse_and() + left = nodes.Or(left, right, lineno=lineno) + lineno = self.stream.current.lineno + return left + + def parse_and(self): + lineno = self.stream.current.lineno + left = self.parse_not() + while self.stream.skip_if("name:and"): + right = self.parse_not() + left = nodes.And(left, right, lineno=lineno) + lineno = self.stream.current.lineno + return left + + def parse_not(self): + if self.stream.current.test("name:not"): + lineno = next(self.stream).lineno + return nodes.Not(self.parse_not(), lineno=lineno) + return self.parse_compare() + + def parse_compare(self): + lineno = self.stream.current.lineno + expr = self.parse_math1() + ops = [] + while 1: + token_type = self.stream.current.type + if token_type in _compare_operators: + next(self.stream) + ops.append(nodes.Operand(token_type, self.parse_math1())) + elif self.stream.skip_if("name:in"): + ops.append(nodes.Operand("in", self.parse_math1())) + elif self.stream.current.test("name:not") and self.stream.look().test( + "name:in" + ): + self.stream.skip(2) + ops.append(nodes.Operand("notin", self.parse_math1())) + else: + break + lineno = self.stream.current.lineno + if not ops: + return expr + return nodes.Compare(expr, ops, lineno=lineno) + + def parse_math1(self): + lineno = self.stream.current.lineno + left = self.parse_concat() + while self.stream.current.type in ("add", "sub"): + cls = _math_nodes[self.stream.current.type] + next(self.stream) + right = self.parse_concat() + left = cls(left, right, lineno=lineno) + lineno = self.stream.current.lineno + return left + + def parse_concat(self): + lineno = self.stream.current.lineno + args = [self.parse_math2()] + while self.stream.current.type == "tilde": + next(self.stream) + args.append(self.parse_math2()) + if len(args) == 1: + return args[0] + return nodes.Concat(args, lineno=lineno) + + def parse_math2(self): + lineno = self.stream.current.lineno + left = self.parse_pow() + while self.stream.current.type in ("mul", "div", "floordiv", "mod"): + cls = _math_nodes[self.stream.current.type] + next(self.stream) + right = self.parse_pow() + left = cls(left, right, lineno=lineno) + lineno = self.stream.current.lineno + return left + + def parse_pow(self): + lineno = self.stream.current.lineno + left = self.parse_unary() + while self.stream.current.type == "pow": + next(self.stream) + right = self.parse_unary() + left = nodes.Pow(left, right, lineno=lineno) + lineno = self.stream.current.lineno + return left + + def parse_unary(self, with_filter=True): + token_type = self.stream.current.type + lineno = self.stream.current.lineno + if token_type == "sub": + next(self.stream) + node = nodes.Neg(self.parse_unary(False), lineno=lineno) + elif token_type == "add": + next(self.stream) + node = nodes.Pos(self.parse_unary(False), lineno=lineno) + else: + node = self.parse_primary() + node = self.parse_postfix(node) + if with_filter: + node = self.parse_filter_expr(node) + return node + + def parse_primary(self): + token = self.stream.current + if token.type == "name": + if token.value in ("true", "false", "True", "False"): + node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno) + elif token.value in ("none", "None"): + node = nodes.Const(None, lineno=token.lineno) + else: + node = nodes.Name(token.value, "load", lineno=token.lineno) + next(self.stream) + elif token.type == "string": + next(self.stream) + buf = [token.value] + lineno = token.lineno + while self.stream.current.type == "string": + buf.append(self.stream.current.value) + next(self.stream) + node = nodes.Const("".join(buf), lineno=lineno) + elif token.type in ("integer", "float"): + next(self.stream) + node = nodes.Const(token.value, lineno=token.lineno) + elif token.type == "lparen": + next(self.stream) + node = self.parse_tuple(explicit_parentheses=True) + self.stream.expect("rparen") + elif token.type == "lbracket": + node = self.parse_list() + elif token.type == "lbrace": + node = self.parse_dict() + else: + self.fail(f"unexpected {describe_token(token)!r}", token.lineno) + return node + + def parse_tuple( + self, + simplified=False, + with_condexpr=True, + extra_end_rules=None, + explicit_parentheses=False, + ): + """Works like `parse_expression` but if multiple expressions are + delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created. + This method could also return a regular expression instead of a tuple + if no commas where found. + + The default parsing mode is a full tuple. If `simplified` is `True` + only names and literals are parsed. The `no_condexpr` parameter is + forwarded to :meth:`parse_expression`. + + Because tuples do not require delimiters and may end in a bogus comma + an extra hint is needed that marks the end of a tuple. For example + for loops support tuples between `for` and `in`. In that case the + `extra_end_rules` is set to ``['name:in']``. + + `explicit_parentheses` is true if the parsing was triggered by an + expression in parentheses. This is used to figure out if an empty + tuple is a valid expression or not. + """ + lineno = self.stream.current.lineno + if simplified: + parse = self.parse_primary + elif with_condexpr: + parse = self.parse_expression + else: + + def parse(): + return self.parse_expression(with_condexpr=False) + + args = [] + is_tuple = False + while 1: + if args: + self.stream.expect("comma") + if self.is_tuple_end(extra_end_rules): + break + args.append(parse()) + if self.stream.current.type == "comma": + is_tuple = True + else: + break + lineno = self.stream.current.lineno + + if not is_tuple: + if args: + return args[0] + + # if we don't have explicit parentheses, an empty tuple is + # not a valid expression. This would mean nothing (literally + # nothing) in the spot of an expression would be an empty + # tuple. + if not explicit_parentheses: + self.fail( + "Expected an expression," + f" got {describe_token(self.stream.current)!r}" + ) + + return nodes.Tuple(args, "load", lineno=lineno) + + def parse_list(self): + token = self.stream.expect("lbracket") + items = [] + while self.stream.current.type != "rbracket": + if items: + self.stream.expect("comma") + if self.stream.current.type == "rbracket": + break + items.append(self.parse_expression()) + self.stream.expect("rbracket") + return nodes.List(items, lineno=token.lineno) + + def parse_dict(self): + token = self.stream.expect("lbrace") + items = [] + while self.stream.current.type != "rbrace": + if items: + self.stream.expect("comma") + if self.stream.current.type == "rbrace": + break + key = self.parse_expression() + self.stream.expect("colon") + value = self.parse_expression() + items.append(nodes.Pair(key, value, lineno=key.lineno)) + self.stream.expect("rbrace") + return nodes.Dict(items, lineno=token.lineno) + + def parse_postfix(self, node): + while 1: + token_type = self.stream.current.type + if token_type == "dot" or token_type == "lbracket": + node = self.parse_subscript(node) + # calls are valid both after postfix expressions (getattr + # and getitem) as well as filters and tests + elif token_type == "lparen": + node = self.parse_call(node) + else: + break + return node + + def parse_filter_expr(self, node): + while 1: + token_type = self.stream.current.type + if token_type == "pipe": + node = self.parse_filter(node) + elif token_type == "name" and self.stream.current.value == "is": + node = self.parse_test(node) + # calls are valid both after postfix expressions (getattr + # and getitem) as well as filters and tests + elif token_type == "lparen": + node = self.parse_call(node) + else: + break + return node + + def parse_subscript(self, node): + token = next(self.stream) + if token.type == "dot": + attr_token = self.stream.current + next(self.stream) + if attr_token.type == "name": + return nodes.Getattr( + node, attr_token.value, "load", lineno=token.lineno + ) + elif attr_token.type != "integer": + self.fail("expected name or number", attr_token.lineno) + arg = nodes.Const(attr_token.value, lineno=attr_token.lineno) + return nodes.Getitem(node, arg, "load", lineno=token.lineno) + if token.type == "lbracket": + args = [] + while self.stream.current.type != "rbracket": + if args: + self.stream.expect("comma") + args.append(self.parse_subscribed()) + self.stream.expect("rbracket") + if len(args) == 1: + arg = args[0] + else: + arg = nodes.Tuple(args, "load", lineno=token.lineno) + return nodes.Getitem(node, arg, "load", lineno=token.lineno) + self.fail("expected subscript expression", token.lineno) + + def parse_subscribed(self): + lineno = self.stream.current.lineno + + if self.stream.current.type == "colon": + next(self.stream) + args = [None] + else: + node = self.parse_expression() + if self.stream.current.type != "colon": + return node + next(self.stream) + args = [node] + + if self.stream.current.type == "colon": + args.append(None) + elif self.stream.current.type not in ("rbracket", "comma"): + args.append(self.parse_expression()) + else: + args.append(None) + + if self.stream.current.type == "colon": + next(self.stream) + if self.stream.current.type not in ("rbracket", "comma"): + args.append(self.parse_expression()) + else: + args.append(None) + else: + args.append(None) + + return nodes.Slice(lineno=lineno, *args) + + def parse_call(self, node): + token = self.stream.expect("lparen") + args = [] + kwargs = [] + dyn_args = dyn_kwargs = None + require_comma = False + + def ensure(expr): + if not expr: + self.fail("invalid syntax for function call expression", token.lineno) + + while self.stream.current.type != "rparen": + if require_comma: + self.stream.expect("comma") + # support for trailing comma + if self.stream.current.type == "rparen": + break + if self.stream.current.type == "mul": + ensure(dyn_args is None and dyn_kwargs is None) + next(self.stream) + dyn_args = self.parse_expression() + elif self.stream.current.type == "pow": + ensure(dyn_kwargs is None) + next(self.stream) + dyn_kwargs = self.parse_expression() + else: + if ( + self.stream.current.type == "name" + and self.stream.look().type == "assign" + ): + # Parsing a kwarg + ensure(dyn_kwargs is None) + key = self.stream.current.value + self.stream.skip(2) + value = self.parse_expression() + kwargs.append(nodes.Keyword(key, value, lineno=value.lineno)) + else: + # Parsing an arg + ensure(dyn_args is None and dyn_kwargs is None and not kwargs) + args.append(self.parse_expression()) + + require_comma = True + self.stream.expect("rparen") + + if node is None: + return args, kwargs, dyn_args, dyn_kwargs + return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno) + + def parse_filter(self, node, start_inline=False): + while self.stream.current.type == "pipe" or start_inline: + if not start_inline: + next(self.stream) + token = self.stream.expect("name") + name = token.value + while self.stream.current.type == "dot": + next(self.stream) + name += "." + self.stream.expect("name").value + if self.stream.current.type == "lparen": + args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None) + else: + args = [] + kwargs = [] + dyn_args = dyn_kwargs = None + node = nodes.Filter( + node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno + ) + start_inline = False + return node + + def parse_test(self, node): + token = next(self.stream) + if self.stream.current.test("name:not"): + next(self.stream) + negated = True + else: + negated = False + name = self.stream.expect("name").value + while self.stream.current.type == "dot": + next(self.stream) + name += "." + self.stream.expect("name").value + dyn_args = dyn_kwargs = None + kwargs = [] + if self.stream.current.type == "lparen": + args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None) + elif self.stream.current.type in ( + "name", + "string", + "integer", + "float", + "lparen", + "lbracket", + "lbrace", + ) and not self.stream.current.test_any("name:else", "name:or", "name:and"): + if self.stream.current.test("name:is"): + self.fail("You cannot chain multiple tests with is") + arg_node = self.parse_primary() + arg_node = self.parse_postfix(arg_node) + args = [arg_node] + else: + args = [] + node = nodes.Test( + node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno + ) + if negated: + node = nodes.Not(node, lineno=token.lineno) + return node + + def subparse(self, end_tokens=None): + body = [] + data_buffer = [] + add_data = data_buffer.append + + if end_tokens is not None: + self._end_token_stack.append(end_tokens) + + def flush_data(): + if data_buffer: + lineno = data_buffer[0].lineno + body.append(nodes.Output(data_buffer[:], lineno=lineno)) + del data_buffer[:] + + try: + while self.stream: + token = self.stream.current + if token.type == "data": + if token.value: + add_data(nodes.TemplateData(token.value, lineno=token.lineno)) + next(self.stream) + elif token.type == "variable_begin": + next(self.stream) + add_data(self.parse_tuple(with_condexpr=True)) + self.stream.expect("variable_end") + elif token.type == "block_begin": + flush_data() + next(self.stream) + if end_tokens is not None and self.stream.current.test_any( + *end_tokens + ): + return body + rv = self.parse_statement() + if isinstance(rv, list): + body.extend(rv) + else: + body.append(rv) + self.stream.expect("block_end") + else: + raise AssertionError("internal parsing error") + + flush_data() + finally: + if end_tokens is not None: + self._end_token_stack.pop() + + return body + + def parse(self): + """Parse the whole template into a `Template` node.""" + result = nodes.Template(self.subparse(), lineno=1) + result.set_environment(self.environment) + return result diff --git a/src/jinja2/runtime.py b/src/jinja2/runtime.py new file mode 100644 index 00000000..7b5925b1 --- /dev/null +++ b/src/jinja2/runtime.py @@ -0,0 +1,919 @@ +"""The runtime functions and state used by compiled templates.""" +import sys +from collections import abc +from itertools import chain +from types import MethodType + +from markupsafe import escape # noqa: F401 +from markupsafe import Markup +from markupsafe import soft_str + +from .exceptions import TemplateNotFound # noqa: F401 +from .exceptions import TemplateRuntimeError # noqa: F401 +from .exceptions import UndefinedError +from .nodes import EvalContext +from .utils import concat +from .utils import evalcontextfunction +from .utils import internalcode +from .utils import missing +from .utils import Namespace # noqa: F401 +from .utils import object_type_repr + +# these variables are exported to the template runtime +exported = [ + "LoopContext", + "TemplateReference", + "Macro", + "Markup", + "TemplateRuntimeError", + "missing", + "concat", + "escape", + "markup_join", + "str_join", + "identity", + "TemplateNotFound", + "Namespace", + "Undefined", +] + + +def identity(x): + """Returns its argument. Useful for certain things in the + environment. + """ + return x + + +def markup_join(seq): + """Concatenation that escapes if necessary and converts to string.""" + buf = [] + iterator = map(soft_str, seq) + for arg in iterator: + buf.append(arg) + if hasattr(arg, "__html__"): + return Markup("").join(chain(buf, iterator)) + return concat(buf) + + +def str_join(seq): + """Simple args to string conversion and concatenation.""" + return concat(map(str, seq)) + + +def unicode_join(seq): + import warnings + + warnings.warn( + "This template must be recompiled with at least Jinja 3.0, or" + " it will fail in 3.1.", + DeprecationWarning, + stacklevel=2, + ) + return str_join(seq) + + +def new_context( + environment, + template_name, + blocks, + vars=None, + shared=None, + globals=None, + locals=None, +): + """Internal helper for context creation.""" + if vars is None: + vars = {} + if shared: + parent = vars + else: + parent = dict(globals or (), **vars) + if locals: + # if the parent is shared a copy should be created because + # we don't want to modify the dict passed + if shared: + parent = dict(parent) + for key, value in locals.items(): + if value is not missing: + parent[key] = value + return environment.context_class( + environment, parent, template_name, blocks, globals=globals + ) + + +class TemplateReference: + """The `self` in templates.""" + + def __init__(self, context): + self.__context = context + + def __getitem__(self, name): + blocks = self.__context.blocks[name] + return BlockReference(name, self.__context, blocks, 0) + + def __repr__(self): + return f"<{self.__class__.__name__} {self.__context.name!r}>" + + +def _get_func(x): + return getattr(x, "__func__", x) + + +class ContextMeta(type): + def __new__(mcs, name, bases, d): + rv = type.__new__(mcs, name, bases, d) + if bases == (): + return rv + + resolve = _get_func(rv.resolve) + default_resolve = _get_func(Context.resolve) + resolve_or_missing = _get_func(rv.resolve_or_missing) + default_resolve_or_missing = _get_func(Context.resolve_or_missing) + + # If we have a changed resolve but no changed default or missing + # resolve we invert the call logic. + if ( + resolve is not default_resolve + and resolve_or_missing is default_resolve_or_missing + ): + rv._legacy_resolve_mode = True + elif ( + resolve is default_resolve + and resolve_or_missing is default_resolve_or_missing + ): + rv._fast_resolve_mode = True + + return rv + + +def resolve_or_missing(context, key, missing=missing): + if key in context.vars: + return context.vars[key] + if key in context.parent: + return context.parent[key] + return missing + + +@abc.Mapping.register +class Context(metaclass=ContextMeta): + """The template context holds the variables of a template. It stores the + values passed to the template and also the names the template exports. + Creating instances is neither supported nor useful as it's created + automatically at various stages of the template evaluation and should not + be created by hand. + + The context is immutable. Modifications on :attr:`parent` **must not** + happen and modifications on :attr:`vars` are allowed from generated + template code only. Template filters and global functions marked as + :func:`contextfunction`\\s get the active context passed as first argument + and are allowed to access the context read-only. + + The template context supports read only dict operations (`get`, + `keys`, `values`, `items`, `iterkeys`, `itervalues`, `iteritems`, + `__getitem__`, `__contains__`). Additionally there is a :meth:`resolve` + method that doesn't fail with a `KeyError` but returns an + :class:`Undefined` object for missing variables. + """ + + # XXX: we want to eventually make this be a deprecation warning and + # remove it. + _legacy_resolve_mode = False + _fast_resolve_mode = False + + def __init__(self, environment, parent, name, blocks, globals=None): + self.parent = parent + self.vars = {} + self.environment = environment + self.eval_ctx = EvalContext(self.environment, name) + self.exported_vars = set() + self.name = name + self.globals_keys = set() if globals is None else set(globals) + + # create the initial mapping of blocks. Whenever template inheritance + # takes place the runtime will update this mapping with the new blocks + # from the template. + self.blocks = {k: [v] for k, v in blocks.items()} + + # In case we detect the fast resolve mode we can set up an alias + # here that bypasses the legacy code logic. + if self._fast_resolve_mode: + self.resolve_or_missing = MethodType(resolve_or_missing, self) + + def super(self, name, current): + """Render a parent block.""" + try: + blocks = self.blocks[name] + index = blocks.index(current) + 1 + blocks[index] + except LookupError: + return self.environment.undefined( + f"there is no parent block called {name!r}.", name="super" + ) + return BlockReference(name, self, blocks, index) + + def get(self, key, default=None): + """Returns an item from the template context, if it doesn't exist + `default` is returned. + """ + try: + return self[key] + except KeyError: + return default + + def resolve(self, key): + """Looks up a variable like `__getitem__` or `get` but returns an + :class:`Undefined` object with the name of the name looked up. + """ + if self._legacy_resolve_mode: + rv = resolve_or_missing(self, key) + else: + rv = self.resolve_or_missing(key) + if rv is missing: + return self.environment.undefined(name=key) + return rv + + def resolve_or_missing(self, key): + """Resolves a variable like :meth:`resolve` but returns the + special `missing` value if it cannot be found. + """ + if self._legacy_resolve_mode: + rv = self.resolve(key) + if isinstance(rv, Undefined): + rv = missing + return rv + return resolve_or_missing(self, key) + + def get_exported(self): + """Get a new dict with the exported variables.""" + return {k: self.vars[k] for k in self.exported_vars} + + def get_all(self): + """Return the complete context as dict including the exported + variables. For optimizations reasons this might not return an + actual copy so be careful with using it. + """ + if not self.vars: + return self.parent + if not self.parent: + return self.vars + return dict(self.parent, **self.vars) + + @internalcode + def call(__self, __obj, *args, **kwargs): # noqa: B902 + """Call the callable with the arguments and keyword arguments + provided but inject the active context or environment as first + argument if the callable is a :func:`contextfunction` or + :func:`environmentfunction`. + """ + if __debug__: + __traceback_hide__ = True # noqa + + # Allow callable classes to take a context + if hasattr(__obj, "__call__"): # noqa: B004 + fn = __obj.__call__ + for fn_type in ( + "contextfunction", + "evalcontextfunction", + "environmentfunction", + ): + if hasattr(fn, fn_type): + __obj = fn + break + + if callable(__obj): + if getattr(__obj, "contextfunction", False) is True: + args = (__self,) + args + elif getattr(__obj, "evalcontextfunction", False) is True: + args = (__self.eval_ctx,) + args + elif getattr(__obj, "environmentfunction", False) is True: + args = (__self.environment,) + args + try: + return __obj(*args, **kwargs) + except StopIteration: + return __self.environment.undefined( + "value was undefined because a callable raised a" + " StopIteration exception" + ) + + def derived(self, locals=None): + """Internal helper function to create a derived context. This is + used in situations where the system needs a new context in the same + template that is independent. + """ + context = new_context( + self.environment, self.name, {}, self.get_all(), True, None, locals + ) + context.eval_ctx = self.eval_ctx + context.blocks.update((k, list(v)) for k, v in self.blocks.items()) + return context + + def _all(meth): # noqa: B902 + def proxy(self): + return getattr(self.get_all(), meth)() + + proxy.__doc__ = getattr(dict, meth).__doc__ + proxy.__name__ = meth + return proxy + + keys = _all("keys") + values = _all("values") + items = _all("items") + del _all + + def __contains__(self, name): + return name in self.vars or name in self.parent + + def __getitem__(self, key): + """Lookup a variable or raise `KeyError` if the variable is + undefined. + """ + item = self.resolve_or_missing(key) + if item is missing: + raise KeyError(key) + return item + + def __repr__(self): + return f"<{self.__class__.__name__} {self.get_all()!r} of {self.name!r}>" + + +class BlockReference: + """One block on a template reference.""" + + def __init__(self, name, context, stack, depth): + self.name = name + self._context = context + self._stack = stack + self._depth = depth + + @property + def super(self): + """Super the block.""" + if self._depth + 1 >= len(self._stack): + return self._context.environment.undefined( + f"there is no parent block called {self.name!r}.", name="super" + ) + return BlockReference(self.name, self._context, self._stack, self._depth + 1) + + @internalcode + def __call__(self): + rv = concat(self._stack[self._depth](self._context)) + if self._context.eval_ctx.autoescape: + rv = Markup(rv) + return rv + + +class LoopContext: + """A wrapper iterable for dynamic ``for`` loops, with information + about the loop and iteration. + """ + + #: Current iteration of the loop, starting at 0. + index0 = -1 + + _length = None + _after = missing + _current = missing + _before = missing + _last_changed_value = missing + + def __init__(self, iterable, undefined, recurse=None, depth0=0): + """ + :param iterable: Iterable to wrap. + :param undefined: :class:`Undefined` class to use for next and + previous items. + :param recurse: The function to render the loop body when the + loop is marked recursive. + :param depth0: Incremented when looping recursively. + """ + self._iterable = iterable + self._iterator = self._to_iterator(iterable) + self._undefined = undefined + self._recurse = recurse + #: How many levels deep a recursive loop currently is, starting at 0. + self.depth0 = depth0 + + @staticmethod + def _to_iterator(iterable): + return iter(iterable) + + @property + def length(self): + """Length of the iterable. + + If the iterable is a generator or otherwise does not have a + size, it is eagerly evaluated to get a size. + """ + if self._length is not None: + return self._length + + try: + self._length = len(self._iterable) + except TypeError: + iterable = list(self._iterator) + self._iterator = self._to_iterator(iterable) + self._length = len(iterable) + self.index + (self._after is not missing) + + return self._length + + def __len__(self): + return self.length + + @property + def depth(self): + """How many levels deep a recursive loop currently is, starting at 1.""" + return self.depth0 + 1 + + @property + def index(self): + """Current iteration of the loop, starting at 1.""" + return self.index0 + 1 + + @property + def revindex0(self): + """Number of iterations from the end of the loop, ending at 0. + + Requires calculating :attr:`length`. + """ + return self.length - self.index + + @property + def revindex(self): + """Number of iterations from the end of the loop, ending at 1. + + Requires calculating :attr:`length`. + """ + return self.length - self.index0 + + @property + def first(self): + """Whether this is the first iteration of the loop.""" + return self.index0 == 0 + + def _peek_next(self): + """Return the next element in the iterable, or :data:`missing` + if the iterable is exhausted. Only peeks one item ahead, caching + the result in :attr:`_last` for use in subsequent checks. The + cache is reset when :meth:`__next__` is called. + """ + if self._after is not missing: + return self._after + + self._after = next(self._iterator, missing) + return self._after + + @property + def last(self): + """Whether this is the last iteration of the loop. + + Causes the iterable to advance early. See + :func:`itertools.groupby` for issues this can cause. + The :func:`groupby` filter avoids that issue. + """ + return self._peek_next() is missing + + @property + def previtem(self): + """The item in the previous iteration. Undefined during the + first iteration. + """ + if self.first: + return self._undefined("there is no previous item") + + return self._before + + @property + def nextitem(self): + """The item in the next iteration. Undefined during the last + iteration. + + Causes the iterable to advance early. See + :func:`itertools.groupby` for issues this can cause. + The :func:`groupby` filter avoids that issue. + """ + rv = self._peek_next() + + if rv is missing: + return self._undefined("there is no next item") + + return rv + + def cycle(self, *args): + """Return a value from the given args, cycling through based on + the current :attr:`index0`. + + :param args: One or more values to cycle through. + """ + if not args: + raise TypeError("no items for cycling given") + + return args[self.index0 % len(args)] + + def changed(self, *value): + """Return ``True`` if previously called with a different value + (including when called for the first time). + + :param value: One or more values to compare to the last call. + """ + if self._last_changed_value != value: + self._last_changed_value = value + return True + + return False + + def __iter__(self): + return self + + def __next__(self): + if self._after is not missing: + rv = self._after + self._after = missing + else: + rv = next(self._iterator) + + self.index0 += 1 + self._before = self._current + self._current = rv + return rv, self + + @internalcode + def __call__(self, iterable): + """When iterating over nested data, render the body of the loop + recursively with the given inner iterable data. + + The loop must have the ``recursive`` marker for this to work. + """ + if self._recurse is None: + raise TypeError( + "The loop must have the 'recursive' marker to be called recursively." + ) + + return self._recurse(iterable, self._recurse, depth=self.depth) + + def __repr__(self): + return f"<{self.__class__.__name__} {self.index}/{self.length}>" + + +class Macro: + """Wraps a macro function.""" + + def __init__( + self, + environment, + func, + name, + arguments, + catch_kwargs, + catch_varargs, + caller, + default_autoescape=None, + ): + self._environment = environment + self._func = func + self._argument_count = len(arguments) + self.name = name + self.arguments = arguments + self.catch_kwargs = catch_kwargs + self.catch_varargs = catch_varargs + self.caller = caller + self.explicit_caller = "caller" in arguments + if default_autoescape is None: + default_autoescape = environment.autoescape + self._default_autoescape = default_autoescape + + @internalcode + @evalcontextfunction + def __call__(self, *args, **kwargs): + # This requires a bit of explanation, In the past we used to + # decide largely based on compile-time information if a macro is + # safe or unsafe. While there was a volatile mode it was largely + # unused for deciding on escaping. This turns out to be + # problematic for macros because whether a macro is safe depends not + # on the escape mode when it was defined, but rather when it was used. + # + # Because however we export macros from the module system and + # there are historic callers that do not pass an eval context (and + # will continue to not pass one), we need to perform an instance + # check here. + # + # This is considered safe because an eval context is not a valid + # argument to callables otherwise anyway. Worst case here is + # that if no eval context is passed we fall back to the compile + # time autoescape flag. + if args and isinstance(args[0], EvalContext): + autoescape = args[0].autoescape + args = args[1:] + else: + autoescape = self._default_autoescape + + # try to consume the positional arguments + arguments = list(args[: self._argument_count]) + off = len(arguments) + + # For information why this is necessary refer to the handling + # of caller in the `macro_body` handler in the compiler. + found_caller = False + + # if the number of arguments consumed is not the number of + # arguments expected we start filling in keyword arguments + # and defaults. + if off != self._argument_count: + for name in self.arguments[len(arguments) :]: + try: + value = kwargs.pop(name) + except KeyError: + value = missing + if name == "caller": + found_caller = True + arguments.append(value) + else: + found_caller = self.explicit_caller + + # it's important that the order of these arguments does not change + # if not also changed in the compiler's `function_scoping` method. + # the order is caller, keyword arguments, positional arguments! + if self.caller and not found_caller: + caller = kwargs.pop("caller", None) + if caller is None: + caller = self._environment.undefined("No caller defined", name="caller") + arguments.append(caller) + + if self.catch_kwargs: + arguments.append(kwargs) + elif kwargs: + if "caller" in kwargs: + raise TypeError( + f"macro {self.name!r} was invoked with two values for the special" + " caller argument. This is most likely a bug." + ) + raise TypeError( + f"macro {self.name!r} takes no keyword argument {next(iter(kwargs))!r}" + ) + if self.catch_varargs: + arguments.append(args[self._argument_count :]) + elif len(args) > self._argument_count: + raise TypeError( + f"macro {self.name!r} takes not more than" + f" {len(self.arguments)} argument(s)" + ) + + return self._invoke(arguments, autoescape) + + def _invoke(self, arguments, autoescape): + """This method is being swapped out by the async implementation.""" + rv = self._func(*arguments) + if autoescape: + rv = Markup(rv) + return rv + + def __repr__(self): + name = "anonymous" if self.name is None else repr(self.name) + return f"<{self.__class__.__name__} {name}>" + + +class Undefined: + """The default undefined type. This undefined type can be printed and + iterated over, but every other access will raise an :exc:`UndefinedError`: + + >>> foo = Undefined(name='foo') + >>> str(foo) + '' + >>> not foo + True + >>> foo + 42 + Traceback (most recent call last): + ... + jinja2.exceptions.UndefinedError: 'foo' is undefined + """ + + __slots__ = ( + "_undefined_hint", + "_undefined_obj", + "_undefined_name", + "_undefined_exception", + ) + + def __init__(self, hint=None, obj=missing, name=None, exc=UndefinedError): + self._undefined_hint = hint + self._undefined_obj = obj + self._undefined_name = name + self._undefined_exception = exc + + @property + def _undefined_message(self): + """Build a message about the undefined value based on how it was + accessed. + """ + if self._undefined_hint: + return self._undefined_hint + + if self._undefined_obj is missing: + return f"{self._undefined_name!r} is undefined" + + if not isinstance(self._undefined_name, str): + return ( + f"{object_type_repr(self._undefined_obj)} has no" + f" element {self._undefined_name!r}" + ) + + return ( + f"{object_type_repr(self._undefined_obj)!r} has no" + f" attribute {self._undefined_name!r}" + ) + + @internalcode + def _fail_with_undefined_error(self, *args, **kwargs): + """Raise an :exc:`UndefinedError` when operations are performed + on the undefined value. + """ + raise self._undefined_exception(self._undefined_message) + + @internalcode + def __getattr__(self, name): + if name[:2] == "__": + raise AttributeError(name) + return self._fail_with_undefined_error() + + __add__ = __radd__ = __sub__ = __rsub__ = _fail_with_undefined_error + __mul__ = __rmul__ = __div__ = __rdiv__ = _fail_with_undefined_error + __truediv__ = __rtruediv__ = _fail_with_undefined_error + __floordiv__ = __rfloordiv__ = _fail_with_undefined_error + __mod__ = __rmod__ = _fail_with_undefined_error + __pos__ = __neg__ = _fail_with_undefined_error + __call__ = __getitem__ = _fail_with_undefined_error + __lt__ = __le__ = __gt__ = __ge__ = _fail_with_undefined_error + __int__ = __float__ = __complex__ = _fail_with_undefined_error + __pow__ = __rpow__ = _fail_with_undefined_error + + def __eq__(self, other): + return type(self) is type(other) + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return id(type(self)) + + def __str__(self): + return "" + + def __len__(self): + return 0 + + def __iter__(self): + if 0: + yield None + + def __bool__(self): + return False + + def __repr__(self): + return "Undefined" + + +def make_logging_undefined(logger=None, base=None): + """Given a logger object this returns a new undefined class that will + log certain failures. It will log iterations and printing. If no + logger is given a default logger is created. + + Example:: + + logger = logging.getLogger(__name__) + LoggingUndefined = make_logging_undefined( + logger=logger, + base=Undefined + ) + + .. versionadded:: 2.8 + + :param logger: the logger to use. If not provided, a default logger + is created. + :param base: the base class to add logging functionality to. This + defaults to :class:`Undefined`. + """ + if logger is None: + import logging + + logger = logging.getLogger(__name__) + logger.addHandler(logging.StreamHandler(sys.stderr)) + if base is None: + base = Undefined + + def _log_message(undef): + logger.warning("Template variable warning: %s", undef._undefined_message) + + class LoggingUndefined(base): + def _fail_with_undefined_error(self, *args, **kwargs): + try: + return super()._fail_with_undefined_error(*args, **kwargs) + except self._undefined_exception as e: + logger.error(f"Template variable error: %s", e) + raise e + + def __str__(self): + _log_message(self) + return super().__str__() + + def __iter__(self): + _log_message(self) + return super().__iter__() + + def __bool__(self): + _log_message(self) + return super().__bool__() + + return LoggingUndefined + + +class ChainableUndefined(Undefined): + """An undefined that is chainable, where both ``__getattr__`` and + ``__getitem__`` return itself rather than raising an + :exc:`UndefinedError`. + + >>> foo = ChainableUndefined(name='foo') + >>> str(foo.bar['baz']) + '' + >>> foo.bar['baz'] + 42 + Traceback (most recent call last): + ... + jinja2.exceptions.UndefinedError: 'foo' is undefined + + .. versionadded:: 2.11.0 + """ + + __slots__ = () + + def __html__(self): + return self.__str__() + + def __getattr__(self, _): + return self + + __getitem__ = __getattr__ + + +class DebugUndefined(Undefined): + """An undefined that returns the debug info when printed. + + >>> foo = DebugUndefined(name='foo') + >>> str(foo) + '{{ foo }}' + >>> not foo + True + >>> foo + 42 + Traceback (most recent call last): + ... + jinja2.exceptions.UndefinedError: 'foo' is undefined + """ + + __slots__ = () + + def __str__(self): + if self._undefined_hint: + message = f"undefined value printed: {self._undefined_hint}" + + elif self._undefined_obj is missing: + message = self._undefined_name + + else: + message = ( + f"no such element: {object_type_repr(self._undefined_obj)}" + f"[{self._undefined_name!r}]" + ) + + return f"{{{{ {message} }}}}" + + +class StrictUndefined(Undefined): + """An undefined that barks on print and iteration as well as boolean + tests and all kinds of comparisons. In other words: you can do nothing + with it except checking if it's defined using the `defined` test. + + >>> foo = StrictUndefined(name='foo') + >>> str(foo) + Traceback (most recent call last): + ... + jinja2.exceptions.UndefinedError: 'foo' is undefined + >>> not foo + Traceback (most recent call last): + ... + jinja2.exceptions.UndefinedError: 'foo' is undefined + >>> foo + 42 + Traceback (most recent call last): + ... + jinja2.exceptions.UndefinedError: 'foo' is undefined + """ + + __slots__ = () + __iter__ = __str__ = __len__ = Undefined._fail_with_undefined_error + __eq__ = __ne__ = __bool__ = __hash__ = Undefined._fail_with_undefined_error + + +# Remove slots attributes, after the metaclass is applied they are +# unneeded and contain wrong data for subclasses. +del ( + Undefined.__slots__, + ChainableUndefined.__slots__, + DebugUndefined.__slots__, + StrictUndefined.__slots__, +) diff --git a/src/jinja2/sandbox.py b/src/jinja2/sandbox.py new file mode 100644 index 00000000..5c6d0946 --- /dev/null +++ b/src/jinja2/sandbox.py @@ -0,0 +1,419 @@ +"""A sandbox layer that ensures unsafe operations cannot be performed. +Useful when the template itself comes from an untrusted source. +""" +import operator +import types +from _string import formatter_field_name_split +from collections import abc +from collections import deque +from string import Formatter + +from markupsafe import EscapeFormatter +from markupsafe import Markup + +from .environment import Environment +from .exceptions import SecurityError + +#: maximum number of items a range may produce +MAX_RANGE = 100000 + +#: Unsafe function attributes. +UNSAFE_FUNCTION_ATTRIBUTES = set() + +#: Unsafe method attributes. Function attributes are unsafe for methods too. +UNSAFE_METHOD_ATTRIBUTES = set() + +#: unsafe generator attributes. +UNSAFE_GENERATOR_ATTRIBUTES = {"gi_frame", "gi_code"} + +#: unsafe attributes on coroutines +UNSAFE_COROUTINE_ATTRIBUTES = {"cr_frame", "cr_code"} + +#: unsafe attributes on async generators +UNSAFE_ASYNC_GENERATOR_ATTRIBUTES = {"ag_code", "ag_frame"} + +_mutable_spec = ( + ( + abc.MutableSet, + frozenset( + [ + "add", + "clear", + "difference_update", + "discard", + "pop", + "remove", + "symmetric_difference_update", + "update", + ] + ), + ), + ( + abc.MutableMapping, + frozenset(["clear", "pop", "popitem", "setdefault", "update"]), + ), + ( + abc.MutableSequence, + frozenset(["append", "reverse", "insert", "sort", "extend", "remove"]), + ), + ( + deque, + frozenset( + [ + "append", + "appendleft", + "clear", + "extend", + "extendleft", + "pop", + "popleft", + "remove", + "rotate", + ] + ), + ), +) + + +def inspect_format_method(callable): + if not isinstance( + callable, (types.MethodType, types.BuiltinMethodType) + ) or callable.__name__ not in ("format", "format_map"): + return None + obj = callable.__self__ + if isinstance(obj, str): + return obj + + +def safe_range(*args): + """A range that can't generate ranges with a length of more than + MAX_RANGE items. + """ + rng = range(*args) + + if len(rng) > MAX_RANGE: + raise OverflowError( + "Range too big. The sandbox blocks ranges larger than" + f" MAX_RANGE ({MAX_RANGE})." + ) + + return rng + + +def unsafe(f): + """Marks a function or method as unsafe. + + .. code-block: python + + @unsafe + def delete(self): + pass + """ + f.unsafe_callable = True + return f + + +def is_internal_attribute(obj, attr): + """Test if the attribute given is an internal python attribute. For + example this function returns `True` for the `func_code` attribute of + python objects. This is useful if the environment method + :meth:`~SandboxedEnvironment.is_safe_attribute` is overridden. + + >>> from jinja2.sandbox import is_internal_attribute + >>> is_internal_attribute(str, "mro") + True + >>> is_internal_attribute(str, "upper") + False + """ + if isinstance(obj, types.FunctionType): + if attr in UNSAFE_FUNCTION_ATTRIBUTES: + return True + elif isinstance(obj, types.MethodType): + if attr in UNSAFE_FUNCTION_ATTRIBUTES or attr in UNSAFE_METHOD_ATTRIBUTES: + return True + elif isinstance(obj, type): + if attr == "mro": + return True + elif isinstance(obj, (types.CodeType, types.TracebackType, types.FrameType)): + return True + elif isinstance(obj, types.GeneratorType): + if attr in UNSAFE_GENERATOR_ATTRIBUTES: + return True + elif hasattr(types, "CoroutineType") and isinstance(obj, types.CoroutineType): + if attr in UNSAFE_COROUTINE_ATTRIBUTES: + return True + elif hasattr(types, "AsyncGeneratorType") and isinstance( + obj, types.AsyncGeneratorType + ): + if attr in UNSAFE_ASYNC_GENERATOR_ATTRIBUTES: + return True + return attr.startswith("__") + + +def modifies_known_mutable(obj, attr): + """This function checks if an attribute on a builtin mutable object + (list, dict, set or deque) or the corresponding ABCs would modify it + if called. + + >>> modifies_known_mutable({}, "clear") + True + >>> modifies_known_mutable({}, "keys") + False + >>> modifies_known_mutable([], "append") + True + >>> modifies_known_mutable([], "index") + False + + If called with an unsupported object, ``False`` is returned. + + >>> modifies_known_mutable("foo", "upper") + False + """ + for typespec, unsafe in _mutable_spec: + if isinstance(obj, typespec): + return attr in unsafe + return False + + +class SandboxedEnvironment(Environment): + """The sandboxed environment. It works like the regular environment but + tells the compiler to generate sandboxed code. Additionally subclasses of + this environment may override the methods that tell the runtime what + attributes or functions are safe to access. + + If the template tries to access insecure code a :exc:`SecurityError` is + raised. However also other exceptions may occur during the rendering so + the caller has to ensure that all exceptions are caught. + """ + + sandboxed = True + + #: default callback table for the binary operators. A copy of this is + #: available on each instance of a sandboxed environment as + #: :attr:`binop_table` + default_binop_table = { + "+": operator.add, + "-": operator.sub, + "*": operator.mul, + "/": operator.truediv, + "//": operator.floordiv, + "**": operator.pow, + "%": operator.mod, + } + + #: default callback table for the unary operators. A copy of this is + #: available on each instance of a sandboxed environment as + #: :attr:`unop_table` + default_unop_table = {"+": operator.pos, "-": operator.neg} + + #: a set of binary operators that should be intercepted. Each operator + #: that is added to this set (empty by default) is delegated to the + #: :meth:`call_binop` method that will perform the operator. The default + #: operator callback is specified by :attr:`binop_table`. + #: + #: The following binary operators are interceptable: + #: ``//``, ``%``, ``+``, ``*``, ``-``, ``/``, and ``**`` + #: + #: The default operation form the operator table corresponds to the + #: builtin function. Intercepted calls are always slower than the native + #: operator call, so make sure only to intercept the ones you are + #: interested in. + #: + #: .. versionadded:: 2.6 + intercepted_binops = frozenset() + + #: a set of unary operators that should be intercepted. Each operator + #: that is added to this set (empty by default) is delegated to the + #: :meth:`call_unop` method that will perform the operator. The default + #: operator callback is specified by :attr:`unop_table`. + #: + #: The following unary operators are interceptable: ``+``, ``-`` + #: + #: The default operation form the operator table corresponds to the + #: builtin function. Intercepted calls are always slower than the native + #: operator call, so make sure only to intercept the ones you are + #: interested in. + #: + #: .. versionadded:: 2.6 + intercepted_unops = frozenset() + + def intercept_unop(self, operator): + """Called during template compilation with the name of a unary + operator to check if it should be intercepted at runtime. If this + method returns `True`, :meth:`call_unop` is executed for this unary + operator. The default implementation of :meth:`call_unop` will use + the :attr:`unop_table` dictionary to perform the operator with the + same logic as the builtin one. + + The following unary operators are interceptable: ``+`` and ``-`` + + Intercepted calls are always slower than the native operator call, + so make sure only to intercept the ones you are interested in. + + .. versionadded:: 2.6 + """ + return False + + def __init__(self, *args, **kwargs): + Environment.__init__(self, *args, **kwargs) + self.globals["range"] = safe_range + self.binop_table = self.default_binop_table.copy() + self.unop_table = self.default_unop_table.copy() + + def is_safe_attribute(self, obj, attr, value): + """The sandboxed environment will call this method to check if the + attribute of an object is safe to access. Per default all attributes + starting with an underscore are considered private as well as the + special attributes of internal python objects as returned by the + :func:`is_internal_attribute` function. + """ + return not (attr.startswith("_") or is_internal_attribute(obj, attr)) + + def is_safe_callable(self, obj): + """Check if an object is safely callable. Per default a function is + considered safe unless the `unsafe_callable` attribute exists and is + True. Override this method to alter the behavior, but this won't + affect the `unsafe` decorator from this module. + """ + return not ( + getattr(obj, "unsafe_callable", False) or getattr(obj, "alters_data", False) + ) + + def call_binop(self, context, operator, left, right): + """For intercepted binary operator calls (:meth:`intercepted_binops`) + this function is executed instead of the builtin operator. This can + be used to fine tune the behavior of certain operators. + + .. versionadded:: 2.6 + """ + return self.binop_table[operator](left, right) + + def call_unop(self, context, operator, arg): + """For intercepted unary operator calls (:meth:`intercepted_unops`) + this function is executed instead of the builtin operator. This can + be used to fine tune the behavior of certain operators. + + .. versionadded:: 2.6 + """ + return self.unop_table[operator](arg) + + def getitem(self, obj, argument): + """Subscribe an object from sandboxed code.""" + try: + return obj[argument] + except (TypeError, LookupError): + if isinstance(argument, str): + try: + attr = str(argument) + except Exception: + pass + else: + try: + value = getattr(obj, attr) + except AttributeError: + pass + else: + if self.is_safe_attribute(obj, argument, value): + return value + return self.unsafe_undefined(obj, argument) + return self.undefined(obj=obj, name=argument) + + def getattr(self, obj, attribute): + """Subscribe an object from sandboxed code and prefer the + attribute. The attribute passed *must* be a bytestring. + """ + try: + value = getattr(obj, attribute) + except AttributeError: + try: + return obj[attribute] + except (TypeError, LookupError): + pass + else: + if self.is_safe_attribute(obj, attribute, value): + return value + return self.unsafe_undefined(obj, attribute) + return self.undefined(obj=obj, name=attribute) + + def unsafe_undefined(self, obj, attribute): + """Return an undefined object for unsafe attributes.""" + return self.undefined( + f"access to attribute {attribute!r} of" + f" {obj.__class__.__name__!r} object is unsafe.", + name=attribute, + obj=obj, + exc=SecurityError, + ) + + def format_string(self, s, args, kwargs, format_func=None): + """If a format call is detected, then this is routed through this + method so that our safety sandbox can be used for it. + """ + if isinstance(s, Markup): + formatter = SandboxedEscapeFormatter(self, s.escape) + else: + formatter = SandboxedFormatter(self) + + if format_func is not None and format_func.__name__ == "format_map": + if len(args) != 1 or kwargs: + raise TypeError( + "format_map() takes exactly one argument" + f" {len(args) + (kwargs is not None)} given" + ) + + kwargs = args[0] + args = None + + rv = formatter.vformat(s, args, kwargs) + return type(s)(rv) + + def call(__self, __context, __obj, *args, **kwargs): # noqa: B902 + """Call an object from sandboxed code.""" + fmt = inspect_format_method(__obj) + if fmt is not None: + return __self.format_string(fmt, args, kwargs, __obj) + + # the double prefixes are to avoid double keyword argument + # errors when proxying the call. + if not __self.is_safe_callable(__obj): + raise SecurityError(f"{__obj!r} is not safely callable") + return __context.call(__obj, *args, **kwargs) + + +class ImmutableSandboxedEnvironment(SandboxedEnvironment): + """Works exactly like the regular `SandboxedEnvironment` but does not + permit modifications on the builtin mutable objects `list`, `set`, and + `dict` by using the :func:`modifies_known_mutable` function. + """ + + def is_safe_attribute(self, obj, attr, value): + if not SandboxedEnvironment.is_safe_attribute(self, obj, attr, value): + return False + return not modifies_known_mutable(obj, attr) + + +class SandboxedFormatterMixin: + def __init__(self, env): + self._env = env + + def get_field(self, field_name, args, kwargs): + first, rest = formatter_field_name_split(field_name) + obj = self.get_value(first, args, kwargs) + for is_attr, i in rest: + if is_attr: + obj = self._env.getattr(obj, i) + else: + obj = self._env.getitem(obj, i) + return obj, first + + +class SandboxedFormatter(SandboxedFormatterMixin, Formatter): + def __init__(self, env): + SandboxedFormatterMixin.__init__(self, env) + Formatter.__init__(self) + + +class SandboxedEscapeFormatter(SandboxedFormatterMixin, EscapeFormatter): + def __init__(self, env, escape): + SandboxedFormatterMixin.__init__(self, env) + EscapeFormatter.__init__(self, escape) diff --git a/src/jinja2/tests.py b/src/jinja2/tests.py new file mode 100644 index 00000000..bc763268 --- /dev/null +++ b/src/jinja2/tests.py @@ -0,0 +1,211 @@ +"""Built-in template tests used with the ``is`` operator.""" +import operator +import re +from collections import abc +from numbers import Number + +from .runtime import Undefined + +number_re = re.compile(r"^-?\d+(\.\d+)?$") +regex_type = type(number_re) +test_callable = callable + + +def test_odd(value): + """Return true if the variable is odd.""" + return value % 2 == 1 + + +def test_even(value): + """Return true if the variable is even.""" + return value % 2 == 0 + + +def test_divisibleby(value, num): + """Check if a variable is divisible by a number.""" + return value % num == 0 + + +def test_defined(value): + """Return true if the variable is defined: + + .. sourcecode:: jinja + + {% if variable is defined %} + value of variable: {{ variable }} + {% else %} + variable is not defined + {% endif %} + + See the :func:`default` filter for a simple way to set undefined + variables. + """ + return not isinstance(value, Undefined) + + +def test_undefined(value): + """Like :func:`defined` but the other way round.""" + return isinstance(value, Undefined) + + +def test_none(value): + """Return true if the variable is none.""" + return value is None + + +def test_boolean(value): + """Return true if the object is a boolean value. + + .. versionadded:: 2.11 + """ + return value is True or value is False + + +def test_false(value): + """Return true if the object is False. + + .. versionadded:: 2.11 + """ + return value is False + + +def test_true(value): + """Return true if the object is True. + + .. versionadded:: 2.11 + """ + return value is True + + +# NOTE: The existing 'number' test matches booleans and floats +def test_integer(value): + """Return true if the object is an integer. + + .. versionadded:: 2.11 + """ + return isinstance(value, int) and value is not True and value is not False + + +# NOTE: The existing 'number' test matches booleans and integers +def test_float(value): + """Return true if the object is a float. + + .. versionadded:: 2.11 + """ + return isinstance(value, float) + + +def test_lower(value): + """Return true if the variable is lowercased.""" + return str(value).islower() + + +def test_upper(value): + """Return true if the variable is uppercased.""" + return str(value).isupper() + + +def test_string(value): + """Return true if the object is a string.""" + return isinstance(value, str) + + +def test_mapping(value): + """Return true if the object is a mapping (dict etc.). + + .. versionadded:: 2.6 + """ + return isinstance(value, abc.Mapping) + + +def test_number(value): + """Return true if the variable is a number.""" + return isinstance(value, Number) + + +def test_sequence(value): + """Return true if the variable is a sequence. Sequences are variables + that are iterable. + """ + try: + len(value) + value.__getitem__ + except Exception: + return False + return True + + +def test_sameas(value, other): + """Check if an object points to the same memory address than another + object: + + .. sourcecode:: jinja + + {% if foo.attribute is sameas false %} + the foo attribute really is the `False` singleton + {% endif %} + """ + return value is other + + +def test_iterable(value): + """Check if it's possible to iterate over an object.""" + try: + iter(value) + except TypeError: + return False + return True + + +def test_escaped(value): + """Check if the value is escaped.""" + return hasattr(value, "__html__") + + +def test_in(value, seq): + """Check if value is in seq. + + .. versionadded:: 2.10 + """ + return value in seq + + +TESTS = { + "odd": test_odd, + "even": test_even, + "divisibleby": test_divisibleby, + "defined": test_defined, + "undefined": test_undefined, + "none": test_none, + "boolean": test_boolean, + "false": test_false, + "true": test_true, + "integer": test_integer, + "float": test_float, + "lower": test_lower, + "upper": test_upper, + "string": test_string, + "mapping": test_mapping, + "number": test_number, + "sequence": test_sequence, + "iterable": test_iterable, + "callable": test_callable, + "sameas": test_sameas, + "escaped": test_escaped, + "in": test_in, + "==": operator.eq, + "eq": operator.eq, + "equalto": operator.eq, + "!=": operator.ne, + "ne": operator.ne, + ">": operator.gt, + "gt": operator.gt, + "greaterthan": operator.gt, + "ge": operator.ge, + ">=": operator.ge, + "<": operator.lt, + "lt": operator.lt, + "lessthan": operator.lt, + "<=": operator.le, + "le": operator.le, +} diff --git a/src/jinja2/utils.py b/src/jinja2/utils.py new file mode 100644 index 00000000..8ee02958 --- /dev/null +++ b/src/jinja2/utils.py @@ -0,0 +1,666 @@ +import json +import os +import re +from collections import abc +from collections import deque +from random import choice +from random import randrange +from threading import Lock +from urllib.parse import quote_from_bytes + +from markupsafe import escape +from markupsafe import Markup + +_word_split_re = re.compile(r"(\s+)") +_lead_pattern = "|".join(map(re.escape, ("(", "<", "<"))) +_trail_pattern = "|".join(map(re.escape, (".", ",", ")", ">", "\n", ">"))) +_punctuation_re = re.compile( + fr"^(?P<lead>(?:{_lead_pattern})*)(?P<middle>.*?)(?P<trail>(?:{_trail_pattern})*)$" +) +_simple_email_re = re.compile(r"^\S+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+$") +_striptags_re = re.compile(r"(<!--.*?-->|<[^>]*>)") +_entity_re = re.compile(r"&([^;]+);") +_letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +_digits = "0123456789" + +# special singleton representing missing values for the runtime +missing = type("MissingType", (), {"__repr__": lambda x: "missing"})() + +# internal code +internal_code = set() + +concat = "".join + +_slash_escape = "\\/" not in json.dumps("/") + + +def contextfunction(f): + """This decorator can be used to mark a function or method context callable. + A context callable is passed the active :class:`Context` as first argument when + called from the template. This is useful if a function wants to get access + to the context or functions provided on the context object. For example + a function that returns a sorted list of template variables the current + template exports could look like this:: + + @contextfunction + def get_exported_names(context): + return sorted(context.exported_vars) + """ + f.contextfunction = True + return f + + +def evalcontextfunction(f): + """This decorator can be used to mark a function or method as an eval + context callable. This is similar to the :func:`contextfunction` + but instead of passing the context, an evaluation context object is + passed. For more information about the eval context, see + :ref:`eval-context`. + + .. versionadded:: 2.4 + """ + f.evalcontextfunction = True + return f + + +def environmentfunction(f): + """This decorator can be used to mark a function or method as environment + callable. This decorator works exactly like the :func:`contextfunction` + decorator just that the first argument is the active :class:`Environment` + and not context. + """ + f.environmentfunction = True + return f + + +def internalcode(f): + """Marks the function as internally used""" + internal_code.add(f.__code__) + return f + + +def is_undefined(obj): + """Check if the object passed is undefined. This does nothing more than + performing an instance check against :class:`Undefined` but looks nicer. + This can be used for custom filters or tests that want to react to + undefined variables. For example a custom default filter can look like + this:: + + def default(var, default=''): + if is_undefined(var): + return default + return var + """ + from .runtime import Undefined + + return isinstance(obj, Undefined) + + +def consume(iterable): + """Consumes an iterable without doing anything with it.""" + for _ in iterable: + pass + + +def clear_caches(): + """Jinja keeps internal caches for environments and lexers. These are + used so that Jinja doesn't have to recreate environments and lexers all + the time. Normally you don't have to care about that but if you are + measuring memory consumption you may want to clean the caches. + """ + from .environment import _spontaneous_environments + from .lexer import _lexer_cache + + _spontaneous_environments.clear() + _lexer_cache.clear() + + +def import_string(import_name, silent=False): + """Imports an object based on a string. This is useful if you want to + use import paths as endpoints or something similar. An import path can + be specified either in dotted notation (``xml.sax.saxutils.escape``) + or with a colon as object delimiter (``xml.sax.saxutils:escape``). + + If the `silent` is True the return value will be `None` if the import + fails. + + :return: imported object + """ + try: + if ":" in import_name: + module, obj = import_name.split(":", 1) + elif "." in import_name: + module, _, obj = import_name.rpartition(".") + else: + return __import__(import_name) + return getattr(__import__(module, None, None, [obj]), obj) + except (ImportError, AttributeError): + if not silent: + raise + + +def open_if_exists(filename, mode="rb"): + """Returns a file descriptor for the filename if that file exists, + otherwise ``None``. + """ + if not os.path.isfile(filename): + return None + + return open(filename, mode) + + +def object_type_repr(obj): + """Returns the name of the object's type. For some recognized + singletons the name of the object is returned instead. (For + example for `None` and `Ellipsis`). + """ + if obj is None: + return "None" + elif obj is Ellipsis: + return "Ellipsis" + + cls = type(obj) + + if cls.__module__ == "builtins": + return f"{cls.__name__} object" + + return f"{cls.__module__}.{cls.__name__} object" + + +def pformat(obj): + """Format an object using :func:`pprint.pformat`. + """ + from pprint import pformat + + return pformat(obj) + + +def urlize(text, trim_url_limit=None, rel=None, target=None): + """Converts any URLs in text into clickable links. Works on http://, + https:// and www. links. Links can have trailing punctuation (periods, + commas, close-parens) and leading punctuation (opening parens) and + it'll still do the right thing. + + If trim_url_limit is not None, the URLs in link text will be limited + to trim_url_limit characters. + + If nofollow is True, the URLs in link text will get a rel="nofollow" + attribute. + + If target is not None, a target attribute will be added to the link. + """ + + def trim_url(x, limit=trim_url_limit): + if limit is not None: + return x[:limit] + ("..." if len(x) >= limit else "") + + return x + + words = _word_split_re.split(str(escape(text))) + rel_attr = f' rel="{escape(rel)}"' if rel else "" + target_attr = f' target="{escape(target)}"' if target else "" + + for i, word in enumerate(words): + match = _punctuation_re.match(word) + if match: + lead, middle, trail = match.groups() + if middle.startswith("www.") or ( + "@" not in middle + and not middle.startswith("http://") + and not middle.startswith("https://") + and len(middle) > 0 + and middle[0] in _letters + _digits + and ( + middle.endswith(".org") + or middle.endswith(".net") + or middle.endswith(".com") + ) + ): + middle = ( + f'<a href="http://{middle}"{rel_attr}{target_attr}>' + f"{trim_url(middle)}</a>" + ) + if middle.startswith("http://") or middle.startswith("https://"): + middle = ( + f'<a href="{middle}"{rel_attr}{target_attr}>{trim_url(middle)}</a>' + ) + if ( + "@" in middle + and not middle.startswith("www.") + and ":" not in middle + and _simple_email_re.match(middle) + ): + middle = f'<a href="mailto:{middle}">{middle}</a>' + if lead + middle + trail != word: + words[i] = lead + middle + trail + return "".join(words) + + +def generate_lorem_ipsum(n=5, html=True, min=20, max=100): + """Generate some lorem ipsum for the template.""" + from .constants import LOREM_IPSUM_WORDS + + words = LOREM_IPSUM_WORDS.split() + result = [] + + for _ in range(n): + next_capitalized = True + last_comma = last_fullstop = 0 + word = None + last = None + p = [] + + # each paragraph contains out of 20 to 100 words. + for idx, _ in enumerate(range(randrange(min, max))): + while True: + word = choice(words) + if word != last: + last = word + break + if next_capitalized: + word = word.capitalize() + next_capitalized = False + # add commas + if idx - randrange(3, 8) > last_comma: + last_comma = idx + last_fullstop += 2 + word += "," + # add end of sentences + if idx - randrange(10, 20) > last_fullstop: + last_comma = last_fullstop = idx + word += "." + next_capitalized = True + p.append(word) + + # ensure that the paragraph ends with a dot. + p = " ".join(p) + if p.endswith(","): + p = p[:-1] + "." + elif not p.endswith("."): + p += "." + result.append(p) + + if not html: + return "\n\n".join(result) + return Markup("\n".join(f"<p>{escape(x)}</p>" for x in result)) + + +def url_quote(obj, charset="utf-8", for_qs=False): + """Quote a string for use in a URL using the given charset. + + This function is misnamed, it is a wrapper around + :func:`urllib.parse.quote`. + + :param obj: String or bytes to quote. Other types are converted to + string then encoded to bytes using the given charset. + :param charset: Encode text to bytes using this charset. + :param for_qs: Quote "/" and use "+" for spaces. + """ + if not isinstance(obj, bytes): + if not isinstance(obj, str): + obj = str(obj) + + obj = obj.encode(charset) + + safe = b"" if for_qs else b"/" + rv = quote_from_bytes(obj, safe) + + if for_qs: + rv = rv.replace("%20", "+") + + return rv + + +def unicode_urlencode(obj, charset="utf-8", for_qs=False): + import warnings + + warnings.warn( + "'unicode_urlencode' has been renamed to 'url_quote'. The old" + " name will be removed in version 3.1.", + DeprecationWarning, + stacklevel=2, + ) + return url_quote(obj, charset=charset, for_qs=for_qs) + + +@abc.MutableMapping.register +class LRUCache: + """A simple LRU Cache implementation.""" + + # this is fast for small capacities (something below 1000) but doesn't + # scale. But as long as it's only used as storage for templates this + # won't do any harm. + + def __init__(self, capacity): + self.capacity = capacity + self._mapping = {} + self._queue = deque() + self._postinit() + + def _postinit(self): + # alias all queue methods for faster lookup + self._popleft = self._queue.popleft + self._pop = self._queue.pop + self._remove = self._queue.remove + self._wlock = Lock() + self._append = self._queue.append + + def __getstate__(self): + return { + "capacity": self.capacity, + "_mapping": self._mapping, + "_queue": self._queue, + } + + def __setstate__(self, d): + self.__dict__.update(d) + self._postinit() + + def __getnewargs__(self): + return (self.capacity,) + + def copy(self): + """Return a shallow copy of the instance.""" + rv = self.__class__(self.capacity) + rv._mapping.update(self._mapping) + rv._queue.extend(self._queue) + return rv + + def get(self, key, default=None): + """Return an item from the cache dict or `default`""" + try: + return self[key] + except KeyError: + return default + + def setdefault(self, key, default=None): + """Set `default` if the key is not in the cache otherwise + leave unchanged. Return the value of this key. + """ + try: + return self[key] + except KeyError: + self[key] = default + return default + + def clear(self): + """Clear the cache.""" + self._wlock.acquire() + try: + self._mapping.clear() + self._queue.clear() + finally: + self._wlock.release() + + def __contains__(self, key): + """Check if a key exists in this cache.""" + return key in self._mapping + + def __len__(self): + """Return the current size of the cache.""" + return len(self._mapping) + + def __repr__(self): + return f"<{self.__class__.__name__} {self._mapping!r}>" + + def __getitem__(self, key): + """Get an item from the cache. Moves the item up so that it has the + highest priority then. + + Raise a `KeyError` if it does not exist. + """ + self._wlock.acquire() + try: + rv = self._mapping[key] + if self._queue[-1] != key: + try: + self._remove(key) + except ValueError: + # if something removed the key from the container + # when we read, ignore the ValueError that we would + # get otherwise. + pass + self._append(key) + return rv + finally: + self._wlock.release() + + def __setitem__(self, key, value): + """Sets the value for an item. Moves the item up so that it + has the highest priority then. + """ + self._wlock.acquire() + try: + if key in self._mapping: + self._remove(key) + elif len(self._mapping) == self.capacity: + del self._mapping[self._popleft()] + self._append(key) + self._mapping[key] = value + finally: + self._wlock.release() + + def __delitem__(self, key): + """Remove an item from the cache dict. + Raise a `KeyError` if it does not exist. + """ + self._wlock.acquire() + try: + del self._mapping[key] + try: + self._remove(key) + except ValueError: + pass + finally: + self._wlock.release() + + def items(self): + """Return a list of items.""" + result = [(key, self._mapping[key]) for key in list(self._queue)] + result.reverse() + return result + + def values(self): + """Return a list of all values.""" + return [x[1] for x in self.items()] + + def keys(self): + """Return a list of all keys ordered by most recent usage.""" + return list(self) + + def __iter__(self): + return reversed(tuple(self._queue)) + + def __reversed__(self): + """Iterate over the keys in the cache dict, oldest items + coming first. + """ + return iter(tuple(self._queue)) + + __copy__ = copy + + +def select_autoescape( + enabled_extensions=("html", "htm", "xml"), + disabled_extensions=(), + default_for_string=True, + default=False, +): + """Intelligently sets the initial value of autoescaping based on the + filename of the template. This is the recommended way to configure + autoescaping if you do not want to write a custom function yourself. + + If you want to enable it for all templates created from strings or + for all templates with `.html` and `.xml` extensions:: + + from jinja2 import Environment, select_autoescape + env = Environment(autoescape=select_autoescape( + enabled_extensions=('html', 'xml'), + default_for_string=True, + )) + + Example configuration to turn it on at all times except if the template + ends with `.txt`:: + + from jinja2 import Environment, select_autoescape + env = Environment(autoescape=select_autoescape( + disabled_extensions=('txt',), + default_for_string=True, + default=True, + )) + + The `enabled_extensions` is an iterable of all the extensions that + autoescaping should be enabled for. Likewise `disabled_extensions` is + a list of all templates it should be disabled for. If a template is + loaded from a string then the default from `default_for_string` is used. + If nothing matches then the initial value of autoescaping is set to the + value of `default`. + + For security reasons this function operates case insensitive. + + .. versionadded:: 2.9 + """ + enabled_patterns = tuple(f".{x.lstrip('.').lower()}" for x in enabled_extensions) + disabled_patterns = tuple(f".{x.lstrip('.').lower()}" for x in disabled_extensions) + + def autoescape(template_name): + if template_name is None: + return default_for_string + template_name = template_name.lower() + if template_name.endswith(enabled_patterns): + return True + if template_name.endswith(disabled_patterns): + return False + return default + + return autoescape + + +def htmlsafe_json_dumps(obj, dumper=None, **kwargs): + """Works exactly like :func:`dumps` but is safe for use in ``<script>`` + tags. It accepts the same arguments and returns a JSON string. Note that + this is available in templates through the ``|tojson`` filter which will + also mark the result as safe. Due to how this function escapes certain + characters this is safe even if used outside of ``<script>`` tags. + + The following characters are escaped in strings: + + - ``<`` + - ``>`` + - ``&`` + - ``'`` + + This makes it safe to embed such strings in any place in HTML with the + notable exception of double quoted attributes. In that case single + quote your attributes or HTML escape it in addition. + """ + if dumper is None: + dumper = json.dumps + rv = ( + dumper(obj, **kwargs) + .replace("<", "\\u003c") + .replace(">", "\\u003e") + .replace("&", "\\u0026") + .replace("'", "\\u0027") + ) + return Markup(rv) + + +class Cycler: + """Cycle through values by yield them one at a time, then restarting + once the end is reached. Available as ``cycler`` in templates. + + Similar to ``loop.cycle``, but can be used outside loops or across + multiple loops. For example, render a list of folders and files in a + list, alternating giving them "odd" and "even" classes. + + .. code-block:: html+jinja + + {% set row_class = cycler("odd", "even") %} + <ul class="browser"> + {% for folder in folders %} + <li class="folder {{ row_class.next() }}">{{ folder }} + {% endfor %} + {% for file in files %} + <li class="file {{ row_class.next() }}">{{ file }} + {% endfor %} + </ul> + + :param items: Each positional argument will be yielded in the order + given for each cycle. + + .. versionadded:: 2.1 + """ + + def __init__(self, *items): + if not items: + raise RuntimeError("at least one item has to be provided") + self.items = items + self.pos = 0 + + def reset(self): + """Resets the current item to the first item.""" + self.pos = 0 + + @property + def current(self): + """Return the current item. Equivalent to the item that will be + returned next time :meth:`next` is called. + """ + return self.items[self.pos] + + def next(self): + """Return the current item, then advance :attr:`current` to the + next item. + """ + rv = self.current + self.pos = (self.pos + 1) % len(self.items) + return rv + + __next__ = next + + +class Joiner: + """A joining helper for templates.""" + + def __init__(self, sep=", "): + self.sep = sep + self.used = False + + def __call__(self): + if not self.used: + self.used = True + return "" + return self.sep + + +class Namespace: + """A namespace object that can hold arbitrary attributes. It may be + initialized from a dictionary or with keyword arguments.""" + + def __init__(*args, **kwargs): # noqa: B902 + self, args = args[0], args[1:] + self.__attrs = dict(*args, **kwargs) + + def __getattribute__(self, name): + # __class__ is needed for the awaitable check in async mode + if name in {"_Namespace__attrs", "__class__"}: + return object.__getattribute__(self, name) + try: + return self.__attrs[name] + except KeyError: + raise AttributeError(name) + + def __setitem__(self, name, value): + self.__attrs[name] = value + + def __repr__(self): + return f"<Namespace {self.__attrs!r}>" + + +# does this python version support async for in and async generators? +try: + exec("async def _():\n async for _ in ():\n yield _") + have_async_gen = True +except SyntaxError: + have_async_gen = False diff --git a/src/jinja2/visitor.py b/src/jinja2/visitor.py new file mode 100644 index 00000000..590fa9eb --- /dev/null +++ b/src/jinja2/visitor.py @@ -0,0 +1,79 @@ +"""API for traversing the AST nodes. Implemented by the compiler and +meta introspection. +""" +from .nodes import Node + + +class NodeVisitor: + """Walks the abstract syntax tree and call visitor functions for every + node found. The visitor functions may return values which will be + forwarded by the `visit` method. + + Per default the visitor functions for the nodes are ``'visit_'`` + + class name of the node. So a `TryFinally` node visit function would + be `visit_TryFinally`. This behavior can be changed by overriding + the `get_visitor` function. If no visitor function exists for a node + (return value `None`) the `generic_visit` visitor is used instead. + """ + + def get_visitor(self, node): + """Return the visitor function for this node or `None` if no visitor + exists for this node. In that case the generic visit function is + used instead. + """ + return getattr(self, f"visit_{node.__class__.__name__}", None) + + def visit(self, node, *args, **kwargs): + """Visit a node.""" + f = self.get_visitor(node) + if f is not None: + return f(node, *args, **kwargs) + return self.generic_visit(node, *args, **kwargs) + + def generic_visit(self, node, *args, **kwargs): + """Called if no explicit visitor function exists for a node.""" + for node in node.iter_child_nodes(): + self.visit(node, *args, **kwargs) + + +class NodeTransformer(NodeVisitor): + """Walks the abstract syntax tree and allows modifications of nodes. + + The `NodeTransformer` will walk the AST and use the return value of the + visitor functions to replace or remove the old node. If the return + value of the visitor function is `None` the node will be removed + from the previous location otherwise it's replaced with the return + value. The return value may be the original node in which case no + replacement takes place. + """ + + def generic_visit(self, node, *args, **kwargs): + for field, old_value in node.iter_fields(): + if isinstance(old_value, list): + new_values = [] + for value in old_value: + if isinstance(value, Node): + value = self.visit(value, *args, **kwargs) + if value is None: + continue + elif not isinstance(value, Node): + new_values.extend(value) + continue + new_values.append(value) + old_value[:] = new_values + elif isinstance(old_value, Node): + new_node = self.visit(old_value, *args, **kwargs) + if new_node is None: + delattr(node, field) + else: + setattr(node, field, new_node) + return node + + def visit_list(self, node, *args, **kwargs): + """As transformers may return lists in some places this method + can be used to enforce a list as return value. + """ + rv = self.visit(node, *args, **kwargs) + if not isinstance(rv, list): + rv = [rv] + return rv diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..ce30d8b2 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,56 @@ +import os + +import pytest + +from jinja2 import Environment +from jinja2 import loaders +from jinja2.utils import have_async_gen + + +def pytest_ignore_collect(path): + if "async" in path.basename and not have_async_gen: + return True + return False + + +@pytest.fixture +def env(): + """returns a new environment.""" + return Environment() + + +@pytest.fixture +def dict_loader(): + """returns DictLoader""" + return loaders.DictLoader({"justdict.html": "FOO"}) + + +@pytest.fixture +def package_loader(): + """returns PackageLoader initialized from templates""" + return loaders.PackageLoader("res", "templates") + + +@pytest.fixture +def filesystem_loader(): + """returns FileSystemLoader initialized to res/templates directory""" + here = os.path.dirname(os.path.abspath(__file__)) + return loaders.FileSystemLoader(here + "/res/templates") + + +@pytest.fixture +def function_loader(): + """returns a FunctionLoader""" + return loaders.FunctionLoader({"justfunction.html": "FOO"}.get) + + +@pytest.fixture +def choice_loader(dict_loader, package_loader): + """returns a ChoiceLoader""" + return loaders.ChoiceLoader([dict_loader, package_loader]) + + +@pytest.fixture +def prefix_loader(filesystem_loader, dict_loader): + """returns a PrefixLoader""" + return loaders.PrefixLoader({"a": filesystem_loader, "b": dict_loader}) diff --git a/tests/res/__init__.py b/tests/res/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/res/__init__.py diff --git a/tests/res/package.zip b/tests/res/package.zip Binary files differnew file mode 100644 index 00000000..d4c9ce9c --- /dev/null +++ b/tests/res/package.zip diff --git a/tests/res/templates/broken.html b/tests/res/templates/broken.html new file mode 100644 index 00000000..77669fae --- /dev/null +++ b/tests/res/templates/broken.html @@ -0,0 +1,3 @@ +Before +{{ fail() }} +After diff --git a/tests/res/templates/foo/test.html b/tests/res/templates/foo/test.html new file mode 100644 index 00000000..b7d6715e --- /dev/null +++ b/tests/res/templates/foo/test.html @@ -0,0 +1 @@ +FOO diff --git a/tests/res/templates/mojibake.txt b/tests/res/templates/mojibake.txt new file mode 100644 index 00000000..4b94aa63 --- /dev/null +++ b/tests/res/templates/mojibake.txt @@ -0,0 +1 @@ +文字化け diff --git a/tests/res/templates/syntaxerror.html b/tests/res/templates/syntaxerror.html new file mode 100644 index 00000000..f21b8179 --- /dev/null +++ b/tests/res/templates/syntaxerror.html @@ -0,0 +1,4 @@ +Foo +{% for item in broken %} + ... +{% endif %} diff --git a/tests/res/templates/test.html b/tests/res/templates/test.html new file mode 100644 index 00000000..ba578e48 --- /dev/null +++ b/tests/res/templates/test.html @@ -0,0 +1 @@ +BAR diff --git a/tests/res/templates2/foo b/tests/res/templates2/foo new file mode 100644 index 00000000..1c4ad3e4 --- /dev/null +++ b/tests/res/templates2/foo @@ -0,0 +1,2 @@ +Looks like the start of templates/foo/test.html +Tested by test_filesystem_loader_overlapping_names diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 00000000..2679e8fb --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,433 @@ +import os +import shutil +import tempfile + +import pytest + +from jinja2 import ChainableUndefined +from jinja2 import DebugUndefined +from jinja2 import DictLoader +from jinja2 import Environment +from jinja2 import is_undefined +from jinja2 import make_logging_undefined +from jinja2 import meta +from jinja2 import StrictUndefined +from jinja2 import Template +from jinja2 import TemplatesNotFound +from jinja2 import Undefined +from jinja2 import UndefinedError +from jinja2.compiler import CodeGenerator +from jinja2.runtime import Context +from jinja2.utils import contextfunction +from jinja2.utils import Cycler +from jinja2.utils import environmentfunction +from jinja2.utils import evalcontextfunction + + +class TestExtendedAPI: + def test_item_and_attribute(self, env): + from jinja2.sandbox import SandboxedEnvironment + + for env in Environment(), SandboxedEnvironment(): + tmpl = env.from_string("{{ foo.items()|list }}") + assert tmpl.render(foo={"items": 42}) == "[('items', 42)]" + tmpl = env.from_string('{{ foo|attr("items")()|list }}') + assert tmpl.render(foo={"items": 42}) == "[('items', 42)]" + tmpl = env.from_string('{{ foo["items"] }}') + assert tmpl.render(foo={"items": 42}) == "42" + + def test_finalize(self): + e = Environment(finalize=lambda v: "" if v is None else v) + t = e.from_string("{% for item in seq %}|{{ item }}{% endfor %}") + assert t.render(seq=(None, 1, "foo")) == "||1|foo" + + def test_finalize_constant_expression(self): + e = Environment(finalize=lambda v: "" if v is None else v) + t = e.from_string("<{{ none }}>") + assert t.render() == "<>" + + def test_no_finalize_template_data(self): + e = Environment(finalize=lambda v: type(v).__name__) + t = e.from_string("<{{ value }}>") + # If template data was finalized, it would print "strintstr". + assert t.render(value=123) == "<int>" + + def test_context_finalize(self): + @contextfunction + def finalize(context, value): + return value * context["scale"] + + e = Environment(finalize=finalize) + t = e.from_string("{{ value }}") + assert t.render(value=5, scale=3) == "15" + + def test_eval_finalize(self): + @evalcontextfunction + def finalize(eval_ctx, value): + return str(eval_ctx.autoescape) + value + + e = Environment(finalize=finalize, autoescape=True) + t = e.from_string("{{ value }}") + assert t.render(value="<script>") == "True<script>" + + def test_env_autoescape(self): + @environmentfunction + def finalize(env, value): + return " ".join( + (env.variable_start_string, repr(value), env.variable_end_string) + ) + + e = Environment(finalize=finalize) + t = e.from_string("{{ value }}") + assert t.render(value="hello") == "{{ 'hello' }}" + + def test_cycler(self, env): + items = 1, 2, 3 + c = Cycler(*items) + for item in items + items: + assert c.current == item + assert next(c) == item + next(c) + assert c.current == 2 + c.reset() + assert c.current == 1 + + def test_expressions(self, env): + expr = env.compile_expression("foo") + assert expr() is None + assert expr(foo=42) == 42 + expr2 = env.compile_expression("foo", undefined_to_none=False) + assert is_undefined(expr2()) + + expr = env.compile_expression("42 + foo") + assert expr(foo=42) == 84 + + def test_template_passthrough(self, env): + t = Template("Content") + assert env.get_template(t) is t + assert env.select_template([t]) is t + assert env.get_or_select_template([t]) is t + assert env.get_or_select_template(t) is t + + def test_get_template_undefined(self, env): + """Passing Undefined to get/select_template raises an + UndefinedError or shows the undefined message in the list. + """ + env.loader = DictLoader({}) + t = Undefined(name="no_name_1") + + with pytest.raises(UndefinedError): + env.get_template(t) + + with pytest.raises(UndefinedError): + env.get_or_select_template(t) + + with pytest.raises(UndefinedError): + env.select_template(t) + + with pytest.raises(TemplatesNotFound) as exc_info: + env.select_template([t, "no_name_2"]) + + exc_message = str(exc_info.value) + assert "'no_name_1' is undefined" in exc_message + assert "no_name_2" in exc_message + + def test_autoescape_autoselect(self, env): + def select_autoescape(name): + if name is None or "." not in name: + return False + return name.endswith(".html") + + env = Environment( + autoescape=select_autoescape, + loader=DictLoader({"test.txt": "{{ foo }}", "test.html": "{{ foo }}"}), + ) + t = env.get_template("test.txt") + assert t.render(foo="<foo>") == "<foo>" + t = env.get_template("test.html") + assert t.render(foo="<foo>") == "<foo>" + t = env.from_string("{{ foo }}") + assert t.render(foo="<foo>") == "<foo>" + + def test_sandbox_max_range(self, env): + from jinja2.sandbox import SandboxedEnvironment, MAX_RANGE + + env = SandboxedEnvironment() + t = env.from_string("{% for item in range(total) %}{{ item }}{% endfor %}") + + with pytest.raises(OverflowError): + t.render(total=MAX_RANGE + 1) + + +class TestMeta: + def test_find_undeclared_variables(self, env): + ast = env.parse("{% set foo = 42 %}{{ bar + foo }}") + x = meta.find_undeclared_variables(ast) + assert x == {"bar"} + + ast = env.parse( + "{% set foo = 42 %}{{ bar + foo }}" + "{% macro meh(x) %}{{ x }}{% endmacro %}" + "{% for item in seq %}{{ muh(item) + meh(seq) }}" + "{% endfor %}" + ) + x = meta.find_undeclared_variables(ast) + assert x == {"bar", "seq", "muh"} + + ast = env.parse("{% for x in range(5) %}{{ x }}{% endfor %}{{ foo }}") + x = meta.find_undeclared_variables(ast) + assert x == {"foo"} + + def test_find_refererenced_templates(self, env): + ast = env.parse('{% extends "layout.html" %}{% include helper %}') + i = meta.find_referenced_templates(ast) + assert next(i) == "layout.html" + assert next(i) is None + assert list(i) == [] + + ast = env.parse( + '{% extends "layout.html" %}' + '{% from "test.html" import a, b as c %}' + '{% import "meh.html" as meh %}' + '{% include "muh.html" %}' + ) + i = meta.find_referenced_templates(ast) + assert list(i) == ["layout.html", "test.html", "meh.html", "muh.html"] + + def test_find_included_templates(self, env): + ast = env.parse('{% include ["foo.html", "bar.html"] %}') + i = meta.find_referenced_templates(ast) + assert list(i) == ["foo.html", "bar.html"] + + ast = env.parse('{% include ("foo.html", "bar.html") %}') + i = meta.find_referenced_templates(ast) + assert list(i) == ["foo.html", "bar.html"] + + ast = env.parse('{% include ["foo.html", "bar.html", foo] %}') + i = meta.find_referenced_templates(ast) + assert list(i) == ["foo.html", "bar.html", None] + + ast = env.parse('{% include ("foo.html", "bar.html", foo) %}') + i = meta.find_referenced_templates(ast) + assert list(i) == ["foo.html", "bar.html", None] + + +class TestStreaming: + def test_basic_streaming(self, env): + t = env.from_string( + "<ul>{% for item in seq %}<li>{{ loop.index }} - {{ item }}</li>" + "{%- endfor %}</ul>" + ) + stream = t.stream(seq=list(range(3))) + assert next(stream) == "<ul>" + assert "".join(stream) == "<li>1 - 0</li><li>2 - 1</li><li>3 - 2</li></ul>" + + def test_buffered_streaming(self, env): + tmpl = env.from_string( + "<ul>{% for item in seq %}<li>{{ loop.index }} - {{ item }}</li>" + "{%- endfor %}</ul>" + ) + stream = tmpl.stream(seq=list(range(3))) + stream.enable_buffering(size=3) + assert next(stream) == "<ul><li>1" + assert next(stream) == " - 0</li>" + + def test_streaming_behavior(self, env): + tmpl = env.from_string("") + stream = tmpl.stream() + assert not stream.buffered + stream.enable_buffering(20) + assert stream.buffered + stream.disable_buffering() + assert not stream.buffered + + def test_dump_stream(self, env): + tmp = tempfile.mkdtemp() + try: + tmpl = env.from_string("\u2713") + stream = tmpl.stream() + stream.dump(os.path.join(tmp, "dump.txt"), "utf-8") + with open(os.path.join(tmp, "dump.txt"), "rb") as f: + assert f.read() == b"\xe2\x9c\x93" + finally: + shutil.rmtree(tmp) + + +class TestUndefined: + def test_stopiteration_is_undefined(self): + def test(): + raise StopIteration() + + t = Template("A{{ test() }}B") + assert t.render(test=test) == "AB" + t = Template("A{{ test().missingattribute }}B") + pytest.raises(UndefinedError, t.render, test=test) + + def test_undefined_and_special_attributes(self): + with pytest.raises(AttributeError): + Undefined("Foo").__dict__ + + def test_undefined_attribute_error(self): + # Django's LazyObject turns the __class__ attribute into a + # property that resolves the wrapped function. If that wrapped + # function raises an AttributeError, printing the repr of the + # object in the undefined message would cause a RecursionError. + class Error: + @property + def __class__(self): + raise AttributeError() + + u = Undefined(obj=Error(), name="hello") + + with pytest.raises(UndefinedError): + getattr(u, "recursion", None) + + def test_logging_undefined(self): + _messages = [] + + class DebugLogger: + def warning(self, msg, *args): + _messages.append("W:" + msg % args) + + def error(self, msg, *args): + _messages.append("E:" + msg % args) + + logging_undefined = make_logging_undefined(DebugLogger()) + env = Environment(undefined=logging_undefined) + assert env.from_string("{{ missing }}").render() == "" + pytest.raises(UndefinedError, env.from_string("{{ missing.attribute }}").render) + assert env.from_string("{{ missing|list }}").render() == "[]" + assert env.from_string("{{ missing is not defined }}").render() == "True" + assert env.from_string("{{ foo.missing }}").render(foo=42) == "" + assert env.from_string("{{ not missing }}").render() == "True" + assert _messages == [ + "W:Template variable warning: 'missing' is undefined", + "E:Template variable error: 'missing' is undefined", + "W:Template variable warning: 'missing' is undefined", + "W:Template variable warning: 'int object' has no attribute 'missing'", + "W:Template variable warning: 'missing' is undefined", + ] + + def test_default_undefined(self): + env = Environment(undefined=Undefined) + assert env.from_string("{{ missing }}").render() == "" + pytest.raises(UndefinedError, env.from_string("{{ missing.attribute }}").render) + assert env.from_string("{{ missing|list }}").render() == "[]" + assert env.from_string("{{ missing is not defined }}").render() == "True" + assert env.from_string("{{ foo.missing }}").render(foo=42) == "" + assert env.from_string("{{ not missing }}").render() == "True" + pytest.raises(UndefinedError, env.from_string("{{ missing - 1}}").render) + und1 = Undefined(name="x") + und2 = Undefined(name="y") + assert und1 == und2 + assert und1 != 42 + assert hash(und1) == hash(und2) == hash(Undefined()) + with pytest.raises(AttributeError): + getattr(Undefined, "__slots__") # noqa: B009 + + def test_chainable_undefined(self): + env = Environment(undefined=ChainableUndefined) + # The following tests are copied from test_default_undefined + assert env.from_string("{{ missing }}").render() == "" + assert env.from_string("{{ missing|list }}").render() == "[]" + assert env.from_string("{{ missing is not defined }}").render() == "True" + assert env.from_string("{{ foo.missing }}").render(foo=42) == "" + assert env.from_string("{{ not missing }}").render() == "True" + pytest.raises(UndefinedError, env.from_string("{{ missing - 1}}").render) + with pytest.raises(AttributeError): + getattr(ChainableUndefined, "__slots__") # noqa: B009 + + # The following tests ensure subclass functionality works as expected + assert env.from_string('{{ missing.bar["baz"] }}').render() == "" + assert env.from_string('{{ foo.bar["baz"]._undefined_name }}').render() == "foo" + assert ( + env.from_string('{{ foo.bar["baz"]._undefined_name }}').render(foo=42) + == "bar" + ) + assert ( + env.from_string('{{ foo.bar["baz"]._undefined_name }}').render( + foo={"bar": 42} + ) + == "baz" + ) + + def test_debug_undefined(self): + env = Environment(undefined=DebugUndefined) + assert env.from_string("{{ missing }}").render() == "{{ missing }}" + pytest.raises(UndefinedError, env.from_string("{{ missing.attribute }}").render) + assert env.from_string("{{ missing|list }}").render() == "[]" + assert env.from_string("{{ missing is not defined }}").render() == "True" + assert ( + env.from_string("{{ foo.missing }}").render(foo=42) + == "{{ no such element: int object['missing'] }}" + ) + assert env.from_string("{{ not missing }}").render() == "True" + undefined_hint = "this is testing undefined hint of DebugUndefined" + assert ( + str(DebugUndefined(hint=undefined_hint)) + == f"{{{{ undefined value printed: {undefined_hint} }}}}" + ) + with pytest.raises(AttributeError): + getattr(DebugUndefined, "__slots__") # noqa: B009 + + def test_strict_undefined(self): + env = Environment(undefined=StrictUndefined) + pytest.raises(UndefinedError, env.from_string("{{ missing }}").render) + pytest.raises(UndefinedError, env.from_string("{{ missing.attribute }}").render) + pytest.raises(UndefinedError, env.from_string("{{ missing|list }}").render) + assert env.from_string("{{ missing is not defined }}").render() == "True" + pytest.raises( + UndefinedError, env.from_string("{{ foo.missing }}").render, foo=42 + ) + pytest.raises(UndefinedError, env.from_string("{{ not missing }}").render) + assert ( + env.from_string('{{ missing|default("default", true) }}').render() + == "default" + ) + with pytest.raises(AttributeError): + getattr(StrictUndefined, "__slots__") # noqa: B009 + assert env.from_string('{{ "foo" if false }}').render() == "" + + def test_indexing_gives_undefined(self): + t = Template("{{ var[42].foo }}") + pytest.raises(UndefinedError, t.render, var=0) + + def test_none_gives_proper_error(self): + with pytest.raises(UndefinedError, match="'None' has no attribute 'split'"): + Environment().getattr(None, "split")() + + def test_object_repr(self): + with pytest.raises( + UndefinedError, match="'int object' has no attribute 'upper'" + ): + Undefined(obj=42, name="upper")() + + +class TestLowLevel: + def test_custom_code_generator(self): + class CustomCodeGenerator(CodeGenerator): + def visit_Const(self, node, frame=None): + # This method is pure nonsense, but works fine for testing... + if node.value == "foo": + self.write(repr("bar")) + else: + super().visit_Const(node, frame) + + class CustomEnvironment(Environment): + code_generator_class = CustomCodeGenerator + + env = CustomEnvironment() + tmpl = env.from_string('{% set foo = "foo" %}{{ foo }}') + assert tmpl.render() == "bar" + + def test_custom_context(self): + class CustomContext(Context): + def resolve_or_missing(self, key): + return "resolve-" + key + + class CustomEnvironment(Environment): + context_class = CustomContext + + env = CustomEnvironment() + tmpl = env.from_string("{{ foo }}") + assert tmpl.render() == "resolve-foo" diff --git a/tests/test_async.py b/tests/test_async.py new file mode 100644 index 00000000..bfdcdb21 --- /dev/null +++ b/tests/test_async.py @@ -0,0 +1,590 @@ +import asyncio + +import pytest + +from jinja2 import DictLoader +from jinja2 import Environment +from jinja2 import Template +from jinja2.asyncsupport import auto_aiter +from jinja2.exceptions import TemplateNotFound +from jinja2.exceptions import TemplatesNotFound +from jinja2.exceptions import UndefinedError + + +def run(coro): + loop = asyncio.get_event_loop() + return loop.run_until_complete(coro) + + +def test_basic_async(): + t = Template( + "{% for item in [1, 2, 3] %}[{{ item }}]{% endfor %}", enable_async=True + ) + + async def func(): + return await t.render_async() + + rv = run(func()) + assert rv == "[1][2][3]" + + +def test_await_on_calls(): + t = Template("{{ async_func() + normal_func() }}", enable_async=True) + + async def async_func(): + return 42 + + def normal_func(): + return 23 + + async def func(): + return await t.render_async(async_func=async_func, normal_func=normal_func) + + rv = run(func()) + assert rv == "65" + + +def test_await_on_calls_normal_render(): + t = Template("{{ async_func() + normal_func() }}", enable_async=True) + + async def async_func(): + return 42 + + def normal_func(): + return 23 + + rv = t.render(async_func=async_func, normal_func=normal_func) + + assert rv == "65" + + +def test_await_and_macros(): + t = Template( + "{% macro foo(x) %}[{{ x }}][{{ async_func() }}]{% endmacro %}{{ foo(42) }}", + enable_async=True, + ) + + async def async_func(): + return 42 + + async def func(): + return await t.render_async(async_func=async_func) + + rv = run(func()) + assert rv == "[42][42]" + + +def test_async_blocks(): + t = Template( + "{% block foo %}<Test>{% endblock %}{{ self.foo() }}", + enable_async=True, + autoescape=True, + ) + + async def func(): + return await t.render_async() + + rv = run(func()) + assert rv == "<Test><Test>" + + +def test_async_generate(): + t = Template("{% for x in [1, 2, 3] %}{{ x }}{% endfor %}", enable_async=True) + rv = list(t.generate()) + assert rv == ["1", "2", "3"] + + +def test_async_iteration_in_templates(): + t = Template("{% for x in rng %}{{ x }}{% endfor %}", enable_async=True) + + async def async_iterator(): + for item in [1, 2, 3]: + yield item + + rv = list(t.generate(rng=async_iterator())) + assert rv == ["1", "2", "3"] + + +def test_async_iteration_in_templates_extended(): + t = Template( + "{% for x in rng %}{{ loop.index0 }}/{{ x }}{% endfor %}", enable_async=True + ) + stream = t.generate(rng=auto_aiter(range(1, 4))) + assert next(stream) == "0" + assert "".join(stream) == "/11/22/3" + + +@pytest.fixture +def test_env_async(): + env = Environment( + loader=DictLoader( + dict( + module="{% macro test() %}[{{ foo }}|{{ bar }}]{% endmacro %}", + header="[{{ foo }}|{{ 23 }}]", + o_printer="({{ o }})", + ) + ), + enable_async=True, + ) + env.globals["bar"] = 23 + return env + + +class TestAsyncImports: + def test_context_imports(self, test_env_async): + t = test_env_async.from_string('{% import "module" as m %}{{ m.test() }}') + assert t.render(foo=42) == "[|23]" + t = test_env_async.from_string( + '{% import "module" as m without context %}{{ m.test() }}' + ) + assert t.render(foo=42) == "[|23]" + t = test_env_async.from_string( + '{% import "module" as m with context %}{{ m.test() }}' + ) + assert t.render(foo=42) == "[42|23]" + t = test_env_async.from_string('{% from "module" import test %}{{ test() }}') + assert t.render(foo=42) == "[|23]" + t = test_env_async.from_string( + '{% from "module" import test without context %}{{ test() }}' + ) + assert t.render(foo=42) == "[|23]" + t = test_env_async.from_string( + '{% from "module" import test with context %}{{ test() }}' + ) + assert t.render(foo=42) == "[42|23]" + + def test_trailing_comma(self, test_env_async): + test_env_async.from_string('{% from "foo" import bar, baz with context %}') + test_env_async.from_string('{% from "foo" import bar, baz, with context %}') + test_env_async.from_string('{% from "foo" import bar, with context %}') + test_env_async.from_string('{% from "foo" import bar, with, context %}') + test_env_async.from_string('{% from "foo" import bar, with with context %}') + + def test_exports(self, test_env_async): + m = run( + test_env_async.from_string( + """ + {% macro toplevel() %}...{% endmacro %} + {% macro __private() %}...{% endmacro %} + {% set variable = 42 %} + {% for item in [1] %} + {% macro notthere() %}{% endmacro %} + {% endfor %} + """ + )._get_default_module_async() + ) + assert run(m.toplevel()) == "..." + assert not hasattr(m, "__missing") + assert m.variable == 42 + assert not hasattr(m, "notthere") + + +class TestAsyncIncludes: + def test_context_include(self, test_env_async): + t = test_env_async.from_string('{% include "header" %}') + assert t.render(foo=42) == "[42|23]" + t = test_env_async.from_string('{% include "header" with context %}') + assert t.render(foo=42) == "[42|23]" + t = test_env_async.from_string('{% include "header" without context %}') + assert t.render(foo=42) == "[|23]" + + def test_choice_includes(self, test_env_async): + t = test_env_async.from_string('{% include ["missing", "header"] %}') + assert t.render(foo=42) == "[42|23]" + + t = test_env_async.from_string( + '{% include ["missing", "missing2"] ignore missing %}' + ) + assert t.render(foo=42) == "" + + t = test_env_async.from_string('{% include ["missing", "missing2"] %}') + pytest.raises(TemplateNotFound, t.render) + with pytest.raises(TemplatesNotFound) as e: + t.render() + + assert e.value.templates == ["missing", "missing2"] + assert e.value.name == "missing2" + + def test_includes(t, **ctx): + ctx["foo"] = 42 + assert t.render(ctx) == "[42|23]" + + t = test_env_async.from_string('{% include ["missing", "header"] %}') + test_includes(t) + t = test_env_async.from_string("{% include x %}") + test_includes(t, x=["missing", "header"]) + t = test_env_async.from_string('{% include [x, "header"] %}') + test_includes(t, x="missing") + t = test_env_async.from_string("{% include x %}") + test_includes(t, x="header") + t = test_env_async.from_string("{% include x %}") + test_includes(t, x="header") + t = test_env_async.from_string("{% include [x] %}") + test_includes(t, x="header") + + def test_include_ignoring_missing(self, test_env_async): + t = test_env_async.from_string('{% include "missing" %}') + pytest.raises(TemplateNotFound, t.render) + for extra in "", "with context", "without context": + t = test_env_async.from_string( + '{% include "missing" ignore missing ' + extra + " %}" + ) + assert t.render() == "" + + def test_context_include_with_overrides(self, test_env_async): + env = Environment( + loader=DictLoader( + dict( + main="{% for item in [1, 2, 3] %}{% include 'item' %}{% endfor %}", + item="{{ item }}", + ) + ) + ) + assert env.get_template("main").render() == "123" + + def test_unoptimized_scopes(self, test_env_async): + t = test_env_async.from_string( + """ + {% macro outer(o) %} + {% macro inner() %} + {% include "o_printer" %} + {% endmacro %} + {{ inner() }} + {% endmacro %} + {{ outer("FOO") }} + """ + ) + assert t.render().strip() == "(FOO)" + + def test_unoptimized_scopes_autoescape(self): + env = Environment( + loader=DictLoader(dict(o_printer="({{ o }})",)), + autoescape=True, + enable_async=True, + ) + t = env.from_string( + """ + {% macro outer(o) %} + {% macro inner() %} + {% include "o_printer" %} + {% endmacro %} + {{ inner() }} + {% endmacro %} + {{ outer("FOO") }} + """ + ) + assert t.render().strip() == "(FOO)" + + +class TestAsyncForLoop: + def test_simple(self, test_env_async): + tmpl = test_env_async.from_string("{% for item in seq %}{{ item }}{% endfor %}") + assert tmpl.render(seq=list(range(10))) == "0123456789" + + def test_else(self, test_env_async): + tmpl = test_env_async.from_string( + "{% for item in seq %}XXX{% else %}...{% endfor %}" + ) + assert tmpl.render() == "..." + + def test_empty_blocks(self, test_env_async): + tmpl = test_env_async.from_string( + "<{% for item in seq %}{% else %}{% endfor %}>" + ) + assert tmpl.render() == "<>" + + @pytest.mark.parametrize( + "transform", [lambda x: x, iter, reversed, lambda x: (i for i in x), auto_aiter] + ) + def test_context_vars(self, test_env_async, transform): + t = test_env_async.from_string( + "{% for item in seq %}{{ loop.index }}|{{ loop.index0 }}" + "|{{ loop.revindex }}|{{ loop.revindex0 }}|{{ loop.first }}" + "|{{ loop.last }}|{{ loop.length }}\n{% endfor %}" + ) + out = t.render(seq=transform([42, 24])) + assert out == "1|0|2|1|True|False|2\n2|1|1|0|False|True|2\n" + + def test_cycling(self, test_env_async): + tmpl = test_env_async.from_string( + """{% for item in seq %}{{ + loop.cycle('<1>', '<2>') }}{% endfor %}{% + for item in seq %}{{ loop.cycle(*through) }}{% endfor %}""" + ) + output = tmpl.render(seq=list(range(4)), through=("<1>", "<2>")) + assert output == "<1><2>" * 4 + + def test_lookaround(self, test_env_async): + tmpl = test_env_async.from_string( + """{% for item in seq -%} + {{ loop.previtem|default('x') }}-{{ item }}-{{ + loop.nextitem|default('x') }}| + {%- endfor %}""" + ) + output = tmpl.render(seq=list(range(4))) + assert output == "x-0-1|0-1-2|1-2-3|2-3-x|" + + def test_changed(self, test_env_async): + tmpl = test_env_async.from_string( + """{% for item in seq -%} + {{ loop.changed(item) }}, + {%- endfor %}""" + ) + output = tmpl.render(seq=[None, None, 1, 2, 2, 3, 4, 4, 4]) + assert output == "True,False,True,True,False,True,True,False,False," + + def test_scope(self, test_env_async): + tmpl = test_env_async.from_string("{% for item in seq %}{% endfor %}{{ item }}") + output = tmpl.render(seq=list(range(10))) + assert not output + + def test_varlen(self, test_env_async): + def inner(): + yield from range(5) + + tmpl = test_env_async.from_string( + "{% for item in iter %}{{ item }}{% endfor %}" + ) + output = tmpl.render(iter=inner()) + assert output == "01234" + + def test_noniter(self, test_env_async): + tmpl = test_env_async.from_string("{% for item in none %}...{% endfor %}") + pytest.raises(TypeError, tmpl.render) + + def test_recursive(self, test_env_async): + tmpl = test_env_async.from_string( + """{% for item in seq recursive -%} + [{{ item.a }}{% if item.b %}<{{ loop(item.b) }}>{% endif %}] + {%- endfor %}""" + ) + assert ( + tmpl.render( + seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a="a")]), + ] + ) + == "[1<[1][2]>][2<[1][2]>][3<[a]>]" + ) + + def test_recursive_lookaround(self, test_env_async): + tmpl = test_env_async.from_string( + """{% for item in seq recursive -%} + [{{ loop.previtem.a if loop.previtem is defined else 'x' }}.{{ + item.a }}.{{ loop.nextitem.a if loop.nextitem is defined else 'x' + }}{% if item.b %}<{{ loop(item.b) }}>{% endif %}] + {%- endfor %}""" + ) + assert ( + tmpl.render( + seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a="a")]), + ] + ) + == "[x.1.2<[x.1.2][1.2.x]>][1.2.3<[x.1.2][1.2.x]>][2.3.x<[x.a.x]>]" + ) + + def test_recursive_depth0(self, test_env_async): + tmpl = test_env_async.from_string( + "{% for item in seq recursive %}[{{ loop.depth0 }}:{{ item.a }}" + "{% if item.b %}<{{ loop(item.b) }}>{% endif %}]{% endfor %}" + ) + assert ( + tmpl.render( + seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a="a")]), + ] + ) + == "[0:1<[1:1][1:2]>][0:2<[1:1][1:2]>][0:3<[1:a]>]" + ) + + def test_recursive_depth(self, test_env_async): + tmpl = test_env_async.from_string( + "{% for item in seq recursive %}[{{ loop.depth }}:{{ item.a }}" + "{% if item.b %}<{{ loop(item.b) }}>{% endif %}]{% endfor %}" + ) + assert ( + tmpl.render( + seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a="a")]), + ] + ) + == "[1:1<[2:1][2:2]>][1:2<[2:1][2:2]>][1:3<[2:a]>]" + ) + + def test_looploop(self, test_env_async): + tmpl = test_env_async.from_string( + """{% for row in table %} + {%- set rowloop = loop -%} + {% for cell in row -%} + [{{ rowloop.index }}|{{ loop.index }}] + {%- endfor %} + {%- endfor %}""" + ) + assert tmpl.render(table=["ab", "cd"]) == "[1|1][1|2][2|1][2|2]" + + def test_reversed_bug(self, test_env_async): + tmpl = test_env_async.from_string( + "{% for i in items %}{{ i }}" + "{% if not loop.last %}" + ",{% endif %}{% endfor %}" + ) + assert tmpl.render(items=reversed([3, 2, 1])) == "1,2,3" + + def test_loop_errors(self, test_env_async): + tmpl = test_env_async.from_string( + """{% for item in [1] if loop.index + == 0 %}...{% endfor %}""" + ) + pytest.raises(UndefinedError, tmpl.render) + tmpl = test_env_async.from_string( + """{% for item in [] %}...{% else + %}{{ loop }}{% endfor %}""" + ) + assert tmpl.render() == "" + + def test_loop_filter(self, test_env_async): + tmpl = test_env_async.from_string( + "{% for item in range(10) if item is even %}[{{ item }}]{% endfor %}" + ) + assert tmpl.render() == "[0][2][4][6][8]" + tmpl = test_env_async.from_string( + """ + {%- for item in range(10) if item is even %}[{{ + loop.index }}:{{ item }}]{% endfor %}""" + ) + assert tmpl.render() == "[1:0][2:2][3:4][4:6][5:8]" + + def test_scoped_special_var(self, test_env_async): + t = test_env_async.from_string( + "{% for s in seq %}[{{ loop.first }}{% for c in s %}" + "|{{ loop.first }}{% endfor %}]{% endfor %}" + ) + assert t.render(seq=("ab", "cd")) == "[True|True|False][False|True|False]" + + def test_scoped_loop_var(self, test_env_async): + t = test_env_async.from_string( + "{% for x in seq %}{{ loop.first }}" + "{% for y in seq %}{% endfor %}{% endfor %}" + ) + assert t.render(seq="ab") == "TrueFalse" + t = test_env_async.from_string( + "{% for x in seq %}{% for y in seq %}" + "{{ loop.first }}{% endfor %}{% endfor %}" + ) + assert t.render(seq="ab") == "TrueFalseTrueFalse" + + def test_recursive_empty_loop_iter(self, test_env_async): + t = test_env_async.from_string( + """ + {%- for item in foo recursive -%}{%- endfor -%} + """ + ) + assert t.render(dict(foo=[])) == "" + + def test_call_in_loop(self, test_env_async): + t = test_env_async.from_string( + """ + {%- macro do_something() -%} + [{{ caller() }}] + {%- endmacro %} + + {%- for i in [1, 2, 3] %} + {%- call do_something() -%} + {{ i }} + {%- endcall %} + {%- endfor -%} + """ + ) + assert t.render() == "[1][2][3]" + + def test_scoping_bug(self, test_env_async): + t = test_env_async.from_string( + """ + {%- for item in foo %}...{{ item }}...{% endfor %} + {%- macro item(a) %}...{{ a }}...{% endmacro %} + {{- item(2) -}} + """ + ) + assert t.render(foo=(1,)) == "...1......2..." + + def test_unpacking(self, test_env_async): + tmpl = test_env_async.from_string( + "{% for a, b, c in [[1, 2, 3]] %}{{ a }}|{{ b }}|{{ c }}{% endfor %}" + ) + assert tmpl.render() == "1|2|3" + + def test_recursive_loop_filter(self, test_env_async): + t = test_env_async.from_string( + """ + <?xml version="1.0" encoding="UTF-8"?> + <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> + {%- for page in [site.root] if page.url != this recursive %} + <url><loc>{{ page.url }}</loc></url> + {{- loop(page.children) }} + {%- endfor %} + </urlset> + """ + ) + sm = t.render( + this="/foo", + site={"root": {"url": "/", "children": [{"url": "/foo"}, {"url": "/bar"}]}}, + ) + lines = [x.strip() for x in sm.splitlines() if x.strip()] + assert lines == [ + '<?xml version="1.0" encoding="UTF-8"?>', + '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">', + "<url><loc>/</loc></url>", + "<url><loc>/bar</loc></url>", + "</urlset>", + ] + + def test_nonrecursive_loop_filter(self, test_env_async): + t = test_env_async.from_string( + """ + <?xml version="1.0" encoding="UTF-8"?> + <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> + {%- for page in items if page.url != this %} + <url><loc>{{ page.url }}</loc></url> + {%- endfor %} + </urlset> + """ + ) + sm = t.render( + this="/foo", items=[{"url": "/"}, {"url": "/foo"}, {"url": "/bar"}] + ) + lines = [x.strip() for x in sm.splitlines() if x.strip()] + assert lines == [ + '<?xml version="1.0" encoding="UTF-8"?>', + '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">', + "<url><loc>/</loc></url>", + "<url><loc>/bar</loc></url>", + "</urlset>", + ] + + def test_bare_async(self, test_env_async): + t = test_env_async.from_string('{% extends "header" %}') + assert t.render(foo=42) == "[42|23]" + + def test_awaitable_property_slicing(self, test_env_async): + t = test_env_async.from_string("{% for x in a.b[:1] %}{{ x }}{% endfor %}") + assert t.render(a=dict(b=[1, 2, 3])) == "1" + + +def test_namespace_awaitable(test_env_async): + async def _test(): + t = test_env_async.from_string( + '{% set ns = namespace(foo="Bar") %}{{ ns.foo }}' + ) + actual = await t.render_async() + assert actual == "Bar" + + run(_test()) diff --git a/tests/test_asyncfilters.py b/tests/test_asyncfilters.py new file mode 100644 index 00000000..7c737c83 --- /dev/null +++ b/tests/test_asyncfilters.py @@ -0,0 +1,224 @@ +from collections import namedtuple + +import pytest + +from jinja2 import Environment +from jinja2.utils import Markup + + +async def make_aiter(iter): + for item in iter: + yield item + + +def mark_dualiter(parameter, factory): + def decorator(f): + return pytest.mark.parametrize( + parameter, [lambda: factory(), lambda: make_aiter(factory())] + )(f) + + return decorator + + +@pytest.fixture +def env_async(): + return Environment(enable_async=True) + + +@mark_dualiter("foo", lambda: range(10)) +def test_first(env_async, foo): + tmpl = env_async.from_string("{{ foo()|first }}") + out = tmpl.render(foo=foo) + assert out == "0" + + +@mark_dualiter( + "items", + lambda: [ + {"foo": 1, "bar": 2}, + {"foo": 2, "bar": 3}, + {"foo": 1, "bar": 1}, + {"foo": 3, "bar": 4}, + ], +) +def test_groupby(env_async, items): + tmpl = env_async.from_string( + """ + {%- for grouper, list in items()|groupby('foo') -%} + {{ grouper }}{% for x in list %}: {{ x.foo }}, {{ x.bar }}{% endfor %}| + {%- endfor %}""" + ) + assert tmpl.render(items=items).split("|") == [ + "1: 1, 2: 1, 1", + "2: 2, 3", + "3: 3, 4", + "", + ] + + +@mark_dualiter("items", lambda: [("a", 1), ("a", 2), ("b", 1)]) +def test_groupby_tuple_index(env_async, items): + tmpl = env_async.from_string( + """ + {%- for grouper, list in items()|groupby(0) -%} + {{ grouper }}{% for x in list %}:{{ x.1 }}{% endfor %}| + {%- endfor %}""" + ) + assert tmpl.render(items=items) == "a:1:2|b:1|" + + +def make_articles(): + Date = namedtuple("Date", "day,month,year") + Article = namedtuple("Article", "title,date") + return [ + Article("aha", Date(1, 1, 1970)), + Article("interesting", Date(2, 1, 1970)), + Article("really?", Date(3, 1, 1970)), + Article("totally not", Date(1, 1, 1971)), + ] + + +@mark_dualiter("articles", make_articles) +def test_groupby_multidot(env_async, articles): + tmpl = env_async.from_string( + """ + {%- for year, list in articles()|groupby('date.year') -%} + {{ year }}{% for x in list %}[{{ x.title }}]{% endfor %}| + {%- endfor %}""" + ) + assert tmpl.render(articles=articles).split("|") == [ + "1970[aha][interesting][really?]", + "1971[totally not]", + "", + ] + + +@mark_dualiter("int_items", lambda: [1, 2, 3]) +def test_join_env_int(env_async, int_items): + tmpl = env_async.from_string('{{ items()|join("|") }}') + out = tmpl.render(items=int_items) + assert out == "1|2|3" + + +@mark_dualiter("string_items", lambda: ["<foo>", Markup("<span>foo</span>")]) +def test_join_string_list(string_items): + env2 = Environment(autoescape=True, enable_async=True) + tmpl = env2.from_string('{{ ["<foo>", "<span>foo</span>"|safe]|join }}') + assert tmpl.render(items=string_items) == "<foo><span>foo</span>" + + +def make_users(): + User = namedtuple("User", "username") + return map(User, ["foo", "bar"]) + + +@mark_dualiter("users", make_users) +def test_join_attribute(env_async, users): + tmpl = env_async.from_string("""{{ users()|join(', ', 'username') }}""") + assert tmpl.render(users=users) == "foo, bar" + + +@mark_dualiter("items", lambda: [1, 2, 3, 4, 5]) +def test_simple_reject(env_async, items): + tmpl = env_async.from_string('{{ items()|reject("odd")|join("|") }}') + assert tmpl.render(items=items) == "2|4" + + +@mark_dualiter("items", lambda: [None, False, 0, 1, 2, 3, 4, 5]) +def test_bool_reject(env_async, items): + tmpl = env_async.from_string('{{ items()|reject|join("|") }}') + assert tmpl.render(items=items) == "None|False|0" + + +@mark_dualiter("items", lambda: [1, 2, 3, 4, 5]) +def test_simple_select(env_async, items): + tmpl = env_async.from_string('{{ items()|select("odd")|join("|") }}') + assert tmpl.render(items=items) == "1|3|5" + + +@mark_dualiter("items", lambda: [None, False, 0, 1, 2, 3, 4, 5]) +def test_bool_select(env_async, items): + tmpl = env_async.from_string('{{ items()|select|join("|") }}') + assert tmpl.render(items=items) == "1|2|3|4|5" + + +def make_users(): + User = namedtuple("User", "name,is_active") + return [ + User("john", True), + User("jane", True), + User("mike", False), + ] + + +@mark_dualiter("users", make_users) +def test_simple_select_attr(env_async, users): + tmpl = env_async.from_string( + '{{ users()|selectattr("is_active")|map(attribute="name")|join("|") }}' + ) + assert tmpl.render(users=users) == "john|jane" + + +@mark_dualiter("items", lambda: list("123")) +def test_simple_map(env_async, items): + tmpl = env_async.from_string('{{ items()|map("int")|sum }}') + assert tmpl.render(items=items) == "6" + + +def test_map_sum(env_async): # async map + async filter + tmpl = env_async.from_string('{{ [[1,2], [3], [4,5,6]]|map("sum")|list }}') + assert tmpl.render() == "[3, 3, 15]" + + +@mark_dualiter("users", make_users) +def test_attribute_map(env_async, users): + tmpl = env_async.from_string('{{ users()|map(attribute="name")|join("|") }}') + assert tmpl.render(users=users) == "john|jane|mike" + + +def test_empty_map(env_async): + tmpl = env_async.from_string('{{ none|map("upper")|list }}') + assert tmpl.render() == "[]" + + +@mark_dualiter("items", lambda: [1, 2, 3, 4, 5, 6]) +def test_sum(env_async, items): + tmpl = env_async.from_string("""{{ items()|sum }}""") + assert tmpl.render(items=items) == "21" + + +@mark_dualiter("items", lambda: [{"value": 23}, {"value": 1}, {"value": 18}]) +def test_sum_attributes(env_async, items): + tmpl = env_async.from_string("""{{ items()|sum('value') }}""") + assert tmpl.render(items=items) + + +def test_sum_attributes_nested(env_async): + tmpl = env_async.from_string("""{{ values|sum('real.value') }}""") + assert ( + tmpl.render( + values=[ + {"real": {"value": 23}}, + {"real": {"value": 1}}, + {"real": {"value": 18}}, + ] + ) + == "42" + ) + + +def test_sum_attributes_tuple(env_async): + tmpl = env_async.from_string("""{{ values.items()|sum('1') }}""") + assert tmpl.render(values={"foo": 23, "bar": 1, "baz": 18}) == "42" + + +@mark_dualiter("items", lambda: range(10)) +def test_slice(env_async, items): + tmpl = env_async.from_string( + "{{ items()|slice(3)|list }}|{{ items()|slice(3, 'X')|list }}" + ) + out = tmpl.render(items=items) + assert out == ( + "[[0, 1, 2, 3], [4, 5, 6], [7, 8, 9]]|" + "[[0, 1, 2, 3], [4, 5, 6, 'X'], [7, 8, 9, 'X']]" + ) diff --git a/tests/test_bytecode_cache.py b/tests/test_bytecode_cache.py new file mode 100644 index 00000000..5b9eb0ff --- /dev/null +++ b/tests/test_bytecode_cache.py @@ -0,0 +1,77 @@ +import pytest + +from jinja2 import Environment +from jinja2.bccache import Bucket +from jinja2.bccache import FileSystemBytecodeCache +from jinja2.bccache import MemcachedBytecodeCache +from jinja2.exceptions import TemplateNotFound + + +@pytest.fixture +def env(package_loader, tmp_path): + bytecode_cache = FileSystemBytecodeCache(str(tmp_path)) + return Environment(loader=package_loader, bytecode_cache=bytecode_cache) + + +class TestByteCodeCache: + def test_simple(self, env): + tmpl = env.get_template("test.html") + assert tmpl.render().strip() == "BAR" + pytest.raises(TemplateNotFound, env.get_template, "missing.html") + + +class MockMemcached: + class Error(Exception): + pass + + key = None + value = None + timeout = None + + def get(self, key): + return self.value + + def set(self, key, value, timeout=None): + self.key = key + self.value = value + self.timeout = timeout + + def get_side_effect(self, key): + raise self.Error() + + def set_side_effect(self, *args): + raise self.Error() + + +class TestMemcachedBytecodeCache: + def test_dump_load(self): + memcached = MockMemcached() + m = MemcachedBytecodeCache(memcached) + + b = Bucket(None, "key", "") + b.code = "code" + m.dump_bytecode(b) + assert memcached.key == "jinja2/bytecode/key" + + b = Bucket(None, "key", "") + m.load_bytecode(b) + assert b.code == "code" + + def test_exception(self): + memcached = MockMemcached() + memcached.get = memcached.get_side_effect + memcached.set = memcached.set_side_effect + m = MemcachedBytecodeCache(memcached) + b = Bucket(None, "key", "") + b.code = "code" + + m.dump_bytecode(b) + m.load_bytecode(b) + + m.ignore_memcache_errors = False + + with pytest.raises(MockMemcached.Error): + m.dump_bytecode(b) + + with pytest.raises(MockMemcached.Error): + m.load_bytecode(b) diff --git a/tests/test_core_tags.py b/tests/test_core_tags.py new file mode 100644 index 00000000..4bb95e02 --- /dev/null +++ b/tests/test_core_tags.py @@ -0,0 +1,595 @@ +import pytest + +from jinja2 import DictLoader +from jinja2 import Environment +from jinja2 import TemplateRuntimeError +from jinja2 import TemplateSyntaxError +from jinja2 import UndefinedError + + +@pytest.fixture +def env_trim(): + return Environment(trim_blocks=True) + + +class TestForLoop: + def test_simple(self, env): + tmpl = env.from_string("{% for item in seq %}{{ item }}{% endfor %}") + assert tmpl.render(seq=list(range(10))) == "0123456789" + + def test_else(self, env): + tmpl = env.from_string("{% for item in seq %}XXX{% else %}...{% endfor %}") + assert tmpl.render() == "..." + + def test_else_scoping_item(self, env): + tmpl = env.from_string("{% for item in [] %}{% else %}{{ item }}{% endfor %}") + assert tmpl.render(item=42) == "42" + + def test_empty_blocks(self, env): + tmpl = env.from_string("<{% for item in seq %}{% else %}{% endfor %}>") + assert tmpl.render() == "<>" + + def test_context_vars(self, env): + slist = [42, 24] + for seq in [slist, iter(slist), reversed(slist), (_ for _ in slist)]: + tmpl = env.from_string( + """{% for item in seq -%} + {{ loop.index }}|{{ loop.index0 }}|{{ loop.revindex }}|{{ + loop.revindex0 }}|{{ loop.first }}|{{ loop.last }}|{{ + loop.length }}###{% endfor %}""" + ) + one, two, _ = tmpl.render(seq=seq).split("###") + ( + one_index, + one_index0, + one_revindex, + one_revindex0, + one_first, + one_last, + one_length, + ) = one.split("|") + ( + two_index, + two_index0, + two_revindex, + two_revindex0, + two_first, + two_last, + two_length, + ) = two.split("|") + + assert int(one_index) == 1 and int(two_index) == 2 + assert int(one_index0) == 0 and int(two_index0) == 1 + assert int(one_revindex) == 2 and int(two_revindex) == 1 + assert int(one_revindex0) == 1 and int(two_revindex0) == 0 + assert one_first == "True" and two_first == "False" + assert one_last == "False" and two_last == "True" + assert one_length == two_length == "2" + + def test_cycling(self, env): + tmpl = env.from_string( + """{% for item in seq %}{{ + loop.cycle('<1>', '<2>') }}{% endfor %}{% + for item in seq %}{{ loop.cycle(*through) }}{% endfor %}""" + ) + output = tmpl.render(seq=list(range(4)), through=("<1>", "<2>")) + assert output == "<1><2>" * 4 + + def test_lookaround(self, env): + tmpl = env.from_string( + """{% for item in seq -%} + {{ loop.previtem|default('x') }}-{{ item }}-{{ + loop.nextitem|default('x') }}| + {%- endfor %}""" + ) + output = tmpl.render(seq=list(range(4))) + assert output == "x-0-1|0-1-2|1-2-3|2-3-x|" + + def test_changed(self, env): + tmpl = env.from_string( + """{% for item in seq -%} + {{ loop.changed(item) }}, + {%- endfor %}""" + ) + output = tmpl.render(seq=[None, None, 1, 2, 2, 3, 4, 4, 4]) + assert output == "True,False,True,True,False,True,True,False,False," + + def test_scope(self, env): + tmpl = env.from_string("{% for item in seq %}{% endfor %}{{ item }}") + output = tmpl.render(seq=list(range(10))) + assert not output + + def test_varlen(self, env): + tmpl = env.from_string("{% for item in iter %}{{ item }}{% endfor %}") + output = tmpl.render(iter=range(5)) + assert output == "01234" + + def test_noniter(self, env): + tmpl = env.from_string("{% for item in none %}...{% endfor %}") + pytest.raises(TypeError, tmpl.render) + + def test_recursive(self, env): + tmpl = env.from_string( + """{% for item in seq recursive -%} + [{{ item.a }}{% if item.b %}<{{ loop(item.b) }}>{% endif %}] + {%- endfor %}""" + ) + assert ( + tmpl.render( + seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a="a")]), + ] + ) + == "[1<[1][2]>][2<[1][2]>][3<[a]>]" + ) + + def test_recursive_lookaround(self, env): + tmpl = env.from_string( + """{% for item in seq recursive -%} + [{{ loop.previtem.a if loop.previtem is defined else 'x' }}.{{ + item.a }}.{{ loop.nextitem.a if loop.nextitem is defined else 'x' + }}{% if item.b %}<{{ loop(item.b) }}>{% endif %}] + {%- endfor %}""" + ) + assert ( + tmpl.render( + seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a="a")]), + ] + ) + == "[x.1.2<[x.1.2][1.2.x]>][1.2.3<[x.1.2][1.2.x]>][2.3.x<[x.a.x]>]" + ) + + def test_recursive_depth0(self, env): + tmpl = env.from_string( + """{% for item in seq recursive -%} + [{{ loop.depth0 }}:{{ item.a }}{% if item.b %}<{{ loop(item.b) }}>{% endif %}] + {%- endfor %}""" + ) + assert ( + tmpl.render( + seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a="a")]), + ] + ) + == "[0:1<[1:1][1:2]>][0:2<[1:1][1:2]>][0:3<[1:a]>]" + ) + + def test_recursive_depth(self, env): + tmpl = env.from_string( + """{% for item in seq recursive -%} + [{{ loop.depth }}:{{ item.a }}{% if item.b %}<{{ loop(item.b) }}>{% endif %}] + {%- endfor %}""" + ) + assert ( + tmpl.render( + seq=[ + dict(a=1, b=[dict(a=1), dict(a=2)]), + dict(a=2, b=[dict(a=1), dict(a=2)]), + dict(a=3, b=[dict(a="a")]), + ] + ) + == "[1:1<[2:1][2:2]>][1:2<[2:1][2:2]>][1:3<[2:a]>]" + ) + + def test_looploop(self, env): + tmpl = env.from_string( + """{% for row in table %} + {%- set rowloop = loop -%} + {% for cell in row -%} + [{{ rowloop.index }}|{{ loop.index }}] + {%- endfor %} + {%- endfor %}""" + ) + assert tmpl.render(table=["ab", "cd"]) == "[1|1][1|2][2|1][2|2]" + + def test_reversed_bug(self, env): + tmpl = env.from_string( + "{% for i in items %}{{ i }}" + "{% if not loop.last %}" + ",{% endif %}{% endfor %}" + ) + assert tmpl.render(items=reversed([3, 2, 1])) == "1,2,3" + + def test_loop_errors(self, env): + tmpl = env.from_string( + """{% for item in [1] if loop.index + == 0 %}...{% endfor %}""" + ) + pytest.raises(UndefinedError, tmpl.render) + tmpl = env.from_string( + """{% for item in [] %}...{% else + %}{{ loop }}{% endfor %}""" + ) + assert tmpl.render() == "" + + def test_loop_filter(self, env): + tmpl = env.from_string( + "{% for item in range(10) if item is even %}[{{ item }}]{% endfor %}" + ) + assert tmpl.render() == "[0][2][4][6][8]" + tmpl = env.from_string( + """ + {%- for item in range(10) if item is even %}[{{ + loop.index }}:{{ item }}]{% endfor %}""" + ) + assert tmpl.render() == "[1:0][2:2][3:4][4:6][5:8]" + + def test_loop_unassignable(self, env): + pytest.raises( + TemplateSyntaxError, env.from_string, "{% for loop in seq %}...{% endfor %}" + ) + + def test_scoped_special_var(self, env): + t = env.from_string( + "{% for s in seq %}[{{ loop.first }}{% for c in s %}" + "|{{ loop.first }}{% endfor %}]{% endfor %}" + ) + assert t.render(seq=("ab", "cd")) == "[True|True|False][False|True|False]" + + def test_scoped_loop_var(self, env): + t = env.from_string( + "{% for x in seq %}{{ loop.first }}" + "{% for y in seq %}{% endfor %}{% endfor %}" + ) + assert t.render(seq="ab") == "TrueFalse" + t = env.from_string( + "{% for x in seq %}{% for y in seq %}" + "{{ loop.first }}{% endfor %}{% endfor %}" + ) + assert t.render(seq="ab") == "TrueFalseTrueFalse" + + def test_recursive_empty_loop_iter(self, env): + t = env.from_string( + """ + {%- for item in foo recursive -%}{%- endfor -%} + """ + ) + assert t.render(dict(foo=[])) == "" + + def test_call_in_loop(self, env): + t = env.from_string( + """ + {%- macro do_something() -%} + [{{ caller() }}] + {%- endmacro %} + + {%- for i in [1, 2, 3] %} + {%- call do_something() -%} + {{ i }} + {%- endcall %} + {%- endfor -%} + """ + ) + assert t.render() == "[1][2][3]" + + def test_scoping_bug(self, env): + t = env.from_string( + """ + {%- for item in foo %}...{{ item }}...{% endfor %} + {%- macro item(a) %}...{{ a }}...{% endmacro %} + {{- item(2) -}} + """ + ) + assert t.render(foo=(1,)) == "...1......2..." + + def test_unpacking(self, env): + tmpl = env.from_string( + "{% for a, b, c in [[1, 2, 3]] %}{{ a }}|{{ b }}|{{ c }}{% endfor %}" + ) + assert tmpl.render() == "1|2|3" + + def test_intended_scoping_with_set(self, env): + tmpl = env.from_string( + "{% for item in seq %}{{ x }}{% set x = item %}{{ x }}{% endfor %}" + ) + assert tmpl.render(x=0, seq=[1, 2, 3]) == "010203" + + tmpl = env.from_string( + "{% set x = 9 %}{% for item in seq %}{{ x }}" + "{% set x = item %}{{ x }}{% endfor %}" + ) + assert tmpl.render(x=0, seq=[1, 2, 3]) == "919293" + + +class TestIfCondition: + def test_simple(self, env): + tmpl = env.from_string("""{% if true %}...{% endif %}""") + assert tmpl.render() == "..." + + def test_elif(self, env): + tmpl = env.from_string( + """{% if false %}XXX{% elif true + %}...{% else %}XXX{% endif %}""" + ) + assert tmpl.render() == "..." + + def test_elif_deep(self, env): + elifs = "\n".join(f"{{% elif a == {i} %}}{i}" for i in range(1, 1000)) + tmpl = env.from_string(f"{{% if a == 0 %}}0{elifs}{{% else %}}x{{% endif %}}") + for x in (0, 10, 999): + assert tmpl.render(a=x).strip() == str(x) + assert tmpl.render(a=1000).strip() == "x" + + def test_else(self, env): + tmpl = env.from_string("{% if false %}XXX{% else %}...{% endif %}") + assert tmpl.render() == "..." + + def test_empty(self, env): + tmpl = env.from_string("[{% if true %}{% else %}{% endif %}]") + assert tmpl.render() == "[]" + + def test_complete(self, env): + tmpl = env.from_string( + "{% if a %}A{% elif b %}B{% elif c == d %}C{% else %}D{% endif %}" + ) + assert tmpl.render(a=0, b=False, c=42, d=42.0) == "C" + + def test_no_scope(self, env): + tmpl = env.from_string("{% if a %}{% set foo = 1 %}{% endif %}{{ foo }}") + assert tmpl.render(a=True) == "1" + tmpl = env.from_string("{% if true %}{% set foo = 1 %}{% endif %}{{ foo }}") + assert tmpl.render() == "1" + + +class TestMacros: + def test_simple(self, env_trim): + tmpl = env_trim.from_string( + """\ +{% macro say_hello(name) %}Hello {{ name }}!{% endmacro %} +{{ say_hello('Peter') }}""" + ) + assert tmpl.render() == "Hello Peter!" + + def test_scoping(self, env_trim): + tmpl = env_trim.from_string( + """\ +{% macro level1(data1) %} +{% macro level2(data2) %}{{ data1 }}|{{ data2 }}{% endmacro %} +{{ level2('bar') }}{% endmacro %} +{{ level1('foo') }}""" + ) + assert tmpl.render() == "foo|bar" + + def test_arguments(self, env_trim): + tmpl = env_trim.from_string( + """\ +{% macro m(a, b, c='c', d='d') %}{{ a }}|{{ b }}|{{ c }}|{{ d }}{% endmacro %} +{{ m() }}|{{ m('a') }}|{{ m('a', 'b') }}|{{ m(1, 2, 3) }}""" + ) + assert tmpl.render() == "||c|d|a||c|d|a|b|c|d|1|2|3|d" + + def test_arguments_defaults_nonsense(self, env_trim): + pytest.raises( + TemplateSyntaxError, + env_trim.from_string, + """\ +{% macro m(a, b=1, c) %}a={{ a }}, b={{ b }}, c={{ c }}{% endmacro %}""", + ) + + def test_caller_defaults_nonsense(self, env_trim): + pytest.raises( + TemplateSyntaxError, + env_trim.from_string, + """\ +{% macro a() %}{{ caller() }}{% endmacro %} +{% call(x, y=1, z) a() %}{% endcall %}""", + ) + + def test_varargs(self, env_trim): + tmpl = env_trim.from_string( + """\ +{% macro test() %}{{ varargs|join('|') }}{% endmacro %}\ +{{ test(1, 2, 3) }}""" + ) + assert tmpl.render() == "1|2|3" + + def test_simple_call(self, env_trim): + tmpl = env_trim.from_string( + """\ +{% macro test() %}[[{{ caller() }}]]{% endmacro %}\ +{% call test() %}data{% endcall %}""" + ) + assert tmpl.render() == "[[data]]" + + def test_complex_call(self, env_trim): + tmpl = env_trim.from_string( + """\ +{% macro test() %}[[{{ caller('data') }}]]{% endmacro %}\ +{% call(data) test() %}{{ data }}{% endcall %}""" + ) + assert tmpl.render() == "[[data]]" + + def test_caller_undefined(self, env_trim): + tmpl = env_trim.from_string( + """\ +{% set caller = 42 %}\ +{% macro test() %}{{ caller is not defined }}{% endmacro %}\ +{{ test() }}""" + ) + assert tmpl.render() == "True" + + def test_include(self, env_trim): + env_trim = Environment( + loader=DictLoader( + {"include": "{% macro test(foo) %}[{{ foo }}]{% endmacro %}"} + ) + ) + tmpl = env_trim.from_string('{% from "include" import test %}{{ test("foo") }}') + assert tmpl.render() == "[foo]" + + def test_macro_api(self, env_trim): + tmpl = env_trim.from_string( + "{% macro foo(a, b) %}{% endmacro %}" + "{% macro bar() %}{{ varargs }}{{ kwargs }}{% endmacro %}" + "{% macro baz() %}{{ caller() }}{% endmacro %}" + ) + assert tmpl.module.foo.arguments == ("a", "b") + assert tmpl.module.foo.name == "foo" + assert not tmpl.module.foo.caller + assert not tmpl.module.foo.catch_kwargs + assert not tmpl.module.foo.catch_varargs + assert tmpl.module.bar.arguments == () + assert not tmpl.module.bar.caller + assert tmpl.module.bar.catch_kwargs + assert tmpl.module.bar.catch_varargs + assert tmpl.module.baz.caller + + def test_callself(self, env_trim): + tmpl = env_trim.from_string( + "{% macro foo(x) %}{{ x }}{% if x > 1 %}|" + "{{ foo(x - 1) }}{% endif %}{% endmacro %}" + "{{ foo(5) }}" + ) + assert tmpl.render() == "5|4|3|2|1" + + def test_macro_defaults_self_ref(self, env): + tmpl = env.from_string( + """ + {%- set x = 42 %} + {%- macro m(a, b=x, x=23) %}{{ a }}|{{ b }}|{{ x }}{% endmacro -%} + """ + ) + assert tmpl.module.m(1) == "1||23" + assert tmpl.module.m(1, 2) == "1|2|23" + assert tmpl.module.m(1, 2, 3) == "1|2|3" + assert tmpl.module.m(1, x=7) == "1|7|7" + + +class TestSet: + def test_normal(self, env_trim): + tmpl = env_trim.from_string("{% set foo = 1 %}{{ foo }}") + assert tmpl.render() == "1" + assert tmpl.module.foo == 1 + + def test_block(self, env_trim): + tmpl = env_trim.from_string("{% set foo %}42{% endset %}{{ foo }}") + assert tmpl.render() == "42" + assert tmpl.module.foo == "42" + + def test_block_escaping(self): + env = Environment(autoescape=True) + tmpl = env.from_string( + "{% set foo %}<em>{{ test }}</em>{% endset %}foo: {{ foo }}" + ) + assert tmpl.render(test="<unsafe>") == "foo: <em><unsafe></em>" + + def test_set_invalid(self, env_trim): + pytest.raises( + TemplateSyntaxError, env_trim.from_string, "{% set foo['bar'] = 1 %}" + ) + tmpl = env_trim.from_string("{% set foo.bar = 1 %}") + exc_info = pytest.raises(TemplateRuntimeError, tmpl.render, foo={}) + assert "non-namespace object" in exc_info.value.message + + def test_namespace_redefined(self, env_trim): + tmpl = env_trim.from_string("{% set ns = namespace() %}{% set ns.bar = 'hi' %}") + exc_info = pytest.raises(TemplateRuntimeError, tmpl.render, namespace=dict) + assert "non-namespace object" in exc_info.value.message + + def test_namespace(self, env_trim): + tmpl = env_trim.from_string( + "{% set ns = namespace() %}{% set ns.bar = '42' %}{{ ns.bar }}" + ) + assert tmpl.render() == "42" + + def test_namespace_block(self, env_trim): + tmpl = env_trim.from_string( + "{% set ns = namespace() %}{% set ns.bar %}42{% endset %}{{ ns.bar }}" + ) + assert tmpl.render() == "42" + + def test_init_namespace(self, env_trim): + tmpl = env_trim.from_string( + "{% set ns = namespace(d, self=37) %}" + "{% set ns.b = 42 %}" + "{{ ns.a }}|{{ ns.self }}|{{ ns.b }}" + ) + assert tmpl.render(d={"a": 13}) == "13|37|42" + + def test_namespace_loop(self, env_trim): + tmpl = env_trim.from_string( + "{% set ns = namespace(found=false) %}" + "{% for x in range(4) %}" + "{% if x == v %}" + "{% set ns.found = true %}" + "{% endif %}" + "{% endfor %}" + "{{ ns.found }}" + ) + assert tmpl.render(v=3) == "True" + assert tmpl.render(v=4) == "False" + + def test_namespace_macro(self, env_trim): + tmpl = env_trim.from_string( + "{% set ns = namespace() %}" + "{% set ns.a = 13 %}" + "{% macro magic(x) %}" + "{% set x.b = 37 %}" + "{% endmacro %}" + "{{ magic(ns) }}" + "{{ ns.a }}|{{ ns.b }}" + ) + assert tmpl.render() == "13|37" + + def test_block_escaping_filtered(self): + env = Environment(autoescape=True) + tmpl = env.from_string( + "{% set foo | trim %}<em>{{ test }}</em> {% endset %}foo: {{ foo }}" + ) + assert tmpl.render(test="<unsafe>") == "foo: <em><unsafe></em>" + + def test_block_filtered(self, env_trim): + tmpl = env_trim.from_string( + "{% set foo | trim | length | string %} 42 {% endset %}{{ foo }}" + ) + assert tmpl.render() == "2" + assert tmpl.module.foo == "2" + + def test_block_filtered_set(self, env_trim): + def _myfilter(val, arg): + assert arg == " xxx " + return val + + env_trim.filters["myfilter"] = _myfilter + tmpl = env_trim.from_string( + '{% set a = " xxx " %}' + "{% set foo | myfilter(a) | trim | length | string %}" + ' {% set b = " yy " %} 42 {{ a }}{{ b }} ' + "{% endset %}" + "{{ foo }}" + ) + assert tmpl.render() == "11" + assert tmpl.module.foo == "11" + + +class TestWith: + def test_with(self, env): + tmpl = env.from_string( + """\ + {% with a=42, b=23 -%} + {{ a }} = {{ b }} + {% endwith -%} + {{ a }} = {{ b }}\ + """ + ) + assert [x.strip() for x in tmpl.render(a=1, b=2).splitlines()] == [ + "42 = 23", + "1 = 2", + ] + + def test_with_argument_scoping(self, env): + tmpl = env.from_string( + """\ + {%- with a=1, b=2, c=b, d=e, e=5 -%} + {{ a }}|{{ b }}|{{ c }}|{{ d }}|{{ e }} + {%- endwith -%} + """ + ) + assert tmpl.render(b=3, e=4) == "1|2|3|4|5" diff --git a/tests/test_debug.py b/tests/test_debug.py new file mode 100644 index 00000000..0aec78ae --- /dev/null +++ b/tests/test_debug.py @@ -0,0 +1,114 @@ +import pickle +import re +from traceback import format_exception + +import pytest + +from jinja2 import ChoiceLoader +from jinja2 import DictLoader +from jinja2 import Environment +from jinja2 import TemplateSyntaxError + + +@pytest.fixture +def fs_env(filesystem_loader): + """returns a new environment.""" + return Environment(loader=filesystem_loader) + + +class TestDebug: + def assert_traceback_matches(self, callback, expected_tb): + with pytest.raises(Exception) as exc_info: + callback() + + tb = format_exception(exc_info.type, exc_info.value, exc_info.tb) + m = re.search(expected_tb.strip(), "".join(tb)) + assert ( + m is not None + ), "Traceback did not match:\n\n{''.join(tb)}\nexpected:\n{expected_tb}" + + def test_runtime_error(self, fs_env): + def test(): + tmpl.render(fail=lambda: 1 / 0) + + tmpl = fs_env.get_template("broken.html") + self.assert_traceback_matches( + test, + r""" + File ".*?broken.html", line 2, in (top-level template code|<module>) + \{\{ fail\(\) \}\} + File ".*debug?.pyc?", line \d+, in <lambda> + tmpl\.render\(fail=lambda: 1 / 0\) +ZeroDivisionError: (int(eger)? )?division (or modulo )?by zero +""", + ) + + def test_syntax_error(self, fs_env): + # The trailing .*? is for PyPy 2 and 3, which don't seem to + # clear the exception's original traceback, leaving the syntax + # error in the middle of other compiler frames. + self.assert_traceback_matches( + lambda: fs_env.get_template("syntaxerror.html"), + """(?sm) + File ".*?syntaxerror.html", line 4, in (template|<module>) + \\{% endif %\\}.*? +(jinja2\\.exceptions\\.)?TemplateSyntaxError: Encountered unknown tag 'endif'. Jinja \ +was looking for the following tags: 'endfor' or 'else'. The innermost block that needs \ +to be closed is 'for'. + """, + ) + + def test_regular_syntax_error(self, fs_env): + def test(): + raise TemplateSyntaxError("wtf", 42) + + self.assert_traceback_matches( + test, + r""" + File ".*debug.pyc?", line \d+, in test + raise TemplateSyntaxError\("wtf", 42\) +(jinja2\.exceptions\.)?TemplateSyntaxError: wtf + line 42""", + ) + + def test_pickleable_syntax_error(self, fs_env): + original = TemplateSyntaxError("bad template", 42, "test", "test.txt") + unpickled = pickle.loads(pickle.dumps(original)) + assert str(original) == str(unpickled) + assert original.name == unpickled.name + + def test_include_syntax_error_source(self, filesystem_loader): + e = Environment( + loader=ChoiceLoader( + [ + filesystem_loader, + DictLoader({"inc": "a\n{% include 'syntaxerror.html' %}\nb"}), + ] + ) + ) + t = e.get_template("inc") + + with pytest.raises(TemplateSyntaxError) as exc_info: + t.render() + + assert exc_info.value.source is not None + + def test_local_extraction(self): + from jinja2.debug import get_template_locals + from jinja2.runtime import missing + + locals = get_template_locals( + { + "l_0_foo": 42, + "l_1_foo": 23, + "l_2_foo": 13, + "l_0_bar": 99, + "l_1_bar": missing, + "l_0_baz": missing, + } + ) + assert locals == {"foo": 13, "bar": 99} + + def test_get_corresponding_lineno_traceback(self, fs_env): + tmpl = fs_env.get_template("test.html") + assert tmpl.get_corresponding_lineno(1) == 1 diff --git a/tests/test_ext.py b/tests/test_ext.py new file mode 100644 index 00000000..94d20bee --- /dev/null +++ b/tests/test_ext.py @@ -0,0 +1,640 @@ +import re +from io import BytesIO + +import pytest + +from jinja2 import contextfunction +from jinja2 import DictLoader +from jinja2 import Environment +from jinja2 import nodes +from jinja2.exceptions import TemplateAssertionError +from jinja2.ext import Extension +from jinja2.lexer import count_newlines +from jinja2.lexer import Token + +importable_object = 23 + +_gettext_re = re.compile(r"_\((.*?)\)", re.DOTALL) + + +i18n_templates = { + "master.html": '<title>{{ page_title|default(_("missing")) }}</title>' + "{% block body %}{% endblock %}", + "child.html": '{% extends "master.html" %}{% block body %}' + "{% trans %}watch out{% endtrans %}{% endblock %}", + "plural.html": "{% trans user_count %}One user online{% pluralize %}" + "{{ user_count }} users online{% endtrans %}", + "plural2.html": "{% trans user_count=get_user_count() %}{{ user_count }}s" + "{% pluralize %}{{ user_count }}p{% endtrans %}", + "stringformat.html": '{{ _("User: %(num)s")|format(num=user_count) }}', +} + +newstyle_i18n_templates = { + "master.html": '<title>{{ page_title|default(_("missing")) }}</title>' + "{% block body %}{% endblock %}", + "child.html": '{% extends "master.html" %}{% block body %}' + "{% trans %}watch out{% endtrans %}{% endblock %}", + "plural.html": "{% trans user_count %}One user online{% pluralize %}" + "{{ user_count }} users online{% endtrans %}", + "stringformat.html": '{{ _("User: %(num)s", num=user_count) }}', + "ngettext.html": '{{ ngettext("%(num)s apple", "%(num)s apples", apples) }}', + "ngettext_long.html": "{% trans num=apples %}{{ num }} apple{% pluralize %}" + "{{ num }} apples{% endtrans %}", + "transvars1.html": "{% trans %}User: {{ num }}{% endtrans %}", + "transvars2.html": "{% trans num=count %}User: {{ num }}{% endtrans %}", + "transvars3.html": "{% trans count=num %}User: {{ count }}{% endtrans %}", + "novars.html": "{% trans %}%(hello)s{% endtrans %}", + "vars.html": "{% trans %}{{ foo }}%(foo)s{% endtrans %}", + "explicitvars.html": '{% trans foo="42" %}%(foo)s{% endtrans %}', +} + + +languages = { + "de": { + "missing": "fehlend", + "watch out": "pass auf", + "One user online": "Ein Benutzer online", + "%(user_count)s users online": "%(user_count)s Benutzer online", + "User: %(num)s": "Benutzer: %(num)s", + "User: %(count)s": "Benutzer: %(count)s", + "%(num)s apple": "%(num)s Apfel", + "%(num)s apples": "%(num)s Äpfel", + } +} + + +@contextfunction +def gettext(context, string): + language = context.get("LANGUAGE", "en") + return languages.get(language, {}).get(string, string) + + +@contextfunction +def ngettext(context, s, p, n): + language = context.get("LANGUAGE", "en") + if n != 1: + return languages.get(language, {}).get(p, p) + return languages.get(language, {}).get(s, s) + + +i18n_env = Environment( + loader=DictLoader(i18n_templates), extensions=["jinja2.ext.i18n"] +) +i18n_env.globals.update({"_": gettext, "gettext": gettext, "ngettext": ngettext}) +i18n_env_trimmed = Environment(extensions=["jinja2.ext.i18n"]) +i18n_env_trimmed.policies["ext.i18n.trimmed"] = True +i18n_env_trimmed.globals.update( + {"_": gettext, "gettext": gettext, "ngettext": ngettext} +) + +newstyle_i18n_env = Environment( + loader=DictLoader(newstyle_i18n_templates), extensions=["jinja2.ext.i18n"] +) +newstyle_i18n_env.install_gettext_callables(gettext, ngettext, newstyle=True) + + +class ExampleExtension(Extension): + tags = {"test"} + ext_attr = 42 + context_reference_node_cls = nodes.ContextReference + + def parse(self, parser): + return nodes.Output( + [ + self.call_method( + "_dump", + [ + nodes.EnvironmentAttribute("sandboxed"), + self.attr("ext_attr"), + nodes.ImportedName(__name__ + ".importable_object"), + self.context_reference_node_cls(), + ], + ) + ] + ).set_lineno(next(parser.stream).lineno) + + def _dump(self, sandboxed, ext_attr, imported_object, context): + return ( + f"{sandboxed}|{ext_attr}|{imported_object}|{context.blocks}" + f"|{context.get('test_var')}" + ) + + +class DerivedExampleExtension(ExampleExtension): + context_reference_node_cls = nodes.DerivedContextReference + + +class PreprocessorExtension(Extension): + def preprocess(self, source, name, filename=None): + return source.replace("[[TEST]]", "({{ foo }})") + + +class StreamFilterExtension(Extension): + def filter_stream(self, stream): + for token in stream: + if token.type == "data": + yield from self.interpolate(token) + else: + yield token + + def interpolate(self, token): + pos = 0 + end = len(token.value) + lineno = token.lineno + while 1: + match = _gettext_re.search(token.value, pos) + if match is None: + break + value = token.value[pos : match.start()] + if value: + yield Token(lineno, "data", value) + lineno += count_newlines(token.value) + yield Token(lineno, "variable_begin", None) + yield Token(lineno, "name", "gettext") + yield Token(lineno, "lparen", None) + yield Token(lineno, "string", match.group(1)) + yield Token(lineno, "rparen", None) + yield Token(lineno, "variable_end", None) + pos = match.end() + if pos < end: + yield Token(lineno, "data", token.value[pos:]) + + +class TestExtensions: + def test_extend_late(self): + env = Environment() + env.add_extension("jinja2.ext.autoescape") + t = env.from_string('{% autoescape true %}{{ "<test>" }}{% endautoescape %}') + assert t.render() == "<test>" + + def test_loop_controls(self): + env = Environment(extensions=["jinja2.ext.loopcontrols"]) + + tmpl = env.from_string( + """ + {%- for item in [1, 2, 3, 4] %} + {%- if item % 2 == 0 %}{% continue %}{% endif -%} + {{ item }} + {%- endfor %}""" + ) + assert tmpl.render() == "13" + + tmpl = env.from_string( + """ + {%- for item in [1, 2, 3, 4] %} + {%- if item > 2 %}{% break %}{% endif -%} + {{ item }} + {%- endfor %}""" + ) + assert tmpl.render() == "12" + + def test_do(self): + env = Environment(extensions=["jinja2.ext.do"]) + tmpl = env.from_string( + """ + {%- set items = [] %} + {%- for char in "foo" %} + {%- do items.append(loop.index0 ~ char) %} + {%- endfor %}{{ items|join(', ') }}""" + ) + assert tmpl.render() == "0f, 1o, 2o" + + def test_extension_nodes(self): + env = Environment(extensions=[ExampleExtension]) + tmpl = env.from_string("{% test %}") + assert tmpl.render() == "False|42|23|{}|None" + + def test_contextreference_node_passes_context(self): + env = Environment(extensions=[ExampleExtension]) + tmpl = env.from_string('{% set test_var="test_content" %}{% test %}') + assert tmpl.render() == "False|42|23|{}|test_content" + + def test_contextreference_node_can_pass_locals(self): + env = Environment(extensions=[DerivedExampleExtension]) + tmpl = env.from_string( + '{% for test_var in ["test_content"] %}{% test %}{% endfor %}' + ) + assert tmpl.render() == "False|42|23|{}|test_content" + + def test_identifier(self): + assert ExampleExtension.identifier == __name__ + ".ExampleExtension" + + def test_rebinding(self): + original = Environment(extensions=[ExampleExtension]) + overlay = original.overlay() + for env in original, overlay: + for ext in env.extensions.values(): + assert ext.environment is env + + def test_preprocessor_extension(self): + env = Environment(extensions=[PreprocessorExtension]) + tmpl = env.from_string("{[[TEST]]}") + assert tmpl.render(foo=42) == "{(42)}" + + def test_streamfilter_extension(self): + env = Environment(extensions=[StreamFilterExtension]) + env.globals["gettext"] = lambda x: x.upper() + tmpl = env.from_string("Foo _(bar) Baz") + out = tmpl.render() + assert out == "Foo BAR Baz" + + def test_extension_ordering(self): + class T1(Extension): + priority = 1 + + class T2(Extension): + priority = 2 + + env = Environment(extensions=[T1, T2]) + ext = list(env.iter_extensions()) + assert ext[0].__class__ is T1 + assert ext[1].__class__ is T2 + + def test_debug(self): + env = Environment(extensions=["jinja2.ext.debug"]) + t = env.from_string("Hello\n{% debug %}\nGoodbye") + out = t.render() + + for value in ("context", "cycler", "filters", "abs", "tests", "!="): + assert f"'{value}'" in out + + +class TestInternationalization: + def test_trans(self): + tmpl = i18n_env.get_template("child.html") + assert tmpl.render(LANGUAGE="de") == "<title>fehlend</title>pass auf" + + def test_trans_plural(self): + tmpl = i18n_env.get_template("plural.html") + assert tmpl.render(LANGUAGE="de", user_count=1) == "Ein Benutzer online" + assert tmpl.render(LANGUAGE="de", user_count=2) == "2 Benutzer online" + + def test_trans_plural_with_functions(self): + tmpl = i18n_env.get_template("plural2.html") + + def get_user_count(): + get_user_count.called += 1 + return 1 + + get_user_count.called = 0 + assert tmpl.render(LANGUAGE="de", get_user_count=get_user_count) == "1s" + assert get_user_count.called == 1 + + def test_complex_plural(self): + tmpl = i18n_env.from_string( + "{% trans foo=42, count=2 %}{{ count }} item{% " + "pluralize count %}{{ count }} items{% endtrans %}" + ) + assert tmpl.render() == "2 items" + pytest.raises( + TemplateAssertionError, + i18n_env.from_string, + "{% trans foo %}...{% pluralize bar %}...{% endtrans %}", + ) + + def test_trans_stringformatting(self): + tmpl = i18n_env.get_template("stringformat.html") + assert tmpl.render(LANGUAGE="de", user_count=5) == "Benutzer: 5" + + def test_trimmed(self): + tmpl = i18n_env.from_string( + "{%- trans trimmed %} hello\n world {% endtrans -%}" + ) + assert tmpl.render() == "hello world" + + def test_trimmed_policy(self): + s = "{%- trans %} hello\n world {% endtrans -%}" + tmpl = i18n_env.from_string(s) + trimmed_tmpl = i18n_env_trimmed.from_string(s) + assert tmpl.render() == " hello\n world " + assert trimmed_tmpl.render() == "hello world" + + def test_trimmed_policy_override(self): + tmpl = i18n_env_trimmed.from_string( + "{%- trans notrimmed %} hello\n world {% endtrans -%}" + ) + assert tmpl.render() == " hello\n world " + + def test_trimmed_vars(self): + tmpl = i18n_env.from_string( + '{%- trans trimmed x="world" %} hello\n {{ x }} {% endtrans -%}' + ) + assert tmpl.render() == "hello world" + + def test_trimmed_varname_trimmed(self): + # unlikely variable name, but when used as a variable + # it should not enable trimming + tmpl = i18n_env.from_string( + "{%- trans trimmed = 'world' %} hello\n {{ trimmed }} {% endtrans -%}" + ) + assert tmpl.render() == " hello\n world " + + def test_extract(self): + from jinja2.ext import babel_extract + + source = BytesIO( + b""" + {{ gettext('Hello World') }} + {% trans %}Hello World{% endtrans %} + {% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %} + """ + ) + assert list(babel_extract(source, ("gettext", "ngettext", "_"), [], {})) == [ + (2, "gettext", "Hello World", []), + (3, "gettext", "Hello World", []), + (4, "ngettext", ("%(users)s user", "%(users)s users", None), []), + ] + + def test_extract_trimmed(self): + from jinja2.ext import babel_extract + + source = BytesIO( + b""" + {{ gettext(' Hello \n World') }} + {% trans trimmed %} Hello \n World{% endtrans %} + {% trans trimmed %}{{ users }} \n user + {%- pluralize %}{{ users }} \n users{% endtrans %} + """ + ) + assert list(babel_extract(source, ("gettext", "ngettext", "_"), [], {})) == [ + (2, "gettext", " Hello \n World", []), + (4, "gettext", "Hello World", []), + (6, "ngettext", ("%(users)s user", "%(users)s users", None), []), + ] + + def test_extract_trimmed_option(self): + from jinja2.ext import babel_extract + + source = BytesIO( + b""" + {{ gettext(' Hello \n World') }} + {% trans %} Hello \n World{% endtrans %} + {% trans %}{{ users }} \n user + {%- pluralize %}{{ users }} \n users{% endtrans %} + """ + ) + opts = {"trimmed": "true"} + assert list(babel_extract(source, ("gettext", "ngettext", "_"), [], opts)) == [ + (2, "gettext", " Hello \n World", []), + (4, "gettext", "Hello World", []), + (6, "ngettext", ("%(users)s user", "%(users)s users", None), []), + ] + + def test_comment_extract(self): + from jinja2.ext import babel_extract + + source = BytesIO( + b""" + {# trans first #} + {{ gettext('Hello World') }} + {% trans %}Hello World{% endtrans %}{# trans second #} + {#: third #} + {% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %} + """ + ) + assert list( + babel_extract(source, ("gettext", "ngettext", "_"), ["trans", ":"], {}) + ) == [ + (3, "gettext", "Hello World", ["first"]), + (4, "gettext", "Hello World", ["second"]), + (6, "ngettext", ("%(users)s user", "%(users)s users", None), ["third"]), + ] + + +class TestScope: + def test_basic_scope_behavior(self): + # This is what the old with statement compiled down to + class ScopeExt(Extension): + tags = {"scope"} + + def parse(self, parser): + node = nodes.Scope(lineno=next(parser.stream).lineno) + assignments = [] + while parser.stream.current.type != "block_end": + lineno = parser.stream.current.lineno + if assignments: + parser.stream.expect("comma") + target = parser.parse_assign_target() + parser.stream.expect("assign") + expr = parser.parse_expression() + assignments.append(nodes.Assign(target, expr, lineno=lineno)) + node.body = assignments + list( + parser.parse_statements(("name:endscope",), drop_needle=True) + ) + return node + + env = Environment(extensions=[ScopeExt]) + tmpl = env.from_string( + """\ + {%- scope a=1, b=2, c=b, d=e, e=5 -%} + {{ a }}|{{ b }}|{{ c }}|{{ d }}|{{ e }} + {%- endscope -%} + """ + ) + assert tmpl.render(b=3, e=4) == "1|2|2|4|5" + + +class TestNewstyleInternationalization: + def test_trans(self): + tmpl = newstyle_i18n_env.get_template("child.html") + assert tmpl.render(LANGUAGE="de") == "<title>fehlend</title>pass auf" + + def test_trans_plural(self): + tmpl = newstyle_i18n_env.get_template("plural.html") + assert tmpl.render(LANGUAGE="de", user_count=1) == "Ein Benutzer online" + assert tmpl.render(LANGUAGE="de", user_count=2) == "2 Benutzer online" + + def test_complex_plural(self): + tmpl = newstyle_i18n_env.from_string( + "{% trans foo=42, count=2 %}{{ count }} item{% " + "pluralize count %}{{ count }} items{% endtrans %}" + ) + assert tmpl.render() == "2 items" + pytest.raises( + TemplateAssertionError, + i18n_env.from_string, + "{% trans foo %}...{% pluralize bar %}...{% endtrans %}", + ) + + def test_trans_stringformatting(self): + tmpl = newstyle_i18n_env.get_template("stringformat.html") + assert tmpl.render(LANGUAGE="de", user_count=5) == "Benutzer: 5" + + def test_newstyle_plural(self): + tmpl = newstyle_i18n_env.get_template("ngettext.html") + assert tmpl.render(LANGUAGE="de", apples=1) == "1 Apfel" + assert tmpl.render(LANGUAGE="de", apples=5) == "5 Äpfel" + + def test_autoescape_support(self): + env = Environment(extensions=["jinja2.ext.autoescape", "jinja2.ext.i18n"]) + env.install_gettext_callables( + lambda x: "<strong>Wert: %(name)s</strong>", + lambda s, p, n: s, + newstyle=True, + ) + t = env.from_string( + '{% autoescape ae %}{{ gettext("foo", name=' + '"<test>") }}{% endautoescape %}' + ) + assert t.render(ae=True) == "<strong>Wert: <test></strong>" + assert t.render(ae=False) == "<strong>Wert: <test></strong>" + + def test_autoescape_macros(self): + env = Environment(autoescape=False, extensions=["jinja2.ext.autoescape"]) + template = ( + "{% macro m() %}<html>{% endmacro %}" + "{% autoescape true %}{{ m() }}{% endautoescape %}" + ) + assert env.from_string(template).render() == "<html>" + + def test_num_used_twice(self): + tmpl = newstyle_i18n_env.get_template("ngettext_long.html") + assert tmpl.render(apples=5, LANGUAGE="de") == "5 Äpfel" + + def test_num_called_num(self): + source = newstyle_i18n_env.compile( + """ + {% trans num=3 %}{{ num }} apple{% pluralize + %}{{ num }} apples{% endtrans %} + """, + raw=True, + ) + # quite hacky, but the only way to properly test that. The idea is + # that the generated code does not pass num twice (although that + # would work) for better performance. This only works on the + # newstyle gettext of course + assert ( + re.search(r"u?'%\(num\)s apple', u?'%\(num\)s apples', 3", source) + is not None + ) + + def test_trans_vars(self): + t1 = newstyle_i18n_env.get_template("transvars1.html") + t2 = newstyle_i18n_env.get_template("transvars2.html") + t3 = newstyle_i18n_env.get_template("transvars3.html") + assert t1.render(num=1, LANGUAGE="de") == "Benutzer: 1" + assert t2.render(count=23, LANGUAGE="de") == "Benutzer: 23" + assert t3.render(num=42, LANGUAGE="de") == "Benutzer: 42" + + def test_novars_vars_escaping(self): + t = newstyle_i18n_env.get_template("novars.html") + assert t.render() == "%(hello)s" + t = newstyle_i18n_env.get_template("vars.html") + assert t.render(foo="42") == "42%(foo)s" + t = newstyle_i18n_env.get_template("explicitvars.html") + assert t.render() == "%(foo)s" + + +class TestAutoEscape: + def test_scoped_setting(self): + env = Environment(extensions=["jinja2.ext.autoescape"], autoescape=True) + tmpl = env.from_string( + """ + {{ "<HelloWorld>" }} + {% autoescape false %} + {{ "<HelloWorld>" }} + {% endautoescape %} + {{ "<HelloWorld>" }} + """ + ) + assert tmpl.render().split() == [ + "<HelloWorld>", + "<HelloWorld>", + "<HelloWorld>", + ] + + env = Environment(extensions=["jinja2.ext.autoescape"], autoescape=False) + tmpl = env.from_string( + """ + {{ "<HelloWorld>" }} + {% autoescape true %} + {{ "<HelloWorld>" }} + {% endautoescape %} + {{ "<HelloWorld>" }} + """ + ) + assert tmpl.render().split() == [ + "<HelloWorld>", + "<HelloWorld>", + "<HelloWorld>", + ] + + def test_nonvolatile(self): + env = Environment(extensions=["jinja2.ext.autoescape"], autoescape=True) + tmpl = env.from_string('{{ {"foo": "<test>"}|xmlattr|escape }}') + assert tmpl.render() == ' foo="<test>"' + tmpl = env.from_string( + '{% autoescape false %}{{ {"foo": "<test>"}' + "|xmlattr|escape }}{% endautoescape %}" + ) + assert tmpl.render() == " foo="&lt;test&gt;"" + + def test_volatile(self): + env = Environment(extensions=["jinja2.ext.autoescape"], autoescape=True) + tmpl = env.from_string( + '{% autoescape foo %}{{ {"foo": "<test>"}' + "|xmlattr|escape }}{% endautoescape %}" + ) + assert tmpl.render(foo=False) == " foo="&lt;test&gt;"" + assert tmpl.render(foo=True) == ' foo="<test>"' + + def test_scoping(self): + env = Environment(extensions=["jinja2.ext.autoescape"]) + tmpl = env.from_string( + '{% autoescape true %}{% set x = "<x>" %}{{ x }}' + '{% endautoescape %}{{ x }}{{ "<y>" }}' + ) + assert tmpl.render(x=1) == "<x>1<y>" + + def test_volatile_scoping(self): + env = Environment(extensions=["jinja2.ext.autoescape"]) + tmplsource = """ + {% autoescape val %} + {% macro foo(x) %} + [{{ x }}] + {% endmacro %} + {{ foo().__class__.__name__ }} + {% endautoescape %} + {{ '<testing>' }} + """ + tmpl = env.from_string(tmplsource) + assert tmpl.render(val=True).split()[0] == "Markup" + assert tmpl.render(val=False).split()[0] == "str" + + # looking at the source we should see <testing> there in raw + # (and then escaped as well) + env = Environment(extensions=["jinja2.ext.autoescape"]) + pysource = env.compile(tmplsource, raw=True) + assert "<testing>\\n" in pysource + + env = Environment(extensions=["jinja2.ext.autoescape"], autoescape=True) + pysource = env.compile(tmplsource, raw=True) + assert "<testing>\\n" in pysource + + def test_overlay_scopes(self): + class MagicScopeExtension(Extension): + tags = {"overlay"} + + def parse(self, parser): + node = nodes.OverlayScope(lineno=next(parser.stream).lineno) + node.body = list( + parser.parse_statements(("name:endoverlay",), drop_needle=True) + ) + node.context = self.call_method("get_scope") + return node + + def get_scope(self): + return {"x": [1, 2, 3]} + + env = Environment(extensions=[MagicScopeExtension]) + + tmpl = env.from_string( + """ + {{- x }}|{% set z = 99 %} + {%- overlay %} + {{- y }}|{{ z }}|{% for item in x %}[{{ item }}]{% endfor %} + {%- endoverlay %}| + {{- x -}} + """ + ) + assert tmpl.render(x=42, y=23) == "42|23|99|[1][2][3]|42" diff --git a/tests/test_features.py b/tests/test_features.py new file mode 100644 index 00000000..4f36458a --- /dev/null +++ b/tests/test_features.py @@ -0,0 +1,14 @@ +import pytest + +from jinja2 import Template + + +# Python < 3.7 +def test_generator_stop(): + class X: + def __getattr__(self, name): + raise StopIteration() + + t = Template("a{{ bad.bar() }}b") + with pytest.raises(RuntimeError): + t.render(bad=X()) diff --git a/tests/test_filters.py b/tests/test_filters.py new file mode 100644 index 00000000..8087a248 --- /dev/null +++ b/tests/test_filters.py @@ -0,0 +1,745 @@ +import random +from collections import namedtuple + +import pytest + +from jinja2 import Environment +from jinja2 import Markup +from jinja2 import StrictUndefined +from jinja2 import UndefinedError + + +class Magic: + def __init__(self, value): + self.value = value + + def __str__(self): + return str(self.value) + + +class Magic2: + def __init__(self, value1, value2): + self.value1 = value1 + self.value2 = value2 + + def __str__(self): + return f"({self.value1},{self.value2})" + + +class TestFilter: + def test_filter_calling(self, env): + rv = env.call_filter("sum", [1, 2, 3]) + assert rv == 6 + + def test_capitalize(self, env): + tmpl = env.from_string('{{ "foo bar"|capitalize }}') + assert tmpl.render() == "Foo bar" + + def test_center(self, env): + tmpl = env.from_string('{{ "foo"|center(9) }}') + assert tmpl.render() == " foo " + + def test_default(self, env): + tmpl = env.from_string( + "{{ missing|default('no') }}|{{ false|default('no') }}|" + "{{ false|default('no', true) }}|{{ given|default('no') }}" + ) + assert tmpl.render(given="yes") == "no|False|no|yes" + + @pytest.mark.parametrize( + "args,expect", + ( + ("", "[('aa', 0), ('AB', 3), ('b', 1), ('c', 2)]"), + ("true", "[('AB', 3), ('aa', 0), ('b', 1), ('c', 2)]"), + ('by="value"', "[('aa', 0), ('b', 1), ('c', 2), ('AB', 3)]"), + ("reverse=true", "[('c', 2), ('b', 1), ('AB', 3), ('aa', 0)]"), + ), + ) + def test_dictsort(self, env, args, expect): + t = env.from_string(f"{{{{ foo|dictsort({args}) }}}}") + out = t.render(foo={"aa": 0, "b": 1, "c": 2, "AB": 3}) + assert out == expect + + def test_batch(self, env): + tmpl = env.from_string("{{ foo|batch(3)|list }}|{{ foo|batch(3, 'X')|list }}") + out = tmpl.render(foo=list(range(10))) + assert out == ( + "[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]|" + "[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 'X', 'X']]" + ) + + def test_slice(self, env): + tmpl = env.from_string("{{ foo|slice(3)|list }}|{{ foo|slice(3, 'X')|list }}") + out = tmpl.render(foo=list(range(10))) + assert out == ( + "[[0, 1, 2, 3], [4, 5, 6], [7, 8, 9]]|" + "[[0, 1, 2, 3], [4, 5, 6, 'X'], [7, 8, 9, 'X']]" + ) + + def test_escape(self, env): + tmpl = env.from_string("""{{ '<">&'|escape }}""") + out = tmpl.render() + assert out == "<">&" + + @pytest.mark.parametrize( + ("chars", "expect"), [(None, "..stays.."), (".", " ..stays"), (" .", "stays")] + ) + def test_trim(self, env, chars, expect): + tmpl = env.from_string("{{ foo|trim(chars) }}") + out = tmpl.render(foo=" ..stays..", chars=chars) + assert out == expect + + def test_striptags(self, env): + tmpl = env.from_string("""{{ foo|striptags }}""") + out = tmpl.render( + foo=' <p>just a small \n <a href="#">' + "example</a> link</p>\n<p>to a webpage</p> " + "<!-- <p>and some commented stuff</p> -->" + ) + assert out == "just a small example link to a webpage" + + def test_filesizeformat(self, env): + tmpl = env.from_string( + "{{ 100|filesizeformat }}|" + "{{ 1000|filesizeformat }}|" + "{{ 1000000|filesizeformat }}|" + "{{ 1000000000|filesizeformat }}|" + "{{ 1000000000000|filesizeformat }}|" + "{{ 100|filesizeformat(true) }}|" + "{{ 1000|filesizeformat(true) }}|" + "{{ 1000000|filesizeformat(true) }}|" + "{{ 1000000000|filesizeformat(true) }}|" + "{{ 1000000000000|filesizeformat(true) }}" + ) + out = tmpl.render() + assert out == ( + "100 Bytes|1.0 kB|1.0 MB|1.0 GB|1.0 TB|100 Bytes|" + "1000 Bytes|976.6 KiB|953.7 MiB|931.3 GiB" + ) + + def test_filesizeformat_issue59(self, env): + tmpl = env.from_string( + "{{ 300|filesizeformat }}|" + "{{ 3000|filesizeformat }}|" + "{{ 3000000|filesizeformat }}|" + "{{ 3000000000|filesizeformat }}|" + "{{ 3000000000000|filesizeformat }}|" + "{{ 300|filesizeformat(true) }}|" + "{{ 3000|filesizeformat(true) }}|" + "{{ 3000000|filesizeformat(true) }}" + ) + out = tmpl.render() + assert out == ( + "300 Bytes|3.0 kB|3.0 MB|3.0 GB|3.0 TB|300 Bytes|2.9 KiB|2.9 MiB" + ) + + def test_first(self, env): + tmpl = env.from_string("{{ foo|first }}") + out = tmpl.render(foo=list(range(10))) + assert out == "0" + + @pytest.mark.parametrize( + ("value", "expect"), (("42", "42.0"), ("abc", "0.0"), ("32.32", "32.32")) + ) + def test_float(self, env, value, expect): + t = env.from_string("{{ value|float }}") + assert t.render(value=value) == expect + + def test_float_default(self, env): + t = env.from_string("{{ value|float(default=1.0) }}") + assert t.render(value="abc") == "1.0" + + def test_format(self, env): + tmpl = env.from_string("{{ '%s|%s'|format('a', 'b') }}") + out = tmpl.render() + assert out == "a|b" + + @staticmethod + def _test_indent_multiline_template(env, markup=False): + text = "\n".join(["", "foo bar", '"baz"', ""]) + if markup: + text = Markup(text) + t = env.from_string("{{ foo|indent(2, false, false) }}") + assert t.render(foo=text) == '\n foo bar\n "baz"\n' + t = env.from_string("{{ foo|indent(2, false, true) }}") + assert t.render(foo=text) == '\n foo bar\n "baz"\n ' + t = env.from_string("{{ foo|indent(2, true, false) }}") + assert t.render(foo=text) == ' \n foo bar\n "baz"\n' + t = env.from_string("{{ foo|indent(2, true, true) }}") + assert t.render(foo=text) == ' \n foo bar\n "baz"\n ' + + def test_indent(self, env): + self._test_indent_multiline_template(env) + t = env.from_string('{{ "jinja"|indent }}') + assert t.render() == "jinja" + t = env.from_string('{{ "jinja"|indent(first=true) }}') + assert t.render() == " jinja" + t = env.from_string('{{ "jinja"|indent(blank=true) }}') + assert t.render() == "jinja" + + def test_indent_markup_input(self, env): + """ + Tests cases where the filter input is a Markup type + """ + self._test_indent_multiline_template(env, markup=True) + + @pytest.mark.parametrize( + ("value", "expect"), + ( + ("42", "42"), + ("abc", "0"), + ("32.32", "32"), + ("12345678901234567890", "12345678901234567890"), + ), + ) + def test_int(self, env, value, expect): + t = env.from_string("{{ value|int }}") + assert t.render(value=value) == expect + + @pytest.mark.parametrize( + ("value", "base", "expect"), + (("0x4d32", 16, "19762"), ("011", 8, "9"), ("0x33Z", 16, "0"),), + ) + def test_int_base(self, env, value, base, expect): + t = env.from_string("{{ value|int(base=base) }}") + assert t.render(value=value, base=base) == expect + + def test_int_default(self, env): + t = env.from_string("{{ value|int(default=1) }}") + assert t.render(value="abc") == "1" + + def test_int_special_method(self, env): + class IntIsh: + def __int__(self): + return 42 + + t = env.from_string("{{ value|int }}") + assert t.render(value=IntIsh()) == "42" + + def test_join(self, env): + tmpl = env.from_string('{{ [1, 2, 3]|join("|") }}') + out = tmpl.render() + assert out == "1|2|3" + + env2 = Environment(autoescape=True) + tmpl = env2.from_string('{{ ["<foo>", "<span>foo</span>"|safe]|join }}') + assert tmpl.render() == "<foo><span>foo</span>" + + def test_join_attribute(self, env): + User = namedtuple("User", "username") + tmpl = env.from_string("""{{ users|join(', ', 'username') }}""") + assert tmpl.render(users=map(User, ["foo", "bar"])) == "foo, bar" + + def test_last(self, env): + tmpl = env.from_string("""{{ foo|last }}""") + out = tmpl.render(foo=list(range(10))) + assert out == "9" + + def test_length(self, env): + tmpl = env.from_string("""{{ "hello world"|length }}""") + out = tmpl.render() + assert out == "11" + + def test_lower(self, env): + tmpl = env.from_string("""{{ "FOO"|lower }}""") + out = tmpl.render() + assert out == "foo" + + def test_pprint(self, env): + from pprint import pformat + + tmpl = env.from_string("""{{ data|pprint }}""") + data = list(range(1000)) + assert tmpl.render(data=data) == pformat(data) + + def test_random(self, env, request): + # restore the random state when the test ends + state = random.getstate() + request.addfinalizer(lambda: random.setstate(state)) + # generate the random values from a known seed + random.seed("jinja") + expected = [random.choice("1234567890") for _ in range(10)] + + # check that the random sequence is generated again by a template + # ensures that filter result is not constant folded + random.seed("jinja") + t = env.from_string('{{ "1234567890"|random }}') + + for value in expected: + assert t.render() == value + + def test_reverse(self, env): + tmpl = env.from_string( + "{{ 'foobar'|reverse|join }}|{{ [1, 2, 3]|reverse|list }}" + ) + assert tmpl.render() == "raboof|[3, 2, 1]" + + def test_string(self, env): + x = [1, 2, 3, 4, 5] + tmpl = env.from_string("""{{ obj|string }}""") + assert tmpl.render(obj=x) == str(x) + + def test_title(self, env): + tmpl = env.from_string("""{{ "foo bar"|title }}""") + assert tmpl.render() == "Foo Bar" + tmpl = env.from_string("""{{ "foo's bar"|title }}""") + assert tmpl.render() == "Foo's Bar" + tmpl = env.from_string("""{{ "foo bar"|title }}""") + assert tmpl.render() == "Foo Bar" + tmpl = env.from_string("""{{ "f bar f"|title }}""") + assert tmpl.render() == "F Bar F" + tmpl = env.from_string("""{{ "foo-bar"|title }}""") + assert tmpl.render() == "Foo-Bar" + tmpl = env.from_string("""{{ "foo\tbar"|title }}""") + assert tmpl.render() == "Foo\tBar" + tmpl = env.from_string("""{{ "FOO\tBAR"|title }}""") + assert tmpl.render() == "Foo\tBar" + tmpl = env.from_string("""{{ "foo (bar)"|title }}""") + assert tmpl.render() == "Foo (Bar)" + tmpl = env.from_string("""{{ "foo {bar}"|title }}""") + assert tmpl.render() == "Foo {Bar}" + tmpl = env.from_string("""{{ "foo [bar]"|title }}""") + assert tmpl.render() == "Foo [Bar]" + tmpl = env.from_string("""{{ "foo <bar>"|title }}""") + assert tmpl.render() == "Foo <Bar>" + + class Foo: + def __str__(self): + return "foo-bar" + + tmpl = env.from_string("""{{ data|title }}""") + out = tmpl.render(data=Foo()) + assert out == "Foo-Bar" + + def test_truncate(self, env): + tmpl = env.from_string( + '{{ data|truncate(15, true, ">>>") }}|' + '{{ data|truncate(15, false, ">>>") }}|' + "{{ smalldata|truncate(15) }}" + ) + out = tmpl.render(data="foobar baz bar" * 1000, smalldata="foobar baz bar") + assert out == "foobar baz b>>>|foobar baz>>>|foobar baz bar" + + def test_truncate_very_short(self, env): + tmpl = env.from_string( + '{{ "foo bar baz"|truncate(9) }}|{{ "foo bar baz"|truncate(9, true) }}' + ) + out = tmpl.render() + assert out == "foo bar baz|foo bar baz" + + def test_truncate_end_length(self, env): + tmpl = env.from_string('{{ "Joel is a slug"|truncate(7, true) }}') + out = tmpl.render() + assert out == "Joel..." + + def test_upper(self, env): + tmpl = env.from_string('{{ "foo"|upper }}') + assert tmpl.render() == "FOO" + + def test_urlize(self, env): + tmpl = env.from_string('{{ "foo http://www.example.com/ bar"|urlize }}') + assert tmpl.render() == ( + 'foo <a href="http://www.example.com/" rel="noopener">' + "http://www.example.com/</a> bar" + ) + + def test_urlize_rel_policy(self): + env = Environment() + env.policies["urlize.rel"] = None + tmpl = env.from_string('{{ "foo http://www.example.com/ bar"|urlize }}') + assert tmpl.render() == ( + 'foo <a href="http://www.example.com/">http://www.example.com/</a> bar' + ) + + def test_urlize_target_parameter(self, env): + tmpl = env.from_string( + '{{ "foo http://www.example.com/ bar"|urlize(target="_blank") }}' + ) + assert ( + tmpl.render() + == 'foo <a href="http://www.example.com/" rel="noopener" target="_blank">' + "http://www.example.com/</a> bar" + ) + + def test_wordcount(self, env): + tmpl = env.from_string('{{ "foo bar baz"|wordcount }}') + assert tmpl.render() == "3" + + strict_env = Environment(undefined=StrictUndefined) + t = strict_env.from_string("{{ s|wordcount }}") + with pytest.raises(UndefinedError): + t.render() + + def test_block(self, env): + tmpl = env.from_string("{% filter lower|escape %}<HEHE>{% endfilter %}") + assert tmpl.render() == "<hehe>" + + def test_chaining(self, env): + tmpl = env.from_string("""{{ ['<foo>', '<bar>']|first|upper|escape }}""") + assert tmpl.render() == "<FOO>" + + def test_sum(self, env): + tmpl = env.from_string("""{{ [1, 2, 3, 4, 5, 6]|sum }}""") + assert tmpl.render() == "21" + + def test_sum_attributes(self, env): + tmpl = env.from_string("""{{ values|sum('value') }}""") + assert tmpl.render(values=[{"value": 23}, {"value": 1}, {"value": 18}]) == "42" + + def test_sum_attributes_nested(self, env): + tmpl = env.from_string("""{{ values|sum('real.value') }}""") + assert ( + tmpl.render( + values=[ + {"real": {"value": 23}}, + {"real": {"value": 1}}, + {"real": {"value": 18}}, + ] + ) + == "42" + ) + + def test_sum_attributes_tuple(self, env): + tmpl = env.from_string("""{{ values.items()|sum('1') }}""") + assert tmpl.render(values={"foo": 23, "bar": 1, "baz": 18}) == "42" + + def test_abs(self, env): + tmpl = env.from_string("""{{ -1|abs }}|{{ 1|abs }}""") + assert tmpl.render() == "1|1", tmpl.render() + + def test_round_positive(self, env): + tmpl = env.from_string( + "{{ 2.7|round }}|{{ 2.1|round }}|" + "{{ 2.1234|round(3, 'floor') }}|" + "{{ 2.1|round(0, 'ceil') }}" + ) + assert tmpl.render() == "3.0|2.0|2.123|3.0", tmpl.render() + + def test_round_negative(self, env): + tmpl = env.from_string( + "{{ 21.3|round(-1)}}|" + "{{ 21.3|round(-1, 'ceil')}}|" + "{{ 21.3|round(-1, 'floor')}}" + ) + assert tmpl.render() == "20.0|30.0|20.0", tmpl.render() + + def test_xmlattr(self, env): + tmpl = env.from_string( + "{{ {'foo': 42, 'bar': 23, 'fish': none, " + "'spam': missing, 'blub:blub': '<?>'}|xmlattr }}" + ) + out = tmpl.render().split() + assert len(out) == 3 + assert 'foo="42"' in out + assert 'bar="23"' in out + assert 'blub:blub="<?>"' in out + + def test_sort1(self, env): + tmpl = env.from_string("{{ [2, 3, 1]|sort }}|{{ [2, 3, 1]|sort(true) }}") + assert tmpl.render() == "[1, 2, 3]|[3, 2, 1]" + + def test_sort2(self, env): + tmpl = env.from_string('{{ "".join(["c", "A", "b", "D"]|sort) }}') + assert tmpl.render() == "AbcD" + + def test_sort3(self, env): + tmpl = env.from_string("""{{ ['foo', 'Bar', 'blah']|sort }}""") + assert tmpl.render() == "['Bar', 'blah', 'foo']" + + def test_sort4(self, env): + tmpl = env.from_string("""{{ items|sort(attribute='value')|join }}""") + assert tmpl.render(items=map(Magic, [3, 2, 4, 1])) == "1234" + + def test_sort5(self, env): + tmpl = env.from_string("""{{ items|sort(attribute='value.0')|join }}""") + assert tmpl.render(items=map(Magic, [[3], [2], [4], [1]])) == "[1][2][3][4]" + + def test_sort6(self, env): + tmpl = env.from_string("""{{ items|sort(attribute='value1,value2')|join }}""") + assert ( + tmpl.render( + items=map( + lambda x: Magic2(x[0], x[1]), [(3, 1), (2, 2), (2, 1), (2, 5)] + ) + ) + == "(2,1)(2,2)(2,5)(3,1)" + ) + + def test_sort7(self, env): + tmpl = env.from_string("""{{ items|sort(attribute='value2,value1')|join }}""") + assert ( + tmpl.render( + items=map( + lambda x: Magic2(x[0], x[1]), [(3, 1), (2, 2), (2, 1), (2, 5)] + ) + ) + == "(2,1)(3,1)(2,2)(2,5)" + ) + + def test_sort8(self, env): + tmpl = env.from_string( + """{{ items|sort(attribute='value1.0,value2.0')|join }}""" + ) + assert ( + tmpl.render( + items=map( + lambda x: Magic2(x[0], x[1]), + [([3], [1]), ([2], [2]), ([2], [1]), ([2], [5])], + ) + ) + == "([2],[1])([2],[2])([2],[5])([3],[1])" + ) + + def test_unique(self, env): + t = env.from_string('{{ "".join(["b", "A", "a", "b"]|unique) }}') + assert t.render() == "bA" + + def test_unique_case_sensitive(self, env): + t = env.from_string('{{ "".join(["b", "A", "a", "b"]|unique(true)) }}') + assert t.render() == "bAa" + + def test_unique_attribute(self, env): + t = env.from_string("{{ items|unique(attribute='value')|join }}") + assert t.render(items=map(Magic, [3, 2, 4, 1, 2])) == "3241" + + @pytest.mark.parametrize( + "source,expect", + ( + ('{{ ["a", "B"]|min }}', "a"), + ('{{ ["a", "B"]|min(case_sensitive=true) }}', "B"), + ("{{ []|min }}", ""), + ('{{ ["a", "B"]|max }}', "B"), + ('{{ ["a", "B"]|max(case_sensitive=true) }}', "a"), + ("{{ []|max }}", ""), + ), + ) + def test_min_max(self, env, source, expect): + t = env.from_string(source) + assert t.render() == expect + + @pytest.mark.parametrize("name,expect", (("min", "1"), ("max", "9"),)) + def test_min_max_attribute(self, env, name, expect): + t = env.from_string("{{ items|" + name + '(attribute="value") }}') + assert t.render(items=map(Magic, [5, 1, 9])) == expect + + def test_groupby(self, env): + tmpl = env.from_string( + """ + {%- for grouper, list in [{'foo': 1, 'bar': 2}, + {'foo': 2, 'bar': 3}, + {'foo': 1, 'bar': 1}, + {'foo': 3, 'bar': 4}]|groupby('foo') -%} + {{ grouper }}{% for x in list %}: {{ x.foo }}, {{ x.bar }}{% endfor %}| + {%- endfor %}""" + ) + assert tmpl.render().split("|") == ["1: 1, 2: 1, 1", "2: 2, 3", "3: 3, 4", ""] + + def test_groupby_tuple_index(self, env): + tmpl = env.from_string( + """ + {%- for grouper, list in [('a', 1), ('a', 2), ('b', 1)]|groupby(0) -%} + {{ grouper }}{% for x in list %}:{{ x.1 }}{% endfor %}| + {%- endfor %}""" + ) + assert tmpl.render() == "a:1:2|b:1|" + + def test_groupby_multidot(self, env): + Date = namedtuple("Date", "day,month,year") + Article = namedtuple("Article", "title,date") + articles = [ + Article("aha", Date(1, 1, 1970)), + Article("interesting", Date(2, 1, 1970)), + Article("really?", Date(3, 1, 1970)), + Article("totally not", Date(1, 1, 1971)), + ] + tmpl = env.from_string( + """ + {%- for year, list in articles|groupby('date.year') -%} + {{ year }}{% for x in list %}[{{ x.title }}]{% endfor %}| + {%- endfor %}""" + ) + assert tmpl.render(articles=articles).split("|") == [ + "1970[aha][interesting][really?]", + "1971[totally not]", + "", + ] + + def test_filtertag(self, env): + tmpl = env.from_string( + "{% filter upper|replace('FOO', 'foo') %}foobar{% endfilter %}" + ) + assert tmpl.render() == "fooBAR" + + def test_replace(self, env): + env = Environment() + tmpl = env.from_string('{{ string|replace("o", 42) }}') + assert tmpl.render(string="<foo>") == "<f4242>" + env = Environment(autoescape=True) + tmpl = env.from_string('{{ string|replace("o", 42) }}') + assert tmpl.render(string="<foo>") == "<f4242>" + tmpl = env.from_string('{{ string|replace("<", 42) }}') + assert tmpl.render(string="<foo>") == "42foo>" + tmpl = env.from_string('{{ string|replace("o", ">x<") }}') + assert tmpl.render(string=Markup("foo")) == "f>x<>x<" + + def test_forceescape(self, env): + tmpl = env.from_string("{{ x|forceescape }}") + assert tmpl.render(x=Markup("<div />")) == "<div />" + + def test_safe(self, env): + env = Environment(autoescape=True) + tmpl = env.from_string('{{ "<div>foo</div>"|safe }}') + assert tmpl.render() == "<div>foo</div>" + tmpl = env.from_string('{{ "<div>foo</div>" }}') + assert tmpl.render() == "<div>foo</div>" + + @pytest.mark.parametrize( + ("value", "expect"), + [ + ("Hello, world!", "Hello%2C%20world%21"), + ("Hello, world\u203d", "Hello%2C%20world%E2%80%BD"), + ({"f": 1}, "f=1"), + ([("f", 1), ("z", 2)], "f=1&z=2"), + ({"\u203d": 1}, "%E2%80%BD=1"), + ({0: 1}, "0=1"), + ([("a b/c", "a b/c")], "a+b%2Fc=a+b%2Fc"), + ("a b/c", "a%20b/c"), + ], + ) + def test_urlencode(self, value, expect): + e = Environment(autoescape=True) + t = e.from_string("{{ value|urlencode }}") + assert t.render(value=value) == expect + + def test_simple_map(self, env): + env = Environment() + tmpl = env.from_string('{{ ["1", "2", "3"]|map("int")|sum }}') + assert tmpl.render() == "6" + + def test_map_sum(self, env): + tmpl = env.from_string('{{ [[1,2], [3], [4,5,6]]|map("sum")|list }}') + assert tmpl.render() == "[3, 3, 15]" + + def test_attribute_map(self, env): + User = namedtuple("User", "name") + env = Environment() + users = [ + User("john"), + User("jane"), + User("mike"), + ] + tmpl = env.from_string('{{ users|map(attribute="name")|join("|") }}') + assert tmpl.render(users=users) == "john|jane|mike" + + def test_empty_map(self, env): + env = Environment() + tmpl = env.from_string('{{ none|map("upper")|list }}') + assert tmpl.render() == "[]" + + def test_map_default(self, env): + Fullname = namedtuple("Fullname", "firstname,lastname") + Firstname = namedtuple("Firstname", "firstname") + env = Environment() + tmpl = env.from_string( + '{{ users|map(attribute="lastname", default="smith")|join(", ") }}' + ) + users = [ + Fullname("john", "lennon"), + Fullname("jane", "edwards"), + Fullname("jon", None), + Firstname("mike"), + ] + assert tmpl.render(users=users) == "lennon, edwards, None, smith" + + def test_simple_select(self, env): + env = Environment() + tmpl = env.from_string('{{ [1, 2, 3, 4, 5]|select("odd")|join("|") }}') + assert tmpl.render() == "1|3|5" + + def test_bool_select(self, env): + env = Environment() + tmpl = env.from_string('{{ [none, false, 0, 1, 2, 3, 4, 5]|select|join("|") }}') + assert tmpl.render() == "1|2|3|4|5" + + def test_simple_reject(self, env): + env = Environment() + tmpl = env.from_string('{{ [1, 2, 3, 4, 5]|reject("odd")|join("|") }}') + assert tmpl.render() == "2|4" + + def test_bool_reject(self, env): + env = Environment() + tmpl = env.from_string('{{ [none, false, 0, 1, 2, 3, 4, 5]|reject|join("|") }}') + assert tmpl.render() == "None|False|0" + + def test_simple_select_attr(self, env): + User = namedtuple("User", "name,is_active") + env = Environment() + users = [ + User("john", True), + User("jane", True), + User("mike", False), + ] + tmpl = env.from_string( + '{{ users|selectattr("is_active")|map(attribute="name")|join("|") }}' + ) + assert tmpl.render(users=users) == "john|jane" + + def test_simple_reject_attr(self, env): + User = namedtuple("User", "name,is_active") + env = Environment() + users = [ + User("john", True), + User("jane", True), + User("mike", False), + ] + tmpl = env.from_string( + '{{ users|rejectattr("is_active")|map(attribute="name")|join("|") }}' + ) + assert tmpl.render(users=users) == "mike" + + def test_func_select_attr(self, env): + User = namedtuple("User", "id,name") + env = Environment() + users = [ + User(1, "john"), + User(2, "jane"), + User(3, "mike"), + ] + tmpl = env.from_string( + '{{ users|selectattr("id", "odd")|map(attribute="name")|join("|") }}' + ) + assert tmpl.render(users=users) == "john|mike" + + def test_func_reject_attr(self, env): + User = namedtuple("User", "id,name") + env = Environment() + users = [ + User(1, "john"), + User(2, "jane"), + User(3, "mike"), + ] + tmpl = env.from_string( + '{{ users|rejectattr("id", "odd")|map(attribute="name")|join("|") }}' + ) + assert tmpl.render(users=users) == "jane" + + def test_json_dump(self): + env = Environment(autoescape=True) + t = env.from_string("{{ x|tojson }}") + assert t.render(x={"foo": "bar"}) == '{"foo": "bar"}' + assert t.render(x="\"ba&r'") == r'"\"ba\u0026r\u0027"' + assert t.render(x="<bar>") == r'"\u003cbar\u003e"' + + def my_dumps(value, **options): + assert options == {"foo": "bar"} + return "42" + + env.policies["json.dumps_function"] = my_dumps + env.policies["json.dumps_kwargs"] = {"foo": "bar"} + assert t.render(x=23) == "42" + + def test_wordwrap(self, env): + env.newline_sequence = "\n" + t = env.from_string("{{ s|wordwrap(20) }}") + result = t.render(s="Hello!\nThis is Jinja saying something.") + assert result == "Hello!\nThis is Jinja saying\nsomething." diff --git a/tests/test_idtracking.py b/tests/test_idtracking.py new file mode 100644 index 00000000..8a884671 --- /dev/null +++ b/tests/test_idtracking.py @@ -0,0 +1,289 @@ +from jinja2 import nodes +from jinja2.idtracking import symbols_for_node + + +def test_basics(): + for_loop = nodes.For( + nodes.Name("foo", "store"), + nodes.Name("seq", "load"), + [nodes.Output([nodes.Name("foo", "load")])], + [], + None, + False, + ) + tmpl = nodes.Template( + [nodes.Assign(nodes.Name("foo", "store"), nodes.Name("bar", "load")), for_loop] + ) + + sym = symbols_for_node(tmpl) + assert sym.refs == { + "foo": "l_0_foo", + "bar": "l_0_bar", + "seq": "l_0_seq", + } + assert sym.loads == { + "l_0_foo": ("undefined", None), + "l_0_bar": ("resolve", "bar"), + "l_0_seq": ("resolve", "seq"), + } + + sym = symbols_for_node(for_loop, sym) + assert sym.refs == { + "foo": "l_1_foo", + } + assert sym.loads == { + "l_1_foo": ("param", None), + } + + +def test_complex(): + title_block = nodes.Block( + "title", [nodes.Output([nodes.TemplateData("Page Title")])], False + ) + + render_title_macro = nodes.Macro( + "render_title", + [nodes.Name("title", "param")], + [], + [ + nodes.Output( + [ + nodes.TemplateData('\n <div class="title">\n <h1>'), + nodes.Name("title", "load"), + nodes.TemplateData("</h1>\n <p>"), + nodes.Name("subtitle", "load"), + nodes.TemplateData("</p>\n "), + ] + ), + nodes.Assign( + nodes.Name("subtitle", "store"), nodes.Const("something else") + ), + nodes.Output( + [ + nodes.TemplateData("\n <p>"), + nodes.Name("subtitle", "load"), + nodes.TemplateData("</p>\n </div>\n"), + nodes.If( + nodes.Name("something", "load"), + [ + nodes.Assign( + nodes.Name("title_upper", "store"), + nodes.Filter( + nodes.Name("title", "load"), + "upper", + [], + [], + None, + None, + ), + ), + nodes.Output( + [ + nodes.Name("title_upper", "load"), + nodes.Call( + nodes.Name("render_title", "load"), + [nodes.Const("Aha")], + [], + None, + None, + ), + ] + ), + ], + [], + [], + ), + ] + ), + ], + ) + + for_loop = nodes.For( + nodes.Name("item", "store"), + nodes.Name("seq", "load"), + [ + nodes.Output( + [ + nodes.TemplateData("\n <li>"), + nodes.Name("item", "load"), + nodes.TemplateData("</li>\n <span>"), + ] + ), + nodes.Include(nodes.Const("helper.html"), True, False), + nodes.Output([nodes.TemplateData("</span>\n ")]), + ], + [], + None, + False, + ) + + body_block = nodes.Block( + "body", + [ + nodes.Output( + [ + nodes.TemplateData("\n "), + nodes.Call( + nodes.Name("render_title", "load"), + [nodes.Name("item", "load")], + [], + None, + None, + ), + nodes.TemplateData("\n <ul>\n "), + ] + ), + for_loop, + nodes.Output([nodes.TemplateData("\n </ul>\n")]), + ], + False, + ) + + tmpl = nodes.Template( + [ + nodes.Extends(nodes.Const("layout.html")), + title_block, + render_title_macro, + body_block, + ] + ) + + tmpl_sym = symbols_for_node(tmpl) + assert tmpl_sym.refs == { + "render_title": "l_0_render_title", + } + assert tmpl_sym.loads == { + "l_0_render_title": ("undefined", None), + } + assert tmpl_sym.stores == {"render_title"} + assert tmpl_sym.dump_stores() == { + "render_title": "l_0_render_title", + } + + macro_sym = symbols_for_node(render_title_macro, tmpl_sym) + assert macro_sym.refs == { + "subtitle": "l_1_subtitle", + "something": "l_1_something", + "title": "l_1_title", + "title_upper": "l_1_title_upper", + } + assert macro_sym.loads == { + "l_1_subtitle": ("resolve", "subtitle"), + "l_1_something": ("resolve", "something"), + "l_1_title": ("param", None), + "l_1_title_upper": ("resolve", "title_upper"), + } + assert macro_sym.stores == {"title", "title_upper", "subtitle"} + assert macro_sym.find_ref("render_title") == "l_0_render_title" + assert macro_sym.dump_stores() == { + "title": "l_1_title", + "title_upper": "l_1_title_upper", + "subtitle": "l_1_subtitle", + "render_title": "l_0_render_title", + } + + body_sym = symbols_for_node(body_block) + assert body_sym.refs == { + "item": "l_0_item", + "seq": "l_0_seq", + "render_title": "l_0_render_title", + } + assert body_sym.loads == { + "l_0_item": ("resolve", "item"), + "l_0_seq": ("resolve", "seq"), + "l_0_render_title": ("resolve", "render_title"), + } + assert body_sym.stores == set() + + for_sym = symbols_for_node(for_loop, body_sym) + assert for_sym.refs == { + "item": "l_1_item", + } + assert for_sym.loads == { + "l_1_item": ("param", None), + } + assert for_sym.stores == {"item"} + assert for_sym.dump_stores() == { + "item": "l_1_item", + } + + +def test_if_branching_stores(): + tmpl = nodes.Template( + [ + nodes.If( + nodes.Name("expression", "load"), + [nodes.Assign(nodes.Name("variable", "store"), nodes.Const(42))], + [], + [], + ) + ] + ) + + sym = symbols_for_node(tmpl) + assert sym.refs == {"variable": "l_0_variable", "expression": "l_0_expression"} + assert sym.stores == {"variable"} + assert sym.loads == { + "l_0_variable": ("resolve", "variable"), + "l_0_expression": ("resolve", "expression"), + } + assert sym.dump_stores() == { + "variable": "l_0_variable", + } + + +def test_if_branching_stores_undefined(): + tmpl = nodes.Template( + [ + nodes.Assign(nodes.Name("variable", "store"), nodes.Const(23)), + nodes.If( + nodes.Name("expression", "load"), + [nodes.Assign(nodes.Name("variable", "store"), nodes.Const(42))], + [], + [], + ), + ] + ) + + sym = symbols_for_node(tmpl) + assert sym.refs == {"variable": "l_0_variable", "expression": "l_0_expression"} + assert sym.stores == {"variable"} + assert sym.loads == { + "l_0_variable": ("undefined", None), + "l_0_expression": ("resolve", "expression"), + } + assert sym.dump_stores() == { + "variable": "l_0_variable", + } + + +def test_if_branching_multi_scope(): + for_loop = nodes.For( + nodes.Name("item", "store"), + nodes.Name("seq", "load"), + [ + nodes.If( + nodes.Name("expression", "load"), + [nodes.Assign(nodes.Name("x", "store"), nodes.Const(42))], + [], + [], + ), + nodes.Include(nodes.Const("helper.html"), True, False), + ], + [], + None, + False, + ) + + tmpl = nodes.Template( + [nodes.Assign(nodes.Name("x", "store"), nodes.Const(23)), for_loop] + ) + + tmpl_sym = symbols_for_node(tmpl) + for_sym = symbols_for_node(for_loop, tmpl_sym) + assert for_sym.stores == {"item", "x"} + assert for_sym.loads == { + "l_1_x": ("alias", "l_0_x"), + "l_1_item": ("param", None), + "l_1_expression": ("resolve", "expression"), + } diff --git a/tests/test_imports.py b/tests/test_imports.py new file mode 100644 index 00000000..054c9010 --- /dev/null +++ b/tests/test_imports.py @@ -0,0 +1,223 @@ +import pytest + +from jinja2.environment import Environment +from jinja2.exceptions import TemplateNotFound +from jinja2.exceptions import TemplatesNotFound +from jinja2.exceptions import TemplateSyntaxError +from jinja2.exceptions import UndefinedError +from jinja2.loaders import DictLoader + + +@pytest.fixture +def test_env(): + env = Environment( + loader=DictLoader( + dict( + module="{% macro test() %}[{{ foo }}|{{ bar }}]{% endmacro %}", + header="[{{ foo }}|{{ 23 }}]", + o_printer="({{ o }})", + ) + ) + ) + env.globals["bar"] = 23 + return env + + +class TestImports: + def test_context_imports(self, test_env): + t = test_env.from_string('{% import "module" as m %}{{ m.test() }}') + assert t.render(foo=42) == "[|23]" + t = test_env.from_string( + '{% import "module" as m without context %}{{ m.test() }}' + ) + assert t.render(foo=42) == "[|23]" + t = test_env.from_string( + '{% import "module" as m with context %}{{ m.test() }}' + ) + assert t.render(foo=42) == "[42|23]" + t = test_env.from_string('{% from "module" import test %}{{ test() }}') + assert t.render(foo=42) == "[|23]" + t = test_env.from_string( + '{% from "module" import test without context %}{{ test() }}' + ) + assert t.render(foo=42) == "[|23]" + t = test_env.from_string( + '{% from "module" import test with context %}{{ test() }}' + ) + assert t.render(foo=42) == "[42|23]" + + def test_import_needs_name(self, test_env): + test_env.from_string('{% from "foo" import bar %}') + test_env.from_string('{% from "foo" import bar, baz %}') + + with pytest.raises(TemplateSyntaxError): + test_env.from_string('{% from "foo" import %}') + + def test_no_trailing_comma(self, test_env): + with pytest.raises(TemplateSyntaxError): + test_env.from_string('{% from "foo" import bar, %}') + + with pytest.raises(TemplateSyntaxError): + test_env.from_string('{% from "foo" import bar,, %}') + + with pytest.raises(TemplateSyntaxError): + test_env.from_string('{% from "foo" import, %}') + + def test_trailing_comma_with_context(self, test_env): + test_env.from_string('{% from "foo" import bar, baz with context %}') + test_env.from_string('{% from "foo" import bar, baz, with context %}') + test_env.from_string('{% from "foo" import bar, with context %}') + test_env.from_string('{% from "foo" import bar, with, context %}') + test_env.from_string('{% from "foo" import bar, with with context %}') + + with pytest.raises(TemplateSyntaxError): + test_env.from_string('{% from "foo" import bar,, with context %}') + + with pytest.raises(TemplateSyntaxError): + test_env.from_string('{% from "foo" import bar with context, %}') + + def test_exports(self, test_env): + m = test_env.from_string( + """ + {% macro toplevel() %}...{% endmacro %} + {% macro __private() %}...{% endmacro %} + {% set variable = 42 %} + {% for item in [1] %} + {% macro notthere() %}{% endmacro %} + {% endfor %} + """ + ).module + assert m.toplevel() == "..." + assert not hasattr(m, "__missing") + assert m.variable == 42 + assert not hasattr(m, "notthere") + + def test_not_exported(self, test_env): + t = test_env.from_string("{% from 'module' import nothing %}{{ nothing() }}") + + with pytest.raises(UndefinedError, match="does not export the requested name"): + t.render() + + def test_import_with_globals(self, test_env): + env = Environment( + loader=DictLoader( + { + "macros": "{% macro test() %}foo: {{ foo }}{% endmacro %}", + "test": "{% import 'macros' as m %}{{ m.test() }}", + "test1": "{% import 'macros' as m %}{{ m.test() }}", + } + ) + ) + tmpl = env.get_template("test", globals={"foo": "bar"}) + assert tmpl.render() == "foo: bar" + + tmpl = env.get_template("test1") + assert tmpl.render() == "foo: " + + def test_import_with_globals_override(self, test_env): + env = Environment( + loader=DictLoader( + { + "macros": "{% set foo = '42' %}{% macro test() %}" + "foo: {{ foo }}{% endmacro %}", + "test": "{% from 'macros' import test %}{{ test() }}", + } + ) + ) + tmpl = env.get_template("test", globals={"foo": "bar"}) + assert tmpl.render() == "foo: 42" + + def test_from_import_with_globals(self, test_env): + env = Environment( + loader=DictLoader( + { + "macros": "{% macro testing() %}foo: {{ foo }}{% endmacro %}", + "test": "{% from 'macros' import testing %}{{ testing() }}", + } + ) + ) + tmpl = env.get_template("test", globals={"foo": "bar"}) + assert tmpl.render() == "foo: bar" + + +class TestIncludes: + def test_context_include(self, test_env): + t = test_env.from_string('{% include "header" %}') + assert t.render(foo=42) == "[42|23]" + t = test_env.from_string('{% include "header" with context %}') + assert t.render(foo=42) == "[42|23]" + t = test_env.from_string('{% include "header" without context %}') + assert t.render(foo=42) == "[|23]" + + def test_choice_includes(self, test_env): + t = test_env.from_string('{% include ["missing", "header"] %}') + assert t.render(foo=42) == "[42|23]" + + t = test_env.from_string('{% include ["missing", "missing2"] ignore missing %}') + assert t.render(foo=42) == "" + + t = test_env.from_string('{% include ["missing", "missing2"] %}') + pytest.raises(TemplateNotFound, t.render) + with pytest.raises(TemplatesNotFound) as e: + t.render() + + assert e.value.templates == ["missing", "missing2"] + assert e.value.name == "missing2" + + def test_includes(t, **ctx): + ctx["foo"] = 42 + assert t.render(ctx) == "[42|23]" + + t = test_env.from_string('{% include ["missing", "header"] %}') + test_includes(t) + t = test_env.from_string("{% include x %}") + test_includes(t, x=["missing", "header"]) + t = test_env.from_string('{% include [x, "header"] %}') + test_includes(t, x="missing") + t = test_env.from_string("{% include x %}") + test_includes(t, x="header") + t = test_env.from_string("{% include [x] %}") + test_includes(t, x="header") + + def test_include_ignoring_missing(self, test_env): + t = test_env.from_string('{% include "missing" %}') + pytest.raises(TemplateNotFound, t.render) + for extra in "", "with context", "without context": + t = test_env.from_string( + '{% include "missing" ignore missing ' + extra + " %}" + ) + assert t.render() == "" + + def test_context_include_with_overrides(self, test_env): + env = Environment( + loader=DictLoader( + dict( + main="{% for item in [1, 2, 3] %}{% include 'item' %}{% endfor %}", + item="{{ item }}", + ) + ) + ) + assert env.get_template("main").render() == "123" + + def test_unoptimized_scopes(self, test_env): + t = test_env.from_string( + """ + {% macro outer(o) %} + {% macro inner() %} + {% include "o_printer" %} + {% endmacro %} + {{ inner() }} + {% endmacro %} + {{ outer("FOO") }} + """ + ) + assert t.render().strip() == "(FOO)" + + def test_import_from_with_context(self): + env = Environment( + loader=DictLoader({"a": "{% macro x() %}{{ foobar }}{% endmacro %}"}) + ) + t = env.from_string( + "{% set foobar = 42 %}{% from 'a' import x with context %}{{ x() }}" + ) + assert t.render() == "42" diff --git a/tests/test_inheritance.py b/tests/test_inheritance.py new file mode 100644 index 00000000..b95c47db --- /dev/null +++ b/tests/test_inheritance.py @@ -0,0 +1,284 @@ +import pytest + +from jinja2 import DictLoader +from jinja2 import Environment +from jinja2 import TemplateRuntimeError + +LAYOUTTEMPLATE = """\ +|{% block block1 %}block 1 from layout{% endblock %} +|{% block block2 %}block 2 from layout{% endblock %} +|{% block block3 %} +{% block block4 %}nested block 4 from layout{% endblock %} +{% endblock %}|""" + +LEVEL1TEMPLATE = """\ +{% extends "layout" %} +{% block block1 %}block 1 from level1{% endblock %}""" + +LEVEL2TEMPLATE = """\ +{% extends "level1" %} +{% block block2 %}{% block block5 %}nested block 5 from level2{% +endblock %}{% endblock %}""" + +LEVEL3TEMPLATE = """\ +{% extends "level2" %} +{% block block5 %}block 5 from level3{% endblock %} +{% block block4 %}block 4 from level3{% endblock %} +""" + +LEVEL4TEMPLATE = """\ +{% extends "level3" %} +{% block block3 %}block 3 from level4{% endblock %} +""" + +WORKINGTEMPLATE = """\ +{% extends "layout" %} +{% block block1 %} + {% if false %} + {% block block2 %} + this should workd + {% endblock %} + {% endif %} +{% endblock %} +""" + +DOUBLEEXTENDS = """\ +{% extends "layout" %} +{% extends "layout" %} +{% block block1 %} + {% if false %} + {% block block2 %} + this should workd + {% endblock %} + {% endif %} +{% endblock %} +""" + + +@pytest.fixture +def env(): + return Environment( + loader=DictLoader( + { + "layout": LAYOUTTEMPLATE, + "level1": LEVEL1TEMPLATE, + "level2": LEVEL2TEMPLATE, + "level3": LEVEL3TEMPLATE, + "level4": LEVEL4TEMPLATE, + "working": WORKINGTEMPLATE, + "doublee": DOUBLEEXTENDS, + } + ), + trim_blocks=True, + ) + + +class TestInheritance: + def test_layout(self, env): + tmpl = env.get_template("layout") + assert tmpl.render() == ( + "|block 1 from layout|block 2 from layout|nested block 4 from layout|" + ) + + def test_level1(self, env): + tmpl = env.get_template("level1") + assert tmpl.render() == ( + "|block 1 from level1|block 2 from layout|nested block 4 from layout|" + ) + + def test_level2(self, env): + tmpl = env.get_template("level2") + assert tmpl.render() == ( + "|block 1 from level1|nested block 5 from " + "level2|nested block 4 from layout|" + ) + + def test_level3(self, env): + tmpl = env.get_template("level3") + assert tmpl.render() == ( + "|block 1 from level1|block 5 from level3|block 4 from level3|" + ) + + def test_level4(self, env): + tmpl = env.get_template("level4") + assert tmpl.render() == ( + "|block 1 from level1|block 5 from level3|block 3 from level4|" + ) + + def test_super(self, env): + env = Environment( + loader=DictLoader( + { + "a": "{% block intro %}INTRO{% endblock %}|" + "BEFORE|{% block data %}INNER{% endblock %}|AFTER", + "b": '{% extends "a" %}{% block data %}({{ ' + "super() }}){% endblock %}", + "c": '{% extends "b" %}{% block intro %}--{{ ' + "super() }}--{% endblock %}\n{% block data " + "%}[{{ super() }}]{% endblock %}", + } + ) + ) + tmpl = env.get_template("c") + assert tmpl.render() == "--INTRO--|BEFORE|[(INNER)]|AFTER" + + def test_working(self, env): + env.get_template("working") + + def test_reuse_blocks(self, env): + tmpl = env.from_string( + "{{ self.foo() }}|{% block foo %}42{% endblock %}|{{ self.foo() }}" + ) + assert tmpl.render() == "42|42|42" + + def test_preserve_blocks(self, env): + env = Environment( + loader=DictLoader( + { + "a": "{% if false %}{% block x %}A{% endblock %}" + "{% endif %}{{ self.x() }}", + "b": '{% extends "a" %}{% block x %}B{{ super() }}{% endblock %}', + } + ) + ) + tmpl = env.get_template("b") + assert tmpl.render() == "BA" + + def test_dynamic_inheritance(self, env): + env = Environment( + loader=DictLoader( + { + "master1": "MASTER1{% block x %}{% endblock %}", + "master2": "MASTER2{% block x %}{% endblock %}", + "child": "{% extends master %}{% block x %}CHILD{% endblock %}", + } + ) + ) + tmpl = env.get_template("child") + for m in range(1, 3): + assert tmpl.render(master=f"master{m}") == f"MASTER{m}CHILD" + + def test_multi_inheritance(self, env): + env = Environment( + loader=DictLoader( + { + "master1": "MASTER1{% block x %}{% endblock %}", + "master2": "MASTER2{% block x %}{% endblock %}", + "child": """{% if master %}{% extends master %}{% else %}{% extends + 'master1' %}{% endif %}{% block x %}CHILD{% endblock %}""", + } + ) + ) + tmpl = env.get_template("child") + assert tmpl.render(master="master2") == "MASTER2CHILD" + assert tmpl.render(master="master1") == "MASTER1CHILD" + assert tmpl.render() == "MASTER1CHILD" + + def test_scoped_block(self, env): + env = Environment( + loader=DictLoader( + { + "master.html": "{% for item in seq %}[{% block item scoped %}" + "{% endblock %}]{% endfor %}" + } + ) + ) + t = env.from_string( + "{% extends 'master.html' %}{% block item %}{{ item }}{% endblock %}" + ) + assert t.render(seq=list(range(5))) == "[0][1][2][3][4]" + + def test_super_in_scoped_block(self, env): + env = Environment( + loader=DictLoader( + { + "master.html": "{% for item in seq %}[{% block item scoped %}" + "{{ item }}{% endblock %}]{% endfor %}" + } + ) + ) + t = env.from_string( + '{% extends "master.html" %}{% block item %}' + "{{ super() }}|{{ item * 2 }}{% endblock %}" + ) + assert t.render(seq=list(range(5))) == "[0|0][1|2][2|4][3|6][4|8]" + + def test_scoped_block_after_inheritance(self, env): + env = Environment( + loader=DictLoader( + { + "layout.html": """ + {% block useless %}{% endblock %} + """, + "index.html": """ + {%- extends 'layout.html' %} + {% from 'helpers.html' import foo with context %} + {% block useless %} + {% for x in [1, 2, 3] %} + {% block testing scoped %} + {{ foo(x) }} + {% endblock %} + {% endfor %} + {% endblock %} + """, + "helpers.html": """ + {% macro foo(x) %}{{ the_foo + x }}{% endmacro %} + """, + } + ) + ) + rv = env.get_template("index.html").render(the_foo=42).split() + assert rv == ["43", "44", "45"] + + +class TestBugFix: + def test_fixed_macro_scoping_bug(self, env): + assert ( + Environment( + loader=DictLoader( + { + "test.html": """\ + {% extends 'details.html' %} + + {% macro my_macro() %} + my_macro + {% endmacro %} + + {% block inner_box %} + {{ my_macro() }} + {% endblock %} + """, + "details.html": """\ + {% extends 'standard.html' %} + + {% macro my_macro() %} + my_macro + {% endmacro %} + + {% block content %} + {% block outer_box %} + outer_box + {% block inner_box %} + inner_box + {% endblock %} + {% endblock %} + {% endblock %} + """, + "standard.html": """ + {% block content %} {% endblock %} + """, + } + ) + ) + .get_template("test.html") + .render() + .split() + == ["outer_box", "my_macro"] + ) + + def test_double_extends(self, env): + """Ensures that a template with more than 1 {% extends ... %} usage + raises a ``TemplateError``. + """ + with pytest.raises(TemplateRuntimeError, match="extended multiple times"): + env.get_template("doublee").render() diff --git a/tests/test_lexnparse.py b/tests/test_lexnparse.py new file mode 100644 index 00000000..96e134d1 --- /dev/null +++ b/tests/test_lexnparse.py @@ -0,0 +1,1023 @@ +import pytest + +from jinja2 import Environment +from jinja2 import nodes +from jinja2 import Template +from jinja2 import TemplateSyntaxError +from jinja2 import UndefinedError +from jinja2.lexer import Token +from jinja2.lexer import TOKEN_BLOCK_BEGIN +from jinja2.lexer import TOKEN_BLOCK_END +from jinja2.lexer import TOKEN_EOF +from jinja2.lexer import TokenStream + + +class TestTokenStream: + test_tokens = [ + Token(1, TOKEN_BLOCK_BEGIN, ""), + Token(2, TOKEN_BLOCK_END, ""), + ] + + def test_simple(self, env): + ts = TokenStream(self.test_tokens, "foo", "bar") + assert ts.current.type is TOKEN_BLOCK_BEGIN + assert bool(ts) + assert not bool(ts.eos) + next(ts) + assert ts.current.type is TOKEN_BLOCK_END + assert bool(ts) + assert not bool(ts.eos) + next(ts) + assert ts.current.type is TOKEN_EOF + assert not bool(ts) + assert bool(ts.eos) + + def test_iter(self, env): + token_types = [t.type for t in TokenStream(self.test_tokens, "foo", "bar")] + assert token_types == [ + "block_begin", + "block_end", + ] + + +class TestLexer: + def test_raw1(self, env): + tmpl = env.from_string( + "{% raw %}foo{% endraw %}|" + "{%raw%}{{ bar }}|{% baz %}{% endraw %}" + ) + assert tmpl.render() == "foo|{{ bar }}|{% baz %}" + + def test_raw2(self, env): + tmpl = env.from_string("1 {%- raw -%} 2 {%- endraw -%} 3") + assert tmpl.render() == "123" + + def test_raw3(self, env): + # The second newline after baz exists because it is AFTER the + # {% raw %} and is ignored. + env = Environment(lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string("bar\n{% raw %}\n {{baz}}2 spaces\n{% endraw %}\nfoo") + assert tmpl.render(baz="test") == "bar\n\n {{baz}}2 spaces\nfoo" + + def test_raw4(self, env): + # The trailing dash of the {% raw -%} cleans both the spaces and + # newlines up to the first character of data. + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string( + "bar\n{%- raw -%}\n\n \n 2 spaces\n space{%- endraw -%}\nfoo" + ) + assert tmpl.render() == "bar2 spaces\n spacefoo" + + def test_balancing(self, env): + env = Environment("{%", "%}", "${", "}") + tmpl = env.from_string( + """{% for item in seq + %}${{'foo': item}|upper}{% endfor %}""" + ) + assert tmpl.render(seq=list(range(3))) == "{'FOO': 0}{'FOO': 1}{'FOO': 2}" + + def test_comments(self, env): + env = Environment("<!--", "-->", "{", "}") + tmpl = env.from_string( + """\ +<ul> +<!--- for item in seq --> + <li>{item}</li> +<!--- endfor --> +</ul>""" + ) + assert tmpl.render(seq=list(range(3))) == ( + "<ul>\n <li>0</li>\n <li>1</li>\n <li>2</li>\n</ul>" + ) + + def test_string_escapes(self, env): + for char in "\0", "\u2668", "\xe4", "\t", "\r", "\n": + tmpl = env.from_string(f"{{{{ {char!r} }}}}") + assert tmpl.render() == char + assert env.from_string('{{ "\N{HOT SPRINGS}" }}').render() == "\u2668" + + def test_bytefallback(self, env): + from pprint import pformat + + tmpl = env.from_string("""{{ 'foo'|pprint }}|{{ 'bär'|pprint }}""") + assert tmpl.render() == pformat("foo") + "|" + pformat("bär") + + def test_operators(self, env): + from jinja2.lexer import operators + + for test, expect in operators.items(): + if test in "([{}])": + continue + stream = env.lexer.tokenize(f"{{{{ {test} }}}}") + next(stream) + assert stream.current.type == expect + + def test_normalizing(self, env): + for seq in "\r", "\r\n", "\n": + env = Environment(newline_sequence=seq) + tmpl = env.from_string("1\n2\r\n3\n4\n") + result = tmpl.render() + assert result.replace(seq, "X") == "1X2X3X4" + + def test_trailing_newline(self, env): + for keep in [True, False]: + env = Environment(keep_trailing_newline=keep) + for template, expected in [ + ("", {}), + ("no\nnewline", {}), + ("with\nnewline\n", {False: "with\nnewline"}), + ("with\nseveral\n\n\n", {False: "with\nseveral\n\n"}), + ]: + tmpl = env.from_string(template) + expect = expected.get(keep, template) + result = tmpl.render() + assert result == expect, (keep, template, result, expect) + + @pytest.mark.parametrize( + ("name", "valid"), + [ + ("foo", True), + ("föö", True), + ("き", True), + ("_", True), + ("1a", False), # invalid ascii start + ("a-", False), # invalid ascii continue + ("\U0001f40da", False), # invalid unicode start + ("a🐍\U0001f40d", False), # invalid unicode continue + # start characters not matched by \w + ("\u1885", True), + ("\u1886", True), + ("\u2118", True), + ("\u212e", True), + # continue character not matched by \w + ("\xb7", False), + ("a\xb7", True), + ], + ) + def test_name(self, env, name, valid): + t = "{{ " + name + " }}" + + if valid: + # valid for version being tested, shouldn't raise + env.from_string(t) + else: + pytest.raises(TemplateSyntaxError, env.from_string, t) + + def test_lineno_with_strip(self, env): + tokens = env.lex( + """\ +<html> + <body> + {%- block content -%} + <hr> + {{ item }} + {% endblock %} + </body> +</html>""" + ) + for tok in tokens: + lineno, token_type, value = tok + if token_type == "name" and value == "item": + assert lineno == 5 + break + + +class TestParser: + def test_php_syntax(self, env): + env = Environment("<?", "?>", "<?=", "?>", "<!--", "-->") + tmpl = env.from_string( + """\ +<!-- I'm a comment, I'm not interesting -->\ +<? for item in seq -?> + <?= item ?> +<?- endfor ?>""" + ) + assert tmpl.render(seq=list(range(5))) == "01234" + + def test_erb_syntax(self, env): + env = Environment("<%", "%>", "<%=", "%>", "<%#", "%>") + tmpl = env.from_string( + """\ +<%# I'm a comment, I'm not interesting %>\ +<% for item in seq -%> + <%= item %> +<%- endfor %>""" + ) + assert tmpl.render(seq=list(range(5))) == "01234" + + def test_comment_syntax(self, env): + env = Environment("<!--", "-->", "${", "}", "<!--#", "-->") + tmpl = env.from_string( + """\ +<!--# I'm a comment, I'm not interesting -->\ +<!-- for item in seq ---> + ${item} +<!--- endfor -->""" + ) + assert tmpl.render(seq=list(range(5))) == "01234" + + def test_balancing(self, env): + tmpl = env.from_string("""{{{'foo':'bar'}.foo}}""") + assert tmpl.render() == "bar" + + def test_start_comment(self, env): + tmpl = env.from_string( + """{# foo comment +and bar comment #} +{% macro blub() %}foo{% endmacro %} +{{ blub() }}""" + ) + assert tmpl.render().strip() == "foo" + + def test_line_syntax(self, env): + env = Environment("<%", "%>", "${", "}", "<%#", "%>", "%") + tmpl = env.from_string( + """\ +<%# regular comment %> +% for item in seq: + ${item} +% endfor""" + ) + assert [ + int(x.strip()) for x in tmpl.render(seq=list(range(5))).split() + ] == list(range(5)) + + env = Environment("<%", "%>", "${", "}", "<%#", "%>", "%", "##") + tmpl = env.from_string( + """\ +<%# regular comment %> +% for item in seq: + ${item} ## the rest of the stuff +% endfor""" + ) + assert [ + int(x.strip()) for x in tmpl.render(seq=list(range(5))).split() + ] == list(range(5)) + + def test_line_syntax_priority(self, env): + # XXX: why is the whitespace there in front of the newline? + env = Environment("{%", "%}", "${", "}", "/*", "*/", "##", "#") + tmpl = env.from_string( + """\ +/* ignore me. + I'm a multiline comment */ +## for item in seq: +* ${item} # this is just extra stuff +## endfor""" + ) + assert tmpl.render(seq=[1, 2]).strip() == "* 1\n* 2" + env = Environment("{%", "%}", "${", "}", "/*", "*/", "#", "##") + tmpl = env.from_string( + """\ +/* ignore me. + I'm a multiline comment */ +# for item in seq: +* ${item} ## this is just extra stuff + ## extra stuff i just want to ignore +# endfor""" + ) + assert tmpl.render(seq=[1, 2]).strip() == "* 1\n\n* 2" + + def test_error_messages(self, env): + def assert_error(code, expected): + with pytest.raises(TemplateSyntaxError, match=expected): + Template(code) + + assert_error( + "{% for item in seq %}...{% endif %}", + "Encountered unknown tag 'endif'. Jinja was looking " + "for the following tags: 'endfor' or 'else'. The " + "innermost block that needs to be closed is 'for'.", + ) + assert_error( + "{% if foo %}{% for item in seq %}...{% endfor %}{% endfor %}", + "Encountered unknown tag 'endfor'. Jinja was looking for " + "the following tags: 'elif' or 'else' or 'endif'. The " + "innermost block that needs to be closed is 'if'.", + ) + assert_error( + "{% if foo %}", + "Unexpected end of template. Jinja was looking for the " + "following tags: 'elif' or 'else' or 'endif'. The " + "innermost block that needs to be closed is 'if'.", + ) + assert_error( + "{% for item in seq %}", + "Unexpected end of template. Jinja was looking for the " + "following tags: 'endfor' or 'else'. The innermost block " + "that needs to be closed is 'for'.", + ) + assert_error( + "{% block foo-bar-baz %}", + "Block names in Jinja have to be valid Python identifiers " + "and may not contain hyphens, use an underscore instead.", + ) + assert_error("{% unknown_tag %}", "Encountered unknown tag 'unknown_tag'.") + + +class TestSyntax: + def test_call(self, env): + env = Environment() + env.globals["foo"] = lambda a, b, c, e, g: a + b + c + e + g + tmpl = env.from_string("{{ foo('a', c='d', e='f', *['b'], **{'g': 'h'}) }}") + assert tmpl.render() == "abdfh" + + def test_slicing(self, env): + tmpl = env.from_string("{{ [1, 2, 3][:] }}|{{ [1, 2, 3][::-1] }}") + assert tmpl.render() == "[1, 2, 3]|[3, 2, 1]" + + def test_attr(self, env): + tmpl = env.from_string("{{ foo.bar }}|{{ foo['bar'] }}") + assert tmpl.render(foo={"bar": 42}) == "42|42" + + def test_subscript(self, env): + tmpl = env.from_string("{{ foo[0] }}|{{ foo[-1] }}") + assert tmpl.render(foo=[0, 1, 2]) == "0|2" + + def test_tuple(self, env): + tmpl = env.from_string("{{ () }}|{{ (1,) }}|{{ (1, 2) }}") + assert tmpl.render() == "()|(1,)|(1, 2)" + + def test_math(self, env): + tmpl = env.from_string("{{ (1 + 1 * 2) - 3 / 2 }}|{{ 2**3 }}") + assert tmpl.render() == "1.5|8" + + def test_div(self, env): + tmpl = env.from_string("{{ 3 // 2 }}|{{ 3 / 2 }}|{{ 3 % 2 }}") + assert tmpl.render() == "1|1.5|1" + + def test_unary(self, env): + tmpl = env.from_string("{{ +3 }}|{{ -3 }}") + assert tmpl.render() == "3|-3" + + def test_concat(self, env): + tmpl = env.from_string("{{ [1, 2] ~ 'foo' }}") + assert tmpl.render() == "[1, 2]foo" + + @pytest.mark.parametrize( + ("a", "op", "b"), + [ + (1, ">", 0), + (1, ">=", 1), + (2, "<", 3), + (3, "<=", 4), + (4, "==", 4), + (4, "!=", 5), + ], + ) + def test_compare(self, env, a, op, b): + t = env.from_string(f"{{{{ {a} {op} {b} }}}}") + assert t.render() == "True" + + def test_compare_parens(self, env): + t = env.from_string("{{ i * (j < 5) }}") + assert t.render(i=2, j=3) == "2" + + @pytest.mark.parametrize( + ("src", "expect"), + [ + ("{{ 4 < 2 < 3 }}", "False"), + ("{{ a < b < c }}", "False"), + ("{{ 4 > 2 > 3 }}", "False"), + ("{{ a > b > c }}", "False"), + ("{{ 4 > 2 < 3 }}", "True"), + ("{{ a > b < c }}", "True"), + ], + ) + def test_compare_compound(self, env, src, expect): + t = env.from_string(src) + assert t.render(a=4, b=2, c=3) == expect + + def test_inop(self, env): + tmpl = env.from_string("{{ 1 in [1, 2, 3] }}|{{ 1 not in [1, 2, 3] }}") + assert tmpl.render() == "True|False" + + @pytest.mark.parametrize("value", ("[]", "{}", "()")) + def test_collection_literal(self, env, value): + t = env.from_string(f"{{{{ {value} }}}}") + assert t.render() == value + + @pytest.mark.parametrize( + ("value", "expect"), + ( + ("1", "1"), + ("123", "123"), + ("12_34_56", "123456"), + ("1.2", "1.2"), + ("34.56", "34.56"), + ("3_4.5_6", "34.56"), + ("1e0", "1.0"), + ("10e1", "100.0"), + ("2.5e100", "2.5e+100"), + ("2.5e+100", "2.5e+100"), + ("25.6e-10", "2.56e-09"), + ("1_2.3_4e5_6", "1.234e+57"), + ), + ) + def test_numeric_literal(self, env, value, expect): + t = env.from_string(f"{{{{ {value} }}}}") + assert t.render() == expect + + def test_bool(self, env): + tmpl = env.from_string( + "{{ true and false }}|{{ false or true }}|{{ not false }}" + ) + assert tmpl.render() == "False|True|True" + + def test_grouping(self, env): + tmpl = env.from_string( + "{{ (true and false) or (false and true) and not false }}" + ) + assert tmpl.render() == "False" + + def test_django_attr(self, env): + tmpl = env.from_string("{{ [1, 2, 3].0 }}|{{ [[1]].0.0 }}") + assert tmpl.render() == "1|1" + + def test_conditional_expression(self, env): + tmpl = env.from_string("""{{ 0 if true else 1 }}""") + assert tmpl.render() == "0" + + def test_short_conditional_expression(self, env): + tmpl = env.from_string("<{{ 1 if false }}>") + assert tmpl.render() == "<>" + + tmpl = env.from_string("<{{ (1 if false).bar }}>") + pytest.raises(UndefinedError, tmpl.render) + + def test_filter_priority(self, env): + tmpl = env.from_string('{{ "foo"|upper + "bar"|upper }}') + assert tmpl.render() == "FOOBAR" + + def test_function_calls(self, env): + tests = [ + (True, "*foo, bar"), + (True, "*foo, *bar"), + (True, "**foo, *bar"), + (True, "**foo, bar"), + (True, "**foo, **bar"), + (True, "**foo, bar=42"), + (False, "foo, bar"), + (False, "foo, bar=42"), + (False, "foo, bar=23, *args"), + (False, "foo, *args, bar=23"), + (False, "a, b=c, *d, **e"), + (False, "*foo, bar=42"), + (False, "*foo, **bar"), + (False, "*foo, bar=42, **baz"), + (False, "foo, *args, bar=23, **baz"), + ] + for should_fail, sig in tests: + if should_fail: + with pytest.raises(TemplateSyntaxError): + env.from_string(f"{{{{ foo({sig}) }}}}") + else: + env.from_string(f"foo({sig})") + + def test_tuple_expr(self, env): + for tmpl in [ + "{{ () }}", + "{{ (1, 2) }}", + "{{ (1, 2,) }}", + "{{ 1, }}", + "{{ 1, 2 }}", + "{% for foo, bar in seq %}...{% endfor %}", + "{% for x in foo, bar %}...{% endfor %}", + "{% for x in foo, %}...{% endfor %}", + ]: + assert env.from_string(tmpl) + + def test_trailing_comma(self, env): + tmpl = env.from_string("{{ (1, 2,) }}|{{ [1, 2,] }}|{{ {1: 2,} }}") + assert tmpl.render().lower() == "(1, 2)|[1, 2]|{1: 2}" + + def test_block_end_name(self, env): + env.from_string("{% block foo %}...{% endblock foo %}") + pytest.raises( + TemplateSyntaxError, env.from_string, "{% block x %}{% endblock y %}" + ) + + def test_constant_casing(self, env): + for const in True, False, None: + const = str(const) + tmpl = env.from_string( + f"{{{{ {const} }}}}|{{{{ {const.lower()} }}}}|{{{{ {const.upper()} }}}}" + ) + assert tmpl.render() == f"{const}|{const}|" + + def test_test_chaining(self, env): + pytest.raises( + TemplateSyntaxError, env.from_string, "{{ foo is string is sequence }}" + ) + assert env.from_string("{{ 42 is string or 42 is number }}").render() == "True" + + def test_string_concatenation(self, env): + tmpl = env.from_string('{{ "foo" "bar" "baz" }}') + assert tmpl.render() == "foobarbaz" + + def test_notin(self, env): + bar = range(100) + tmpl = env.from_string("""{{ not 42 in bar }}""") + assert tmpl.render(bar=bar) == "False" + + def test_operator_precedence(self, env): + tmpl = env.from_string("""{{ 2 * 3 + 4 % 2 + 1 - 2 }}""") + assert tmpl.render() == "5" + + def test_implicit_subscribed_tuple(self, env): + class Foo: + def __getitem__(self, x): + return x + + t = env.from_string("{{ foo[1, 2] }}") + assert t.render(foo=Foo()) == "(1, 2)" + + def test_raw2(self, env): + tmpl = env.from_string("{% raw %}{{ FOO }} and {% BAR %}{% endraw %}") + assert tmpl.render() == "{{ FOO }} and {% BAR %}" + + def test_const(self, env): + tmpl = env.from_string( + "{{ true }}|{{ false }}|{{ none }}|" + "{{ none is defined }}|{{ missing is defined }}" + ) + assert tmpl.render() == "True|False|None|True|False" + + def test_neg_filter_priority(self, env): + node = env.parse("{{ -1|foo }}") + assert isinstance(node.body[0].nodes[0], nodes.Filter) + assert isinstance(node.body[0].nodes[0].node, nodes.Neg) + + def test_const_assign(self, env): + constass1 = """{% set true = 42 %}""" + constass2 = """{% for none in seq %}{% endfor %}""" + for tmpl in constass1, constass2: + pytest.raises(TemplateSyntaxError, env.from_string, tmpl) + + def test_localset(self, env): + tmpl = env.from_string( + """{% set foo = 0 %}\ +{% for item in [1, 2] %}{% set foo = 1 %}{% endfor %}\ +{{ foo }}""" + ) + assert tmpl.render() == "0" + + def test_parse_unary(self, env): + tmpl = env.from_string('{{ -foo["bar"] }}') + assert tmpl.render(foo={"bar": 42}) == "-42" + tmpl = env.from_string('{{ -foo["bar"]|abs }}') + assert tmpl.render(foo={"bar": 42}) == "42" + + +class TestLstripBlocks: + def test_lstrip(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(""" {% if True %}\n {% endif %}""") + assert tmpl.render() == "\n" + + def test_lstrip_trim(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string(""" {% if True %}\n {% endif %}""") + assert tmpl.render() == "" + + def test_no_lstrip(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(""" {%+ if True %}\n {%+ endif %}""") + assert tmpl.render() == " \n " + + def test_lstrip_blocks_false_with_no_lstrip(self, env): + # Test that + is a NOP (but does not cause an error) if lstrip_blocks=False + env = Environment(lstrip_blocks=False, trim_blocks=False) + tmpl = env.from_string(""" {% if True %}\n {% endif %}""") + assert tmpl.render() == " \n " + tmpl = env.from_string(""" {%+ if True %}\n {%+ endif %}""") + assert tmpl.render() == " \n " + + def test_lstrip_endline(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(""" hello{% if True %}\n goodbye{% endif %}""") + assert tmpl.render() == " hello\n goodbye" + + def test_lstrip_inline(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(""" {% if True %}hello {% endif %}""") + assert tmpl.render() == "hello " + + def test_lstrip_nested(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string( + """ {% if True %}a {% if True %}b {% endif %}c {% endif %}""" + ) + assert tmpl.render() == "a b c " + + def test_lstrip_left_chars(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string( + """ abc {% if True %} + hello{% endif %}""" + ) + assert tmpl.render() == " abc \n hello" + + def test_lstrip_embeded_strings(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(""" {% set x = " {% str %} " %}{{ x }}""") + assert tmpl.render() == " {% str %} " + + def test_lstrip_preserve_leading_newlines(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string("""\n\n\n{% set hello = 1 %}""") + assert tmpl.render() == "\n\n\n" + + def test_lstrip_comment(self, env): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string( + """ {# if True #} +hello + {#endif#}""" + ) + assert tmpl.render() == "\nhello\n" + + def test_lstrip_angle_bracket_simple(self, env): + env = Environment( + "<%", + "%>", + "${", + "}", + "<%#", + "%>", + "%", + "##", + lstrip_blocks=True, + trim_blocks=True, + ) + tmpl = env.from_string(""" <% if True %>hello <% endif %>""") + assert tmpl.render() == "hello " + + def test_lstrip_angle_bracket_comment(self, env): + env = Environment( + "<%", + "%>", + "${", + "}", + "<%#", + "%>", + "%", + "##", + lstrip_blocks=True, + trim_blocks=True, + ) + tmpl = env.from_string(""" <%# if True %>hello <%# endif %>""") + assert tmpl.render() == "hello " + + def test_lstrip_angle_bracket(self, env): + env = Environment( + "<%", + "%>", + "${", + "}", + "<%#", + "%>", + "%", + "##", + lstrip_blocks=True, + trim_blocks=True, + ) + tmpl = env.from_string( + """\ + <%# regular comment %> + <% for item in seq %> +${item} ## the rest of the stuff + <% endfor %>""" + ) + assert tmpl.render(seq=range(5)) == "".join(f"{x}\n" for x in range(5)) + + def test_lstrip_angle_bracket_compact(self, env): + env = Environment( + "<%", + "%>", + "${", + "}", + "<%#", + "%>", + "%", + "##", + lstrip_blocks=True, + trim_blocks=True, + ) + tmpl = env.from_string( + """\ + <%#regular comment%> + <%for item in seq%> +${item} ## the rest of the stuff + <%endfor%>""" + ) + assert tmpl.render(seq=range(5)) == "".join(f"{x}\n" for x in range(5)) + + def test_lstrip_blocks_outside_with_new_line(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string( + " {% if kvs %}(\n" + " {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}\n" + " ){% endif %}" + ) + out = tmpl.render(kvs=[("a", 1), ("b", 2)]) + assert out == "(\na=1 b=2 \n )" + + def test_lstrip_trim_blocks_outside_with_new_line(self): + env = Environment(lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string( + " {% if kvs %}(\n" + " {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}\n" + " ){% endif %}" + ) + out = tmpl.render(kvs=[("a", 1), ("b", 2)]) + assert out == "(\na=1 b=2 )" + + def test_lstrip_blocks_inside_with_new_line(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string( + " ({% if kvs %}\n" + " {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}\n" + " {% endif %})" + ) + out = tmpl.render(kvs=[("a", 1), ("b", 2)]) + assert out == " (\na=1 b=2 \n)" + + def test_lstrip_trim_blocks_inside_with_new_line(self): + env = Environment(lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string( + " ({% if kvs %}\n" + " {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}\n" + " {% endif %})" + ) + out = tmpl.render(kvs=[("a", 1), ("b", 2)]) + assert out == " (a=1 b=2 )" + + def test_lstrip_blocks_without_new_line(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string( + " {% if kvs %}" + " {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}" + " {% endif %}" + ) + out = tmpl.render(kvs=[("a", 1), ("b", 2)]) + assert out == " a=1 b=2 " + + def test_lstrip_trim_blocks_without_new_line(self): + env = Environment(lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string( + " {% if kvs %}" + " {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}" + " {% endif %}" + ) + out = tmpl.render(kvs=[("a", 1), ("b", 2)]) + assert out == " a=1 b=2 " + + def test_lstrip_blocks_consume_after_without_new_line(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string( + " {% if kvs -%}" + " {% for k, v in kvs %}{{ k }}={{ v }} {% endfor -%}" + " {% endif -%}" + ) + out = tmpl.render(kvs=[("a", 1), ("b", 2)]) + assert out == "a=1 b=2 " + + def test_lstrip_trim_blocks_consume_before_without_new_line(self): + env = Environment(lstrip_blocks=False, trim_blocks=False) + tmpl = env.from_string( + " {%- if kvs %}" + " {%- for k, v in kvs %}{{ k }}={{ v }} {% endfor -%}" + " {%- endif %}" + ) + out = tmpl.render(kvs=[("a", 1), ("b", 2)]) + assert out == "a=1 b=2 " + + def test_lstrip_trim_blocks_comment(self): + env = Environment(lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string(" {# 1 space #}\n {# 2 spaces #} {# 4 spaces #}") + out = tmpl.render() + assert out == " " * 4 + + def test_lstrip_trim_blocks_raw(self): + env = Environment(lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string("{{x}}\n{%- raw %} {% endraw -%}\n{{ y }}") + out = tmpl.render(x=1, y=2) + assert out == "1 2" + + def test_php_syntax_with_manual(self, env): + env = Environment( + "<?", "?>", "<?=", "?>", "<!--", "-->", lstrip_blocks=True, trim_blocks=True + ) + tmpl = env.from_string( + """\ + <!-- I'm a comment, I'm not interesting --> + <? for item in seq -?> + <?= item ?> + <?- endfor ?>""" + ) + assert tmpl.render(seq=range(5)) == "01234" + + def test_php_syntax(self, env): + env = Environment( + "<?", "?>", "<?=", "?>", "<!--", "-->", lstrip_blocks=True, trim_blocks=True + ) + tmpl = env.from_string( + """\ + <!-- I'm a comment, I'm not interesting --> + <? for item in seq ?> + <?= item ?> + <? endfor ?>""" + ) + assert tmpl.render(seq=range(5)) == "".join(f" {x}\n" for x in range(5)) + + def test_php_syntax_compact(self, env): + env = Environment( + "<?", "?>", "<?=", "?>", "<!--", "-->", lstrip_blocks=True, trim_blocks=True + ) + tmpl = env.from_string( + """\ + <!-- I'm a comment, I'm not interesting --> + <?for item in seq?> + <?=item?> + <?endfor?>""" + ) + assert tmpl.render(seq=range(5)) == "".join(f" {x}\n" for x in range(5)) + + def test_erb_syntax(self, env): + env = Environment( + "<%", "%>", "<%=", "%>", "<%#", "%>", lstrip_blocks=True, trim_blocks=True + ) + tmpl = env.from_string( + """\ +<%# I'm a comment, I'm not interesting %> + <% for item in seq %> + <%= item %> + <% endfor %> +""" + ) + assert tmpl.render(seq=range(5)) == "".join(f" {x}\n" for x in range(5)) + + def test_erb_syntax_with_manual(self, env): + env = Environment( + "<%", "%>", "<%=", "%>", "<%#", "%>", lstrip_blocks=True, trim_blocks=True + ) + tmpl = env.from_string( + """\ +<%# I'm a comment, I'm not interesting %> + <% for item in seq -%> + <%= item %> + <%- endfor %>""" + ) + assert tmpl.render(seq=range(5)) == "01234" + + def test_erb_syntax_no_lstrip(self, env): + env = Environment( + "<%", "%>", "<%=", "%>", "<%#", "%>", lstrip_blocks=True, trim_blocks=True + ) + tmpl = env.from_string( + """\ +<%# I'm a comment, I'm not interesting %> + <%+ for item in seq -%> + <%= item %> + <%- endfor %>""" + ) + assert tmpl.render(seq=range(5)) == " 01234" + + def test_comment_syntax(self, env): + env = Environment( + "<!--", + "-->", + "${", + "}", + "<!--#", + "-->", + lstrip_blocks=True, + trim_blocks=True, + ) + tmpl = env.from_string( + """\ +<!--# I'm a comment, I'm not interesting -->\ +<!-- for item in seq ---> + ${item} +<!--- endfor -->""" + ) + assert tmpl.render(seq=range(5)) == "01234" + + +class TestTrimBlocks: + def test_trim(self, env): + env = Environment(trim_blocks=True, lstrip_blocks=False) + tmpl = env.from_string(" {% if True %}\n {% endif %}") + assert tmpl.render() == " " + + def test_no_trim(self, env): + env = Environment(trim_blocks=True, lstrip_blocks=False) + tmpl = env.from_string(" {% if True +%}\n {% endif %}") + assert tmpl.render() == " \n " + + def test_no_trim_outer(self, env): + env = Environment(trim_blocks=True, lstrip_blocks=False) + tmpl = env.from_string("{% if True %}X{% endif +%}\nmore things") + assert tmpl.render() == "X\nmore things" + + def test_lstrip_no_trim(self, env): + env = Environment(trim_blocks=True, lstrip_blocks=True) + tmpl = env.from_string(" {% if True +%}\n {% endif %}") + assert tmpl.render() == "\n" + + def test_trim_blocks_false_with_no_trim(self, env): + # Test that + is a NOP (but does not cause an error) if trim_blocks=False + env = Environment(trim_blocks=False, lstrip_blocks=False) + tmpl = env.from_string(" {% if True %}\n {% endif %}") + assert tmpl.render() == " \n " + tmpl = env.from_string(" {% if True +%}\n {% endif %}") + assert tmpl.render() == " \n " + + tmpl = env.from_string(" {# comment #}\n ") + assert tmpl.render() == " \n " + tmpl = env.from_string(" {# comment +#}\n ") + assert tmpl.render() == " \n " + + tmpl = env.from_string(" {% raw %}{% endraw %}\n ") + assert tmpl.render() == " \n " + tmpl = env.from_string(" {% raw %}{% endraw +%}\n ") + assert tmpl.render() == " \n " + + def test_trim_nested(self, env): + env = Environment(trim_blocks=True, lstrip_blocks=True) + tmpl = env.from_string( + " {% if True %}\na {% if True %}\nb {% endif %}\nc {% endif %}" + ) + assert tmpl.render() == "a b c " + + def test_no_trim_nested(self, env): + env = Environment(trim_blocks=True, lstrip_blocks=True) + tmpl = env.from_string( + " {% if True +%}\na {% if True +%}\nb {% endif +%}\nc {% endif %}" + ) + assert tmpl.render() == "\na \nb \nc " + + def test_comment_trim(self, env): + env = Environment(trim_blocks=True, lstrip_blocks=True) + tmpl = env.from_string(""" {# comment #}\n\n """) + assert tmpl.render() == "\n " + + def test_comment_no_trim(self, env): + env = Environment(trim_blocks=True, lstrip_blocks=True) + tmpl = env.from_string(""" {# comment +#}\n\n """) + assert tmpl.render() == "\n\n " + + def test_multiple_comment_trim_lstrip(self, env): + env = Environment(trim_blocks=True, lstrip_blocks=True) + tmpl = env.from_string( + " {# comment #}\n\n{# comment2 #}\n \n{# comment3 #}\n\n " + ) + assert tmpl.render() == "\n \n\n " + + def test_multiple_comment_no_trim_lstrip(self, env): + env = Environment(trim_blocks=True, lstrip_blocks=True) + tmpl = env.from_string( + " {# comment +#}\n\n{# comment2 +#}\n \n{# comment3 +#}\n\n " + ) + assert tmpl.render() == "\n\n\n \n\n\n " + + def test_raw_trim_lstrip(self, env): + env = Environment(trim_blocks=True, lstrip_blocks=True) + tmpl = env.from_string("{{x}}{% raw %}\n\n {% endraw %}\n\n{{ y }}") + assert tmpl.render(x=1, y=2) == "1\n\n\n2" + + def test_raw_no_trim_lstrip(self, env): + env = Environment(trim_blocks=False, lstrip_blocks=True) + tmpl = env.from_string("{{x}}{% raw %}\n\n {% endraw +%}\n\n{{ y }}") + assert tmpl.render(x=1, y=2) == "1\n\n\n\n2" + + # raw blocks do not process inner text, so start tag cannot ignore trim + with pytest.raises(TemplateSyntaxError): + tmpl = env.from_string("{{x}}{% raw +%}\n\n {% endraw +%}\n\n{{ y }}") + + def test_no_trim_angle_bracket(self, env): + env = Environment( + "<%", "%>", "${", "}", "<%#", "%>", lstrip_blocks=True, trim_blocks=True, + ) + tmpl = env.from_string(" <% if True +%>\n\n <% endif %>") + assert tmpl.render() == "\n\n" + + tmpl = env.from_string(" <%# comment +%>\n\n ") + assert tmpl.render() == "\n\n " + + def test_no_trim_php_syntax(self, env): + env = Environment( + "<?", + "?>", + "<?=", + "?>", + "<!--", + "-->", + lstrip_blocks=False, + trim_blocks=True, + ) + tmpl = env.from_string(" <? if True +?>\n\n <? endif ?>") + assert tmpl.render() == " \n\n " + tmpl = env.from_string(" <!-- comment +-->\n\n ") + assert tmpl.render() == " \n\n " diff --git a/tests/test_loader.py b/tests/test_loader.py new file mode 100644 index 00000000..8ca1289a --- /dev/null +++ b/tests/test_loader.py @@ -0,0 +1,383 @@ +import importlib.abc +import importlib.machinery +import importlib.util +import os +import platform +import shutil +import sys +import tempfile +import time +import weakref + +import pytest + +from jinja2 import Environment +from jinja2 import loaders +from jinja2 import PackageLoader +from jinja2.exceptions import TemplateNotFound +from jinja2.loaders import split_template_path + + +class TestLoaders: + def test_dict_loader(self, dict_loader): + env = Environment(loader=dict_loader) + tmpl = env.get_template("justdict.html") + assert tmpl.render().strip() == "FOO" + pytest.raises(TemplateNotFound, env.get_template, "missing.html") + + def test_package_loader(self, package_loader): + env = Environment(loader=package_loader) + tmpl = env.get_template("test.html") + assert tmpl.render().strip() == "BAR" + pytest.raises(TemplateNotFound, env.get_template, "missing.html") + + def test_filesystem_loader_overlapping_names(self, filesystem_loader): + res = os.path.dirname(filesystem_loader.searchpath[0]) + t2_dir = os.path.join(res, "templates2") + # Make "foo" show up before "foo/test.html". + filesystem_loader.searchpath.insert(0, t2_dir) + e = Environment(loader=filesystem_loader) + e.get_template("foo") + # This would raise NotADirectoryError if "t2/foo" wasn't skipped. + e.get_template("foo/test.html") + + def test_choice_loader(self, choice_loader): + env = Environment(loader=choice_loader) + tmpl = env.get_template("justdict.html") + assert tmpl.render().strip() == "FOO" + tmpl = env.get_template("test.html") + assert tmpl.render().strip() == "BAR" + pytest.raises(TemplateNotFound, env.get_template, "missing.html") + + def test_function_loader(self, function_loader): + env = Environment(loader=function_loader) + tmpl = env.get_template("justfunction.html") + assert tmpl.render().strip() == "FOO" + pytest.raises(TemplateNotFound, env.get_template, "missing.html") + + def test_prefix_loader(self, prefix_loader): + env = Environment(loader=prefix_loader) + tmpl = env.get_template("a/test.html") + assert tmpl.render().strip() == "BAR" + tmpl = env.get_template("b/justdict.html") + assert tmpl.render().strip() == "FOO" + pytest.raises(TemplateNotFound, env.get_template, "missing") + + def test_caching(self): + changed = False + + class TestLoader(loaders.BaseLoader): + def get_source(self, environment, template): + return "foo", None, lambda: not changed + + env = Environment(loader=TestLoader(), cache_size=-1) + tmpl = env.get_template("template") + assert tmpl is env.get_template("template") + changed = True + assert tmpl is not env.get_template("template") + changed = False + + def test_no_cache(self): + mapping = {"foo": "one"} + env = Environment(loader=loaders.DictLoader(mapping), cache_size=0) + assert env.get_template("foo") is not env.get_template("foo") + + def test_limited_size_cache(self): + mapping = {"one": "foo", "two": "bar", "three": "baz"} + loader = loaders.DictLoader(mapping) + env = Environment(loader=loader, cache_size=2) + t1 = env.get_template("one") + t2 = env.get_template("two") + assert t2 is env.get_template("two") + assert t1 is env.get_template("one") + env.get_template("three") + loader_ref = weakref.ref(loader) + assert (loader_ref, "one") in env.cache + assert (loader_ref, "two") not in env.cache + assert (loader_ref, "three") in env.cache + + def test_cache_loader_change(self): + loader1 = loaders.DictLoader({"foo": "one"}) + loader2 = loaders.DictLoader({"foo": "two"}) + env = Environment(loader=loader1, cache_size=2) + assert env.get_template("foo").render() == "one" + env.loader = loader2 + assert env.get_template("foo").render() == "two" + + def test_dict_loader_cache_invalidates(self): + mapping = {"foo": "one"} + env = Environment(loader=loaders.DictLoader(mapping)) + assert env.get_template("foo").render() == "one" + mapping["foo"] = "two" + assert env.get_template("foo").render() == "two" + + def test_split_template_path(self): + assert split_template_path("foo/bar") == ["foo", "bar"] + assert split_template_path("./foo/bar") == ["foo", "bar"] + pytest.raises(TemplateNotFound, split_template_path, "../foo") + + +class TestFileSystemLoader: + searchpath = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "res", "templates" + ) + + @staticmethod + def _test_common(env): + tmpl = env.get_template("test.html") + assert tmpl.render().strip() == "BAR" + tmpl = env.get_template("foo/test.html") + assert tmpl.render().strip() == "FOO" + pytest.raises(TemplateNotFound, env.get_template, "missing.html") + + def test_searchpath_as_str(self): + filesystem_loader = loaders.FileSystemLoader(self.searchpath) + + env = Environment(loader=filesystem_loader) + self._test_common(env) + + def test_searchpath_as_pathlib(self): + import pathlib + + searchpath = pathlib.Path(self.searchpath) + filesystem_loader = loaders.FileSystemLoader(searchpath) + env = Environment(loader=filesystem_loader) + self._test_common(env) + + def test_searchpath_as_list_including_pathlib(self): + import pathlib + + searchpath = pathlib.Path(self.searchpath) + filesystem_loader = loaders.FileSystemLoader(["/tmp/templates", searchpath]) + env = Environment(loader=filesystem_loader) + self._test_common(env) + + def test_caches_template_based_on_mtime(self): + filesystem_loader = loaders.FileSystemLoader(self.searchpath) + + env = Environment(loader=filesystem_loader) + tmpl1 = env.get_template("test.html") + tmpl2 = env.get_template("test.html") + assert tmpl1 is tmpl2 + + os.utime(os.path.join(self.searchpath, "test.html"), (time.time(), time.time())) + tmpl3 = env.get_template("test.html") + assert tmpl1 is not tmpl3 + + @pytest.mark.parametrize( + ("encoding", "expect"), + [ + ("utf-8", "文字化け"), + ("iso-8859-1", "æ\x96\x87\xe5\xad\x97\xe5\x8c\x96\xe3\x81\x91"), + ], + ) + def test_uses_specified_encoding(self, encoding, expect): + loader = loaders.FileSystemLoader(self.searchpath, encoding=encoding) + e = Environment(loader=loader) + t = e.get_template("mojibake.txt") + assert t.render() == expect + + +class TestModuleLoader: + archive = None + + def compile_down(self, prefix_loader, zip="deflated"): + log = [] + self.reg_env = Environment(loader=prefix_loader) + if zip is not None: + fd, self.archive = tempfile.mkstemp(suffix=".zip") + os.close(fd) + else: + self.archive = tempfile.mkdtemp() + self.reg_env.compile_templates(self.archive, zip=zip, log_function=log.append) + self.mod_env = Environment(loader=loaders.ModuleLoader(self.archive)) + return "".join(log) + + def teardown(self): + if hasattr(self, "mod_env"): + if os.path.isfile(self.archive): + os.remove(self.archive) + else: + shutil.rmtree(self.archive) + self.archive = None + + def test_log(self, prefix_loader): + log = self.compile_down(prefix_loader) + assert ( + 'Compiled "a/foo/test.html" as ' + "tmpl_a790caf9d669e39ea4d280d597ec891c4ef0404a" in log + ) + assert "Finished compiling templates" in log + assert ( + 'Could not compile "a/syntaxerror.html": ' + "Encountered unknown tag 'endif'" in log + ) + + def _test_common(self): + tmpl1 = self.reg_env.get_template("a/test.html") + tmpl2 = self.mod_env.get_template("a/test.html") + assert tmpl1.render() == tmpl2.render() + + tmpl1 = self.reg_env.get_template("b/justdict.html") + tmpl2 = self.mod_env.get_template("b/justdict.html") + assert tmpl1.render() == tmpl2.render() + + def test_deflated_zip_compile(self, prefix_loader): + self.compile_down(prefix_loader, zip="deflated") + self._test_common() + + def test_stored_zip_compile(self, prefix_loader): + self.compile_down(prefix_loader, zip="stored") + self._test_common() + + def test_filesystem_compile(self, prefix_loader): + self.compile_down(prefix_loader, zip=None) + self._test_common() + + def test_weak_references(self, prefix_loader): + self.compile_down(prefix_loader) + self.mod_env.get_template("a/test.html") + key = loaders.ModuleLoader.get_template_key("a/test.html") + name = self.mod_env.loader.module.__name__ + + assert hasattr(self.mod_env.loader.module, key) + assert name in sys.modules + + # unset all, ensure the module is gone from sys.modules + self.mod_env = None + + try: + import gc + + gc.collect() + except BaseException: + pass + + assert name not in sys.modules + + def test_choice_loader(self, prefix_loader): + self.compile_down(prefix_loader) + self.mod_env.loader = loaders.ChoiceLoader( + [self.mod_env.loader, loaders.DictLoader({"DICT_SOURCE": "DICT_TEMPLATE"})] + ) + tmpl1 = self.mod_env.get_template("a/test.html") + assert tmpl1.render() == "BAR" + tmpl2 = self.mod_env.get_template("DICT_SOURCE") + assert tmpl2.render() == "DICT_TEMPLATE" + + def test_prefix_loader(self, prefix_loader): + self.compile_down(prefix_loader) + self.mod_env.loader = loaders.PrefixLoader( + { + "MOD": self.mod_env.loader, + "DICT": loaders.DictLoader({"test.html": "DICT_TEMPLATE"}), + } + ) + tmpl1 = self.mod_env.get_template("MOD/a/test.html") + assert tmpl1.render() == "BAR" + tmpl2 = self.mod_env.get_template("DICT/test.html") + assert tmpl2.render() == "DICT_TEMPLATE" + + def test_path_as_pathlib(self, prefix_loader): + self.compile_down(prefix_loader) + + mod_path = self.mod_env.loader.module.__path__[0] + + import pathlib + + mod_loader = loaders.ModuleLoader(pathlib.Path(mod_path)) + self.mod_env = Environment(loader=mod_loader) + + self._test_common() + + def test_supports_pathlib_in_list_of_paths(self, prefix_loader): + self.compile_down(prefix_loader) + + mod_path = self.mod_env.loader.module.__path__[0] + + import pathlib + + mod_loader = loaders.ModuleLoader([pathlib.Path(mod_path), "/tmp/templates"]) + self.mod_env = Environment(loader=mod_loader) + + self._test_common() + + +@pytest.fixture() +def package_dir_loader(monkeypatch): + monkeypatch.syspath_prepend(os.path.dirname(__file__)) + return PackageLoader("res") + + +@pytest.mark.parametrize( + ("template", "expect"), [("foo/test.html", "FOO"), ("test.html", "BAR")] +) +def test_package_dir_source(package_dir_loader, template, expect): + source, name, up_to_date = package_dir_loader.get_source(None, template) + assert source.rstrip() == expect + assert name.endswith(os.path.join(*split_template_path(template))) + assert up_to_date() + + +def test_package_dir_list(package_dir_loader): + templates = package_dir_loader.list_templates() + assert "foo/test.html" in templates + assert "test.html" in templates + + +@pytest.fixture() +def package_zip_loader(monkeypatch): + monkeypatch.syspath_prepend( + os.path.join(os.path.dirname(__file__), "res", "package.zip") + ) + return PackageLoader("t_pack") + + +@pytest.mark.parametrize( + ("template", "expect"), [("foo/test.html", "FOO"), ("test.html", "BAR")] +) +def test_package_zip_source(package_zip_loader, template, expect): + source, name, up_to_date = package_zip_loader.get_source(None, template) + assert source.rstrip() == expect + assert name.endswith(os.path.join(*split_template_path(template))) + assert up_to_date is None + + +@pytest.mark.xfail( + platform.python_implementation() == "PyPy", + reason="PyPy's zipimporter doesn't have a '_files' attribute.", + raises=TypeError, +) +def test_package_zip_list(package_zip_loader): + assert package_zip_loader.list_templates() == ["foo/test.html", "test.html"] + + +def test_pep_451_import_hook(): + class ImportHook(importlib.abc.MetaPathFinder, importlib.abc.Loader): + def find_spec(self, name, path=None, target=None): + if name != "res": + return None + + spec = importlib.machinery.PathFinder.find_spec(name) + return importlib.util.spec_from_file_location( + name, + spec.origin, + loader=self, + submodule_search_locations=spec.submodule_search_locations, + ) + + def create_module(self, spec): + return None # default behaviour is fine + + def exec_module(self, module): + return None # we need this to satisfy the interface, it's wrong + + # ensure we restore `sys.meta_path` after putting in our loader + before = sys.meta_path[:] + + try: + sys.meta_path.insert(0, ImportHook()) + package_loader = PackageLoader("res") + assert "test.html" in package_loader.list_templates() + finally: + sys.meta_path[:] = before diff --git a/tests/test_nativetypes.py b/tests/test_nativetypes.py new file mode 100644 index 00000000..22581813 --- /dev/null +++ b/tests/test_nativetypes.py @@ -0,0 +1,149 @@ +import math + +import pytest + +from jinja2.exceptions import UndefinedError +from jinja2.nativetypes import NativeEnvironment +from jinja2.nativetypes import NativeTemplate +from jinja2.runtime import Undefined + + +@pytest.fixture +def env(): + return NativeEnvironment() + + +def test_is_defined_native_return(env): + t = env.from_string("{{ missing is defined }}") + assert not t.render() + + +def test_undefined_native_return(env): + t = env.from_string("{{ missing }}") + assert isinstance(t.render(), Undefined) + + +def test_adding_undefined_native_return(env): + t = env.from_string("{{ 3 + missing }}") + + with pytest.raises(UndefinedError): + t.render() + + +def test_cast_int(env): + t = env.from_string("{{ value|int }}") + result = t.render(value="3") + assert isinstance(result, int) + assert result == 3 + + +def test_list_add(env): + t = env.from_string("{{ a + b }}") + result = t.render(a=["a", "b"], b=["c", "d"]) + assert isinstance(result, list) + assert result == ["a", "b", "c", "d"] + + +def test_multi_expression_add(env): + t = env.from_string("{{ a }} + {{ b }}") + result = t.render(a=["a", "b"], b=["c", "d"]) + assert not isinstance(result, list) + assert result == "['a', 'b'] + ['c', 'd']" + + +def test_loops(env): + t = env.from_string("{% for x in value %}{{ x }}{% endfor %}") + result = t.render(value=["a", "b", "c", "d"]) + assert isinstance(result, str) + assert result == "abcd" + + +def test_loops_with_ints(env): + t = env.from_string("{% for x in value %}{{ x }}{% endfor %}") + result = t.render(value=[1, 2, 3, 4]) + assert isinstance(result, int) + assert result == 1234 + + +def test_loop_look_alike(env): + t = env.from_string("{% for x in value %}{{ x }}{% endfor %}") + result = t.render(value=[1]) + assert isinstance(result, int) + assert result == 1 + + +@pytest.mark.parametrize( + ("source", "expect"), + ( + ("{{ value }}", True), + ("{{ value }}", False), + ("{{ 1 == 1 }}", True), + ("{{ 2 + 2 == 5 }}", False), + ("{{ None is none }}", True), + ("{{ '' == None }}", False), + ), +) +def test_booleans(env, source, expect): + t = env.from_string(source) + result = t.render(value=expect) + assert isinstance(result, bool) + assert result is expect + + +def test_variable_dunder(env): + t = env.from_string("{{ x.__class__ }}") + result = t.render(x=True) + assert isinstance(result, type) + + +def test_constant_dunder(env): + t = env.from_string("{{ true.__class__ }}") + result = t.render() + assert isinstance(result, type) + + +def test_constant_dunder_to_string(env): + t = env.from_string("{{ true.__class__|string }}") + result = t.render() + assert not isinstance(result, type) + assert result in {"<type 'bool'>", "<class 'bool'>"} + + +def test_string_literal_var(env): + t = env.from_string("[{{ 'all' }}]") + result = t.render() + assert isinstance(result, str) + assert result == "[all]" + + +def test_string_top_level(env): + t = env.from_string("'Jinja'") + result = t.render() + assert result == "Jinja" + + +def test_tuple_of_variable_strings(env): + t = env.from_string("'{{ a }}', 'data', '{{ b }}', b'{{ c }}'") + result = t.render(a=1, b=2, c="bytes") + assert isinstance(result, tuple) + assert result == ("1", "data", "2", b"bytes") + + +def test_concat_strings_with_quotes(env): + t = env.from_string("--host='{{ host }}' --user \"{{ user }}\"") + result = t.render(host="localhost", user="Jinja") + assert result == "--host='localhost' --user \"Jinja\"" + + +def test_no_intermediate_eval(env): + t = env.from_string("0.000{{ a }}") + result = t.render(a=7) + assert isinstance(result, float) + # If intermediate eval happened, 0.000 would render 0.0, then 7 + # would be appended, resulting in 0.07. + assert math.isclose(result, 0.0007) + + +def test_spontaneous_env(): + t = NativeTemplate("{{ true }}") + assert isinstance(t.environment, NativeEnvironment) diff --git a/tests/test_regression.py b/tests/test_regression.py new file mode 100644 index 00000000..65ace87f --- /dev/null +++ b/tests/test_regression.py @@ -0,0 +1,613 @@ +import pytest + +from jinja2 import DictLoader +from jinja2 import Environment +from jinja2 import PrefixLoader +from jinja2 import Template +from jinja2 import TemplateAssertionError +from jinja2 import TemplateNotFound +from jinja2 import TemplateSyntaxError + + +class TestCorner: + def test_assigned_scoping(self, env): + t = env.from_string( + """ + {%- for item in (1, 2, 3, 4) -%} + [{{ item }}] + {%- endfor %} + {{- item -}} + """ + ) + assert t.render(item=42) == "[1][2][3][4]42" + + t = env.from_string( + """ + {%- for item in (1, 2, 3, 4) -%} + [{{ item }}] + {%- endfor %} + {%- set item = 42 %} + {{- item -}} + """ + ) + assert t.render() == "[1][2][3][4]42" + + t = env.from_string( + """ + {%- set item = 42 %} + {%- for item in (1, 2, 3, 4) -%} + [{{ item }}] + {%- endfor %} + {{- item -}} + """ + ) + assert t.render() == "[1][2][3][4]42" + + def test_closure_scoping(self, env): + t = env.from_string( + """ + {%- set wrapper = "<FOO>" %} + {%- for item in (1, 2, 3, 4) %} + {%- macro wrapper() %}[{{ item }}]{% endmacro %} + {{- wrapper() }} + {%- endfor %} + {{- wrapper -}} + """ + ) + assert t.render() == "[1][2][3][4]<FOO>" + + t = env.from_string( + """ + {%- for item in (1, 2, 3, 4) %} + {%- macro wrapper() %}[{{ item }}]{% endmacro %} + {{- wrapper() }} + {%- endfor %} + {%- set wrapper = "<FOO>" %} + {{- wrapper -}} + """ + ) + assert t.render() == "[1][2][3][4]<FOO>" + + t = env.from_string( + """ + {%- for item in (1, 2, 3, 4) %} + {%- macro wrapper() %}[{{ item }}]{% endmacro %} + {{- wrapper() }} + {%- endfor %} + {{- wrapper -}} + """ + ) + assert t.render(wrapper=23) == "[1][2][3][4]23" + + +class TestBug: + def test_keyword_folding(self, env): + env = Environment() + env.filters["testing"] = lambda value, some: value + some + assert ( + env.from_string("{{ 'test'|testing(some='stuff') }}").render() + == "teststuff" + ) + + def test_extends_output_bugs(self, env): + env = Environment( + loader=DictLoader({"parent.html": "(({% block title %}{% endblock %}))"}) + ) + + t = env.from_string( + '{% if expr %}{% extends "parent.html" %}{% endif %}' + "[[{% block title %}title{% endblock %}]]" + "{% for item in [1, 2, 3] %}({{ item }}){% endfor %}" + ) + assert t.render(expr=False) == "[[title]](1)(2)(3)" + assert t.render(expr=True) == "((title))" + + def test_urlize_filter_escaping(self, env): + tmpl = env.from_string('{{ "http://www.example.org/<foo"|urlize }}') + assert ( + tmpl.render() == '<a href="http://www.example.org/<foo" rel="noopener">' + "http://www.example.org/<foo</a>" + ) + + def test_loop_call_loop(self, env): + tmpl = env.from_string( + """ + + {% macro test() %} + {{ caller() }} + {% endmacro %} + + {% for num1 in range(5) %} + {% call test() %} + {% for num2 in range(10) %} + {{ loop.index }} + {% endfor %} + {% endcall %} + {% endfor %} + + """ + ) + + assert tmpl.render().split() == [str(x) for x in range(1, 11)] * 5 + + def test_weird_inline_comment(self, env): + env = Environment(line_statement_prefix="%") + pytest.raises( + TemplateSyntaxError, + env.from_string, + "% for item in seq {# missing #}\n...% endfor", + ) + + def test_old_macro_loop_scoping_bug(self, env): + tmpl = env.from_string( + "{% for i in (1, 2) %}{{ i }}{% endfor %}" + "{% macro i() %}3{% endmacro %}{{ i() }}" + ) + assert tmpl.render() == "123" + + def test_partial_conditional_assignments(self, env): + tmpl = env.from_string("{% if b %}{% set a = 42 %}{% endif %}{{ a }}") + assert tmpl.render(a=23) == "23" + assert tmpl.render(b=True) == "42" + + def test_stacked_locals_scoping_bug(self, env): + env = Environment(line_statement_prefix="#") + t = env.from_string( + """\ +# for j in [1, 2]: +# set x = 1 +# for i in [1, 2]: +# print x +# if i % 2 == 0: +# set x = x + 1 +# endif +# endfor +# endfor +# if a +# print 'A' +# elif b +# print 'B' +# elif c == d +# print 'C' +# else +# print 'D' +# endif + """ + ) + assert t.render(a=0, b=False, c=42, d=42.0) == "1111C" + + def test_stacked_locals_scoping_bug_twoframe(self, env): + t = Template( + """ + {% set x = 1 %} + {% for item in foo %} + {% if item == 1 %} + {% set x = 2 %} + {% endif %} + {% endfor %} + {{ x }} + """ + ) + rv = t.render(foo=[1]).strip() + assert rv == "1" + + def test_call_with_args(self, env): + t = Template( + """{% macro dump_users(users) -%} + <ul> + {%- for user in users -%} + <li><p>{{ user.username|e }}</p>{{ caller(user) }}</li> + {%- endfor -%} + </ul> + {%- endmacro -%} + + {% call(user) dump_users(list_of_user) -%} + <dl> + <dl>Realname</dl> + <dd>{{ user.realname|e }}</dd> + <dl>Description</dl> + <dd>{{ user.description }}</dd> + </dl> + {% endcall %}""" + ) + + assert [ + x.strip() + for x in t.render( + list_of_user=[ + { + "username": "apo", + "realname": "something else", + "description": "test", + } + ] + ).splitlines() + ] == [ + "<ul><li><p>apo</p><dl>", + "<dl>Realname</dl>", + "<dd>something else</dd>", + "<dl>Description</dl>", + "<dd>test</dd>", + "</dl>", + "</li></ul>", + ] + + def test_empty_if_condition_fails(self, env): + pytest.raises(TemplateSyntaxError, Template, "{% if %}....{% endif %}") + pytest.raises( + TemplateSyntaxError, Template, "{% if foo %}...{% elif %}...{% endif %}" + ) + pytest.raises(TemplateSyntaxError, Template, "{% for x in %}..{% endfor %}") + + def test_recursive_loop_compile(self, env): + Template( + """ + {% for p in foo recursive%} + {{p.bar}} + {% for f in p.fields recursive%} + {{f.baz}} + {{p.bar}} + {% if f.rec %} + {{ loop(f.sub) }} + {% endif %} + {% endfor %} + {% endfor %} + """ + ) + Template( + """ + {% for p in foo%} + {{p.bar}} + {% for f in p.fields recursive%} + {{f.baz}} + {{p.bar}} + {% if f.rec %} + {{ loop(f.sub) }} + {% endif %} + {% endfor %} + {% endfor %} + """ + ) + + def test_else_loop_bug(self, env): + t = Template( + """ + {% for x in y %} + {{ loop.index0 }} + {% else %} + {% for i in range(3) %}{{ i }}{% endfor %} + {% endfor %} + """ + ) + assert t.render(y=[]).strip() == "012" + + def test_correct_prefix_loader_name(self, env): + env = Environment(loader=PrefixLoader({"foo": DictLoader({})})) + with pytest.raises(TemplateNotFound) as e: + env.get_template("foo/bar.html") + + assert e.value.name == "foo/bar.html" + + def test_contextfunction_callable_classes(self, env): + from jinja2.utils import contextfunction + + class CallableClass: + @contextfunction + def __call__(self, ctx): + return ctx.resolve("hello") + + tpl = Template("""{{ callableclass() }}""") + output = tpl.render(callableclass=CallableClass(), hello="TEST") + expected = "TEST" + + assert output == expected + + def test_block_set_with_extends(self): + env = Environment( + loader=DictLoader({"main": "{% block body %}[{{ x }}]{% endblock %}"}) + ) + t = env.from_string('{% extends "main" %}{% set x %}42{% endset %}') + assert t.render() == "[42]" + + def test_nested_for_else(self, env): + tmpl = env.from_string( + "{% for x in y %}{{ loop.index0 }}{% else %}" + "{% for i in range(3) %}{{ i }}{% endfor %}" + "{% endfor %}" + ) + assert tmpl.render() == "012" + + def test_macro_var_bug(self, env): + tmpl = env.from_string( + """ + {% set i = 1 %} + {% macro test() %} + {% for i in range(0, 10) %}{{ i }}{% endfor %} + {% endmacro %}{{ test() }} + """ + ) + assert tmpl.render().strip() == "0123456789" + + def test_macro_var_bug_advanced(self, env): + tmpl = env.from_string( + """ + {% macro outer() %} + {% set i = 1 %} + {% macro test() %} + {% for i in range(0, 10) %}{{ i }}{% endfor %} + {% endmacro %}{{ test() }} + {% endmacro %}{{ outer() }} + """ + ) + assert tmpl.render().strip() == "0123456789" + + def test_callable_defaults(self): + env = Environment() + env.globals["get_int"] = lambda: 42 + t = env.from_string( + """ + {% macro test(a, b, c=get_int()) -%} + {{ a + b + c }} + {%- endmacro %} + {{ test(1, 2) }}|{{ test(1, 2, 3) }} + """ + ) + assert t.render().strip() == "45|6" + + def test_macro_escaping(self): + env = Environment( + autoescape=lambda x: False, extensions=["jinja2.ext.autoescape"] + ) + template = "{% macro m() %}<html>{% endmacro %}" + template += "{% autoescape true %}{{ m() }}{% endautoescape %}" + assert env.from_string(template).render() + + def test_macro_scoping(self, env): + tmpl = env.from_string( + """ + {% set n=[1,2,3,4,5] %} + {% for n in [[1,2,3], [3,4,5], [5,6,7]] %} + + {% macro x(l) %} + {{ l.pop() }} + {% if l %}{{ x(l) }}{% endif %} + {% endmacro %} + + {{ x(n) }} + + {% endfor %} + """ + ) + assert list(map(int, tmpl.render().split())) == [3, 2, 1, 5, 4, 3, 7, 6, 5] + + def test_scopes_and_blocks(self): + env = Environment( + loader=DictLoader( + { + "a.html": """ + {%- set foo = 'bar' -%} + {% include 'x.html' -%} + """, + "b.html": """ + {%- set foo = 'bar' -%} + {% block test %}{% include 'x.html' %}{% endblock -%} + """, + "c.html": """ + {%- set foo = 'bar' -%} + {% block test %}{% set foo = foo + %}{% include 'x.html' %}{% endblock -%} + """, + "x.html": """{{ foo }}|{{ test }}""", + } + ) + ) + + a = env.get_template("a.html") + b = env.get_template("b.html") + c = env.get_template("c.html") + + assert a.render(test="x").strip() == "bar|x" + assert b.render(test="x").strip() == "bar|x" + assert c.render(test="x").strip() == "bar|x" + + def test_scopes_and_include(self): + env = Environment( + loader=DictLoader( + { + "include.html": "{{ var }}", + "base.html": '{% include "include.html" %}', + "child.html": '{% extends "base.html" %}{% set var = 42 %}', + } + ) + ) + t = env.get_template("child.html") + assert t.render() == "42" + + def test_caller_scoping(self, env): + t = env.from_string( + """ + {% macro detail(icon, value) -%} + {% if value -%} + <p><span class="fa fa-fw fa-{{ icon }}"></span> + {%- if caller is undefined -%} + {{ value }} + {%- else -%} + {{ caller(value, *varargs) }} + {%- endif -%}</p> + {%- endif %} + {%- endmacro %} + + + {% macro link_detail(icon, value, href) -%} + {% call(value, href) detail(icon, value, href) -%} + <a href="{{ href }}">{{ value }}</a> + {%- endcall %} + {%- endmacro %} + """ + ) + + assert t.module.link_detail("circle", "Index", "/") == ( + '<p><span class="fa fa-fw fa-circle"></span><a href="/">Index</a></p>' + ) + + def test_variable_reuse(self, env): + t = env.from_string("{% for x in x.y %}{{ x }}{% endfor %}") + assert t.render(x={"y": [0, 1, 2]}) == "012" + + t = env.from_string("{% for x in x.y %}{{ loop.index0 }}|{{ x }}{% endfor %}") + assert t.render(x={"y": [0, 1, 2]}) == "0|01|12|2" + + t = env.from_string("{% for x in x.y recursive %}{{ x }}{% endfor %}") + assert t.render(x={"y": [0, 1, 2]}) == "012" + + def test_double_caller(self, env): + t = env.from_string( + "{% macro x(caller=none) %}[{% if caller %}" + "{{ caller() }}{% endif %}]{% endmacro %}" + "{{ x() }}{% call x() %}aha!{% endcall %}" + ) + assert t.render() == "[][aha!]" + + def test_double_caller_no_default(self, env): + with pytest.raises(TemplateAssertionError) as exc_info: + env.from_string( + "{% macro x(caller) %}[{% if caller %}" + "{{ caller() }}{% endif %}]{% endmacro %}" + ) + assert exc_info.match( + r'"caller" argument must be omitted or ' r"be given a default" + ) + + t = env.from_string( + "{% macro x(caller=none) %}[{% if caller %}" + "{{ caller() }}{% endif %}]{% endmacro %}" + ) + with pytest.raises(TypeError) as exc_info: + t.module.x(None, caller=lambda: 42) + assert exc_info.match( + r"\'x\' was invoked with two values for the " r"special caller argument" + ) + + def test_macro_blocks(self, env): + t = env.from_string( + "{% macro x() %}{% block foo %}x{% endblock %}{% endmacro %}{{ x() }}" + ) + assert t.render() == "x" + + def test_scoped_block(self, env): + t = env.from_string( + "{% set x = 1 %}{% with x = 2 %}{% block y scoped %}" + "{{ x }}{% endblock %}{% endwith %}" + ) + assert t.render() == "2" + + def test_recursive_loop_filter(self, env): + t = env.from_string( + """ + <?xml version="1.0" encoding="UTF-8"?> + <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> + {%- for page in [site.root] if page.url != this recursive %} + <url><loc>{{ page.url }}</loc></url> + {{- loop(page.children) }} + {%- endfor %} + </urlset> + """ + ) + sm = t.render( + this="/foo", + site={"root": {"url": "/", "children": [{"url": "/foo"}, {"url": "/bar"}]}}, + ) + lines = [x.strip() for x in sm.splitlines() if x.strip()] + assert lines == [ + '<?xml version="1.0" encoding="UTF-8"?>', + '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">', + "<url><loc>/</loc></url>", + "<url><loc>/bar</loc></url>", + "</urlset>", + ] + + def test_empty_if(self, env): + t = env.from_string("{% if foo %}{% else %}42{% endif %}") + assert t.render(foo=False) == "42" + + def test_subproperty_if(self, env): + t = env.from_string( + "{% if object1.subproperty1 is eq object2.subproperty2 %}42{% endif %}" + ) + assert ( + t.render( + object1={"subproperty1": "value"}, object2={"subproperty2": "value"} + ) + == "42" + ) + + def test_set_and_include(self): + env = Environment( + loader=DictLoader( + { + "inc": "bar", + "main": '{% set foo = "foo" %}{{ foo }}{% include "inc" %}', + } + ) + ) + assert env.get_template("main").render() == "foobar" + + def test_loop_include(self): + env = Environment( + loader=DictLoader( + { + "inc": "{{ i }}", + "main": '{% for i in [1, 2, 3] %}{% include "inc" %}{% endfor %}', + } + ) + ) + assert env.get_template("main").render() == "123" + + def test_grouper_repr(self): + from jinja2.filters import _GroupTuple + + t = _GroupTuple("foo", [1, 2]) + assert t.grouper == "foo" + assert t.list == [1, 2] + assert repr(t) == "('foo', [1, 2])" + assert str(t) == "('foo', [1, 2])" + + def test_custom_context(self, env): + from jinja2.runtime import Context + + class MyContext(Context): + pass + + class MyEnvironment(Environment): + context_class = MyContext + + loader = DictLoader({"base": "{{ foobar }}", "test": '{% extends "base" %}'}) + env = MyEnvironment(loader=loader) + assert env.get_template("test").render(foobar="test") == "test" + + def test_legacy_custom_context(self, env): + from jinja2.runtime import Context, missing + + class MyContext(Context): + def resolve(self, name): + if name == "foo": + return 42 + return super().resolve(name) + + x = MyContext(env, parent={"bar": 23}, name="foo", blocks={}) + assert x._legacy_resolve_mode + assert x.resolve_or_missing("foo") == 42 + assert x.resolve_or_missing("bar") == 23 + assert x.resolve_or_missing("baz") is missing + + def test_recursive_loop_bug(self, env): + tmpl = env.from_string( + "{%- for value in values recursive %}1{% else %}0{% endfor -%}" + ) + assert tmpl.render(values=[]) == "0" + + def test_markup_and_chainable_undefined(self): + from jinja2 import Markup + from jinja2.runtime import ChainableUndefined + + assert str(Markup(ChainableUndefined())) == "" diff --git a/tests/test_runtime.py b/tests/test_runtime.py new file mode 100644 index 00000000..db95899d --- /dev/null +++ b/tests/test_runtime.py @@ -0,0 +1,75 @@ +import itertools + +from jinja2 import Template +from jinja2.runtime import LoopContext + +TEST_IDX_TEMPLATE_STR_1 = ( + "[{% for i in lst|reverse %}(len={{ loop.length }}," + " revindex={{ loop.revindex }}, index={{ loop.index }}, val={{ i }}){% endfor %}]" +) +TEST_IDX0_TEMPLATE_STR_1 = ( + "[{% for i in lst|reverse %}(len={{ loop.length }}," + " revindex0={{ loop.revindex0 }}, index0={{ loop.index0 }}, val={{ i }})" + "{% endfor %}]" +) + + +def test_loop_idx(): + t = Template(TEST_IDX_TEMPLATE_STR_1) + lst = [10] + excepted_render = "[(len=1, revindex=1, index=1, val=10)]" + assert excepted_render == t.render(lst=lst) + + +def test_loop_idx0(): + t = Template(TEST_IDX0_TEMPLATE_STR_1) + lst = [10] + excepted_render = "[(len=1, revindex0=0, index0=0, val=10)]" + assert excepted_render == t.render(lst=lst) + + +def test_loopcontext0(): + in_lst = [] + lc = LoopContext(reversed(in_lst), None) + assert lc.length == len(in_lst) + + +def test_loopcontext1(): + in_lst = [10] + lc = LoopContext(reversed(in_lst), None) + assert lc.length == len(in_lst) + + +def test_loopcontext2(): + in_lst = [10, 11] + lc = LoopContext(reversed(in_lst), None) + assert lc.length == len(in_lst) + + +def test_iterator_not_advanced_early(): + t = Template("{% for _, g in gs %}{{ loop.index }} {{ g|list }}\n{% endfor %}") + out = t.render( + gs=itertools.groupby([(1, "a"), (1, "b"), (2, "c"), (3, "d")], lambda x: x[0]) + ) + # groupby groups depend on the current position of the iterator. If + # it was advanced early, the lists would appear empty. + assert out == "1 [(1, 'a'), (1, 'b')]\n2 [(2, 'c')]\n3 [(3, 'd')]\n" + + +def test_mock_not_contextfunction(): + """If a callable class has a ``__getattr__`` that returns True-like + values for arbitrary attrs, it should not be incorrectly identified + as a ``contextfunction``. + """ + + class Calc: + def __getattr__(self, item): + return object() + + def __call__(self, *args, **kwargs): + return len(args) + len(kwargs) + + t = Template("{{ calc() }}") + out = t.render(calc=Calc()) + # Would be "1" if context argument was passed. + assert out == "0" diff --git a/tests/test_security.py b/tests/test_security.py new file mode 100644 index 00000000..1b64cd37 --- /dev/null +++ b/tests/test_security.py @@ -0,0 +1,173 @@ +import pytest + +from jinja2 import Environment +from jinja2 import escape +from jinja2.exceptions import SecurityError +from jinja2.exceptions import TemplateRuntimeError +from jinja2.exceptions import TemplateSyntaxError +from jinja2.nodes import EvalContext +from jinja2.sandbox import ImmutableSandboxedEnvironment +from jinja2.sandbox import SandboxedEnvironment +from jinja2.sandbox import unsafe + + +class PrivateStuff: + def bar(self): + return 23 + + @unsafe + def foo(self): + return 42 + + def __repr__(self): + return "PrivateStuff" + + +class PublicStuff: + def bar(self): + return 23 + + def _foo(self): + return 42 + + def __repr__(self): + return "PublicStuff" + + +class TestSandbox: + def test_unsafe(self, env): + env = SandboxedEnvironment() + pytest.raises( + SecurityError, env.from_string("{{ foo.foo() }}").render, foo=PrivateStuff() + ) + assert env.from_string("{{ foo.bar() }}").render(foo=PrivateStuff()) == "23" + + pytest.raises( + SecurityError, env.from_string("{{ foo._foo() }}").render, foo=PublicStuff() + ) + assert env.from_string("{{ foo.bar() }}").render(foo=PublicStuff()) == "23" + assert env.from_string("{{ foo.__class__ }}").render(foo=42) == "" + assert env.from_string("{{ foo.func_code }}").render(foo=lambda: None) == "" + # security error comes from __class__ already. + pytest.raises( + SecurityError, + env.from_string("{{ foo.__class__.__subclasses__() }}").render, + foo=42, + ) + + def test_immutable_environment(self, env): + env = ImmutableSandboxedEnvironment() + pytest.raises(SecurityError, env.from_string("{{ [].append(23) }}").render) + pytest.raises(SecurityError, env.from_string("{{ {1:2}.clear() }}").render) + + def test_restricted(self, env): + env = SandboxedEnvironment() + pytest.raises( + TemplateSyntaxError, + env.from_string, + "{% for item.attribute in seq %}...{% endfor %}", + ) + pytest.raises( + TemplateSyntaxError, + env.from_string, + "{% for foo, bar.baz in seq %}...{% endfor %}", + ) + + def test_template_data(self, env): + env = Environment(autoescape=True) + t = env.from_string( + "{% macro say_hello(name) %}" + "<p>Hello {{ name }}!</p>{% endmacro %}" + '{{ say_hello("<blink>foo</blink>") }}' + ) + escaped_out = "<p>Hello <blink>foo</blink>!</p>" + assert t.render() == escaped_out + assert str(t.module) == escaped_out + assert escape(t.module) == escaped_out + assert t.module.say_hello("<blink>foo</blink>") == escaped_out + assert ( + escape(t.module.say_hello(EvalContext(env), "<blink>foo</blink>")) + == escaped_out + ) + assert escape(t.module.say_hello("<blink>foo</blink>")) == escaped_out + + def test_attr_filter(self, env): + env = SandboxedEnvironment() + tmpl = env.from_string('{{ cls|attr("__subclasses__")() }}') + pytest.raises(SecurityError, tmpl.render, cls=int) + + def test_binary_operator_intercepting(self, env): + def disable_op(left, right): + raise TemplateRuntimeError("that operator so does not work") + + for expr, ctx, rv in ("1 + 2", {}, "3"), ("a + 2", {"a": 2}, "4"): + env = SandboxedEnvironment() + env.binop_table["+"] = disable_op + t = env.from_string(f"{{{{ {expr} }}}}") + assert t.render(ctx) == rv + env.intercepted_binops = frozenset(["+"]) + t = env.from_string(f"{{{{ {expr} }}}}") + with pytest.raises(TemplateRuntimeError): + t.render(ctx) + + def test_unary_operator_intercepting(self, env): + def disable_op(arg): + raise TemplateRuntimeError("that operator so does not work") + + for expr, ctx, rv in ("-1", {}, "-1"), ("-a", {"a": 2}, "-2"): + env = SandboxedEnvironment() + env.unop_table["-"] = disable_op + t = env.from_string(f"{{{{ {expr} }}}}") + assert t.render(ctx) == rv + env.intercepted_unops = frozenset(["-"]) + t = env.from_string(f"{{{{ {expr} }}}}") + with pytest.raises(TemplateRuntimeError): + t.render(ctx) + + +class TestStringFormat: + def test_basic_format_safety(self): + env = SandboxedEnvironment() + t = env.from_string('{{ "a{0.__class__}b".format(42) }}') + assert t.render() == "ab" + + def test_basic_format_all_okay(self): + env = SandboxedEnvironment() + t = env.from_string('{{ "a{0.foo}b".format({"foo": 42}) }}') + assert t.render() == "a42b" + + def test_safe_format_safety(self): + env = SandboxedEnvironment() + t = env.from_string('{{ ("a{0.__class__}b{1}"|safe).format(42, "<foo>") }}') + assert t.render() == "ab<foo>" + + def test_safe_format_all_okay(self): + env = SandboxedEnvironment() + t = env.from_string('{{ ("a{0.foo}b{1}"|safe).format({"foo": 42}, "<foo>") }}') + assert t.render() == "a42b<foo>" + + def test_empty_braces_format(self): + env = SandboxedEnvironment() + t1 = env.from_string('{{ ("a{}b{}").format("foo", "42")}}') + t2 = env.from_string('{{ ("a{}b{}"|safe).format(42, "<foo>") }}') + assert t1.render() == "afoob42" + assert t2.render() == "a42b<foo>" + + +class TestStringFormatMap: + def test_basic_format_safety(self): + env = SandboxedEnvironment() + t = env.from_string('{{ "a{x.__class__}b".format_map({"x":42}) }}') + assert t.render() == "ab" + + def test_basic_format_all_okay(self): + env = SandboxedEnvironment() + t = env.from_string('{{ "a{x.foo}b".format_map({"x":{"foo": 42}}) }}') + assert t.render() == "a42b" + + def test_safe_format_all_okay(self): + env = SandboxedEnvironment() + t = env.from_string( + '{{ ("a{x.foo}b{y}"|safe).format_map({"x":{"foo": 42}, "y":"<foo>"}) }}' + ) + assert t.render() == "a42b<foo>" diff --git a/tests/test_tests.py b/tests/test_tests.py new file mode 100644 index 00000000..d363653d --- /dev/null +++ b/tests/test_tests.py @@ -0,0 +1,208 @@ +import pytest + +from jinja2 import Environment +from jinja2 import Markup + + +class MyDict(dict): + pass + + +class TestTestsCase: + def test_defined(self, env): + tmpl = env.from_string("{{ missing is defined }}|{{ true is defined }}") + assert tmpl.render() == "False|True" + + def test_even(self, env): + tmpl = env.from_string("""{{ 1 is even }}|{{ 2 is even }}""") + assert tmpl.render() == "False|True" + + def test_odd(self, env): + tmpl = env.from_string("""{{ 1 is odd }}|{{ 2 is odd }}""") + assert tmpl.render() == "True|False" + + def test_lower(self, env): + tmpl = env.from_string("""{{ "foo" is lower }}|{{ "FOO" is lower }}""") + assert tmpl.render() == "True|False" + + # Test type checks + @pytest.mark.parametrize( + "op,expect", + ( + ("none is none", True), + ("false is none", False), + ("true is none", False), + ("42 is none", False), + ("none is true", False), + ("false is true", False), + ("true is true", True), + ("0 is true", False), + ("1 is true", False), + ("42 is true", False), + ("none is false", False), + ("false is false", True), + ("true is false", False), + ("0 is false", False), + ("1 is false", False), + ("42 is false", False), + ("none is boolean", False), + ("false is boolean", True), + ("true is boolean", True), + ("0 is boolean", False), + ("1 is boolean", False), + ("42 is boolean", False), + ("0.0 is boolean", False), + ("1.0 is boolean", False), + ("3.14159 is boolean", False), + ("none is integer", False), + ("false is integer", False), + ("true is integer", False), + ("42 is integer", True), + ("3.14159 is integer", False), + ("(10 ** 100) is integer", True), + ("none is float", False), + ("false is float", False), + ("true is float", False), + ("42 is float", False), + ("4.2 is float", True), + ("(10 ** 100) is float", False), + ("none is number", False), + ("false is number", True), + ("true is number", True), + ("42 is number", True), + ("3.14159 is number", True), + ("complex is number", True), + ("(10 ** 100) is number", True), + ("none is string", False), + ("false is string", False), + ("true is string", False), + ("42 is string", False), + ('"foo" is string', True), + ("none is sequence", False), + ("false is sequence", False), + ("42 is sequence", False), + ('"foo" is sequence', True), + ("[] is sequence", True), + ("[1, 2, 3] is sequence", True), + ("{} is sequence", True), + ("none is mapping", False), + ("false is mapping", False), + ("42 is mapping", False), + ('"foo" is mapping', False), + ("[] is mapping", False), + ("{} is mapping", True), + ("mydict is mapping", True), + ("none is iterable", False), + ("false is iterable", False), + ("42 is iterable", False), + ('"foo" is iterable', True), + ("[] is iterable", True), + ("{} is iterable", True), + ("range(5) is iterable", True), + ("none is callable", False), + ("false is callable", False), + ("42 is callable", False), + ('"foo" is callable', False), + ("[] is callable", False), + ("{} is callable", False), + ("range is callable", True), + ), + ) + def test_types(self, env, op, expect): + t = env.from_string(f"{{{{ {op} }}}}") + assert t.render(mydict=MyDict(), complex=complex(1, 2)) == str(expect) + + def test_upper(self, env): + tmpl = env.from_string('{{ "FOO" is upper }}|{{ "foo" is upper }}') + assert tmpl.render() == "True|False" + + def test_equalto(self, env): + tmpl = env.from_string( + "{{ foo is eq 12 }}|" + "{{ foo is eq 0 }}|" + "{{ foo is eq (3 * 4) }}|" + '{{ bar is eq "baz" }}|' + '{{ bar is eq "zab" }}|' + '{{ bar is eq ("ba" + "z") }}|' + "{{ bar is eq bar }}|" + "{{ bar is eq foo }}" + ) + assert ( + tmpl.render(foo=12, bar="baz") + == "True|False|True|True|False|True|True|False" + ) + + @pytest.mark.parametrize( + "op,expect", + ( + ("eq 2", True), + ("eq 3", False), + ("ne 3", True), + ("ne 2", False), + ("lt 3", True), + ("lt 2", False), + ("le 2", True), + ("le 1", False), + ("gt 1", True), + ("gt 2", False), + ("ge 2", True), + ("ge 3", False), + ), + ) + def test_compare_aliases(self, env, op, expect): + t = env.from_string(f"{{{{ 2 is {op} }}}}") + assert t.render() == str(expect) + + def test_sameas(self, env): + tmpl = env.from_string("{{ foo is sameas false }}|{{ 0 is sameas false }}") + assert tmpl.render(foo=False) == "True|False" + + def test_no_paren_for_arg1(self, env): + tmpl = env.from_string("{{ foo is sameas none }}") + assert tmpl.render(foo=None) == "True" + + def test_escaped(self, env): + env = Environment(autoescape=True) + tmpl = env.from_string("{{ x is escaped }}|{{ y is escaped }}") + assert tmpl.render(x="foo", y=Markup("foo")) == "False|True" + + def test_greaterthan(self, env): + tmpl = env.from_string("{{ 1 is greaterthan 0 }}|{{ 0 is greaterthan 1 }}") + assert tmpl.render() == "True|False" + + def test_lessthan(self, env): + tmpl = env.from_string("{{ 0 is lessthan 1 }}|{{ 1 is lessthan 0 }}") + assert tmpl.render() == "True|False" + + def test_multiple_tests(self): + items = [] + + def matching(x, y): + items.append((x, y)) + return False + + env = Environment() + env.tests["matching"] = matching + tmpl = env.from_string( + "{{ 'us-west-1' is matching '(us-east-1|ap-northeast-1)'" + " or 'stage' is matching '(dev|stage)' }}" + ) + assert tmpl.render() == "False" + assert items == [ + ("us-west-1", "(us-east-1|ap-northeast-1)"), + ("stage", "(dev|stage)"), + ] + + def test_in(self, env): + tmpl = env.from_string( + '{{ "o" is in "foo" }}|' + '{{ "foo" is in "foo" }}|' + '{{ "b" is in "foo" }}|' + "{{ 1 is in ((1, 2)) }}|" + "{{ 3 is in ((1, 2)) }}|" + "{{ 1 is in [1, 2] }}|" + "{{ 3 is in [1, 2] }}|" + '{{ "foo" is in {"foo": 1}}}|' + '{{ "baz" is in {"bar": 1}}}' + ) + assert tmpl.render() == "True|True|False|True|False|True|False|True|False" diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000..3e24166c --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,190 @@ +import pickle +import random +from collections import deque +from copy import copy as shallow_copy + +import pytest +from markupsafe import Markup + +from jinja2.utils import consume +from jinja2.utils import generate_lorem_ipsum +from jinja2.utils import LRUCache +from jinja2.utils import missing +from jinja2.utils import object_type_repr +from jinja2.utils import select_autoescape +from jinja2.utils import urlize + + +class TestLRUCache: + def test_simple(self): + d = LRUCache(3) + d["a"] = 1 + d["b"] = 2 + d["c"] = 3 + d["a"] + d["d"] = 4 + assert len(d) == 3 + assert "a" in d and "c" in d and "d" in d and "b" not in d + + def test_itervalues(self): + cache = LRUCache(3) + cache["b"] = 1 + cache["a"] = 2 + values = [v for v in cache.values()] + assert len(values) == 2 + assert 1 in values + assert 2 in values + + def test_itervalues_empty(self): + cache = LRUCache(2) + values = [v for v in cache.values()] + assert len(values) == 0 + + def test_pickleable(self): + cache = LRUCache(2) + cache["foo"] = 42 + cache["bar"] = 23 + cache["foo"] + + for protocol in range(3): + copy = pickle.loads(pickle.dumps(cache, protocol)) + assert copy.capacity == cache.capacity + assert copy._mapping == cache._mapping + assert copy._queue == cache._queue + + @pytest.mark.parametrize("copy_func", [LRUCache.copy, shallow_copy]) + def test_copy(self, copy_func): + cache = LRUCache(2) + cache["a"] = 1 + cache["b"] = 2 + copy = copy_func(cache) + assert copy._queue == cache._queue + copy["c"] = 3 + assert copy._queue != cache._queue + assert "a" not in copy and "b" in copy and "c" in copy + + def test_clear(self): + d = LRUCache(3) + d["a"] = 1 + d["b"] = 2 + d["c"] = 3 + d.clear() + assert d.__getstate__() == {"capacity": 3, "_mapping": {}, "_queue": deque([])} + + def test_repr(self): + d = LRUCache(3) + d["a"] = 1 + d["b"] = 2 + d["c"] = 3 + # Sort the strings - mapping is unordered + assert sorted(repr(d)) == sorted("<LRUCache {'a': 1, 'b': 2, 'c': 3}>") + + def test_items(self): + """Test various items, keys, values and iterators of LRUCache.""" + d = LRUCache(3) + d["a"] = 1 + d["b"] = 2 + d["c"] = 3 + assert d.items() == [("c", 3), ("b", 2), ("a", 1)] + assert d.keys() == ["c", "b", "a"] + assert d.values() == [3, 2, 1] + assert list(reversed(d)) == ["a", "b", "c"] + + # Change the cache a little + d["b"] + d["a"] = 4 + assert d.items() == [("a", 4), ("b", 2), ("c", 3)] + assert d.keys() == ["a", "b", "c"] + assert d.values() == [4, 2, 3] + assert list(reversed(d)) == ["c", "b", "a"] + + def test_setdefault(self): + d = LRUCache(3) + assert len(d) == 0 + assert d.setdefault("a") is None + assert d.setdefault("a", 1) is None + assert len(d) == 1 + assert d.setdefault("b", 2) == 2 + assert len(d) == 2 + + +class TestHelpers: + def test_object_type_repr(self): + class X: + pass + + assert object_type_repr(42) == "int object" + assert object_type_repr([]) == "list object" + assert object_type_repr(X()) == "test_utils.X object" + assert object_type_repr(None) == "None" + assert object_type_repr(Ellipsis) == "Ellipsis" + + def test_autoescape_select(self): + func = select_autoescape( + enabled_extensions=("html", ".htm"), + disabled_extensions=("txt",), + default_for_string="STRING", + default="NONE", + ) + + assert func(None) == "STRING" + assert func("unknown.foo") == "NONE" + assert func("foo.html") + assert func("foo.htm") + assert not func("foo.txt") + assert func("FOO.HTML") + assert not func("FOO.TXT") + + +class TestEscapeUrlizeTarget: + def test_escape_urlize_target(self): + url = "http://example.org" + target = "<script>" + assert urlize(url, target=target) == ( + '<a href="http://example.org"' + ' target="<script>">' + "http://example.org</a>" + ) + + +class TestLoremIpsum: + def test_lorem_ipsum_markup(self): + """Test that output of lorem_ipsum is Markup by default.""" + assert isinstance(generate_lorem_ipsum(), Markup) + + def test_lorem_ipsum_html(self): + """Test that output of lorem_ipsum is a string_type when not html.""" + assert isinstance(generate_lorem_ipsum(html=False), str) + + def test_lorem_ipsum_n(self): + """Test that the n (number of lines) works as expected.""" + assert generate_lorem_ipsum(n=0, html=False) == "" + for n in range(1, 50): + assert generate_lorem_ipsum(n=n, html=False).count("\n") == (n - 1) * 2 + + def test_lorem_ipsum_min(self): + """Test that at least min words are in the output of each line""" + for _ in range(5): + m = random.randrange(20, 99) + for _ in range(10): + assert generate_lorem_ipsum(n=1, min=m, html=False).count(" ") >= m - 1 + + def test_lorem_ipsum_max(self): + """Test that at least max words are in the output of each line""" + for _ in range(5): + m = random.randrange(21, 100) + for _ in range(10): + assert generate_lorem_ipsum(n=1, max=m, html=False).count(" ") < m - 1 + + +def test_missing(): + """Test the repr of missing.""" + assert repr(missing) == "missing" + + +def test_consume(): + """Test that consume consumes an iterator.""" + x = iter([1, 2, 3, 4, 5]) + consume(x) + with pytest.raises(StopIteration): + next(x) diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..9b6d4713 --- /dev/null +++ b/tox.ini @@ -0,0 +1,19 @@ +[tox] +envlist = + py{38,37,36,py3} + style + docs +skip_missing_interpreters = true + +[testenv] +deps = -r requirements/tests.txt +commands = pytest --tb=short --basetemp={envtmpdir} {posargs} + +[testenv:style] +deps = pre-commit +skip_install = true +commands = pre-commit run --all-files --show-diff-on-failure + +[testenv:docs] +deps = -r requirements/docs.txt +commands = sphinx-build -W -b html -d {envtmpdir}/doctrees docs {envtmpdir}/html |
