diff --git a/django/template/base.py b/django/template/base.py index 4a292c55a7..9dafd3eb59 100644 --- a/django/template/base.py +++ b/django/template/base.py @@ -3,9 +3,10 @@ This is the Django template system. How it works: -The Lexer.tokenize() function converts a template string (i.e., a string containing -markup with custom template tags) to tokens, which can be either plain text -(TOKEN_TEXT), variables (TOKEN_VAR) or block statements (TOKEN_BLOCK). +The Lexer.tokenize() method converts a template string (i.e., a string +containing markup with custom template tags) to tokens, which can be either +plain text (TokenType.TEXT), variables (TokenType.VAR), or block statements +(TokenType.BLOCK). The Parser() class takes a list of tokens in its constructor, and its parse() method returns a compiled template -- which is, under the hood, a list of @@ -51,6 +52,7 @@ times with multiple contexts) import logging import re +from enum import Enum from inspect import getcallargs, getfullargspec from django.template.context import ( # NOQA: imported for backwards compatibility @@ -67,17 +69,6 @@ from django.utils.translation import gettext_lazy, pgettext_lazy from .exceptions import TemplateSyntaxError -TOKEN_TEXT = 0 -TOKEN_VAR = 1 -TOKEN_BLOCK = 2 -TOKEN_COMMENT = 3 -TOKEN_MAPPING = { - TOKEN_TEXT: 'Text', - TOKEN_VAR: 'Var', - TOKEN_BLOCK: 'Block', - TOKEN_COMMENT: 'Comment', -} - # template syntax constants FILTER_SEPARATOR = '|' FILTER_ARGUMENT_SEPARATOR = ':' @@ -106,6 +97,13 @@ tag_re = (re.compile('(%s.*?%s|%s.*?%s|%s.*?%s)' % logger = logging.getLogger('django.template') +class TokenType(Enum): + TEXT = 0 + VAR = 1 + BLOCK = 2 + COMMENT = 3 + + class VariableDoesNotExist(Exception): def __init__(self, msg, params=()): @@ -293,7 +291,7 @@ class Token: A token representing a string from the template. token_type - One of TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK, or TOKEN_COMMENT. + A TokenType, either .TEXT, .VAR, .BLOCK, or .COMMENT. contents The token source string. @@ -312,7 +310,7 @@ class Token: self.position = position def __str__(self): - token_name = TOKEN_MAPPING[self.token_type] + token_name = self.token_type.name.capitalize() return ('<%s token: "%s...">' % (token_name, self.contents[:20].replace('\n', ''))) @@ -367,18 +365,18 @@ class Lexer: self.verbatim = False if in_tag and not self.verbatim: if token_string.startswith(VARIABLE_TAG_START): - return Token(TOKEN_VAR, token_string[2:-2].strip(), position, lineno) + return Token(TokenType.VAR, token_string[2:-2].strip(), position, lineno) elif token_string.startswith(BLOCK_TAG_START): if block_content[:9] in ('verbatim', 'verbatim '): self.verbatim = 'end%s' % block_content - return Token(TOKEN_BLOCK, block_content, position, lineno) + return Token(TokenType.BLOCK, block_content, position, lineno) elif token_string.startswith(COMMENT_TAG_START): content = '' if token_string.find(TRANSLATOR_COMMENT_MARK): content = token_string[2:-2].strip() - return Token(TOKEN_COMMENT, content, position, lineno) + return Token(TokenType.COMMENT, content, position, lineno) else: - return Token(TOKEN_TEXT, token_string, position, lineno) + return Token(TokenType.TEXT, token_string, position, lineno) class DebugLexer(Lexer): @@ -439,10 +437,10 @@ class Parser: nodelist = NodeList() while self.tokens: token = self.next_token() - # Use the raw values here for TOKEN_* for a tiny performance boost. - if token.token_type == 0: # TOKEN_TEXT + # Use the raw values here for TokenType.* for a tiny performance boost. + if token.token_type.value == 0: # TokenType.TEXT self.extend_nodelist(nodelist, TextNode(token.contents), token) - elif token.token_type == 1: # TOKEN_VAR + elif token.token_type.value == 1: # TokenType.VAR if not token.contents: raise self.error(token, 'Empty variable tag on line %d' % token.lineno) try: @@ -451,7 +449,7 @@ class Parser: raise self.error(token, e) var_node = VariableNode(filter_expression) self.extend_nodelist(nodelist, var_node, token) - elif token.token_type == 2: # TOKEN_BLOCK + elif token.token_type.value == 2: # TokenType.BLOCK try: command = token.contents.split()[0] except IndexError: @@ -488,7 +486,7 @@ class Parser: def skip_past(self, endtag): while self.tokens: token = self.next_token() - if token.token_type == TOKEN_BLOCK and token.contents == endtag: + if token.token_type == TokenType.BLOCK and token.contents == endtag: return self.unclosed_block_tag([endtag]) diff --git a/django/templatetags/i18n.py b/django/templatetags/i18n.py index b949e459f0..a1dc6e5fd4 100644 --- a/django/templatetags/i18n.py +++ b/django/templatetags/i18n.py @@ -1,6 +1,6 @@ from django.conf import settings from django.template import Library, Node, TemplateSyntaxError, Variable -from django.template.base import TOKEN_TEXT, TOKEN_VAR, render_value_in_context +from django.template.base import TokenType, render_value_in_context from django.template.defaulttags import token_kwargs from django.utils import translation from django.utils.safestring import SafeData, mark_safe @@ -112,9 +112,9 @@ class BlockTranslateNode(Node): result = [] vars = [] for token in tokens: - if token.token_type == TOKEN_TEXT: + if token.token_type == TokenType.TEXT: result.append(token.contents.replace('%', '%%')) - elif token.token_type == TOKEN_VAR: + elif token.token_type == TokenType.VAR: result.append('%%(%s)s' % token.contents) vars.append(token.contents) msg = ''.join(result) @@ -510,7 +510,7 @@ def do_block_translate(parser, token): plural = [] while parser.tokens: token = parser.next_token() - if token.token_type in (TOKEN_VAR, TOKEN_TEXT): + if token.token_type in (TokenType.VAR, TokenType.TEXT): singular.append(token) else: break @@ -519,7 +519,7 @@ def do_block_translate(parser, token): raise TemplateSyntaxError("'blocktrans' doesn't allow other block tags inside it") while parser.tokens: token = parser.next_token() - if token.token_type in (TOKEN_VAR, TOKEN_TEXT): + if token.token_type in (TokenType.VAR, TokenType.TEXT): plural.append(token) else: break diff --git a/django/utils/translation/template.py b/django/utils/translation/template.py index 3659fab3b6..aa849b0937 100644 --- a/django/utils/translation/template.py +++ b/django/utils/translation/template.py @@ -2,10 +2,7 @@ import re import warnings from io import StringIO -from django.template.base import ( - TOKEN_BLOCK, TOKEN_COMMENT, TOKEN_TEXT, TOKEN_VAR, TRANSLATOR_COMMENT_MARK, - Lexer, -) +from django.template.base import TRANSLATOR_COMMENT_MARK, Lexer, TokenType from . import TranslatorCommentWarning, trim_whitespace @@ -63,7 +60,7 @@ def templatize(src, origin=None): for t in Lexer(src).tokenize(): if incomment: - if t.token_type == TOKEN_BLOCK and t.contents == 'endcomment': + if t.token_type == TokenType.BLOCK and t.contents == 'endcomment': content = ''.join(comment) translators_comment_start = None for lineno, line in enumerate(content.splitlines(True)): @@ -79,7 +76,7 @@ def templatize(src, origin=None): else: comment.append(t.contents) elif intrans: - if t.token_type == TOKEN_BLOCK: + if t.token_type == TokenType.BLOCK: endbmatch = endblock_re.match(t.contents) pluralmatch = plural_re.match(t.contents) if endbmatch: @@ -130,12 +127,12 @@ def templatize(src, origin=None): "Translation blocks must not include other block tags: " "%s (%sline %d)" % (t.contents, filemsg, t.lineno) ) - elif t.token_type == TOKEN_VAR: + elif t.token_type == TokenType.VAR: if inplural: plural.append('%%(%s)s' % t.contents) else: singular.append('%%(%s)s' % t.contents) - elif t.token_type == TOKEN_TEXT: + elif t.token_type == TokenType.TEXT: contents = t.contents.replace('%', '%%') if inplural: plural.append(contents) @@ -147,7 +144,7 @@ def templatize(src, origin=None): if comment_lineno_cache is not None: cur_lineno = t.lineno + t.contents.count('\n') if comment_lineno_cache == cur_lineno: - if t.token_type != TOKEN_COMMENT: + if t.token_type != TokenType.COMMENT: for c in lineno_comment_map[comment_lineno_cache]: filemsg = '' if origin: @@ -163,7 +160,7 @@ def templatize(src, origin=None): out.write('# %s' % ' | '.join(lineno_comment_map[comment_lineno_cache])) comment_lineno_cache = None - if t.token_type == TOKEN_BLOCK: + if t.token_type == TokenType.BLOCK: imatch = inline_re.match(t.contents) bmatch = block_re.match(t.contents) cmatches = constant_re.findall(t.contents) @@ -211,7 +208,7 @@ def templatize(src, origin=None): incomment = True else: out.write(blankout(t.contents, 'B')) - elif t.token_type == TOKEN_VAR: + elif t.token_type == TokenType.VAR: parts = t.contents.split('|') cmatch = constant_re.match(parts[0]) if cmatch: @@ -221,7 +218,7 @@ def templatize(src, origin=None): out.write(' %s ' % p.split(':', 1)[1]) else: out.write(blankout(p, 'F')) - elif t.token_type == TOKEN_COMMENT: + elif t.token_type == TokenType.COMMENT: if t.contents.lstrip().startswith(TRANSLATOR_COMMENT_MARK): lineno_comment_map.setdefault(t.lineno, []).append(t.contents) comment_lineno_cache = t.lineno diff --git a/tests/template_tests/test_parser.py b/tests/template_tests/test_parser.py index 3dc731e0a9..2370e0263e 100644 --- a/tests/template_tests/test_parser.py +++ b/tests/template_tests/test_parser.py @@ -3,7 +3,7 @@ Testing some internals of the template processing. These are *not* examples to b """ from django.template import Library, TemplateSyntaxError from django.template.base import ( - TOKEN_BLOCK, FilterExpression, Parser, Token, Variable, + FilterExpression, Parser, Token, TokenType, Variable, ) from django.template.defaultfilters import register as filter_library from django.test import SimpleTestCase @@ -15,7 +15,7 @@ class ParserTests(SimpleTestCase): """ #7027 -- _() syntax should work with spaces """ - token = Token(TOKEN_BLOCK, 'sometag _("Page not found") value|yesno:_("yes,no")') + token = Token(TokenType.BLOCK, 'sometag _("Page not found") value|yesno:_("yes,no")') split = token.split_contents() self.assertEqual(split, ["sometag", '_("Page not found")', 'value|yesno:_("yes,no")'])