mirror of
https://github.com/django/django.git
synced 2024-12-22 17:16:24 +00:00
401 lines
12 KiB
Python
401 lines
12 KiB
Python
"""Tests for jslex."""
|
||
# originally from https://bitbucket.org/ned/jslex
|
||
|
||
from django.test import SimpleTestCase
|
||
from django.utils.jslex import JsLexer, prepare_js_for_gettext
|
||
|
||
|
||
class JsTokensTest(SimpleTestCase):
|
||
LEX_CASES = [
|
||
# ids
|
||
("a ABC $ _ a123", ["id a", "id ABC", "id $", "id _", "id a123"]),
|
||
(
|
||
"\\u1234 abc\\u0020 \\u0065_\\u0067",
|
||
["id \\u1234", "id abc\\u0020", "id \\u0065_\\u0067"],
|
||
),
|
||
# numbers
|
||
(
|
||
"123 1.234 0.123e-3 0 1E+40 1e1 .123",
|
||
[
|
||
"dnum 123",
|
||
"dnum 1.234",
|
||
"dnum 0.123e-3",
|
||
"dnum 0",
|
||
"dnum 1E+40",
|
||
"dnum 1e1",
|
||
"dnum .123",
|
||
],
|
||
),
|
||
("0x1 0xabCD 0XABcd", ["hnum 0x1", "hnum 0xabCD", "hnum 0XABcd"]),
|
||
("010 0377 090", ["onum 010", "onum 0377", "dnum 0", "dnum 90"]),
|
||
("0xa123ghi", ["hnum 0xa123", "id ghi"]),
|
||
# keywords
|
||
(
|
||
"function Function FUNCTION",
|
||
["keyword function", "id Function", "id FUNCTION"],
|
||
),
|
||
(
|
||
"const constructor in inherits",
|
||
["keyword const", "id constructor", "keyword in", "id inherits"],
|
||
),
|
||
("true true_enough", ["reserved true", "id true_enough"]),
|
||
# strings
|
||
(""" 'hello' "hello" """, ["string 'hello'", 'string "hello"']),
|
||
(
|
||
r""" 'don\'t' "don\"t" '"' "'" '\'' "\"" """,
|
||
[
|
||
r"""string 'don\'t'""",
|
||
r'''string "don\"t"''',
|
||
r"""string '"'""",
|
||
r'''string "'"''',
|
||
r"""string '\''""",
|
||
r'''string "\""''',
|
||
],
|
||
),
|
||
(r'"ƃuıxǝ⅂ ʇdıɹɔsɐʌɐſ\""', [r'string "ƃuıxǝ⅂ ʇdıɹɔsɐʌɐſ\""']),
|
||
# comments
|
||
("a//b", ["id a", "linecomment //b"]),
|
||
(
|
||
"/****/a/=2//hello",
|
||
["comment /****/", "id a", "punct /=", "dnum 2", "linecomment //hello"],
|
||
),
|
||
(
|
||
"/*\n * Header\n */\na=1;",
|
||
["comment /*\n * Header\n */", "id a", "punct =", "dnum 1", "punct ;"],
|
||
),
|
||
# punctuation
|
||
("a+++b", ["id a", "punct ++", "punct +", "id b"]),
|
||
# regex
|
||
(r"a=/a*/,1", ["id a", "punct =", "regex /a*/", "punct ,", "dnum 1"]),
|
||
(r"a=/a*[^/]+/,1", ["id a", "punct =", "regex /a*[^/]+/", "punct ,", "dnum 1"]),
|
||
(r"a=/a*\[^/,1", ["id a", "punct =", r"regex /a*\[^/", "punct ,", "dnum 1"]),
|
||
(r"a=/\//,1", ["id a", "punct =", r"regex /\//", "punct ,", "dnum 1"]),
|
||
# next two are from https://www-archive.mozilla.org/js/language/js20-2002-04/rationale/syntax.html#regular-expressions # NOQA
|
||
(
|
||
'for (var x = a in foo && "</x>" || mot ? z:/x:3;x<5;y</g/i) {xyz(x++);}',
|
||
[
|
||
"keyword for",
|
||
"punct (",
|
||
"keyword var",
|
||
"id x",
|
||
"punct =",
|
||
"id a",
|
||
"keyword in",
|
||
"id foo",
|
||
"punct &&",
|
||
'string "</x>"',
|
||
"punct ||",
|
||
"id mot",
|
||
"punct ?",
|
||
"id z",
|
||
"punct :",
|
||
"regex /x:3;x<5;y</g",
|
||
"punct /",
|
||
"id i",
|
||
"punct )",
|
||
"punct {",
|
||
"id xyz",
|
||
"punct (",
|
||
"id x",
|
||
"punct ++",
|
||
"punct )",
|
||
"punct ;",
|
||
"punct }",
|
||
],
|
||
),
|
||
(
|
||
'for (var x = a in foo && "</x>" || mot ? z/x:3;x<5;y</g/i) {xyz(x++);}',
|
||
[
|
||
"keyword for",
|
||
"punct (",
|
||
"keyword var",
|
||
"id x",
|
||
"punct =",
|
||
"id a",
|
||
"keyword in",
|
||
"id foo",
|
||
"punct &&",
|
||
'string "</x>"',
|
||
"punct ||",
|
||
"id mot",
|
||
"punct ?",
|
||
"id z",
|
||
"punct /",
|
||
"id x",
|
||
"punct :",
|
||
"dnum 3",
|
||
"punct ;",
|
||
"id x",
|
||
"punct <",
|
||
"dnum 5",
|
||
"punct ;",
|
||
"id y",
|
||
"punct <",
|
||
"regex /g/i",
|
||
"punct )",
|
||
"punct {",
|
||
"id xyz",
|
||
"punct (",
|
||
"id x",
|
||
"punct ++",
|
||
"punct )",
|
||
"punct ;",
|
||
"punct }",
|
||
],
|
||
),
|
||
# Various "illegal" regexes that are valid according to the std.
|
||
(
|
||
r"""/????/, /++++/, /[----]/ """,
|
||
["regex /????/", "punct ,", "regex /++++/", "punct ,", "regex /[----]/"],
|
||
),
|
||
# Stress cases from https://stackoverflow.com/questions/5533925/what-javascript-constructs-does-jslex-incorrectly-lex/5573409#5573409 # NOQA
|
||
(r"""/\[/""", [r"""regex /\[/"""]),
|
||
(r"""/[i]/""", [r"""regex /[i]/"""]),
|
||
(r"""/[\]]/""", [r"""regex /[\]]/"""]),
|
||
(r"""/a[\]]/""", [r"""regex /a[\]]/"""]),
|
||
(r"""/a[\]]b/""", [r"""regex /a[\]]b/"""]),
|
||
(r"""/[\]/]/gi""", [r"""regex /[\]/]/gi"""]),
|
||
(r"""/\[[^\]]+\]/gi""", [r"""regex /\[[^\]]+\]/gi"""]),
|
||
(
|
||
r"""
|
||
rexl.re = {
|
||
NAME: /^(?![0-9])(?:\w)+|^"(?:[^"]|"")+"/,
|
||
UNQUOTED_LITERAL: /^@(?:(?![0-9])(?:\w|\:)+|^"(?:[^"]|"")+")\[[^\]]+\]/,
|
||
QUOTED_LITERAL: /^'(?:[^']|'')*'/,
|
||
NUMERIC_LITERAL: /^[0-9]+(?:\.[0-9]*(?:[eE][-+][0-9]+)?)?/,
|
||
SYMBOL: /^(?:==|=|<>|<=|<|>=|>|!~~|!~|~~|~|!==|!=|!~=|!~|!|&|\||\.|\:|,|\(|\)|\[|\]|\{|\}|\?|\:|;|@|\^|\/\+|\/|\*|\+|-)/
|
||
};
|
||
""", # NOQA
|
||
[
|
||
"id rexl",
|
||
"punct .",
|
||
"id re",
|
||
"punct =",
|
||
"punct {",
|
||
"id NAME",
|
||
"punct :",
|
||
r"""regex /^(?![0-9])(?:\w)+|^"(?:[^"]|"")+"/""",
|
||
"punct ,",
|
||
"id UNQUOTED_LITERAL",
|
||
"punct :",
|
||
r"""regex /^@(?:(?![0-9])(?:\w|\:)+|^"(?:[^"]|"")+")\[[^\]]+\]/""",
|
||
"punct ,",
|
||
"id QUOTED_LITERAL",
|
||
"punct :",
|
||
r"""regex /^'(?:[^']|'')*'/""",
|
||
"punct ,",
|
||
"id NUMERIC_LITERAL",
|
||
"punct :",
|
||
r"""regex /^[0-9]+(?:\.[0-9]*(?:[eE][-+][0-9]+)?)?/""",
|
||
"punct ,",
|
||
"id SYMBOL",
|
||
"punct :",
|
||
r"""regex /^(?:==|=|<>|<=|<|>=|>|!~~|!~|~~|~|!==|!=|!~=|!~|!|&|\||\.|\:|,|\(|\)|\[|\]|\{|\}|\?|\:|;|@|\^|\/\+|\/|\*|\+|-)/""", # NOQA
|
||
"punct }",
|
||
"punct ;",
|
||
],
|
||
),
|
||
(
|
||
r"""
|
||
rexl.re = {
|
||
NAME: /^(?![0-9])(?:\w)+|^"(?:[^"]|"")+"/,
|
||
UNQUOTED_LITERAL: /^@(?:(?![0-9])(?:\w|\:)+|^"(?:[^"]|"")+")\[[^\]]+\]/,
|
||
QUOTED_LITERAL: /^'(?:[^']|'')*'/,
|
||
NUMERIC_LITERAL: /^[0-9]+(?:\.[0-9]*(?:[eE][-+][0-9]+)?)?/,
|
||
SYMBOL: /^(?:==|=|<>|<=|<|>=|>|!~~|!~|~~|~|!==|!=|!~=|!~|!|&|\||\.|\:|,|\(|\)|\[|\]|\{|\}|\?|\:|;|@|\^|\/\+|\/|\*|\+|-)/
|
||
};
|
||
str = '"';
|
||
""", # NOQA
|
||
[
|
||
"id rexl",
|
||
"punct .",
|
||
"id re",
|
||
"punct =",
|
||
"punct {",
|
||
"id NAME",
|
||
"punct :",
|
||
r"""regex /^(?![0-9])(?:\w)+|^"(?:[^"]|"")+"/""",
|
||
"punct ,",
|
||
"id UNQUOTED_LITERAL",
|
||
"punct :",
|
||
r"""regex /^@(?:(?![0-9])(?:\w|\:)+|^"(?:[^"]|"")+")\[[^\]]+\]/""",
|
||
"punct ,",
|
||
"id QUOTED_LITERAL",
|
||
"punct :",
|
||
r"""regex /^'(?:[^']|'')*'/""",
|
||
"punct ,",
|
||
"id NUMERIC_LITERAL",
|
||
"punct :",
|
||
r"""regex /^[0-9]+(?:\.[0-9]*(?:[eE][-+][0-9]+)?)?/""",
|
||
"punct ,",
|
||
"id SYMBOL",
|
||
"punct :",
|
||
r"""regex /^(?:==|=|<>|<=|<|>=|>|!~~|!~|~~|~|!==|!=|!~=|!~|!|&|\||\.|\:|,|\(|\)|\[|\]|\{|\}|\?|\:|;|@|\^|\/\+|\/|\*|\+|-)/""", # NOQA
|
||
"punct }",
|
||
"punct ;",
|
||
"id str",
|
||
"punct =",
|
||
"""string '"'""",
|
||
"punct ;",
|
||
],
|
||
),
|
||
(
|
||
r' this._js = "e.str(\"" + this.value.replace(/\\/g, "\\\\")'
|
||
r'.replace(/"/g, "\\\"") + "\")"; ',
|
||
[
|
||
"keyword this",
|
||
"punct .",
|
||
"id _js",
|
||
"punct =",
|
||
r'''string "e.str(\""''',
|
||
"punct +",
|
||
"keyword this",
|
||
"punct .",
|
||
"id value",
|
||
"punct .",
|
||
"id replace",
|
||
"punct (",
|
||
r"regex /\\/g",
|
||
"punct ,",
|
||
r'string "\\\\"',
|
||
"punct )",
|
||
"punct .",
|
||
"id replace",
|
||
"punct (",
|
||
r'regex /"/g',
|
||
"punct ,",
|
||
r'string "\\\""',
|
||
"punct )",
|
||
"punct +",
|
||
r'string "\")"',
|
||
"punct ;",
|
||
],
|
||
),
|
||
]
|
||
|
||
|
||
def make_function(input, toks):
|
||
def test_func(self):
|
||
lexer = JsLexer()
|
||
result = [
|
||
"%s %s" % (name, tok) for name, tok in lexer.lex(input) if name != "ws"
|
||
]
|
||
self.assertEqual(result, toks)
|
||
|
||
return test_func
|
||
|
||
|
||
for i, (input, toks) in enumerate(JsTokensTest.LEX_CASES):
|
||
setattr(JsTokensTest, "test_case_%d" % i, make_function(input, toks))
|
||
|
||
|
||
GETTEXT_CASES = (
|
||
(
|
||
r"""
|
||
a = 1; /* /[0-9]+/ */
|
||
b = 0x2a0b / 1; // /[0-9]+/
|
||
c = 3;
|
||
""",
|
||
r"""
|
||
a = 1; /* /[0-9]+/ */
|
||
b = 0x2a0b / 1; // /[0-9]+/
|
||
c = 3;
|
||
""",
|
||
),
|
||
(
|
||
r"""
|
||
a = 1.234e-5;
|
||
/*
|
||
* /[0-9+/
|
||
*/
|
||
b = .0123;
|
||
""",
|
||
r"""
|
||
a = 1.234e-5;
|
||
/*
|
||
* /[0-9+/
|
||
*/
|
||
b = .0123;
|
||
""",
|
||
),
|
||
(
|
||
r"""
|
||
x = y / z;
|
||
alert(gettext("hello"));
|
||
x /= 3;
|
||
""",
|
||
r"""
|
||
x = y / z;
|
||
alert(gettext("hello"));
|
||
x /= 3;
|
||
""",
|
||
),
|
||
(
|
||
r"""
|
||
s = "Hello \"th/foo/ere\"";
|
||
s = 'He\x23llo \'th/foo/ere\'';
|
||
s = 'slash quote \", just quote "';
|
||
""",
|
||
r"""
|
||
s = "Hello \"th/foo/ere\"";
|
||
s = "He\x23llo \'th/foo/ere\'";
|
||
s = "slash quote \", just quote \"";
|
||
""",
|
||
),
|
||
(
|
||
r"""
|
||
s = "Line continuation\
|
||
continued /hello/ still the string";/hello/;
|
||
""",
|
||
r"""
|
||
s = "Line continuation\
|
||
continued /hello/ still the string";"REGEX";
|
||
""",
|
||
),
|
||
(
|
||
r"""
|
||
var regex = /pattern/;
|
||
var regex2 = /matter/gm;
|
||
var regex3 = /[*/]+/gm.foo("hey");
|
||
""",
|
||
r"""
|
||
var regex = "REGEX";
|
||
var regex2 = "REGEX";
|
||
var regex3 = "REGEX".foo("hey");
|
||
""",
|
||
),
|
||
(
|
||
r"""
|
||
for (var x = a in foo && "</x>" || mot ? z:/x:3;x<5;y</g/i) {xyz(x++);}
|
||
for (var x = a in foo && "</x>" || mot ? z/x:3;x<5;y</g/i) {xyz(x++);}
|
||
""",
|
||
r"""
|
||
for (var x = a in foo && "</x>" || mot ? z:"REGEX"/i) {xyz(x++);}
|
||
for (var x = a in foo && "</x>" || mot ? z/x:3;x<5;y<"REGEX") {xyz(x++);}
|
||
""",
|
||
),
|
||
(
|
||
"""
|
||
\\u1234xyz = gettext('Hello there');
|
||
""",
|
||
r"""
|
||
Uu1234xyz = gettext("Hello there");
|
||
""",
|
||
),
|
||
)
|
||
|
||
|
||
class JsToCForGettextTest(SimpleTestCase):
|
||
pass
|
||
|
||
|
||
def make_function(js, c):
|
||
def test_func(self):
|
||
self.assertEqual(prepare_js_for_gettext(js), c)
|
||
|
||
return test_func
|
||
|
||
|
||
for i, pair in enumerate(GETTEXT_CASES):
|
||
setattr(JsToCForGettextTest, "test_case_%d" % i, make_function(*pair))
|