From a0ce42a87cd307bc004f4ffc6a0e6de5354472d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Gu=C3=A9rin?= Date: Mon, 28 May 2018 12:17:15 +0200 Subject: [PATCH 1/2] Percent signs escaping in templatetags Closes https://github.com/python-babel/django-babel/issues/43 In Django >= 1.9, `%` signs are escaped to `%%` during extraction then replaced by `%` during the rendering. [1] Before Django 1.9, only single percent signs were escaped. The extraction now behaves in the same way. [1] https://github.com/django/django/commit/b7508896fbe19ec2cdeb81565cd587091b6b68d0 --- django_babel/extract.py | 18 ++++++++++++++++-- tests/test_extract.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/django_babel/extract.py b/django_babel/extract.py index edb42c7..a5fec7c 100644 --- a/django_babel/extract.py +++ b/django_babel/extract.py @@ -11,6 +11,12 @@ from django.utils.translation.template import ( inline_re, block_re, endblock_re, plural_re, constant_re) +try: + from django.utils.translation.trans_real import one_percent_re +except ImportError: + # Django 1.9+ + one_percent_re = None + def join_tokens(tokens, trim=False): message = ''.join(tokens) @@ -112,10 +118,14 @@ def extract_django(fileobj, keywords, comment_tags, options): else: singular.append('%%(%s)s' % t.contents) elif t.token_type == TOKEN_TEXT: + if one_percent_re: + contents = one_percent_re.sub('%%', t.contents) + else: + contents = t.contents.replace('%', '%%') if inplural: - plural.append(t.contents) + plural.append(contents) else: - singular.append(t.contents) + singular.append(contents) else: if t.token_type == TOKEN_BLOCK: imatch = inline_re.match(t.contents) @@ -124,6 +134,10 @@ def extract_django(fileobj, keywords, comment_tags, options): if imatch: g = imatch.group(1) g = strip_quotes(g) + if one_percent_re: + g = one_percent_re.sub('%%', g) + else: + g = g.replace('%', '%%') message_context = imatch.group(3) if message_context: # strip quotes diff --git a/tests/test_extract.py b/tests/test_extract.py index aca1a58..96eeed2 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -218,3 +218,33 @@ def test_blocktrans_with_whitespace_trimmed(self): buf = BytesIO(test_tmpl) messages = list(extract_django(buf, default_keys, [], {})) self.assertEqual([(4, None, u'foo bar', [])], messages) + + @pytest.mark.skipif(django.VERSION >= (1, 9), + reason='%-sign escaping changed in django 1.9') + def test_extract_trans_percents_old_way(self): + """Before Django 1.9, only a signle %-sign was escaped to %%""" + buf = BytesIO(b'{% trans "1 %, 2 %%, 3 %%%" %}') + messages = list(extract_django(buf, default_keys, [], {})) + self.assertEqual([(1, None, u'1 %%, 2 %%, 3 %%%', [])], messages) + + @pytest.mark.skipif(django.VERSION >= (1, 9), + reason='%-sign escaping changed in django 1.9') + def test_extract_blocktrans_percents_old_way(self): + """Before Django 1.9, only a signle %-sign was escaped to %%""" + buf = BytesIO(b'{% blocktrans %}1 %, 2 %%, 3 %%%{% endblocktrans %}') + messages = list(extract_django(buf, default_keys, [], {})) + self.assertEqual([(1, None, u'1 %%, 2 %%, 3 %%%', [])], messages) + + @pytest.mark.skipif(django.VERSION < (1, 9), + reason='%-sign escaping changed in django 1.9') + def test_extract_trans_percents(self): + buf = BytesIO(b'{% trans "1 %, 2 %%, 3 %%%" %}') + messages = list(extract_django(buf, default_keys, [], {})) + self.assertEqual([(1, None, u'1 %%, 2 %%%%, 3 %%%%%%', [])], messages) + + @pytest.mark.skipif(django.VERSION < (1, 9), + reason='%-sign escaping changed in django 1.9') + def test_extract_blocktrans_percents(self): + buf = BytesIO(b'{% blocktrans %}1 %, 2 %%, 3 %%%{% endblocktrans %}') + messages = list(extract_django(buf, default_keys, [], {})) + self.assertEqual([(1, None, u'1 %%, 2 %%%%, 3 %%%%%%', [])], messages) From 93253c45c3ce3e922b8d8b89539cc5a26363ebc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Gu=C3=A9rin?= Date: Mon, 28 May 2018 17:22:42 +0200 Subject: [PATCH 2/2] Fix replaced TOKEN_* constants by TokenType enums Since Django 2.1a1 [1], TOKEN_* constants have been replaced by an Enum [1] https://github.com/django/django/commit/9c4ea63e878c053600c284e32d5f32d27a59b63a --- django_babel/extract.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/django_babel/extract.py b/django_babel/extract.py index a5fec7c..0f19511 100644 --- a/django_babel/extract.py +++ b/django_babel/extract.py @@ -1,8 +1,17 @@ # -*- coding: utf-8 -*- -from django.template.base import Lexer, TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK +from django.template.base import Lexer from django.utils.translation import trim_whitespace from django.utils.encoding import smart_text +try: + from django.template.base import TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK +except ImportError: + # Django 2.1+ + from django.template.base import TokenType + TOKEN_TEXT = TokenType.TEXT + TOKEN_VAR = TokenType.VAR + TOKEN_BLOCK = TokenType.BLOCK + try: from django.utils.translation.trans_real import ( inline_re, block_re, endblock_re, plural_re, constant_re)