From ded485464214a3f69b64402b7d82221279f80008 Mon Sep 17 00:00:00 2001 From: Oleg Sverdlov Date: Thu, 5 Dec 2024 14:12:45 +0100 Subject: [PATCH] Fixed #35944 -- Handled serialization of Unicode values in ArrayField and HStoreField. --- django/contrib/postgres/fields/array.py | 2 +- django/contrib/postgres/fields/hstore.py | 2 +- tests/postgres_tests/test_array.py | 26 +++++++++ tests/postgres_tests/test_hstore.py | 68 ++++++++++++++---------- 4 files changed, 69 insertions(+), 29 deletions(-) diff --git a/django/contrib/postgres/fields/array.py b/django/contrib/postgres/fields/array.py index 4171af82f9..a7e40703a3 100644 --- a/django/contrib/postgres/fields/array.py +++ b/django/contrib/postgres/fields/array.py @@ -169,7 +169,7 @@ class ArrayField(CheckFieldDefaultMixin, Field): else: obj = AttributeSetter(base_field.attname, val) values.append(base_field.value_to_string(obj)) - return json.dumps(values) + return json.dumps(values, ensure_ascii=False) def get_transform(self, name): transform = super().get_transform(name) diff --git a/django/contrib/postgres/fields/hstore.py b/django/contrib/postgres/fields/hstore.py index cfc156ab59..300458c0b1 100644 --- a/django/contrib/postgres/fields/hstore.py +++ b/django/contrib/postgres/fields/hstore.py @@ -43,7 +43,7 @@ class HStoreField(CheckFieldDefaultMixin, Field): return value def value_to_string(self, obj): - return json.dumps(self.value_from_object(obj)) + return json.dumps(self.value_from_object(obj), ensure_ascii=False) def formfield(self, **kwargs): return super().formfield( diff --git a/tests/postgres_tests/test_array.py b/tests/postgres_tests/test_array.py index ea7807687e..ba7151d4a2 100644 --- a/tests/postgres_tests/test_array.py +++ b/tests/postgres_tests/test_array.py @@ -1008,6 +1008,32 @@ class TestSerialization(PostgreSQLSimpleTestCase): self.assertEqual(instance.field, [1, 2, None]) +class TestStringSerialization(PostgreSQLSimpleTestCase): + field_values = [["Django", "Python", None], ["Джанго", "פייתון", None, "król"]] + + @staticmethod + def create_json_data(array_field_value): + fields = {"field": json.dumps(array_field_value, ensure_ascii=False)} + return json.dumps( + [{"model": "postgres_tests.chararraymodel", "pk": None, "fields": fields}] + ) + + def test_encode(self): + for field_value in self.field_values: + with self.subTest(field_value=field_value): + instance = CharArrayModel(field=field_value) + data = serializers.serialize("json", [instance]) + json_data = self.create_json_data(field_value) + self.assertEqual(json.loads(data), json.loads(json_data)) + + def test_decode(self): + for field_value in self.field_values: + with self.subTest(field_value=field_value): + json_data = self.create_json_data(field_value) + instance = list(serializers.deserialize("json", json_data))[0].object + self.assertEqual(instance.field, field_value) + + class TestValidation(PostgreSQLSimpleTestCase): def test_unbounded(self): field = ArrayField(models.IntegerField()) diff --git a/tests/postgres_tests/test_hstore.py b/tests/postgres_tests/test_hstore.py index cac3eb742a..2d19364736 100644 --- a/tests/postgres_tests/test_hstore.py +++ b/tests/postgres_tests/test_hstore.py @@ -297,39 +297,53 @@ class TestChecks(PostgreSQLSimpleTestCase): class TestSerialization(PostgreSQLSimpleTestCase): - test_data = json.dumps( - [ - { - "model": "postgres_tests.hstoremodel", - "pk": None, - "fields": { - "field": json.dumps({"a": "b"}), - "array_field": json.dumps( - [ - json.dumps({"a": "b"}), - json.dumps({"b": "a"}), - ] - ), - }, - } - ] - ) + field_values = [ + ({"a": "b"}, [{"a": "b"}, {"b": "a"}]), + ( + {"все": "Трурль и Клапауций"}, + [{"Трурль": "Клапауций"}, {"Клапауций": "Трурль"}], + ), + ] + + @staticmethod + def create_json_data(field_value, array_field_value): + fields = { + "field": json.dumps(field_value, ensure_ascii=False), + "array_field": json.dumps( + [json.dumps(item, ensure_ascii=False) for item in array_field_value], + ensure_ascii=False, + ), + } + return json.dumps( + [{"model": "postgres_tests.hstoremodel", "pk": None, "fields": fields}] + ) def test_dumping(self): - instance = HStoreModel(field={"a": "b"}, array_field=[{"a": "b"}, {"b": "a"}]) - data = serializers.serialize("json", [instance]) - self.assertEqual(json.loads(data), json.loads(self.test_data)) + for field_value, array_field_value in self.field_values: + with self.subTest(field_value=field_value, array_value=array_field_value): + instance = HStoreModel(field=field_value, array_field=array_field_value) + data = serializers.serialize("json", [instance]) + json_data = self.create_json_data(field_value, array_field_value) + self.assertEqual(json.loads(data), json.loads(json_data)) def test_loading(self): - instance = list(serializers.deserialize("json", self.test_data))[0].object - self.assertEqual(instance.field, {"a": "b"}) - self.assertEqual(instance.array_field, [{"a": "b"}, {"b": "a"}]) + for field_value, array_field_value in self.field_values: + with self.subTest(field_value=field_value, array_value=array_field_value): + json_data = self.create_json_data(field_value, array_field_value) + instance = list(serializers.deserialize("json", json_data))[0].object + self.assertEqual(instance.field, field_value) + self.assertEqual(instance.array_field, array_field_value) def test_roundtrip_with_null(self): - instance = HStoreModel(field={"a": "b", "c": None}) - data = serializers.serialize("json", [instance]) - new_instance = list(serializers.deserialize("json", data))[0].object - self.assertEqual(instance.field, new_instance.field) + for field_value in [ + {"a": "b", "c": None}, + {"Енеїда": "Ти знаєш, він який суціга", "Зефір": None}, + ]: + with self.subTest(field_value=field_value): + instance = HStoreModel(field=field_value) + data = serializers.serialize("json", [instance]) + new_instance = list(serializers.deserialize("json", data))[0].object + self.assertEqual(instance.field, new_instance.field) class TestValidation(PostgreSQLSimpleTestCase):