Added JSON Parser and request.data

django/core/handlers/asgi.py

@@ -115,6 +115,7 @@ class ASGIRequest(HttpRequest):
         self._parsers = [
             parsers.FormParser(),
             parsers.MultiPartParser(),
+            parsers.JSONParser(),
         ]
 
     @cached_property

django/core/handlers/wsgi.py

@@ -81,6 +81,7 @@ class WSGIRequest(HttpRequest):
         self._parsers = [
             parsers.FormParser(),
             parsers.MultiPartParser(),
+            parsers.JSONParser(),
         ]
 
     def _get_scheme(self):

django/http/multipartparser.py

@@ -54,7 +54,7 @@ class MultiPartParser:
 
     boundary_re = _lazy_re_compile(r"[ -~]{0,200}[!-~]")
 
-    def __init__(self, META, input_data, upload_handlers, encoding=None):
+    def __init__(self, META, input_data, upload_handlers, encoding=None, parsers=None):
         """
         Initialize the MultiPartParser object.
 
@@ -112,6 +112,7 @@ class MultiPartParser:
         self._encoding = encoding or settings.DEFAULT_CHARSET
         self._content_length = content_length
         self._upload_handlers = upload_handlers
+        self._parsers = parsers
 
     def parse(self):
         # Call the actual parse routine and close all open files in case of
@@ -236,6 +237,23 @@ class MultiPartParser:
                         data = field_stream.read(size=read_size)
                         num_bytes_read += len(data)
 
+                    try:
+                        content_type = meta_data["content-type"][0].strip()
+                    except KeyError:
+                        content_type = None
+                    selected_parser = None
+                    if content_type:
+                        for parser in self._parsers:
+                            if parser.can_handle(content_type):
+                                selected_parser = parser
+                                break
+                    if selected_parser:
+                        # TODO maybe .parse() shouldn't return an empty MultiValueDict
+                        # for files if it's not needed
+                        self._post.appendlist(
+                            field_name, selected_parser.parse(data)[0]
+                        )
+                    else:
                         # Add two here to make the check consistent with the
                         # x-www-form-urlencoded check that includes '&='.
                         num_bytes_read += len(field_name) + 2

django/http/parsers.py

@@ -1,3 +1,4 @@
+import json
 from io import BytesIO
 
 from django.core.exceptions import BadRequest
@@ -7,18 +8,24 @@ from django.utils.datastructures import ImmutableList, MultiValueDict
 
 class BaseParser:
     media_type = None
+    parsers = None
 
     def can_handle(self, media_type):
         return media_type == self.media_type
 
-    def parse(self, request):
+    def parse(self, data, request=None):
         pass
 
+    @property
+    def _supports_form_parsing(self):
+        form_media = ("application/x-www-form-urlencoded", "multipart/form-data")
+        return self.media_type in form_media
+
 
 class FormParser(BaseParser):
     media_type = "application/x-www-form-urlencoded"
 
-    def parse(self, request):
+    def parse(self, data, request=None):
         from django.http import QueryDict
 
         # According to RFC 1866, the "application/x-www-form-urlencoded"
@@ -35,13 +42,17 @@ class FormParser(BaseParser):
 class MultiPartParser(BaseParser):
     media_type = "multipart/form-data"
 
-    def parse(self, request):
+    def parse(self, data, request=None):
         if hasattr(request, "_body"):
             # Use already read data
             data = BytesIO(request._body)
         else:
             data = request
 
+        # TODO - POST and data can be called on the same request. This parser can be
+        # called multiple times on the same request. While `_post` `_data` are different
+        # _files is the same. Allow parsing them twice, but don't change the handlers?
+        if not hasattr(request, "_files"):
             request.upload_handlers = ImmutableList(
                 request.upload_handlers,
                 warning=(
@@ -50,7 +61,16 @@ class MultiPartParser(BaseParser):
                 ),
             )
         parser = _MultiPartParser(
-            request.META, data, request.upload_handlers, request.encoding
+            request.META, data, request.upload_handlers, request.encoding, self.parsers
         )
+        # TODO _post could also be _data
         _post, _files = parser.parse()
         return _post, _files
+
+
+class JSONParser(BaseParser):
+    media_type = "application/json"
+
+    def parse(self, data, request=None):
+        # TODO enable strict mode. Like DRF.
+        return json.loads(data), MultiValueDict()

django/http/request.py

@@ -30,6 +30,8 @@ host_validation_re = _lazy_re_compile(
     r"^([a-z0-9.-]+|\[[a-f0-9]*:[a-f0-9.:]+\])(?::([0-9]+))?$"
 )
 
+POST_PARSERS = (parsers.FormParser(), parsers.MultiPartParser())
+
 
 class UnreadablePostError(OSError):
     pass
@@ -72,6 +74,7 @@ class HttpRequest:
         self._parsers = [
             parsers.FormParser(),
             parsers.MultiPartParser(),
+            parsers.JSONParser(),
         ]
 
     def __repr__(self):
@@ -350,43 +353,52 @@ class HttpRequest:
         self._post = QueryDict()
         self._files = MultiValueDict()
 
-    def _load_post_and_files(self):
+    def _load_post_and_files(
-        """Populate self._post and self._files if the content-type is a form type"""
+        self, data_attr="_post", parsers=POST_PARSERS, methods=("POST",)
-        if self.method != "POST":
+    ):
+        if methods and self.method not in methods:
             self._post, self._files = (
                 QueryDict(encoding=self._encoding),
                 MultiValueDict(),
             )
             return
+
         if self._read_started and not hasattr(self, "_body"):
-            self._mark_post_parse_error()
+            setattr(self, data_attr, QueryDict())
+            self._files = MultiValueDict()
             return
 
-        # TODO create a parsers setter/getter/initializer like upload_handlers
-        parser_list = (parsers.FormParser(), parsers.MultiPartParser())
-
         selected_parser = None
-        for parser in parser_list:
+        for parser in parsers:
             if parser.can_handle(self.content_type):
                 selected_parser = parser
                 break
 
         if selected_parser:
+            selected_parser.parsers = parsers
             try:
-                self._post, self._files = parser.parse(self)
+                if selected_parser._supports_form_parsing:
+                    # TODO Not sure how to make these consistent.
+                    data, self._files = parser.parse(None, self)
+                    setattr(self, data_attr, data)
+                else:
+                    data, self._files = parser.parse(self.body, self)
+                    setattr(self, data_attr, data)
             except Exception as e:
                 # TODO 'application/x-www-form-urlencoded' didn't do this.
                 # An error occurred while parsing POST data. Since when
                 # formatting the error the request handler might access
                 # self.POST, set self._post and self._file to prevent
                 # attempts to parse POST data again.
-                self._mark_post_parse_error()
+                data_attr = QueryDict()
+                self._files = MultiValueDict()
                 raise e
         else:
-            self._post, self._files = (
+            data, self._files = (
                 QueryDict(encoding=self._encoding),
                 MultiValueDict(),
             )
+            setattr(self, data_attr, data)
 
     def close(self):
         if hasattr(self, "_files"):
@@ -427,13 +439,23 @@ class HttpRequest:
 
     @parsers.setter
     def parsers(self, parsers):
-        # TODO Also check for _data once added
+        if hasattr(self, "_data") or hasattr(self, "_files"):
-        if hasattr(self, "_files"):
             raise AttributeError(
                 "You cannot change parsers after processing the request's content."
             )
         self._parsers = parsers
 
+    # TODO should this property be on [WSGI|ASGI]Request?
+    @property
+    def data(self):
+        if not hasattr(self, "_data"):
+            self._load_post_and_files("_data", self.parsers, methods=None)
+        return self._data
+
+    @data.setter
+    def data(self, data):
+        self._data = data
+
 
 class HttpHeaders(CaseInsensitiveMapping):
     HTTP_PREFIX = "HTTP_"

docs/ref/parsers.txt

@@ -57,6 +57,16 @@ Parses HTML form content (). The ``parse()`` method returns a
 Parses multipart form content and supports file uploads. The method returns
 a ``QueryDict`` for ``data`` and an ``MultiValueDict`` for ``FILES``.
 
+.. class:: JSONParser
+
+    .. attribute:: media_type
+
+      ``"application/json"``
+
+The ``parse()`` method deserializes JSON to a Python dictionary. This is
+returned for ``data`` and empty ``MultiValueDict`` is provided for ``FILES``.
+
+
 HttpRequest.Parsers
 ===================
 

docs/ref/request-response.txt

@@ -119,6 +119,15 @@ All attributes should be considered read-only, unless stated otherwise.
 
     ``POST`` does *not* include file-upload information. See :attr:`FILES`.
 
+.. attribute:: HttpRequest.data
+
+    Similar to :attr:`HttpRequest.POST` but parses the :attr:`HttpRequest.body`
+    with the parsers returned by :attr:`HttpRequest.parsers``. By default this
+    will result in ``application/json`` data being parsed in addition to form
+    data.
+
+    In addition data will be parsed for all :attr:`HttpRequest.method` methods.
+
 .. attribute:: HttpRequest.COOKIES
 
     A dictionary containing all cookies. Keys and values are strings.

tests/requests_tests/test_parsers.py

@@ -1,6 +1,6 @@
 from django.core.handlers.wsgi import WSGIRequest
 from django.http import HttpRequest
-from django.http.parsers import BaseParser, FormParser, MultiPartParser
+from django.http.parsers import BaseParser, FormParser, JSONParser, MultiPartParser
 from django.test import SimpleTestCase
 from django.test.client import FakePayload
 from django.utils.http import urlencode
@@ -31,9 +31,10 @@ class TestParsers(SimpleTestCase):
 
     def test_request_parser_no_setting(self):
         request = HttpRequest()
-        form, multipart = request.parsers
+        form, multipart, json = request.parsers
         self.assertIsInstance(form, FormParser)
         self.assertIsInstance(multipart, MultiPartParser)
+        self.assertIsInstance(json, JSONParser)
 
     def test_set_parser(self):
         request = HttpRequest()

tests/requests_tests/tests.py

@@ -14,7 +14,7 @@ from django.http import (
     UnreadablePostError,
 )
 from django.http.multipartparser import MAX_TOTAL_HEADER_SIZE, MultiPartParserError
-from django.http.request import split_domain_port
+from django.http.request import QueryDict, split_domain_port
 from django.test import RequestFactory, SimpleTestCase, override_settings
 from django.test.client import BOUNDARY, MULTIPART_CONTENT, FakePayload
 
@@ -99,6 +99,7 @@ class RequestsTests(SimpleTestCase):
         )
         self.assertEqual(list(request.GET), [])
         self.assertEqual(list(request.POST), [])
+        self.assertEqual(list(request.data), [])
         self.assertEqual(list(request.COOKIES), [])
         self.assertEqual(
             set(request.META),
@@ -339,7 +340,7 @@ class RequestsTests(SimpleTestCase):
     def test_read_after_value(self):
         """
         Reading from request is allowed after accessing request contents as
-        POST or body.
+        POST, data or body.
         """
         payload = FakePayload("name=value")
         request = WSGIRequest(
@@ -351,12 +352,13 @@ class RequestsTests(SimpleTestCase):
             }
         )
         self.assertEqual(request.POST, {"name": ["value"]})
+        self.assertEqual(request.data, {"name": ["value"]})
         self.assertEqual(request.body, b"name=value")
         self.assertEqual(request.read(), b"name=value")
 
     def test_value_after_read(self):
         """
-        Construction of POST or body is not allowed after reading
+        Construction of POST, data or body is not allowed after reading
         from request.
         """
         payload = FakePayload("name=value")
@@ -372,6 +374,7 @@ class RequestsTests(SimpleTestCase):
         with self.assertRaises(RawPostDataException):
             request.body
         self.assertEqual(request.POST, {})
+        self.assertEqual(request.data, {})
 
     def test_non_ascii_POST(self):
         payload = FakePayload(urlencode({"key": "España"}))
@@ -384,6 +387,7 @@ class RequestsTests(SimpleTestCase):
             }
         )
         self.assertEqual(request.POST, {"key": ["España"]})
+        self.assertEqual(request.data, {"key": ["España"]})
 
     def test_non_utf8_charset_POST_bad_request(self):
         payload = FakePayload(urlencode({"key": "España".encode("latin-1")}))
@@ -400,6 +404,8 @@ class RequestsTests(SimpleTestCase):
         )
         with self.assertRaisesMessage(BadRequest, msg):
             request.POST
+        with self.assertRaisesMessage(BadRequest, msg):
+            request.data
         request = WSGIRequest(environ)
         with self.assertRaisesMessage(BadRequest, msg):
             request.FILES
@@ -419,6 +425,7 @@ class RequestsTests(SimpleTestCase):
                     }
                 )
                 self.assertEqual(request.POST, {"key": ["España"]})
+                self.assertEqual(request.data, {"key": ["España"]})
 
     def test_body_after_POST_multipart_form_data(self):
         """
@@ -477,6 +484,7 @@ class RequestsTests(SimpleTestCase):
             }
         )
         self.assertEqual(request.POST, {})
+        self.assertEqual(request.data, {})
         self.assertEqual(request.body, payload_data)
 
     def test_POST_multipart_with_content_length_zero(self):
@@ -506,6 +514,7 @@ class RequestsTests(SimpleTestCase):
             }
         )
         self.assertEqual(request.POST, {})
+        self.assertEqual(request.data, {})
 
     @override_settings(
         FILE_UPLOAD_HANDLERS=["requests_tests.tests.ErrorFileUploadHandler"]
@@ -559,7 +568,7 @@ class RequestsTests(SimpleTestCase):
         self.assertEqual(request.POST, "_POST")
         self.assertEqual(request.FILES, "_FILES")
 
-    def test_request_methods_with_content(self):
+    def test_request_methods_with_content_POST(self):
         for method in ["GET", "PUT", "DELETE"]:
             with self.subTest(method=method):
                 payload = FakePayload(urlencode({"key": "value"}))
@@ -573,6 +582,20 @@ class RequestsTests(SimpleTestCase):
                 )
                 self.assertEqual(request.POST, {})
 
+    def test_request_methods_with_content_data(self):
+        for method in ["GET", "PUT", "DELETE"]:
+            with self.subTest(method=method):
+                payload = FakePayload(urlencode({"key": "value"}))
+                request = WSGIRequest(
+                    {
+                        "REQUEST_METHOD": method,
+                        "CONTENT_LENGTH": len(payload),
+                        "CONTENT_TYPE": "application/x-www-form-urlencoded",
+                        "wsgi.input": payload,
+                    }
+                )
+                self.assertEqual(request.data, QueryDict("key=value"))
+
     def test_POST_content_type_json(self):
         payload = FakePayload(
             "\r\n".join(
@@ -593,12 +616,39 @@ class RequestsTests(SimpleTestCase):
         self.assertEqual(request.POST, {})
         self.assertEqual(request.FILES, {})
 
+    def test_data_content_type_json(self):
+        payload = FakePayload(
+            "\r\n".join(
+                [
+                    '{"pk": 1, "model": "store.book", "fields": {"name": "Mostly Ha'
+                    'rmless", "author": ["Douglas", "Adams"]}}',
+                ]
+            )
+        )
+        request = WSGIRequest(
+            {
+                "REQUEST_METHOD": "POST",
+                "CONTENT_TYPE": "application/json",
+                "CONTENT_LENGTH": len(payload),
+                "wsgi.input": payload,
+            }
+        )
+        self.assertEqual(
+            request.data,
+            {
+                "pk": 1,
+                "model": "store.book",
+                "fields": {"name": "Mostly Harmless", "author": ["Douglas", "Adams"]},
+            },
+        )
+        self.assertEqual(request.FILES, {})
+
     _json_payload = [
         'Content-Disposition: form-data; name="JSON"',
         "Content-Type: application/json",
         "",
         '{"pk": 1, "model": "store.book", "fields": {"name": "Mostly Harmless", '
-        '"author": ["Douglas", Adams"]}}',
+        '"author": ["Douglas", "Adams"]}}',
     ]
 
     def test_POST_form_data_json(self):
@@ -618,7 +668,7 @@ class RequestsTests(SimpleTestCase):
             {
                 "JSON": [
                     '{"pk": 1, "model": "store.book", "fields": {"name": "Mostly '
-                    'Harmless", "author": ["Douglas", Adams"]}}'
+                    'Harmless", "author": ["Douglas", "Adams"]}}'
                 ],
             },
         )
@@ -651,7 +701,46 @@ class RequestsTests(SimpleTestCase):
                 "name": ["value"],
                 "JSON": [
                     '{"pk": 1, "model": "store.book", "fields": {"name": "Mostly '
-                    'Harmless", "author": ["Douglas", Adams"]}}'
+                    'Harmless", "author": ["Douglas", "Adams"]}}'
+                ],
+            },
+        )
+
+    def test_data_multipart_json(self):
+        payload = FakePayload(
+            "\r\n".join(
+                [
+                    f"--{BOUNDARY}",
+                    'Content-Disposition: form-data; name="name"',
+                    "",
+                    "value",
+                    f"--{BOUNDARY}",
+                    *self._json_payload,
+                    f"--{BOUNDARY}--",
+                ]
+            )
+        )
+        request = WSGIRequest(
+            {
+                "REQUEST_METHOD": "POST",
+                "CONTENT_TYPE": MULTIPART_CONTENT,
+                "CONTENT_LENGTH": len(payload),
+                "wsgi.input": payload,
+            }
+        )
+        self.assertEqual(
+            request.data,
+            {
+                "name": ["value"],
+                "JSON": [
+                    {
+                        "pk": 1,
+                        "model": "store.book",
+                        "fields": {
+                            "name": "Mostly Harmless",
+                            "author": ["Douglas", "Adams"],
+                        },
+                    }
                 ],
             },
         )
@@ -689,7 +778,52 @@ class RequestsTests(SimpleTestCase):
                 "name": ["value"],
                 "JSON": [
                     '{"pk": 1, "model": "store.book", "fields": {"name": "Mostly '
-                    'Harmless", "author": ["Douglas", Adams"]}}'
+                    'Harmless", "author": ["Douglas", "Adams"]}}'
+                ],
+                "CSV": ["Framework,ID.Django,1.Flask,2."],
+            },
+        )
+
+    def test_data_multipart_json_csv(self):
+        payload = FakePayload(
+            "\r\n".join(
+                [
+                    f"--{BOUNDARY}",
+                    'Content-Disposition: form-data; name="name"',
+                    "",
+                    "value",
+                    f"--{BOUNDARY}",
+                    *self._json_payload,
+                    f"--{BOUNDARY}",
+                    'Content-Disposition: form-data; name="CSV"',
+                    "Content-Type: text/csv",
+                    "",
+                    "Framework,ID.Django,1.Flask,2.",
+                    f"--{BOUNDARY}--",
+                ]
+            )
+        )
+        request = WSGIRequest(
+            {
+                "REQUEST_METHOD": "POST",
+                "CONTENT_TYPE": MULTIPART_CONTENT,
+                "CONTENT_LENGTH": len(payload),
+                "wsgi.input": payload,
+            }
+        )
+        self.assertEqual(
+            request.data,
+            {
+                "name": ["value"],
+                "JSON": [
+                    {
+                        "pk": 1,
+                        "model": "store.book",
+                        "fields": {
+                            "name": "Mostly Harmless",
+                            "author": ["Douglas", "Adams"],
+                        },
+                    }
                 ],
                 "CSV": ["Framework,ID.Django,1.Flask,2."],
             },
@@ -730,7 +864,55 @@ class RequestsTests(SimpleTestCase):
                 "name": ["value"],
                 "JSON": [
                     '{"pk": 1, "model": "store.book", "fields": {"name": "Mostly '
-                    'Harmless", "author": ["Douglas", Adams"]}}'
+                    'Harmless", "author": ["Douglas", "Adams"]}}'
+                ],
+            },
+        )
+        self.assertEqual(len(request.FILES), 1)
+        self.assertIsInstance((request.FILES["File"]), InMemoryUploadedFile)
+
+    def test_data_multipart_with_file(self):
+        payload = FakePayload(
+            "\r\n".join(
+                [
+                    f"--{BOUNDARY}",
+                    'Content-Disposition: form-data; name="name"',
+                    "",
+                    "value",
+                    f"--{BOUNDARY}",
+                    *self._json_payload,
+                    f"--{BOUNDARY}",
+                    'Content-Disposition: form-data; name="File"; filename="test.csv"',
+                    "Content-Type: application/octet-stream",
+                    "",
+                    "Framework,ID",
+                    "Django,1",
+                    "Flask,2",
+                    f"--{BOUNDARY}--",
+                ]
+            )
+        )
+        request = WSGIRequest(
+            {
+                "REQUEST_METHOD": "POST",
+                "CONTENT_TYPE": MULTIPART_CONTENT,
+                "CONTENT_LENGTH": len(payload),
+                "wsgi.input": payload,
+            }
+        )
+        self.assertEqual(
+            request.data,
+            {
+                "name": ["value"],
+                "JSON": [
+                    {
+                        "pk": 1,
+                        "model": "store.book",
+                        "fields": {
+                            "name": "Mostly Harmless",
+                            "author": ["Douglas", "Adams"],
+                        },
+                    }
                 ],
             },
         )
@@ -794,9 +976,9 @@ class RequestsTests(SimpleTestCase):
         )
         self.assertEqual(list(request), [b"name=value"])
 
-    def test_POST_after_body_read(self):
+    def test_POST_and_data_after_body_read(self):
         """
-        POST should be populated even if body is read first
+        POST and data should be populated even if body is read first
         """
         payload = FakePayload("name=value")
         request = WSGIRequest(
@@ -809,10 +991,11 @@ class RequestsTests(SimpleTestCase):
         )
         request.body  # evaluate
         self.assertEqual(request.POST, {"name": ["value"]})
+        self.assertEqual(request.data, {"name": ["value"]})
 
-    def test_POST_after_body_read_and_stream_read(self):
+    def test_POST_and_data_after_body_read_and_stream_read(self):
         """
-        POST should be populated even if body is read first, and then
+        POST and data should be populated even if body is read first, and then
         the stream is read second.
         """
         payload = FakePayload("name=value")
@@ -827,6 +1010,7 @@ class RequestsTests(SimpleTestCase):
         request.body  # evaluate
         self.assertEqual(request.read(1), b"n")
         self.assertEqual(request.POST, {"name": ["value"]})
+        self.assertEqual(request.data, {"name": ["value"]})
 
     def test_multipart_post_field_with_base64(self):
         payload = FakePayload(