diff --git a/django/core/handlers/asgi.py b/django/core/handlers/asgi.py index 46945b41e6..3187897380 100644 --- a/django/core/handlers/asgi.py +++ b/django/core/handlers/asgi.py @@ -115,6 +115,7 @@ class ASGIRequest(HttpRequest): self._parsers = [ parsers.FormParser(), parsers.MultiPartParser(), + parsers.JSONParser(), ] @cached_property diff --git a/django/core/handlers/wsgi.py b/django/core/handlers/wsgi.py index 5a372625e2..5518b46e2c 100644 --- a/django/core/handlers/wsgi.py +++ b/django/core/handlers/wsgi.py @@ -81,6 +81,7 @@ class WSGIRequest(HttpRequest): self._parsers = [ parsers.FormParser(), parsers.MultiPartParser(), + parsers.JSONParser(), ] def _get_scheme(self): diff --git a/django/http/multipartparser.py b/django/http/multipartparser.py index dfd406c35e..e0dd4c7c29 100644 --- a/django/http/multipartparser.py +++ b/django/http/multipartparser.py @@ -54,7 +54,7 @@ class MultiPartParser: boundary_re = _lazy_re_compile(r"[ -~]{0,200}[!-~]") - def __init__(self, META, input_data, upload_handlers, encoding=None): + def __init__(self, META, input_data, upload_handlers, encoding=None, parsers=None): """ Initialize the MultiPartParser object. @@ -112,6 +112,7 @@ class MultiPartParser: self._encoding = encoding or settings.DEFAULT_CHARSET self._content_length = content_length self._upload_handlers = upload_handlers + self._parsers = parsers def parse(self): # Call the actual parse routine and close all open files in case of @@ -236,21 +237,38 @@ class MultiPartParser: data = field_stream.read(size=read_size) num_bytes_read += len(data) - # Add two here to make the check consistent with the - # x-www-form-urlencoded check that includes '&='. - num_bytes_read += len(field_name) + 2 - if ( - settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None - and num_bytes_read > settings.DATA_UPLOAD_MAX_MEMORY_SIZE - ): - raise RequestDataTooBig( - "Request body exceeded " - "settings.DATA_UPLOAD_MAX_MEMORY_SIZE." + try: + content_type = meta_data["content-type"][0].strip() + except KeyError: + content_type = None + selected_parser = None + if content_type: + for parser in self._parsers: + if parser.can_handle(content_type): + selected_parser = parser + break + if selected_parser: + # TODO maybe .parse() shouldn't return an empty MultiValueDict + # for files if it's not needed + self._post.appendlist( + field_name, selected_parser.parse(data)[0] ) + else: + # Add two here to make the check consistent with the + # x-www-form-urlencoded check that includes '&='. + num_bytes_read += len(field_name) + 2 + if ( + settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None + and num_bytes_read > settings.DATA_UPLOAD_MAX_MEMORY_SIZE + ): + raise RequestDataTooBig( + "Request body exceeded " + "settings.DATA_UPLOAD_MAX_MEMORY_SIZE." + ) - self._post.appendlist( - field_name, force_str(data, encoding, errors="replace") - ) + self._post.appendlist( + field_name, force_str(data, encoding, errors="replace") + ) elif item_type == FILE: # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FILES. num_files += 1 diff --git a/django/http/parsers.py b/django/http/parsers.py index bd86c2dac7..0fb2271fa2 100644 --- a/django/http/parsers.py +++ b/django/http/parsers.py @@ -1,3 +1,4 @@ +import json from io import BytesIO from django.core.exceptions import BadRequest @@ -7,18 +8,24 @@ from django.utils.datastructures import ImmutableList, MultiValueDict class BaseParser: media_type = None + parsers = None def can_handle(self, media_type): return media_type == self.media_type - def parse(self, request): + def parse(self, data, request=None): pass + @property + def _supports_form_parsing(self): + form_media = ("application/x-www-form-urlencoded", "multipart/form-data") + return self.media_type in form_media + class FormParser(BaseParser): media_type = "application/x-www-form-urlencoded" - def parse(self, request): + def parse(self, data, request=None): from django.http import QueryDict # According to RFC 1866, the "application/x-www-form-urlencoded" @@ -35,22 +42,35 @@ class FormParser(BaseParser): class MultiPartParser(BaseParser): media_type = "multipart/form-data" - def parse(self, request): + def parse(self, data, request=None): if hasattr(request, "_body"): # Use already read data data = BytesIO(request._body) else: data = request - request.upload_handlers = ImmutableList( - request.upload_handlers, - warning=( - "You cannot alter upload handlers after the upload has been " - "processed." - ), - ) + # TODO - POST and data can be called on the same request. This parser can be + # called multiple times on the same request. While `_post` `_data` are different + # _files is the same. Allow parsing them twice, but don't change the handlers? + if not hasattr(request, "_files"): + request.upload_handlers = ImmutableList( + request.upload_handlers, + warning=( + "You cannot alter upload handlers after the upload has been " + "processed." + ), + ) parser = _MultiPartParser( - request.META, data, request.upload_handlers, request.encoding + request.META, data, request.upload_handlers, request.encoding, self.parsers ) + # TODO _post could also be _data _post, _files = parser.parse() return _post, _files + + +class JSONParser(BaseParser): + media_type = "application/json" + + def parse(self, data, request=None): + # TODO enable strict mode. Like DRF. + return json.loads(data), MultiValueDict() diff --git a/django/http/request.py b/django/http/request.py index bc9662282e..75779c2764 100644 --- a/django/http/request.py +++ b/django/http/request.py @@ -30,6 +30,8 @@ host_validation_re = _lazy_re_compile( r"^([a-z0-9.-]+|\[[a-f0-9]*:[a-f0-9.:]+\])(?::([0-9]+))?$" ) +POST_PARSERS = (parsers.FormParser(), parsers.MultiPartParser()) + class UnreadablePostError(OSError): pass @@ -72,6 +74,7 @@ class HttpRequest: self._parsers = [ parsers.FormParser(), parsers.MultiPartParser(), + parsers.JSONParser(), ] def __repr__(self): @@ -350,43 +353,52 @@ class HttpRequest: self._post = QueryDict() self._files = MultiValueDict() - def _load_post_and_files(self): - """Populate self._post and self._files if the content-type is a form type""" - if self.method != "POST": + def _load_post_and_files( + self, data_attr="_post", parsers=POST_PARSERS, methods=("POST",) + ): + if methods and self.method not in methods: self._post, self._files = ( QueryDict(encoding=self._encoding), MultiValueDict(), ) return + if self._read_started and not hasattr(self, "_body"): - self._mark_post_parse_error() + setattr(self, data_attr, QueryDict()) + self._files = MultiValueDict() return - # TODO create a parsers setter/getter/initializer like upload_handlers - parser_list = (parsers.FormParser(), parsers.MultiPartParser()) - selected_parser = None - for parser in parser_list: + for parser in parsers: if parser.can_handle(self.content_type): selected_parser = parser break if selected_parser: + selected_parser.parsers = parsers try: - self._post, self._files = parser.parse(self) + if selected_parser._supports_form_parsing: + # TODO Not sure how to make these consistent. + data, self._files = parser.parse(None, self) + setattr(self, data_attr, data) + else: + data, self._files = parser.parse(self.body, self) + setattr(self, data_attr, data) except Exception as e: # TODO 'application/x-www-form-urlencoded' didn't do this. # An error occurred while parsing POST data. Since when # formatting the error the request handler might access # self.POST, set self._post and self._file to prevent # attempts to parse POST data again. - self._mark_post_parse_error() + data_attr = QueryDict() + self._files = MultiValueDict() raise e else: - self._post, self._files = ( + data, self._files = ( QueryDict(encoding=self._encoding), MultiValueDict(), ) + setattr(self, data_attr, data) def close(self): if hasattr(self, "_files"): @@ -427,13 +439,23 @@ class HttpRequest: @parsers.setter def parsers(self, parsers): - # TODO Also check for _data once added - if hasattr(self, "_files"): + if hasattr(self, "_data") or hasattr(self, "_files"): raise AttributeError( "You cannot change parsers after processing the request's content." ) self._parsers = parsers + # TODO should this property be on [WSGI|ASGI]Request? + @property + def data(self): + if not hasattr(self, "_data"): + self._load_post_and_files("_data", self.parsers, methods=None) + return self._data + + @data.setter + def data(self, data): + self._data = data + class HttpHeaders(CaseInsensitiveMapping): HTTP_PREFIX = "HTTP_" diff --git a/docs/ref/parsers.txt b/docs/ref/parsers.txt index fd732f51e5..b433bd3735 100644 --- a/docs/ref/parsers.txt +++ b/docs/ref/parsers.txt @@ -57,6 +57,16 @@ Parses HTML form content (). The ``parse()`` method returns a Parses multipart form content and supports file uploads. The method returns a ``QueryDict`` for ``data`` and an ``MultiValueDict`` for ``FILES``. +.. class:: JSONParser + + .. attribute:: media_type + + ``"application/json"`` + +The ``parse()`` method deserializes JSON to a Python dictionary. This is +returned for ``data`` and empty ``MultiValueDict`` is provided for ``FILES``. + + HttpRequest.Parsers =================== diff --git a/docs/ref/request-response.txt b/docs/ref/request-response.txt index 0a73e644da..34583af11e 100644 --- a/docs/ref/request-response.txt +++ b/docs/ref/request-response.txt @@ -119,6 +119,15 @@ All attributes should be considered read-only, unless stated otherwise. ``POST`` does *not* include file-upload information. See :attr:`FILES`. +.. attribute:: HttpRequest.data + + Similar to :attr:`HttpRequest.POST` but parses the :attr:`HttpRequest.body` + with the parsers returned by :attr:`HttpRequest.parsers``. By default this + will result in ``application/json`` data being parsed in addition to form + data. + + In addition data will be parsed for all :attr:`HttpRequest.method` methods. + .. attribute:: HttpRequest.COOKIES A dictionary containing all cookies. Keys and values are strings. diff --git a/tests/requests_tests/test_parsers.py b/tests/requests_tests/test_parsers.py index 8f23ec3988..c9ba5e001a 100644 --- a/tests/requests_tests/test_parsers.py +++ b/tests/requests_tests/test_parsers.py @@ -1,6 +1,6 @@ from django.core.handlers.wsgi import WSGIRequest from django.http import HttpRequest -from django.http.parsers import BaseParser, FormParser, MultiPartParser +from django.http.parsers import BaseParser, FormParser, JSONParser, MultiPartParser from django.test import SimpleTestCase from django.test.client import FakePayload from django.utils.http import urlencode @@ -31,9 +31,10 @@ class TestParsers(SimpleTestCase): def test_request_parser_no_setting(self): request = HttpRequest() - form, multipart = request.parsers + form, multipart, json = request.parsers self.assertIsInstance(form, FormParser) self.assertIsInstance(multipart, MultiPartParser) + self.assertIsInstance(json, JSONParser) def test_set_parser(self): request = HttpRequest() diff --git a/tests/requests_tests/tests.py b/tests/requests_tests/tests.py index 95ddf9bb27..4f9b32d838 100644 --- a/tests/requests_tests/tests.py +++ b/tests/requests_tests/tests.py @@ -14,7 +14,7 @@ from django.http import ( UnreadablePostError, ) from django.http.multipartparser import MAX_TOTAL_HEADER_SIZE, MultiPartParserError -from django.http.request import split_domain_port +from django.http.request import QueryDict, split_domain_port from django.test import RequestFactory, SimpleTestCase, override_settings from django.test.client import BOUNDARY, MULTIPART_CONTENT, FakePayload @@ -99,6 +99,7 @@ class RequestsTests(SimpleTestCase): ) self.assertEqual(list(request.GET), []) self.assertEqual(list(request.POST), []) + self.assertEqual(list(request.data), []) self.assertEqual(list(request.COOKIES), []) self.assertEqual( set(request.META), @@ -339,7 +340,7 @@ class RequestsTests(SimpleTestCase): def test_read_after_value(self): """ Reading from request is allowed after accessing request contents as - POST or body. + POST, data or body. """ payload = FakePayload("name=value") request = WSGIRequest( @@ -351,12 +352,13 @@ class RequestsTests(SimpleTestCase): } ) self.assertEqual(request.POST, {"name": ["value"]}) + self.assertEqual(request.data, {"name": ["value"]}) self.assertEqual(request.body, b"name=value") self.assertEqual(request.read(), b"name=value") def test_value_after_read(self): """ - Construction of POST or body is not allowed after reading + Construction of POST, data or body is not allowed after reading from request. """ payload = FakePayload("name=value") @@ -372,6 +374,7 @@ class RequestsTests(SimpleTestCase): with self.assertRaises(RawPostDataException): request.body self.assertEqual(request.POST, {}) + self.assertEqual(request.data, {}) def test_non_ascii_POST(self): payload = FakePayload(urlencode({"key": "España"})) @@ -384,6 +387,7 @@ class RequestsTests(SimpleTestCase): } ) self.assertEqual(request.POST, {"key": ["España"]}) + self.assertEqual(request.data, {"key": ["España"]}) def test_non_utf8_charset_POST_bad_request(self): payload = FakePayload(urlencode({"key": "España".encode("latin-1")})) @@ -400,6 +404,8 @@ class RequestsTests(SimpleTestCase): ) with self.assertRaisesMessage(BadRequest, msg): request.POST + with self.assertRaisesMessage(BadRequest, msg): + request.data request = WSGIRequest(environ) with self.assertRaisesMessage(BadRequest, msg): request.FILES @@ -419,6 +425,7 @@ class RequestsTests(SimpleTestCase): } ) self.assertEqual(request.POST, {"key": ["España"]}) + self.assertEqual(request.data, {"key": ["España"]}) def test_body_after_POST_multipart_form_data(self): """ @@ -477,6 +484,7 @@ class RequestsTests(SimpleTestCase): } ) self.assertEqual(request.POST, {}) + self.assertEqual(request.data, {}) self.assertEqual(request.body, payload_data) def test_POST_multipart_with_content_length_zero(self): @@ -506,6 +514,7 @@ class RequestsTests(SimpleTestCase): } ) self.assertEqual(request.POST, {}) + self.assertEqual(request.data, {}) @override_settings( FILE_UPLOAD_HANDLERS=["requests_tests.tests.ErrorFileUploadHandler"] @@ -559,7 +568,7 @@ class RequestsTests(SimpleTestCase): self.assertEqual(request.POST, "_POST") self.assertEqual(request.FILES, "_FILES") - def test_request_methods_with_content(self): + def test_request_methods_with_content_POST(self): for method in ["GET", "PUT", "DELETE"]: with self.subTest(method=method): payload = FakePayload(urlencode({"key": "value"})) @@ -573,6 +582,20 @@ class RequestsTests(SimpleTestCase): ) self.assertEqual(request.POST, {}) + def test_request_methods_with_content_data(self): + for method in ["GET", "PUT", "DELETE"]: + with self.subTest(method=method): + payload = FakePayload(urlencode({"key": "value"})) + request = WSGIRequest( + { + "REQUEST_METHOD": method, + "CONTENT_LENGTH": len(payload), + "CONTENT_TYPE": "application/x-www-form-urlencoded", + "wsgi.input": payload, + } + ) + self.assertEqual(request.data, QueryDict("key=value")) + def test_POST_content_type_json(self): payload = FakePayload( "\r\n".join( @@ -593,12 +616,39 @@ class RequestsTests(SimpleTestCase): self.assertEqual(request.POST, {}) self.assertEqual(request.FILES, {}) + def test_data_content_type_json(self): + payload = FakePayload( + "\r\n".join( + [ + '{"pk": 1, "model": "store.book", "fields": {"name": "Mostly Ha' + 'rmless", "author": ["Douglas", "Adams"]}}', + ] + ) + ) + request = WSGIRequest( + { + "REQUEST_METHOD": "POST", + "CONTENT_TYPE": "application/json", + "CONTENT_LENGTH": len(payload), + "wsgi.input": payload, + } + ) + self.assertEqual( + request.data, + { + "pk": 1, + "model": "store.book", + "fields": {"name": "Mostly Harmless", "author": ["Douglas", "Adams"]}, + }, + ) + self.assertEqual(request.FILES, {}) + _json_payload = [ 'Content-Disposition: form-data; name="JSON"', "Content-Type: application/json", "", '{"pk": 1, "model": "store.book", "fields": {"name": "Mostly Harmless", ' - '"author": ["Douglas", Adams"]}}', + '"author": ["Douglas", "Adams"]}}', ] def test_POST_form_data_json(self): @@ -618,7 +668,7 @@ class RequestsTests(SimpleTestCase): { "JSON": [ '{"pk": 1, "model": "store.book", "fields": {"name": "Mostly ' - 'Harmless", "author": ["Douglas", Adams"]}}' + 'Harmless", "author": ["Douglas", "Adams"]}}' ], }, ) @@ -651,7 +701,46 @@ class RequestsTests(SimpleTestCase): "name": ["value"], "JSON": [ '{"pk": 1, "model": "store.book", "fields": {"name": "Mostly ' - 'Harmless", "author": ["Douglas", Adams"]}}' + 'Harmless", "author": ["Douglas", "Adams"]}}' + ], + }, + ) + + def test_data_multipart_json(self): + payload = FakePayload( + "\r\n".join( + [ + f"--{BOUNDARY}", + 'Content-Disposition: form-data; name="name"', + "", + "value", + f"--{BOUNDARY}", + *self._json_payload, + f"--{BOUNDARY}--", + ] + ) + ) + request = WSGIRequest( + { + "REQUEST_METHOD": "POST", + "CONTENT_TYPE": MULTIPART_CONTENT, + "CONTENT_LENGTH": len(payload), + "wsgi.input": payload, + } + ) + self.assertEqual( + request.data, + { + "name": ["value"], + "JSON": [ + { + "pk": 1, + "model": "store.book", + "fields": { + "name": "Mostly Harmless", + "author": ["Douglas", "Adams"], + }, + } ], }, ) @@ -689,7 +778,52 @@ class RequestsTests(SimpleTestCase): "name": ["value"], "JSON": [ '{"pk": 1, "model": "store.book", "fields": {"name": "Mostly ' - 'Harmless", "author": ["Douglas", Adams"]}}' + 'Harmless", "author": ["Douglas", "Adams"]}}' + ], + "CSV": ["Framework,ID.Django,1.Flask,2."], + }, + ) + + def test_data_multipart_json_csv(self): + payload = FakePayload( + "\r\n".join( + [ + f"--{BOUNDARY}", + 'Content-Disposition: form-data; name="name"', + "", + "value", + f"--{BOUNDARY}", + *self._json_payload, + f"--{BOUNDARY}", + 'Content-Disposition: form-data; name="CSV"', + "Content-Type: text/csv", + "", + "Framework,ID.Django,1.Flask,2.", + f"--{BOUNDARY}--", + ] + ) + ) + request = WSGIRequest( + { + "REQUEST_METHOD": "POST", + "CONTENT_TYPE": MULTIPART_CONTENT, + "CONTENT_LENGTH": len(payload), + "wsgi.input": payload, + } + ) + self.assertEqual( + request.data, + { + "name": ["value"], + "JSON": [ + { + "pk": 1, + "model": "store.book", + "fields": { + "name": "Mostly Harmless", + "author": ["Douglas", "Adams"], + }, + } ], "CSV": ["Framework,ID.Django,1.Flask,2."], }, @@ -730,7 +864,55 @@ class RequestsTests(SimpleTestCase): "name": ["value"], "JSON": [ '{"pk": 1, "model": "store.book", "fields": {"name": "Mostly ' - 'Harmless", "author": ["Douglas", Adams"]}}' + 'Harmless", "author": ["Douglas", "Adams"]}}' + ], + }, + ) + self.assertEqual(len(request.FILES), 1) + self.assertIsInstance((request.FILES["File"]), InMemoryUploadedFile) + + def test_data_multipart_with_file(self): + payload = FakePayload( + "\r\n".join( + [ + f"--{BOUNDARY}", + 'Content-Disposition: form-data; name="name"', + "", + "value", + f"--{BOUNDARY}", + *self._json_payload, + f"--{BOUNDARY}", + 'Content-Disposition: form-data; name="File"; filename="test.csv"', + "Content-Type: application/octet-stream", + "", + "Framework,ID", + "Django,1", + "Flask,2", + f"--{BOUNDARY}--", + ] + ) + ) + request = WSGIRequest( + { + "REQUEST_METHOD": "POST", + "CONTENT_TYPE": MULTIPART_CONTENT, + "CONTENT_LENGTH": len(payload), + "wsgi.input": payload, + } + ) + self.assertEqual( + request.data, + { + "name": ["value"], + "JSON": [ + { + "pk": 1, + "model": "store.book", + "fields": { + "name": "Mostly Harmless", + "author": ["Douglas", "Adams"], + }, + } ], }, ) @@ -794,9 +976,9 @@ class RequestsTests(SimpleTestCase): ) self.assertEqual(list(request), [b"name=value"]) - def test_POST_after_body_read(self): + def test_POST_and_data_after_body_read(self): """ - POST should be populated even if body is read first + POST and data should be populated even if body is read first """ payload = FakePayload("name=value") request = WSGIRequest( @@ -809,10 +991,11 @@ class RequestsTests(SimpleTestCase): ) request.body # evaluate self.assertEqual(request.POST, {"name": ["value"]}) + self.assertEqual(request.data, {"name": ["value"]}) - def test_POST_after_body_read_and_stream_read(self): + def test_POST_and_data_after_body_read_and_stream_read(self): """ - POST should be populated even if body is read first, and then + POST and data should be populated even if body is read first, and then the stream is read second. """ payload = FakePayload("name=value") @@ -827,6 +1010,7 @@ class RequestsTests(SimpleTestCase): request.body # evaluate self.assertEqual(request.read(1), b"n") self.assertEqual(request.POST, {"name": ["value"]}) + self.assertEqual(request.data, {"name": ["value"]}) def test_multipart_post_field_with_base64(self): payload = FakePayload(