mirrors/django
1
0
Fork 0
mirror of https://github.com/django/django.git synced 2025-04-06 06:26:41 +00:00
Code Issues 32 Releases Wiki Activity

Added JSON Parser and request.data

Browse Source
This commit is contained in:
David Smith 2023-12-04 17:20:38 +00:00
parent 31aabb20c2
commit 32ea23f53b
9 changed files with 319 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
django/core/handlers/asgi.py
View File

@ -115,6 +115,7 @@ class ASGIRequest(HttpRequest):
self._parsers = [
parsers.FormParser(),
parsers.MultiPartParser(),
parsers.JSONParser(),
]
@cached_property

1
django/core/handlers/wsgi.py
View File

@ -81,6 +81,7 @@ class WSGIRequest(HttpRequest):
self._parsers = [
parsers.FormParser(),
parsers.MultiPartParser(),
parsers.JSONParser(),
]
def _get_scheme(self):

46
django/http/multipartparser.py
View File

@ -54,7 +54,7 @@ class MultiPartParser:
boundary_re = _lazy_re_compile(r"[ -~]{0,200}[!-~]")
def __init__(self, META, input_data, upload_handlers, encoding=None):
def __init__(self, META, input_data, upload_handlers, encoding=None, parsers=None):
"""
Initialize the MultiPartParser object.
@ -112,6 +112,7 @@ class MultiPartParser:
self._encoding = encoding or settings.DEFAULT_CHARSET
self._content_length = content_length
self._upload_handlers = upload_handlers
self._parsers = parsers
def parse(self):
# Call the actual parse routine and close all open files in case of
@ -236,21 +237,38 @@ class MultiPartParser:
data = field_stream.read(size=read_size)
num_bytes_read += len(data)
# Add two here to make the check consistent with the
# x-www-form-urlencoded check that includes '&='.
num_bytes_read += len(field_name) + 2
if (
settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None
and num_bytes_read > settings.DATA_UPLOAD_MAX_MEMORY_SIZE
):
raise RequestDataTooBig(
"Request body exceeded "
"settings.DATA_UPLOAD_MAX_MEMORY_SIZE."
try:
content_type = meta_data["content-type"][0].strip()
except KeyError:
content_type = None
selected_parser = None
if content_type:
for parser in self._parsers:
if parser.can_handle(content_type):
selected_parser = parser
break
if selected_parser:
# TODO maybe .parse() shouldn't return an empty MultiValueDict
# for files if it's not needed
self._post.appendlist(
field_name, selected_parser.parse(data)[0]
)
else:
# Add two here to make the check consistent with the
# x-www-form-urlencoded check that includes '&='.
num_bytes_read += len(field_name) + 2
if (
settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None
and num_bytes_read > settings.DATA_UPLOAD_MAX_MEMORY_SIZE
):
raise RequestDataTooBig(
"Request body exceeded "
"settings.DATA_UPLOAD_MAX_MEMORY_SIZE."
)
self._post.appendlist(
field_name, force_str(data, encoding, errors="replace")
)
self._post.appendlist(
field_name, force_str(data, encoding, errors="replace")
)
elif item_type == FILE:
# Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FILES.
num_files += 1

42
django/http/parsers.py
View File

@ -1,3 +1,4 @@
import json
from io import BytesIO
from django.core.exceptions import BadRequest
@ -7,18 +8,24 @@ from django.utils.datastructures import ImmutableList, MultiValueDict
class BaseParser:
media_type = None
parsers = None
def can_handle(self, media_type):
return media_type == self.media_type
def parse(self, request):
def parse(self, data, request=None):
pass
@property
def _supports_form_parsing(self):
form_media = ("application/x-www-form-urlencoded", "multipart/form-data")
return self.media_type in form_media
class FormParser(BaseParser):
media_type = "application/x-www-form-urlencoded"
def parse(self, request):
def parse(self, data, request=None):
from django.http import QueryDict
# According to RFC 1866, the "application/x-www-form-urlencoded"
@ -35,22 +42,35 @@ class FormParser(BaseParser):
class MultiPartParser(BaseParser):
media_type = "multipart/form-data"
def parse(self, request):
def parse(self, data, request=None):
if hasattr(request, "_body"):
# Use already read data
data = BytesIO(request._body)
else:
data = request
request.upload_handlers = ImmutableList(
request.upload_handlers,
warning=(
"You cannot alter upload handlers after the upload has been "
"processed."
),
)
# TODO - POST and data can be called on the same request. This parser can be
# called multiple times on the same request. While `_post` `_data` are different
# _files is the same. Allow parsing them twice, but don't change the handlers?
if not hasattr(request, "_files"):
request.upload_handlers = ImmutableList(
request.upload_handlers,
warning=(
"You cannot alter upload handlers after the upload has been "
"processed."
),
)
parser = _MultiPartParser(
request.META, data, request.upload_handlers, request.encoding
request.META, data, request.upload_handlers, request.encoding, self.parsers
)
# TODO _post could also be _data
_post, _files = parser.parse()
return _post, _files
class JSONParser(BaseParser):
media_type = "application/json"
def parse(self, data, request=None):
# TODO enable strict mode. Like DRF.
return json.loads(data), MultiValueDict()

48
django/http/request.py
View File

@ -30,6 +30,8 @@ host_validation_re = _lazy_re_compile(
r"^([a-z0-9.-]+|\[[a-f0-9]*:[a-f0-9.:]+\])(?::([0-9]+))?$"
)
POST_PARSERS = (parsers.FormParser(), parsers.MultiPartParser())
class UnreadablePostError(OSError):
pass
@ -72,6 +74,7 @@ class HttpRequest:
self._parsers = [
parsers.FormParser(),
parsers.MultiPartParser(),
parsers.JSONParser(),
]
def __repr__(self):
@ -350,43 +353,52 @@ class HttpRequest:
self._post = QueryDict()
self._files = MultiValueDict()
def _load_post_and_files(self):
"""Populate self._post and self._files if the content-type is a form type"""
if self.method != "POST":
def _load_post_and_files(
self, data_attr="_post", parsers=POST_PARSERS, methods=("POST",)
):
if methods and self.method not in methods:
self._post, self._files = (
QueryDict(encoding=self._encoding),
MultiValueDict(),
)
return
if self._read_started and not hasattr(self, "_body"):
self._mark_post_parse_error()
setattr(self, data_attr, QueryDict())
self._files = MultiValueDict()
return
# TODO create a parsers setter/getter/initializer like upload_handlers
parser_list = (parsers.FormParser(), parsers.MultiPartParser())
selected_parser = None
for parser in parser_list:
for parser in parsers:
if parser.can_handle(self.content_type):
selected_parser = parser
break
if selected_parser:
selected_parser.parsers = parsers
try:
self._post, self._files = parser.parse(self)
if selected_parser._supports_form_parsing:
# TODO Not sure how to make these consistent.
data, self._files = parser.parse(None, self)
setattr(self, data_attr, data)
else:
data, self._files = parser.parse(self.body, self)
setattr(self, data_attr, data)
except Exception as e:
# TODO 'application/x-www-form-urlencoded' didn't do this.
# An error occurred while parsing POST data. Since when
# formatting the error the request handler might access
# self.POST, set self._post and self._file to prevent
# attempts to parse POST data again.
self._mark_post_parse_error()
data_attr = QueryDict()
self._files = MultiValueDict()
raise e
else:
self._post, self._files = (
data, self._files = (
QueryDict(encoding=self._encoding),
MultiValueDict(),
)
setattr(self, data_attr, data)
def close(self):
if hasattr(self, "_files"):
@ -427,13 +439,23 @@ class HttpRequest:
@parsers.setter
def parsers(self, parsers):
# TODO Also check for _data once added
if hasattr(self, "_files"):
if hasattr(self, "_data") or hasattr(self, "_files"):
raise AttributeError(
"You cannot change parsers after processing the request's content."
)
self._parsers = parsers
# TODO should this property be on [WSGI|ASGI]Request?
@property
def data(self):
if not hasattr(self, "_data"):
self._load_post_and_files("_data", self.parsers, methods=None)
return self._data
@data.setter
def data(self, data):
self._data = data
class HttpHeaders(CaseInsensitiveMapping):
HTTP_PREFIX = "HTTP_"

10
docs/ref/parsers.txt
View File

@ -57,6 +57,16 @@ Parses HTML form content (). The ``parse()`` method returns a
Parses multipart form content and supports file uploads. The method returns
a ``QueryDict`` for ``data`` and an ``MultiValueDict`` for ``FILES``.
.. class:: JSONParser
.. attribute:: media_type
``"application/json"``
The ``parse()`` method deserializes JSON to a Python dictionary. This is
returned for ``data`` and empty ``MultiValueDict`` is provided for ``FILES``.
HttpRequest.Parsers
===================

9
docs/ref/request-response.txt
View File

@ -119,6 +119,15 @@ All attributes should be considered read-only, unless stated otherwise.
``POST`` does *not* include file-upload information. See :attr:`FILES`.
.. attribute:: HttpRequest.data
Similar to :attr:`HttpRequest.POST` but parses the :attr:`HttpRequest.body`
with the parsers returned by :attr:`HttpRequest.parsers``. By default this
will result in ``application/json`` data being parsed in addition to form
data.
In addition data will be parsed for all :attr:`HttpRequest.method` methods.
.. attribute:: HttpRequest.COOKIES
A dictionary containing all cookies. Keys and values are strings.

5
tests/requests_tests/test_parsers.py
View File

@ -1,6 +1,6 @@
from django.core.handlers.wsgi import WSGIRequest
from django.http import HttpRequest
from django.http.parsers import BaseParser, FormParser, MultiPartParser
from django.http.parsers import BaseParser, FormParser, JSONParser, MultiPartParser
from django.test import SimpleTestCase
from django.test.client import FakePayload
from django.utils.http import urlencode
@ -31,9 +31,10 @@ class TestParsers(SimpleTestCase):
def test_request_parser_no_setting(self):
request = HttpRequest()
form, multipart = request.parsers
form, multipart, json = request.parsers
self.assertIsInstance(form, FormParser)
self.assertIsInstance(multipart, MultiPartParser)
self.assertIsInstance(json, JSONParser)
def test_set_parser(self):
request = HttpRequest()

210
tests/requests_tests/tests.py
View File

@ -14,7 +14,7 @@ from django.http import (
UnreadablePostError,
)
from django.http.multipartparser import MAX_TOTAL_HEADER_SIZE, MultiPartParserError
from django.http.request import split_domain_port
from django.http.request import QueryDict, split_domain_port
from django.test import RequestFactory, SimpleTestCase, override_settings
from django.test.client import BOUNDARY, MULTIPART_CONTENT, FakePayload
@ -99,6 +99,7 @@ class RequestsTests(SimpleTestCase):
)
self.assertEqual(list(request.GET), [])
self.assertEqual(list(request.POST), [])
self.assertEqual(list(request.data), [])
self.assertEqual(list(request.COOKIES), [])
self.assertEqual(
set(request.META),
@ -339,7 +340,7 @@ class RequestsTests(SimpleTestCase):
def test_read_after_value(self):
"""
Reading from request is allowed after accessing request contents as
POST or body.
POST, data or body.
"""
payload = FakePayload("name=value")
request = WSGIRequest(
@ -351,12 +352,13 @@ class RequestsTests(SimpleTestCase):
}
)
self.assertEqual(request.POST, {"name": ["value"]})
self.assertEqual(request.data, {"name": ["value"]})
self.assertEqual(request.body, b"name=value")
self.assertEqual(request.read(), b"name=value")
def test_value_after_read(self):
"""
Construction of POST or body is not allowed after reading
Construction of POST, data or body is not allowed after reading
from request.
"""
payload = FakePayload("name=value")
@ -372,6 +374,7 @@ class RequestsTests(SimpleTestCase):
with self.assertRaises(RawPostDataException):
request.body
self.assertEqual(request.POST, {})
self.assertEqual(request.data, {})
def test_non_ascii_POST(self):
payload = FakePayload(urlencode({"key": "España"}))
@ -384,6 +387,7 @@ class RequestsTests(SimpleTestCase):
}
)
self.assertEqual(request.POST, {"key": ["España"]})
self.assertEqual(request.data, {"key": ["España"]})
def test_non_utf8_charset_POST_bad_request(self):
payload = FakePayload(urlencode({"key": "España".encode("latin-1")}))
@ -400,6 +404,8 @@ class RequestsTests(SimpleTestCase):
)
with self.assertRaisesMessage(BadRequest, msg):
request.POST
with self.assertRaisesMessage(BadRequest, msg):
request.data
request = WSGIRequest(environ)
with self.assertRaisesMessage(BadRequest, msg):
request.FILES
@ -419,6 +425,7 @@ class RequestsTests(SimpleTestCase):
}
)
self.assertEqual(request.POST, {"key": ["España"]})
self.assertEqual(request.data, {"key": ["España"]})
def test_body_after_POST_multipart_form_data(self):
"""
@ -477,6 +484,7 @@ class RequestsTests(SimpleTestCase):
}
)
self.assertEqual(request.POST, {})
self.assertEqual(request.data, {})
self.assertEqual(request.body, payload_data)
def test_POST_multipart_with_content_length_zero(self):
@ -506,6 +514,7 @@ class RequestsTests(SimpleTestCase):
}
)
self.assertEqual(request.POST, {})
self.assertEqual(request.data, {})
@override_settings(
FILE_UPLOAD_HANDLERS=["requests_tests.tests.ErrorFileUploadHandler"]
@ -559,7 +568,7 @@ class RequestsTests(SimpleTestCase):
self.assertEqual(request.POST, "_POST")
self.assertEqual(request.FILES, "_FILES")
def test_request_methods_with_content(self):
def test_request_methods_with_content_POST(self):
for method in ["GET", "PUT", "DELETE"]:
with self.subTest(method=method):
payload = FakePayload(urlencode({"key": "value"}))
@ -573,6 +582,20 @@ class RequestsTests(SimpleTestCase):
)
self.assertEqual(request.POST, {})
def test_request_methods_with_content_data(self):
for method in ["GET", "PUT", "DELETE"]:
with self.subTest(method=method):
payload = FakePayload(urlencode({"key": "value"}))
request = WSGIRequest(
{
"REQUEST_METHOD": method,
"CONTENT_LENGTH": len(payload),
"CONTENT_TYPE": "application/x-www-form-urlencoded",
"wsgi.input": payload,
}
)
self.assertEqual(request.data, QueryDict("key=value"))
def test_POST_content_type_json(self):
payload = FakePayload(
"\r\n".join(
@ -593,12 +616,39 @@ class RequestsTests(SimpleTestCase):
self.assertEqual(request.POST, {})
self.assertEqual(request.FILES, {})
def test_data_content_type_json(self):
payload = FakePayload(
"\r\n".join(
[
'{"pk": 1, "model": "store.book", "fields": {"name": "Mostly Ha'
'rmless", "author": ["Douglas", "Adams"]}}',
]
)
)
request = WSGIRequest(
{
"REQUEST_METHOD": "POST",
"CONTENT_TYPE": "application/json",
"CONTENT_LENGTH": len(payload),
"wsgi.input": payload,
}
)
self.assertEqual(
request.data,
{
"pk": 1,
"model": "store.book",
"fields": {"name": "Mostly Harmless", "author": ["Douglas", "Adams"]},
},
)
self.assertEqual(request.FILES, {})
_json_payload = [
'Content-Disposition: form-data; name="JSON"',
"Content-Type: application/json",
"",
'{"pk": 1, "model": "store.book", "fields": {"name": "Mostly Harmless", '
'"author": ["Douglas", Adams"]}}',
'"author": ["Douglas", "Adams"]}}',
]
def test_POST_form_data_json(self):
@ -618,7 +668,7 @@ class RequestsTests(SimpleTestCase):
{
"JSON": [
'{"pk": 1, "model": "store.book", "fields": {"name": "Mostly '
'Harmless", "author": ["Douglas", Adams"]}}'
'Harmless", "author": ["Douglas", "Adams"]}}'
],
},
)
@ -651,7 +701,46 @@ class RequestsTests(SimpleTestCase):
"name": ["value"],
"JSON": [
'{"pk": 1, "model": "store.book", "fields": {"name": "Mostly '
'Harmless", "author": ["Douglas", Adams"]}}'
'Harmless", "author": ["Douglas", "Adams"]}}'
],
},
)
def test_data_multipart_json(self):
payload = FakePayload(
"\r\n".join(
[
f"--{BOUNDARY}",
'Content-Disposition: form-data; name="name"',
"",
"value",
f"--{BOUNDARY}",
*self._json_payload,
f"--{BOUNDARY}--",
]
)
)
request = WSGIRequest(
{
"REQUEST_METHOD": "POST",
"CONTENT_TYPE": MULTIPART_CONTENT,
"CONTENT_LENGTH": len(payload),
"wsgi.input": payload,
}
)
self.assertEqual(
request.data,
{
"name": ["value"],
"JSON": [
{
"pk": 1,
"model": "store.book",
"fields": {
"name": "Mostly Harmless",
"author": ["Douglas", "Adams"],
},
}
],
},
)
@ -689,7 +778,52 @@ class RequestsTests(SimpleTestCase):
"name": ["value"],
"JSON": [
'{"pk": 1, "model": "store.book", "fields": {"name": "Mostly '
'Harmless", "author": ["Douglas", Adams"]}}'
'Harmless", "author": ["Douglas", "Adams"]}}'
],
"CSV": ["Framework,ID.Django,1.Flask,2."],
},
)
def test_data_multipart_json_csv(self):
payload = FakePayload(
"\r\n".join(
[
f"--{BOUNDARY}",
'Content-Disposition: form-data; name="name"',
"",
"value",
f"--{BOUNDARY}",
*self._json_payload,
f"--{BOUNDARY}",
'Content-Disposition: form-data; name="CSV"',
"Content-Type: text/csv",
"",
"Framework,ID.Django,1.Flask,2.",
f"--{BOUNDARY}--",
]
)
)
request = WSGIRequest(
{
"REQUEST_METHOD": "POST",
"CONTENT_TYPE": MULTIPART_CONTENT,
"CONTENT_LENGTH": len(payload),
"wsgi.input": payload,
}
)
self.assertEqual(
request.data,
{
"name": ["value"],
"JSON": [
{
"pk": 1,
"model": "store.book",
"fields": {
"name": "Mostly Harmless",
"author": ["Douglas", "Adams"],
},
}
],
"CSV": ["Framework,ID.Django,1.Flask,2."],
},
@ -730,7 +864,55 @@ class RequestsTests(SimpleTestCase):
"name": ["value"],
"JSON": [
'{"pk": 1, "model": "store.book", "fields": {"name": "Mostly '
'Harmless", "author": ["Douglas", Adams"]}}'
'Harmless", "author": ["Douglas", "Adams"]}}'
],
},
)
self.assertEqual(len(request.FILES), 1)
self.assertIsInstance((request.FILES["File"]), InMemoryUploadedFile)
def test_data_multipart_with_file(self):
payload = FakePayload(
"\r\n".join(
[
f"--{BOUNDARY}",
'Content-Disposition: form-data; name="name"',
"",
"value",
f"--{BOUNDARY}",
*self._json_payload,
f"--{BOUNDARY}",
'Content-Disposition: form-data; name="File"; filename="test.csv"',
"Content-Type: application/octet-stream",
"",
"Framework,ID",
"Django,1",
"Flask,2",
f"--{BOUNDARY}--",
]
)
)
request = WSGIRequest(
{
"REQUEST_METHOD": "POST",
"CONTENT_TYPE": MULTIPART_CONTENT,
"CONTENT_LENGTH": len(payload),
"wsgi.input": payload,
}
)
self.assertEqual(
request.data,
{
"name": ["value"],
"JSON": [
{
"pk": 1,
"model": "store.book",
"fields": {
"name": "Mostly Harmless",
"author": ["Douglas", "Adams"],
},
}
],
},
)
@ -794,9 +976,9 @@ class RequestsTests(SimpleTestCase):
)
self.assertEqual(list(request), [b"name=value"])
def test_POST_after_body_read(self):
def test_POST_and_data_after_body_read(self):
"""
POST should be populated even if body is read first
POST and data should be populated even if body is read first
"""
payload = FakePayload("name=value")
request = WSGIRequest(
@ -809,10 +991,11 @@ class RequestsTests(SimpleTestCase):
)
request.body # evaluate
self.assertEqual(request.POST, {"name": ["value"]})
self.assertEqual(request.data, {"name": ["value"]})
def test_POST_after_body_read_and_stream_read(self):
def test_POST_and_data_after_body_read_and_stream_read(self):
"""
POST should be populated even if body is read first, and then
POST and data should be populated even if body is read first, and then
the stream is read second.
"""
payload = FakePayload("name=value")
@ -827,6 +1010,7 @@ class RequestsTests(SimpleTestCase):
request.body # evaluate
self.assertEqual(request.read(1), b"n")
self.assertEqual(request.POST, {"name": ["value"]})
self.assertEqual(request.data, {"name": ["value"]})
def test_multipart_post_field_with_base64(self):
payload = FakePayload(