From 1c6e8ec4ed6d9c374161eda965160e4782c7d71e Mon Sep 17 00:00:00 2001 From: Standa Opichal Date: Fri, 10 Nov 2023 17:40:24 +0100 Subject: [PATCH] Fixed #34968 -- Made multipart parsing of headers raise an error on too long headers. This also allow customizing the maximum size of headers via MAX_TOTAL_HEADER_SIZE. --- django/http/multipartparser.py | 36 ++++++++++++++--------- tests/file_uploads/tests.py | 52 ++++++++++++++++++++++++++++++++++ tests/requests_tests/tests.py | 27 +++++++++++++++++- 3 files changed, 101 insertions(+), 14 deletions(-) diff --git a/django/http/multipartparser.py b/django/http/multipartparser.py index 5ab63455ef..dfd406c35e 100644 --- a/django/http/multipartparser.py +++ b/django/http/multipartparser.py @@ -41,6 +41,7 @@ RAW = "raw" FILE = "file" FIELD = "field" FIELD_TYPES = frozenset([FIELD, RAW]) +MAX_TOTAL_HEADER_SIZE = 1024 class MultiPartParser: @@ -682,21 +683,30 @@ def parse_boundary_stream(stream, max_header_size): """ Parse one and exactly one stream that encapsulates a boundary. """ - # Stream at beginning of header, look for end of header - # and parse it if found. The header must fit within one - # chunk. - chunk = stream.read(max_header_size) - # 'find' returns the top of these four bytes, so we'll - # need to munch them later to prevent them from polluting - # the payload. - header_end = chunk.find(b"\r\n\r\n") + # Look for the end of headers and if not found extend the search to double + # the size up to the MAX_TOTAL_HEADER_SIZE. + headers_chunk_size = 1024 + while True: + if headers_chunk_size > max_header_size: + raise MultiPartParserError("Request max total header size exceeded.") - if header_end == -1: - # we find no header, so we just mark this fact and pass on - # the stream verbatim + # Stream at beginning of header, look for end of header and parse it if + # found. The header must fit within one chunk. + chunk = stream.read(headers_chunk_size) + # 'find' returns the top of these four bytes, so munch them later to + # prevent them from polluting the payload. + header_end = chunk.find(b"\r\n\r\n") + if header_end != -1: + break + + # Find no header, mark this fact and pass on the stream verbatim. stream.unget(chunk) - return (RAW, {}, stream) + # No more data to read. + if len(chunk) < headers_chunk_size: + return (RAW, {}, stream) + # Double the chunk size. + headers_chunk_size *= 2 header = chunk[:header_end] @@ -740,4 +750,4 @@ class Parser: boundarystream = InterBoundaryIter(self._stream, self._separator) for sub_stream in boundarystream: # Iterate over each part - yield parse_boundary_stream(sub_stream, 1024) + yield parse_boundary_stream(sub_stream, MAX_TOTAL_HEADER_SIZE) diff --git a/tests/file_uploads/tests.py b/tests/file_uploads/tests.py index 693efc4c62..9fdef57324 100644 --- a/tests/file_uploads/tests.py +++ b/tests/file_uploads/tests.py @@ -16,6 +16,7 @@ from django.core.files.storage import default_storage from django.core.files.uploadedfile import SimpleUploadedFile, UploadedFile from django.http.multipartparser import ( FILE, + MAX_TOTAL_HEADER_SIZE, MultiPartParser, MultiPartParserError, Parser, @@ -603,6 +604,57 @@ class FileUploadTests(TestCase): temp_path = response.json()["temp_path"] self.assertIs(os.path.exists(temp_path), False) + def test_upload_large_header_fields(self): + payload = client.FakePayload( + "\r\n".join( + [ + "--" + client.BOUNDARY, + 'Content-Disposition: form-data; name="my_file"; ' + 'filename="test.txt"', + "Content-Type: text/plain", + "X-Long-Header: %s" % ("-" * 500), + "", + "file contents", + "--" + client.BOUNDARY + "--\r\n", + ] + ), + ) + r = { + "CONTENT_LENGTH": len(payload), + "CONTENT_TYPE": client.MULTIPART_CONTENT, + "PATH_INFO": "/echo_content/", + "REQUEST_METHOD": "POST", + "wsgi.input": payload, + } + response = self.client.request(**r) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), {"my_file": "file contents"}) + + def test_upload_header_fields_too_large(self): + payload = client.FakePayload( + "\r\n".join( + [ + "--" + client.BOUNDARY, + 'Content-Disposition: form-data; name="my_file"; ' + 'filename="test.txt"', + "Content-Type: text/plain", + "X-Long-Header: %s" % ("-" * (MAX_TOTAL_HEADER_SIZE + 1)), + "", + "file contents", + "--" + client.BOUNDARY + "--\r\n", + ] + ), + ) + r = { + "CONTENT_LENGTH": len(payload), + "CONTENT_TYPE": client.MULTIPART_CONTENT, + "PATH_INFO": "/echo_content/", + "REQUEST_METHOD": "POST", + "wsgi.input": payload, + } + response = self.client.request(**r) + self.assertEqual(response.status_code, 400) + def test_fileupload_getlist(self): file = tempfile.NamedTemporaryFile with file() as file1, file() as file2, file() as file2a: diff --git a/tests/requests_tests/tests.py b/tests/requests_tests/tests.py index 228198ae8a..035552713c 100644 --- a/tests/requests_tests/tests.py +++ b/tests/requests_tests/tests.py @@ -11,7 +11,7 @@ from django.http import ( RawPostDataException, UnreadablePostError, ) -from django.http.multipartparser import MultiPartParserError +from django.http.multipartparser import MAX_TOTAL_HEADER_SIZE, MultiPartParserError from django.http.request import split_domain_port from django.test import RequestFactory, SimpleTestCase, override_settings from django.test.client import BOUNDARY, MULTIPART_CONTENT, FakePayload @@ -691,6 +691,31 @@ class RequestsTests(SimpleTestCase): with self.assertRaisesMessage(MultiPartParserError, msg): request.POST + def test_multipart_with_header_fields_too_large(self): + payload = FakePayload( + "\r\n".join( + [ + "--boundary", + 'Content-Disposition: form-data; name="name"', + "X-Long-Header: %s" % ("-" * (MAX_TOTAL_HEADER_SIZE + 1)), + "", + "value", + "--boundary--", + ] + ) + ) + request = WSGIRequest( + { + "REQUEST_METHOD": "POST", + "CONTENT_TYPE": "multipart/form-data; boundary=boundary", + "CONTENT_LENGTH": len(payload), + "wsgi.input": payload, + } + ) + msg = "Request max total header size exceeded." + with self.assertRaisesMessage(MultiPartParserError, msg): + request.POST + def test_POST_connection_error(self): """ If wsgi.input.read() raises an exception while trying to read() the