diff --git a/django/core/management/commands/dumpdata.py b/django/core/management/commands/dumpdata.py index 194c169f67..925a23a56d 100644 --- a/django/core/management/commands/dumpdata.py +++ b/django/core/management/commands/dumpdata.py @@ -1,3 +1,5 @@ +import gzip +import os import warnings from django.apps import apps @@ -6,6 +8,18 @@ from django.core.management.base import BaseCommand, CommandError from django.core.management.utils import parse_apps_and_model_labels from django.db import DEFAULT_DB_ALIAS, router +try: + import bz2 + has_bz2 = True +except ImportError: + has_bz2 = False + +try: + import lzma + has_lzma = True +except ImportError: + has_lzma = False + class ProxyModelWarning(Warning): pass @@ -184,7 +198,36 @@ class Command(BaseCommand): if output and self.stdout.isatty() and options['verbosity'] > 0: progress_output = self.stdout object_count = sum(get_objects(count_only=True)) - stream = open(output, 'w') if output else None + if output: + file_root, file_ext = os.path.splitext(output) + compression_formats = { + '.bz2': (open, {}, file_root), + '.gz': (gzip.open, {}, output), + '.lzma': (open, {}, file_root), + '.xz': (open, {}, file_root), + '.zip': (open, {}, file_root), + } + if has_bz2: + compression_formats['.bz2'] = (bz2.open, {}, output) + if has_lzma: + compression_formats['.lzma'] = ( + lzma.open, {'format': lzma.FORMAT_ALONE}, output + ) + compression_formats['.xz'] = (lzma.open, {}, output) + try: + open_method, kwargs, file_path = compression_formats[file_ext] + except KeyError: + open_method, kwargs, file_path = (open, {}, output) + if file_path != output: + file_name = os.path.basename(file_path) + warnings.warn( + f"Unsupported file extension ({file_ext}). " + f"Fixtures saved in '{file_name}'.", + RuntimeWarning, + ) + stream = open_method(file_path, 'wt', **kwargs) + else: + stream = None try: serializers.serialize( format, get_objects(), indent=indent, diff --git a/docs/ref/django-admin.txt b/docs/ref/django-admin.txt index b2c885431f..c916ff96c8 100644 --- a/docs/ref/django-admin.txt +++ b/docs/ref/django-admin.txt @@ -364,6 +364,17 @@ standard output. When this option is set and ``--verbosity`` is greater than 0 (the default), a progress bar is shown in the terminal. +Fixtures compression +~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 3.2 + +The output file can be compressed with one of the ``bz2``, ``gz``, ``lzma``, or +``xz`` formats by ending the filename with the corresponding extension. +For example, to output the data as a compressed JSON file:: + + django-admin dumpdata -o mydata.json.gz + ``flush`` --------- diff --git a/docs/releases/3.2.txt b/docs/releases/3.2.txt index 45f42bc214..8796419c62 100644 --- a/docs/releases/3.2.txt +++ b/docs/releases/3.2.txt @@ -338,6 +338,9 @@ Management Commands * :djadmin:`loaddata` now supports fixtures stored in XZ archives (``.xz``) and LZMA archives (``.lzma``). +* :djadmin:`dumpdata` now can compress data in the ``bz2``, ``gz``, ``lzma``, + or ``xz`` formats. + * :djadmin:`makemigrations` can now be called without an active database connection. In that case, check for a consistent migration history is skipped. diff --git a/tests/fixtures/tests.py b/tests/fixtures/tests.py index e072844508..06ae3d183e 100644 --- a/tests/fixtures/tests.py +++ b/tests/fixtures/tests.py @@ -1,3 +1,4 @@ +import gzip import os import sys import tempfile @@ -80,9 +81,26 @@ class DumpDataAssertMixin: primary_keys=primary_keys, ) if filename: - with open(filename) as f: + file_root, file_ext = os.path.splitext(filename) + compression_formats = { + '.bz2': (open, file_root), + '.gz': (gzip.open, filename), + '.lzma': (open, file_root), + '.xz': (open, file_root), + '.zip': (open, file_root), + } + if HAS_BZ2: + compression_formats['.bz2'] = (bz2.open, filename) + if HAS_LZMA: + compression_formats['.lzma'] = (lzma.open, filename) + compression_formats['.xz'] = (lzma.open, filename) + try: + open_method, file_path = compression_formats[file_ext] + except KeyError: + open_method, file_path = open, filename + with open_method(file_path, 'rt') as f: command_output = f.read() - os.remove(filename) + os.remove(file_path) else: command_output = new_io.getvalue().strip() if format == "json": @@ -492,6 +510,66 @@ class FixtureLoadingTests(DumpDataAssertMixin, TestCase): filename='dumpdata.json' ) + def test_dumpdata_with_file_gzip_output(self): + management.call_command('loaddata', 'fixture1.json', verbosity=0) + self._dumpdata_assert( + ['fixtures'], + '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": ' + '"News Stories"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place ' + 'on ESPN", "pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": ' + '{"headline": "Time to reform copyright", "pub_date": "2006-06-16T13:00:00"}}]', + filename='dumpdata.json.gz', + ) + + @unittest.skipUnless(HAS_BZ2, 'No bz2 library detected.') + def test_dumpdata_with_file_bz2_output(self): + management.call_command('loaddata', 'fixture1.json', verbosity=0) + self._dumpdata_assert( + ['fixtures'], + '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": ' + '"News Stories"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place ' + 'on ESPN", "pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": ' + '{"headline": "Time to reform copyright", "pub_date": "2006-06-16T13:00:00"}}]', + filename='dumpdata.json.bz2', + ) + + @unittest.skipUnless(HAS_LZMA, 'No lzma library detected.') + def test_dumpdata_with_file_lzma_output(self): + management.call_command('loaddata', 'fixture1.json', verbosity=0) + self._dumpdata_assert( + ['fixtures'], + '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": ' + '"News Stories"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place ' + 'on ESPN", "pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": ' + '{"headline": "Time to reform copyright", "pub_date": "2006-06-16T13:00:00"}}]', + filename='dumpdata.json.lzma', + ) + + @unittest.skipUnless(HAS_LZMA, 'No lzma library detected.') + def test_dumpdata_with_file_xz_output(self): + management.call_command('loaddata', 'fixture1.json', verbosity=0) + self._dumpdata_assert( + ['fixtures'], + '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": ' + '"News Stories"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place ' + 'on ESPN", "pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": ' + '{"headline": "Time to reform copyright", "pub_date": "2006-06-16T13:00:00"}}]', + filename='dumpdata.json.xz', + ) + + def test_dumpdata_with_file_zip_output(self): + management.call_command('loaddata', 'fixture1.json', verbosity=0) + msg = "Unsupported file extension (.zip). Fixtures saved in 'dumpdata.json'." + with self.assertWarnsMessage(RuntimeWarning, msg): + self._dumpdata_assert( + ['fixtures'], + '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": ' + '"News Stories"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place ' + 'on ESPN", "pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": ' + '{"headline": "Time to reform copyright", "pub_date": "2006-06-16T13:00:00"}}]', + filename='dumpdata.json.zip', + ) + def test_dumpdata_progressbar(self): """ Dumpdata shows a progress bar on the command line when --output is set,