From 39676e8f56312646b246a27d789397bebcd22731 Mon Sep 17 00:00:00 2001 From: Kevin Kubasik Date: Tue, 2 Jun 2009 14:52:29 +0000 Subject: [PATCH] Merge branch 'gsoc' into local/gsoc git-svn-id: http://code.djangoproject.com/svn/django/branches/soc2009/test-improvements@10888 bcc190cf-cafb-0310-a4f2-bffc1f526a37 --- .gitignore | 1 + django/conf/global_settings.py | 51 ++- django/core/management/commands/test.py | 12 +- django/db/models/fields/files.py | 133 ++++-- django/test/coverage_report/__init__.py | 18 + django/test/coverage_report/data_storage.py | 57 +++ .../coverage_report/html_module_detail.py | 91 ++++ .../coverage_report/html_module_errors.py | 53 +++ .../coverage_report/html_module_exceptions.py | 32 ++ .../coverage_report/html_module_excludes.py | 55 +++ django/test/coverage_report/html_report.py | 143 +++++++ .../coverage_report/templates/__init__.py | 17 + .../templates/default_module_detail.py | 193 +++++++++ .../templates/default_module_errors.py | 23 + .../templates/default_module_exceptions.py | 89 ++++ .../templates/default_module_excludes.py | 23 + .../templates/default_module_index.py | 195 +++++++++ django/test/simple.py | 90 ++-- django/test/test_coverage.py | 106 +++++ django/test/utils.py | 14 +- django/utils/module_tools/__init__.py | 19 + django/utils/module_tools/data_storage.py | 42 ++ django/utils/module_tools/module_loader.py | 79 ++++ django/utils/module_tools/module_walker.py | 135 ++++++ tests/modeltests/model_forms/models.py | 21 +- tests/regressiontests/file_storage/models.py | 93 ---- tests/regressiontests/model_fields/4x8.png | Bin 0 -> 87 bytes tests/regressiontests/model_fields/8x4.png | Bin 0 -> 87 bytes .../model_fields/imagefield.py | 403 ++++++++++++++++++ tests/regressiontests/model_fields/models.py | 107 ++++- tests/regressiontests/model_fields/tests.py | 16 +- tests/runtests.py | 14 +- 32 files changed, 2123 insertions(+), 202 deletions(-) create mode 100644 .gitignore create mode 100644 django/test/coverage_report/__init__.py create mode 100644 django/test/coverage_report/data_storage.py create mode 100644 django/test/coverage_report/html_module_detail.py create mode 100644 django/test/coverage_report/html_module_errors.py create mode 100644 django/test/coverage_report/html_module_exceptions.py create mode 100644 django/test/coverage_report/html_module_excludes.py create mode 100644 django/test/coverage_report/html_report.py create mode 100644 django/test/coverage_report/templates/__init__.py create mode 100644 django/test/coverage_report/templates/default_module_detail.py create mode 100644 django/test/coverage_report/templates/default_module_errors.py create mode 100644 django/test/coverage_report/templates/default_module_exceptions.py create mode 100644 django/test/coverage_report/templates/default_module_excludes.py create mode 100644 django/test/coverage_report/templates/default_module_index.py create mode 100644 django/test/test_coverage.py create mode 100644 django/utils/module_tools/__init__.py create mode 100644 django/utils/module_tools/data_storage.py create mode 100644 django/utils/module_tools/module_loader.py create mode 100644 django/utils/module_tools/module_walker.py create mode 100644 tests/regressiontests/model_fields/4x8.png create mode 100644 tests/regressiontests/model_fields/8x4.png create mode 100644 tests/regressiontests/model_fields/imagefield.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..0d20b6487c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index 99fc72e468..431b5208b8 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -379,7 +379,7 @@ PASSWORD_RESET_TIMEOUT_DAYS = 3 ########### # The name of the method to use to invoke the test suite -TEST_RUNNER = 'django.test.simple.run_tests' +TEST_RUNNER = 'django.test.simple.DefaultTestRunner' # The name of the database to use for testing purposes. # If None, a name of 'test_' + DATABASE_NAME will be assumed @@ -392,6 +392,55 @@ TEST_DATABASE_NAME = None TEST_DATABASE_CHARSET = None TEST_DATABASE_COLLATION = None +############ +# COVERAGE # +############ + + +# Specify the coverage test runner +COVERAGE_TEST_RUNNER = 'django.test.test_coverage.BaseCoverageRunner' + +# Specify regular expressions of code blocks the coverage analyzer should +# ignore as statements (e.g. ``raise NotImplemented``). +# These statements are not figured in as part of the coverage statistics. +# This setting is optional. +COVERAGE_CODE_EXCLUDES = [ + 'def __unicode__\(self\):', 'def get_absolute_url\(self\):', + 'from .* import .*', 'import .*', + ] + +# Specify a list of regular expressions of paths to exclude from +# coverage analysis. +# Note these paths are ignored by the module introspection tool and take +# precedence over any package/module settings such as: +# TODO: THE SETTING FOR MODULES +# Use this to exclude subdirectories like ``r'.svn'``, for example. +# This setting is optional. +COVERAGE_PATH_EXCLUDES = [r'.svn'] + +# Specify a list of additional module paths to include +# in the coverage analysis. By default, only modules within installed +# apps are reported. If you have utility modules outside of the app +# structure, you can include them here. +# Note this list is *NOT* regular expression, so you have to be explicit, +# such as 'myproject.utils', and not 'utils$'. +# This setting is optional. +COVERAGE_ADDITIONAL_MODULES = [] + +# Specify a list of regular expressions of module paths to exclude +# from the coverage analysis. Examples are ``'tests$'`` and ``'urls$'``. +# This setting is optional. +COVERAGE_MODULE_EXCLUDES = ['tests$', 'settings$','urls$', 'common.views.test', + '__init__', 'django'] + +# Specify the directory where you would like the coverage report to create +# the HTML files. +# You'll need to make sure this directory exists and is writable by the +# user account running the test. +# You should probably set this one explicitly in your own settings file. +COVERAGE_REPORT_HTML_OUTPUT_DIR = 'test_html' + + ############ # FIXTURES # ############ diff --git a/django/core/management/commands/test.py b/django/core/management/commands/test.py index 8ebf3daea6..a710d74e08 100644 --- a/django/core/management/commands/test.py +++ b/django/core/management/commands/test.py @@ -6,6 +6,10 @@ class Command(BaseCommand): option_list = BaseCommand.option_list + ( make_option('--noinput', action='store_false', dest='interactive', default=True, help='Tells Django to NOT prompt the user for input of any kind.'), + make_option('--coverage', action='store_true', dest='coverage', default=False, + help='Tells Django to run the coverage runner'), + make_option('--reports', action='store_true', dest='reports', default=False, + help='Tells Django to output coverage results as HTML reports'), ) help = 'Runs the test suite for the specified applications, or the entire site if no apps are specified.' args = '[appname ...]' @@ -18,8 +22,10 @@ class Command(BaseCommand): verbosity = int(options.get('verbosity', 1)) interactive = options.get('interactive', True) - test_runner = get_runner(settings) - - failures = test_runner(test_labels, verbosity=verbosity, interactive=interactive) + cover = options.get('coverage', False) + report = options.get('reports', False) + test_runner = get_runner(settings, coverage=cover, reports=report) + tr = test_runner() + failures = tr.run_tests(test_labels, verbosity=verbosity, interactive=interactive) if failures: sys.exit(failures) diff --git a/django/db/models/fields/files.py b/django/db/models/fields/files.py index 61a903e36c..aab4f3789f 100644 --- a/django/db/models/fields/files.py +++ b/django/db/models/fields/files.py @@ -142,13 +142,13 @@ class FileDescriptor(object): """ The descriptor for the file attribute on the model instance. Returns a FieldFile when accessed so you can do stuff like:: - + >>> instance.file.size - + Assigns a file object on assignment so you can do:: - + >>> instance.file = File(...) - + """ def __init__(self, field): self.field = field @@ -156,9 +156,9 @@ class FileDescriptor(object): def __get__(self, instance=None, owner=None): if instance is None: raise AttributeError( - "The '%s' attribute can only be accessed from %s instances." + "The '%s' attribute can only be accessed from %s instances." % (self.field.name, owner.__name__)) - + # This is slightly complicated, so worth an explanation. # instance.file`needs to ultimately return some instance of `File`, # probably a subclass. Additionally, this returned object needs to have @@ -168,8 +168,8 @@ class FileDescriptor(object): # peek below you can see that we're not. So depending on the current # value of the field we have to dynamically construct some sort of # "thing" to return. - - # The instance dict contains whatever was originally assigned + + # The instance dict contains whatever was originally assigned # in __set__. file = instance.__dict__[self.field.name] @@ -186,14 +186,14 @@ class FileDescriptor(object): # Other types of files may be assigned as well, but they need to have # the FieldFile interface added to the. Thus, we wrap any other type of - # File inside a FieldFile (well, the field's attr_class, which is + # File inside a FieldFile (well, the field's attr_class, which is # usually FieldFile). elif isinstance(file, File) and not isinstance(file, FieldFile): file_copy = self.field.attr_class(instance, self.field, file.name) file_copy.file = file file_copy._committed = False instance.__dict__[self.field.name] = file_copy - + # Finally, because of the (some would say boneheaded) way pickle works, # the underlying FieldFile might not actually itself have an associated # file. So we need to reset the details of the FieldFile in those cases. @@ -201,7 +201,7 @@ class FileDescriptor(object): file.instance = instance file.field = self.field file.storage = self.field.storage - + # That was fun, wasn't it? return instance.__dict__[self.field.name] @@ -212,7 +212,7 @@ class FileField(Field): # The class to wrap instance attributes in. Accessing the file object off # the instance will always return an instance of attr_class. attr_class = FieldFile - + # The descriptor to use for accessing the attribute off of the class. descriptor_class = FileDescriptor @@ -300,40 +300,20 @@ class ImageFileDescriptor(FileDescriptor): assigning the width/height to the width_field/height_field, if appropriate. """ def __set__(self, instance, value): + previous_file = instance.__dict__.get(self.field.name) super(ImageFileDescriptor, self).__set__(instance, value) - - # The rest of this method deals with width/height fields, so we can - # bail early if neither is used. - if not self.field.width_field and not self.field.height_field: - return - - # We need to call the descriptor's __get__ to coerce this assigned - # value into an instance of the right type (an ImageFieldFile, in this - # case). - value = self.__get__(instance) - - if not value: - return - - # Get the image dimensions, making sure to leave the file in the same - # state (opened or closed) that we got it in. However, we *don't* rewind - # the file pointer if the file is already open. This is in keeping with - # most Python standard library file operations that leave it up to the - # user code to reset file pointers after operations that move it. - from django.core.files.images import get_image_dimensions - close = value.closed - value.open() - try: - width, height = get_image_dimensions(value) - finally: - if close: - value.close() - - # Update the width and height fields - if self.field.width_field: - setattr(value.instance, self.field.width_field, width) - if self.field.height_field: - setattr(value.instance, self.field.height_field, height) + + # To prevent recalculating image dimensions when we are instantiating + # an object from the database (bug #11084), only update dimensions if + # the field had a value before this assignment. Since the default + # value for FileField subclasses is an instance of field.attr_class, + # previous_file will only be None when we are called from + # Model.__init__(). The ImageField.update_dimension_fields method + # hooked up to the post_init signal handles the Model.__init__() cases. + # Assignment happening outside of Model.__init__() will trigger the + # update right here. + if previous_file is not None: + self.field.update_dimension_fields(instance, force=True) class ImageFieldFile(ImageFile, FieldFile): def delete(self, save=True): @@ -350,6 +330,69 @@ class ImageField(FileField): self.width_field, self.height_field = width_field, height_field FileField.__init__(self, verbose_name, name, **kwargs) + def contribute_to_class(self, cls, name): + super(ImageField, self).contribute_to_class(cls, name) + # Attach update_dimension_fields so that dimension fields declared + # after their corresponding image field don't stay cleared by + # Model.__init__, see bug #11196. + signals.post_init.connect(self.update_dimension_fields, sender=cls) + + def update_dimension_fields(self, instance, force=False, *args, **kwargs): + """ + Updates field's width and height fields, if defined. + + This method is hooked up to model's post_init signal to update + dimensions after instantiating a model instance. However, dimensions + won't be updated if the dimensions fields are already populated. This + avoids unnecessary recalculation when loading an object from the + database. + + Dimensions can be forced to update with force=True, which is how + ImageFileDescriptor.__set__ calls this method. + """ + # Nothing to update if the field doesn't have have dimension fields. + has_dimension_fields = self.width_field or self.height_field + if not has_dimension_fields: + return + + # getattr will call the ImageFileDescriptor's __get__ method, which + # coerces the assigned value into an instance of self.attr_class + # (ImageFieldFile in this case). + file = getattr(instance, self.attname) + + # Nothing to update if we have no file and not being forced to update. + if not file and not force: + return + + dimension_fields_filled = not( + (self.width_field and not getattr(instance, self.width_field)) + or (self.height_field and not getattr(instance, self.height_field)) + ) + # When both dimension fields have values, we are most likely loading + # data from the database or updating an image field that already had + # an image stored. In the first case, we don't want to update the + # dimension fields because we are already getting their values from the + # database. In the second case, we do want to update the dimensions + # fields and will skip this return because force will be True since we + # were called from ImageFileDescriptor.__set__. + if dimension_fields_filled and not force: + return + + # file should be an instance of ImageFieldFile or should be None. + if file: + width = file.width + height = file.height + else: + # No file, so clear dimensions fields. + width = None + height = None + + # Update the width and height fields. + if self.width_field: + setattr(instance, self.width_field, width) + if self.height_field: + setattr(instance, self.height_field, height) + def formfield(self, **kwargs): defaults = {'form_class': forms.ImageField} defaults.update(kwargs) diff --git a/django/test/coverage_report/__init__.py b/django/test/coverage_report/__init__.py new file mode 100644 index 0000000000..87e6eb8821 --- /dev/null +++ b/django/test/coverage_report/__init__.py @@ -0,0 +1,18 @@ +""" +Copyright 2009 55 Minutes (http://www.55minutes.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from html_report import * + diff --git a/django/test/coverage_report/data_storage.py b/django/test/coverage_report/data_storage.py new file mode 100644 index 0000000000..30f03fab70 --- /dev/null +++ b/django/test/coverage_report/data_storage.py @@ -0,0 +1,57 @@ +""" +Copyright 2009 55 Minutes (http://www.55minutes.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import coverage, time + +try: + set +except: + from sets import Set as set + + +class ModuleVars(object): + modules = dict() + def __new__(cls, module_name, module=None): + if cls.modules.get(module_name, None): + return cls.modules.get(module_name) + else: + obj=super(ModuleVars, cls).__new__(cls) + obj._init(module_name, module) + cls.modules[module_name] = obj + return obj + + def _init(self, module_name, module): + source_file, stmts, excluded, missed, missed_display = coverage.analysis2(module) + executed = list(set(stmts).difference(missed)) + total = list(set(stmts).union(excluded)) + total.sort() + title = module.__name__ + total_count = len(total) + executed_count = len(executed) + excluded_count = len(excluded) + missed_count = len(missed) + try: + percent_covered = float(len(executed))/len(stmts)*100 + except ZeroDivisionError: + percent_covered = 100 + test_timestamp = time.strftime('%a %Y-%m-%d %H:%M %Z') + severity = 'normal' + if percent_covered < 75: severity = 'warning' + if percent_covered < 50: severity = 'critical' + + for k, v in locals().iteritems(): + setattr(self, k, v) + diff --git a/django/test/coverage_report/html_module_detail.py b/django/test/coverage_report/html_module_detail.py new file mode 100644 index 0000000000..7c5372e3c3 --- /dev/null +++ b/django/test/coverage_report/html_module_detail.py @@ -0,0 +1,91 @@ +""" +Copyright 2009 55 Minutes (http://www.55minutes.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import cgi, os + +from data_storage import ModuleVars +from templates import default_module_detail as module_detail + +def html_module_detail(filename, module_name, nav=None): + """ + Creates a module detail report based on coverage testing at the specified + filename. If ``nav`` is specified, the nav template will be used as well. + + It uses `templates.default_module_detail` to create the page. The template + contains the following sections which need to be rendered and assembled into + the final HTML. + + TOP: Contains the HTML declaration and head information, as well as the + inline stylesheet. It requires the following variable: + * %(title)s The module name is probably fitting for this. + + CONTENT_HEADER: The header portion of the body. Requires the following variable: + * %(title)s + * %(source_file)s File path to the module + * %(total_count)d + * %(executed_count)d + * %(excluded_count)d + * %(ignored_count)d + * %(percent_covered)0.1f + * %(test_timestamp)s + + CONTENT_BODY: Annotated module source code listing. Requires the following variable: + * ``%(source_lines)s`` The actual source listing which is generated by + looping through each line and concatenanting together rendered + ``SOURCE_LINE`` template (see below). + + BOTTOM: Just a closing ```` + + SOURCE_LINE: Used to assemble the content of ``%(source_lines)s`` for ``CONTENT_BODY``. + Requires the following variables: + * ``%(line_status)s`` (ignored, executed, missed, excluded) used as CSS class + identifier to style the each source line. + * ``%(source_line)s`` + """ + if not nav: + nav = {} + m_vars = ModuleVars(module_name) + + m_vars.source_lines = source_lines = list() + i = 0 + for i, source_line in enumerate( + [cgi.escape(l.rstrip()) for l in file(m_vars.source_file, 'rb').readlines()]): + line_status = 'ignored' + if i+1 in m_vars.executed: line_status = 'executed' + if i+1 in m_vars.excluded: line_status = 'excluded' + if i+1 in m_vars.missed: line_status = 'missed' + source_lines.append(module_detail.SOURCE_LINE %vars()) + m_vars.ignored_count = i+1 - m_vars.total_count + m_vars.source_lines = os.linesep.join(source_lines) + + if 'prev_link' in nav and 'next_link' in nav: + nav_html = module_detail.NAV %nav + elif 'prev_link' in nav: + nav_html = module_detail.NAV_NO_NEXT %nav + elif 'next_link' in nav: + nav_html = module_detail.NAV_NO_PREV %nav + + fo = file(filename, 'wb+') + print >>fo, module_detail.TOP %m_vars.__dict__ + if nav: + print >>fo, nav_html + print >>fo, module_detail.CONTENT_HEADER %m_vars.__dict__ + print >>fo, module_detail.CONTENT_BODY %m_vars.__dict__ + if nav: + print >>fo, nav_html + print >>fo, module_detail.BOTTOM + fo.close() + diff --git a/django/test/coverage_report/html_module_errors.py b/django/test/coverage_report/html_module_errors.py new file mode 100644 index 0000000000..2578e1d097 --- /dev/null +++ b/django/test/coverage_report/html_module_errors.py @@ -0,0 +1,53 @@ +""" +Copyright 2009 55 Minutes (http://www.55minutes.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from html_module_exceptions import html_module_exceptions +from templates import default_module_errors as module_errors + +def html_module_errors(filename, errors): + """ + Creates an index page of packages and modules which had problems being imported + for coverage analysis at the specified filename. + + It uses `templates.default_module_errors` to create the page. The template + contains the following sections which need to be rendered and assembled into + the final HTML. + + TOP: Contains the HTML declaration and head information, as well as the + inline stylesheet. + + CONTENT_HEADER: The header portion of the body. + + CONTENT_BODY: A list of excluded packages and modules. Requires the following + variable: + * ``%(long_desc)s`` A long description of what this page + is about. + * ``%(exception_list)s`` List of package and module names + which is generated by looping through each line and + concatenanting together rendered ``EXCEPTION_LINE`` + template (see below). + + BOTTOM: Just a closing ```` + + EXCEPTION_LINE: Used to assemble the content of ``%(exception_list)s`` for ``CONTENT_BODY``. + Requires the following variable: + * ``%(module_name)s`` + """ + long_desc = """\ + test_coverage.utils.module_tools.find_or_load_module had + problems importing these packages and modules: + """ + html_module_exceptions(filename, errors, module_errors, long_desc) diff --git a/django/test/coverage_report/html_module_exceptions.py b/django/test/coverage_report/html_module_exceptions.py new file mode 100644 index 0000000000..43ad26d5d9 --- /dev/null +++ b/django/test/coverage_report/html_module_exceptions.py @@ -0,0 +1,32 @@ +""" +Copyright 2009 55 Minutes (http://www.55minutes.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os + +def html_module_exceptions(filename, exceptions, template, long_desc): + exception_list = [] + exceptions.sort() + for module_name in exceptions: + exception_list.append(template.EXCEPTION_LINE %vars()) + exception_list = os.linesep.join(exception_list) + + fo = file(filename, 'wb+') + print >>fo, template.TOP + print >>fo, template.CONTENT_HEADER + print >>fo, template.CONTENT_BODY %vars() + print >>fo, template.BOTTOM + fo.close() + diff --git a/django/test/coverage_report/html_module_excludes.py b/django/test/coverage_report/html_module_excludes.py new file mode 100644 index 0000000000..86b8204814 --- /dev/null +++ b/django/test/coverage_report/html_module_excludes.py @@ -0,0 +1,55 @@ +""" +Copyright 2009 55 Minutes (http://www.55minutes.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from html_module_exceptions import html_module_exceptions +from templates import default_module_excludes as module_excludes + +def html_module_excludes(filename, excludes): + """ + Creates an index page of packages and modules which were excluded from the + coverage analysis at the specified filename. + + It uses `templates.default_module_excludes` to create the page. The template + contains the following sections which need to be rendered and assembled into + the final HTML. + + TOP: Contains the HTML declaration and head information, as well as the + inline stylesheet. + + CONTENT_HEADER: The header portion of the body. + + CONTENT_BODY: A list of excluded packages and modules. Requires the following + variable: + * ``%(long_desc)s`` A long description of what this page + is about. + * ``%(exception_list)s`` List of package and module names + which is generated by looping through each line and + concatenanting together rendered ``EXCEPTION_LINE`` + template (see below). + + BOTTOM: Just a closing ```` + + EXCEPTION_LINE: Used to assemble the content of ``%(exclude_list)s`` for ``CONTENT_BODY``. + Requires the following variable: + * ``%(module_name)s`` + """ + long_desc = """\ + These packages and modules were excluded from the coverage analysis in + django.conf.settings.COVERAGE_MODULE_EXCLUDES or they do + not contain any executable statements: + """ + html_module_exceptions(filename, excludes, module_excludes, long_desc) + diff --git a/django/test/coverage_report/html_report.py b/django/test/coverage_report/html_report.py new file mode 100644 index 0000000000..30c40a0a18 --- /dev/null +++ b/django/test/coverage_report/html_report.py @@ -0,0 +1,143 @@ +""" +Copyright 2009 55 Minutes (http://www.55minutes.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os, time +from urllib import pathname2url as p2url + +from data_storage import ModuleVars +from html_module_detail import html_module_detail +from html_module_errors import html_module_errors +from html_module_excludes import html_module_excludes +from templates import default_module_index as module_index + +def html_report(outdir, modules, excludes=None, errors=None): + """ + Creates an ``index.html`` in the specified ``outdir``. Also attempts to create + a ``modules`` subdirectory to create module detail html pages, which are named + `module.__name__` + '.html'. + + It uses `templates.default_module_index` to create the index page. The template + contains the following sections which need to be rendered and assembled into + `index.html`. + + TOP: Contains the HTML declaration and head information, as well as the + inline stylesheet. It doesn't require any variables. + + CONTENT_HEADER: The header portion of the body. Requires the following variable: + * ``%(test_timestamp)s`` + + CONTENT_BODY: A table of contents to the different module detail pages, with some + basic stats. Requires the following variables: + * ``%(module_stats)s`` This is the actual content of the table and is + generated by looping through the modules we want to report on and + concatenanting together rendered ``MODULE_STAT`` template (see below). + * ``%(total_lines)d`` + * ``%(total_executed)d`` + * ``%(total_excluded)d`` + * ``%(overall_covered)0.1f`` + + EXCEPTIONS_LINK: Link to the excludes and errors index page which shows + packages and modules which were not part of the coverage + analysis. Requires the following variable: + * ``%(exceptions_link)s`` Link to the index page. + * ``%(exceptions_desc)s`` Describe the exception. + + ERRORS_LINK: Link to the errors index page which shows packages and modules which + had problems being imported. Requires the following variable: + * ``%(errors_link)s`` Link to the index page. + + BOTTOM: Just a closing ```` + + MODULE_STAT: Used to assemble the content of ``%(module_stats)s`` for ``CONTENT_BODY``. + Requires the following variables: + * ``%(severity)s`` (normal, warning, critical) used as CSS class identifier + to style the coverage percentage. + * ``%(module_link)s`` + * ``%(module_name)s`` + * ``%(total_count)d`` + * ``%(executed_count)d`` + * ``%(excluded_count)d`` + * ``%(percent_covered)0.1f`` + """ + # TODO: More robust directory checking and creation + outdir = os.path.abspath(outdir) + test_timestamp = time.strftime('%a %Y-%m-%d %H:%M %Z') + m_subdirname = 'modules' + m_dir = os.path.join(outdir, m_subdirname) + try: + os.mkdir(m_dir) + except OSError: + pass + + total_lines = 0 + total_executed = 0 + total_excluded = 0 + total_stmts = 0 + module_stats = list() + m_names = modules.keys() + m_names.sort() + for n in m_names: + m_vars = ModuleVars(n, modules[n]) + if not m_vars.total_count: + excludes.append(m_vars.module_name) + del modules[n] + continue + m_vars.module_link = p2url(os.path.join(m_subdirname, m_vars.module_name + '.html')) + module_stats.append(module_index.MODULE_STAT %m_vars.__dict__) + total_lines += m_vars.total_count + total_executed += m_vars.executed_count + total_excluded += m_vars.excluded_count + total_stmts += len(m_vars.stmts) + module_stats = os.linesep.join(module_stats) + overall_covered = float(total_executed)/total_stmts*100 + + m_names = modules.keys() + m_names.sort() + i = 0 + for i, n in enumerate(m_names): + m_vars = ModuleVars(n) + nav = dict(up_link=p2url(os.path.join('..', 'index.html')), + up_label='index') + if i > 0: + m = ModuleVars(m_names[i-1]) + nav['prev_link'] = os.path.basename(m.module_link) + nav['prev_label'] = m.module_name + if i+1 < len(modules): + m = ModuleVars(m_names[i+1]) + nav['next_link'] = os.path.basename(m.module_link) + nav['next_label'] = m.module_name + html_module_detail( + os.path.join(m_dir, m_vars.module_name + '.html'), n, nav) + + fo = file(os.path.join(outdir, 'index.html'), 'wb+') + print >>fo, module_index.TOP + print >>fo, module_index.CONTENT_HEADER %vars() + print >>fo, module_index.CONTENT_BODY %vars() + if excludes: + _file = 'excludes.html' + exceptions_link = _file + exception_desc = "Excluded packages and modules" + print >>fo, module_index.EXCEPTIONS_LINK %vars() + html_module_excludes(os.path.join(outdir, _file), excludes) + if errors: + _file = 'errors.html' + exceptions_link = _file + exception_desc = "Error packages and modules" + print >>fo, module_index.EXCEPTIONS_LINK %vars() + html_module_errors(os.path.join(outdir, _file), errors) + print >>fo, module_index.BOTTOM + fo.close() + diff --git a/django/test/coverage_report/templates/__init__.py b/django/test/coverage_report/templates/__init__.py new file mode 100644 index 0000000000..2d6f79d98f --- /dev/null +++ b/django/test/coverage_report/templates/__init__.py @@ -0,0 +1,17 @@ +""" +Copyright 2009 55 Minutes (http://www.55minutes.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + + diff --git a/django/test/coverage_report/templates/default_module_detail.py b/django/test/coverage_report/templates/default_module_detail.py new file mode 100644 index 0000000000..e942bf0454 --- /dev/null +++ b/django/test/coverage_report/templates/default_module_detail.py @@ -0,0 +1,193 @@ +""" +Copyright 2009 55 Minutes (http://www.55minutes.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +TOP = """\ + + + + + Test coverage report: %(title)s + + + + +""" + +NAV = """\ +
+ %(prev_label)s << + %(up_label)s + >> %(next_label)s +
+""" + +NAV_NO_PREV = """\ +
+ %(up_label)s + >> %(next_label)s +
+""" + +NAV_NO_NEXT = """\ +
+ %(prev_label)s << + %(up_label)s +
+""" + +CONTENT_HEADER = """\ +
+

+ %(title)s: + %(total_count)d total statements, + %(percent_covered)0.1f%% covered +

+

Generated: %(test_timestamp)s

+

Source file: %(source_file)s

+

+ Stats: + %(executed_count)d executed, + %(missed_count)d missed, + %(excluded_count)d excluded, + %(ignored_count)d ignored +

+
+""" + +CONTENT_BODY = """\ +
+
    + %(source_lines)s +
+
+""" + +SOURCE_LINE = '
  • %(source_line)s
    • ' + +BOTTOM = """\ + + +""" diff --git a/django/test/coverage_report/templates/default_module_errors.py b/django/test/coverage_report/templates/default_module_errors.py new file mode 100644 index 0000000000..2c8c4a6ed9 --- /dev/null +++ b/django/test/coverage_report/templates/default_module_errors.py @@ -0,0 +1,23 @@ +""" +Copyright 2009 55 Minutes (http://www.55minutes.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from default_module_exceptions import * + +title = "error packages and modules" + +TOP = TOP %vars() +CONTENT_HEADER = CONTENT_HEADER %vars() + diff --git a/django/test/coverage_report/templates/default_module_exceptions.py b/django/test/coverage_report/templates/default_module_exceptions.py new file mode 100644 index 0000000000..ff4eab8a1e --- /dev/null +++ b/django/test/coverage_report/templates/default_module_exceptions.py @@ -0,0 +1,89 @@ +""" +Copyright 2009 55 Minutes (http://www.55minutes.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import time + +test_timestamp = time.strftime('%a %Y-%m-%d %H:%M %Z') + +TOP = """\ + + + + + Test coverage report: %(title)s + + + + +""" + +CONTENT_HEADER = """\ +
    +

    Test Coverage Report: %(title)s

    """ +CONTENT_HEADER += "

    Generated: %(test_timestamp)s

    " %vars() +CONTENT_HEADER += "
    " + +CONTENT_BODY = """\ +
    +

    %(long_desc)s

    + + Back to index. +
    +""" + +EXCEPTION_LINE = "
  • %(module_name)s
    • " + +BOTTOM = """\ + + +""" diff --git a/django/test/coverage_report/templates/default_module_excludes.py b/django/test/coverage_report/templates/default_module_excludes.py new file mode 100644 index 0000000000..66e951c65d --- /dev/null +++ b/django/test/coverage_report/templates/default_module_excludes.py @@ -0,0 +1,23 @@ +""" +Copyright 2009 55 Minutes (http://www.55minutes.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from default_module_exceptions import * + +title = "excluded packages and modules" + +TOP = TOP %vars() +CONTENT_HEADER = CONTENT_HEADER %vars() + diff --git a/django/test/coverage_report/templates/default_module_index.py b/django/test/coverage_report/templates/default_module_index.py new file mode 100644 index 0000000000..1e76d57f56 --- /dev/null +++ b/django/test/coverage_report/templates/default_module_index.py @@ -0,0 +1,195 @@ +""" +Copyright 2009 55 Minutes (http://www.55minutes.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +TOP = """\ + + + + + Test coverage report + + + + +""" + +CONTENT_HEADER = """\ +
    +

    Test Coverage Report

    +

    Generated: %(test_timestamp)s

    +
    +""" + +CONTENT_BODY = """\ +
    + + + + + + + + + + + + + + + + + + + + + + + + + %(module_stats)s + +
     Statements
    Moduletotalexecutedexcluded%% covered
    Total%(total_lines)d%(total_executed)d%(total_excluded)d%(overall_covered)0.1f%%
    +
    +""" + +MODULE_STAT = """\ + + %(module_name)s + %(total_count)d + %(executed_count)d + %(excluded_count)d + %(percent_covered)0.1f%% + +""" + +EXCEPTIONS_LINK = """\ +
    + + %(exception_desc)s + +
    +""" + +BOTTOM = """\ + + +""" diff --git a/django/test/simple.py b/django/test/simple.py index f3c48bae33..af46e76aef 100644 --- a/django/test/simple.py +++ b/django/test/simple.py @@ -146,52 +146,64 @@ def reorder_suite(suite, classes): bins[0].addTests(bins[i+1]) return bins[0] -def run_tests(test_labels, verbosity=1, interactive=True, extra_tests=[]): +class DefaultTestRunner(object): """ - Run the unit tests for all the test labels in the provided list. - Labels must be of the form: - - app.TestClass.test_method - Run a single specific test method - - app.TestClass - Run all the test methods in a given class - - app - Search for doctests and unittests in the named application. - - When looking for tests, the test runner will look in the models and - tests modules for the application. - - A list of 'extra' tests may also be provided; these tests - will be added to the test suite. - - Returns the number of tests that failed. + The original test runner. No coverage reporting. """ - setup_test_environment() - settings.DEBUG = False - suite = unittest.TestSuite() + def __init__(self): + """ + Placeholder constructor. Want to make it obvious that it can + be overridden. + """ + pass - if test_labels: - for label in test_labels: - if '.' in label: - suite.addTest(build_test(label)) - else: - app = get_app(label) + def run_tests(self, test_labels, verbosity=1, interactive=True, extra_tests=[]): + """ + Run the unit tests for all the test labels in the provided list. + Labels must be of the form: + - app.TestClass.test_method + Run a single specific test method + - app.TestClass + Run all the test methods in a given class + - app + Search for doctests and unittests in the named application. + + When looking for tests, the test runner will look in the models and + tests modules for the application. + + A list of 'extra' tests may also be provided; these tests + will be added to the test suite. + + Returns the number of tests that failed. + """ + setup_test_environment() + + settings.DEBUG = False + suite = unittest.TestSuite() + + if test_labels: + for label in test_labels: + if '.' in label: + suite.addTest(build_test(label)) + else: + app = get_app(label) + suite.addTest(build_suite(app)) + else: + for app in get_apps(): suite.addTest(build_suite(app)) - else: - for app in get_apps(): - suite.addTest(build_suite(app)) - for test in extra_tests: - suite.addTest(test) + for test in extra_tests: + suite.addTest(test) - suite = reorder_suite(suite, (TestCase,)) + suite = reorder_suite(suite, (TestCase,)) - old_name = settings.DATABASE_NAME - from django.db import connection - connection.creation.create_test_db(verbosity, autoclobber=not interactive) - result = unittest.TextTestRunner(verbosity=verbosity).run(suite) - connection.creation.destroy_test_db(old_name, verbosity) + old_name = settings.DATABASE_NAME + from django.db import connection + connection.creation.create_test_db(verbosity, autoclobber=not interactive) + result = unittest.TextTestRunner(verbosity=verbosity).run(suite) + connection.creation.destroy_test_db(old_name, verbosity) - teardown_test_environment() + teardown_test_environment() - return len(result.failures) + len(result.errors) + return len(result.failures) + len(result.errors) diff --git a/django/test/test_coverage.py b/django/test/test_coverage.py new file mode 100644 index 0000000000..791b4d097e --- /dev/null +++ b/django/test/test_coverage.py @@ -0,0 +1,106 @@ +import coverage +import os, sys + +from django.conf import settings +from django.db.models import get_app, get_apps +from django.test.simple import DefaultTestRunner as base_run_tests + +from django.utils.module_tools import get_all_modules +from django.test.coverage_report import html_report +from django.utils.translation import ugettext as _ + +def _get_app_package(app_model_module): + """ + Returns the app module name from the app model module. + """ + return '.'.join(app_model_module.__name__.split('.')[:-1]) + + +class BaseCoverageRunner(object): + """ + Placeholder class for coverage runners. Intended to be easily extended. + """ + + def __init__(self): + """Placeholder (since it is overrideable)""" + pass + + def run_tests(self, test_labels, verbosity=1, interactive=True, + extra_tests=[]): + """ + Runs the specified tests while generating code coverage statistics. Upon + the tests' completion, the results are printed to stdout. + """ + #Allow an on-disk cache of coverage stats. + #coverage.use_cache(0) + for e in getattr(settings, 'COVERAGE_CODE_EXCLUDES', []): + coverage.exclude(e) + + coverage.start() + brt = base_run_tests() + results = brt.run_tests(test_labels, verbosity, interactive, extra_tests) + coverage.stop() + + coverage_modules = [] + if test_labels: + for label in test_labels: + label = label.split('.')[0] + app = get_app(label) + coverage_modules.append(_get_app_package(app)) + else: + for app in get_apps(): + coverage_modules.append(_get_app_package(app)) + + coverage_modules.extend(getattr(settings, 'COVERAGE_ADDITIONAL_MODULES', [])) + + packages, self.modules, self.excludes, self.errors = get_all_modules( + coverage_modules, getattr(settings, 'COVERAGE_MODULE_EXCLUDES', []), + getattr(settings, 'COVERAGE_PATH_EXCLUDES', [])) + + coverage.report(self.modules.values(), show_missing=1) + if self.excludes: + print >> sys.stdout + print >> sys.stdout, _("The following packages or modules were excluded:"), + for e in self.excludes: + print >> sys.stdout, e, + print >>sys.stdout + if self.errors: + print >> sys.stdout + print >> sys.stderr, _("There were problems with the following packages or modules:"), + for e in self.errors: + print >> sys.stderr, e, + print >> sys.stdout + return results + + +class ReportingCoverageRunner(BaseCoverageRunner): + """Runs coverage.py analysis, as well as generating detailed HTML reports.""" + + def __init__(self, outdir = None): + """ + Constructor, overrides BaseCoverageRunner. Sets output directory + for reports. Parameter or setting. + """ + if(outdir): + self.outdir = outdir + else: + # Realistically, we aren't going to ship the entire reporting framework.. + # but for the time being I have left it in. + self.outdir = getattr(settings, 'COVERAGE_REPORT_HTML_OUTPUT_DIR', 'test_html') + self.outdir = os.path.abspath(self.outdir) + # Create directory + if( not os.path.exists(self.outdir)): + os.mkdir(self.outdir) + + + def run_tests(self, *args, **kwargs): + """ + Overrides BaseCoverageRunner.run_tests, and adds html report generation + with the results + """ + res = BaseCoverageRunner.run_tests(self, *args, **kwargs) + html_report(self.outdir, self.modules, self.excludes, self.errors) + print >>sys.stdout + print >>sys.stdout, _("HTML reports were output to '%s'") %self.outdir + + return res diff --git a/django/test/utils.py b/django/test/utils.py index d34dd33d15..5d6686f210 100644 --- a/django/test/utils.py +++ b/django/test/utils.py @@ -80,8 +80,18 @@ def teardown_test_environment(): del mail.outbox -def get_runner(settings): - test_path = settings.TEST_RUNNER.split('.') +def get_runner(settings, coverage = False, reports = False): + """ + Based on the settings and parameters, returns the appropriate test + runner class. + """ + if(coverage): + if(reports): + test_path = 'django.test.test_coverage.ReportingCoverageRunner'.split('.') + else: + test_path = settings.COVERAGE_TEST_RUNNER.split('.') + else: + test_path = settings.TEST_RUNNER.split('.') # Allow for Python 2.5 relative paths if len(test_path) > 1: test_module_name = '.'.join(test_path[:-1]) diff --git a/django/utils/module_tools/__init__.py b/django/utils/module_tools/__init__.py new file mode 100644 index 0000000000..d940906d2d --- /dev/null +++ b/django/utils/module_tools/__init__.py @@ -0,0 +1,19 @@ +""" +Copyright 2009 55 Minutes (http://www.55minutes.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from module_loader import * +from module_walker import * + diff --git a/django/utils/module_tools/data_storage.py b/django/utils/module_tools/data_storage.py new file mode 100644 index 0000000000..aed5980e6b --- /dev/null +++ b/django/utils/module_tools/data_storage.py @@ -0,0 +1,42 @@ +""" +Copyright 2009 55 Minutes (http://www.55minutes.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +__all__ = ('Packages', 'Modules', 'Excluded', 'Errors') + +class SingletonType(type): + def __call__(cls, *args, **kwargs): + if getattr(cls, '__instance__', None) is None: + instance = cls.__new__(cls) + instance.__init__(*args, **kwargs) + cls.__instance__ = instance + return cls.__instance__ + +class Packages(object): + __metaclass__ = SingletonType + packages = {} + +class Modules(object): + __metaclass__ = SingletonType + modules = {} + +class Excluded(object): + __metaclass__ = SingletonType + excluded = [] + +class Errors(object): + __metaclass__ = SingletonType + errors = [] + diff --git a/django/utils/module_tools/module_loader.py b/django/utils/module_tools/module_loader.py new file mode 100644 index 0000000000..e6dd6ce4b8 --- /dev/null +++ b/django/utils/module_tools/module_loader.py @@ -0,0 +1,79 @@ +""" +Copyright 2009 55 Minutes (http://www.55minutes.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import imp, sys, types + +__all__ = ('find_or_load_module',) + +def _brute_force_find_module(module_name, module_path, module_type): + for m in [m for n, m in sys.modules.iteritems() if type(m) == types.ModuleType]: + m_path = [] + try: + if module_type in (imp.PY_COMPILED, imp.PY_SOURCE): + m_path = [m.__file__] + elif module_type==imp.PKG_DIRECTORY: + m_path = m.__path__ + except AttributeError: + pass + for p in m_path: + if p.startswith(module_path): + return m + return None + +def _load_module(module_name, fo, fp, desc): + suffix, mode, mtype = desc + if module_name in sys.modules and \ + sys.modules[module_name].__file__.startswith(fp): + module = sys.modules[module_name] + else: + module = _brute_force_find_module(module_name, fp, mtype) + if not module: + try: + module = imp.load_module(module_name, fo, fp, desc) + except: + raise ImportError + return module + +def _load_package(pkg_name, fp, desc): + suffix, mode, mtype = desc + if pkg_name in sys.modules: + if fp in sys.modules[pkg_name].__path__: + pkg = sys.modules[pkg_name] + else: + pkg = _brute_force_find_module(pkg_name, fp, mtype) + if not pkg: + pkg = imp.load_module(pkg_name, None, fp, desc) + return pkg + +def find_or_load_module(module_name, path=None): + """ + Attempts to lookup ``module_name`` in ``sys.modules``, else uses the + facilities in the ``imp`` module to load the module. + + If module_name specified is not of type ``imp.PY_SOURCE`` or + ``imp.PKG_DIRECTORY``, raise ``ImportError`` since we don't know + what to do with those. + """ + fo, fp, desc = imp.find_module(module_name.split('.')[-1], path) + suffix, mode, mtype = desc + if mtype in (imp.PY_SOURCE, imp.PY_COMPILED): + module = _load_module(module_name, fo, fp, desc) + elif mtype==imp.PKG_DIRECTORY: + module = _load_package(module_name, fp, desc) + else: + raise ImportError("Don't know how to handle this module type.") + return module + diff --git a/django/utils/module_tools/module_walker.py b/django/utils/module_tools/module_walker.py new file mode 100644 index 0000000000..442150e689 --- /dev/null +++ b/django/utils/module_tools/module_walker.py @@ -0,0 +1,135 @@ +""" +Copyright 2009 55 Minutes (http://www.55minutes.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os, re, sys +from glob import glob + +from data_storage import * +from module_loader import find_or_load_module + +try: + set +except: + from sets import Set as set + +__all__ = ('get_all_modules',) + +def _build_pkg_path(pkg_name, pkg, path): + for rp in [x for x in pkg.__path__ if path.startswith(x)]: + p = path.replace(rp, '').replace(os.path.sep, '.') + return pkg_name + p + +def _build_module_path(pkg_name, pkg, path): + return _build_pkg_path(pkg_name, pkg, os.path.splitext(path)[0]) + +def _prune_whitelist(whitelist, blacklist): + excluded = Excluded().excluded + + for wp in whitelist[:]: + for bp in blacklist: + if re.search(bp, wp): + whitelist.remove(wp) + excluded.append(wp) + break + return whitelist + +def _parse_module_list(m_list): + packages = Packages().packages + modules = Modules().modules + excluded = Excluded().excluded + errors = Errors().errors + + for m in m_list: + components = m.split('.') + m_name = '' + search_path = [] + processed=False + for i, c in enumerate(components): + m_name = '.'.join([x for x in m_name.split('.') if x] + [c]) + try: + module = find_or_load_module(m_name, search_path or None) + except ImportError: + processed=True + errors.append(m) + break + try: + search_path.extend(module.__path__) + except AttributeError: + processed = True + if i+1==len(components): + modules[m_name] = module + else: + errors.append(m) + break + if not processed: + packages[m_name] = module + +def prune_dirs(root, dirs, exclude_dirs): + _dirs = [os.path.join(root, d) for d in dirs] + for i, p in enumerate(_dirs): + for e in exclude_dirs: + if re.search(e, p): + del dirs[i] + break + +def _get_all_packages(pkg_name, pkg, blacklist, exclude_dirs): + packages = Packages().packages + errors = Errors().errors + + for path in pkg.__path__: + for root, dirs, files in os.walk(path): + prune_dirs(root, dirs, exclude_dirs or []) + m_name = _build_pkg_path(pkg_name, pkg, root) + try: + if _prune_whitelist([m_name], blacklist): + m = find_or_load_module(m_name, [os.path.split(root)[0]]) + packages[m_name] = m + else: + for d in dirs[:]: + dirs.remove(d) + except ImportError: + errors.append(m_name) + for d in dirs[:]: + dirs.remove(d) + +def _get_all_modules(pkg_name, pkg, blacklist): + modules = Modules().modules + errors = Errors().errors + + for p in pkg.__path__: + for f in glob('%s/*.py' %p): + m_name = _build_module_path(pkg_name, pkg, f) + try: + if _prune_whitelist([m_name], blacklist): + m = find_or_load_module(m_name, [p]) + modules[m_name] = m + except ImportError: + errors.append(m_name) + +def get_all_modules(whitelist, blacklist=None, exclude_dirs=None): + packages = Packages().packages + modules = Modules().modules + excluded = Excluded().excluded + errors = Errors().errors + + whitelist = _prune_whitelist(whitelist, blacklist or []) + _parse_module_list(whitelist) + for pkg_name, pkg in packages.copy().iteritems(): + _get_all_packages(pkg_name, pkg, blacklist, exclude_dirs) + for pkg_name, pkg in packages.copy().iteritems(): + _get_all_modules(pkg_name, pkg, blacklist) + return packages, modules, list(set(excluded)), list(set(errors)) + diff --git a/tests/modeltests/model_forms/models.py b/tests/modeltests/model_forms/models.py index 95fda273f0..0fd24c18ad 100644 --- a/tests/modeltests/model_forms/models.py +++ b/tests/modeltests/model_forms/models.py @@ -1175,8 +1175,9 @@ True >>> instance.height 16 -# Delete the current file since this is not done by Django. ->>> instance.image.delete() +# Delete the current file since this is not done by Django, but don't save +# because the dimension fields are not null=True. +>>> instance.image.delete(save=False) >>> f = ImageFileForm(data={'description': u'An image'}, files={'image': SimpleUploadedFile('test.png', image_data)}) >>> f.is_valid() @@ -1207,9 +1208,9 @@ True >>> instance.width 16 -# Delete the current image since this is not done by Django. - ->>> instance.image.delete() +# Delete the current file since this is not done by Django, but don't save +# because the dimension fields are not null=True. +>>> instance.image.delete(save=False) # Override the file by uploading a new one. @@ -1224,8 +1225,9 @@ True >>> instance.width 48 -# Delete the current file since this is not done by Django. ->>> instance.image.delete() +# Delete the current file since this is not done by Django, but don't save +# because the dimension fields are not null=True. +>>> instance.image.delete(save=False) >>> instance.delete() >>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': SimpleUploadedFile('test2.png', image_data2)}) @@ -1239,8 +1241,9 @@ True >>> instance.width 48 -# Delete the current file since this is not done by Django. ->>> instance.image.delete() +# Delete the current file since this is not done by Django, but don't save +# because the dimension fields are not null=True. +>>> instance.image.delete(save=False) >>> instance.delete() # Test the non-required ImageField diff --git a/tests/regressiontests/file_storage/models.py b/tests/regressiontests/file_storage/models.py index 32af5d7588..e69de29bb2 100644 --- a/tests/regressiontests/file_storage/models.py +++ b/tests/regressiontests/file_storage/models.py @@ -1,93 +0,0 @@ -import os -import tempfile -import shutil -from django.db import models -from django.core.files.storage import FileSystemStorage -from django.core.files.base import ContentFile - -# Test for correct behavior of width_field/height_field. -# Of course, we can't run this without PIL. - -try: - # Checking for the existence of Image is enough for CPython, but - # for PyPy, you need to check for the underlying modules - from PIL import Image, _imaging -except ImportError: - Image = None - -# If we have PIL, do these tests -if Image: - temp_storage_dir = tempfile.mkdtemp() - temp_storage = FileSystemStorage(temp_storage_dir) - - class Person(models.Model): - name = models.CharField(max_length=50) - mugshot = models.ImageField(storage=temp_storage, upload_to='tests', - height_field='mug_height', - width_field='mug_width') - mug_height = models.PositiveSmallIntegerField() - mug_width = models.PositiveSmallIntegerField() - - __test__ = {'API_TESTS': """ ->>> from django.core.files import File ->>> image_data = open(os.path.join(os.path.dirname(__file__), "test.png"), 'rb').read() ->>> p = Person(name="Joe") ->>> p.mugshot.save("mug", ContentFile(image_data)) ->>> p.mugshot.width -16 ->>> p.mugshot.height -16 ->>> p.mug_height -16 ->>> p.mug_width -16 - -# Bug #9786: Ensure '==' and '!=' work correctly. ->>> image_data = open(os.path.join(os.path.dirname(__file__), "test1.png"), 'rb').read() ->>> p1 = Person(name="Bob") ->>> p1.mugshot.save("mug", ContentFile(image_data)) ->>> p2 = Person.objects.get(name="Joe") ->>> p.mugshot == p2.mugshot -True ->>> p.mugshot != p2.mugshot -False ->>> p.mugshot != p1.mugshot -True - -Bug #9508: Similarly to the previous test, make sure hash() works as expected -(equal items must hash to the same value). ->>> hash(p.mugshot) == hash(p2.mugshot) -True - -# Bug #8175: correctly delete files that have been removed off the file system. ->>> import os ->>> p2 = Person(name="Fred") ->>> p2.mugshot.save("shot", ContentFile(image_data)) ->>> os.remove(p2.mugshot.path) ->>> p2.delete() - -# Bug #8534: FileField.size should not leave the file open. ->>> p3 = Person(name="Joan") ->>> p3.mugshot.save("shot", ContentFile(image_data)) - -# Get a "clean" model instance ->>> p3 = Person.objects.get(name="Joan") - -# It won't have an opened file. ->>> p3.mugshot.closed -True - -# After asking for the size, the file should still be closed. ->>> _ = p3.mugshot.size ->>> p3.mugshot.closed -True - -# Make sure that wrapping the file in a file still works ->>> p3.mugshot.file.open() ->>> p = Person.objects.create(name="Bob The Builder", mugshot=File(p3.mugshot.file)) ->>> p.save() ->>> p3.mugshot.file.close() - -# Delete all test files ->>> shutil.rmtree(temp_storage_dir) -"""} diff --git a/tests/regressiontests/model_fields/4x8.png b/tests/regressiontests/model_fields/4x8.png new file mode 100644 index 0000000000000000000000000000000000000000..ffce444d487d9c39a6c060b200e5bf7a1b18c9c0 GIT binary patch literal 87 zcmeAS@N?(olHy`uVBq!ia0vp^EI`b`!2~1&15bhk7>k44ofy`glX(f`2zt6WhHzX@ i{&T*8!H37>F#|)EBO|}3Wal!VB!j1`pUXO@geCyuc@ypc literal 0 HcmV?d00001 diff --git a/tests/regressiontests/model_fields/8x4.png b/tests/regressiontests/model_fields/8x4.png new file mode 100644 index 0000000000000000000000000000000000000000..60e6d69ee1da4a154ab87c29e347e76cdb76169c GIT binary patch literal 87 zcmeAS@N?(olHy`uVBq!ia0vp^96-#%!2~32*1ud1q!^2X+?^QKos)S9