From 860c2c8bc5c77194c41464655851379bf512a052 Mon Sep 17 00:00:00 2001
From: Aymeric Augustin <aymeric.augustin@m4x.org>
Date: Wed, 11 Dec 2013 21:44:27 +0100
Subject: [PATCH] Moved django.db.models.loading to django.apps.cache.
This commit doesn't contain any code changes; it's purely a refactoring.
---
django/apps/__init__.py | 0
django/apps/cache.py | 396 ++++++++++++++++++
django/contrib/auth/tests/test_management.py | 2 +-
.../management/commands/makemigrations.py | 2 +-
django/core/management/commands/migrate.py | 2 +-
django/core/management/commands/shell.py | 2 +-
django/core/management/validation.py | 2 +-
django/db/backends/sqlite3/schema.py | 2 +-
django/db/migrations/loader.py | 2 +-
django/db/migrations/questioner.py | 2 +-
django/db/migrations/recorder.py | 2 +-
django/db/migrations/state.py | 2 +-
django/db/migrations/writer.py | 4 +-
django/db/models/__init__.py | 4 +-
django/db/models/base.py | 2 +-
django/db/models/fields/__init__.py | 2 +-
django/db/models/loading.py | 384 +----------------
django/db/models/options.py | 2 +-
django/db/models/signals.py | 2 +-
django/test/testcases.py | 2 +-
docs/internals/deprecation.txt | 3 +
tests/app_cache/models.py | 2 +-
tests/app_cache/tests.py | 6 +-
tests/app_loading/tests.py | 2 +-
tests/contenttypes_tests/tests.py | 2 +-
tests/defer_regress/tests.py | 2 +-
tests/empty/tests.py | 2 +-
tests/invalid_models/tests.py | 2 +-
tests/managers_regress/tests.py | 2 +-
tests/migrations/models.py | 2 +-
tests/migrations/test_commands.py | 2 +-
tests/migrations/test_state.py | 4 +-
tests/migrations/test_writer.py | 2 +-
tests/proxy_model_inheritance/tests.py | 2 +-
tests/proxy_models/tests.py | 2 +-
tests/runtests.py | 4 +-
tests/schema/models.py | 2 +-
tests/swappable_models/tests.py | 2 +-
tests/tablespaces/tests.py | 2 +-
tests/validation/test_unique.py | 2 +-
40 files changed, 448 insertions(+), 419 deletions(-)
create mode 100644 django/apps/__init__.py
create mode 100644 django/apps/cache.py
diff --git a/django/apps/__init__.py b/django/apps/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/django/apps/cache.py b/django/apps/cache.py
new file mode 100644
index 0000000000..a5b645646f
--- /dev/null
+++ b/django/apps/cache.py
@@ -0,0 +1,396 @@
+"Utilities for loading models and the modules that contain them."
+
+from collections import OrderedDict
+import copy
+import imp
+from importlib import import_module
+import os
+import sys
+
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+from django.utils.module_loading import module_has_submodule
+from django.utils._os import upath
+from django.utils import six
+
+__all__ = ('get_apps', 'get_app', 'get_models', 'get_model', 'register_models',
+ 'load_app', 'app_cache_ready')
+
+MODELS_MODULE_NAME = 'models'
+
+
+class ModelDict(OrderedDict):
+ """
+ We need to special-case the deepcopy for this, as the keys are modules,
+ which can't be deep copied.
+ """
+ def __deepcopy__(self, memo):
+ return self.__class__([(key, copy.deepcopy(value, memo))
+ for key, value in self.items()])
+
+
+class UnavailableApp(Exception):
+ pass
+
+
+def _initialize():
+ """
+ Returns a dictionary to be used as the initial value of the
+ [shared] state of the app cache.
+ """
+ return dict(
+ # Keys of app_store are the model modules for each application.
+ app_store=ModelDict(),
+
+ # Mapping of installed app_labels to model modules for that app.
+ app_labels={},
+
+ # Mapping of app_labels to a dictionary of model names to model code.
+ # May contain apps that are not installed.
+ app_models=ModelDict(),
+
+ # Mapping of app_labels to errors raised when trying to import the app.
+ app_errors={},
+
+ # Pending lookups for lazy relations
+ pending_lookups={},
+
+ # List of app_labels that allows restricting the set of apps.
+ # Used by TransactionTestCase.available_apps for performance reasons.
+ available_apps=None,
+
+ # -- Everything below here is only used when populating the cache --
+ loads_installed=True,
+ loaded=False,
+ handled=set(),
+ postponed=[],
+ nesting_level=0,
+ _get_models_cache={},
+ )
+
+
+class BaseAppCache(object):
+ """
+ A cache that stores installed applications and their models. Used to
+ provide reverse-relations and for app introspection (e.g. admin).
+
+ This provides the base (non-Borg) AppCache class - the AppCache
+ subclass adds borg-like behaviour for the few cases where it's needed.
+ """
+
+ def __init__(self):
+ self.__dict__ = _initialize()
+ # This stops _populate loading from INSTALLED_APPS and ignores the
+ # only_installed arguments to get_model[s]
+ self.loads_installed = False
+
+ def _populate(self):
+ """
+ Fill in all the cache information. This method is threadsafe, in the
+ sense that every caller will see the same state upon return, and if the
+ cache is already initialised, it does no work.
+ """
+ if self.loaded:
+ return
+ if not self.loads_installed:
+ self.loaded = True
+ return
+ # Note that we want to use the import lock here - the app loading is
+ # in many cases initiated implicitly by importing, and thus it is
+ # possible to end up in deadlock when one thread initiates loading
+ # without holding the importer lock and another thread then tries to
+ # import something which also launches the app loading. For details of
+ # this situation see #18251.
+ imp.acquire_lock()
+ try:
+ if self.loaded:
+ return
+ for app_name in settings.INSTALLED_APPS:
+ if app_name in self.handled:
+ continue
+ self.load_app(app_name, True)
+ if not self.nesting_level:
+ for app_name in self.postponed:
+ self.load_app(app_name)
+ self.loaded = True
+ finally:
+ imp.release_lock()
+
+ def _label_for(self, app_mod):
+ """
+ Return app_label for given models module.
+
+ """
+ return app_mod.__name__.split('.')[-2]
+
+ def load_app(self, app_name, can_postpone=False):
+ """
+ Loads the app with the provided fully qualified name, and returns the
+ model module.
+ """
+ app_module = import_module(app_name)
+ self.handled.add(app_name)
+ self.nesting_level += 1
+ try:
+ models = import_module('%s.%s' % (app_name, MODELS_MODULE_NAME))
+ except ImportError:
+ self.nesting_level -= 1
+ # If the app doesn't have a models module, we can just ignore the
+ # ImportError and return no models for it.
+ if not module_has_submodule(app_module, MODELS_MODULE_NAME):
+ return None
+ # But if the app does have a models module, we need to figure out
+ # whether to suppress or propagate the error. If can_postpone is
+ # True then it may be that the package is still being imported by
+ # Python and the models module isn't available yet. So we add the
+ # app to the postponed list and we'll try it again after all the
+ # recursion has finished (in populate). If can_postpone is False
+ # then it's time to raise the ImportError.
+ else:
+ if can_postpone:
+ self.postponed.append(app_name)
+ return None
+ else:
+ raise
+
+ self.nesting_level -= 1
+ if models not in self.app_store:
+ self.app_store[models] = len(self.app_store)
+ self.app_labels[self._label_for(models)] = models
+ return models
+
+ def app_cache_ready(self):
+ """
+ Returns true if the model cache is fully populated.
+
+ Useful for code that wants to cache the results of get_models() for
+ themselves once it is safe to do so.
+ """
+ return self.loaded
+
+ def get_apps(self):
+ """
+ Returns a list of all installed modules that contain models.
+ """
+ self._populate()
+
+ apps = self.app_store.items()
+ if self.available_apps is not None:
+ apps = [elt for elt in apps
+ if self._label_for(elt[0]) in self.available_apps]
+
+ # Ensure the returned list is always in the same order (with new apps
+ # added at the end). This avoids unstable ordering on the admin app
+ # list page, for example.
+ apps = sorted(apps, key=lambda elt: elt[1])
+
+ return [elt[0] for elt in apps]
+
+ def _get_app_package(self, app):
+ return '.'.join(app.__name__.split('.')[:-1])
+
+ def get_app_package(self, app_label):
+ return self._get_app_package(self.get_app(app_label))
+
+ def _get_app_path(self, app):
+ if hasattr(app, '__path__'): # models/__init__.py package
+ app_path = app.__path__[0]
+ else: # models.py module
+ app_path = app.__file__
+ return os.path.dirname(upath(app_path))
+
+ def get_app_path(self, app_label):
+ return self._get_app_path(self.get_app(app_label))
+
+ def get_app_paths(self):
+ """
+ Returns a list of paths to all installed apps.
+
+ Useful for discovering files at conventional locations inside apps
+ (static files, templates, etc.)
+ """
+ self._populate()
+
+ app_paths = []
+ for app in self.get_apps():
+ app_paths.append(self._get_app_path(app))
+ return app_paths
+
+ def get_app(self, app_label, emptyOK=False):
+ """
+ Returns the module containing the models for the given app_label.
+
+ Returns None if the app has no models in it and emptyOK is True.
+
+ Raises UnavailableApp when set_available_apps() in in effect and
+ doesn't include app_label.
+ """
+ self._populate()
+ imp.acquire_lock()
+ try:
+ for app_name in settings.INSTALLED_APPS:
+ if app_label == app_name.split('.')[-1]:
+ mod = self.load_app(app_name, False)
+ if mod is None and not emptyOK:
+ raise ImproperlyConfigured("App with label %s is missing a models.py module." % app_label)
+ if self.available_apps is not None and app_label not in self.available_apps:
+ raise UnavailableApp("App with label %s isn't available." % app_label)
+ return mod
+ raise ImproperlyConfigured("App with label %s could not be found" % app_label)
+ finally:
+ imp.release_lock()
+
+ def get_app_errors(self):
+ "Returns the map of known problems with the INSTALLED_APPS."
+ self._populate()
+ return self.app_errors
+
+ def get_models(self, app_mod=None,
+ include_auto_created=False, include_deferred=False,
+ only_installed=True, include_swapped=False):
+ """
+ Given a module containing models, returns a list of the models.
+ Otherwise returns a list of all installed models.
+
+ By default, auto-created models (i.e., m2m models without an
+ explicit intermediate table) are not included. However, if you
+ specify include_auto_created=True, they will be.
+
+ By default, models created to satisfy deferred attribute
+ queries are *not* included in the list of models. However, if
+ you specify include_deferred, they will be.
+
+ By default, models that aren't part of installed apps will *not*
+ be included in the list of models. However, if you specify
+ only_installed=False, they will be. If you're using a non-default
+ AppCache, this argument does nothing - all models will be included.
+
+ By default, models that have been swapped out will *not* be
+ included in the list of models. However, if you specify
+ include_swapped, they will be.
+ """
+ if not self.loads_installed:
+ only_installed = False
+ cache_key = (app_mod, include_auto_created, include_deferred, only_installed, include_swapped)
+ model_list = None
+ try:
+ model_list = self._get_models_cache[cache_key]
+ if self.available_apps is not None and only_installed:
+ model_list = [m for m in model_list if m._meta.app_label in self.available_apps]
+ return model_list
+ except KeyError:
+ pass
+ self._populate()
+ if app_mod:
+ if app_mod in self.app_store:
+ app_list = [self.app_models.get(self._label_for(app_mod), ModelDict())]
+ else:
+ app_list = []
+ else:
+ if only_installed:
+ app_list = [self.app_models.get(app_label, ModelDict())
+ for app_label in six.iterkeys(self.app_labels)]
+ else:
+ app_list = six.itervalues(self.app_models)
+ model_list = []
+ for app in app_list:
+ model_list.extend(
+ model for model in app.values()
+ if ((not model._deferred or include_deferred) and
+ (not model._meta.auto_created or include_auto_created) and
+ (not model._meta.swapped or include_swapped))
+ )
+ self._get_models_cache[cache_key] = model_list
+ if self.available_apps is not None and only_installed:
+ model_list = [m for m in model_list if m._meta.app_label in self.available_apps]
+ return model_list
+
+ def get_model(self, app_label, model_name,
+ seed_cache=True, only_installed=True):
+ """
+ Returns the model matching the given app_label and case-insensitive
+ model_name.
+
+ Returns None if no model is found.
+
+ Raises UnavailableApp when set_available_apps() in in effect and
+ doesn't include app_label.
+ """
+ if not self.loads_installed:
+ only_installed = False
+ if seed_cache:
+ self._populate()
+ if only_installed and app_label not in self.app_labels:
+ return None
+ if (self.available_apps is not None and only_installed
+ and app_label not in self.available_apps):
+ raise UnavailableApp("App with label %s isn't available." % app_label)
+ try:
+ return self.app_models[app_label][model_name.lower()]
+ except KeyError:
+ return None
+
+ def register_models(self, app_label, *models):
+ """
+ Register a set of models as belonging to an app.
+ """
+ for model in models:
+ # Store as 'name: model' pair in a dictionary
+ # in the app_models dictionary
+ model_name = model._meta.model_name
+ model_dict = self.app_models.setdefault(app_label, ModelDict())
+ if model_name in model_dict:
+ # The same model may be imported via different paths (e.g.
+ # appname.models and project.appname.models). We use the source
+ # filename as a means to detect identity.
+ fname1 = os.path.abspath(upath(sys.modules[model.__module__].__file__))
+ fname2 = os.path.abspath(upath(sys.modules[model_dict[model_name].__module__].__file__))
+ # Since the filename extension could be .py the first time and
+ # .pyc or .pyo the second time, ignore the extension when
+ # comparing.
+ if os.path.splitext(fname1)[0] == os.path.splitext(fname2)[0]:
+ continue
+ model_dict[model_name] = model
+ self._get_models_cache.clear()
+
+ def set_available_apps(self, available):
+ if not set(available).issubset(set(settings.INSTALLED_APPS)):
+ extra = set(available) - set(settings.INSTALLED_APPS)
+ raise ValueError("Available apps isn't a subset of installed "
+ "apps, extra apps: " + ", ".join(extra))
+ self.available_apps = set(app.rsplit('.', 1)[-1] for app in available)
+
+ def unset_available_apps(self):
+ self.available_apps = None
+
+
+class AppCache(BaseAppCache):
+ """
+ A cache that stores installed applications and their models. Used to
+ provide reverse-relations and for app introspection (e.g. admin).
+
+ Borg version of the BaseAppCache class.
+ """
+
+ __shared_state = _initialize()
+
+ def __init__(self):
+ self.__dict__ = self.__shared_state
+
+
+cache = AppCache()
+
+
+# These methods were always module level, so are kept that way for backwards
+# compatibility.
+get_apps = cache.get_apps
+get_app_package = cache.get_app_package
+get_app_path = cache.get_app_path
+get_app_paths = cache.get_app_paths
+get_app = cache.get_app
+get_app_errors = cache.get_app_errors
+get_models = cache.get_models
+get_model = cache.get_model
+register_models = cache.register_models
+load_app = cache.load_app
+app_cache_ready = cache.app_cache_ready
diff --git a/django/contrib/auth/tests/test_management.py b/django/contrib/auth/tests/test_management.py
index bd51f39977..698947d57c 100644
--- a/django/contrib/auth/tests/test_management.py
+++ b/django/contrib/auth/tests/test_management.py
@@ -1,6 +1,7 @@
from __future__ import unicode_literals
from datetime import date
+from django.apps.cache import get_app
from django.contrib.auth import models, management
from django.contrib.auth.management import create_permissions
from django.contrib.auth.management.commands import changepassword
@@ -12,7 +13,6 @@ from django.core import exceptions
from django.core.management import call_command
from django.core.management.base import CommandError
from django.core.management.validation import get_validation_errors
-from django.db.models.loading import get_app
from django.test import TestCase
from django.test.utils import override_settings
from django.utils import six
diff --git a/django/core/management/commands/makemigrations.py b/django/core/management/commands/makemigrations.py
index 239a0a416c..7c4bb4f7cf 100644
--- a/django/core/management/commands/makemigrations.py
+++ b/django/core/management/commands/makemigrations.py
@@ -3,6 +3,7 @@ import os
import operator
from optparse import make_option
+from django.apps.cache import cache
from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import ImproperlyConfigured
from django.db import connections, DEFAULT_DB_ALIAS, migrations
@@ -11,7 +12,6 @@ from django.db.migrations.autodetector import MigrationAutodetector
from django.db.migrations.questioner import MigrationQuestioner, InteractiveMigrationQuestioner
from django.db.migrations.state import ProjectState
from django.db.migrations.writer import MigrationWriter
-from django.db.models.loading import cache
from django.utils.six.moves import reduce
diff --git a/django/core/management/commands/migrate.py b/django/core/management/commands/migrate.py
index 093c8a79d0..1966223042 100644
--- a/django/core/management/commands/migrate.py
+++ b/django/core/management/commands/migrate.py
@@ -6,6 +6,7 @@ from importlib import import_module
import itertools
import traceback
+from django.apps.cache import cache
from django.conf import settings
from django.core.management import call_command
from django.core.management.base import BaseCommand, CommandError
@@ -16,7 +17,6 @@ from django.db.migrations.executor import MigrationExecutor
from django.db.migrations.loader import MigrationLoader, AmbiguityError
from django.db.migrations.state import ProjectState
from django.db.migrations.autodetector import MigrationAutodetector
-from django.db.models.loading import cache
from django.utils.module_loading import module_has_submodule
diff --git a/django/core/management/commands/shell.py b/django/core/management/commands/shell.py
index 00a6602c0b..0d84c9b2ec 100644
--- a/django/core/management/commands/shell.py
+++ b/django/core/management/commands/shell.py
@@ -66,7 +66,7 @@ class Command(NoArgsCommand):
def handle_noargs(self, **options):
# XXX: (Temporary) workaround for ticket #1796: force early loading of all
# models from installed apps.
- from django.db.models.loading import get_models
+ from django.apps.cache import get_models
get_models()
use_plain = options.get('plain', False)
diff --git a/django/core/management/validation.py b/django/core/management/validation.py
index 6bdcf853d4..459923a088 100644
--- a/django/core/management/validation.py
+++ b/django/core/management/validation.py
@@ -26,8 +26,8 @@ def get_validation_errors(outfile, app=None):
validates all models of all installed apps. Writes errors, if any, to outfile.
Returns number of errors.
"""
+ from django.apps.cache import get_app_errors
from django.db import connection, models
- from django.db.models.loading import get_app_errors
from django.db.models.deletion import SET_NULL, SET_DEFAULT
e = ModelErrorCollection(outfile)
diff --git a/django/db/backends/sqlite3/schema.py b/django/db/backends/sqlite3/schema.py
index f04095507f..69e3b61b87 100644
--- a/django/db/backends/sqlite3/schema.py
+++ b/django/db/backends/sqlite3/schema.py
@@ -1,6 +1,6 @@
+from django.apps.cache import BaseAppCache
from django.db.backends.schema import BaseDatabaseSchemaEditor
from django.db.models.fields.related import ManyToManyField
-from django.db.models.loading import BaseAppCache
class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):
diff --git a/django/db/migrations/loader.py b/django/db/migrations/loader.py
index f2510b8368..101588c4a1 100644
--- a/django/db/migrations/loader.py
+++ b/django/db/migrations/loader.py
@@ -1,7 +1,7 @@
import os
import sys
from importlib import import_module
-from django.db.models.loading import cache
+from django.apps.cache import cache
from django.db.migrations.recorder import MigrationRecorder
from django.db.migrations.graph import MigrationGraph
from django.utils import six
diff --git a/django/db/migrations/questioner.py b/django/db/migrations/questioner.py
index 7e798d3105..7874f23ed0 100644
--- a/django/db/migrations/questioner.py
+++ b/django/db/migrations/questioner.py
@@ -2,7 +2,7 @@ import importlib
import os
import sys
-from django.db.models.loading import cache
+from django.apps.cache import cache
from django.utils import datetime_safe
from django.utils.six.moves import input
from django.core.exceptions import ImproperlyConfigured
diff --git a/django/db/migrations/recorder.py b/django/db/migrations/recorder.py
index be804e52aa..bf1cd225c2 100644
--- a/django/db/migrations/recorder.py
+++ b/django/db/migrations/recorder.py
@@ -1,5 +1,5 @@
+from django.apps.cache import BaseAppCache
from django.db import models
-from django.db.models.loading import BaseAppCache
from django.utils.timezone import now
diff --git a/django/db/migrations/state.py b/django/db/migrations/state.py
index 769b0005f8..6aa44ba108 100644
--- a/django/db/migrations/state.py
+++ b/django/db/migrations/state.py
@@ -1,5 +1,5 @@
+from django.apps.cache import BaseAppCache
from django.db import models
-from django.db.models.loading import BaseAppCache
from django.db.models.options import DEFAULT_NAMES, normalize_unique_together
from django.utils import six
from django.utils.module_loading import import_by_path
diff --git a/django/db/migrations/writer.py b/django/db/migrations/writer.py
index 22b0977ba4..150455b09c 100644
--- a/django/db/migrations/writer.py
+++ b/django/db/migrations/writer.py
@@ -3,12 +3,12 @@ import datetime
import types
import os
from importlib import import_module
-from django.utils import six
+from django.apps.cache import cache
from django.db import models
-from django.db.models.loading import cache
from django.db.migrations.loader import MigrationLoader
from django.utils.encoding import force_text
from django.utils.functional import Promise
+from django.utils import six
class MigrationWriter(object):
diff --git a/django/db/models/__init__.py b/django/db/models/__init__.py
index 12c31f89fd..ad76347494 100644
--- a/django/db/models/__init__.py
+++ b/django/db/models/__init__.py
@@ -1,9 +1,9 @@
from functools import wraps
-from django.core.exceptions import ObjectDoesNotExist, ImproperlyConfigured # NOQA
-from django.db.models.loading import ( # NOQA
+from django.apps.cache import ( # NOQA
get_apps, get_app_path, get_app_paths, get_app, get_models, get_model,
register_models, UnavailableApp)
+from django.core.exceptions import ObjectDoesNotExist, ImproperlyConfigured # NOQA
from django.db.models.query import Q, QuerySet, Prefetch # NOQA
from django.db.models.expressions import F # NOQA
from django.db.models.manager import Manager # NOQA
diff --git a/django/db/models/base.py b/django/db/models/base.py
index dd4850c3d4..f22506aa92 100644
--- a/django/db/models/base.py
+++ b/django/db/models/base.py
@@ -5,6 +5,7 @@ import sys
from functools import update_wrapper
from django.utils.six.moves import zip
+from django.apps.cache import get_model, MODELS_MODULE_NAME
import django.db.models.manager # NOQA: Imported to register signal handler.
from django.conf import settings
from django.core.exceptions import (ObjectDoesNotExist,
@@ -19,7 +20,6 @@ from django.db.models.query_utils import DeferredAttribute, deferred_class_facto
from django.db.models.deletion import Collector
from django.db.models.options import Options
from django.db.models import signals
-from django.db.models.loading import get_model, MODELS_MODULE_NAME
from django.utils.translation import ugettext_lazy as _
from django.utils.functional import curry
from django.utils.encoding import force_str, force_text
diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py
index 33adaedc7e..d69a4346e2 100644
--- a/django/db/models/fields/__init__.py
+++ b/django/db/models/fields/__init__.py
@@ -9,8 +9,8 @@ import warnings
from base64 import b64decode, b64encode
from itertools import tee
+from django.apps.cache import get_model
from django.db import connection
-from django.db.models.loading import get_model
from django.db.models.query_utils import QueryWrapper
from django.conf import settings
from django import forms
diff --git a/django/db/models/loading.py b/django/db/models/loading.py
index 21440216e0..795de130b5 100644
--- a/django/db/models/loading.py
+++ b/django/db/models/loading.py
@@ -1,387 +1,15 @@
-"Utilities for loading models and the modules that contain them."
+import warnings
-from collections import OrderedDict
-import copy
-import imp
-from importlib import import_module
-import os
-import sys
+from django.apps.cache import cache
-from django.conf import settings
-from django.core.exceptions import ImproperlyConfigured
-from django.utils.module_loading import module_has_submodule
-from django.utils._os import upath
-from django.utils import six
+warnings.warn(
+ "The utilities in django.db.models.loading are deprecated "
+ "in favor of the new application loading system.",
+ PendingDeprecationWarning, stacklevel=2)
__all__ = ('get_apps', 'get_app', 'get_models', 'get_model', 'register_models',
'load_app', 'app_cache_ready')
-MODELS_MODULE_NAME = 'models'
-
-
-class ModelDict(OrderedDict):
- """
- We need to special-case the deepcopy for this, as the keys are modules,
- which can't be deep copied.
- """
- def __deepcopy__(self, memo):
- return self.__class__([(key, copy.deepcopy(value, memo))
- for key, value in self.items()])
-
-
-class UnavailableApp(Exception):
- pass
-
-
-def _initialize():
- """
- Returns a dictionary to be used as the initial value of the
- [shared] state of the app cache.
- """
- return dict(
- # Keys of app_store are the model modules for each application.
- app_store=ModelDict(),
-
- # Mapping of installed app_labels to model modules for that app.
- app_labels={},
-
- # Mapping of app_labels to a dictionary of model names to model code.
- # May contain apps that are not installed.
- app_models=ModelDict(),
-
- # Mapping of app_labels to errors raised when trying to import the app.
- app_errors={},
-
- # Pending lookups for lazy relations
- pending_lookups={},
-
- # List of app_labels that allows restricting the set of apps.
- # Used by TransactionTestCase.available_apps for performance reasons.
- available_apps=None,
-
- # -- Everything below here is only used when populating the cache --
- loads_installed=True,
- loaded=False,
- handled=set(),
- postponed=[],
- nesting_level=0,
- _get_models_cache={},
- )
-
-
-class BaseAppCache(object):
- """
- A cache that stores installed applications and their models. Used to
- provide reverse-relations and for app introspection (e.g. admin).
-
- This provides the base (non-Borg) AppCache class - the AppCache
- subclass adds borg-like behaviour for the few cases where it's needed.
- """
-
- def __init__(self):
- self.__dict__ = _initialize()
- # This stops _populate loading from INSTALLED_APPS and ignores the
- # only_installed arguments to get_model[s]
- self.loads_installed = False
-
- def _populate(self):
- """
- Fill in all the cache information. This method is threadsafe, in the
- sense that every caller will see the same state upon return, and if the
- cache is already initialised, it does no work.
- """
- if self.loaded:
- return
- if not self.loads_installed:
- self.loaded = True
- return
- # Note that we want to use the import lock here - the app loading is
- # in many cases initiated implicitly by importing, and thus it is
- # possible to end up in deadlock when one thread initiates loading
- # without holding the importer lock and another thread then tries to
- # import something which also launches the app loading. For details of
- # this situation see #18251.
- imp.acquire_lock()
- try:
- if self.loaded:
- return
- for app_name in settings.INSTALLED_APPS:
- if app_name in self.handled:
- continue
- self.load_app(app_name, True)
- if not self.nesting_level:
- for app_name in self.postponed:
- self.load_app(app_name)
- self.loaded = True
- finally:
- imp.release_lock()
-
- def _label_for(self, app_mod):
- """
- Return app_label for given models module.
-
- """
- return app_mod.__name__.split('.')[-2]
-
- def load_app(self, app_name, can_postpone=False):
- """
- Loads the app with the provided fully qualified name, and returns the
- model module.
- """
- app_module = import_module(app_name)
- self.handled.add(app_name)
- self.nesting_level += 1
- try:
- models = import_module('%s.%s' % (app_name, MODELS_MODULE_NAME))
- except ImportError:
- self.nesting_level -= 1
- # If the app doesn't have a models module, we can just ignore the
- # ImportError and return no models for it.
- if not module_has_submodule(app_module, MODELS_MODULE_NAME):
- return None
- # But if the app does have a models module, we need to figure out
- # whether to suppress or propagate the error. If can_postpone is
- # True then it may be that the package is still being imported by
- # Python and the models module isn't available yet. So we add the
- # app to the postponed list and we'll try it again after all the
- # recursion has finished (in populate). If can_postpone is False
- # then it's time to raise the ImportError.
- else:
- if can_postpone:
- self.postponed.append(app_name)
- return None
- else:
- raise
-
- self.nesting_level -= 1
- if models not in self.app_store:
- self.app_store[models] = len(self.app_store)
- self.app_labels[self._label_for(models)] = models
- return models
-
- def app_cache_ready(self):
- """
- Returns true if the model cache is fully populated.
-
- Useful for code that wants to cache the results of get_models() for
- themselves once it is safe to do so.
- """
- return self.loaded
-
- def get_apps(self):
- """
- Returns a list of all installed modules that contain models.
- """
- self._populate()
-
- apps = self.app_store.items()
- if self.available_apps is not None:
- apps = [elt for elt in apps
- if self._label_for(elt[0]) in self.available_apps]
-
- # Ensure the returned list is always in the same order (with new apps
- # added at the end). This avoids unstable ordering on the admin app
- # list page, for example.
- apps = sorted(apps, key=lambda elt: elt[1])
-
- return [elt[0] for elt in apps]
-
- def _get_app_package(self, app):
- return '.'.join(app.__name__.split('.')[:-1])
-
- def get_app_package(self, app_label):
- return self._get_app_package(self.get_app(app_label))
-
- def _get_app_path(self, app):
- if hasattr(app, '__path__'): # models/__init__.py package
- app_path = app.__path__[0]
- else: # models.py module
- app_path = app.__file__
- return os.path.dirname(upath(app_path))
-
- def get_app_path(self, app_label):
- return self._get_app_path(self.get_app(app_label))
-
- def get_app_paths(self):
- """
- Returns a list of paths to all installed apps.
-
- Useful for discovering files at conventional locations inside apps
- (static files, templates, etc.)
- """
- self._populate()
-
- app_paths = []
- for app in self.get_apps():
- app_paths.append(self._get_app_path(app))
- return app_paths
-
- def get_app(self, app_label, emptyOK=False):
- """
- Returns the module containing the models for the given app_label.
-
- Returns None if the app has no models in it and emptyOK is True.
-
- Raises UnavailableApp when set_available_apps() in in effect and
- doesn't include app_label.
- """
- self._populate()
- imp.acquire_lock()
- try:
- for app_name in settings.INSTALLED_APPS:
- if app_label == app_name.split('.')[-1]:
- mod = self.load_app(app_name, False)
- if mod is None and not emptyOK:
- raise ImproperlyConfigured("App with label %s is missing a models.py module." % app_label)
- if self.available_apps is not None and app_label not in self.available_apps:
- raise UnavailableApp("App with label %s isn't available." % app_label)
- return mod
- raise ImproperlyConfigured("App with label %s could not be found" % app_label)
- finally:
- imp.release_lock()
-
- def get_app_errors(self):
- "Returns the map of known problems with the INSTALLED_APPS."
- self._populate()
- return self.app_errors
-
- def get_models(self, app_mod=None,
- include_auto_created=False, include_deferred=False,
- only_installed=True, include_swapped=False):
- """
- Given a module containing models, returns a list of the models.
- Otherwise returns a list of all installed models.
-
- By default, auto-created models (i.e., m2m models without an
- explicit intermediate table) are not included. However, if you
- specify include_auto_created=True, they will be.
-
- By default, models created to satisfy deferred attribute
- queries are *not* included in the list of models. However, if
- you specify include_deferred, they will be.
-
- By default, models that aren't part of installed apps will *not*
- be included in the list of models. However, if you specify
- only_installed=False, they will be. If you're using a non-default
- AppCache, this argument does nothing - all models will be included.
-
- By default, models that have been swapped out will *not* be
- included in the list of models. However, if you specify
- include_swapped, they will be.
- """
- if not self.loads_installed:
- only_installed = False
- cache_key = (app_mod, include_auto_created, include_deferred, only_installed, include_swapped)
- model_list = None
- try:
- model_list = self._get_models_cache[cache_key]
- if self.available_apps is not None and only_installed:
- model_list = [m for m in model_list if m._meta.app_label in self.available_apps]
-
- return model_list
- except KeyError:
- pass
- self._populate()
- if app_mod:
- if app_mod in self.app_store:
- app_list = [self.app_models.get(self._label_for(app_mod), ModelDict())]
- else:
- app_list = []
- else:
- if only_installed:
- app_list = [self.app_models.get(app_label, ModelDict())
- for app_label in six.iterkeys(self.app_labels)]
- else:
- app_list = six.itervalues(self.app_models)
- model_list = []
- for app in app_list:
- model_list.extend(
- model for model in app.values()
- if ((not model._deferred or include_deferred) and
- (not model._meta.auto_created or include_auto_created) and
- (not model._meta.swapped or include_swapped))
- )
- self._get_models_cache[cache_key] = model_list
- if self.available_apps is not None and only_installed:
- model_list = [m for m in model_list if m._meta.app_label in self.available_apps]
- return model_list
-
- def get_model(self, app_label, model_name,
- seed_cache=True, only_installed=True):
- """
- Returns the model matching the given app_label and case-insensitive
- model_name.
-
- Returns None if no model is found.
-
- Raises UnavailableApp when set_available_apps() in in effect and
- doesn't include app_label.
- """
- if not self.loads_installed:
- only_installed = False
- if seed_cache:
- self._populate()
- if only_installed and app_label not in self.app_labels:
- return None
- if (self.available_apps is not None and only_installed
- and app_label not in self.available_apps):
- raise UnavailableApp("App with label %s isn't available." % app_label)
- try:
- return self.app_models[app_label][model_name.lower()]
- except KeyError:
- return None
-
- def register_models(self, app_label, *models):
- """
- Register a set of models as belonging to an app.
- """
- for model in models:
- # Store as 'name: model' pair in a dictionary
- # in the app_models dictionary
- model_name = model._meta.model_name
- model_dict = self.app_models.setdefault(app_label, ModelDict())
- if model_name in model_dict:
- # The same model may be imported via different paths (e.g.
- # appname.models and project.appname.models). We use the source
- # filename as a means to detect identity.
- fname1 = os.path.abspath(upath(sys.modules[model.__module__].__file__))
- fname2 = os.path.abspath(upath(sys.modules[model_dict[model_name].__module__].__file__))
- # Since the filename extension could be .py the first time and
- # .pyc or .pyo the second time, ignore the extension when
- # comparing.
- if os.path.splitext(fname1)[0] == os.path.splitext(fname2)[0]:
- continue
- model_dict[model_name] = model
- self._get_models_cache.clear()
-
- def set_available_apps(self, available):
- if not set(available).issubset(set(settings.INSTALLED_APPS)):
- extra = set(available) - set(settings.INSTALLED_APPS)
- raise ValueError("Available apps isn't a subset of installed "
- "apps, extra apps: " + ", ".join(extra))
- self.available_apps = set(app.rsplit('.', 1)[-1] for app in available)
-
- def unset_available_apps(self):
- self.available_apps = None
-
-
-class AppCache(BaseAppCache):
- """
- A cache that stores installed applications and their models. Used to
- provide reverse-relations and for app introspection (e.g. admin).
-
- Borg version of the BaseAppCache class.
- """
-
- __shared_state = _initialize()
-
- def __init__(self):
- self.__dict__ = self.__shared_state
-
-
-cache = AppCache()
-
-
# These methods were always module level, so are kept that way for backwards
# compatibility.
get_apps = cache.get_apps
diff --git a/django/db/models/options.py b/django/db/models/options.py
index b14e61573c..5d99066343 100644
--- a/django/db/models/options.py
+++ b/django/db/models/options.py
@@ -5,11 +5,11 @@ import re
from bisect import bisect
import warnings
+from django.apps.cache import app_cache_ready, cache
from django.conf import settings
from django.db.models.fields.related import ManyToManyRel
from django.db.models.fields import AutoField, FieldDoesNotExist
from django.db.models.fields.proxy import OrderWrt
-from django.db.models.loading import app_cache_ready, cache
from django.utils import six
from django.utils.functional import cached_property
from django.utils.encoding import force_text, smart_text, python_2_unicode_compatible
diff --git a/django/db/models/signals.py b/django/db/models/signals.py
index 6b011c2099..2543cf5f4a 100644
--- a/django/db/models/signals.py
+++ b/django/db/models/signals.py
@@ -1,6 +1,6 @@
from collections import defaultdict
-from django.db.models.loading import get_model
+from django.apps.cache import get_model
from django.dispatch import Signal
from django.utils import six
diff --git a/django/test/testcases.py b/django/test/testcases.py
index 4dbff55204..52900ec478 100644
--- a/django/test/testcases.py
+++ b/django/test/testcases.py
@@ -15,6 +15,7 @@ import unittest
from unittest import skipIf # NOQA: Imported here for backward compatibility
from unittest.util import safe_repr
+from django.apps.cache import cache
from django.conf import settings
from django.core import mail
from django.core.exceptions import ValidationError, ImproperlyConfigured
@@ -25,7 +26,6 @@ from django.core.management.commands import flush
from django.core.servers.basehttp import WSGIRequestHandler, WSGIServer
from django.core.urlresolvers import clear_url_caches, set_urlconf
from django.db import connection, connections, DEFAULT_DB_ALIAS, transaction
-from django.db.models.loading import cache
from django.forms.fields import CharField
from django.http import QueryDict
from django.test.client import Client
diff --git a/docs/internals/deprecation.txt b/docs/internals/deprecation.txt
index 2e53901536..d548f275fe 100644
--- a/docs/internals/deprecation.txt
+++ b/docs/internals/deprecation.txt
@@ -222,6 +222,9 @@ these changes.
* ``django.core.cache.get_cache`` will be removed. Add suitable entries
to :setting:`CACHES` and use :data:`django.core.cache.caches` instead.
+* ``django.db.models.loading`` will be removed. Use the new application
+ loading APIs instead.
+
2.0
---
diff --git a/tests/app_cache/models.py b/tests/app_cache/models.py
index 1b4d33c2f9..cc092390ab 100644
--- a/tests/app_cache/models.py
+++ b/tests/app_cache/models.py
@@ -1,5 +1,5 @@
+from django.apps.cache import BaseAppCache
from django.db import models
-from django.db.models.loading import BaseAppCache
# We're testing app cache presence on load, so this is handy.
diff --git a/tests/app_cache/tests.py b/tests/app_cache/tests.py
index b72b862de3..564f9f6f48 100644
--- a/tests/app_cache/tests.py
+++ b/tests/app_cache/tests.py
@@ -1,7 +1,9 @@
from __future__ import absolute_import
-from django.test import TestCase
-from django.db.models.loading import cache, BaseAppCache
+
+from django.apps.cache import cache, BaseAppCache
from django.db import models
+from django.test import TestCase
+
from .models import TotallyNormal, SoAlternative, new_app_cache
diff --git a/tests/app_loading/tests.py b/tests/app_loading/tests.py
index 20ec064d69..4795ca4139 100644
--- a/tests/app_loading/tests.py
+++ b/tests/app_loading/tests.py
@@ -5,7 +5,7 @@ import os
import sys
from unittest import TestCase
-from django.db.models.loading import cache, load_app, get_model, get_models, AppCache
+from django.apps.cache import cache, load_app, get_model, get_models, AppCache
from django.test.utils import override_settings
from django.utils._os import upath
diff --git a/tests/contenttypes_tests/tests.py b/tests/contenttypes_tests/tests.py
index 9d1e1f77ea..a56d196933 100644
--- a/tests/contenttypes_tests/tests.py
+++ b/tests/contenttypes_tests/tests.py
@@ -1,8 +1,8 @@
from __future__ import unicode_literals
+from django.apps.cache import BaseAppCache
from django.contrib.contenttypes.models import ContentType
from django.db import models
-from django.db.models.loading import BaseAppCache
from django.test import TestCase
from .models import Author, Article
diff --git a/tests/defer_regress/tests.py b/tests/defer_regress/tests.py
index c03388b50e..0107e8167c 100644
--- a/tests/defer_regress/tests.py
+++ b/tests/defer_regress/tests.py
@@ -2,10 +2,10 @@ from __future__ import unicode_literals
from operator import attrgetter
+from django.apps.cache import cache
from django.contrib.contenttypes.models import ContentType
from django.contrib.sessions.backends.db import SessionStore
from django.db.models import Count
-from django.db.models.loading import cache
from django.test import TestCase
from django.test.utils import override_settings
diff --git a/tests/empty/tests.py b/tests/empty/tests.py
index 007d04c363..2a9f568aea 100644
--- a/tests/empty/tests.py
+++ b/tests/empty/tests.py
@@ -1,5 +1,5 @@
+from django.apps.cache import get_app
from django.core.exceptions import ImproperlyConfigured
-from django.db.models.loading import get_app
from django.test import TestCase
from django.test.utils import override_settings
from django.utils import six
diff --git a/tests/invalid_models/tests.py b/tests/invalid_models/tests.py
index 9c9db91da9..2f7815f96d 100644
--- a/tests/invalid_models/tests.py
+++ b/tests/invalid_models/tests.py
@@ -2,8 +2,8 @@ import copy
import sys
import unittest
+from django.apps.cache import cache, load_app
from django.core.management.validation import get_validation_errors
-from django.db.models.loading import cache, load_app
from django.test.utils import override_settings
from django.utils.six import StringIO
diff --git a/tests/managers_regress/tests.py b/tests/managers_regress/tests.py
index 3798b91ef5..11ad52ce44 100644
--- a/tests/managers_regress/tests.py
+++ b/tests/managers_regress/tests.py
@@ -1,8 +1,8 @@
from __future__ import unicode_literals
import copy
+from django.apps.cache import cache
from django.db import models
-from django.db.models.loading import cache
from django.template import Context, Template
from django.test import TestCase
from django.test.utils import override_settings
diff --git a/tests/migrations/models.py b/tests/migrations/models.py
index 3bb50289be..9726e4457a 100644
--- a/tests/migrations/models.py
+++ b/tests/migrations/models.py
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
+from django.apps.cache import BaseAppCache
from django.db import models
-from django.db.models.loading import BaseAppCache
from django.utils.encoding import python_2_unicode_compatible
diff --git a/tests/migrations/test_commands.py b/tests/migrations/test_commands.py
index 48fb68b03d..fa8a212533 100644
--- a/tests/migrations/test_commands.py
+++ b/tests/migrations/test_commands.py
@@ -6,8 +6,8 @@ import copy
import os
import shutil
+from django.apps.cache import cache
from django.core.management import call_command, CommandError
-from django.db.models.loading import cache
from django.test.utils import override_settings
from django.utils import six
from django.utils._os import upath
diff --git a/tests/migrations/test_state.py b/tests/migrations/test_state.py
index 9cbbbf294d..13e575862d 100644
--- a/tests/migrations/test_state.py
+++ b/tests/migrations/test_state.py
@@ -1,7 +1,7 @@
-from django.test import TestCase
+from django.apps.cache import BaseAppCache
from django.db import models
-from django.db.models.loading import BaseAppCache
from django.db.migrations.state import ProjectState, ModelState, InvalidBasesError
+from django.test import TestCase
class StateTests(TestCase):
diff --git a/tests/migrations/test_writer.py b/tests/migrations/test_writer.py
index 0d64d40350..11109c8186 100644
--- a/tests/migrations/test_writer.py
+++ b/tests/migrations/test_writer.py
@@ -6,10 +6,10 @@ import copy
import datetime
import os
+from django.apps.cache import cache
from django.core.validators import RegexValidator, EmailValidator
from django.db import models, migrations
from django.db.migrations.writer import MigrationWriter
-from django.db.models.loading import cache
from django.test import TestCase, override_settings
from django.utils import six
from django.utils.deconstruct import deconstructible
diff --git a/tests/proxy_model_inheritance/tests.py b/tests/proxy_model_inheritance/tests.py
index 9941506303..af22e7caed 100644
--- a/tests/proxy_model_inheritance/tests.py
+++ b/tests/proxy_model_inheritance/tests.py
@@ -3,9 +3,9 @@ from __future__ import unicode_literals
import os
import sys
+from django.apps.cache import cache, load_app
from django.conf import settings
from django.core.management import call_command
-from django.db.models.loading import cache, load_app
from django.test import TestCase, TransactionTestCase
from django.test.utils import override_settings
from django.utils._os import upath
diff --git a/tests/proxy_models/tests.py b/tests/proxy_models/tests.py
index 8be7929ac4..a900366744 100644
--- a/tests/proxy_models/tests.py
+++ b/tests/proxy_models/tests.py
@@ -1,13 +1,13 @@
from __future__ import unicode_literals
import copy
+from django.apps.cache import cache
from django.contrib import admin
from django.contrib.contenttypes.models import ContentType
from django.core import management
from django.core.exceptions import FieldError
from django.db import models, DEFAULT_DB_ALIAS
from django.db.models import signals
-from django.db.models.loading import cache
from django.test import TestCase
from django.test.utils import override_settings
diff --git a/tests/runtests.py b/tests/runtests.py
index f37c0e9dda..8bcb06e522 100755
--- a/tests/runtests.py
+++ b/tests/runtests.py
@@ -80,14 +80,14 @@ def get_test_modules():
def get_installed():
- from django.db.models.loading import get_apps
+ from django.apps.cache import get_apps
return [app.__name__.rsplit('.', 1)[0] for app in get_apps()]
def setup(verbosity, test_labels):
import django
+ from django.apps.cache import get_apps, load_app
from django.conf import settings
- from django.db.models.loading import get_apps, load_app
from django.test import TransactionTestCase, TestCase
print("Testing against Django installed in '%s'" % os.path.dirname(django.__file__))
diff --git a/tests/schema/models.py b/tests/schema/models.py
index 06ba8fb760..2df97b935c 100644
--- a/tests/schema/models.py
+++ b/tests/schema/models.py
@@ -1,5 +1,5 @@
+from django.apps.cache import BaseAppCache
from django.db import models
-from django.db.models.loading import BaseAppCache
# Because we want to test creation and deletion of these as separate things,
# these models are all inserted into a separate AppCache so the main test
diff --git a/tests/swappable_models/tests.py b/tests/swappable_models/tests.py
index 85484615a3..ae42366745 100644
--- a/tests/swappable_models/tests.py
+++ b/tests/swappable_models/tests.py
@@ -2,10 +2,10 @@ from __future__ import unicode_literals
from django.utils.six import StringIO
+from django.apps.cache import cache
from django.contrib.auth.models import Permission
from django.contrib.contenttypes.models import ContentType
from django.core import management
-from django.db.models.loading import cache
from django.test import TestCase
from django.test.utils import override_settings
diff --git a/tests/tablespaces/tests.py b/tests/tablespaces/tests.py
index 6a81643a0c..0ee1b0e742 100644
--- a/tests/tablespaces/tests.py
+++ b/tests/tablespaces/tests.py
@@ -2,9 +2,9 @@ from __future__ import unicode_literals
import copy
+from django.apps.cache import cache
from django.conf import settings
from django.db import connection
-from django.db.models.loading import cache
from django.core.management.color import no_style
from django.test import TestCase, skipIfDBFeature, skipUnlessDBFeature
diff --git a/tests/validation/test_unique.py b/tests/validation/test_unique.py
index b3d389e4ec..995949fed7 100644
--- a/tests/validation/test_unique.py
+++ b/tests/validation/test_unique.py
@@ -3,9 +3,9 @@ from __future__ import unicode_literals
import datetime
import unittest
+from django.apps.cache import BaseAppCache
from django.core.exceptions import ValidationError
from django.db import models
-from django.db.models.loading import BaseAppCache
from django.test import TestCase
from .models import (CustomPKModel, UniqueTogetherModel, UniqueFieldsModel,