[multi-db] Merge trunk to [3875]. Some tests still failing.

git-svn-id: http://code.djangoproject.com/svn/django/branches/multiple-db-support@4151 bcc190cf-cafb-0310-a4f2-bffc1f526a37

AUTHORS

@@ -64,6 +64,7 @@ answer newbie questions, and generally made Django that much better:
     Ian Clelland <clelland@gmail.com>
     crankycoder@gmail.com
     Matt Croydon <http://www.postneo.com/>
+    dackze+django@gmail.com
     Jonathan Daugherty (cygnus) <http://www.cprogrammer.org/>
     Jason Davies (Esaj) <http://www.jasondavies.com/>
     Alex Dedul
@@ -104,7 +105,9 @@ answer newbie questions, and generally made Django that much better:
     Eugene Lazutkin <http://lazutkin.com/blog/>
     Jeong-Min Lee
     Christopher Lenz <http://www.cmlenz.net/>
+    lerouxb@gmail.com
     limodou
+    mattmcc
     Martin Maney <http://www.chipy.org/Martin_Maney>
     Manuzhai
     Petar Marić
@@ -116,6 +119,7 @@ answer newbie questions, and generally made Django that much better:
     Eric Moritz <http://eric.themoritzfamily.com/>
     Robin Munn <http://www.geekforgod.com/>
     Nebojša Dorđević
+    Fraser Nevett <mail@nevett.org>
     Sam Newman <http://www.magpiebrain.com/>
     Neal Norwitz <nnorwitz@google.com>
     oggie rob <oz.robharvey@gmail.com>
@@ -143,6 +147,7 @@ answer newbie questions, and generally made Django that much better:
     Radek Švarz <http://www.svarz.cz/translate/>
     Swaroop C H <http://www.swaroopch.info>
     Aaron Swartz <http://www.aaronsw.com/>
+    Tyson Tate <tyson@fallingbullets.com>
     Tom Tobin
     Tom Insam
     Joe Topjian <http://joe.terrarum.net/geek/code/python/django/>

django/bin/daily_cleanup.py

@@ -1,16 +1,17 @@
-"Daily cleanup file"
+"""
+Daily cleanup job.
+
+Can be run as a cronjob to clean out old data from the database (only expired
+sessions at the moment).
+"""
 
 from django.db import backend, connection, transaction
 
-DOCUMENTATION_DIRECTORY = '/home/html/documentation/'
-
 def clean_up():
     # Clean up old database records
     cursor = connection.cursor()
     cursor.execute("DELETE FROM %s WHERE %s < NOW()" % \
-        (backend.quote_name('core_sessions'), backend.quote_name('expire_date')))
+        (backend.quote_name('django_session'), backend.quote_name('expire_date')))
-    cursor.execute("DELETE FROM %s WHERE %s < NOW() - INTERVAL '1 week'" % \
-        (backend.quote_name('registration_challenges'), backend.quote_name('request_date')))
     transaction.commit_unless_managed()
 
 if __name__ == "__main__":

django/contrib/admin/templates/admin_doc/view_detail.html

@@ -8,7 +8,7 @@
 
 <h1>{{ name }}</h1>
 
-<h2 class="subhead">{{ summary|escape }}</h2>
+<h2 class="subhead">{{ summary }}</h2>
 
 <p>{{ body }}</p>
 

django/contrib/auth/create_superuser.py

@@ -46,6 +46,7 @@ def createsuperuser(username=None, email=None, password=None):
             if not username.isalnum():
                 sys.stderr.write("Error: That username is invalid. Use only letters, digits and underscores.\n")
                 username = None
+                continue
             try:
                 User.objects.get(username=username)
             except User.DoesNotExist:

django/contrib/auth/forms.py

@@ -4,7 +4,6 @@ from django.contrib.sites.models import Site
 from django.template import Context, loader
 from django.core import validators
 from django import forms
-from django.utils.translation import gettext_lazy as _
 
 class UserCreationForm(forms.Manipulator):
     "A form that creates a user, with no privileges, from the given username and password."

django/contrib/sitemaps/templates/sitemap.xml

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
+{% spaceless %}
 {% for url in urlset %}
   <url>
     <loc>{{ url.location|escape }}</loc>
@@ -8,4 +9,5 @@
     {% if url.priority %}<priority>{{ url.priority }}</priority>{% endif %}
    </url>
 {% endfor %}
+{% endspaceless %}
 </urlset>

django/contrib/sitemaps/templates/sitemap_index.xml

@@ -1,8 +1,4 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <sitemapindex xmlns="http://www.google.com/schemas/sitemap/0.84">
-{% for location in sitemaps %}
+{% for location in sitemaps %}<sitemap><loc>{{ location|escape }}</loc></sitemap>{% endfor %}
-	<sitemap>
-		<loc>{{ location|escape }}</loc>
-	</sitemap>
-{% endfor %}
 </sitemapindex>

django/core/handlers/base.py

@@ -48,7 +48,7 @@ class BaseHandler(object):
             if hasattr(mw_instance, 'process_exception'):
                 self._exception_middleware.insert(0, mw_instance.process_exception)
 
-    def get_response(self, path, request):
+    def get_response(self, request):
         "Returns an HttpResponse object for the given HttpRequest"
         from django.core import exceptions, urlresolvers
         from django.core.mail import mail_admins
@@ -62,7 +62,7 @@ class BaseHandler(object):
 
         resolver = urlresolvers.RegexURLResolver(r'^/', settings.ROOT_URLCONF)
         try:
-            callback, callback_args, callback_kwargs = resolver.resolve(path)
+            callback, callback_args, callback_kwargs = resolver.resolve(request.path)
 
             # Apply view middleware
             for middleware_method in self._view_middleware:
@@ -105,7 +105,7 @@ class BaseHandler(object):
                 exc_info = sys.exc_info()
                 receivers = dispatcher.send(signal=signals.got_request_exception)
                 # When DEBUG is False, send an error message to the admins.
-                subject = 'Error (%s IP): %s' % ((request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS and 'internal' or 'EXTERNAL'), getattr(request, 'path', ''))
+                subject = 'Error (%s IP): %s' % ((request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS and 'internal' or 'EXTERNAL'), request.path)
                 try:
                     request_repr = repr(request)
                 except:

django/core/handlers/modpython.py

@@ -102,7 +102,7 @@ class ModPythonRequest(http.HttpRequest):
                 'REQUEST_METHOD':    self._req.method,
                 'SCRIPT_NAME':       None, # Not supported
                 'SERVER_NAME':       self._req.server.server_hostname,
-                'SERVER_PORT':       self._req.server.port,
+                'SERVER_PORT':       str(self._req.connection.local_addr[1]),
                 'SERVER_PROTOCOL':   self._req.protocol,
                 'SERVER_SOFTWARE':   'mod_python'
             }
@@ -150,7 +150,7 @@ class ModPythonHandler(BaseHandler):
         dispatcher.send(signal=signals.request_started)
         try:
             request = ModPythonRequest(req)
-            response = self.get_response(req.uri, request)
+            response = self.get_response(request)
 
             # Apply response middleware
             for middleware_method in self._response_middleware:

django/core/handlers/wsgi.py

@@ -186,7 +186,7 @@ class WSGIHandler(BaseHandler):
         dispatcher.send(signal=signals.request_started)
         try:
             request = WSGIRequest(environ)
-            response = self.get_response(request.path, request)
+            response = self.get_response(request)
 
             # Apply response middleware
             for middleware_method in self._response_middleware:

django/core/management.py

@@ -669,7 +669,8 @@ def get_validation_errors(outfile, app=None):
     validates all models of all installed apps. Writes errors, if any, to outfile.
     Returns number of errors.
     """
-    from django.db import models, model_connection_name
+    from django.conf import settings
+    from django.db import connections, models, model_connection_name
     from django.db.models.loading import get_app_errors
     from django.db.models.fields.related import RelatedObject
 
@@ -681,6 +682,7 @@ def get_validation_errors(outfile, app=None):
     for cls in models.get_models(app):
         opts = cls._meta
         connection_name = model_connection_name(cls)
+        connection = connections[connection_name]
         
         # Do field-specific validation.
         for f in opts.fields:
@@ -712,6 +714,12 @@ def get_validation_errors(outfile, app=None):
             if f.db_index not in (None, True, False):
                 e.add(opts, '"%s": "db_index" should be either None, True or False.' % f.name)
 
+            # Check that maxlength <= 255 if using older MySQL versions.
+            if settings.DATABASE_ENGINE == 'mysql':
+                db_version = connection.connection.get_server_version()
+                if db_version < (5, 0, 3) and isinstance(f, (models.CharField, models.CommaSeparatedIntegerField, models.SlugField)) and f.maxlength > 255:
+                    e.add(opts, '"%s": %s cannot have a "maxlength" greater than 255 when you are using a version of MySQL prior to 5.0.3 (you are using %s).' % (f.name, f.__class__.__name__, '.'.join([str(n) for n in db_version[:3]])))
+
             # Check to see if the related field will clash with any
             # existing fields, m2m fields, m2m related objects or related objects
             if f.rel:

django/db/backends/mysql/base.py

@@ -13,6 +13,7 @@ except ImportError, e:
 from MySQLdb.converters import conversions
 from MySQLdb.constants import FIELD_TYPE
 import types
+import re
 
 DatabaseError = Database.DatabaseError
 
@@ -24,6 +25,12 @@ django_conversions.update({
     FIELD_TYPE.TIME: util.typecast_time,
 })
 
+# This should match the numerical portion of the version numbers (we can treat
+# versions like 5.0.24 and 5.0.24a as the same). Based on the list of version
+# at http://dev.mysql.com/doc/refman/4.1/en/news.html and
+# http://dev.mysql.com/doc/refman/5.0/en/news.html .
+server_version_re = re.compile(r'(\d{1,2})\.(\d{1,2})\.(\d{1,2})')
+
 # This is an extra debug layer over MySQL queries, to display warnings.
 # It's only used when DEBUG=True.
 class MysqlDebugWrapper:
@@ -56,6 +63,7 @@ class DatabaseWrapper(object):
         self.settings = settings
         self.connection = None
         self.queries = []
+        self.server_version = None
 
     def _valid_connection(self):
         if self.connection is not None:
@@ -106,6 +114,16 @@ class DatabaseWrapper(object):
             self.connection.close()
             self.connection = None
 
+    def get_server_version(self):
+        if not self.server_version:
+            if not self._valid_connection():
+                self.cursor()
+            m = server_version_re.match(self.connection.get_server_info())
+            if not m:
+                raise Exception('Unable to determine MySQL version from version string %r' % self.connection.get_server_info())
+            self.server_version = tuple([int(x) for x in m.groups()])
+        return self.server_version
+
 supports_constraints = True
 supports_compound_statements = True
 

django/db/models/fields/__init__.py

@@ -5,6 +5,7 @@ from django.core import validators
 from django import forms
 from django.core.exceptions import ObjectDoesNotExist
 from django.utils.functional import curry
+from django.utils.itercompat import tee
 from django.utils.text import capfirst
 from django.utils.translation import gettext, gettext_lazy
 import datetime, os, time
@@ -80,7 +81,7 @@ class Field(object):
         self.prepopulate_from = prepopulate_from
         self.unique_for_date, self.unique_for_month = unique_for_date, unique_for_month
         self.unique_for_year = unique_for_year
-        self.choices = choices or []
+        self._choices = choices or []
         self.radio_admin = radio_admin
         self.help_text = help_text
         self.db_column = db_column
@@ -324,6 +325,14 @@ class Field(object):
     def bind(self, fieldmapping, original, bound_field_class):
         return bound_field_class(self, fieldmapping, original)
 
+    def _get_choices(self):
+        if hasattr(self._choices, 'next'):
+            choices, self._choices = tee(self._choices)
+            return choices
+        else:
+            return self._choices
+    choices = property(_get_choices)
+
 class AutoField(Field):
     empty_strings_allowed = False
     def __init__(self, *args, **kwargs):

django/db/models/manipulators.py

@@ -177,7 +177,7 @@ class AutomaticManipulator(forms.Manipulator):
                         # case, because they'll be dealt with later.
 
                         if f == related.field:
-                            param = getattr(new_object, related.field.rel.field_name)
+                            param = getattr(new_object, related.field.rel.get_related_field().attname)
                         elif (not self.change) and isinstance(f, AutoField):
                             param = None
                         elif self.change and (isinstance(f, FileField) or not child_follow.get(f.name, None)):
@@ -215,7 +215,10 @@ class AutomaticManipulator(forms.Manipulator):
                         # Save many-to-many objects.
                         for f in related.opts.many_to_many:
                             if child_follow.get(f.name, None) and not f.rel.edit_inline:
-                                setattr(new_rel_obj, f.name, f.rel.to.objects.filter(pk__in=rel_new_data[f.attname]))
+                                new_value = rel_new_data[f.attname]
+                                if f.rel.raw_id_admin:
+                                    new_value = new_value[0]
+                                setattr(new_rel_obj, f.name, f.rel.to.objects.filter(pk__in=new_value))
                                 if self.change:
                                     self.fields_changed.append('%s for %s "%s"' % (f.verbose_name, related.opts.verbose_name, new_rel_obj))
 
@@ -300,7 +303,7 @@ def manipulator_validator_unique_together(field_name_list, opts, self, field_dat
         pass
     else:
         raise validators.ValidationError, _("%(object)s with this %(type)s already exists for the given %(field)s.") % \
-            {'object': capfirst(opts.verbose_name), 'type': field_list[0].verbose_name, 'field': get_text_list(field_name_list[1:], 'and')}
+            {'object': capfirst(opts.verbose_name), 'type': field_list[0].verbose_name, 'field': get_text_list([f.verbose_name for f in field_list[1:]], 'and')}
 
 def manipulator_validator_unique_for_date(from_field, date_field, opts, lookup_type, self, field_data, all_data):
     from django.db.models.fields.related import ManyToOneRel

django/middleware/common.py

@@ -2,6 +2,7 @@ from django.conf import settings
 from django import http
 from django.core.mail import mail_managers
 import md5
+import re
 
 class CommonMiddleware(object):
     """
@@ -61,7 +62,7 @@ class CommonMiddleware(object):
                 # send a note to the managers.
                 domain = http.get_host(request)
                 referer = request.META.get('HTTP_REFERER', None)
-                is_internal = referer and (domain in referer)
+                is_internal = _is_internal_request(domain, referer)
                 path = request.get_full_path()
                 if referer and not _is_ignorable_404(path) and (is_internal or '?' not in referer):
                     ua = request.META.get('HTTP_USER_AGENT', '<none>')
@@ -88,3 +89,8 @@ def _is_ignorable_404(uri):
         if uri.endswith(end):
             return True
     return False
+
+def _is_internal_request(domain, referer):
+    "Return true if the referring URL is the same domain as the current request"
+    # Different subdomains are treated as different domains.
+    return referer is not None and re.match("^https?://%s/" % re.escape(domain), referer)

django/template/__init__.py

@@ -532,7 +532,7 @@ class FilterExpression(object):
                 constant_arg, i18n_arg, var_arg = match.group("constant_arg", "i18n_arg", "var_arg")
                 if i18n_arg:
                     args.append((False, _(i18n_arg.replace(r'\"', '"'))))
-                elif constant_arg:
+                elif constant_arg is not None:
                     args.append((False, constant_arg.replace(r'\"', '"')))
                 elif var_arg:
                     args.append((True, var_arg))

django/test/client.py

@@ -24,7 +24,7 @@ class ClientHandler(BaseHandler):
         dispatcher.send(signal=signals.request_started)
         try:
             request = WSGIRequest(environ)
-            response = self.get_response(request.path, request)
+            response = self.get_response(request)
 
             # Apply response middleware
             for middleware_method in self._response_middleware:

django/utils/itercompat.py

@@ -0,0 +1,31 @@
+"""
+Providing iterator functions that are not in all version of Python we support.
+Where possible, we try to use the system-native version and only fall back to
+these implementations if necessary.
+"""
+
+import itertools
+
+def compat_tee(iterable):
+    """Return two independent iterators from a single iterable.
+
+    Based on http://www.python.org/doc/2.3.5/lib/itertools-example.html
+    """
+    # Note: Using a dictionary and a list as the default arguments here is
+    # deliberate and safe in this instance.
+    def gen(next, data={}, cnt=[0]):
+        dpop = data.pop
+        for i in itertools.count():
+            if i == cnt[0]:
+                item = data[i] = next()
+                cnt[0] += 1
+            else:
+                item = dpop(i)
+            yield item
+    next = iter(iterable).next
+    return gen(next), gen(next)
+
+if hasattr(itertools, 'tee'):
+    tee = itertools.tee
+else:
+    tee = compat_tee

docs/faq.txt

@@ -313,6 +313,18 @@ PostgreSQL fans, and MySQL_ and `SQLite 3`_ are also supported.
 .. _MySQL: http://www.mysql.com/
 .. _`SQLite 3`: http://www.sqlite.org/
 
+Do I lose anything by using Python 2.3 versus newer Python versions, such as Python 2.5?
+----------------------------------------------------------------------------------------
+
+No. Django itself is guaranteed to work with any version of Python from 2.3
+and higher.
+
+If you use a Python version newer than 2.3, you will, of course, be able to
+take advantage of newer Python features in your own code, along with the speed
+improvements and other optimizations that have been made to the Python language
+itself. But the Django framework itself should work equally well on 2.3 as it
+does on 2.4 or 2.5.
+
 Do I have to use mod_python?
 ----------------------------
 

docs/templates_python.txt

@@ -366,6 +366,18 @@ If ``TEMPLATE_CONTEXT_PROCESSORS`` contains this processor, every
 `HttpRequest object`_. Note that this processor is not enabled by default;
 you'll have to activate it.
 
+Writing your own context processors
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A context processor has a very simple interface: It's just a Python function
+that takes one argument, an ``HttpRequest`` object, and returns a dictionary
+that gets added to the template context. Each context processor *must* return
+a dictionary.
+
+Custom context processors can live anywhere in your code base. All Django cares
+about is that your custom context processors are pointed-to by your
+``TEMPLATE_CONTEXT_PROCESSORS`` setting.
+
 Loading templates
 -----------------
 

tests/regressiontests/dateformat/tests.py

@@ -21,22 +21,22 @@ r"""
 '7'
 >>> format(my_birthday, 'N')
 'July'
->>> format(my_birthday, 'O')
+>>> no_tz or format(my_birthday, 'O') == '+0100'
-'+0100'
+True
 >>> format(my_birthday, 'P')
 '10 p.m.'
->>> format(my_birthday, 'r')
+>>> no_tz or format(my_birthday, 'r') == 'Sun, 8 Jul 1979 22:00:00 +0100'
-'Sun, 8 Jul 1979 22:00:00 +0100'
+True
 >>> format(my_birthday, 's')
 '00'
 >>> format(my_birthday, 'S')
 'th'
 >>> format(my_birthday, 't')
 '31'
->>> format(my_birthday, 'T')
+>>> no_tz or format(my_birthday, 'T') == 'CET'
-'CET'
+True
->>> format(my_birthday, 'U')
+>>> no_tz or format(my_birthday, 'U') == '300531600'
-'300531600'
+True
 >>> format(my_birthday, 'w')
 '0'
 >>> format(my_birthday, 'W')
@@ -47,17 +47,17 @@ r"""
 '1979'
 >>> format(my_birthday, 'z')
 '189'
->>> format(my_birthday, 'Z')
+>>> no_tz or format(my_birthday, 'Z') == '3600'
-'3600'
+True
 
->>> format(summertime, 'I')
+>>> no_tz or format(summertime, 'I') == '1'
-'1'
+True
->>> format(summertime, 'O')
+>>> no_tz or format(summertime, 'O') == '+0200'
-'+0200'
+True
->>> format(wintertime, 'I')
+>>> no_tz or format(wintertime, 'I') == '0'
-'0'
+True
->>> format(wintertime, 'O')
+>>> no_tz or format(wintertime, 'O') == '+0100'
-'+0100'
+True
 
 >>> format(my_birthday, r'Y z \C\E\T')
 '1979 189 CET'
@@ -73,7 +73,11 @@ format = dateformat.format
 os.environ['TZ'] = 'Europe/Copenhagen'
 translation.activate('en-us')
 
+try:
     time.tzset()
+    no_tz = False
+except AttributeError:
+    no_tz = True
 
 my_birthday = datetime.datetime(1979, 7, 8, 22, 00)
 summertime = datetime.datetime(2005, 10, 30, 1, 00)

tests/regressiontests/templates/tests.py

@@ -170,6 +170,9 @@ class Templates(unittest.TestCase):
             # Escaped backslash using known escape char
             'basic-syntax35': (r'{{ var|default_if_none:"foo\now" }}', {"var": None}, r'foo\now'),
 
+            # Empty strings can be passed as arguments to filters
+            'basic-syntax36': (r'{{ var|join:"" }}', {'var': ['a', 'b', 'c']}, 'abc'),
+
             ### COMMENT TAG ###########################################################
             'comment-tag01': ("{% comment %}this is hidden{% endcomment %}hello", {}, "hello"),
             'comment-tag02': ("{% comment %}this is hidden{% endcomment %}hello{% comment %}foo{% endcomment %}", {}, "hello"),