newforms-admin: Merged from trunk up to [6775].

git-svn-id: http://code.djangoproject.com/svn/django/branches/newforms-admin@6777 bcc190cf-cafb-0310-a4f2-bffc1f526a37

AUTHORS

@@ -72,6 +72,7 @@ answer newbie questions, and generally made Django that much better:
     Jonathan Buchanan <jonathan.buchanan@gmail.com>
     Trevor Caira <trevor@caira.com>
     Ricardo Javier Cárdenes Medina <ricardo.cardenes@gmail.com>
+    Graham Carlyle <graham.carlyle@maplecroft.net>
     Antonio Cavedoni <http://cavedoni.com/>
     C8E
     cedric@terramater.net
@@ -101,6 +102,7 @@ answer newbie questions, and generally made Django that much better:
     Alex Dedul
     deric@monowerks.com
     Max Derkachev <mderk@yandex.ru>
+    Rajesh Dhawan <rajesh.dhawan@gmail.com>
     Sander Dijkhuis <sander.dijkhuis@gmail.com>
     Jordan Dimov <s3x3y1@gmail.com>
     dne@mayonnaise.net
@@ -189,6 +191,7 @@ answer newbie questions, and generally made Django that much better:
     krzysiek.pawlik@silvermedia.pl
     Joseph Kocherhans
     konrad@gwu.edu
+    knox <christobzr@gmail.com>
     kurtiss@meetro.com
     lakin.wecker@gmail.com
     Nick Lane <nick.lane.au@gmail.com>
@@ -278,6 +281,7 @@ answer newbie questions, and generally made Django that much better:
     Vinay Sajip <vinay_sajip@yahoo.co.uk>
     David Schein
     scott@staplefish.com
+    Ilya Semenov <semenov@inetss.com>
     serbaut@gmail.com
     John Shaffer <jshaffer2112@gmail.com>
     Pete Shinners <pete@shinners.org>

README

@@ -34,4 +34,3 @@ To contribute to Django:
 
     * Check out http://www.djangoproject.com/community/ for information
       about getting involved.
-

django/bin/compile-messages.py

@@ -11,10 +11,9 @@ except NameError:
 
 
 def compile_messages(locale=None):
-    basedirs = [os.path.join('conf', 'locale'), 'locale']
+    basedirs = (os.path.join('conf', 'locale'), 'locale')
     if os.environ.get('DJANGO_SETTINGS_MODULE'):
         from django.conf import settings
-        if hasattr(settings, 'LOCALE_PATHS'):
         basedirs += settings.LOCALE_PATHS
 
     # Gather existing directories.

django/conf/global_settings.py

@@ -90,6 +90,8 @@ LANGUAGES_BIDI = ("he", "ar", "fa")
 # to load the internationalization machinery.
 USE_I18N = True
 
+LOCALE_PATHS = ()
+
 # Not-necessarily-technical managers of the site. They get broken link
 # notifications and other various e-mails.
 MANAGERS = ADMINS

django/conf/locale/en/LC_MESSAGES/djangojs.po

@@ -7,12 +7,12 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2005-12-09 11:51+0100\n"
+"POT-Creation-Date: 2007-11-29 10:58-0600\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
 "MIME-Version: 1.0\n"
-"Content-Type: text/plain; charset=utf-8\n"
+"Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 
 #: contrib/admin/media/js/SelectFilter2.js:33
@@ -45,64 +45,73 @@ msgstr ""
 msgid "Clear all"
 msgstr ""
 
-#: contrib/admin/media/js/dateparse.js:26
 #: contrib/admin/media/js/calendar.js:24
+#: contrib/admin/media/js/dateparse.js:32
 msgid ""
 "January February March April May June July August September October November "
 "December"
 msgstr ""
 
-#: contrib/admin/media/js/dateparse.js:27
-msgid "Sunday Monday Tuesday Wednesday Thursday Friday Saturday"
-msgstr ""
-
 #: contrib/admin/media/js/calendar.js:25
 msgid "S M T W T F S"
 msgstr ""
 
-#: contrib/admin/media/js/admin/DateTimeShortcuts.js:45
+#: contrib/admin/media/js/dateparse.js:33
-#: contrib/admin/media/js/admin/DateTimeShortcuts.js:80
+msgid "Sunday Monday Tuesday Wednesday Thursday Friday Saturday"
+msgstr ""
+
+#: contrib/admin/media/js/admin/CollapsedFieldsets.js:34
+#: contrib/admin/media/js/admin/CollapsedFieldsets.js:72
+msgid "Show"
+msgstr ""
+
+#: contrib/admin/media/js/admin/CollapsedFieldsets.js:63
+msgid "Hide"
+msgstr ""
+
+#: contrib/admin/media/js/admin/DateTimeShortcuts.js:47
+#: contrib/admin/media/js/admin/DateTimeShortcuts.js:81
 msgid "Now"
 msgstr ""
 
-#: contrib/admin/media/js/admin/DateTimeShortcuts.js:48
+#: contrib/admin/media/js/admin/DateTimeShortcuts.js:51
 msgid "Clock"
 msgstr ""
 
-#: contrib/admin/media/js/admin/DateTimeShortcuts.js:77
+#: contrib/admin/media/js/admin/DateTimeShortcuts.js:78
 msgid "Choose a time"
 msgstr ""
 
-#: contrib/admin/media/js/admin/DateTimeShortcuts.js:81
+#: contrib/admin/media/js/admin/DateTimeShortcuts.js:82
 msgid "Midnight"
 msgstr ""
 
-#: contrib/admin/media/js/admin/DateTimeShortcuts.js:82
+#: contrib/admin/media/js/admin/DateTimeShortcuts.js:83
 msgid "6 a.m."
 msgstr ""
 
-#: contrib/admin/media/js/admin/DateTimeShortcuts.js:83
+#: contrib/admin/media/js/admin/DateTimeShortcuts.js:84
 msgid "Noon"
 msgstr ""
 
-#: contrib/admin/media/js/admin/DateTimeShortcuts.js:87
+#: contrib/admin/media/js/admin/DateTimeShortcuts.js:88
-#: contrib/admin/media/js/admin/DateTimeShortcuts.js:168
+#: contrib/admin/media/js/admin/DateTimeShortcuts.js:183
 msgid "Cancel"
 msgstr ""
 
-#: contrib/admin/media/js/admin/DateTimeShortcuts.js:111
+#: contrib/admin/media/js/admin/DateTimeShortcuts.js:128
-#: contrib/admin/media/js/admin/DateTimeShortcuts.js:162
+#: contrib/admin/media/js/admin/DateTimeShortcuts.js:177
 msgid "Today"
 msgstr ""
 
-#: contrib/admin/media/js/admin/DateTimeShortcuts.js:114
+#: contrib/admin/media/js/admin/DateTimeShortcuts.js:132
 msgid "Calendar"
 msgstr ""
 
-#: contrib/admin/media/js/admin/DateTimeShortcuts.js:160
+#: contrib/admin/media/js/admin/DateTimeShortcuts.js:175
 msgid "Yesterday"
 msgstr ""
 
-#: contrib/admin/media/js/admin/DateTimeShortcuts.js:164
+#: contrib/admin/media/js/admin/DateTimeShortcuts.js:179
 msgid "Tomorrow"
 msgstr ""

django/contrib/admin/media/css/base.css

@@ -4,7 +4,7 @@
 */
 
 /* Block IE 5 */
-@import "null?\"\{";
+@import "null.css?\"\{";
 
 /* Import other styles */
 @import url('global.css');

django/contrib/admin/media/js/admin/RelatedObjectLookups.js

@@ -1,6 +1,16 @@
 // Handles related-objects functionality: lookup link for raw_id_fields
 // and Add Another links.
 
+function html_unescape(text) {
+    // Unescape a string that was escaped using django.utils.html.escape.
+    text = text.replace(/&lt;/g, '<');
+    text = text.replace(/&gt;/g, '>');
+    text = text.replace(/&quot;/g, '"');
+    text = text.replace(/&#39;/g, "'");
+    text = text.replace(/&amp;/g, '&');
+    return text;
+}
+
 function showRelatedObjectLookupPopup(triggeringLink) {
     var name = triggeringLink.id.replace(/^lookup_/, '');
     // IE doesn't like periods in the window name, so convert temporarily.
@@ -42,6 +52,10 @@ function showAddAnotherPopup(triggeringLink) {
 }
 
 function dismissAddAnotherPopup(win, newId, newRepr) {
+    // newId and newRepr are expected to have previously been escaped by
+    // django.utils.html.escape.
+    newId = html_unescape(newId);
+    newRepr = html_unescape(newRepr);
     var name = win.name.replace(/___/g, '.');
     var elem = document.getElementById(name);
     if (elem) {

django/contrib/admin/options.py

@@ -386,7 +386,8 @@ class ModelAdmin(BaseModelAdmin):
             if type(pk_value) is str: # Quote if string, so JavaScript doesn't think it's a variable.
                 pk_value = '"%s"' % pk_value.replace('"', '\\"')
             return HttpResponse('<script type="text/javascript">opener.dismissAddAnotherPopup(window, %s, "%s");</script>' % \
-                (pk_value, str(new_object).replace('"', '\\"')))
+                # escape() calls force_unicode.
+                (escape(pk_value), escape(new_object)))
         elif request.POST.has_key("_addanother"):
             request.user.message_set.create(message=msg + ' ' + (_("You may add another %s below.") % opts.verbose_name))
             return HttpResponseRedirect(request.path)

django/contrib/admin/templates/admin_doc/template_filter_index.html

@@ -14,7 +14,7 @@
 {% for library in filter_libraries %}
 <div class="module">
     <h2>{% if library.grouper %}{{ library.grouper }}{% else %}Built-in filters{% endif %}</h2>
-    {% if library.grouper %}<p class="small quiet">To use these filters, put <code>{% templatetag openblock %} load {{ library.grouper }} {% templatetag closeblock %}</code> in your template before using the filter.</p><hr>{% endif %}
+    {% if library.grouper %}<p class="small quiet">To use these filters, put <code>{% templatetag openblock %} load {{ library.grouper }} {% templatetag closeblock %}</code> in your template before using the filter.</p><hr />{% endif %}
     {% for filter in library.list|dictsort:"name" %}
     <h3 id="{{ filter.name }}">{{ filter.name }}</h3>
     <p>{{ filter.title }}</p>

django/contrib/admin/templates/admin_doc/template_tag_index.html

@@ -14,7 +14,7 @@
 {% for library in tag_libraries %}
 <div class="module">
     <h2>{% if library.grouper %}{{ library.grouper }}{% else %}Built-in tags{% endif %}</h2>
-    {% if library.grouper %}<p class="small quiet">To use these tags, put <code>{% templatetag openblock %} load {{ library.grouper }} {% templatetag closeblock %}</code> in your template before using the tag.</p><hr>{% endif %}
+    {% if library.grouper %}<p class="small quiet">To use these tags, put <code>{% templatetag openblock %} load {{ library.grouper }} {% templatetag closeblock %}</code> in your template before using the tag.</p><hr />{% endif %}
     {% for tag in library.list|dictsort:"name" %}
     <h3 id="{{ tag.name }}">{{ tag.name }}</h3>
     <h4>{{ tag.title }}</h4>

django/contrib/admin/templates/admin_doc/view_index.html

@@ -30,10 +30,10 @@
 
 {% for view in site_views.list|dictsort:"url" %}
 {% ifchanged %}
-<h3><a href="{{ view.module }}.{{ view.name }}/"/>{{ view.url|escape }}</a></h3>
+<h3><a href="{{ view.module }}.{{ view.name }}/">{{ view.url|escape }}</a></h3>
 <p class="small quiet">View function: {{ view.module }}.{{ view.name }}</p>
 <p>{{ view.title }}</p>
-<hr>
+<hr />
 {% endifchanged %}
 {% endfor %}
 </div>

django/contrib/admin/templatetags/admin_list.py

@@ -148,6 +148,8 @@ def items_for_result(cl, result):
                 # function has an "allow_tags" attribute set to True.
                 if not allow_tags:
                     result_repr = escape(result_repr)
+                else:
+                    result_repr = mark_safe(result_repr)
         else:
             field_val = getattr(result, f.attname)
 
@@ -185,7 +187,7 @@ def items_for_result(cl, result):
             else:
                 result_repr = escape(field_val)
         if force_unicode(result_repr) == '':
-            result_repr = ' '
+            result_repr = mark_safe(' ')
         # If list_display_links not defined, add the link tag to the first field
         if (first and not cl.list_display_links) or field_name in cl.list_display_links:
             table_tag = {True:'th', False:'td'}[first]

django/contrib/admin/templatetags/admin_modify.py

@@ -81,7 +81,7 @@ class FieldWrapper(object):
         return not isinstance(self.field, models.AutoField)
 
     def header_class_attribute(self):
-        return self.field.blank and ' class="optional"' or ''
+        return self.field.blank and mark_safe(' class="optional"') or ''
 
     def use_raw_id_admin(self):
         return isinstance(self.field.rel, (models.ManyToOneRel, models.ManyToManyRel)) \

django/contrib/admin/views/main.py

@@ -100,7 +100,7 @@ class AdminBoundField(object):
         return repr(self.__dict__)
 
     def html_error_list(self):
-        return " ".join([form_field.html_error_list() for form_field in self.form_fields if form_field.errors])
+        return mark_safe(" ".join([form_field.html_error_list() for form_field in self.form_fields if form_field.errors]))
 
     def original_url(self):
         if self.is_file_field and self.original and self.field.attname:

django/contrib/comments/models.py

@@ -92,7 +92,7 @@ class Comment(models.Model):
         verbose_name_plural = _('comments')
         ordering = ('-submit_date',)
 
-    def __repr__(self):
+    def __unicode__(self):
         return "%s: %s..." % (self.user.username, self.comment[:100])
 
     def get_absolute_url(self):
@@ -171,7 +171,7 @@ class FreeComment(models.Model):
         verbose_name_plural = _('free comments')
         ordering = ('-submit_date',)
 
-    def __repr__(self):
+    def __unicode__(self):
         return "%s: %s..." % (self.person_name, self.comment[:100])
 
     def get_absolute_url(self):
@@ -226,7 +226,7 @@ class KarmaScore(models.Model):
         verbose_name_plural = _('karma scores')
         unique_together = (('user', 'comment'),)
 
-    def __repr__(self):
+    def __unicode__(self):
         return _("%(score)d rating by %(user)s") % {'score': self.score, 'user': self.user}
 
 class UserFlagManager(models.Manager):
@@ -258,7 +258,7 @@ class UserFlag(models.Model):
         verbose_name_plural = _('user flags')
         unique_together = (('user', 'comment'),)
 
-    def __repr__(self):
+    def __unicode__(self):
         return _("Flag by %r") % self.user
 
 class ModeratorDeletion(models.Model):
@@ -271,7 +271,7 @@ class ModeratorDeletion(models.Model):
         verbose_name_plural = _('moderator deletions')
         unique_together = (('user', 'comment'),)
 
-    def __repr__(self):
+    def __unicode__(self):
         return _("Moderator deletion by %r") % self.user
 
 # Register the admin options for these models.

django/contrib/comments/tests.py

@@ -0,0 +1,13 @@
+# coding: utf-8
+
+r"""
+>>> from django.contrib.comments.models import Comment
+>>> from django.contrib.auth.models import User
+>>> u = User.objects.create_user('commenttestuser', 'commenttest@example.com', 'testpw')
+>>> c = Comment(user=u, comment=u'\xe2')
+>>> c
+<Comment: commenttestuser: â...>
+>>> print c
+commenttestuser: â...
+"""
+

django/contrib/formtools/models.py

@@ -0,0 +1 @@
+""" models.py (even empty) currently required by the runtests.py to enable unit tests """

django/contrib/formtools/preview.py

@@ -109,7 +109,7 @@ class FormPreview(object):
         data = [(bf.name, bf.data) for bf in form] + [settings.SECRET_KEY]
         # Use HIGHEST_PROTOCOL because it's the most efficient. It requires
         # Python 2.3, but Django requires 2.3 anyway, so that's OK.
-        pickled = pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL)
+        pickled = pickle.dumps(data, pickle.HIGHEST_PROTOCOL)
         return md5.new(pickled).hexdigest()
 
     def failed_hash(self, request):

django/contrib/formtools/test_urls.py

@@ -0,0 +1,12 @@
+"""
+
+This is a urlconf to be loaded by tests.py. Add any urls needed
+for tests only.
+
+"""
+from django.conf.urls.defaults import *
+from django.contrib.formtools.tests import *
+
+urlpatterns = patterns('',
+                       (r'^test1/', TestFormPreview(TestForm)),
+                      )

django/contrib/formtools/tests.py

@@ -0,0 +1,93 @@
+from django import newforms as forms
+from django.contrib.formtools import preview
+from django import http
+from django.conf import settings
+from django.test import TestCase
+from django.test.client import Client
+
+
+success_string = "Done was called!"
+test_data = {'field1': u'foo',
+             'field1_': u'asdf'}
+
+
+class TestFormPreview(preview.FormPreview):
+
+    def done(self, request, cleaned_data):
+        return http.HttpResponse(success_string)
+
+
+class TestForm(forms.Form):
+    field1 = forms.CharField()
+    field1_ = forms.CharField()
+
+
+class PreviewTests(TestCase):
+
+    def setUp(self):
+        settings.ROOT_URLCONF = 'django.contrib.formtools.test_urls'
+        # Create a FormPreview instance to share between tests
+        self.preview = preview.FormPreview(TestForm)
+        input_template = '<input type="hidden" name="%s" value="%s" />'
+        self.input = input_template % (self.preview.unused_name('stage'), "%d")
+
+    def test_unused_name(self):
+        """
+        Verifies name mangling to get uniue field name.
+        """
+        self.assertEqual(self.preview.unused_name('field1'), 'field1__')
+
+    def test_form_get(self):
+        """
+        Test contrib.formtools.preview form retrieval.
+
+        Use the client library to see if we can sucessfully retrieve
+        the form (mostly testing the setup ROOT_URLCONF
+        process). Verify that an additional  hidden input field
+        is created to manage the stage.
+
+        """
+        response = self.client.get('/test1/')
+        stage = self.input % 1
+        self.assertContains(response, stage, 1)
+
+    def test_form_preview(self):
+        """
+        Test contrib.formtools.preview form preview rendering.
+
+        Use the client library to POST to the form to see if a preview
+        is returned.  If we do get a form back check that the hidden
+        value is correctly managing the state of the form.
+
+        """
+        # Pass strings for form submittal and add stage variable to
+        # show we previously saw first stage of the form.
+        test_data.update({'stage': 1})
+        response = self.client.post('/test1/', test_data)
+        # Check to confirm stage is set to 2 in output form.
+        stage = self.input % 2
+        self.assertContains(response, stage, 1)
+
+    def test_form_submit(self):
+        """
+        Test contrib.formtools.preview form submittal.
+
+        Use the client library to POST to the form with stage set to 3
+        to see if our forms done() method is called. Check first
+        without the security hash, verify failure, retry with security
+        hash and verify sucess.
+
+        """
+        # Pass strings for form submittal and add stage variable to
+        # show we previously saw first stage of the form.
+        test_data.update({'stage': 2})
+        response = self.client.post('/test1/', test_data)
+        self.failIfEqual(response.content, success_string)
+        hash = self.preview.security_hash(None, TestForm(test_data))
+        test_data.update({'hash': hash})
+        response = self.client.post('/test1/', test_data)
+        self.assertEqual(response.content, success_string)
+
+
+if __name__ == '__main__':
+    unittest.main()

django/contrib/localflavor/ca/ca_provinces.py

@@ -33,6 +33,8 @@ PROVINCES_NORMALIZED = {
     'british columbia': 'BC',
     'mb': 'MB',
     'manitoba': 'MB',
+    'nb': 'NB',
+    'new brunswick': 'NB',
     'nf': 'NF',
     'newfoundland': 'NF',
     'newfoundland and labrador': 'NF',

django/core/cache/backends/base.py

@@ -64,4 +64,3 @@ class BaseCache(object):
         return self.get(key) is not None
 
     __contains__ = has_key
-

django/core/cache/backends/locmem.py

@@ -16,8 +16,12 @@ class CacheClass(SimpleCacheClass):
 
     def add(self, key, value, timeout=None):
         self._lock.writer_enters()
+        # Python 2.3 and 2.4 don't allow combined try-except-finally blocks.
         try:
-            SimpleCacheClass.add(self, key, value, timeout)
+            try:
+                super(CacheClass, self).add(key, pickle.dumps(value), timeout)
+            except pickle.PickleError:
+                pass
         finally:
             self._lock.writer_leaves()
 
@@ -49,6 +53,7 @@ class CacheClass(SimpleCacheClass):
 
     def set(self, key, value, timeout=None):
         self._lock.writer_enters()
+        # Python 2.3 and 2.4 don't allow combined try-except-finally blocks.
         try:
             try:
                 super(CacheClass, self).set(key, pickle.dumps(value), timeout)

django/core/management/__init__.py

@@ -52,7 +52,7 @@ def load_command_class(app_name, name):
     return getattr(__import__('%s.management.commands.%s' % (app_name, name),
                    {}, {}, ['Command']), 'Command')()
 
-def get_commands(load_user_commands=True, project_directory=None):
+def get_commands():
     """
     Returns a dictionary of commands against the application in which
     those commands can be found. This works by looking for a
@@ -60,10 +60,10 @@ def get_commands(load_user_commands=True, project_directory=None):
     application -- if a commands package exists, all commands in that
     package are registered.
 
-    Core commands are always included; user-defined commands will also
+    Core commands are always included. If a settings module has been
-    be included if ``load_user_commands`` is True. If a project directory
+    specified, user-defined commands will also be included, the
-    is provided, the startproject command will be disabled, and the
+    startproject command will be disabled, and the startapp command
-    startapp command will be modified to use that directory.
+    will be modified to use the directory in which that module appears.
 
     The dictionary is in the format {command_name: app_name}. Key-value
     pairs from this dictionary can then be used in calls to
@@ -80,10 +80,14 @@ def get_commands(load_user_commands=True, project_directory=None):
     if _commands is None:
         _commands = dict([(name, 'django.core')
                           for name in find_commands(__path__[0])])
-        if load_user_commands:
         # Get commands from all installed apps.
+        try:
             from django.conf import settings
-            for app_name in settings.INSTALLED_APPS:
+            apps = settings.INSTALLED_APPS
+        except (AttributeError, EnvironmentError):
+            apps = []
+
+        for app_name in apps:
             try:
                 path = find_management_module(app_name)
                 _commands.update(dict([(name, app_name)
@@ -91,6 +95,13 @@ def get_commands(load_user_commands=True, project_directory=None):
             except ImportError:
                 pass # No management module - ignore this app
 
+        # Try to determine the project directory
+        try:
+            from django.conf import settings
+            project_directory = setup_environ(__import__(settings.SETTINGS_MODULE))
+        except (AttributeError, EnvironmentError, ImportError):
+            project_directory = None
+
         if project_directory:
             # Remove the "startproject" command from self.commands, because
             # that's a django-admin.py command, not a manage.py command.
@@ -146,8 +157,6 @@ class ManagementUtility(object):
     def __init__(self, argv=None):
         self.argv = argv or sys.argv[:]
         self.prog_name = os.path.basename(self.argv[0])
-        self.project_directory = None
-        self.user_commands = False
 
     def main_help_text(self):
         """
@@ -159,8 +168,7 @@ class ManagementUtility(object):
         usage.append("Type '%s help <subcommand>' for help on a specific"
                      " subcommand." % self.prog_name)
         usage.append('Available subcommands:')
-        commands = get_commands(self.user_commands,
+        commands = get_commands().keys()
-                                self.project_directory).keys()
         commands.sort()
         for cmd in commands:
             usage.append('  %s' % cmd)
@@ -173,8 +181,7 @@ class ManagementUtility(object):
         django-admin.py or manage.py) if it can't be found.
         """
         try:
-            app_name = get_commands(self.user_commands,
+            app_name = get_commands()[subcommand]
-                                    self.project_directory)[subcommand]
             if isinstance(app_name, BaseCommand):
                 # If the command is already loaded, use it directly.
                 klass = app_name
@@ -235,8 +242,6 @@ class ProjectManagementUtility(ManagementUtility):
     """
     def __init__(self, argv, project_directory):
         super(ProjectManagementUtility, self).__init__(argv)
-        self.project_directory = project_directory
-        self.user_commands = True
 
 def setup_environ(settings_mod):
     """

django/core/management/base.py

@@ -172,14 +172,13 @@ def copy_helper(style, app_or_project, name, directory, other_name=''):
     """
     Copies either a Django application layout template or a Django project
     layout template into the specified directory.
-
-    * style - A color style object (see django.core.management.color).
-    * app_or_project - The string 'app' or 'project'.
-    * name - The name of the application or project.
-    * directory - The directory to copy the layout template to.
-    * other_name - When copying an application layout, this should be the name
-                   of the project.
     """
+    # style -- A color style object (see django.core.management.color).
+    # app_or_project -- The string 'app' or 'project'.
+    # name -- The name of the application or project.
+    # directory -- The directory to which the layout template should be copied.
+    # other_name -- When copying an application layout, this should be the name
+    #               of the project.
     import re
     import shutil
     other = {'project': 'app', 'app': 'project'}[app_or_project]

django/core/paginator.py

@@ -91,7 +91,7 @@ class ObjectPaginator(object):
         a template for loop.
         """
         if self._page_range is None:
-            self._page_range = range(1, self._pages + 1)
+            self._page_range = range(1, self.pages + 1)
         return self._page_range
 
     hits = property(_get_hits)

django/db/models/base.py

@@ -466,5 +466,5 @@ def method_get_order(ordered_obj, self):
 # HELPER FUNCTIONS (CURRIED MODEL FUNCTIONS) #
 ##############################################
 
-def get_absolute_url(opts, func, self):
+def get_absolute_url(opts, func, self, *args, **kwargs):
-    return settings.ABSOLUTE_URL_OVERRIDES.get('%s.%s' % (opts.app_label, opts.module_name), func)(self)
+    return settings.ABSOLUTE_URL_OVERRIDES.get('%s.%s' % (opts.app_label, opts.module_name), func)(self, *args, **kwargs)

django/db/models/fields/__init__.py

@@ -390,7 +390,7 @@ class Field(object):
         "Returns a django.newforms.Field instance for this database Field."
         defaults = {'required': not self.blank, 'label': capfirst(self.verbose_name), 'help_text': self.help_text}
         if self.choices:
-            defaults['widget'] = forms.Select(choices=self.get_choices())
+            defaults['widget'] = forms.Select(choices=self.get_choices(include_blank=self.blank or not (self.has_default() or 'initial' in kwargs)))
         if self.has_default():
             defaults['initial'] = self.get_default()
         defaults.update(kwargs)

django/db/models/fields/subclassing.py

@@ -28,10 +28,10 @@ class Creator(object):
     def __get__(self, obj, type=None):
         if obj is None:
             raise AttributeError('Can only be accessed via an instance.')
-        return self.value
+        return obj.__dict__[self.field.name]        
 
     def __set__(self, obj, value):
-        self.value = self.field.to_python(value)
+        obj.__dict__[self.field.name] = self.field.to_python(value)
 
 def make_contrib(func=None):
     """

django/middleware/cache.py

@@ -1,14 +1,18 @@
 from django.conf import settings
 from django.core.cache import cache
-from django.utils.cache import get_cache_key, learn_cache_key, patch_response_headers
+from django.utils.cache import get_cache_key, learn_cache_key, patch_response_headers, get_max_age
 
 class CacheMiddleware(object):
     """
     Cache middleware. If this is enabled, each Django-powered page will be
-    cached for CACHE_MIDDLEWARE_SECONDS seconds. Cache is based on URLs.
+    cached (based on URLs).
 
     Only parameter-less GET or HEAD-requests with status code 200 are cached.
 
+    The number of seconds each page is stored for is set by the
+    "max-age" section of the response's "Cache-Control" header, falling back to
+    the CACHE_MIDDLEWARE_SECONDS setting if the section was not found.
+
     If CACHE_MIDDLEWARE_ANONYMOUS_ONLY is set to True, only anonymous requests
     (i.e., those not made by a logged-in user) will be cached. This is a
     simple and effective way of avoiding the caching of the Django admin (and
@@ -78,7 +82,16 @@ class CacheMiddleware(object):
             return response
         if not response.status_code == 200:
             return response
-        patch_response_headers(response, self.cache_timeout)
+        # Try to get the timeout from the "max-age" section of the "Cache-
-        cache_key = learn_cache_key(request, response, self.cache_timeout, self.key_prefix)
+        # Control" header before reverting to using the default cache_timeout
-        cache.set(cache_key, response, self.cache_timeout)
+        # length.
+        timeout = get_max_age(response)
+        if timeout == None:
+            timeout = self.cache_timeout
+        elif timeout == 0:
+            # max-age was set to 0, don't bother caching.
+            return response
+        patch_response_headers(response, timeout)
+        cache_key = learn_cache_key(request, response, timeout, self.key_prefix)
+        cache.set(cache_key, response, timeout)
         return response

django/middleware/gzip.py

@@ -1,4 +1,5 @@
 import re
+
 from django.utils.text import compress_string
 from django.utils.cache import patch_vary_headers
 
@@ -11,18 +12,21 @@ class GZipMiddleware(object):
     on the Accept-Encoding header.
     """
     def process_response(self, request, response):
+        # It's not worth compressing non-OK or really short responses.
         if response.status_code != 200 or len(response.content) < 200:
-            # Not worth compressing really short responses or 304 status
-            # responses, etc.
             return response
 
         patch_vary_headers(response, ('Accept-Encoding',))
 
-        # Avoid gzipping if we've already got a content-encoding or if the
+        # Avoid gzipping if we've already got a content-encoding.
-        # content-type is Javascript and the user's browser is IE.
+        if response.has_header('Content-Encoding'):
-        is_js = ("msie" in request.META.get('HTTP_USER_AGENT', '').lower() and
+            return response
-                "javascript" in response.get('Content-Type', '').lower())
+
-        if response.has_header('Content-Encoding') or is_js:
+        # Older versions of IE have issues with gzipped javascript.
+        # See http://code.djangoproject.com/ticket/2449
+        is_ie = "msie" in request.META.get('HTTP_USER_AGENT', '').lower()
+        is_js = "javascript" in response.get('Content-Type', '').lower()
+        if is_ie and is_js:
             return response
 
         ae = request.META.get('HTTP_ACCEPT_ENCODING', '')

django/newforms/extras/widgets.py

@@ -6,6 +6,7 @@ import datetime
 
 from django.newforms.widgets import Widget, Select
 from django.utils.dates import MONTHS
+from django.utils.safestring import mark_safe
 
 __all__ = ('SelectDateWidget',)
 
@@ -51,7 +52,7 @@ class SelectDateWidget(Widget):
         select_html = Select(choices=year_choices).render(self.year_field % name, year_val)
         output.append(select_html)
 
-        return u'\n'.join(output)
+        return mark_safe(u'\n'.join(output))
 
     def value_from_datadict(self, data, files, name):
         y, m, d = data.get(self.year_field % name), data.get(self.month_field % name), data.get(self.day_field % name)

django/newforms/fields.py

@@ -17,7 +17,7 @@ except NameError:
     from sets import Set as set
 
 from django.utils.translation import ugettext_lazy as _
-from django.utils.encoding import StrAndUnicode, smart_unicode
+from django.utils.encoding import StrAndUnicode, smart_unicode, smart_str
 
 from util import ErrorList, ValidationError
 from widgets import TextInput, PasswordInput, HiddenInput, MultipleHiddenInput, FileInput, CheckboxInput, Select, NullBooleanSelect, SelectMultiple, DateTimeInput
@@ -235,7 +235,7 @@ class DecimalField(Field):
         super(DecimalField, self).clean(value)
         if not self.required and value in EMPTY_VALUES:
             return None
-        value = str(value).strip()
+        value = smart_str(value).strip()
         try:
             value = Decimal(value)
         except DecimalException:
@@ -536,11 +536,12 @@ class BooleanField(Field):
     widget = CheckboxInput
 
     def clean(self, value):
-        "Returns a Python boolean object."
+        """Returns a Python boolean object."""
         super(BooleanField, self).clean(value)
-        # Explicitly check for the string '0', which is what as hidden field
+        # Explicitly check for the string 'False', which is what a hidden field
-        # will submit for False.
+        # will submit for False (since bool("True") == True we don't need to
-        if value == '0':
+        # handle that explicitly).
+        if value == 'False':
             return False
         return bool(value)
 

django/newforms/models.py

@@ -3,13 +3,13 @@ Helper functions for creating Form classes from Django models
 and database field objects.
 """
 
-from django.utils.translation import ugettext
+from django.utils.translation import ugettext_lazy as _
 from django.utils.encoding import smart_unicode
 from django.utils.datastructures import SortedDict
 
 from util import ValidationError
 from forms import BaseForm
-from fields import Field, ChoiceField, IntegerField
+from fields import Field, ChoiceField, IntegerField, EMPTY_VALUES
 from formsets import BaseFormSet, formset_for_form, DELETION_FIELD_NAME
 from widgets import Select, SelectMultiple, HiddenInput, MultipleHiddenInput
 
@@ -153,15 +153,20 @@ class ModelChoiceField(ChoiceField):
     """A ChoiceField whose choices are a model QuerySet."""
     # This class is a subclass of ChoiceField for purity, but it doesn't
     # actually use any of ChoiceField's implementation.
+    default_error_messages = {
+        'invalid_choice': _(u'Select a valid choice. That choice is not one of'
+                            u' the available choices.'),
+    }
 
     def __init__(self, queryset, empty_label=u"---------", cache_choices=False,
                  required=True, widget=Select, label=None, initial=None,
-                 help_text=None):
+                 help_text=None, *args, **kwargs):
         self.empty_label = empty_label
         self.cache_choices = cache_choices
         # Call Field instead of ChoiceField __init__() because we don't need
         # ChoiceField.__init__().
-        Field.__init__(self, required, widget, label, initial, help_text)
+        Field.__init__(self, required, widget, label, initial, help_text,
+                       *args, **kwargs)
         self.queryset = queryset
 
     def _get_queryset(self):
@@ -197,41 +202,43 @@ class ModelChoiceField(ChoiceField):
 
     def clean(self, value):
         Field.clean(self, value)
-        if value in ('', None):
+        if value in EMPTY_VALUES:
             return None
         try:
             value = self.queryset.get(pk=value)
         except self.queryset.model.DoesNotExist:
-            raise ValidationError(ugettext(u'Select a valid choice. That'
+            raise ValidationError(self.error_messages['invalid_choice'])
-                                           u' choice is not one of the'
-                                           u' available choices.'))
         return value
 
 class ModelMultipleChoiceField(ModelChoiceField):
     """A MultipleChoiceField whose choices are a model QuerySet."""
     hidden_widget = MultipleHiddenInput
+    default_error_messages = {
+        'list': _(u'Enter a list of values.'),
+        'invalid_choice': _(u'Select a valid choice. %s is not one of the'
+                            u' available choices.'),
+    }
 
     def __init__(self, queryset, cache_choices=False, required=True,
                  widget=SelectMultiple, label=None, initial=None,
-                 help_text=None):
+                 help_text=None, *args, **kwargs):
         super(ModelMultipleChoiceField, self).__init__(queryset, None,
-            cache_choices, required, widget, label, initial, help_text)
+            cache_choices, required, widget, label, initial, help_text,
+            *args, **kwargs)
 
     def clean(self, value):
         if self.required and not value:
-            raise ValidationError(ugettext(u'This field is required.'))
+            raise ValidationError(self.error_messages['required'])
         elif not self.required and not value:
             return []
         if not isinstance(value, (list, tuple)):
-            raise ValidationError(ugettext(u'Enter a list of values.'))
+            raise ValidationError(self.error_messages['list'])
         final_values = []
         for val in value:
             try:
                 obj = self.queryset.get(pk=val)
             except self.queryset.model.DoesNotExist:
-                raise ValidationError(ugettext(u'Select a valid choice. %s is'
+                raise ValidationError(self.error_messages['invalid_choice'] % val)
-                                               u' not one of the available'
-                                               u' choices.') % val)
             else:
                 final_values.append(obj)
         return final_values

django/newforms/widgets.py

@@ -11,7 +11,7 @@ import copy
 from itertools import chain
 from django.conf import settings
 from django.utils.datastructures import MultiValueDict
-from django.utils.html import escape
+from django.utils.html import escape, conditional_escape
 from django.utils.translation import ugettext
 from django.utils.encoding import StrAndUnicode, force_unicode
 from django.utils.safestring import mark_safe
@@ -257,7 +257,7 @@ class Textarea(Widget):
         value = force_unicode(value)
         final_attrs = self.build_attrs(attrs, name=name)
         return mark_safe(u'<textarea%s>%s</textarea>' % (flatatt(final_attrs),
-                escape(value)))
+                conditional_escape(force_unicode(value))))
 
 class DateTimeInput(Input):
     input_type = 'text'
@@ -319,7 +319,9 @@ class Select(Widget):
         for option_value, option_label in chain(self.choices, choices):
             option_value = force_unicode(option_value)
             selected_html = (option_value == str_value) and u' selected="selected"' or ''
-            output.append(u'<option value="%s"%s>%s</option>' % (escape(option_value), selected_html, escape(force_unicode(option_label))))
+            output.append(u'<option value="%s"%s>%s</option>' % (
+                    escape(option_value), selected_html,
+                    conditional_escape(force_unicode(option_label))))
         output.append(u'</select>')
         return mark_safe(u'\n'.join(output))
 
@@ -356,7 +358,9 @@ class SelectMultiple(Widget):
         for option_value, option_label in chain(self.choices, choices):
             option_value = force_unicode(option_value)
             selected_html = (option_value in str_values) and ' selected="selected"' or ''
-            output.append(u'<option value="%s"%s>%s</option>' % (escape(option_value), selected_html, escape(force_unicode(option_label))))
+            output.append(u'<option value="%s"%s>%s</option>' % (
+                    escape(option_value), selected_html,
+                    conditional_escape(force_unicode(option_label))))
         output.append(u'</select>')
         return mark_safe(u'\n'.join(output))
 
@@ -380,7 +384,7 @@ class RadioInput(StrAndUnicode):
 
     def __unicode__(self):
         return mark_safe(u'<label>%s %s</label>' % (self.tag(),
-                self.choice_label))
+                conditional_escape(force_unicode(self.choice_label))))
 
     def is_checked(self):
         return self.value == self.choice_value
@@ -419,11 +423,13 @@ class RadioFieldRenderer(StrAndUnicode):
                 % force_unicode(w) for w in self]))
 
 class RadioSelect(Select):
+    renderer = RadioFieldRenderer
 
     def __init__(self, *args, **kwargs):
-        self.renderer = kwargs.pop('renderer', None)
+        # Override the default renderer if we were passed one.
-        if not self.renderer:
+        renderer = kwargs.pop('renderer', None)
-            self.renderer = RadioFieldRenderer
+        if renderer:
+            self.renderer = renderer
         super(RadioSelect, self).__init__(*args, **kwargs)
 
     def get_renderer(self, name, value, attrs=None, choices=()):
@@ -463,7 +469,8 @@ class CheckboxSelectMultiple(SelectMultiple):
             cb = CheckboxInput(final_attrs, check_test=lambda value: value in str_values)
             option_value = force_unicode(option_value)
             rendered_cb = cb.render(name, option_value)
-            output.append(u'<li><label>%s %s</label></li>' % (rendered_cb, escape(force_unicode(option_label))))
+            output.append(u'<li><label>%s %s</label></li>' % (rendered_cb,
+                    conditional_escape(force_unicode(option_label))))
         output.append(u'</ul>')
         return mark_safe(u'\n'.join(output))
 

django/template/__init__.py

@@ -547,9 +547,9 @@ class FilterExpression(object):
             if var == None:
                 var, constant, i18n_constant = match.group("var", "constant", "i18n_constant")
                 if i18n_constant:
-                    var = '"%s"' %  _(i18n_constant)
+                    var = '"%s"' %  _(i18n_constant.replace(r'\"', '"'))
                 elif constant:
-                    var = '"%s"' % constant
+                    var = '"%s"' % constant.replace(r'\"', '"')
                 upto = match.end()
                 if var == None:
                     raise TemplateSyntaxError, "Could not find variable at start of %s" % token
@@ -594,7 +594,7 @@ class FilterExpression(object):
             arg_vals = []
             for lookup, arg in args:
                 if not lookup:
-                    arg_vals.append(arg)
+                    arg_vals.append(mark_safe(arg))
                 else:
                     arg_vals.append(arg.resolve(context))
             if getattr(func, 'needs_autoescape', False):
@@ -678,6 +678,7 @@ class Variable(object):
         self.var = var
         self.literal = None
         self.lookups = None
+        self.translate = False
 
         try:
             # First try to treat this variable as a number.
@@ -698,11 +699,15 @@ class Variable(object):
 
         except ValueError:
             # A ValueError means that the variable isn't a number.
+            if var.startswith('_(') and var.endswith(')'):
+                # The result of the lookup should be translated at rendering
+                # time.
+                self.translate = True
+                var = var[2:-1]
             # If it's wrapped with quotes (single or double), then
             # we're also dealing with a literal.
             if var[0] in "\"'" and var[0] == var[-1]:
-                self.literal = var[1:-1]
+                self.literal = mark_safe(var[1:-1])
-
             else:
                 # Otherwise we'll set self.lookups so that resolve() knows we're
                 # dealing with a bonafide variable
@@ -712,10 +717,13 @@ class Variable(object):
         """Resolve this variable against a given context."""
         if self.lookups is not None:
             # We're dealing with a variable that needs to be resolved
-            return self._resolve_lookup(context)
+            value = self._resolve_lookup(context)
         else:
             # We're dealing with a literal, so it's already been "resolved"
-            return self.literal
+            value = self.literal
+        if self.translate:
+            return _(value)
+        return value
 
     def __repr__(self):
         return "<%s: %r>" % (self.__class__.__name__, self.var)

django/template/defaultfilters.py

@@ -25,6 +25,8 @@ def stringfilter(func):
         if args:
             args = list(args)
             args[0] = force_unicode(args[0])
+            if isinstance(args[0], SafeData) and getattr(func, 'is_safe', False):
+                return mark_safe(func(*args, **kwargs))
         return func(*args, **kwargs)
 
     # Include a reference to the real function (used to check original
@@ -89,7 +91,7 @@ def floatformat(text, arg=-1):
     """
     try:
         f = float(text)
-    except ValueError:
+    except (ValueError, TypeError):
         return u''
     try:
         d = int(arg)
@@ -106,6 +108,7 @@ floatformat.is_safe = True
 def iriencode(value):
     """Escapes an IRI value for use in a URL."""
     return force_unicode(iri_to_uri(value))
+iriencode.is_safe = True
 iriencode = stringfilter(iriencode)
 
 def linenumbers(value, autoescape=None):

django/template/loader_tags.py

@@ -2,6 +2,7 @@ from django.template import TemplateSyntaxError, TemplateDoesNotExist, Variable
 from django.template import Library, Node
 from django.template.loader import get_template, get_template_from_string, find_template_source
 from django.conf import settings
+from django.utils.safestring import mark_safe
 
 register = Library()
 
@@ -26,7 +27,7 @@ class BlockNode(Node):
 
     def super(self):
         if self.parent:
-            return self.parent.render(self.context)
+            return mark_safe(self.parent.render(self.context))
         return ''
 
     def add_parent(self, nodelist):

django/templatetags/i18n.py

@@ -1,9 +1,10 @@
 import re
 
-from django.template import Node, Variable
+from django.template import Node, Variable, VariableNode
 from django.template import TemplateSyntaxError, TokenParser, Library
 from django.template import TOKEN_TEXT, TOKEN_VAR
 from django.utils import translation
+from django.utils.encoding import force_unicode
 
 register = Library()
 
@@ -45,7 +46,8 @@ class TranslateNode(Node):
             return translation.ugettext(value)
 
 class BlockTranslateNode(Node):
-    def __init__(self, extra_context, singular, plural=None, countervar=None, counter=None):
+    def __init__(self, extra_context, singular, plural=None, countervar=None,
+            counter=None):
         self.extra_context = extra_context
         self.singular = singular
         self.plural = plural
@@ -54,29 +56,32 @@ class BlockTranslateNode(Node):
 
     def render_token_list(self, tokens):
         result = []
+        vars = []
         for token in tokens:
             if token.token_type == TOKEN_TEXT:
                 result.append(token.contents)
             elif token.token_type == TOKEN_VAR:
-                result.append('%%(%s)s' % token.contents)
+                result.append(u'%%(%s)s' % token.contents)
-        return ''.join(result)
+                vars.append(token.contents)
+        return ''.join(result), vars
 
     def render(self, context):
         context.push()
         for var, val in self.extra_context.items():
-            context[var] = val.resolve(context)
+            context[var] = val.render(context)
-        singular = self.render_token_list(self.singular)
+        singular, vars = self.render_token_list(self.singular)
         if self.plural and self.countervar and self.counter:
             count = self.counter.resolve(context)
             context[self.countervar] = count
-            plural = self.render_token_list(self.plural)
+            plural, vars = self.render_token_list(self.plural)
             result = translation.ungettext(singular, plural, count)
         else:
             result = translation.ugettext(singular)
         # Escape all isolated '%' before substituting in the context.
-        result = re.sub('%(?!\()', '%%', result) % context
+        result = re.sub(u'%(?!\()', u'%%', result)
+        data = dict([(v, force_unicode(context[v])) for v in vars])
         context.pop()
-        return result
+        return result % data
 
 def do_get_available_languages(parser, token):
     """
@@ -198,7 +203,6 @@ def do_block_translate(parser, token):
     This is much like ngettext, only in template syntax.
     """
     class BlockTranslateParser(TokenParser):
-
         def top(self):
             countervar = None
             counter = None
@@ -209,7 +213,8 @@ def do_block_translate(parser, token):
                     value = self.value()
                     if self.tag() != 'as':
                         raise TemplateSyntaxError, "variable bindings in 'blocktrans' must be 'with value as variable'"
-                    extra_context[self.tag()] = parser.compile_filter(value)
+                    extra_context[self.tag()] = VariableNode(
+                            parser.compile_filter(value))
                 elif tag == 'count':
                     counter = parser.compile_filter(self.value())
                     if self.tag() != 'as':
@@ -241,7 +246,8 @@ def do_block_translate(parser, token):
     if token.contents.strip() != 'endblocktrans':
         raise TemplateSyntaxError, "'blocktrans' doesn't allow other block tags (seen %r) inside it" % token.contents
 
-    return BlockTranslateNode(extra_context, singular, plural, countervar, counter)
+    return BlockTranslateNode(extra_context, singular, plural, countervar,
+            counter)
 
 register.tag('get_available_languages', do_get_available_languages)
 register.tag('get_current_language', do_get_current_language)

django/test/testcases.py

@@ -51,9 +51,9 @@ class TestCase(unittest.TestCase):
     def _pre_setup(self):
         """Performs any pre-test setup. This includes:
 
-            * If the Test Case class has a 'fixtures' member, clearing the
+            * Flushing the database.
-              database and installing the named fixtures at the start of each
+            * If the Test Case class has a 'fixtures' member, installing the 
-              test.
+              named fixtures.
             * Clearing the mail test outbox.
         """
         call_command('flush', verbosity=0, interactive=False)

django/utils/cache.py

@@ -20,6 +20,10 @@ An example: i18n middleware would need to distinguish caches by the
 import md5
 import re
 import time
+try:
+    set
+except NameError:
+    from sets import Set as set   # Python 2.3 fallback
 
 from django.conf import settings
 from django.core.cache import cache
@@ -70,7 +74,20 @@ def patch_cache_control(response, **kwargs):
     cc = ', '.join([dictvalue(el) for el in cc.items()])
     response['Cache-Control'] = cc
 
-vary_delim_re = re.compile(r',\s*')
+def get_max_age(response):
+    """
+    Returns the max-age from the response Cache-Control header as an integer
+    (or ``None`` if it wasn't found or wasn't an integer.
+    """
+    if not response.has_header('Cache-Control'):
+        return
+    cc = dict([_to_tuple(el) for el in
+        cc_delim_re.split(response['Cache-Control'])])
+    if 'max-age' in cc:
+        try:
+            return int(cc['max-age'])
+        except (ValueError, TypeError):
+            pass
 
 def patch_response_headers(response, cache_timeout=None):
     """
@@ -109,14 +126,15 @@ def patch_vary_headers(response, newheaders):
     # Note that we need to keep the original order intact, because cache
     # implementations may rely on the order of the Vary contents in, say,
     # computing an MD5 hash.
-    vary = []
     if response.has_header('Vary'):
-        vary = vary_delim_re.split(response['Vary'])
+        vary_headers = cc_delim_re.split(response['Vary'])
-    oldheaders = dict([(el.lower(), 1) for el in vary])
+    else:
-    for newheader in newheaders:
+        vary_headers = []
-        if not newheader.lower() in oldheaders:
+    # Use .lower() here so we treat headers as case-insensitive.
-            vary.append(newheader)
+    existing_headers = set([header.lower() for header in vary_headers])
-    response['Vary'] = ', '.join(vary)
+    additional_headers = [newheader for newheader in newheaders
+                          if newheader.lower() not in existing_headers]
+    response['Vary'] = ', '.join(vary_headers + additional_headers)
 
 def _generate_cache_key(request, headerlist, key_prefix):
     """Returns a cache key from the headers given in the header list."""
@@ -169,7 +187,7 @@ def learn_cache_key(request, response, cache_timeout=None, key_prefix=None):
                     key_prefix, iri_to_uri(request.path))
     if response.has_header('Vary'):
         headerlist = ['HTTP_'+header.upper().replace('-', '_')
-                      for header in vary_delim_re.split(response['Vary'])]
+                      for header in cc_delim_re.split(response['Vary'])]
         cache.set(cache_key, headerlist, cache_timeout)
         return _generate_cache_key(request, headerlist, key_prefix)
     else:
@@ -177,3 +195,10 @@ def learn_cache_key(request, response, cache_timeout=None, key_prefix=None):
         # for the request.path
         cache.set(cache_key, [], cache_timeout)
         return _generate_cache_key(request, [], key_prefix)
+
+
+def _to_tuple(s):
+    t = s.split('=',1)
+    if len(t) == 2:
+        return t[0].lower(), t[1]
+    return t[0].lower(), True

django/utils/datastructures.py

@@ -7,9 +7,9 @@ class MergeDict(object):
         self.dicts = dicts
 
     def __getitem__(self, key):
-        for dict in self.dicts:
+        for dict_ in self.dicts:
             try:
-                return dict[key]
+                return dict_[key]
             except KeyError:
                 pass
         raise KeyError
@@ -24,22 +24,22 @@ class MergeDict(object):
             return default
 
     def getlist(self, key):
-        for dict in self.dicts:
+        for dict_ in self.dicts:
             try:
-                return dict.getlist(key)
+                return dict_.getlist(key)
             except KeyError:
                 pass
         raise KeyError
 
     def items(self):
         item_list = []
-        for dict in self.dicts:
+        for dict_ in self.dicts:
-            item_list.extend(dict.items())
+            item_list.extend(dict_.items())
         return item_list
 
     def has_key(self, key):
-        for dict in self.dicts:
+        for dict_ in self.dicts:
-            if key in dict:
+            if key in dict_:
                 return True
         return False
 
@@ -56,11 +56,14 @@ class SortedDict(dict):
     def __init__(self, data=None):
         if data is None:
             data = {}
-        dict.__init__(self, data)
+        super(SortedDict, self).__init__(data)
         if isinstance(data, dict):
             self.keyOrder = data.keys()
         else:
-            self.keyOrder = [key for key, value in data]
+            self.keyOrder = []
+            for key, value in data:
+                if key not in self.keyOrder:
+                    self.keyOrder.append(key)
 
     def __deepcopy__(self, memo):
         from copy import deepcopy
@@ -68,12 +71,12 @@ class SortedDict(dict):
                                for key, value in self.iteritems()])
 
     def __setitem__(self, key, value):
-        dict.__setitem__(self, key, value)
+        super(SortedDict, self).__setitem__(key, value)
         if key not in self.keyOrder:
             self.keyOrder.append(key)
 
     def __delitem__(self, key):
-        dict.__delitem__(self, key)
+        super(SortedDict, self).__delitem__(key)
         self.keyOrder.remove(key)
 
     def __iter__(self):
@@ -81,7 +84,7 @@ class SortedDict(dict):
             yield k
 
     def pop(self, k, *args):
-        result = dict.pop(self, k, *args)
+        result = super(SortedDict, self).pop(k, *args)
         try:
             self.keyOrder.remove(k)
         except ValueError:
@@ -90,7 +93,7 @@ class SortedDict(dict):
         return result
 
     def popitem(self):
-        result = dict.popitem(self)
+        result = super(SortedDict, self).popitem()
         self.keyOrder.remove(result[0])
         return result
 
@@ -99,7 +102,7 @@ class SortedDict(dict):
 
     def iteritems(self):
         for key in self.keyOrder:
-            yield key, dict.__getitem__(self, key)
+            yield key, super(SortedDict, self).__getitem__(key)
 
     def keys(self):
         return self.keyOrder[:]
@@ -108,20 +111,20 @@ class SortedDict(dict):
         return iter(self.keyOrder)
 
     def values(self):
-        return [dict.__getitem__(self, k) for k in self.keyOrder]
+        return [super(SortedDict, self).__getitem__(k) for k in self.keyOrder]
 
     def itervalues(self):
         for key in self.keyOrder:
-            yield dict.__getitem__(self, key)
+            yield super(SortedDict, self).__getitem__(key)
 
-    def update(self, dict):
+    def update(self, dict_):
-        for k, v in dict.items():
+        for k, v in dict_.items():
             self.__setitem__(k, v)
 
     def setdefault(self, key, default):
         if key not in self.keyOrder:
             self.keyOrder.append(key)
-        return dict.setdefault(self, key, default)
+        return super(SortedDict, self).setdefault(key, default)
 
     def value_for_index(self, index):
         """Returns the value of the item at the given zero-based index."""
@@ -135,7 +138,7 @@ class SortedDict(dict):
             if n < index:
                 index -= 1
         self.keyOrder.insert(index, key)
-        dict.__setitem__(self, key, value)
+        super(SortedDict, self).__setitem__(key, value)
 
     def copy(self):
         """Returns a copy of this object."""
@@ -173,10 +176,11 @@ class MultiValueDict(dict):
     single name-value pairs.
     """
     def __init__(self, key_to_list_mapping=()):
-        dict.__init__(self, key_to_list_mapping)
+        super(MultiValueDict, self).__init__(key_to_list_mapping)
 
     def __repr__(self):
-        return "<%s: %s>" % (self.__class__.__name__, dict.__repr__(self))
+        return "<%s: %s>" % (self.__class__.__name__,
+                             super(MultiValueDict, self).__repr__())
 
     def __getitem__(self, key):
         """
@@ -184,7 +188,7 @@ class MultiValueDict(dict):
         raises KeyError if not found.
         """
         try:
-            list_ = dict.__getitem__(self, key)
+            list_ = super(MultiValueDict, self).__getitem__(key)
         except KeyError:
             raise MultiValueDictKeyError, "Key %r not found in %r" % (key, self)
         try:
@@ -193,10 +197,10 @@ class MultiValueDict(dict):
             return []
 
     def __setitem__(self, key, value):
-        dict.__setitem__(self, key, [value])
+        super(MultiValueDict, self).__setitem__(key, [value])
 
     def __copy__(self):
-        return self.__class__(dict.items(self))
+        return self.__class__(super(MultiValueDict, self).items())
 
     def __deepcopy__(self, memo=None):
         import copy
@@ -210,7 +214,10 @@ class MultiValueDict(dict):
         return result
 
     def get(self, key, default=None):
-        """Returns the default value if the requested data doesn't exist."""
+        """
+        Returns the last data value for the passed key.  If key doesn't exist
+        or value is an empty list, then default is returned.
+        """
         try:
             val = self[key]
         except KeyError:
@@ -220,14 +227,17 @@ class MultiValueDict(dict):
         return val
 
     def getlist(self, key):
-        """Returns an empty list if the requested data doesn't exist."""
+        """
+        Returns the list of values for the passed key.  If key doesn't exist,
+        then an empty list is returned.
+        """
         try:
-            return dict.__getitem__(self, key)
+            return super(MultiValueDict, self).__getitem__(key)
         except KeyError:
             return []
 
     def setlist(self, key, list_):
-        dict.__setitem__(self, key, list_)
+        super(MultiValueDict, self).__setitem__(key, list_)
 
     def setdefault(self, key, default=None):
         if key not in self:
@@ -242,7 +252,7 @@ class MultiValueDict(dict):
     def appendlist(self, key, value):
         """Appends an item to the internal list associated with key."""
         self.setlistdefault(key, [])
-        dict.__setitem__(self, key, self.getlist(key) + [value])
+        super(MultiValueDict, self).__setitem__(key, self.getlist(key) + [value])
 
     def items(self):
         """
@@ -253,7 +263,7 @@ class MultiValueDict(dict):
 
     def lists(self):
         """Returns a list of (key, list) pairs."""
-        return dict.items(self)
+        return super(MultiValueDict, self).items()
 
     def values(self):
         """Returns a list of the last value on every key list."""

django/utils/html.py

@@ -6,6 +6,7 @@ import string
 from django.utils.safestring import SafeData, mark_safe
 from django.utils.encoding import force_unicode
 from django.utils.functional import allow_lazy
+from django.utils.http import urlquote
 
 # Configuration for urlize() function
 LEADING_PUNCTUATION  = ['(', '<', '&lt;']
@@ -101,14 +102,24 @@ def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False):
             if middle.startswith('www.') or ('@' not in middle and not middle.startswith('http://') and \
                     len(middle) > 0 and middle[0] in string.letters + string.digits and \
                     (middle.endswith('.org') or middle.endswith('.net') or middle.endswith('.com'))):
-                middle = '<a href="http://%s"%s>%s</a>' % (middle, nofollow_attr, trim_url(middle))
+                middle = '<a href="http://%s"%s>%s</a>' % (
+                        urlquote(middle, safe='/&=:;#?+'),  nofollow_attr,
+                        trim_url(middle))
             if middle.startswith('http://') or middle.startswith('https://'):
-                middle = '<a href="%s"%s>%s</a>' % (middle, nofollow_attr, trim_url(middle))
+                middle = '<a href="%s"%s>%s</a>' % (
-            if '@' in middle and not middle.startswith('www.') and not ':' in middle \
+                        urlquote(middle, safe='/&=:;#?+'), nofollow_attr,
-                and simple_email_re.match(middle):
+                        trim_url(middle))
+            if '@' in middle and not middle.startswith('www.') and \
+                    not ':' in middle and simple_email_re.match(middle):
                 middle = '<a href="mailto:%s">%s</a>' % (middle, middle)
             if lead + middle + trail != word:
                 words[i] = lead + middle + trail
+            elif autoescape and not safe_input:
+                words[i] = escape(word)
+        elif safe_input:
+            words[i] = mark_safe(word)
+        elif autoescape:
+            words[i] = escape(word)
     return u''.join(words)
 urlize = allow_lazy(urlize, unicode)
 

django/utils/safestring.py

@@ -57,7 +57,6 @@ class SafeString(str, SafeData):
         else:
             return SafeUnicode(data)
 
-    encode = curry(_proxy_method, method = str.encode)
     decode = curry(_proxy_method, method = str.decode)
 
 class SafeUnicode(unicode, SafeData):
@@ -89,7 +88,6 @@ class SafeUnicode(unicode, SafeData):
             return SafeUnicode(data)
 
     encode = curry(_proxy_method, method = unicode.encode)
-    decode = curry(_proxy_method, method = unicode.decode)
 
 def mark_safe(s):
     """

django/utils/translation/trans_null.py

@@ -4,6 +4,7 @@
 
 from django.conf import settings
 from django.utils.encoding import force_unicode
+from django.utils.safestring import mark_safe, SafeData
 
 def ngettext(singular, plural, number):
     if number == 1: return singular
@@ -31,7 +32,10 @@ TECHNICAL_ID_MAP = {
 }
 
 def gettext(message):
-    return TECHNICAL_ID_MAP.get(message, message)
+    result = TECHNICAL_ID_MAP.get(message, message)
+    if isinstance(message, SafeData):
+        return mark_safe(result)
+    return result
 
 def ugettext(message):
     return force_unicode(gettext(message))

django/utils/translation/trans_real.py

@@ -8,6 +8,7 @@ import gettext as gettext_module
 from cStringIO import StringIO
 
 from django.utils.encoding import force_unicode
+from django.utils.safestring import mark_safe, SafeData
 
 try:
     import threading
@@ -167,7 +168,6 @@ def translation(language):
                     res.merge(t)
             return res
 
-        if hasattr(settings, 'LOCALE_PATHS'):
         for localepath in settings.LOCALE_PATHS:
             if os.path.isdir(localepath):
                 res = _merge(localepath)
@@ -271,11 +271,15 @@ def do_translate(message, translation_function):
     global _default, _active
     t = _active.get(currentThread(), None)
     if t is not None:
-        return getattr(t, translation_function)(message)
+        result = getattr(t, translation_function)(message)
+    else:
         if _default is None:
             from django.conf import settings
             _default = translation(settings.LANGUAGE_CODE)
-    return getattr(_default, translation_function)(message)
+        result = getattr(_default, translation_function)(message)
+    if isinstance(message, SafeData):
+        return mark_safe(result)
+    return result
 
 def gettext(message):
     return do_translate(message, 'gettext')

django/utils/tzinfo.py

@@ -54,6 +54,12 @@ class LocalTimezone(tzinfo):
 
     def _isdst(self, dt):
         tt = (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.weekday(), 0, -1)
+        try:
+            stamp = time.mktime(tt)
+        except OverflowError:
+            # 32 bit systems can't handle dates after Jan 2038, so we fake it
+            # in that case (since we only care about the DST flag here).
+            tt = (2037,) + tt[1:]
             stamp = time.mktime(tt)
         tt = time.localtime(stamp)
         return tt.tm_isdst > 0

django/views/debug.py

@@ -422,11 +422,11 @@ TECHNICAL_500_TEMPLATE = """
           {% if frame.context_line %}
             <div class="context" id="c{{ frame.id }}">
               {% if frame.pre_context %}
-                <ol start="{{ frame.pre_context_lineno }}" class="pre-context" id="pre{{ frame.id }}">{% for line in frame.pre_context %}<li onclick="toggle('pre{{ frame.id }}', 'post{{ frame.id }}')">{{ line }}</li>{% endfor %}</ol>
+                <ol start="{{ frame.pre_context_lineno }}" class="pre-context" id="pre{{ frame.id }}">{% for line in frame.pre_context %}<li onclick="toggle('pre{{ frame.id }}', 'post{{ frame.id }}')">{{ line|escape }}</li>{% endfor %}</ol>
               {% endif %}
-              <ol start="{{ frame.lineno }}" class="context-line"><li onclick="toggle('pre{{ frame.id }}', 'post{{ frame.id }}')">{{ frame.context_line }} <span>...</span></li></ol>
+              <ol start="{{ frame.lineno }}" class="context-line"><li onclick="toggle('pre{{ frame.id }}', 'post{{ frame.id }}')">{{ frame.context_line|escape }} <span>...</span></li></ol>
               {% if frame.post_context %}
-                <ol start='{{ frame.lineno|add:"1" }}' class="post-context" id="post{{ frame.id }}">{% for line in frame.post_context %}<li onclick="toggle('pre{{ frame.id }}', 'post{{ frame.id }}')">{{ line }}</li>{% endfor %}</ol>
+                <ol start='{{ frame.lineno|add:"1" }}' class="post-context" id="post{{ frame.id }}">{% for line in frame.post_context %}<li onclick="toggle('pre{{ frame.id }}', 'post{{ frame.id }}')">{{ line|escape }}</li>{% endfor %}</ol>
               {% endif %}
             </div>
           {% endif %}
@@ -445,8 +445,8 @@ TECHNICAL_500_TEMPLATE = """
               <tbody>
                 {% for var in frame.vars|dictsort:"0" %}
                   <tr>
-                    <td>{{ var.0 }}</td>
+                    <td>{{ var.0|escape }}</td>
-                    <td class="code"><div>{{ var.1|pprint }}</div></td>
+                    <td class="code"><div>{{ var.1|pprint|escape }}</div></td>
                   </tr>
                 {% endfor %}
               </tbody>
@@ -466,7 +466,7 @@ Traceback (most recent call last):<br/>
 {% for frame in frames %}
   File "{{ frame.filename }}" in {{ frame.function }}<br/>
   {% if frame.context_line %}
-    &nbsp;&nbsp;{{ frame.lineno }}. {{ frame.context_line }}<br/>
+    &nbsp;&nbsp;{{ frame.lineno }}. {{ frame.context_line|escape }}<br/>
   {% endif %}
 {% endfor %}<br/>
 &nbsp;&nbsp;{{ exception_type }} at {{ request.path|escape }}<br/>

django/views/i18n.py

@@ -120,8 +120,12 @@ def javascript_catalog(request, domain='djangojs', packages=None):
         p = __import__(package, {}, {}, [''])
         path = os.path.join(os.path.dirname(p.__file__), 'locale')
         paths.append(path)
+        try:
             catalog = gettext_module.translation(domain, path, ['en'])
             t.update(catalog._catalog)
+        except IOError:
+            # 'en' catalog was missing. This is harmless.
+            pass
     # next load the settings.LANGUAGE_CODE translations if it isn't english
     if default_locale != 'en':
         for path in paths:

django/views/static.py

@@ -33,6 +33,7 @@ def serve(request, path, document_root=None, show_indexes=False):
 
     # Clean up given path to only allow serving files below document_root.
     path = posixpath.normpath(urllib.unquote(path))
+    path = path.lstrip('/')
     newpath = ''
     for part in path.split('/'):
         if not part:

docs/authentication.txt

@@ -170,7 +170,7 @@ The ``User`` model has a custom manager that has the following helper functions:
 
       If no password is provided, ``set_unusable_password()`` will be called.
 
-      See _`Creating users` for example usage.
+      See `Creating users`_ for example usage.
 
     * ``make_random_password(length=10, allowed_chars='abcdefghjkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789')``
       Returns a random password with the given length and given string of

docs/cache.txt

@@ -263,6 +263,18 @@ See the `middleware documentation`_ for more on middleware.
 
 .. _`middleware documentation`: ../middleware/
 
+**New in Django development version**
+
+If a view sets its own cache expiry time (i.e. it has a ``max-age`` section in
+its ``Cache-Control`` header) then the page will be cached until the expiry
+time, rather than ``CACHE_MIDDLEWARE_SECONDS``. Using the decorators in
+``django.views.decorators.cache`` you can easily set a view's expiry time
+(using the ``cache_control`` decorator) or disable caching for a view (using
+the ``never_cache`` decorator). See the `using other headers`__ section for
+more on these decorators.
+
+__ `Controlling cache: Using other headers`_
+
 The per-view cache
 ==================
 
@@ -291,7 +303,7 @@ minutes.
 Template fragment caching
 =========================
 
-**New in development version**.
+**New in development version**
 
 If you're after even more control, you can also cache template fragments using
 the ``cache`` template tag. To give your template access to this tag, put
@@ -307,18 +319,18 @@ and the name to give the cache fragment. For example::
     {% endcache %}
 
 Sometimes you might want to cache multiple copies of a fragment depending on
-some dynamic data that appears inside the fragment. For example you may want a
+some dynamic data that appears inside the fragment. For example, you might want a
 separate cached copy of the sidebar used in the previous example for every user
-of your site. This can be easily achieved by passing additional arguments to
+of your site. Do this by passing additional arguments to the ``{% cache %}``
-the ``{% cache %}`` template tag to uniquely identify the cache fragment::
+template tag to uniquely identify the cache fragment::
 
     {% load cache %}
     {% cache 500 sidebar request.user.username %}
         .. sidebar for logged in user ..
     {% endcache %}
 
-If you need more than one argument to identify the fragment that's fine, simply
+It's perfectly fine to specify more than one argument to identify the fragment.
-pass as many arguments to ``{% cache %}`` as you need!
+Simply pass as many arguments to ``{% cache %}`` as you need.
 
 The low-level cache API
 =======================
@@ -358,16 +370,16 @@ get() can take a ``default`` argument::
     >>> cache.get('my_key', 'has expired')
     'has expired'
 
-To add a key only if it doesn't already exist, there is an add() method.  It
+**New in Django development version:** To add a key only if it doesn't already
-takes the same parameters as set(), but will not attempt to update the cache
+exist, use the ``add()`` method. It takes the same parameters as ``set()``, but
-if the key specified is already present::
+it will not attempt to update the cache if the key specified is already present::
 
     >>> cache.set('add_key', 'Initial value')
     >>> cache.add('add_key', 'New value')
     >>> cache.get('add_key')
     'Initial value'
 
-There's also a get_many() interface that only hits the cache once. get_many()
+There's also a ``get_many()`` interface that only hits the cache once. ``get_many()``
 returns a dictionary with all the keys you asked for that actually exist in the
 cache (and haven't expired)::
 
@@ -566,7 +578,7 @@ the value of the ``CACHE_MIDDLEWARE_SETTINGS`` setting. If you use a custom
 precedence, and the header values will be merged correctly.)
 
 If you want to use headers to disable caching altogether,
-``django.views.decorators.never_cache`` is a view decorator that adds
+``django.views.decorators.cache.never_cache`` is a view decorator that adds
 headers to ensure the response won't be cached by browsers or other caches. Example::
 
     from django.views.decorators.cache import never_cache

docs/custom_model_fields.txt

@@ -56,7 +56,7 @@ would like to be able to things like this in our models (we assume the
 
 We assign to and retrieve from the ``hand`` attribute in our model just like
 any other Python class. The trick is to tell Django how to handle saving and
-loading such an object
+loading such an object.
 
 In order to use the ``Hand`` class in our models, we **do not** have to change
 this class at all. This is ideal, because it means you can easily write
@@ -98,7 +98,7 @@ For our ``Hand`` example, we could convert the card data to a string of 104
 characters by concatenating all the cards together in a pre-determined order.
 Say, all the *north* cards first, then the *east*, *south* and *west* cards, in
 that order. So ``Hand`` objects can be saved to text or character columns in
-the database
+the database.
 
 What does a field class do?
 ---------------------------
@@ -233,9 +233,9 @@ sure your field subclass uses ``django.db.models.SubfieldBase`` as its
 metaclass. This ensures that the ``to_python()`` method, documented below_,
 will always be called when the attribute is initialised.
 
-Our ``HandleField`` class now looks like this::
+Our ``HandField`` class now looks like this::
 
-    class HandleField(models.Field):
+    class HandField(models.Field):
         __metaclass__ = models.SubfieldBase
 
         def __init__(self, *args, **kwargs):
@@ -549,7 +549,7 @@ we can reuse some existing conversion code::
 Some general advice
 --------------------
 
-Writing a custom field can be a tricky process sometime, particularly if you
+Writing a custom field can be a tricky process sometimes, particularly if you
 are doing complex conversions between your Python types and your database and
 serialization formats. A couple of tips to make things go more smoothly:
 

docs/databases.txt

@@ -172,7 +172,7 @@ storage engine, you have a couple of options.
 .. _AlterModelOnSyncDB: http://code.djangoproject.com/wiki/AlterModelOnSyncDB
 
 
-Oracle Notes
+Oracle notes
 ============
 
 Django supports `Oracle Database Server`_ versions 9i and higher. Oracle
@@ -182,12 +182,22 @@ operators.  You will also need the `cx_Oracle`_ driver, version 4.3.1 or newer.
 .. _`Oracle Database Server`: http://www.oracle.com/
 .. _`cx_Oracle`: http://cx-oracle.sourceforge.net/
 
-To run ``python manage.py syncdb``, you'll need to create an Oracle database
+In order for the ``python manage.py syncdb`` command to work, your Oracle
-user with CREATE TABLE, CREATE SEQUENCE, CREATE PROCEDURE, and CREATE TRIGGER
+database user must have privileges to run the following commands:
-privileges.  To run Django's test suite, the user also needs 
-CREATE and DROP DATABASE and CREATE and DROP TABLESPACE privileges.
 
-Connecting to the Database
+	* CREATE TABLE
+	* CREATE SEQUENCE
+	* CREATE PROCEDURE
+	* CREATE TRIGGER
+
+To run Django's test suite, the user needs these *additional* privileges:
+
+ 	* CREATE DATABASE
+	* DROP DATABASE
+	* CREATE TABLESPACE
+	* DROP TABLESPACE
+
+Connecting to the database
 --------------------------
 
 Your Django settings.py file should look something like this for Oracle::
@@ -213,20 +223,20 @@ and ``DATABASE_PORT`` like so::
 You should supply both ``DATABASE_HOST`` and ``DATABASE_PORT``, or leave both
 as empty strings.
 
-Tablespace Options
+Tablespace options
 ------------------
 
 A common paradigm for optimizing performance in Oracle-based systems is the
 use of `tablespaces`_ to organize disk layout. The Oracle backend supports
 this use case by adding ``db_tablespace`` options to the ``Meta`` and
-``Field`` classes.  (When using a backend that lacks support for tablespaces,
+``Field`` classes.  (When you use a backend that lacks support for tablespaces,
-these options are ignored.)
+Django ignores these options.)
 
 .. _`tablespaces`: http://en.wikipedia.org/wiki/Tablespace
 
 A tablespace can be specified for the table(s) generated by a model by
-supplying the ``db_tablespace`` option inside the model's ``Meta`` class.
+supplying the ``db_tablespace`` option inside the model's ``class Meta``.
-Additionally, the ``db_tablespace`` option can be passed to a ``Field``
+Additionally, you can pass the ``db_tablespace`` option to a ``Field``
 constructor to specify an alternate tablespace for the ``Field``'s column
 index. If no index would be created for the column, the ``db_tablespace``
 option is ignored.
@@ -234,8 +244,8 @@ option is ignored.
 ::
 
     class TablespaceExample(models.Model):
-        name = models.CharField(maxlength=30, db_index=True, db_tablespace="indexes")
+        name = models.CharField(max_length=30, db_index=True, db_tablespace="indexes")
-        data = models.CharField(maxlength=255, db_index=True)
+        data = models.CharField(max_length=255, db_index=True)
         edges = models.ManyToManyField(to="self", db_tablespace="indexes")
 
         class Meta:
@@ -253,14 +263,14 @@ documentation`_ for details on creating and managing tablespaces.
 
 .. _`Oracle's documentation`: http://download.oracle.com/docs/cd/B19306_01/server.102/b14200/statements_7003.htm#SQLRF01403
 
-Naming Issues
+Naming issues
 -------------
 
 Oracle imposes a name length limit of 30 characters. To accommodate this, the
 backend truncates database identifiers to fit, replacing the final four
 characters of the truncated name with a repeatable MD5 hash value.
 
-NULL and Empty Strings
+NULL and empty strings
 ----------------------
 
 Django generally prefers to use the empty string ('') rather than NULL, but
@@ -270,8 +280,8 @@ value.  When fetching from the database, it is assumed that a NULL value in
 one of these fields really means the empty string, and the data is silently
 converted to reflect this assumption.
 
-TextField Limitations
+``TextField`` limitations
----------------------
+-------------------------
 
 The Oracle backend stores ``TextFields`` as ``NCLOB`` columns. Oracle imposes
 some limitations on the usage of such LOB columns in general:
@@ -284,5 +294,5 @@ some limitations on the usage of such LOB columns in general:
     attempting to use the ``QuerySet.distinct`` method on a model that
     includes ``TextField`` columns will result in an error when run against
     Oracle. A workaround to this is to keep ``TextField`` columns out of any
-    models that you foresee performing ``.distinct`` queries on, and to
+    models that you foresee performing ``distinct()`` queries on, and to
     include the ``TextField`` in a related model instead.

docs/django-admin.txt

@@ -184,9 +184,9 @@ is being executed as an unattended, automated script.
 Use ``--verbosity`` to specify the amount of notification and debug information
 that ``django-admin.py`` should print to the console.
 
-	* ``0`` means no input.
+	* ``0`` means no output.
-	* ``1`` means normal input (default).
+	* ``1`` means normal output (default).
-	* ``2`` means verbose input.
+	* ``2`` means verbose output.
 
 Example usage::
 
@@ -651,7 +651,7 @@ To run the test server on port 7000 with ``fixture1`` and ``fixture2``::
 that it doesn't matter whether the options come before or after the fixture
 arguments.)
 
-To run on 1.2.3.4:7000 with a `test` fixture::
+To run on 1.2.3.4:7000 with a ``test`` fixture::
 
     django-admin.py testserver --addrport 1.2.3.4:7000 test
 

docs/flatpages.txt

@@ -113,3 +113,10 @@ Here's a sample ``flatpages/default.html`` template::
     {{ flatpage.content }}
     </body>
     </html>
+
+Since you're already entering raw HTML into the admin page for a flatpage,
+both ``flatpage.title`` and ``flatpage.content`` are marked as **not**
+requiring `automatic HTML escaping`_  in the template.
+
+.. _automatic HTML escaping: ../templates/#automatic-html-escaping
+

docs/i18n.txt

@@ -461,8 +461,8 @@ otherwise, they'll be tacked together without whitespace!
     you'll be using to edit the content. Due to the way the ``gettext`` tools
     work internally and because we want to allow non-ASCII source strings in
     Django's core and your applications, you **must** use UTF-8 as the encoding
-    for your PO file (this means that everybody will be using the same
+    for your PO file. This means that everybody will be using the same
-    encoding, which is important when Django processes the PO files).
+    encoding, which is important when Django processes the PO files.
 
 To reexamine all source code and templates for new translation strings and
 update all message files for **all** languages, run this::
@@ -658,7 +658,7 @@ message file. The choice is yours.
     of the settings file to determine this, and a settings file doesn't exist
     if you're manually configuring your settings.)
 
-.. _settings documentation: ../settings/#using-settings-without-the-django-settings-module-environment-variable
+.. _settings documentation: ../settings/#using-settings-without-setting-django-settings-module
 
 All message file repositories are structured the same way. They are:
 

docs/install.txt

@@ -187,10 +187,10 @@ latest bug fixes and improvements, follow these instructions:
    "Where are my ``site-packages`` stored?" section above.)
 
    Alternatively, you can define your ``PYTHONPATH`` environment variable
-   so that it includes the ``django`` subdirectory of ``django-trunk``.
+   so that it includes the ``django-trunk`` directory. This is perhaps the
-   This is perhaps the most convenient solution on Windows systems, which
+   most convenient solution on Windows systems, which don't support symbolic
-   don't support symbolic links. (Environment variables can be defined on
+   links. (Environment variables can be defined on Windows systems `from the
-   Windows systems `from the Control Panel`_.)
+   Control Panel`_.)
 
    .. admonition:: What about Apache and mod_python?
 
@@ -204,11 +204,18 @@ latest bug fixes and improvements, follow these instructions:
 
       .. _How to use Django with mod_python: ../modpython/
 
-4. Copy the file ``django-trunk/django/bin/django-admin.py`` to somewhere on
+4. On Unix-like systems, create a symbolic link to the file
-   your system path, such as ``/usr/local/bin`` (Unix) or ``C:\Python24\Scripts``
+   ``django-trunk/django/bin/django-admin.py`` in a directory on your system
-   (Windows). This step simply lets you type ``django-admin.py`` from within
+   path, such as ``/usr/local/bin``. For example::
-   any directory, rather than having to qualify the command with the full path
+
-   to the file.
+       ln -s `pwd`/django-trunk/django/bin/django-admin.py /usr/local/bin
+
+   This simply lets you type ``django-admin.py`` from within any directory,
+   rather than having to qualify the command with the full path to the file.
+
+   On Windows systems, the same result can be achieved by copying the file
+   ``django-trunk/django/bin/django-admin.py`` to somewhere on your system
+   path, for example ``C:\Python24\Scripts``.
 
 You *don't* have to run ``python setup.py install``, because you've already
 carried out the equivalent actions in steps 3 and 4.

docs/middleware.txt

@@ -104,8 +104,7 @@ Handles conditional GET operations. If the response has a ``ETag`` or
 ``Last-Modified`` header, and the request has ``If-None-Match`` or
 ``If-Modified-Since``, the response is replaced by an HttpNotModified.
 
-Also removes the content from any response to a HEAD request and sets the
+Also sets the ``Date`` and ``Content-Length`` response-headers.
-``Date`` and ``Content-Length`` response-headers.
 
 django.middleware.http.SetRemoteAddrFromForwardedFor
 ----------------------------------------------------

docs/model-api.txt

@@ -50,7 +50,7 @@ The above ``Person`` model would create a database table like this::
 Some technical notes:
 
     * The name of the table, ``myapp_person``, is automatically derived from
-      some model metadata but can be overridden. See _`Table names` below.
+      some model metadata but can be overridden. See `Table names`_ below.
     * An ``id`` field is added automatically, but this behavior can be
       overriden. See `Automatic primary key fields`_ below.
     * The ``CREATE TABLE`` SQL in this example is formatted using PostgreSQL
@@ -1664,7 +1664,7 @@ Adding extra Manager methods
 
 Adding extra ``Manager`` methods is the preferred way to add "table-level"
 functionality to your models. (For "row-level" functionality -- i.e., functions
-that act on a single instance of a model object -- use _`Model methods`, not
+that act on a single instance of a model object -- use `Model methods`_, not
 custom ``Manager`` methods.)
 
 A custom ``Manager`` method can return anything you want. It doesn't have to

docs/modpython.txt

@@ -89,18 +89,17 @@ path.
 
 .. note::
 
-    If you're using Windows, it is still recommended that you use forward
+    If you're using Windows, we still recommended that you use forward
-    slashes in the pathnames, even though Windows normally uses backslashes
+    slashes in the pathnames, even though Windows normally uses the backslash
-    for its native separator. Apache knows how to convert from the forward
+    character as its native separator. Apache knows how to convert from the
-    slash format to the native format, so this approach is portable and easier
+    forward slash format to the native format, so this approach is portable and
-    to read (it avoids tricky problems with having to double-escape
+    easier to read. (It avoids tricky problems with having to double-escape
-    backslashes).
+    backslashes.)
 
     This is valid even on a Windows system::
 
         PythonPath "['c:/path/to/project'] + sys.path"
 
-
 You can also add directives such as ``PythonAutoReload Off`` for performance.
 See the `mod_python documentation`_ for a full list of options.
 

docs/newforms.txt

@@ -756,6 +756,30 @@ For example::
     </ul>
     </form>
 
+Highlighting required fields in templates
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You may wish to show a visitor which fields are required. Here is the above
+example modified to insert an asterix after the label of each required field::
+
+    <form method="post" action="">
+    <dl>
+    {% for field in form %}
+        <dt>{{ field.label_tag }}{{ field.label }}{% if field.field.required %}*{% endif %}</dt>
+        <dd>{{ field }}</dd>
+        {% if field.help_text %}<dd>{{ field.help_text }}</dd>{% endif %}
+        {% if field.errors %}<dd class="myerrors">{{ field.errors }}</dd>{% endif %}
+    {% endfor %}
+    </dl>
+    <input type="submit" />
+    </form>
+
+The ``{% if field.field.required %}*{% endif %}`` fragment is the relevant
+addition here. It adds the asterix only if the field is required. Note that we
+check ``field.field.required`` and not ``field.required``. In the template,
+``field`` is a ``newforms.forms.BoundField`` instance, which holds the actual
+``Field`` instance in its ``field`` attribute.
+
 Binding uploaded files to a form
 --------------------------------
 
@@ -863,12 +887,11 @@ classes::
     <li>Instrument: <input type="text" name="instrument" /></li>
     <li>Haircut type: <input type="text" name="haircut_type" /></li>
 
-
 Prefixes for forms
 ------------------
 
 You can put several Django forms inside one ``<form>`` tag. To give each
-``Form`` its own namespace you need to use the ``prefix`` keyword argument::
+``Form`` its own namespace, use the ``prefix`` keyword argument::
 
     >>> mother = PersonForm(prefix="mother")
     >>> father = PersonForm(prefix="father")
@@ -879,7 +902,6 @@ You can put several Django forms inside one ``<form>`` tag. To give each
     <li><label for="id_father-first_name">First name:</label> <input type="text" name="father-first_name" id="id_father-first_name" /></li>
     <li><label for="id_father-last_name">Last name:</label> <input type="text" name="father-last_name" id="id_father-last_name" /></li>
 
-
 Fields
 ======
 
@@ -1852,7 +1874,11 @@ In addition, each generated form field has attributes set as follows:
 
     * If the model field has ``choices`` set, then the form field's ``widget``
       will be set to ``Select``, with choices coming from the model field's
-      ``choices``.
+      ``choices``. The choices will normally include the blank choice which is
+      selected by default. If the field is required, this forces the user to
+      make a selection. The blank choice will not be included if the model
+      field has ``blank=False`` and an explicit ``default`` value (the
+      ``default`` value will be initially selected instead).
 
 Finally, note that you can override the form field used for a given model
 field. See "Overriding the default field types" below.
@@ -2098,10 +2124,14 @@ instance instead of a model class::
     # Instantiate the form.
     >>> f = AuthorForm()
 
-When a form created by ``form_for_instance()`` is created, the initial
+When a form created by ``form_for_instance()`` is created, the initial data
-data values for the form fields are drawn from the instance. However,
+values for the form fields are drawn from the instance. However, this data is
-this data is not bound to the form. You will need to bind data to the
+not bound to the form. You will need to bind data to the form before the form
-form before the form can be saved.
+can be saved.
+
+Unlike ``form_for_model()``, a choice field in form created by
+``form_for_instance()`` will not include the blank choice if the respective
+model field has ``blank=False``. The initial choice is drawn from the instance.
 
 When you call ``save()`` on a form created by ``form_for_instance()``,
 the database instance will be updated. As in ``form_for_model()``, ``save()``

docs/release_notes_0.96.txt

@@ -44,7 +44,7 @@ to this::
     DATABASE_ENGINE = "mysql_old"
 
 However, we strongly encourage MySQL users to upgrade to a more recent
-version of `MySQLdb` as soon as possible, The "mysql_old" backend is
+version of ``MySQLdb`` as soon as possible, The "mysql_old" backend is
 provided only to ease this transition, and is considered deprecated;
 aside from any necessary security fixes, it will not be actively
 maintained, and it will be removed in a future release of Django.

docs/settings.txt

@@ -583,7 +583,7 @@ LOCALE_PATHS
 
 Default: ``()`` (Empty tuple)
 
-A list of directories where Django looks for translation files.
+A tuple of directories where Django looks for translation files.
 See the `internationalization docs section`_ explaining the variable and the
 default behavior.
 
@@ -791,10 +791,12 @@ SESSION_COOKIE_PATH
 
 Default: ``'/'``
 
-The path set on the session cookie. Should match the URL path of your Django
+The path set on the session cookie. This should either match the URL path of your
-installation (or be parent of that path). This is useful if you have multiple
+Django installation or be parent of that path.
-Django instances running under the same hostname; they can use different
+
-cookie paths and each instance will only see its own session cookie.
+This is useful if you have multiple Django instances running under the same
+hostname. They can use different cookie paths, and each instance will only see
+its own session cookie.
 
 SESSION_COOKIE_SECURE
 ---------------------

docs/syndication_feeds.txt

@@ -201,6 +201,8 @@ the feed.
 
 An example makes this clear. Here's the code for these beat-specific feeds::
 
+    from django.contrib.syndication import FeedDoesNotExist
+
     class BeatFeed(Feed):
         def get_object(self, bits):
             # In case of "/rss/beats/0613/foo/bar/baz/", or other such clutter,
@@ -213,6 +215,8 @@ An example makes this clear. Here's the code for these beat-specific feeds::
             return "Chicagocrime.org: Crimes for beat %s" % obj.beat
 
         def link(self, obj):
+            if not obj:
+                raise FeedDoesNotExist
             return obj.get_absolute_url()
 
         def description(self, obj):
@@ -246,11 +250,18 @@ request to the URL ``/rss/beats/0613/``:
       each of ``title``, ``link`` and ``description``, Django follows this
       algorithm:
 
-          * First, it tries to call a method, passing the ``obj`` argument, where
+          * First, it tries to call a method, passing the ``obj`` argument,
-            ``obj`` is the object returned by ``get_object()``.
+            where ``obj`` is the object returned by ``get_object()``.
           * Failing that, it tries to call a method with no arguments.
           * Failing that, it uses the class attribute.
 
+      Inside the ``link()`` method, we handle the possibility that ``obj``
+      might be ``None``, which can occur when the URL isn't fully specified. In
+      some cases, you might want to do something else in this case, which would
+      mean you'd need to check for ``obj`` existing in other methods as well
+      (the ``link()`` method is called very early in the feed generation
+      process, so is a good place to bail out early).
+
     * Finally, note that ``items()`` in this example also takes the ``obj``
       argument. The algorithm for ``items`` is the same as described in the
       previous step -- first, it tries ``items(obj)``, then ``items()``, then
@@ -553,15 +564,15 @@ This example illustrates all possible attributes and methods for a ``Feed`` clas
         def ttl(self, obj):
             """
             Takes the object returned by get_object() and returns the feed's
-            TTL (Time to live) as a normal Python string.
+            TTL (Time To Live) as a normal Python string.
             """
 
         def ttl(self):
             """
-            Returns the feed's ttl as a normal Python string.
+            Returns the feed's TTL as a normal Python string.
             """
 
-        ttl = 600 # Hard-coded Time to live.
+        ttl = 600 # Hard-coded Time To Live.
 
         # ITEMS -- One of the following three is required. The framework looks
         # for them in this order.

docs/templates.txt

@@ -2,6 +2,8 @@
 The Django template language: For template authors
 ==================================================
 
+.. admonition:: About this document
+
     This document explains the language syntax of the Django template system. If
     you're looking for a more technical perspective on how it works and how to
     extend it, see `The Django template language: For Python programmers`_.
@@ -280,7 +282,9 @@ Here are some tips for working with inheritance:
     * If you need to get the content of the block from the parent template,
       the ``{{ block.super }}`` variable will do the trick. This is useful
       if you want to add to the contents of a parent block instead of
-      completely overriding it.
+      completely overriding it. Data inserted using ``{{ block.super }}`` will
+      not be automatically escaped (see the `next section`_), since it was
+      already escaped, if necessary, in the parent template.
 
     * For extra readability, you can optionally give a *name* to your
       ``{% endblock %}`` tag. For example::
@@ -299,6 +303,8 @@ it also defines the content that fills the hole in the *parent*. If there were
 two similarly-named ``{% block %}`` tags in a template, that template's parent
 wouldn't know which one of the blocks' content to use.
 
+.. _next section: #automatic-html-escaping
+
 Automatic HTML escaping
 =======================
 
@@ -397,6 +403,32 @@ auto-escaping is on, these extra filters won't change the output -- any
 variables that use the ``escape`` filter do not have further automatic
 escaping applied to them.
 
+String literals and automatic escaping
+--------------------------------------
+
+Sometimes you will pass a string literal as an argument to a filter. For
+example::
+
+    {{ data|default:"This is a string literal." }}
+
+All string literals are inserted **without** any automatic escaping into the
+template, if they are used (it's as if they were all passed through the
+``safe`` filter). The reasoning behind this is that the template author is in
+control of what goes into the string literal, so they can make sure the text
+is correctly escaped when the template is written.
+
+This means you would write ::
+
+    {{ data|default:"3 > 2" }}
+
+...rather than ::
+
+    {{ data|default:"3 > 2" }}  <-- Bad! Don't do this.
+
+This doesn't affect what happens to data coming from the variable itself.
+The variable's contents are still automatically escaped, if necessary, since
+they're beyond the control of the template author.
+
 Using the built-in reference
 ============================
 

docs/templates_python.txt

@@ -755,61 +755,106 @@ inside the template code:
    ``EscapeString`` and ``EscapeUnicode``. You will not normally need to worry
    about these; they exist for the implementation of the ``escape`` filter.
 
-Inside your filter, you will need to think about three areas in order to be
+When you are writing a filter, your code will typically fall into one of two
-auto-escaping compliant:
+situations:
 
- 1. If your filter returns a string that is ready for direct output (it should
+ 1. Your filter does not introduce any HTML-unsafe characters (``<``, ``>``,
-    be considered a "safe" string), you should call
+    ``'``, ``"`` or ``&``) into the result that were not already present. In
-    ``django.utils.safestring.mark_safe()`` on the result prior to returning.
+    this case, you can let Django take care of all the auto-escaping handling
-    This will turn the result into the appropriate ``SafeData`` type. This is
+    for you. All you need to do is put the ``is_safe`` attribute on your
-    often the case when you are returning raw HTML, for example.
+    filter function and set it to ``True``. This attribute tells Django that
+    is a "safe" string is passed into your filter, the result will still be
+    "safe" and if a non-safe string is passed in, Django will automatically
+    escape it, if necessary. The reason ``is_safe`` is necessary is because
+    there are plenty of normal string operations that will turn a ``SafeData``
+    object back into a normal ``str`` or ``unicode`` object and, rather than
+    try to catch them all, which would be very difficult, Django repairs the
+    damage after the filter has completed.
 
- 2. If your filter is given a "safe" string, is it guaranteed to return a
+    For example, suppose you have a filter that adds the string ``xx`` to the
-    "safe" string? If so, set the ``is_safe`` attribute on the function to be
+    end of any input. Since this introduces no dangerous HTML characters into
-    ``True``. For example, a filter that replaced a word consisting only of
+    the result (aside from any that were already present), you should mark
-    digits with the number spelt out in words is going to be
+    your filter with ``is_safe``::
-    safe-string-preserving, since it cannot introduce any of the five dangerous
-    characters: <, >, ", ' or &. We can write::
 
         @register.filter
-        def convert_to_words(value):
+        def add_xx(value):
-            # ... implementation here ...
+            return '%sxx' % value
-            return result
+        add_xx.is_safe = True
 
-        convert_to_words.is_safe = True
+    When this filter is used in a template where auto-escaping is enabled,
+    Django will escape the output whenever the input is not already marked as
+    "safe".
 
-    Note that this filter does not return a universally safe result (it does
+    By default, ``is_safe`` defaults to ``False`` and you can omit it from
-    not return ``mark_safe(result)``) because if it is handed a raw string such
+    any filters where it isn't required.
-    as '<a>', this will need further escaping in an auto-escape environment.
-    The ``is_safe`` attribute only talks about the the result when a safe
-    string is passed into the filter.
 
- 3. Will your filter behave differently depending upon whether auto-escaping
+    Be careful when deciding if your filter really does leave safe strings
-    is currently in effect or not? This is normally a concern when you are
+    as safe. Sometimes if you are *removing* characters, you can
-    returning mixed content (HTML elements mixed with user-supplied content).
+    inadvertently leave unbalanced HTML tags or entities in the result.
-    For example, the ``ordered_list`` filter that ships with Django needs to
+    For example, removing a ``>`` from the input might turn ``<a>`` into
-    know whether to escape its content or not. It will always return a safe
+    ``<a``, which would need to be escaped on output to avoid causing
-    string. Since it returns raw HTML, we cannot apply escaping to the
+    problems. Similarly, removing a semicolon (``;``) can turn ``&amp;``
-    result -- it needs to be done in-situ.
+    into ``&amp``, which is no longer a valid entity and thus needs
+    further escaping. Most cases won't be nearly this tricky, but keep an
+    eye out for any problems like that when reviewing your code.
 
-    For these cases, the filter function needs to be told what the current
+ 2. Alternatively, your filter code can manually take care of any necessary
-    auto-escaping setting is. Set the ``needs_autoescape`` attribute on the
+    escaping. This is usually necessary when you are introducing new HTML
-    filter to ``True`` and have your function take an extra argument called
+    markup into the result. You want to mark the output as safe from further
-    ``autoescape`` with a default value of ``None``. When the filter is called,
+    escaping so that your HTML markup isn't escaped further, so you'll need to
-    the ``autoescape`` keyword argument will be ``True`` if auto-escaping is in
+    handle the input yourself.
-    effect. For example, the ``unordered_list`` filter is written as::
 
-        def unordered_list(value, autoescape=None):
+    To mark the output as a safe string, use
-            # ... lots of code here ...
+    ``django.utils.safestring.mark_safe()``.
 
-            return mark_safe(...)
+    Be careful, though. You need to do more than just mark the output as
+    safe. You need to ensure it really *is* safe and what you do will often
+    depend upon whether or not auto-escaping is in effect. The idea is to
+    write filters than can operate in templates where auto-escaping is either
+    on or off in order to make things easier for your template authors.
 
-        unordered_list.is_safe = True
+    In order for you filter to know the current auto-escaping state, set the
-        unordered_list.needs_autoescape = True
+    ``needs_autoescape`` attribute to ``True`` on your function (if you don't
+    specify this attribute, it defaults to ``False``). This attribute tells
+    Django that your filter function wants to be passed an extra keyword
+    argument, called ``autoescape`` that is ``True`` is auto-escaping is in
+    effect and ``False`` otherwise.
 
-By default, both the ``is_safe`` and ``needs_autoescape`` attributes are
+    An example might make this clearer. Let's write a filter that emphasizes
-``False``. You do not need to specify them if ``False`` is an acceptable
+    the first character of a string::
-value.
+
+        from django.utils.html import conditional_escape
+        from django.utils.safestring import mark_safe
+
+        def initial_letter_filter(text, autoescape=None):
+            first, other = text[0] ,text[1:]
+            if autoescape:
+                esc = conditional_escape
+            else:
+                esc = lambda x: x
+            result = '<strong>%s</strong>%s' % (esc(first), esc(other))
+            return mark_safe(result)
+        initial_letter_filter.needs_autoescape = True
+
+    The ``needs_autoescape`` attribute on the filter function and the
+    ``autoescape`` keyword argument mean that our function will know whether
+    or not automatic escaping is in effect when the filter is called. We use
+    ``autoescape`` to decide whether the input data needs to be passed through
+    ``django.utils.html.conditional_escape`` or not (in the latter case, we
+    just use the identity function as the "escape" function). The
+    ``conditional_escape()`` function is like ``escape()`` except it only
+    escapes input that is **not** a ``SafeData`` instance. If a ``SafeData``
+    instance is passed to ``conditional_escape()``, the data is returned
+    unchanged.
+
+    Finally, in the above example, we remember to mark the result as safe
+    so that our HTML is inserted directly into the template without further
+    escaping.
+
+    There is no need to worry about the ``is_safe`` attribute in this case
+    (although including it wouldn't hurt anything). Whenever you are manually
+    handling the auto-escaping issues and returning a safe string, the
+    ``is_safe`` attribute won't change anything either way.
 
 Writing custom template tags
 ----------------------------
@@ -932,7 +977,9 @@ without having to be parsed multiple times.
 Auto-escaping considerations
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The output from template tags is not automatically run through the
+**New in Django development version**
+
+The output from template tags is **not** automatically run through the
 auto-escaping filters. However, there are still a couple of things you should
 keep in mind when writing a template tag:
 
@@ -1136,7 +1183,7 @@ on the object being edited -- so they're a perfect case for using a small
 template that is filled with details from the current object. (In the admin's
 case, this is the ``submit_row`` tag.)
 
-These sorts of tags are called `inclusion tags`.
+These sorts of tags are called "inclusion tags".
 
 Writing inclusion tags is probably best demonstrated by example. Let's write a
 tag that outputs a list of choices for a given ``Poll`` object, such as was

docs/tutorial01.txt

@@ -47,7 +47,7 @@ will create a ``mysite`` directory in your current directory.
    denied" when you try to run ``django-admin.py startproject``. This
    is because, on Unix-based systems like OS X, a file must be marked
    as "executable" before it can be run as a program. To do this, open
-   Terminal.app and navigate (using the `cd` command) to the directory
+   Terminal.app and navigate (using the ``cd`` command) to the directory
    where ``django-admin.py`` is installed, then run the command
    ``chmod +x django-admin.py``.
 

docs/unicode.txt

@@ -136,7 +136,7 @@ for converting back and forth between Unicode and bytestrings.
     * ``smart_str(s, encoding='utf-8', strings_only=False, errors='strict')``
       is essentially the opposite of ``smart_unicode()``. It forces the first
       argument to a bytestring. The ``strings_only`` parameter has the same
-      behaviour as for ``smart_unicode()`` and ``force_unicode()``. This is
+      behavior as for ``smart_unicode()`` and ``force_unicode()``. This is
       slightly different semantics from Python's builtin ``str()`` function,
       but the difference is needed in a few places within Django's internals.
 

docs/url_dispatch.txt

@@ -237,7 +237,7 @@ include
 -------
 
 A function that takes a full Python import path to another URLconf that should
-be "included" in this place. See _`Including other URLconfs` below.
+be "included" in this place. See `Including other URLconfs`_ below.
 
 Notes on capturing text in URLs
 ===============================

tests/modeltests/field_subclassing/models.py

@@ -103,4 +103,14 @@ TypeError: Invalid lookup type: 'lt'
 >>> obj = list(serializers.deserialize("json", stream))[0]
 >>> obj.object == m
 True
+
+# Test retrieving custom field data
+>>> m.delete()
+>>> m1 = MyModel(name="1", data=Small(1, 2))
+>>> m1.save()
+>>> m2 = MyModel(name="2", data=Small(2, 3))
+>>> m2.save()
+>>> for m in MyModel.objects.all(): print unicode(m.data)
+12
+23
 """}

tests/modeltests/model_forms/models.py

@@ -30,6 +30,23 @@ ARTICLE_STATUS = (
     (3, 'Live'),
 )
 
+STEERING_TYPE = (
+    ('left', 'Left steering wheel'),
+    ('right', 'Right steering wheel'),
+)
+
+FUEL_TYPE = (
+    ('gas', 'Gasoline'),
+    ('diesel', 'Diesel'),
+    ('other', 'Other'),
+)
+
+TRANSMISSION_TYPE = (
+    ('at', 'Automatic'),
+    ('mt', 'Manual'),
+    ('cvt', 'CVT'),
+)
+
 class Category(models.Model):
     name = models.CharField(max_length=20)
     slug = models.SlugField(max_length=20)
@@ -70,6 +87,12 @@ class PhoneNumber(models.Model):
     def __unicode__(self):
         return self.phone
 
+class Car(models.Model):
+    name = models.CharField(max_length=50)
+    steering = models.CharField(max_length=5, choices=STEERING_TYPE, default='left')
+    fuel = models.CharField(max_length=10, choices=FUEL_TYPE)
+    transmission = models.CharField(max_length=3, choices=TRANSMISSION_TYPE, blank=True, help_text='Leave empty if not applicable.')
+
 __test__ = {'API_TESTS': """
 >>> from django.newforms import form_for_model, form_for_instance, save_instance, BaseForm, Form, CharField
 >>> import datetime
@@ -592,4 +615,54 @@ ValidationError: [u'Select a valid choice. 4 is not one of the available choices
 True
 >>> f.cleaned_data
 {'phone': u'312-555-1212', 'description': u'Assistance'}
+
+# form_for_* blank choices ####################################################
+
+Show the form for a new Car. Note that steering field doesn't include the blank choice,
+because the field is obligatory and has an explicit default.
+>>> CarForm = form_for_model(Car)
+>>> f = CarForm(auto_id=False)
+>>> print f
+<tr><th>Name:</th><td><input type="text" name="name" maxlength="50" /></td></tr>
+<tr><th>Steering:</th><td><select name="steering">
+<option value="left" selected="selected">Left steering wheel</option>
+<option value="right">Right steering wheel</option>
+</select></td></tr>
+<tr><th>Fuel:</th><td><select name="fuel">
+<option value="" selected="selected">---------</option>
+<option value="gas">Gasoline</option>
+<option value="diesel">Diesel</option>
+<option value="other">Other</option>
+</select></td></tr>
+<tr><th>Transmission:</th><td><select name="transmission">
+<option value="" selected="selected">---------</option>
+<option value="at">Automatic</option>
+<option value="mt">Manual</option>
+<option value="cvt">CVT</option>
+</select><br />Leave empty if not applicable.</td></tr>
+
+Create a Car, and display the form for modifying it. Note that now the fuel
+selector doesn't include the blank choice as well, since the field is
+obligatory and can not be changed to be blank.
+>>> honda = Car(name='Honda Accord Wagon', steering='right', fuel='gas', transmission='at')
+>>> honda.save()
+>>> HondaForm = form_for_instance(honda)
+>>> f = HondaForm(auto_id=False)
+>>> print f
+<tr><th>Name:</th><td><input type="text" name="name" value="Honda Accord Wagon" maxlength="50" /></td></tr>
+<tr><th>Steering:</th><td><select name="steering">
+<option value="left">Left steering wheel</option>
+<option value="right" selected="selected">Right steering wheel</option>
+</select></td></tr>
+<tr><th>Fuel:</th><td><select name="fuel">
+<option value="gas" selected="selected">Gasoline</option>
+<option value="diesel">Diesel</option>
+<option value="other">Other</option>
+</select></td></tr>
+<tr><th>Transmission:</th><td><select name="transmission">
+<option value="">---------</option>
+<option value="at" selected="selected">Automatic</option>
+<option value="mt">Manual</option>
+<option value="cvt">CVT</option>
+</select><br />Leave empty if not applicable.</td></tr>
 """}

tests/modeltests/pagination/models.py

@@ -78,7 +78,8 @@ True
 >>> paginator.pages
 2
 
-# The paginator can provide a list of all available pages
+# The paginator can provide a list of all available pages.
+>>> paginator = ObjectPaginator(Article.objects.all(), 10)
 >>> paginator.page_range
 [1, 2]
 """}

tests/regressiontests/cache/tests.py

@@ -3,9 +3,12 @@
 # Unit tests for cache framework
 # Uses whatever cache backend is set in the test settings file.
 
-from django.core.cache import cache
 import time, unittest
 
+from django.core.cache import cache
+from django.utils.cache import patch_vary_headers
+from django.http import HttpResponse
+
 # functions/classes for complex data type tests
 def f():
     return 42
@@ -87,5 +90,30 @@ class Cache(unittest.TestCase):
             cache.set(key, value)
             self.assertEqual(cache.get(key), value)
 
+
+class CacheUtils(unittest.TestCase):
+    """TestCase for django.utils.cache functions."""
+    
+    def test_patch_vary_headers(self):
+        headers = ( 
+            # Initial vary, new headers, resulting vary.
+            (None, ('Accept-Encoding',), 'Accept-Encoding'),
+            ('Accept-Encoding', ('accept-encoding',), 'Accept-Encoding'),
+            ('Accept-Encoding', ('ACCEPT-ENCODING',), 'Accept-Encoding'),
+            ('Cookie', ('Accept-Encoding',), 'Cookie, Accept-Encoding'),
+            ('Cookie, Accept-Encoding', ('Accept-Encoding',), 'Cookie, Accept-Encoding'),
+            ('Cookie, Accept-Encoding', ('Accept-Encoding', 'cookie'), 'Cookie, Accept-Encoding'),
+            (None, ('Accept-Encoding', 'COOKIE'), 'Accept-Encoding, COOKIE'),
+            ('Cookie,     Accept-Encoding', ('Accept-Encoding', 'cookie'), 'Cookie, Accept-Encoding'),
+            ('Cookie    ,     Accept-Encoding', ('Accept-Encoding', 'cookie'), 'Cookie, Accept-Encoding'),
+        )
+        for initial_vary, newheaders, resulting_vary in headers:
+            response = HttpResponse()
+            if initial_vary is not None:
+                response['Vary'] = initial_vary
+            patch_vary_headers(response, newheaders)
+            self.assertEqual(response['Vary'], resulting_vary)
+
+
 if __name__ == '__main__':
     unittest.main()

tests/regressiontests/datastructures/tests.py

@@ -25,11 +25,23 @@
 >>> d = MultiValueDict({'name': ['Adrian', 'Simon'], 'position': ['Developer']})
 >>> d['name']
 'Simon'
+>>> d.get('name')
+'Simon'
 >>> d.getlist('name')
 ['Adrian', 'Simon']
+>>> d['lastname']
+Traceback (most recent call last):
+...
+MultiValueDictKeyError: "Key 'lastname' not found in <MultiValueDict: {'position': ['Developer'], 'name': ['Adrian', 'Simon']}>"
+>>> d.get('lastname')
+
 >>> d.get('lastname', 'nonexistent')
 'nonexistent'
+>>> d.getlist('lastname')
+[]
 >>> d.setlist('lastname', ['Holovaty', 'Willison'])
+>>> d.getlist('lastname')
+['Holovaty', 'Willison']
 
 ### SortedDict #################################################################
 

tests/regressiontests/dateformat/tests.py

@@ -66,6 +66,9 @@ u'1979 189 CET'
 
 >>> format(my_birthday, r'jS o\f F')
 u'8th of July'
+
+>>> format(the_future, r'Y')
+u'2100'
 """
 
 from django.utils import dateformat, translation
@@ -84,3 +87,4 @@ except AttributeError:
 my_birthday = datetime.datetime(1979, 7, 8, 22, 00)
 summertime = datetime.datetime(2005, 10, 30, 1, 00)
 wintertime = datetime.datetime(2005, 10, 30, 4, 00)
+the_future = datetime.datetime(2100, 10, 25, 0, 00)

tests/regressiontests/defaultfilters/tests.py

@@ -37,6 +37,8 @@ u''
 u'13.1031'
 >>> floatformat(u'foo', u'bar')
 u''
+>>> floatformat(None)
+u''
 
 >>> addslashes(u'"double quotes" and \'single quotes\'')
 u'\\"double quotes\\" and \\\'single quotes\\\''

tests/regressiontests/forms/error_messages.py

@@ -312,4 +312,49 @@ ValidationError: [u'REQUIRED']
 Traceback (most recent call last):
 ...
 ValidationError: [u'INVALID IP ADDRESS']
+
+###############################################################################
+
+# Create choices for the model choice field tests below.
+
+>>> from regressiontests.forms.models import ChoiceModel
+>>> ChoiceModel.objects.create(pk=1, name='a')
+<ChoiceModel: ChoiceModel object>
+>>> ChoiceModel.objects.create(pk=2, name='b')
+<ChoiceModel: ChoiceModel object>
+>>> ChoiceModel.objects.create(pk=3, name='c')
+<ChoiceModel: ChoiceModel object>
+
+# ModelChoiceField ############################################################
+
+>>> e = {'required': 'REQUIRED'}
+>>> e['invalid_choice'] = 'INVALID CHOICE'
+>>> f = ModelChoiceField(queryset=ChoiceModel.objects.all(), error_messages=e)
+>>> f.clean('')
+Traceback (most recent call last):
+...
+ValidationError: [u'REQUIRED']
+>>> f.clean('4')
+Traceback (most recent call last):
+...
+ValidationError: [u'INVALID CHOICE']
+
+# ModelMultipleChoiceField ####################################################
+
+>>> e = {'required': 'REQUIRED'}
+>>> e['invalid_choice'] = '%s IS INVALID CHOICE'
+>>> e['list'] = 'NOT A LIST OF VALUES'
+>>> f = ModelMultipleChoiceField(queryset=ChoiceModel.objects.all(), error_messages=e)
+>>> f.clean('')
+Traceback (most recent call last):
+...
+ValidationError: [u'REQUIRED']
+>>> f.clean('3')
+Traceback (most recent call last):
+...
+ValidationError: [u'NOT A LIST OF VALUES']
+>>> f.clean(['4'])
+Traceback (most recent call last):
+...
+ValidationError: [u'4 IS INVALID CHOICE']
 """

tests/regressiontests/forms/fields.py

@@ -323,6 +323,10 @@ Decimal("3.14")
 Traceback (most recent call last):
 ...
 ValidationError: [u'Enter a number.']
+>>> f.clean(u'łąść')
+Traceback (most recent call last):
+...
+ValidationError: [u'Enter a number.']
 >>> f.clean('1.0 ')
 Decimal("1.0")
 >>> f.clean(' 1.0')
@@ -914,6 +918,11 @@ False
 >>> f.clean('Django rocks')
 True
 
+>>> f.clean('True')
+True
+>>> f.clean('False')
+False
+
 >>> f = BooleanField(required=False)
 >>> f.clean('')
 False
@@ -930,6 +939,11 @@ False
 >>> f.clean('Django rocks')
 True
 
+A form's BooleanField with a hidden widget will output the string 'False', so
+that should clean to the boolean value False:
+>>> f.clean('False')
+False
+
 # ChoiceField #################################################################
 
 >>> f = ChoiceField(choices=[('1', '1'), ('2', '2')])

tests/regressiontests/forms/localflavor/ca.py

@@ -147,6 +147,10 @@ u'BC'
 u'NS'
 >>> f.clean('  manitoba ')
 u'MB'
+>>> f.clean(' new brunswick ')
+u'NB'
+>>> f.clean('NB')
+u'NB'
 >>> f.clean('T2S 2H7')
 Traceback (most recent call last):
 ...

tests/regressiontests/forms/models.py

@@ -10,6 +10,10 @@ class Defaults(models.Model):
     def_date = models.DateField(default = datetime.date(1980, 1, 1))
     value = models.IntegerField(default=42)
 
+class ChoiceModel(models.Model):
+    """For ModelChoiceField and ModelMultipleChoiceField tests."""
+    name = models.CharField(max_length=10)
+
 __test__ = {'API_TESTS': """
 >>> from django.newforms import form_for_model, form_for_instance
 

tests/regressiontests/forms/widgets.py

@@ -2,6 +2,7 @@
 tests = r"""
 >>> from django.newforms import *
 >>> from django.newforms.widgets import RadioFieldRenderer
+>>> from django.utils.safestring import mark_safe
 >>> import datetime
 >>> import time
 >>> import re
@@ -128,6 +129,13 @@ u'<input type="hidden" class="fun" value="\u0160\u0110\u0106\u017d\u0107\u017e\u
 >>> w.render('email', '', attrs={'class': 'special'})
 u'<input type="hidden" class="special" name="email" />'
 
+Boolean values are rendered to their string forms ("True" and "False").
+>>> w = HiddenInput()
+>>> w.render('get_spam', False)
+u'<input type="hidden" name="get_spam" value="False" />'
+>>> w.render('get_spam', True)
+u'<input type="hidden" name="get_spam" value="True" />'
+
 # MultipleHiddenInput Widget ##################################################
 
 >>> w = MultipleHiddenInput()
@@ -205,6 +213,8 @@ u'<textarea rows="10" cols="40" name="msg"></textarea>'
 u'<textarea rows="10" cols="40" name="msg">value</textarea>'
 >>> w.render('msg', 'some "quoted" & ampersanded value')
 u'<textarea rows="10" cols="40" name="msg">some &quot;quoted&quot; &amp; ampersanded value</textarea>'
+>>> w.render('msg', mark_safe('pre &quot;quoted&quot; value'))
+u'<textarea rows="10" cols="40" name="msg">pre &quot;quoted&quot; value</textarea>'
 >>> w.render('msg', 'value', attrs={'class': 'pretty', 'rows': 20})
 u'<textarea class="pretty" rows="20" cols="40" name="msg">value</textarea>'
 
@@ -375,6 +385,17 @@ If 'choices' is passed to both the constructor and render(), then they'll both b
 <option value="5">5</option>
 </select>
 
+# Choices are escaped correctly
+>>> print w.render('escape', None, choices=(('bad', 'you & me'), ('good', mark_safe('you &gt; me'))))
+<select name="escape">
+<option value="1">1</option>
+<option value="2">2</option>
+<option value="3">3</option>
+<option value="bad">you &amp; me</option>
+<option value="good">you &gt; me</option>
+</select>
+
+# Unicode choices are correctly rendered as HTML
 >>> w.render('email', 'ŠĐĆŽćžšđ', choices=[('ŠĐĆŽćžšđ', 'ŠĐabcĆŽćžšđ'), ('ćžšđ', 'abcćžšđ')])
 u'<select name="email">\n<option value="1">1</option>\n<option value="2">2</option>\n<option value="3">3</option>\n<option value="\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111" selected="selected">\u0160\u0110abc\u0106\u017d\u0107\u017e\u0161\u0111</option>\n<option value="\u0107\u017e\u0161\u0111">abc\u0107\u017e\u0161\u0111</option>\n</select>'
 
@@ -538,6 +559,17 @@ If 'choices' is passed to both the constructor and render(), then they'll both b
 <option value="5">5</option>
 </select>
 
+# Choices are escaped correctly
+>>> print w.render('escape', None, choices=(('bad', 'you & me'), ('good', mark_safe('you &gt; me'))))
+<select multiple="multiple" name="escape">
+<option value="1">1</option>
+<option value="2">2</option>
+<option value="3">3</option>
+<option value="bad">you &amp; me</option>
+<option value="good">you &gt; me</option>
+</select>
+
+# Unicode choices are correctly rendered as HTML
 >>> w.render('nums', ['ŠĐĆŽćžšđ'], choices=[('ŠĐĆŽćžšđ', 'ŠĐabcĆŽćžšđ'), ('ćžšđ', 'abcćžšđ')])
 u'<select multiple="multiple" name="nums">\n<option value="1">1</option>\n<option value="2">2</option>\n<option value="3">3</option>\n<option value="\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111" selected="selected">\u0160\u0110abc\u0106\u017d\u0107\u017e\u0161\u0111</option>\n<option value="\u0107\u017e\u0161\u0111">abc\u0107\u017e\u0161\u0111</option>\n</select>'
 
@@ -663,6 +695,16 @@ You can create your own custom renderers for RadioSelect to use.
 <label><input checked="checked" type="radio" name="beatle" value="G" /> George</label><br />
 <label><input type="radio" name="beatle" value="R" /> Ringo</label>
 
+Or you can use custom RadioSelect fields that use your custom renderer.
+>>> class CustomRadioSelect(RadioSelect):
+...    renderer = MyRenderer
+>>> w = CustomRadioSelect()
+>>> print w.render('beatle', 'G', choices=(('J', 'John'), ('P', 'Paul'), ('G', 'George'), ('R', 'Ringo')))
+<label><input type="radio" name="beatle" value="J" /> John</label><br />
+<label><input type="radio" name="beatle" value="P" /> Paul</label><br />
+<label><input checked="checked" type="radio" name="beatle" value="G" /> George</label><br />
+<label><input type="radio" name="beatle" value="R" /> Ringo</label>
+
 A RadioFieldRenderer object also allows index access to individual RadioInput
 objects.
 >>> w = RadioSelect()
@@ -682,6 +724,14 @@ Traceback (most recent call last):
 ...
 IndexError: list index out of range
 
+# Choices are escaped correctly
+>>> w = RadioSelect()
+>>> print w.render('escape', None, choices=(('bad', 'you & me'), ('good', mark_safe('you &gt; me'))))
+<ul>
+<li><label><input type="radio" name="escape" value="bad" /> you &amp; me</label></li>
+<li><label><input type="radio" name="escape" value="good" /> you &gt; me</label></li>
+</ul>
+
 # Unicode choices are correctly rendered as HTML
 >>> w = RadioSelect()
 >>> unicode(w.render('email', 'ŠĐĆŽćžšđ', choices=[('ŠĐĆŽćžšđ', 'ŠĐabcĆŽćžšđ'), ('ćžšđ', 'abcćžšđ')]))
@@ -811,6 +861,17 @@ If 'choices' is passed to both the constructor and render(), then they'll both b
 <li><label><input type="checkbox" name="nums" value="5" /> 5</label></li>
 </ul>
 
+# Choices are escaped correctly
+>>> print w.render('escape', None, choices=(('bad', 'you & me'), ('good', mark_safe('you &gt; me'))))
+<ul>
+<li><label><input type="checkbox" name="escape" value="1" /> 1</label></li>
+<li><label><input type="checkbox" name="escape" value="2" /> 2</label></li>
+<li><label><input type="checkbox" name="escape" value="3" /> 3</label></li>
+<li><label><input type="checkbox" name="escape" value="bad" /> you &amp; me</label></li>
+<li><label><input type="checkbox" name="escape" value="good" /> you &gt; me</label></li>
+</ul>
+
+# Unicode choices are correctly rendered as HTML
 >>> w.render('nums', ['ŠĐĆŽćžšđ'], choices=[('ŠĐĆŽćžšđ', 'ŠĐabcĆŽćžšđ'), ('ćžšđ', 'abcćžšđ')])
 u'<ul>\n<li><label><input type="checkbox" name="nums" value="1" /> 1</label></li>\n<li><label><input type="checkbox" name="nums" value="2" /> 2</label></li>\n<li><label><input type="checkbox" name="nums" value="3" /> 3</label></li>\n<li><label><input checked="checked" type="checkbox" name="nums" value="\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111" /> \u0160\u0110abc\u0106\u017d\u0107\u017e\u0161\u0111</label></li>\n<li><label><input type="checkbox" name="nums" value="\u0107\u017e\u0161\u0111" /> abc\u0107\u017e\u0161\u0111</label></li>\n</ul>'
 

tests/regressiontests/i18n/tests.py

@@ -4,7 +4,7 @@ import misc
 regressions = ur"""
 Format string interpolation should work with *_lazy objects.
 
->>> from django.utils.translation import ugettext_lazy, activate, deactivate, gettext_lazy
+>>> from django.utils.translation import ugettext, ugettext_lazy, activate, deactivate, gettext_lazy
 >>> s = ugettext_lazy('Add %(name)s')
 >>> d = {'name': 'Ringo'}
 >>> s % d
@@ -39,6 +39,18 @@ unicode(string_concat(...)) should not raise a TypeError - #4796
 <module 'django.utils.translation' from ...>
 >>> unicode(django.utils.translation.string_concat("dja", "ngo"))
 u'django'
+
+Translating a string requiring no auto-escaping shouldn't change the "safe"
+status.
+
+>>> from django.utils.safestring import mark_safe
+>>> s = mark_safe('Password')
+>>> type(s)
+<class 'django.utils.safestring.SafeString'>
+>>> activate('de')
+>>> type(ugettext(s))
+<class 'django.utils.safestring.SafeUnicode'>
+>>> deactivate()
 """
 
 __test__ = {

tests/regressiontests/templates/filters.py

@@ -12,6 +12,15 @@ from datetime import datetime, timedelta
 from django.utils.tzinfo import LocalTimezone
 from django.utils.safestring import mark_safe
 
+# These two classes are used to test auto-escaping of __unicode__ output.
+class UnsafeClass:
+    def __unicode__(self):
+        return u'you & me'
+
+class SafeClass:
+    def __unicode__(self):
+        return mark_safe(u'you > me')
+
 # RESULT SYNTAX --
 # 'template_name': ('template contents', 'context dict',
 #                   'expected string output' or Exception class)
@@ -94,6 +103,11 @@ def get_filter_tests():
         'filter-urlize03': ('{% autoescape off %}{{ a|urlize }}{% endautoescape %}', {"a": mark_safe("a & b")}, 'a & b'),
         'filter-urlize04': ('{{ a|urlize }}', {"a": mark_safe("a & b")}, 'a & b'),
 
+        # This will lead to a nonsense result, but at least it won't be
+        # exploitable for XSS purposes when auto-escaping is on.
+        'filter-urlize05': ('{% autoescape off %}{{ a|urlize }}{% endautoescape %}', {"a": "<script>alert('foo')</script>"}, "<script>alert('foo')</script>"),
+        'filter-urlize06': ('{{ a|urlize }}', {"a": "<script>alert('foo')</script>"}, '&lt;script&gt;alert(&#39;foo&#39;)&lt;/script&gt;'),
+
         'filter-urlizetrunc01': ('{% autoescape off %}{{ a|urlizetrunc:"8" }} {{ b|urlizetrunc:"8" }}{% endautoescape %}', {"a": "http://example.com/x=&y=", "b": mark_safe("http://example.com?x=&y=")}, u'<a href="http://example.com/x=&y=" rel="nofollow">http:...</a> <a href="http://example.com?x=&y=" rel="nofollow">http:...</a>'),
         'filter-urlizetrunc02': ('{{ a|urlizetrunc:"8" }} {{ b|urlizetrunc:"8" }}', {"a": "http://example.com/x=&y=", "b": mark_safe("http://example.com?x=&y=")}, u'<a href="http://example.com/x=&y=" rel="nofollow">http:...</a> <a href="http://example.com?x=&y=" rel="nofollow">http:...</a>'),
 
@@ -177,23 +191,28 @@ def get_filter_tests():
         'filter-unordered_list04': ('{% autoescape off %}{{ a|unordered_list }}{% endautoescape %}', {"a": ["x>", [[mark_safe("<y"), []]]]}, "\t<li>x>\n\t<ul>\n\t\t<li><y</li>\n\t</ul>\n\t</li>"),
         'filter-unordered_list05': ('{% autoescape off %}{{ a|unordered_list }}{% endautoescape %}', {"a": ["x>", [["<y", []]]]}, "\t<li>x>\n\t<ul>\n\t\t<li><y</li>\n\t</ul>\n\t</li>"),
 
-        # If the input to "default" filter is marked as safe, then so is the
+        # Literal string arguments to the default filter are always treated as
-        # output. However, if the default arg is used, auto-escaping kicks in
+        # safe strings, regardless of the auto-escaping state.
-        # (if enabled), because we cannot mark the default as safe.
         #
         # Note: we have to use {"a": ""} here, otherwise the invalid template
         # variable string interferes with the test result.
-        'filter-default01': ('{{ a|default:"x<" }}', {"a": ""}, "x&lt;"),
+        'filter-default01': ('{{ a|default:"x<" }}', {"a": ""}, "x<"),
         'filter-default02': ('{% autoescape off %}{{ a|default:"x<" }}{% endautoescape %}', {"a": ""}, "x<"),
         'filter-default03': ('{{ a|default:"x<" }}', {"a": mark_safe("x>")}, "x>"),
         'filter-default04': ('{% autoescape off %}{{ a|default:"x<" }}{% endautoescape %}', {"a": mark_safe("x>")}, "x>"),
 
-        'filter-default_if_none01': ('{{ a|default:"x<" }}', {"a": None}, "x&lt;"),
+        'filter-default_if_none01': ('{{ a|default:"x<" }}', {"a": None}, "x<"),
         'filter-default_if_none02': ('{% autoescape off %}{{ a|default:"x<" }}{% endautoescape %}', {"a": None}, "x<"),
 
         'filter-phone2numeric01': ('{{ a|phone2numeric }} {{ b|phone2numeric }}', {"a": "<1-800-call-me>", "b": mark_safe("<1-800-call-me>") }, "&lt;1-800-2255-63&gt; <1-800-2255-63>"),
         'filter-phone2numeric02': ('{% autoescape off %}{{ a|phone2numeric }} {{ b|phone2numeric }}{% endautoescape %}', {"a": "<1-800-call-me>", "b": mark_safe("<1-800-call-me>") }, "<1-800-2255-63> <1-800-2255-63>"),
 
+        # Ensure iriencode keeps safe strings:
+        'filter-iriencode01': ('{{ url|iriencode }}', {'url': '?test=1&me=2'}, '?test=1&amp;me=2'),
+        'filter-iriencode02': ('{% autoescape off %}{{ url|iriencode }}{% endautoescape %}', {'url': '?test=1&me=2'}, '?test=1&me=2'),
+        'filter-iriencode03': ('{{ url|iriencode }}', {'url': mark_safe('?test=1&me=2')}, '?test=1&me=2'),
+        'filter-iriencode04': ('{% autoescape off %}{{ url|iriencode }}{% endautoescape %}', {'url': mark_safe('?test=1&me=2')}, '?test=1&me=2'),
+
         # Chaining a bunch of safeness-preserving filters should not alter
         # the safe status either way.
         'chaining01': ('{{ a|capfirst|center:"7" }}.{{ b|capfirst|center:"7" }}', {"a": "a < b", "b": mark_safe("a < b")}, " A &lt; b . A < b "),
@@ -209,12 +228,19 @@ def get_filter_tests():
 
         # Force to safe, then back (also showing why using force_escape too
         # early in a chain can lead to unexpected results).
-        'chaining07': ('{{ a|force_escape|cut:"b" }}', {"a": "a < b"}, "a &lt; "),
+        'chaining07': ('{{ a|force_escape|cut:";" }}', {"a": "a < b"}, "a &amp;lt b"),
-        'chaining08': ('{% autoescape off %}{{ a|force_escape|cut:"b" }}{% endautoescape %}', {"a": "a < b"}, "a &lt; "),
+        'chaining08': ('{% autoescape off %}{{ a|force_escape|cut:";" }}{% endautoescape %}', {"a": "a < b"}, "a &lt b"),
-        'chaining09': ('{{ a|cut:"b"|force_escape }}', {"a": "a < b"}, "a &lt; "),
+        'chaining09': ('{{ a|cut:";"|force_escape }}', {"a": "a < b"}, "a &lt; b"),
-        'chaining10': ('{% autoescape off %}{{ a|cut:"b"|force_escape }}{% endautoescape %}', {"a": "a < b"}, "a &lt; "),
+        'chaining10': ('{% autoescape off %}{{ a|cut:";"|force_escape }}{% endautoescape %}', {"a": "a < b"}, "a &lt; b"),
         'chaining11': ('{{ a|cut:"b"|safe }}', {"a": "a < b"}, "a < "),
         'chaining12': ('{% autoescape off %}{{ a|cut:"b"|safe }}{% endautoescape %}', {"a": "a < b"}, "a < "),
         'chaining13': ('{{ a|safe|force_escape }}', {"a": "a < b"}, "a &lt; b"),
         'chaining14': ('{% autoescape off %}{{ a|safe|force_escape }}{% endautoescape %}', {"a": "a < b"}, "a &lt; b"),
+
+        # Filters decorated with stringfilter still respect is_safe. 
+        'autoescape-stringfilter01': (r'{{ unsafe|capfirst }}', {'unsafe': UnsafeClass()}, 'You &amp; me'),
+        'autoescape-stringfilter02': (r'{% autoescape off %}{{ unsafe|capfirst }}{% endautoescape %}', {'unsafe': UnsafeClass()}, 'You & me'),
+        'autoescape-stringfilter03': (r'{{ safe|capfirst }}', {'safe': SafeClass()}, 'You &gt; me'),
+        'autoescape-stringfilter04': (r'{% autoescape off %}{{ safe|capfirst }}{% endautoescape %}', {'safe': SafeClass()}, 'You &gt; me'),
     }
+

tests/regressiontests/templates/tests.py

@@ -268,6 +268,12 @@ class Templates(unittest.TestCase):
             # Embedded newlines make it not-a-tag.
             'basic-syntax24': ("{{ moo\n }}", {}, "{{ moo\n }}"),
 
+            # Literal strings are permitted inside variables, mostly for i18n
+            # purposes.
+            'basic-syntax25': ('{{ "fred" }}', {}, "fred"),
+            'basic-syntax26': (r'{{ "\"fred\"" }}', {}, "\"fred\""),
+            'basic-syntax27': (r'{{ _("\"fred\"") }}', {}, "\"fred\""),
+
             # List-index syntax allows a template to access a certain item of a subscriptable object.
             'list-index01': ("{{ var.1 }}", {"var": ["first item", "second item"]}, "second item"),
 
@@ -318,9 +324,9 @@ class Templates(unittest.TestCase):
             # Chained filters, with an argument to the first one
             'filter-syntax09': ('{{ var|removetags:"b i"|upper|lower }}', {"var": "<b><i>Yes</i></b>"}, "yes"),
 
-            # Escaped string as argument
+            # Literal string as argument is always "safe" from auto-escaping..
             'filter-syntax10': (r'{{ var|default_if_none:" endquote\" hah" }}',
-                    {"var": None}, ' endquote&quot; hah'),
+                    {"var": None}, ' endquote" hah'),
 
             # Variable as argument
             'filter-syntax11': (r'{{ var|default_if_none:var2 }}', {"var": None, "var2": "happy"}, 'happy'),
@@ -617,7 +623,7 @@ class Templates(unittest.TestCase):
             ### INHERITANCE ###########################################################
 
             # Standard template with no inheritance
-            'inheritance01': ("1{% block first %}_{% endblock %}3{% block second %}_{% endblock %}", {}, '1_3_'),
+            'inheritance01': ("1{% block first %}&{% endblock %}3{% block second %}_{% endblock %}", {}, '1&3_'),
 
             # Standard two-level inheritance
             'inheritance02': ("{% extends 'inheritance01' %}{% block first %}2{% endblock %}{% block second %}4{% endblock %}", {}, '1234'),
@@ -626,7 +632,7 @@ class Templates(unittest.TestCase):
             'inheritance03': ("{% extends 'inheritance02' %}", {}, '1234'),
 
             # Two-level with no redefinitions on second level
-            'inheritance04': ("{% extends 'inheritance01' %}", {}, '1_3_'),
+            'inheritance04': ("{% extends 'inheritance01' %}", {}, '1&3_'),
 
             # Two-level with double quotes instead of single quotes
             'inheritance05': ('{% extends "inheritance02" %}', {}, '1234'),
@@ -635,16 +641,16 @@ class Templates(unittest.TestCase):
             'inheritance06': ("{% extends foo %}", {'foo': 'inheritance02'}, '1234'),
 
             # Two-level with one block defined, one block not defined
-            'inheritance07': ("{% extends 'inheritance01' %}{% block second %}5{% endblock %}", {}, '1_35'),
+            'inheritance07': ("{% extends 'inheritance01' %}{% block second %}5{% endblock %}", {}, '1&35'),
 
             # Three-level with one block defined on this level, two blocks defined next level
             'inheritance08': ("{% extends 'inheritance02' %}{% block second %}5{% endblock %}", {}, '1235'),
 
             # Three-level with second and third levels blank
-            'inheritance09': ("{% extends 'inheritance04' %}", {}, '1_3_'),
+            'inheritance09': ("{% extends 'inheritance04' %}", {}, '1&3_'),
 
             # Three-level with space NOT in a block -- should be ignored
-            'inheritance10': ("{% extends 'inheritance04' %}      ", {}, '1_3_'),
+            'inheritance10': ("{% extends 'inheritance04' %}      ", {}, '1&3_'),
 
             # Three-level with both blocks defined on this level, but none on second level
             'inheritance11': ("{% extends 'inheritance04' %}{% block first %}2{% endblock %}{% block second %}4{% endblock %}", {}, '1234'),
@@ -656,7 +662,7 @@ class Templates(unittest.TestCase):
             'inheritance13': ("{% extends 'inheritance02' %}{% block first %}a{% endblock %}{% block second %}b{% endblock %}", {}, '1a3b'),
 
             # A block defined only in a child template shouldn't be displayed
-            'inheritance14': ("{% extends 'inheritance01' %}{% block newblock %}NO DISPLAY{% endblock %}", {}, '1_3_'),
+            'inheritance14': ("{% extends 'inheritance01' %}{% block newblock %}NO DISPLAY{% endblock %}", {}, '1&3_'),
 
             # A block within another block
             'inheritance15': ("{% extends 'inheritance01' %}{% block first %}2{% block inner %}inner{% endblock %}{% endblock %}", {}, '12inner3_'),
@@ -674,16 +680,16 @@ class Templates(unittest.TestCase):
             'inheritance19': ("{% extends 'inheritance01' %}{% block first %}{% load testtags %}{% echo 400 %}5678{% endblock %}", {}, '140056783_'),
 
             # Two-level inheritance with {{ block.super }}
-            'inheritance20': ("{% extends 'inheritance01' %}{% block first %}{{ block.super }}a{% endblock %}", {}, '1_a3_'),
+            'inheritance20': ("{% extends 'inheritance01' %}{% block first %}{{ block.super }}a{% endblock %}", {}, '1&a3_'),
 
             # Three-level inheritance with {{ block.super }} from parent
             'inheritance21': ("{% extends 'inheritance02' %}{% block first %}{{ block.super }}a{% endblock %}", {}, '12a34'),
 
             # Three-level inheritance with {{ block.super }} from grandparent
-            'inheritance22': ("{% extends 'inheritance04' %}{% block first %}{{ block.super }}a{% endblock %}", {}, '1_a3_'),
+            'inheritance22': ("{% extends 'inheritance04' %}{% block first %}{{ block.super }}a{% endblock %}", {}, '1&a3_'),
 
             # Three-level inheritance with {{ block.super }} from parent and grandparent
-            'inheritance23': ("{% extends 'inheritance20' %}{% block first %}{{ block.super }}b{% endblock %}", {}, '1_ab3_'),
+            'inheritance23': ("{% extends 'inheritance20' %}{% block first %}{{ block.super }}b{% endblock %}", {}, '1&ab3_'),
 
             # Inheritance from local context without use of template loader
             'inheritance24': ("{% extends context_template %}{% block first %}2{% endblock %}{% block second %}4{% endblock %}", {'context_template': template.Template("1{% block first %}_{% endblock %}3{% block second %}_{% endblock %}")}, '1234'),
@@ -705,10 +711,10 @@ class Templates(unittest.TestCase):
             'i18n02': ('{% load i18n %}{% trans "xxxyyyxxx" %}', {}, "xxxyyyxxx"),
 
             # simple translation of a variable
-            'i18n03': ('{% load i18n %}{% blocktrans %}{{ anton }}{% endblocktrans %}', {'anton': 'xxxyyyxxx'}, "xxxyyyxxx"),
+            'i18n03': ('{% load i18n %}{% blocktrans %}{{ anton }}{% endblocktrans %}', {'anton': '\xc3\x85'}, u"Å"),
 
             # simple translation of a variable and filter
-            'i18n04': ('{% load i18n %}{% blocktrans with anton|lower as berta %}{{ berta }}{% endblocktrans %}', {'anton': 'XXXYYYXXX'}, "xxxyyyxxx"),
+            'i18n04': ('{% load i18n %}{% blocktrans with anton|lower as berta %}{{ berta }}{% endblocktrans %}', {'anton': '\xc3\x85'}, u'å'),
 
             # simple translation of a string with interpolation
             'i18n05': ('{% load i18n %}{% blocktrans %}xxx{{ anton }}xxx{% endblocktrans %}', {'anton': 'yyy'}, "xxxyyyxxx"),
@@ -717,10 +723,10 @@ class Templates(unittest.TestCase):
             'i18n06': ('{% load i18n %}{% trans "Page not found" %}', {'LANGUAGE_CODE': 'de'}, "Seite nicht gefunden"),
 
             # translation of singular form
-            'i18n07': ('{% load i18n %}{% blocktrans count number as counter %}singular{% plural %}plural{% endblocktrans %}', {'number': 1}, "singular"),
+            'i18n07': ('{% load i18n %}{% blocktrans count number as counter %}singular{% plural %}{{ counter }} plural{% endblocktrans %}', {'number': 1}, "singular"),
 
             # translation of plural form
-            'i18n08': ('{% load i18n %}{% blocktrans count number as counter %}singular{% plural %}plural{% endblocktrans %}', {'number': 2}, "plural"),
+            'i18n08': ('{% load i18n %}{% blocktrans count number as counter %}singular{% plural %}{{ counter }} plural{% endblocktrans %}', {'number': 2}, "2 plural"),
 
             # simple non-translation (only marking) of a string to german
             'i18n09': ('{% load i18n %}{% trans "Page not found" noop %}', {'LANGUAGE_CODE': 'de'}, "Page not found"),
@@ -734,8 +740,16 @@ class Templates(unittest.TestCase):
             # usage of the get_available_languages tag
             'i18n12': ('{% load i18n %}{% get_available_languages as langs %}{% for lang in langs %}{% ifequal lang.0 "de" %}{{ lang.0 }}{% endifequal %}{% endfor %}', {}, 'de'),
 
-            # translation of a constant string
+            # translation of constant strings
-            'i18n13': ('{{ _("Page not found") }}', {'LANGUAGE_CODE': 'de'}, 'Seite nicht gefunden'),
+            'i18n13': ('{{ _("Password") }}', {'LANGUAGE_CODE': 'de'}, 'Passwort'),
+            'i18n14': ('{% cycle "foo" _("Password") _(\'Password\') as c %} {% cycle c %} {% cycle c %}', {'LANGUAGE_CODE': 'de'}, 'foo Passwort Passwort'),
+            'i18n15': ('{{ absent|default:_("Password") }}', {'LANGUAGE_CODE': 'de', 'absent': ""}, 'Passwort'),
+            'i18n16': ('{{ _("<") }}', {'LANGUAGE_CODE': 'de'}, '<'),
+
+            # Escaping inside blocktrans works as if it was directly in the
+            # template.
+            'i18n17': ('{% load i18n %}{% blocktrans with anton|escape as berta %}{{ berta }}{% endblocktrans %}', {'anton': 'α & β'}, u'α &amp; β'),
+            'i18n18': ('{% load i18n %}{% blocktrans with anton|force_escape as berta %}{{ berta }}{% endblocktrans %}', {'anton': 'α & β'}, u'α &amp; β'),
 
             ### HANDLING OF TEMPLATE_STRING_IF_INVALID ###################################
 
@@ -883,9 +897,14 @@ class Templates(unittest.TestCase):
             'autoescape-tag06': ("{{ first }}", {"first": mark_safe("<b>first</b>")}, "<b>first</b>"),
             'autoescape-tag07': ("{% autoescape on %}{{ first }}{% endautoescape %}", {"first": mark_safe(u"<b>Apple</b>")}, u"<b>Apple</b>"),
 
-            # String arguments to filters, if used in the result, are escaped,
+            # Literal string arguments to filters, if used in the result, are
-            # too.
+            # safe.
-            'basic-syntax08': (r'{% autoescape on %}{{ var|default_if_none:" endquote\" hah" }}{% endautoescape %}', {"var": None}, ' endquote&quot; hah'),
+            'autoescape-tag08': (r'{% autoescape on %}{{ var|default_if_none:" endquote\" hah" }}{% endautoescape %}', {"var": None}, ' endquote" hah'),
+
+            # Objects which return safe strings as their __unicode__ method
+            # won't get double-escaped.
+            'autoescape-tag09': (r'{{ unsafe }}', {'unsafe': filters.UnsafeClass()}, 'you &amp; me'),
+            'autoescape-tag10': (r'{{ safe }}', {'safe': filters.SafeClass()}, 'you &gt; me'),
 
             # The "safe" and "escape" filters cannot work due to internal
             # implementation details (fortunately, the (no)autoescape block

tests/regressiontests/utils/datastructures.py

@@ -0,0 +1,51 @@
+"""
+>>> from django.utils.datastructures import SortedDict
+
+>>> d = SortedDict()
+>>> d[7] = 'seven'
+>>> d[1] = 'one'
+>>> d[9] = 'nine'
+>>> d.keys()
+[7, 1, 9]
+>>> d.values()
+['seven', 'one', 'nine']
+>>> d.items()
+[(7, 'seven'), (1, 'one'), (9, 'nine')]
+
+# Overwriting an item keeps it's place.
+>>> d[1] = 'ONE'
+>>> d.values()
+['seven', 'ONE', 'nine']
+
+# New items go to the end.
+>>> d[0] = 'nil'
+>>> d.keys()
+[7, 1, 9, 0]
+
+# Deleting an item, then inserting the same key again will place it at the end.
+>>> del d[7]
+>>> d.keys()
+[1, 9, 0]
+>>> d[7] = 'lucky number 7'
+>>> d.keys()
+[1, 9, 0, 7]
+
+# Changing the keys won't do anything, it's only a copy of the keys dict.
+>>> k = d.keys()
+>>> k.remove(9)
+>>> d.keys()
+[1, 9, 0, 7]
+
+# Initialising a SortedDict with two keys will just take the first one. A real
+# dict will actually take the second value so we will too, but we'll keep the
+# ordering from the first key found.
+>>> tuples = ((2, 'two'), (1, 'one'), (2, 'second-two'))
+>>> d = SortedDict(tuples)
+>>> d.keys()
+[2, 1]
+>>> real_dict = dict(tuples)
+>>> real_dict.values()
+['one', 'second-two']
+>>> d.values()
+['second-two', 'one']
+"""

tests/regressiontests/utils/tests.py

@@ -6,7 +6,14 @@ from unittest import TestCase
 
 from django.utils import html, checksums
 
-from timesince import timesince_tests
+import timesince
+import datastructures
+
+# Extra tests
+__test__ = {
+    'timesince': timesince,
+    'datastructures': datastructures,
+}
 
 class TestUtilsHtml(TestCase):
 
@@ -142,10 +149,6 @@ class TestUtilsChecksums(TestCase):
         for value, output in items:
             self.check_output(f, value, output)
 
-__test__ = {
-    'timesince_tests': timesince_tests,
-}
-
 if __name__ == "__main__":
     import doctest
     doctest.testmod()

tests/regressiontests/utils/timesince.py

@@ -1,4 +1,4 @@
-timesince_tests = """
+"""
 >>> from datetime import datetime, timedelta
 >>> from django.utils.timesince import timesince
 

tests/regressiontests/views/tests/static.py

@@ -13,3 +13,11 @@ class StaticTests(TestCase):
             response = self.client.get('/views/site_media/%s' % filename)
             file = open(path.join(media_dir, filename))
             self.assertEquals(file.read(), response.content)
+
+    def test_copes_with_empty_path_component(self):
+        file_name = 'file.txt'
+        response = self.client.get('/views/site_media//%s' % file_name)
+        file = open(path.join(media_dir, file_name))
+        self.assertEquals(file.read(), response.content)
+        
+