queryset-refactor: Merged from trunk up to [7002].

git-svn-id: http://code.djangoproject.com/svn/django/branches/queryset-refactor@7004 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/contrib/admin/media/js/urlify.js

@@ -5,7 +5,7 @@ var LATIN_MAP = {
     'O', 'Ő': 'O', 'Ø': 'O', 'Ù': 'U', 'Ú': 'U', 'Û': 'U', 'Ü': 'U', 'Ű': 'U',
     'Ý': 'Y', 'Þ': 'TH', 'ß': 'ss', 'à':'a', 'á':'a', 'â': 'a', 'ã': 'a', 'ä':
     'a', 'å': 'a', 'æ': 'ae', 'ç': 'c', 'è': 'e', 'é': 'e', 'ê': 'e', 'ë': 'e',
-    'ì': 'i', 'í': 'i', 'î': 'i', 'ï': 'i', 'ð': 'o', 'ñ': 'n', 'ò': 'o', 'ó':
+    'ì': 'i', 'í': 'i', 'î': 'i', 'ï': 'i', 'ð': 'd', 'ñ': 'n', 'ò': 'o', 'ó':
     'o', 'ô': 'o', 'õ': 'o', 'ö': 'o', 'ő': 'o', 'ø': 'o', 'ù': 'u', 'ú': 'u',
     'û': 'u', 'ü': 'u', 'ű': 'u', 'ý': 'y', 'þ': 'th', 'ÿ': 'y'
 }

django/contrib/localflavor/ca/forms.py

@@ -80,12 +80,12 @@ class CASocialInsuranceNumberField(Field):
 
     Checks the following rules to determine whether the number is valid:
 
-        * Conforms to the XXX-XXX-XXXX format.
+        * Conforms to the XXX-XXX-XXX format.
         * Passes the check digit process "Luhn Algorithm"
              See: http://en.wikipedia.org/wiki/Social_Insurance_Number
     """
     default_error_messages = {
-        'invalid': ugettext('Enter a valid Canadian Social Insurance number in XXX-XXX-XXXX format.'),
+        'invalid': ugettext('Enter a valid Canadian Social Insurance number in XXX-XXX-XXX format.'),
     }
 
     def clean(self, value):

django/contrib/sessions/backends/db.py

@@ -26,6 +26,8 @@ class SessionStore(SessionBase):
 
             # Save immediately to minimize collision
             self.save()
+            # Ensure the user is notified via a new cookie.
+            self.modified = True
             return {}
 
     def exists(self, session_key):

django/contrib/sessions/backends/file.py

@@ -46,6 +46,8 @@ class SessionStore(SessionBase):
                     self._session_key = self._get_new_session_key()
                     self._session_cache = {}
                     self.save()
+                    # Ensure the user is notified via a new cookie.
+                    self.modified = True
             finally:
                 session_file.close()
         except(IOError):

django/core/mail.py

@@ -67,42 +67,32 @@ def make_msgid(idstring=None):
 class BadHeaderError(ValueError):
     pass
 
+def forbid_multi_line_headers(name, val):
+    "Forbids multi-line headers, to prevent header injection."
+    if '\n' in val or '\r' in val:
+        raise BadHeaderError("Header values can't contain newlines (got %r for header %r)" % (val, name))
+    try:
+        val = force_unicode(val).encode('ascii')
+    except UnicodeEncodeError:
+        if name.lower() in ('to', 'from', 'cc'):
+            result = []
+            for item in val.split(', '):
+                nm, addr = parseaddr(item)
+                nm = str(Header(nm, settings.DEFAULT_CHARSET))
+                result.append(formataddr((nm, str(addr))))
+            val = ', '.join(result)
+        else:
+            val = Header(force_unicode(val), settings.DEFAULT_CHARSET)
+    return name, val
+
 class SafeMIMEText(MIMEText):
     def __setitem__(self, name, val):
-        "Forbids multi-line headers, to prevent header injection."
+        name, val = forbid_multi_line_headers(name, val)
-        if '\n' in val or '\r' in val:
-            raise BadHeaderError, "Header values can't contain newlines (got %r for header %r)" % (val, name)
-        try:
-            val = force_unicode(val).encode('ascii')
-        except UnicodeEncodeError:
-            if name.lower() in ('to', 'from', 'cc'):
-                result = []
-                for item in val.split(', '):
-                    nm, addr = parseaddr(item)
-                    nm = str(Header(nm, settings.DEFAULT_CHARSET))
-                    result.append(formataddr((nm, str(addr))))
-                val = ', '.join(result)
-            else:
-                val = Header(force_unicode(val), settings.DEFAULT_CHARSET)
         MIMEText.__setitem__(self, name, val)
 
 class SafeMIMEMultipart(MIMEMultipart):
     def __setitem__(self, name, val):
-        "Forbids multi-line headers, to prevent header injection."
+        name, val = forbid_multi_line_headers(name, val)
-        if '\n' in val or '\r' in val:
-            raise BadHeaderError, "Header values can't contain newlines (got %r for header %r)" % (val, name)
-        try:
-            val = force_unicode(val).encode('ascii')
-        except UnicodeEncodeError:
-            if name.lower() in ('to', 'from', 'cc'):
-                result = []
-                for item in val.split(', '):
-                    nm, addr = parseaddr(item)
-                    nm = str(Header(nm, settings.DEFAULT_CHARSET))
-                    result.append(formataddr((nm, str(addr))))
-                val = ', '.join(result)
-            else:
-                val = Header(force_unicode(val), settings.DEFAULT_CHARSET)
         MIMEMultipart.__setitem__(self, name, val)
 
 class SMTPConnection(object):

django/core/management/sql.py

@@ -293,7 +293,7 @@ def sql_model_create(model, style, known_models=set()):
             style.SQL_KEYWORD('NULL'))
     for field_constraints in opts.unique_together:
         table_output.append(style.SQL_KEYWORD('UNIQUE') + ' (%s)' % \
-            ", ".join([qn(style.SQL_FIELD(opts.get_field(f).column)) for f in field_constraints]))
+            ", ".join([style.SQL_FIELD(qn(opts.get_field(f).column)) for f in field_constraints]))
 
     full_statement = [style.SQL_KEYWORD('CREATE TABLE') + ' ' + style.SQL_TABLE(qn(opts.db_table)) + ' (']
     for i, line in enumerate(table_output): # Combine and add commas.

django/db/backends/oracle/base.py

@@ -413,6 +413,7 @@ class DatabaseWrapper(BaseDatabaseWrapper):
         return self.connection is not None
 
     def _cursor(self, settings):
+        cursor = None
         if not self._valid_connection():
             if len(settings.DATABASE_HOST.strip()) == 0:
                 settings.DATABASE_HOST = 'localhost'
@@ -422,16 +423,25 @@ class DatabaseWrapper(BaseDatabaseWrapper):
             else:
                 conn_string = "%s/%s@%s" % (settings.DATABASE_USER, settings.DATABASE_PASSWORD, settings.DATABASE_NAME)
                 self.connection = Database.connect(conn_string, **self.options)
+            cursor = FormatStylePlaceholderCursor(self.connection)
+            # Set oracle date to ansi date format.  This only needs to execute
+            # once when we create a new connection.
+            cursor.execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD' "
+                           "NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS.FF'")
             try:
                 self.oracle_version = int(self.connection.version.split('.')[0])
             except ValueError:
                 pass
-        cursor = FormatStylePlaceholderCursor(self.connection)
+            try:
+                self.connection.stmtcachesize = 20
+            except:
+                # Django docs specify cx_Oracle version 4.3.1 or higher, but
+                # stmtcachesize is available only in 4.3.2 and up.
+                pass
+        if not cursor:
+            cursor = FormatStylePlaceholderCursor(self.connection)
         # Default arraysize of 1 is highly sub-optimal.
         cursor.arraysize = 100
-        # Set oracle date to ansi date format.
-        cursor.execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD'")
-        cursor.execute("ALTER SESSION SET NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS.FF'")
         return cursor
 
 class FormatStylePlaceholderCursor(Database.Cursor):
@@ -506,7 +516,10 @@ class FormatStylePlaceholderCursor(Database.Cursor):
         return Database.Cursor.executemany(self, query, new_param_list)
 
     def fetchone(self):
-        return to_unicode(Database.Cursor.fetchone(self))
+        row = Database.Cursor.fetchone(self)
+        if row is None:
+            return row
+        return tuple([to_unicode(e) for e in row])
 
     def fetchmany(self, size=None):
         if size is None:

django/template/__init__.py

@@ -154,15 +154,12 @@ class StringOrigin(Origin):
 
 class Template(object):
     def __init__(self, template_string, origin=None, name='<Unknown Template>'):
-        "Compilation stage"
         try:
             template_string = smart_unicode(template_string)
         except UnicodeDecodeError:
             raise TemplateEncodingError("Templates can only be constructed from unicode or UTF-8 strings.")
-        if settings.TEMPLATE_DEBUG and origin == None:
+        if settings.TEMPLATE_DEBUG and origin is None:
             origin = StringOrigin(template_string)
-            # Could do some crazy stack-frame stuff to record where this string
-            # came from...
         self.nodelist = compile_string(template_string, origin)
         self.name = name
 
@@ -177,13 +174,18 @@ class Template(object):
 
 def compile_string(template_string, origin):
     "Compiles template_string into NodeList ready for rendering"
-    lexer = lexer_factory(template_string, origin)
+    if settings.TEMPLATE_DEBUG:
-    parser = parser_factory(lexer.tokenize())
+        from debug import DebugLexer, DebugParser
+        lexer_class, parser_class = DebugLexer, DebugParser
+    else:
+        lexer_class, parser_class = Lexer, Parser
+    lexer = lexer_class(template_string, origin)
+    parser = parser_class(lexer.tokenize())
     return parser.parse()
 
 class Token(object):
     def __init__(self, token_type, contents):
-        "The token_type must be TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK or TOKEN_COMMENT"
+        # token_type must be TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK or TOKEN_COMMENT.
         self.token_type, self.contents = token_type, contents
 
     def __str__(self):
@@ -200,7 +202,7 @@ class Lexer(object):
         self.origin = origin
 
     def tokenize(self):
-        "Return a list of tokens from a given template_string"
+        "Return a list of tokens from a given template_string."
         in_tag = False
         result = []
         for bit in tag_re.split(self.template_string):
@@ -226,30 +228,6 @@ class Lexer(object):
             token = Token(TOKEN_TEXT, token_string)
         return token
 
-class DebugLexer(Lexer):
-    def __init__(self, template_string, origin):
-        super(DebugLexer, self).__init__(template_string, origin)
-
-    def tokenize(self):
-        "Return a list of tokens from a given template_string"
-        result, upto = [], 0
-        for match in tag_re.finditer(self.template_string):
-            start, end = match.span()
-            if start > upto:
-                result.append(self.create_token(self.template_string[upto:start], (upto, start), False))
-                upto = start
-            result.append(self.create_token(self.template_string[start:end], (start, end), True))
-            upto = end
-        last_bit = self.template_string[upto:]
-        if last_bit:
-            result.append(self.create_token(last_bit, (upto, upto + len(last_bit)), False))
-        return result
-
-    def create_token(self, token_string, source, in_tag):
-        token = super(DebugLexer, self).create_token(token_string, in_tag)
-        token.source = self.origin, source
-        return token
-
 class Parser(object):
     def __init__(self, tokens):
         self.tokens = tokens
@@ -319,17 +297,17 @@ class Parser(object):
     def exit_command(self):
         pass
 
-    def error(self, token, msg ):
+    def error(self, token, msg):
         return TemplateSyntaxError(msg)
 
     def empty_variable(self, token):
-        raise self.error( token, "Empty variable tag")
+        raise self.error(token, "Empty variable tag")
 
     def empty_block_tag(self, token):
-        raise self.error( token, "Empty block tag")
+        raise self.error(token, "Empty block tag")
 
     def invalid_block_tag(self, token, command):
-        raise self.error( token, "Invalid block tag: '%s'" % command)
+        raise self.error(token, "Invalid block tag: '%s'" % command)
 
     def unclosed_block_tag(self, parse_until):
         raise self.error(None, "Unclosed tags: %s " %  ', '.join(parse_until))
@@ -358,57 +336,7 @@ class Parser(object):
         if filter_name in self.filters:
             return self.filters[filter_name]
         else:
-            raise TemplateSyntaxError, "Invalid filter: '%s'" % filter_name
+            raise TemplateSyntaxError("Invalid filter: '%s'" % filter_name)
-
-class DebugParser(Parser):
-    def __init__(self, lexer):
-        super(DebugParser, self).__init__(lexer)
-        self.command_stack = []
-
-    def enter_command(self, command, token):
-        self.command_stack.append( (command, token.source) )
-
-    def exit_command(self):
-        self.command_stack.pop()
-
-    def error(self, token, msg):
-        return self.source_error(token.source, msg)
-
-    def source_error(self, source,msg):
-        e = TemplateSyntaxError(msg)
-        e.source = source
-        return e
-
-    def create_nodelist(self):
-        return DebugNodeList()
-
-    def create_variable_node(self, contents):
-        return DebugVariableNode(contents)
-
-    def extend_nodelist(self, nodelist, node, token):
-        node.source = token.source
-        super(DebugParser, self).extend_nodelist(nodelist, node, token)
-
-    def unclosed_block_tag(self, parse_until):
-        command, source = self.command_stack.pop()
-        msg = "Unclosed tag '%s'. Looking for one of: %s " % (command, ', '.join(parse_until))
-        raise self.source_error( source, msg)
-
-    def compile_function_error(self, token, e):
-        if not hasattr(e, 'source'):
-            e.source = token.source
-
-def lexer_factory(*args, **kwargs):
-    if settings.TEMPLATE_DEBUG:
-        return DebugLexer(*args, **kwargs)
-    else:
-        return Lexer(*args, **kwargs)
-
-def parser_factory(*args, **kwargs):
-    if settings.TEMPLATE_DEBUG:
-        return DebugParser(*args, **kwargs)
-    else:
-        return Parser(*args, **kwargs)
 
 class TokenParser(object):
     """
@@ -426,7 +354,7 @@ class TokenParser(object):
 
     def top(self):
         "Overload this method to do the actual parsing and return the result."
-        raise NotImplemented
+        raise NotImplementedError()
 
     def more(self):
         "Returns True if there is more stuff in the tag."
@@ -435,7 +363,7 @@ class TokenParser(object):
     def back(self):
         "Undoes the last microparser. Use this for lookahead and backtracking."
         if not len(self.backout):
-            raise TemplateSyntaxError, "back called without some previous parsing"
+            raise TemplateSyntaxError("back called without some previous parsing")
         self.pointer = self.backout.pop()
 
     def tag(self):
@@ -443,7 +371,7 @@ class TokenParser(object):
         subject = self.subject
         i = self.pointer
         if i >= len(subject):
-            raise TemplateSyntaxError, "expected another tag, found end of string: %s" % subject
+            raise TemplateSyntaxError("expected another tag, found end of string: %s" % subject)
         p = i
         while i < len(subject) and subject[i] not in (' ', '\t'):
             i += 1
@@ -459,14 +387,14 @@ class TokenParser(object):
         subject = self.subject
         i = self.pointer
         if i >= len(subject):
-            raise TemplateSyntaxError, "Searching for value. Expected another value but found end of string: %s" % subject
+            raise TemplateSyntaxError("Searching for value. Expected another value but found end of string: %s" % subject)
         if subject[i] in ('"', "'"):
             p = i
             i += 1
             while i < len(subject) and subject[i] != subject[p]:
                 i += 1
             if i >= len(subject):
-                raise TemplateSyntaxError, "Searching for value. Unexpected end of string in column %d: %s" % (i, subject)
+                raise TemplateSyntaxError("Searching for value. Unexpected end of string in column %d: %s" % (i, subject))
             i += 1
             res = subject[p:i]
             while i < len(subject) and subject[i] in (' ', '\t'):
@@ -483,7 +411,7 @@ class TokenParser(object):
                     while i < len(subject) and subject[i] != c:
                         i += 1
                     if i >= len(subject):
-                        raise TemplateSyntaxError, "Searching for value. Unexpected end of string in column %d: %s" % (i, subject)
+                        raise TemplateSyntaxError("Searching for value. Unexpected end of string in column %d: %s" % (i, subject))
                 i += 1
             s = subject[p:i]
             while i < len(subject) and subject[i] in (' ', '\t'):
@@ -542,8 +470,8 @@ class FilterExpression(object):
         for match in matches:
             start = match.start()
             if upto != start:
-                raise TemplateSyntaxError, "Could not parse some characters: %s|%s|%s"  % \
+                raise TemplateSyntaxError("Could not parse some characters: %s|%s|%s"  % \
-                                           (token[:upto], token[upto:start], token[start:])
+                                           (token[:upto], token[upto:start], token[start:]))
             if var == None:
                 var, constant, i18n_constant = match.group("var", "constant", "i18n_constant")
                 if i18n_constant:
@@ -552,9 +480,9 @@ class FilterExpression(object):
                     var = '"%s"' % constant.replace(r'\"', '"')
                 upto = match.end()
                 if var == None:
-                    raise TemplateSyntaxError, "Could not find variable at start of %s" % token
+                    raise TemplateSyntaxError("Could not find variable at start of %s" % token)
                 elif var.find(VARIABLE_ATTRIBUTE_SEPARATOR + '_') > -1 or var[0] == '_':
-                    raise TemplateSyntaxError, "Variables and attributes may not begin with underscores: '%s'" % var
+                    raise TemplateSyntaxError("Variables and attributes may not begin with underscores: '%s'" % var)
             else:
                 filter_name = match.group("filter_name")
                 args = []
@@ -570,7 +498,7 @@ class FilterExpression(object):
                 filters.append( (filter_func,args))
                 upto = match.end()
         if upto != len(token):
-            raise TemplateSyntaxError, "Could not parse the remainder: '%s' from '%s'" % (token[upto:], token)
+            raise TemplateSyntaxError("Could not parse the remainder: '%s' from '%s'" % (token[upto:], token))
         self.filters = filters
         self.var = Variable(var)
 
@@ -627,7 +555,7 @@ class FilterExpression(object):
                 provided.pop(0)
         except IndexError:
             # Not enough
-            raise TemplateSyntaxError, "%s requires %d arguments, %d provided" % (name, len(nondefs), plen)
+            raise TemplateSyntaxError("%s requires %d arguments, %d provided" % (name, len(nondefs), plen))
 
         # Defaults can be overridden.
         defaults = defaults and list(defaults) or []
@@ -636,7 +564,7 @@ class FilterExpression(object):
                 defaults.pop(0)
         except IndexError:
             # Too many.
-            raise TemplateSyntaxError, "%s requires %d arguments, %d provided" % (name, len(nondefs), plen)
+            raise TemplateSyntaxError("%s requires %d arguments, %d provided" % (name, len(nondefs), plen))
 
         return True
     args_check = staticmethod(args_check)
@@ -816,22 +744,6 @@ class NodeList(list):
     def render_node(self, node, context):
         return node.render(context)
 
-class DebugNodeList(NodeList):
-    def render_node(self, node, context):
-        try:
-            result = node.render(context)
-        except TemplateSyntaxError, e:
-            if not hasattr(e, 'source'):
-                e.source = node.source
-            raise
-        except Exception, e:
-            from sys import exc_info
-            wrapped = TemplateSyntaxError('Caught an exception while rendering: %s' % e)
-            wrapped.source = node.source
-            wrapped.exc_info = exc_info()
-            raise wrapped
-        return result
-
 class TextNode(Node):
     def __init__(self, s):
         self.s = s
@@ -861,21 +773,6 @@ class VariableNode(Node):
         else:
             return force_unicode(output)
 
-class DebugVariableNode(VariableNode):
-    def render(self, context):
-        try:
-            output = force_unicode(self.filter_expression.resolve(context))
-        except TemplateSyntaxError, e:
-            if not hasattr(e, 'source'):
-                e.source = self.source
-            raise
-        except UnicodeDecodeError:
-            return ''
-        if (context.autoescape and not isinstance(output, SafeData)) or isinstance(output, EscapeData):
-            return escape(output)
-        else:
-            return output
-
 def generic_tag_compiler(params, defaults, name, node_class, parser, token):
     "Returns a template.Node subclass."
     bits = token.split_contents()[1:]
@@ -887,7 +784,7 @@ def generic_tag_compiler(params, defaults, name, node_class, parser, token):
             message = "%s takes %s arguments" % (name, bmin)
         else:
             message = "%s takes between %s and %s arguments" % (name, bmin, bmax)
-        raise TemplateSyntaxError, message
+        raise TemplateSyntaxError(message)
     return node_class(bits)
 
 class Library(object):
@@ -913,7 +810,7 @@ class Library(object):
             self.tags[name] = compile_function
             return compile_function
         else:
-            raise InvalidTemplateLibrary, "Unsupported arguments to Library.tag: (%r, %r)", (name, compile_function)
+            raise InvalidTemplateLibrary("Unsupported arguments to Library.tag: (%r, %r)", (name, compile_function))
 
     def tag_function(self,func):
         self.tags[getattr(func, "_decorated_function", func).__name__] = func
@@ -937,7 +834,7 @@ class Library(object):
             self.filters[name] = filter_func
             return filter_func
         else:
-            raise InvalidTemplateLibrary, "Unsupported arguments to Library.filter: (%r, %r)", (name, filter_func)
+            raise InvalidTemplateLibrary("Unsupported arguments to Library.filter: (%r, %r)", (name, filter_func))
 
     def filter_function(self, func):
         self.filters[getattr(func, "_decorated_function", func).__name__] = func
@@ -966,7 +863,7 @@ class Library(object):
                 if params[0] == 'context':
                     params = params[1:]
                 else:
-                    raise TemplateSyntaxError, "Any tag function decorated with takes_context=True must have a first argument of 'context'"
+                    raise TemplateSyntaxError("Any tag function decorated with takes_context=True must have a first argument of 'context'")
 
             class InclusionNode(Node):
                 def __init__(self, vars_to_resolve):
@@ -1003,12 +900,12 @@ def get_library(module_name):
         try:
             mod = __import__(module_name, {}, {}, [''])
         except ImportError, e:
-            raise InvalidTemplateLibrary, "Could not load template library from %s, %s" % (module_name, e)
+            raise InvalidTemplateLibrary("Could not load template library from %s, %s" % (module_name, e))
         try:
             lib = mod.register
             libraries[module_name] = lib
         except AttributeError:
-            raise InvalidTemplateLibrary, "Template library %s does not have a variable named 'register'" % module_name
+            raise InvalidTemplateLibrary("Template library %s does not have a variable named 'register'" % module_name)
     return lib
 
 def add_to_builtins(module_name):

django/template/context.py

@@ -9,7 +9,6 @@ class ContextPopException(Exception):
 
 class Context(object):
     "A stack container for variable context"
-
     def __init__(self, dict_=None, autoescape=True):
         dict_ = dict_ or {}
         self.dicts = [dict_]
@@ -78,11 +77,11 @@ def get_standard_processors():
             try:
                 mod = __import__(module, {}, {}, [attr])
             except ImportError, e:
-                raise ImproperlyConfigured, 'Error importing request processor module %s: "%s"' % (module, e)
+                raise ImproperlyConfigured('Error importing request processor module %s: "%s"' % (module, e))
             try:
                 func = getattr(mod, attr)
             except AttributeError:
-                raise ImproperlyConfigured, 'Module "%s" does not define a "%s" callable request processor' % (module, attr)
+                raise ImproperlyConfigured('Module "%s" does not define a "%s" callable request processor' % (module, attr))
             processors.append(func)
         _standard_context_processors = tuple(processors)
     return _standard_context_processors
@@ -102,4 +101,3 @@ class RequestContext(Context):
             processors = tuple(processors)
         for processor in get_standard_processors() + processors:
             self.update(processor(request))
-

django/template/debug.py

@@ -0,0 +1,97 @@
+from django.template import Lexer, Parser, tag_re, NodeList, VariableNode, TemplateSyntaxError
+from django.utils.encoding import force_unicode
+from django.utils.html import escape
+from django.utils.safestring import SafeData, EscapeData
+
+class DebugLexer(Lexer):
+    def __init__(self, template_string, origin):
+        super(DebugLexer, self).__init__(template_string, origin)
+
+    def tokenize(self):
+        "Return a list of tokens from a given template_string"
+        result, upto = [], 0
+        for match in tag_re.finditer(self.template_string):
+            start, end = match.span()
+            if start > upto:
+                result.append(self.create_token(self.template_string[upto:start], (upto, start), False))
+                upto = start
+            result.append(self.create_token(self.template_string[start:end], (start, end), True))
+            upto = end
+        last_bit = self.template_string[upto:]
+        if last_bit:
+            result.append(self.create_token(last_bit, (upto, upto + len(last_bit)), False))
+        return result
+
+    def create_token(self, token_string, source, in_tag):
+        token = super(DebugLexer, self).create_token(token_string, in_tag)
+        token.source = self.origin, source
+        return token
+
+class DebugParser(Parser):
+    def __init__(self, lexer):
+        super(DebugParser, self).__init__(lexer)
+        self.command_stack = []
+
+    def enter_command(self, command, token):
+        self.command_stack.append( (command, token.source) )
+
+    def exit_command(self):
+        self.command_stack.pop()
+
+    def error(self, token, msg):
+        return self.source_error(token.source, msg)
+
+    def source_error(self, source,msg):
+        e = TemplateSyntaxError(msg)
+        e.source = source
+        return e
+
+    def create_nodelist(self):
+        return DebugNodeList()
+
+    def create_variable_node(self, contents):
+        return DebugVariableNode(contents)
+
+    def extend_nodelist(self, nodelist, node, token):
+        node.source = token.source
+        super(DebugParser, self).extend_nodelist(nodelist, node, token)
+
+    def unclosed_block_tag(self, parse_until):
+        command, source = self.command_stack.pop()
+        msg = "Unclosed tag '%s'. Looking for one of: %s " % (command, ', '.join(parse_until))
+        raise self.source_error(source, msg)
+
+    def compile_function_error(self, token, e):
+        if not hasattr(e, 'source'):
+            e.source = token.source
+
+class DebugNodeList(NodeList):
+    def render_node(self, node, context):
+        try:
+            result = node.render(context)
+        except TemplateSyntaxError, e:
+            if not hasattr(e, 'source'):
+                e.source = node.source
+            raise
+        except Exception, e:
+            from sys import exc_info
+            wrapped = TemplateSyntaxError('Caught an exception while rendering: %s' % e)
+            wrapped.source = node.source
+            wrapped.exc_info = exc_info()
+            raise wrapped
+        return result
+
+class DebugVariableNode(VariableNode):
+    def render(self, context):
+        try:
+            output = force_unicode(self.filter_expression.resolve(context))
+        except TemplateSyntaxError, e:
+            if not hasattr(e, 'source'):
+                e.source = self.source
+            raise
+        except UnicodeDecodeError:
+            return ''
+        if (context.autoescape and not isinstance(output, SafeData)) or isinstance(output, EscapeData):
+            return escape(output)
+        else:
+            return output

django/template/defaultfilters.py

@@ -433,7 +433,7 @@ def first(value):
         return value[0]
     except IndexError:
         return u''
-first.is_safe = True
+first.is_safe = False
 
 def join(value, arg):
     """Joins a list with a string, like Python's ``str.join(list)``."""
@@ -449,6 +449,14 @@ def join(value, arg):
         return data
 join.is_safe = True
 
+def last(value):
+    "Returns the last item in a list"
+    try:
+        return value[-1]
+    except IndexError:
+        return u''
+last.is_safe = True
+
 def length(value):
     """Returns the length of the value - useful for lists."""
     return len(value)
@@ -800,6 +808,7 @@ register.filter(force_escape)
 register.filter(get_digit)
 register.filter(iriencode)
 register.filter(join)
+register.filter(last)
 register.filter(length)
 register.filter(length_is)
 register.filter(linebreaks)

django/template/defaulttags.py

@@ -84,19 +84,16 @@ class FirstOfNode(Node):
         return u''
 
 class ForNode(Node):
-    def __init__(self, loopvars, sequence, reversed, nodelist_loop):
+    def __init__(self, loopvars, sequence, is_reversed, nodelist_loop):
         self.loopvars, self.sequence = loopvars, sequence
-        self.reversed = reversed
+        self.is_reversed = is_reversed
         self.nodelist_loop = nodelist_loop
 
     def __repr__(self):
-        if self.reversed:
+        reversed_text = self.is_reversed and ' reversed' or ''
-            reversed = ' reversed'
-        else:
-            reversed = ''
         return "<For Node: for %s in %s, tail_len: %d%s>" % \
             (', '.join(self.loopvars), self.sequence, len(self.nodelist_loop),
-             reversed)
+             reversed_text)
 
     def __iter__(self):
         for node in self.nodelist_loop:
@@ -125,22 +122,23 @@ class ForNode(Node):
         if not hasattr(values, '__len__'):
             values = list(values)
         len_values = len(values)
-        if self.reversed:
+        if self.is_reversed:
             values = reversed(values)
         unpack = len(self.loopvars) > 1
+        # Create a forloop value in the context.  We'll update counters on each
+        # iteration just below.
+        loop_dict = context['forloop'] = {'parentloop': parentloop}
         for i, item in enumerate(values):
-            context['forloop'] = {
+            # Shortcuts for current loop iteration number.
-                # Shortcuts for current loop iteration number.
+            loop_dict['counter0'] = i
-                'counter0': i,
+            loop_dict['counter'] = i+1
-                'counter': i+1,
+            # Reverse counter iteration numbers.
-                # Reverse counter iteration numbers.
+            loop_dict['revcounter'] = len_values - i
-                'revcounter': len_values - i,
+            loop_dict['revcounter0'] = len_values - i - 1
-                'revcounter0': len_values - i - 1,
+            # Boolean values designating first and last times through loop.
-                # Boolean values designating first and last times through loop.
+            loop_dict['first'] = (i == 0)
-                'first': (i == 0),
+            loop_dict['last'] = (i == len_values - 1)
-                'last': (i == len_values - 1),
+
-                'parentloop': parentloop,
-            }
             if unpack:
                 # If there are multiple loop variables, unpack the item into
                 # them.
@@ -619,8 +617,8 @@ def do_for(parser, token):
         raise TemplateSyntaxError("'for' statements should have at least four"
                                   " words: %s" % token.contents)
 
-    reversed = bits[-1] == 'reversed'
+    is_reversed = bits[-1] == 'reversed'
-    in_index = reversed and -3 or -2
+    in_index = is_reversed and -3 or -2
     if bits[in_index] != 'in':
         raise TemplateSyntaxError("'for' statements should use the format"
                                   " 'for x in y': %s" % token.contents)
@@ -634,7 +632,7 @@ def do_for(parser, token):
     sequence = parser.compile_filter(bits[in_index+1])
     nodelist_loop = parser.parse(('endfor',))
     parser.delete_first_token()
-    return ForNode(loopvars, sequence, reversed, nodelist_loop)
+    return ForNode(loopvars, sequence, is_reversed, nodelist_loop)
 do_for = register.tag("for", do_for)
 
 def do_ifequal(parser, token, negate):

django/views/debug.py

@@ -507,7 +507,7 @@ Exception Type: {{ exception_type|escape }} at {{ request.path|escape }}
 Exception Value: {{ exception_value|escape }}
 </textarea>
   <br><br>
-  <input type="submit" value="Share this traceback on public Web site">
+  <input type="submit" value="Share this traceback on a public Web site">
   </div>
 </form>
 </div>

docs/django-admin.txt

@@ -716,8 +716,8 @@ in Python package syntax, e.g. ``mysite.settings``. If this isn't provided,
 ``django-admin.py`` will use the ``DJANGO_SETTINGS_MODULE`` environment
 variable.
 
-Note that this option is unnecessary in ``manage.py``, because it takes care of
+Note that this option is unnecessary in ``manage.py``, because it uses
-setting ``DJANGO_SETTINGS_MODULE`` for you.
+``settings.py`` from the current project by default. 
 
 Extra niceties
 ==============

docs/localflavor.txt

@@ -181,7 +181,7 @@ CASocialInsuranceNumberField
 ----------------------------
 
 A form field that validates input as a Canadian Social Insurance Number (SIN).
-A valid number must have the format XXX-XXX-XXXX and pass a `Luhn mod-10
+A valid number must have the format XXX-XXX-XXX and pass a `Luhn mod-10
 checksum`_.
 
 .. _Luhn mod-10 checksum: http://en.wikipedia.org/wiki/Luhn_algorithm

docs/middleware.txt

@@ -154,6 +154,17 @@ every incoming ``HttpRequest`` object. See `Authentication in Web requests`_.
 
 .. _Authentication in Web requests: ../authentication/#authentication-in-web-requests
 
+django.contrib.csrf.middleware.CsrfMiddleware
+---------------------------------------------
+
+**New in Django development version**
+
+Adds protection against Cross Site Request Forgeries by adding hidden form
+fields to POST forms and checking requests for the correct value. See the
+`Cross Site Request Forgery protection documentation`_.
+
+.. _`Cross Site Request Forgery protection documentation`: ../csrf/
+
 django.middleware.transaction.TransactionMiddleware
 ---------------------------------------------------
 

docs/sites.txt

@@ -97,7 +97,7 @@ Hooking into the current site from views
 ----------------------------------------
 
 On a lower level, you can use the sites framework in your Django views to do
-particular things based on what site in which the view is being called.
+particular things based on the site in which the view is being called.
 For example::
 
     from django.conf import settings
@@ -330,13 +330,13 @@ Here's how Django uses the sites framework:
       retrieving flatpages to display.
 
     * In the `syndication framework`_, the templates for ``title`` and
-      ``description`` automatically have access to a variable ``{{{ site }}}``,
+      ``description`` automatically have access to a variable ``{{ site }}``,
       which is the ``Site`` object representing the current site. Also, the
       hook for providing item URLs will use the ``domain`` from the current
       ``Site`` object if you don't specify a fully-qualified domain.
 
     * In the `authentication framework`_, the ``django.contrib.auth.views.login``
-      view passes the current ``Site`` name to the template as ``{{{ site_name }}}``.
+      view passes the current ``Site`` name to the template as ``{{ site_name }}``.
 
     * The shortcut view (``django.views.defaults.shortcut``) uses the domain of
       the current ``Site`` object when calculating an object's URL.

docs/syndication_feeds.txt

@@ -201,7 +201,7 @@ the feed.
 
 An example makes this clear. Here's the code for these beat-specific feeds::
 
-    from django.contrib.syndication import FeedDoesNotExist
+    from django.contrib.syndication.feeds import FeedDoesNotExist
 
     class BeatFeed(Feed):
         def get_object(self, bits):

docs/templates.txt

@@ -1230,7 +1230,7 @@ addslashes
 Adds slashes before quotes. Useful for escaping strings in CSV, for example.
 
 **New in Django development version**: for escaping data in JavaScript strings,
-use the `escapejs` filter instead.
+use the `escapejs`_ filter instead.
 
 capfirst
 ~~~~~~~~
@@ -1403,6 +1403,11 @@ join
 
 Joins a list with a string, like Python's ``str.join(list)``.
 
+last
+~~~~
+
+Returns the last item in a list.
+
 length
 ~~~~~~
 

tests/regressiontests/backends/models.py

@@ -8,6 +8,13 @@ class Square(models.Model):
     def __unicode__(self):
         return "%s ** 2 == %s" % (self.root, self.square)
 
+class Person(models.Model):
+    first_name = models.CharField(max_length=20)
+    last_name = models.CharField(max_length=20)
+
+    def __unicode__(self):
+        return u'%s %s' % (self.first_name, self.last_name)
+
 if connection.features.uses_case_insensitive_names:
     t_convert = lambda x: x.upper()
 else:
@@ -32,4 +39,25 @@ __test__ = {'API_TESTS': """
 >>> Square.objects.count()
 11
 
+#6254: fetchone, fetchmany, fetchall return strings as unicode objects
+>>> Person(first_name="John", last_name="Doe").save()
+>>> Person(first_name="Jane", last_name="Doe").save()
+>>> Person(first_name="Mary", last_name="Agnelline").save()
+>>> Person(first_name="Peter", last_name="Parker").save()
+>>> Person(first_name="Clark", last_name="Kent").save()
+>>> opts2 = Person._meta
+>>> f3, f4 = opts2.get_field('first_name'), opts2.get_field('last_name')
+>>> query2 = ('SELECT %s, %s FROM %s ORDER BY %s'
+...          % (qn(f3.column), qn(f4.column), t_convert(opts2.db_table),
+...             qn(f3.column)))
+>>> cursor.execute(query2) and None or None
+>>> cursor.fetchone()
+(u'Clark', u'Kent')
+
+>>> list(cursor.fetchmany(2))
+[(u'Jane', u'Doe'), (u'John', u'Doe')]
+
+>>> list(cursor.fetchall())
+[(u'Mary', u'Agnelline'), (u'Peter', u'Parker')]
+
 """}

tests/regressiontests/forms/localflavor/ca.py

@@ -213,13 +213,13 @@ u'046-454-286'
 >>> f.clean('046-454-287')
 Traceback (most recent call last):
 ...
-ValidationError: [u'Enter a valid Canadian Social Insurance number in XXX-XXX-XXXX format.']
+ValidationError: [u'Enter a valid Canadian Social Insurance number in XXX-XXX-XXX format.']
 >>> f.clean('046 454 286')
 Traceback (most recent call last):
 ...
-ValidationError: [u'Enter a valid Canadian Social Insurance number in XXX-XXX-XXXX format.']
+ValidationError: [u'Enter a valid Canadian Social Insurance number in XXX-XXX-XXX format.']
 >>> f.clean('046-44-286')
 Traceback (most recent call last):
 ...
-ValidationError: [u'Enter a valid Canadian Social Insurance number in XXX-XXX-XXXX format.']
+ValidationError: [u'Enter a valid Canadian Social Insurance number in XXX-XXX-XXX format.']
 """

tests/regressiontests/middleware/models.py

@@ -0,0 +1 @@
+# models.py file for tests to run.

tests/regressiontests/middleware/tests.py

@@ -1,7 +1,5 @@
 # -*- coding: utf-8 -*-
 
-import unittest
-
 from django.test import TestCase
 from django.http import HttpRequest
 from django.middleware.common import CommonMiddleware
@@ -19,7 +17,7 @@ class CommonMiddlewareTest(TestCase):
 
     def test_append_slash_have_slash(self):
         """
-        tests that urls with slashes go unmolested
+        Tests that URLs with slashes go unmolested.
         """
         settings.APPEND_SLASH = True
         request = self._get_request('slash/')
@@ -27,7 +25,7 @@ class CommonMiddlewareTest(TestCase):
 
     def test_append_slash_slashless_resource(self):
         """
-        tests that matches to explicit slashless urls go unmolested
+        Tests that matches to explicit slashless URLs go unmolested.
         """
         settings.APPEND_SLASH = True
         request = self._get_request('noslash')
@@ -35,7 +33,7 @@ class CommonMiddlewareTest(TestCase):
 
     def test_append_slash_slashless_unknown(self):
         """
-        tests that APPEND_SLASH doesn't redirect to unknown resources
+        Tests that APPEND_SLASH doesn't redirect to unknown resources.
         """
         settings.APPEND_SLASH = True
         request = self._get_request('unknown')
@@ -43,7 +41,7 @@ class CommonMiddlewareTest(TestCase):
 
     def test_append_slash_redirect(self):
         """
-        tests that APPEND_SLASH redirects slashless urls to a valid pattern
+        Tests that APPEND_SLASH redirects slashless URLs to a valid pattern.
         """
         settings.APPEND_SLASH = True
         request = self._get_request('slash')
@@ -53,9 +51,9 @@ class CommonMiddlewareTest(TestCase):
 
     def test_append_slash_no_redirect_on_POST_in_DEBUG(self):
         """
-        tests that while in debug mode, an exception is raised with a warning
+        Tests that while in debug mode, an exception is raised with a warning
-        when a failed attempt is made to POST to an url which would normally be
+        when a failed attempt is made to POST to an URL which would normally be
-        redirected to a slashed version
+        redirected to a slashed version.
         """
         settings.APPEND_SLASH = True
         settings.DEBUG = True
@@ -68,11 +66,12 @@ class CommonMiddlewareTest(TestCase):
         try:
             CommonMiddleware().process_request(request)
         except RuntimeError, e:
-            self.assertTrue('end in a slash' in str(e))
+            self.failUnless('end in a slash' in str(e))
+        settings.DEBUG = False
 
     def test_append_slash_disabled(self):
         """
-        tests disabling append slash functionality
+        Tests disabling append slash functionality.
         """
         settings.APPEND_SLASH = False
         request = self._get_request('slash')
@@ -80,8 +79,8 @@ class CommonMiddlewareTest(TestCase):
 
     def test_append_slash_quoted(self):
         """
-        tests that urls which require quoting are redirected to their slash
+        Tests that URLs which require quoting are redirected to their slash
-        version ok
+        version ok.
         """
         settings.APPEND_SLASH = True
         request = self._get_request('needsquoting#')
@@ -90,4 +89,3 @@ class CommonMiddlewareTest(TestCase):
         self.assertEquals(
             r['Location'],
             'http://testserver/middleware/needsquoting%23/')
-

tests/regressiontests/templates/filters.py

@@ -179,6 +179,9 @@ def get_filter_tests():
         'filter-first01': ('{{ a|first }} {{ b|first }}', {"a": ["a&b", "x"], "b": [mark_safe("a&b"), "x"]}, "a&b a&b"),
         'filter-first02': ('{% autoescape off %}{{ a|first }} {{ b|first }}{% endautoescape %}', {"a": ["a&b", "x"], "b": [mark_safe("a&b"), "x"]}, "a&b a&b"),
 
+        'filter-last01': ('{{ a|last }} {{ b|last }}', {"a": ["x", "a&b"], "b": ["x", mark_safe("a&b")]}, "a&b a&b"),
+        'filter-last02': ('{% autoescape off %}{{ a|last }} {{ b|last }}{% endautoescape %}', {"a": ["x", "a&b"], "b": ["x", mark_safe("a&b")]}, "a&b a&b"),
+
         'filter-random01': ('{{ a|random }} {{ b|random }}', {"a": ["a&b", "a&b"], "b": [mark_safe("a&b"), mark_safe("a&b")]}, "a&b a&b"),
         'filter-random02': ('{% autoescape off %}{{ a|random }} {{ b|random }}{% endautoescape %}', {"a": ["a&b", "a&b"], "b": [mark_safe("a&b"), mark_safe("a&b")]}, "a&b a&b"),
 

tests/regressiontests/templates/tests.py

@@ -441,6 +441,8 @@ class Templates(unittest.TestCase):
             'for-tag-vars02': ("{% for val in values %}{{ forloop.counter0 }}{% endfor %}", {"values": [6, 6, 6]}, "012"),
             'for-tag-vars03': ("{% for val in values %}{{ forloop.revcounter }}{% endfor %}", {"values": [6, 6, 6]}, "321"),
             'for-tag-vars04': ("{% for val in values %}{{ forloop.revcounter0 }}{% endfor %}", {"values": [6, 6, 6]}, "210"),
+            'for-tag-vars05': ("{% for val in values %}{% if forloop.first %}f{% else %}x{% endif %}{% endfor %}", {"values": [6, 6, 6]}, "fxx"),
+            'for-tag-vars06': ("{% for val in values %}{% if forloop.last %}l{% else %}x{% endif %}{% endfor %}", {"values": [6, 6, 6]}, "xxl"),
             'for-tag-unpack01': ("{% for key,value in items %}{{ key }}:{{ value }}/{% endfor %}", {"items": (('one', 1), ('two', 2))}, "one:1/two:2/"),
             'for-tag-unpack03': ("{% for key, value in items %}{{ key }}:{{ value }}/{% endfor %}", {"items": (('one', 1), ('two', 2))}, "one:1/two:2/"),
             'for-tag-unpack04': ("{% for key , value in items %}{{ key }}:{{ value }}/{% endfor %}", {"items": (('one', 1), ('two', 2))}, "one:1/two:2/"),