diff --git a/django/template/__init__.py b/django/template/__init__.py
index 29daa76eac..e60ff64ebf 100644
--- a/django/template/__init__.py
+++ b/django/template/__init__.py
@@ -129,7 +129,10 @@ class VariableDoesNotExist(Exception):
         self.params = params
 
     def __str__(self):
-        return self.msg % self.params
+        return unicode(self).encode('utf-8')
+
+    def __unicode__(self):
+        return self.msg % tuple([force_unicode(p, errors='replace') for p in self.params])
 
 class InvalidTemplateLibrary(Exception):
     pass
diff --git a/django/template/debug.py b/django/template/debug.py
index 008059c28f..c58c854858 100644
--- a/django/template/debug.py
+++ b/django/template/debug.py
@@ -75,7 +75,7 @@ class DebugNodeList(NodeList):
             raise
         except Exception, e:
             from sys import exc_info
-            wrapped = TemplateSyntaxError('Caught an exception while rendering: %s' % e)
+            wrapped = TemplateSyntaxError(u'Caught an exception while rendering: %s' % force_unicode(e, errors='replace'))
             wrapped.source = node.source
             wrapped.exc_info = exc_info()
             raise wrapped