diff --git a/django/core/cache/utils.py b/django/core/cache/utils.py
index 45727fae46..2aead84d60 100644
--- a/django/core/cache/utils.py
+++ b/django/core/cache/utils.py
@@ -1,12 +1,12 @@
 import hashlib
-from urllib.parse import quote
 
 TEMPLATE_FRAGMENT_KEY_TEMPLATE = 'template.cache.%s.%s'
 
 
 def make_template_fragment_key(fragment_name, vary_on=None):
-    if vary_on is None:
-        vary_on = ()
-    key = ':'.join(quote(str(var)) for var in vary_on)
-    args = hashlib.md5(key.encode())
-    return TEMPLATE_FRAGMENT_KEY_TEMPLATE % (fragment_name, args.hexdigest())
+    hasher = hashlib.md5()
+    if vary_on is not None:
+        for arg in vary_on:
+            hasher.update(str(arg).encode())
+            hasher.update(b':')
+    return TEMPLATE_FRAGMENT_KEY_TEMPLATE % (fragment_name, hasher.hexdigest())
diff --git a/docs/releases/3.1.txt b/docs/releases/3.1.txt
index cafd8dbdb0..865ae60ec3 100644
--- a/docs/releases/3.1.txt
+++ b/docs/releases/3.1.txt
@@ -208,7 +208,11 @@ backends.
 Miscellaneous
 -------------
 
-* ...
+* The cache keys used by :ttag:`cache` and generated by
+  :func:`~django.core.cache.utils.make_template_fragment_key` are different
+  from the keys generated by older versions of Django. After upgrading to
+  Django 3.1, the first request to any previously cached template fragment will
+  be a cache miss.
 
 .. _deprecated-features-3.1:
 
diff --git a/tests/cache/tests.py b/tests/cache/tests.py
index 871b1498aa..ec515a297b 100644
--- a/tests/cache/tests.py
+++ b/tests/cache/tests.py
@@ -2306,15 +2306,27 @@ class TestMakeTemplateFragmentKey(SimpleTestCase):
 
     def test_with_one_vary_on(self):
         key = make_template_fragment_key('foo', ['abc'])
-        self.assertEqual(key, 'template.cache.foo.900150983cd24fb0d6963f7d28e17f72')
+        self.assertEqual(key, 'template.cache.foo.493e283d571a73056196f1a68efd0f66')
 
     def test_with_many_vary_on(self):
         key = make_template_fragment_key('bar', ['abc', 'def'])
-        self.assertEqual(key, 'template.cache.bar.4b35f12ab03cec09beec4c21b2d2fa88')
+        self.assertEqual(key, 'template.cache.bar.17c1a507a0cb58384f4c639067a93520')
 
     def test_proper_escaping(self):
         key = make_template_fragment_key('spam', ['abc:def%'])
-        self.assertEqual(key, 'template.cache.spam.f27688177baec990cdf3fbd9d9c3f469')
+        self.assertEqual(key, 'template.cache.spam.06c8ae8e8c430b69fb0a6443504153dc')
+
+    def test_with_ints_vary_on(self):
+        key = make_template_fragment_key('foo', [1, 2, 3, 4, 5])
+        self.assertEqual(key, 'template.cache.foo.7ae8fd2e0d25d651c683bdeebdb29461')
+
+    def test_with_unicode_vary_on(self):
+        key = make_template_fragment_key('foo', ['42º', '😀'])
+        self.assertEqual(key, 'template.cache.foo.7ced1c94e543668590ba39b3c08b0237')
+
+    def test_long_vary_on(self):
+        key = make_template_fragment_key('foo', ['x' * 10000])
+        self.assertEqual(key, 'template.cache.foo.3670b349b5124aa56bdb50678b02b23a')
 
 
 class CacheHandlerTest(SimpleTestCase):