diff --git a/django/utils/autoreload.py b/django/utils/autoreload.py
index 3ac8c1fa8e..e7df8c8938 100644
--- a/django/utils/autoreload.py
+++ b/django/utils/autoreload.py
@@ -91,6 +91,7 @@ def gen_filenames(only_new=False):
     # fail with RuntimeError: cannot mutate dictionary while iterating
     global _cached_modules, _cached_filenames
     module_values = set(sys.modules.values())
+    _cached_filenames = clean_files(_cached_filenames)
     if _cached_modules == module_values:
         # No changes in module list, short-circuit the function
         if only_new:
diff --git a/tests/utils_tests/test_autoreload.py b/tests/utils_tests/test_autoreload.py
index e7ef8537c0..1edf3f9bf8 100644
--- a/tests/utils_tests/test_autoreload.py
+++ b/tests/utils_tests/test_autoreload.py
@@ -1,9 +1,12 @@
+from importlib import import_module
 import os
+import tempfile
 
 from django import conf
 from django.contrib import admin
 from django.test import TestCase, override_settings
 from django.utils.autoreload import gen_filenames
+from django.utils._os import upath
 
 LOCALE_PATH = os.path.join(os.path.dirname(__file__), 'locale')
 
@@ -82,3 +85,13 @@ class TestFilenameGenerator(TestCase):
         self.assertEqual(len(filenames2), 1)
         self.assertTrue(filenames2[0].endswith('fractions.py'))
         self.assertFalse(any(f.endswith('.pyc') for f in gen_filenames()))
+
+    def test_deleted_removed(self):
+        _, filepath = tempfile.mkstemp(dir=os.path.dirname(upath(__file__)), suffix='.py')
+        try:
+            _, filename = os.path.split(filepath)
+            import_module('.%s' % filename.rstrip('.py'), package='utils_tests')
+            self.assertIn(filepath, gen_filenames())
+        finally:
+            os.remove(filepath)
+        self.assertNotIn(filepath, gen_filenames())