diff --git a/django/db/models/query.py b/django/db/models/query.py
index 6a6a82968f..57288f42dd 100644
--- a/django/db/models/query.py
+++ b/django/db/models/query.py
@@ -231,9 +231,6 @@ class QuerySet(object):
             fields = self.model._meta.fields
             pk_idx = self.model._meta.pk_index()
 
-        index_start = len(extra_select)
-        aggregate_start = index_start + len(self.model._meta.fields)
-
         load_fields = []
         # If only/defer clauses have been specified,
         # build the list of fields that are to be loaded.
@@ -253,6 +250,9 @@ class QuerySet(object):
                     # Therefore, we need to load all fields from this model
                     load_fields.append(field.name)
 
+        index_start = len(extra_select)
+        aggregate_start = index_start + len(load_fields or self.model._meta.fields)
+
         skip = None
         if load_fields and not fill_cache:
             # Some fields have been deferred, so we have to initialise
diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py
index 841ec12f2d..b5f170eb49 100644
--- a/django/db/models/sql/compiler.py
+++ b/django/db/models/sql/compiler.py
@@ -169,7 +169,7 @@ class SQLCompiler(object):
                 if isinstance(col, (list, tuple)):
                     alias, column = col
                     table = self.query.alias_map[alias][TABLE_NAME]
-                    if table in only_load and col not in only_load[table]:
+                    if table in only_load and column not in only_load[table]:
                         continue
                     r = '%s.%s' % (qn(alias), qn(column))
                     if with_aliases:
diff --git a/tests/regressiontests/defer_regress/tests.py b/tests/regressiontests/defer_regress/tests.py
index fbcf85e078..b1d88e3f36 100644
--- a/tests/regressiontests/defer_regress/tests.py
+++ b/tests/regressiontests/defer_regress/tests.py
@@ -4,6 +4,7 @@ from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.sessions.backends.db import SessionStore
 from django.db import connection
+from django.db.models import Count
 from django.db.models.loading import cache
 from django.test import TestCase
 
@@ -148,6 +149,10 @@ class DeferRegressionTest(TestCase):
             ]
         )
 
+        # Regression for #16409 - make sure defer() and only() work with annotate()
+        self.assertIsInstance(list(Item.objects.annotate(Count('relateditem')).defer('name')), list)
+        self.assertIsInstance(list(Item.objects.annotate(Count('relateditem')).only('name')), list)
+
     def test_only_and_defer_usage_on_proxy_models(self):
         # Regression for #15790 - only() broken for proxy models
         proxy = Proxy.objects.create(name="proxy", value=42)