diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py
index ef3f5220fb..7d24701438 100644
--- a/django/db/models/sql/query.py
+++ b/django/db/models/sql/query.py
@@ -509,8 +509,10 @@ class Query(object):
         pieces = name.split(LOOKUP_SEP)
         if not alias:
             alias = self.get_initial_alias()
-        field, target, opts, joins = self.setup_joins(pieces, opts, alias,
-                False)
+        result = self.setup_joins(pieces, opts, alias, False, False)
+        if isinstance(result, int):
+            raise FieldError("Cannot order by many-valued field: '%s'" % name)
+        field, target, opts, joins = result
         alias = joins[-1][-1]
         col = target.column
 
diff --git a/docs/db-api.txt b/docs/db-api.txt
index 74adc10457..3cab6574ea 100644
--- a/docs/db-api.txt
+++ b/docs/db-api.txt
@@ -541,6 +541,13 @@ primary key if there is no ``Meta.ordering`` specified. For example::
 
 ...since the ``Blog`` model has no default ordering specified.
 
+You can only order by model fields that have a single value attached to them
+for each instance of the model. For example, non-relations, ``ForeignKey`` and
+``OneToOneField`` fields. Explicitly, you can't order by a ``ManyToManyField``
+or a reverse ``ForeignKey`` relation. There's no naturally correct ordering
+for many-valued fields and a lot of the alternatives are not psosible to
+express in SQL very efficiently.
+
 **New in Django development version:** If you don't want any ordering to be
 applied to a query, not even the default ordering, call ``order_by()`` with no
 parameters.
diff --git a/tests/regressiontests/queries/models.py b/tests/regressiontests/queries/models.py
index ff55550d4e..17476217cd 100644
--- a/tests/regressiontests/queries/models.py
+++ b/tests/regressiontests/queries/models.py
@@ -418,6 +418,13 @@ FieldError: Infinite loop caused by ordering.
 >>> Ranking.objects.all().order_by('rank')
 [<Ranking: 1: a3>, <Ranking: 2: a2>, <Ranking: 3: a1>]
 
+# Ordering by a many-valued attribute (e.g. a many-to-many or reverse
+# ForeignKey) doesn't make sense (there's no natural ordering).
+>>> Item.objects.all().order_by('tags')
+Traceback (most recent call last):
+...
+FieldError: Cannot order by many-valued field: 'tags'
+
 # If we replace the default ordering, Django adjusts the required tables
 # automatically. Item normally requires a join with Note to do the default
 # ordering, but that isn't needed here.