diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py
index 16b0bdad70..0562a71dd1 100644
--- a/django/db/models/sql/compiler.py
+++ b/django/db/models/sql/compiler.py
@@ -424,25 +424,34 @@ class SQLCompiler:
                 src = resolved.expression
                 expr_src = expr.expression
                 for sel_expr, _, col_alias in self.select:
-                    if col_alias and not (
-                        isinstance(expr_src, F) and col_alias == expr_src.name
-                    ):
-                        continue
                     if src == sel_expr:
+                        # When values() is used the exact alias must be used to
+                        # reference annotations.
+                        if (
+                            self.query.has_select_fields
+                            and col_alias in self.query.annotation_select
+                            and not (
+                                isinstance(expr_src, F) and col_alias == expr_src.name
+                            )
+                        ):
+                            continue
                         resolved.set_source_expressions(
                             [Ref(col_alias if col_alias else src.target.column, src)]
                         )
                         break
                 else:
-                    if col_alias:
-                        raise DatabaseError(
-                            "ORDER BY term does not match any column in the result set."
-                        )
                     # Add column used in ORDER BY clause to the selected
                     # columns and to each combined query.
                     order_by_idx = len(self.query.select) + 1
                     col_alias = f"__orderbycol{order_by_idx}"
                     for q in self.query.combined_queries:
+                        # If fields were explicitly selected through values()
+                        # combined queries cannot be augmented.
+                        if q.has_select_fields:
+                            raise DatabaseError(
+                                "ORDER BY term does not match any column in "
+                                "the result set."
+                            )
                         q.add_annotation(expr_src, col_alias)
                     self.query.add_select_col(resolved, col_alias)
                     resolved.set_source_expressions([Ref(col_alias, src)])
@@ -540,7 +549,7 @@ class SQLCompiler:
                             *self.query.annotation_select,
                         )
                     )
-                part_sql, part_args = compiler.as_sql()
+                part_sql, part_args = compiler.as_sql(with_col_aliases=True)
                 if compiler.query.combinator:
                     # Wrap in a subquery if wrapping in parentheses isn't
                     # supported.
@@ -688,8 +697,9 @@ class SQLCompiler:
         """
         refcounts_before = self.query.alias_refcount.copy()
         try:
+            combinator = self.query.combinator
             extra_select, order_by, group_by = self.pre_sql_setup(
-                with_col_aliases=with_col_aliases,
+                with_col_aliases=with_col_aliases or bool(combinator),
             )
             for_update_part = None
             # Is a LIMIT/OFFSET clause needed?
diff --git a/tests/queries/test_qs_combinators.py b/tests/queries/test_qs_combinators.py
index 865e172816..9d4988d6eb 100644
--- a/tests/queries/test_qs_combinators.py
+++ b/tests/queries/test_qs_combinators.py
@@ -304,6 +304,34 @@ class QuerySetSetOperationTests(TestCase):
             operator.itemgetter("num"),
         )
 
+    def test_union_with_select_related_and_order(self):
+        e1 = ExtraInfo.objects.create(value=7, info="e1")
+        a1 = Author.objects.create(name="a1", num=1, extra=e1)
+        a2 = Author.objects.create(name="a2", num=3, extra=e1)
+        Author.objects.create(name="a3", num=2, extra=e1)
+        base_qs = Author.objects.select_related("extra").order_by()
+        qs1 = base_qs.filter(name="a1")
+        qs2 = base_qs.filter(name="a2")
+        self.assertSequenceEqual(qs1.union(qs2).order_by("pk"), [a1, a2])
+
+    @skipUnlessDBFeature("supports_slicing_ordering_in_compound")
+    def test_union_with_select_related_and_first(self):
+        e1 = ExtraInfo.objects.create(value=7, info="e1")
+        a1 = Author.objects.create(name="a1", num=1, extra=e1)
+        Author.objects.create(name="a2", num=3, extra=e1)
+        base_qs = Author.objects.select_related("extra")
+        qs1 = base_qs.filter(name="a1")
+        qs2 = base_qs.filter(name="a2")
+        self.assertEqual(qs1.union(qs2).first(), a1)
+
+    def test_union_with_first(self):
+        e1 = ExtraInfo.objects.create(value=7, info="e1")
+        a1 = Author.objects.create(name="a1", num=1, extra=e1)
+        base_qs = Author.objects.order_by()
+        qs1 = base_qs.filter(name="a1")
+        qs2 = base_qs.filter(name="a2")
+        self.assertEqual(qs1.union(qs2).first(), a1)
+
     def test_union_multiple_models_with_values_list_and_order(self):
         reserved_name = ReservedName.objects.create(name="rn1", order=0)
         qs1 = Celebrity.objects.all()