Rollup merge of #109683 - compiler-errors:self-ty-overflow, r=lcnr

Check for overflow in `assemble_candidates_after_normalizing_self_ty`

Prevents a stack overflow (:warning: :exclamation:) in the new solver when we have param-env candidates that look like: `T: Trait<Assoc = <T as Trait>::Assoc>`

The current error message looks bad, but that's because we don't distinguish overflow and other ambiguity errors. I'll break that out into a separate PR since the fix may be controversial.

r? `@lcnr`

compiler/rustc_trait_selection/src/solve/assembly.rs

@@ -1,5 +1,6 @@
 //! Code shared by trait and projection goals for candidate assembly.
 
+use super::search_graph::OverflowHandler;
 #[cfg(doc)]
 use super::trait_goals::structural_traits::*;
 use super::{EvalCtxt, SolverMode};
@@ -279,25 +280,38 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             return
         };
 
-        self.probe(|ecx| {
-            let normalized_ty = ecx.next_ty_infer();
-            let normalizes_to_goal = goal.with(
-                tcx,
-                ty::Binder::dummy(ty::ProjectionPredicate {
-                    projection_ty,
-                    term: normalized_ty.into(),
-                }),
-            );
-            ecx.add_goal(normalizes_to_goal);
-            if let Ok(_) = ecx.try_evaluate_added_goals() {
-                let normalized_ty = ecx.resolve_vars_if_possible(normalized_ty);
-
-                // NOTE: Alternatively we could call `evaluate_goal` here and only have a `Normalized` candidate.
-                // This doesn't work as long as we use `CandidateSource` in winnowing.
-                let goal = goal.with(tcx, goal.predicate.with_self_ty(tcx, normalized_ty));
-                candidates.extend(ecx.assemble_and_evaluate_candidates(goal));
-            }
+        let normalized_self_candidates: Result<_, NoSolution> = self.probe(|ecx| {
+            ecx.with_incremented_depth(
+                |ecx| {
+                    let result = ecx.evaluate_added_goals_and_make_canonical_response(
+                        Certainty::Maybe(MaybeCause::Overflow),
+                    )?;
+                    Ok(vec![Candidate { source: CandidateSource::BuiltinImpl, result }])
+                },
+                |ecx| {
+                    let normalized_ty = ecx.next_ty_infer();
+                    let normalizes_to_goal = goal.with(
+                        tcx,
+                        ty::Binder::dummy(ty::ProjectionPredicate {
+                            projection_ty,
+                            term: normalized_ty.into(),
+                        }),
+                    );
+                    ecx.add_goal(normalizes_to_goal);
+                    let _ = ecx.try_evaluate_added_goals()?;
+                    let normalized_ty = ecx.resolve_vars_if_possible(normalized_ty);
+                    // NOTE: Alternatively we could call `evaluate_goal` here and only
+                    // have a `Normalized` candidate. This doesn't work as long as we
+                    // use `CandidateSource` in winnowing.
+                    let goal = goal.with(tcx, goal.predicate.with_self_ty(tcx, normalized_ty));
+                    Ok(ecx.assemble_and_evaluate_candidates(goal))
+                },
+            )
         });
+
+        if let Ok(normalized_self_candidates) = normalized_self_candidates {
+            candidates.extend(normalized_self_candidates);
+        }
     }
 
     fn assemble_impl_candidates<G: GoalKind<'tcx>>(

compiler/rustc_trait_selection/src/solve/search_graph/overflow.rs

@@ -73,6 +73,27 @@ pub(in crate::solve) trait OverflowHandler<'tcx> {
         self.search_graph().overflow_data.deal_with_overflow();
         on_overflow(self)
     }
+
+    // Increment the `additional_depth` by one and evaluate `body`, or `on_overflow`
+    // if the depth is overflown.
+    fn with_incremented_depth<T>(
+        &mut self,
+        on_overflow: impl FnOnce(&mut Self) -> T,
+        body: impl FnOnce(&mut Self) -> T,
+    ) -> T {
+        let depth = self.search_graph().stack.len();
+        self.search_graph().overflow_data.additional_depth += 1;
+
+        let result = if self.search_graph().overflow_data.has_overflow(depth) {
+            self.search_graph().overflow_data.deal_with_overflow();
+            on_overflow(self)
+        } else {
+            body(self)
+        };
+
+        self.search_graph().overflow_data.additional_depth -= 1;
+        result
+    }
 }
 
 impl<'tcx> OverflowHandler<'tcx> for EvalCtxt<'_, 'tcx> {

tests/ui/traits/new-solver/recursive-self-normalization-2.rs

@@ -0,0 +1,19 @@
+// compile-flags: -Ztrait-solver=next
+
+trait Foo1 {
+    type Assoc1;
+}
+
+trait Foo2 {
+    type Assoc2;
+}
+
+trait Bar {}
+fn needs_bar<S: Bar>() {}
+
+fn test<T: Foo1<Assoc1 = <T as Foo2>::Assoc2> + Foo2<Assoc2 = <T as Foo1>::Assoc1>>() {
+    needs_bar::<T::Assoc1>();
+    //~^ ERROR type annotations needed
+}
+
+fn main() {}

tests/ui/traits/new-solver/recursive-self-normalization-2.stderr

@@ -0,0 +1,9 @@
+error[E0282]: type annotations needed
+  --> $DIR/recursive-self-normalization-2.rs:15:5
+   |
+LL |     needs_bar::<T::Assoc1>();
+   |     ^^^^^^^^^^^^^^^^^^^^^^ cannot infer type of the type parameter `S` declared on the function `needs_bar`
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0282`.

tests/ui/traits/new-solver/recursive-self-normalization.rs

@@ -0,0 +1,15 @@
+// compile-flags: -Ztrait-solver=next
+
+trait Foo {
+    type Assoc;
+}
+
+trait Bar {}
+fn needs_bar<S: Bar>() {}
+
+fn test<T: Foo<Assoc = <T as Foo>::Assoc>>() {
+    needs_bar::<T::Assoc>();
+    //~^ ERROR type annotations needed
+}
+
+fn main() {}

tests/ui/traits/new-solver/recursive-self-normalization.stderr

@@ -0,0 +1,9 @@
+error[E0282]: type annotations needed
+  --> $DIR/recursive-self-normalization.rs:11:5
+   |
+LL |     needs_bar::<T::Assoc>();
+   |     ^^^^^^^^^^^^^^^^^^^^^ cannot infer type of the type parameter `S` declared on the function `needs_bar`
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0282`.