fix trait objects with a Self-having projection va

This follows ALT2 in the issue.

Fixes rust-lang#56288.

src/librustc/traits/object_safety.rs

@@ -190,7 +190,26 @@ impl<'a, 'tcx> TyCtxt<'a, 'tcx, 'tcx> {
                         // In the case of a trait predicate, we can skip the "self" type.
                         data.skip_binder().input_types().skip(1).any(|t| t.has_self_ty())
                     }
-                    ty::Predicate::Projection(..) |
+                    ty::Predicate::Projection(ref data) => {
+                        // And similarly for projections. This should be redundant with
+                        // the previous check because any projection should have a
+                        // matching `Trait` predicate with the same inputs, but we do
+                        // the check to be safe.
+                        //
+                        // Note that we *do* allow projection *outputs* to contain
+                        // `self` (i.e., `trait Foo: Bar<Output=Self::Result> { type Result; }`),
+                        // we just require the user to specify *both* outputs
+                        // in the object type (i.e., `dyn Foo<Output=(), Result=()>`).
+                        //
+                        // This is ALT2 in issue #56288, see that for discussion of the
+                        // possible alternatives.
+                        data.skip_binder()
+                            .projection_ty
+                            .trait_ref(self)
+                            .input_types()
+                            .skip(1)
+                            .any(|t| t.has_self_ty())
+                    }
                     ty::Predicate::WellFormed(..) |
                     ty::Predicate::ObjectSafe(..) |
                     ty::Predicate::TypeOutlives(..) |

src/librustc_typeck/astconv.rs

@@ -1013,15 +1013,39 @@ impl<'o, 'gcx: 'tcx, 'tcx> dyn AstConv<'gcx, 'tcx> + 'o {
         let mut associated_types = BTreeSet::default();
 
         for tr in traits::elaborate_trait_ref(tcx, principal) {
+            debug!("conv_object_ty_poly_trait_ref: observing object predicate `{:?}`", tr);
             match tr {
                 ty::Predicate::Trait(pred) => {
                     associated_types.extend(tcx.associated_items(pred.def_id())
                                     .filter(|item| item.kind == ty::AssociatedKind::Type)
                                     .map(|item| item.def_id));
                 }
                 ty::Predicate::Projection(pred) => {
-                    // Include projections defined on supertraits.
-                    projection_bounds.push((pred, DUMMY_SP))
+                    // A `Self` within the original bound will be substituted with a
+                    // `TRAIT_OBJECT_DUMMY_SELF`, so check for that.
+                    let references_self =
+                        pred.skip_binder().ty.walk().any(|t| t == dummy_self);
+
+                    // If the projection output contains `Self`, force the user to
+                    // elaborate it explicitly to avoid a bunch of complexity.
+                    //
+                    // The "classicaly useful" case is the following:
+                    // ```
+                    //     trait MyTrait: FnMut() -> <Self as MyTrait>::MyOutput {
+                    //         type MyOutput;
+                    //     }
+                    // ```
+                    //
+                    // Here, the user could theoretically write `dyn MyTrait<Output=X>`,
+                    // but actually supporting that would "expand" to an infinitely-long type
+                    // `fix $ τ → dyn MyTrait<MyOutput=X, Output=<τ as MyTrait>::MyOutput`.
+                    //
+                    // Instead, we force the user to write `dyn MyTrait<MyOutput=X, Output=X>`,
+                    // which is uglier but works. See the discussion in #56288 for alternatives.
+                    if !references_self {
+                        // Include projections defined on supertraits,
+                        projection_bounds.push((pred, DUMMY_SP))
+                    }
                 }
                 _ => ()
             }

src/test/ui/traits/trait-object-with-self-in-projection-output-bad.rs

@@ -0,0 +1,22 @@
+trait Base {
+    type Output;
+}
+
+trait Helper: Base<Output=<Self as Helper>::Target> {
+    type Target;
+}
+
+impl Base for u32
+{
+    type Output = i32;
+}
+
+impl Helper for u32
+{
+    type Target = i32;
+}
+
+fn main() {
+    let _x: Box<dyn Helper<Target=i32>> = Box::new(2u32);
+    //~^ ERROR the value of the associated type `Output` (from the trait `Base`) must be specified
+}

src/test/ui/traits/trait-object-with-self-in-projection-output-bad.stderr

@@ -0,0 +1,12 @@
+error[E0191]: the value of the associated type `Output` (from the trait `Base`) must be specified
+  --> $DIR/trait-object-with-self-in-projection-output-bad.rs:20:17
+   |
+LL |     type Output;
+   |     ------------ `Output` defined here
+...
+LL |     let _x: Box<dyn Helper<Target=i32>> = Box::new(2u32);
+   |                 ^^^^^^^^^^^^^^^^^^^^^^ associated type `Output` must be specified
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0191`.

src/test/ui/traits/trait-object-with-self-in-projection-output-good.rs

@@ -0,0 +1,23 @@
+// compile-pass
+
+trait Base {
+    type Output;
+}
+
+trait Helper: Base<Output=<Self as Helper>::Target> {
+    type Target;
+}
+
+impl Base for u32
+{
+    type Output = i32;
+}
+
+impl Helper for u32
+{
+    type Target = i32;
+}
+
+fn main() {
+    let _x: Box<dyn Helper<Target=i32, Output=i32>> = Box::new(2u32);
+}