compiler: report error when trait object type param reference self

Fixes #139082.

Emits an error when `Self` is found in the projection bounds of a trait
object. In type aliases, `Self` has no meaning, so `type A = &'static
dyn B` where `trait B = Fn() -> Self` will expands to `type A = &'static
Fn() -> Self` which is illegal, causing the region solver to bail out
when hitting the uninferred Self.

Bug: #139082
Signed-off-by: xtex <xtexchooser@duck.com>

Author: xtexx

compiler/rustc_hir_analysis/messages.ftl

@@ -486,6 +486,9 @@ hir_analysis_self_in_impl_self =
     `Self` is not valid in the self type of an impl block
     .note = replace `Self` with a different type
 
+hir_analysis_self_in_type_alias = `Self` is not allowed in type aliases
+    .label = `Self` is only available in impls, traits, and concrete type definitions
+
 hir_analysis_self_ty_not_captured = `impl Trait` must mention the `Self` type of the trait in `use<...>`
     .label = `Self` type parameter is implicitly captured by this `impl Trait`
     .note = currently, all type parameters are required to be mentioned in the precise captures list

compiler/rustc_hir_analysis/src/errors.rs

@@ -1707,3 +1707,11 @@ pub(crate) enum SupertraitItemShadowee {
         traits: DiagSymbolList,
     },
 }
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_self_in_type_alias, code = E0411)]
+pub(crate) struct SelfInTypeAlias {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+}

compiler/rustc_hir_analysis/src/hir_ty_lowering/dyn_compatibility.rs

@@ -16,6 +16,7 @@ use smallvec::{SmallVec, smallvec};
 use tracing::{debug, instrument};
 
 use super::HirTyLowerer;
+use crate::errors::SelfInTypeAlias;
 use crate::hir_ty_lowering::{
     GenericArgCountMismatch, GenericArgCountResult, PredicateFilter, RegionInferReason,
 };
@@ -352,6 +353,16 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                     b.projection_term = replace_dummy_self_with_error(tcx, b.projection_term, guar);
                 }
 
+                if let Some(term_ty) = &b.term.as_type() {
+                    let references_self = term_ty.walk().any(|arg| arg == dummy_self.into());
+                    if references_self {
+                        // With trait alias and type alias combined, type resolver
+                        // may not be able to catch all illegal `Self` usages (issue 139082)
+                        let guar = tcx.dcx().emit_err(SelfInTypeAlias { span });
+                        b.term = replace_dummy_self_with_error(tcx, b.term, guar);
+                    }
+                }
+
                 ty::ExistentialPredicate::Projection(ty::ExistentialProjection::erase_self_ty(
                     tcx, b,
                 ))

tests/ui/dyn-compatibility/trait-alias-self-projection.rs

@@ -0,0 +1,12 @@
+#![feature(trait_alias)]
+trait B = Fn() -> Self;
+type D = &'static dyn B;
+//~^ ERROR E0411
+
+fn a() -> D {
+    unreachable!();
+}
+
+fn main() {
+    _ = a();
+}

tests/ui/dyn-compatibility/trait-alias-self-projection.stderr

@@ -0,0 +1,9 @@
+error[E0411]: `Self` is not allowed in type aliases
+  --> $DIR/trait-alias-self-projection.rs:3:19
+   |
+LL | type D = &'static dyn B;
+   |                   ^^^^^ `Self` is only available in impls, traits, and concrete type definitions
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0411`.