Canonicalize free regions from inputs as placeholders in root univ

compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs

@@ -451,7 +451,7 @@ fn try_extract_error_from_region_constraints<'a, 'tcx>(
             (RePlaceholder(a_p), RePlaceholder(b_p)) => a_p.bound == b_p.bound,
             _ => a_region == b_region,
         };
-    let mut check = |c: &Constraint<'tcx>, cause: &SubregionOrigin<'tcx>, exact| match c.kind {
+    let mut check = |c: Constraint<'tcx>, cause: &SubregionOrigin<'tcx>, exact| match c.kind {
         ConstraintKind::RegSubReg
             if ((exact && c.sup == placeholder_region)
                 || (!exact && regions_the_same(c.sup, placeholder_region)))
@@ -467,13 +467,23 @@ fn try_extract_error_from_region_constraints<'a, 'tcx>(
         {
             Some((c.sub, cause.clone()))
         }
-        _ => None,
+        ConstraintKind::VarSubVar
+        | ConstraintKind::RegSubVar
+        | ConstraintKind::VarSubReg
+        | ConstraintKind::RegSubReg => None,
+
+        ConstraintKind::VarEqVar | ConstraintKind::VarEqReg | ConstraintKind::RegEqReg => {
+            unreachable!()
+        }
     };
 
     let mut find_culprit = |exact_match: bool| {
         region_constraints
             .constraints
             .iter()
+            .flat_map(|(constraint, cause)| {
+                constraint.iter_outlives().map(move |constraint| (constraint, cause))
+            })
             .find_map(|(constraint, cause)| check(constraint, cause, exact_match))
     };
 

compiler/rustc_borrowck/src/type_check/constraint_conversion.rs

@@ -70,12 +70,14 @@ impl<'a, 'tcx> ConstraintConversion<'a, 'tcx> {
 
     #[instrument(skip(self), level = "debug")]
     pub(super) fn convert_all(&mut self, query_constraints: &QueryRegionConstraints<'tcx>) {
-        let QueryRegionConstraints { outlives, assumptions } = query_constraints;
+        let QueryRegionConstraints { constraints, assumptions } = query_constraints;
         let assumptions =
             elaborate::elaborate_outlives_assumptions(self.infcx.tcx, assumptions.iter().copied());
 
-        for &(predicate, constraint_category) in outlives {
-            self.convert(predicate, constraint_category, &assumptions);
+        for &(constraint, constraint_category) in constraints {
+            constraint.iter_outlives().for_each(|predicate| {
+                self.convert(predicate, constraint_category, &assumptions);
+            });
         }
     }
 
@@ -292,8 +294,12 @@ impl<'a, 'tcx> ConstraintConversion<'a, 'tcx> {
         ) {
             Ok(TypeOpOutput { output: ty, constraints, .. }) => {
                 // FIXME(higher_ranked_auto): What should we do with the assumptions here?
-                if let Some(QueryRegionConstraints { outlives, assumptions: _ }) = constraints {
-                    next_outlives_predicates.extend(outlives.iter().copied());
+                if let Some(QueryRegionConstraints { constraints, assumptions: _ }) = constraints {
+                    next_outlives_predicates.extend(constraints.iter().flat_map(
+                        |(constraint, category)| {
+                            constraint.iter_outlives().map(|outlives| (outlives, *category))
+                        },
+                    ));
                 }
                 ty
             }

compiler/rustc_infer/src/infer/canonical/canonicalizer.rs

@@ -229,7 +229,8 @@ impl CanonicalizeMode for CanonicalizeUserTypeAnnotation {
             | ty::ReErased
             | ty::ReStatic
             | ty::ReError(_) => r,
-            ty::ReVar(_) => canonicalizer.canonical_var_for_region_in_root_universe(r),
+            ty::ReVar(_) => canonicalizer
+                .canonical_var_for_region(CanonicalVarKind::Region(ty::UniverseIndex::ROOT), r),
             ty::RePlaceholder(..) | ty::ReBound(..) => {
                 // We only expect region names that the user can type.
                 bug!("unexpected region in query response: `{:?}`", r)
@@ -707,23 +708,26 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
     }
 
     /// Shorthand helper that creates a canonical region variable for
-    /// `r` (always in the root universe). The reason that we always
-    /// put these variables into the root universe is because this
-    /// method is used during **query construction:** in that case, we
-    /// are taking all the regions and just putting them into the most
-    /// generic context we can. This may generate solutions that don't
-    /// fit (e.g., that equate some region variable with a placeholder
-    /// it can't name) on the caller side, but that's ok, the caller
-    /// can figure that out. In the meantime, it maximizes our
+    /// `r` (always as a placeholder in the root universe). The reason
+    /// that we always put these variables into the root universe as a
+    /// placeholder is because this method is used during
+    /// **query construction:** in that case, we are taking all the
+    /// free regions and just putting them into the most generic context
+    /// we can. This makes any region constraints between them, including
+    /// region equalities, to be preserved and conveyed to the caller
+    /// instead of unifying them. In the meantime, it maximizes our
     /// caching.
-    ///
-    /// (This works because unification never fails -- and hence trait
-    /// selection is never affected -- due to a universe mismatch.)
     fn canonical_var_for_region_in_root_universe(
         &mut self,
         r: ty::Region<'tcx>,
     ) -> ty::Region<'tcx> {
-        self.canonical_var_for_region(CanonicalVarKind::Region(ty::UniverseIndex::ROOT), r)
+        self.canonical_var_for_region(
+            CanonicalVarKind::PlaceholderRegion(ty::PlaceholderRegion::new_anon(
+                ty::UniverseIndex::ROOT,
+                self.var_kinds.len().into(),
+            )),
+            r,
+        )
     }
 
     /// Creates a canonical variable (with the given `info`)

compiler/rustc_infer/src/infer/canonical/query_response.rs

@@ -22,7 +22,7 @@ use crate::infer::canonical::{
     Canonical, CanonicalQueryResponse, CanonicalVarValues, Certainty, OriginalQueryValues,
     QueryRegionConstraints, QueryResponse,
 };
-use crate::infer::region_constraints::RegionConstraintData;
+use crate::infer::region_constraints::{ConstraintKind, RegionConstraintData};
 use crate::infer::{
     DefineOpaqueTypes, InferCtxt, InferOk, InferResult, OpaqueTypeStorageEntries, SubregionOrigin,
     TypeOutlivesConstraint,
@@ -188,9 +188,16 @@ impl<'tcx> InferCtxt<'tcx> {
         let InferOk { value: result_args, obligations } =
             self.query_response_instantiation(cause, param_env, original_values, query_response)?;
 
-        for (predicate, _category) in &query_response.value.region_constraints.outlives {
-            let predicate = instantiate_value(self.tcx, &result_args, *predicate);
-            self.register_outlives_constraint(predicate, cause);
+        for (constraint, _category) in &query_response.value.region_constraints.constraints {
+            let constraint = instantiate_value(self.tcx, &result_args, *constraint);
+            match constraint {
+                ty::RegionConstraint::Outlives(predicate) => {
+                    self.register_outlives_constraint(predicate, cause);
+                }
+                ty::RegionConstraint::Eq(predicate) => {
+                    self.register_region_eq_constraint(predicate, cause);
+                }
+            }
         }
 
         for assumption in &query_response.value.region_constraints.assumptions {
@@ -277,14 +284,11 @@ impl<'tcx> InferCtxt<'tcx> {
                 }
 
                 (GenericArgKind::Lifetime(v_o), GenericArgKind::Lifetime(v_r)) => {
-                    // To make `v_o = v_r`, we emit `v_o: v_r` and `v_r: v_o`.
                     if v_o != v_r {
-                        output_query_region_constraints
-                            .outlives
-                            .push((ty::OutlivesPredicate(v_o.into(), v_r), constraint_category));
-                        output_query_region_constraints
-                            .outlives
-                            .push((ty::OutlivesPredicate(v_r.into(), v_o), constraint_category));
+                        output_query_region_constraints.constraints.push((
+                            ty::RegionEqPredicate(v_o.into(), v_r).into(),
+                            constraint_category,
+                        ));
                     }
                 }
 
@@ -311,13 +315,12 @@ impl<'tcx> InferCtxt<'tcx> {
         }
 
         // ...also include the other query region constraints from the query.
-        output_query_region_constraints.outlives.extend(
-            query_response.value.region_constraints.outlives.iter().filter_map(|&r_c| {
+        output_query_region_constraints.constraints.extend(
+            query_response.value.region_constraints.constraints.iter().filter_map(|&r_c| {
                 let r_c = instantiate_value(self.tcx, &result_args, r_c);
 
-                // Screen out `'a: 'a` cases.
-                let ty::OutlivesPredicate(k1, r2) = r_c.0;
-                if k1 != r2.into() { Some(r_c) } else { None }
+                // Screen out `'a: 'a` or `'a == 'a` cases.
+                if r_c.0.is_trivial() { None } else { Some(r_c) }
             }),
         );
 
@@ -611,20 +614,30 @@ pub fn make_query_region_constraints<'tcx>(
 
     debug!(?constraints);
 
-    let outlives: Vec<_> = constraints
+    let constraints: Vec<_> = constraints
         .iter()
-        .map(|(c, origin)| {
-            // Swap regions because we are going from sub (<=) to outlives (>=).
-            let constraint = ty::OutlivesPredicate(c.sup.into(), c.sub);
-            (constraint, origin.to_constraint_category())
+        .map(|(c, origin)| match c.kind {
+            ConstraintKind::VarSubVar
+            | ConstraintKind::RegSubVar
+            | ConstraintKind::VarSubReg
+            | ConstraintKind::RegSubReg => {
+                // Swap regions because we are going from sub (<=) to outlives (>=).
+                let constraint = ty::OutlivesPredicate(c.sup.into(), c.sub).into();
+                (constraint, origin.to_constraint_category())
+            }
+
+            ConstraintKind::VarEqVar | ConstraintKind::VarEqReg | ConstraintKind::RegEqReg => {
+                let constraint = ty::RegionEqPredicate(c.sup, c.sub).into();
+                (constraint, origin.to_constraint_category())
+            }
         })
         .chain(outlives_obligations.into_iter().map(|obl| {
             (
-                ty::OutlivesPredicate(obl.sup_type.into(), obl.sub_region),
+                ty::OutlivesPredicate(obl.sup_type.into(), obl.sub_region).into(),
                 obl.origin.to_constraint_category(),
             )
         }))
         .collect();
 
-    QueryRegionConstraints { outlives, assumptions }
+    QueryRegionConstraints { constraints, assumptions }
 }

compiler/rustc_infer/src/infer/lexical_region_resolve/indexed_edges.rs

@@ -13,39 +13,60 @@ pub(super) enum EdgeDirection {
 
 /// Type alias for the pairs stored in [`RegionConstraintData::constraints`],
 /// which we are indexing.
-type ConstraintPair<'tcx> = (Constraint<'tcx>, SubregionOrigin<'tcx>);
+type ConstraintPair<'data, 'tcx> = (Constraint<'tcx>, &'data SubregionOrigin<'tcx>);
 
 /// An index from region variables to their corresponding constraint edges,
 /// used on some error paths.
 pub(super) struct IndexedConstraintEdges<'data, 'tcx> {
-    out_edges: IndexVec<RegionVid, Vec<&'data ConstraintPair<'tcx>>>,
-    in_edges: IndexVec<RegionVid, Vec<&'data ConstraintPair<'tcx>>>,
+    out_edges: IndexVec<RegionVid, Vec<ConstraintPair<'data, 'tcx>>>,
+    in_edges: IndexVec<RegionVid, Vec<ConstraintPair<'data, 'tcx>>>,
 }
 
 impl<'data, 'tcx> IndexedConstraintEdges<'data, 'tcx> {
     pub(super) fn build_index(num_vars: usize, data: &'data RegionConstraintData<'tcx>) -> Self {
         let mut out_edges = IndexVec::from_fn_n(|_| vec![], num_vars);
         let mut in_edges = IndexVec::from_fn_n(|_| vec![], num_vars);
 
-        for pair @ (c, _) in &data.constraints {
+        for pair @ (c, _) in data
+            .constraints
+            .iter()
+            .flat_map(|(c, origin)| c.iter_outlives().map(move |c| (c, origin)))
+        {
             // Only push a var out-edge for `VarSub...` constraints.
             match c.kind {
                 ConstraintKind::VarSubVar | ConstraintKind::VarSubReg => {
-                    out_edges[c.sub.as_var()].push(pair)
+                    out_edges[c.sub.as_var()].push(pair);
                 }
+
                 ConstraintKind::RegSubVar | ConstraintKind::RegSubReg => {}
+
+                ConstraintKind::VarEqVar | ConstraintKind::VarEqReg | ConstraintKind::RegEqReg => {
+                    unreachable!();
+                }
             }
         }
 
+        // FIXME: We should merge this loop with the above one eventually.
         // Index in-edges in reverse order, to match what current tests expect.
         // (It's unclear whether this is important or not.)
-        for pair @ (c, _) in data.constraints.iter().rev() {
+
+        for pair @ (c, _) in data
+            .constraints
+            .iter()
+            .rev()
+            .flat_map(|(c, origin)| c.iter_outlives().map(move |c| (c, origin)))
+        {
             // Only push a var in-edge for `...SubVar` constraints.
             match c.kind {
                 ConstraintKind::VarSubVar | ConstraintKind::RegSubVar => {
-                    in_edges[c.sup.as_var()].push(pair)
+                    in_edges[c.sup.as_var()].push(pair);
                 }
+
                 ConstraintKind::VarSubReg | ConstraintKind::RegSubReg => {}
+
+                ConstraintKind::VarEqVar | ConstraintKind::VarEqReg | ConstraintKind::RegEqReg => {
+                    unreachable!();
+                }
             }
         }
 
@@ -58,7 +79,7 @@ impl<'data, 'tcx> IndexedConstraintEdges<'data, 'tcx> {
         &self,
         region_vid: RegionVid,
         dir: EdgeDirection,
-    ) -> &[&'data ConstraintPair<'tcx>] {
+    ) -> &[ConstraintPair<'data, 'tcx>] {
         let edges = match dir {
             EdgeDirection::Out => &self.out_edges,
             EdgeDirection::In => &self.in_edges,

compiler/rustc_infer/src/infer/lexical_region_resolve/mod.rs

@@ -34,6 +34,18 @@ pub(crate) fn resolve<'tcx>(
     var_infos: VarInfos<'tcx>,
     data: RegionConstraintData<'tcx>,
 ) -> (LexicalRegionResolutions<'tcx>, Vec<RegionResolutionError<'tcx>>) {
+    assert!(
+        data.constraints.iter().all(|(c, _)| match c.kind {
+            ConstraintKind::VarSubVar
+            | ConstraintKind::RegSubVar
+            | ConstraintKind::VarSubReg
+            | ConstraintKind::RegSubReg => true,
+
+            ConstraintKind::VarEqVar | ConstraintKind::VarEqReg | ConstraintKind::RegEqReg => false,
+        }),
+        "Every constraint should be decomposed into outlives here"
+    );
+
     let mut errors = vec![];
     let mut resolver = LexicalResolver { region_rels, var_infos, data };
     let values = resolver.infer_variable_values(&mut errors);
@@ -279,6 +291,10 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
                     // is done, in `collect_errors`.
                     continue;
                 }
+
+                ConstraintKind::VarEqVar | ConstraintKind::VarEqReg | ConstraintKind::RegEqReg => {
+                    unreachable!()
+                }
             }
         }
 
@@ -575,6 +591,10 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
                         *sub_data = VarValue::ErrorValue;
                     }
                 }
+
+                ConstraintKind::VarEqVar | ConstraintKind::VarEqReg | ConstraintKind::RegEqReg => {
+                    unreachable!()
+                }
             }
         }
 
@@ -852,19 +872,23 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
                     }
 
                     ConstraintKind::RegSubVar => {
-                        let origin = origin.clone();
+                        let origin = (*origin).clone();
                         state.result.push(RegionAndOrigin { region: c.sub, origin });
                     }
 
                     ConstraintKind::VarSubReg => {
-                        let origin = origin.clone();
+                        let origin = (*origin).clone();
                         state.result.push(RegionAndOrigin { region: c.sup, origin });
                     }
 
                     ConstraintKind::RegSubReg => panic!(
                         "cannot reach reg-sub-reg edge in region inference \
                          post-processing"
                     ),
+
+                    ConstraintKind::VarEqVar
+                    | ConstraintKind::VarEqReg
+                    | ConstraintKind::RegEqReg => unreachable!(),
                 }
             }
         }

compiler/rustc_infer/src/infer/mod.rs

@@ -702,6 +702,16 @@ impl<'tcx> InferCtxt<'tcx> {
         self.inner.borrow_mut().unwrap_region_constraints().make_subregion(origin, a, b);
     }
 
+    #[instrument(skip(self), level = "debug")]
+    pub fn equate_regions(
+        &self,
+        origin: SubregionOrigin<'tcx>,
+        a: ty::Region<'tcx>,
+        b: ty::Region<'tcx>,
+    ) {
+        self.inner.borrow_mut().unwrap_region_constraints().make_eqregion(origin, a, b);
+    }
+
     /// Processes a `Coerce` predicate from the fulfillment context.
     /// This is NOT the preferred way to handle coercion, which is to
     /// invoke `FnCtxt::coerce` or a similar method (see `coercion.rs`).