Align the two coset intersection methods; add some comments; restore …

…a removed criterion

lib/csetperm.gi

@@ -555,9 +555,9 @@ InstallMethod(Intersection2, "perm cosets", IsIdenticalObj,
   [IsRightCoset and IsPermCollection,IsRightCoset and IsPermCollection],0,
 function(cos1,cos2)
     local H1, H2, x1, x2, shift, sigma, listMoved_H1, listMoved_H2,
-          listMoved_H12, listMoved_sigma, grpInt, set2, set1, eRepr,
+          listMoved_H12, listMoved_sigma, U, set2, set1, eRepr,
           set2_img, set1_img, H1_sigma, H2_sigma, test, H12, swap,
-          eCos, rho, cosTest, diff12, diff21, fset1, fset2, listMoved_all;
+          eCos, rho, diff12, diff21, fset1, fset2;
     # We set cosInt = cos1 cap cos2 = H1 x1 cap H2 x2
     H1:=ActingDomain(cos1);
     H2:=ActingDomain(cos2);
@@ -581,11 +581,26 @@ function(cos1,cos2)
         listMoved_H2:=MovedPoints(H2);
         listMoved_H12:=Union(listMoved_H1, listMoved_H2);
         listMoved_sigma:=MovedPoints(sigma);
-        # First exclusion case
-        listMoved_all := Union(listMoved_H12, listMoved_sigma);
-        if ForAny(listMoved_all, n -> IsEmpty(Intersection(Set(Orbit(H1,n)), OnSets(Set(Orbit(H2,n)),sigma)))) then
+
+        # If the coset intersection is non-empty, then there is h1 \in H1
+        # and h2 \in H2 such that h1 = h2*sigma, in other words, sigma
+        # is contained in the group H12 generated by H1 and H2. A necessary
+        # condition for this is that the points moved by sigma are a subset
+        # of the points moved by H12.
+        if not IsSubset(listMoved_H12, listMoved_sigma) then
             return [];
         fi;
+
+        # Suppose x is an element of the intersection of the two cosets. Then for
+        # any positive integer n, we know that n^x is contained in n^H1 but
+        # also in (n^H2)^\sigma. Thus if the intersection of n^H1 and
+        # (n^H2)^\sigma is empty, then the intersection of the cosets is also
+        # empty. Clearly the orbit intersection contains n whenever n is fixed
+        # by sigma, so we only have to consider this for n moved by sigma.
+        if ForAny(listMoved_sigma, n -> IsEmpty(Intersection(Orbit(H1,n), OnTuples(Orbit(H2,n),sigma)))) then
+            return [];
+        fi;
+
         # Easy reductions: points that are moved by sigma outside of one group allow us to reduce the problem
         diff12:=Difference(listMoved_H1, listMoved_H2);
         diff21:=Difference(listMoved_H2, listMoved_H1);
@@ -618,14 +633,14 @@ function(cos1,cos2)
         fi;
         # easy termination criterion
         if sigma in H2 then
-            grpInt:=Intersection(H1, H2);
-            return RightCoset(grpInt, shift);
+            U:=Intersection(H1, H2);
+            return RightCoset(U, shift);
         fi;
         if sigma in H1 then
             # cosInt = (H1 \cap H2 sigma) shift
             #        = (H1 sigma^{-1} \cap H2) sigma shift
-            grpInt:=Intersection(H1, H2);
-            return RightCoset(grpInt, sigma * shift);
+            U:=Intersection(H1, H2);
+            return RightCoset(U, sigma * shift);
         fi;
         # reduction on sets which is easy
         fset1:=Difference(listMoved_H1, Union(listMoved_H2, listMoved_sigma));
@@ -664,18 +679,18 @@ function(cos1,cos2)
     # it here in order to avoid errors:
     # --- The naive algorithm for computing H1 \cap H2 sigma is to iterate
     # over elements of H1 and testing if one belongs to H2 sigma. If we find
-    # one such z then the result is the coset RightCoset(grpInt, z). If not
+    # one such z then the result is the coset RightCoset(U, z). If not
     # then it is empty.
-    # --- Since the result is independent of the cosets grpInt, what we can
-    # do is iterate over the RightCosets(H1, grpInt). The algorithm is the
+    # --- Since the result is independent of the cosets U, what we can
+    # do is iterate over the RightCosets(H1, U). The algorithm is the
     # one of Proposition 3.2
-    # for r in RightCosets(H1, grpInt) do
+    # for r in RightCosets(H1, U) do
     #   if r in H1*sigma then
-    #     return RightCoset(grpInt, r * shift)
+    #     return RightCoset(U, r * shift)
     #   fi;
     # od;
     # --- (TODO for future work): The question is how to make it faster.
-    # One idea is to use an ascending chain between grpInt and H1.
+    # One idea is to use an ascending chain between U and H1.
     # Section 3.4 of above paper gives statement related to that but not a
     # useful algorithm. The question deserves further exploration.
     #
@@ -691,11 +706,10 @@ function(cos1,cos2)
         sigma:=Inverse(sigma);
     fi;
     # So now Order(H1) <= Order(H2)
-    cosTest:=RightCoset(H2, sigma);
-    grpInt:=Intersection(H1, H2);
-    for rho in RightTransversal(H1, grpInt) do
-        if rho in cosTest then
-            return RightCoset(grpInt, rho * shift);
+    U:=Intersection(H1, H2);
+    for rho in RightTransversal(H1, U) do
+        if rho / sigma in H2 then
+            return RightCoset(U, rho * shift);
         fi;
     od;
     return [];