diff --git a/.github/workflows/issue-write.yml b/.github/workflows/issue-write.yml
index eebaf89e027be..d9ce8fa94e4d5 100644
--- a/.github/workflows/issue-write.yml
+++ b/.github/workflows/issue-write.yml
@@ -162,4 +162,4 @@ jobs:
         if: >-
           always() &&
           steps.download-artifact.outputs.artifact-ids != ''
-        run: cat comments
+        run: cat comments*
diff --git a/clang-tools-extra/clang-tidy/readability/RedundantCastingCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantCastingCheck.cpp
index d11c41c33d2be..21f481a718219 100644
--- a/clang-tools-extra/clang-tidy/readability/RedundantCastingCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/RedundantCastingCheck.cpp
@@ -9,6 +9,7 @@
 #include "RedundantCastingCheck.h"
 #include "../utils/FixItHintUtils.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/TypeBase.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/Lex/Lexer.h"
 
@@ -29,7 +30,7 @@ static bool areTypesEqual(QualType S, QualType D) {
   const QualType PtrD = D->getPointeeType();
 
   if (!PtrS.isNull() && !PtrD.isNull())
-    return areTypesEqual(PtrS, PtrD);
+    return areTypesEqual(PtrS.IgnoreParens(), PtrD.IgnoreParens());
 
   const DeducedType *DT = S->getContainedDeducedType();
   if (DT && DT->isDeduced())
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 225ddaeedc10d..fdc866d8255bb 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -567,6 +567,10 @@ Changes in existing checks
   <clang-tidy/checks/readability/qualified-auto>` check by adding the option
   `IgnoreAliasing`, that allows not looking at underlying types of type aliases.
 
+- Improved :doc:`readability-redundant-casting
+  <clang-tidy/checks/readability/redundant-casting>` check by fixing false
+  negatives when explicitly cast from function pointer.
+
 - Improved :doc:`readability-uppercase-literal-suffix
   <clang-tidy/checks/readability/uppercase-literal-suffix>` check to recognize
   literal suffixes added in C++23 and C23.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-casting.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-casting.cpp
index fa91995c5615f..3e723b8b61d1d 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-casting.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-casting.cpp
@@ -235,3 +235,13 @@ void testRedundantDependentNTTPCasting() {
   // CHECK-MESSAGES: :[[@LINE-4]]:25: note: source type originates from referencing this non-type template parameter
   // CHECK-FIXES: T a = V;
 }
+
+namespace gh170476 {
+int f(void);
+int g1() {
+  int (*fp)() = (int(*)(void))&f;
+  // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: redundant explicit casting to the same type 'int (*)()' as the sub-expression, remove this casting [readability-redundant-casting]
+  // CHECK-FIXES: int (*fp)() = (&f);
+  return fp();
+}
+} // namespace gh170476
diff --git a/flang-rt/test/Driver/exec.f90 b/flang-rt/test/Driver/exec.f90
index 5a81a1e4c3e45..d220174994979 100644
--- a/flang-rt/test/Driver/exec.f90
+++ b/flang-rt/test/Driver/exec.f90
@@ -1,4 +1,3 @@
-! REQUIRES: flang-rt
 ! UNSUPPORTED: offload-cuda
 
 ! Verify that flang can correctly build executables.
diff --git a/flang/include/flang/Parser/openmp-utils.h b/flang/include/flang/Parser/openmp-utils.h
index b7d990c9e75d6..e164f63aa189b 100644
--- a/flang/include/flang/Parser/openmp-utils.h
+++ b/flang/include/flang/Parser/openmp-utils.h
@@ -14,6 +14,7 @@
 #define FORTRAN_PARSER_OPENMP_UTILS_H
 
 #include "flang/Common/indirection.h"
+#include "flang/Common/template.h"
 #include "flang/Parser/parse-tree.h"
 #include "llvm/Frontend/OpenMP/OMP.h"
 
@@ -127,7 +128,88 @@ template <typename T> struct IsStatement<Statement<T>> {
 std::optional<Label> GetStatementLabel(const ExecutionPartConstruct &x);
 std::optional<Label> GetFinalLabel(const OpenMPConstruct &x);
 
+namespace detail {
+// Clauses with flangClass = "OmpObjectList".
+using MemberObjectListClauses =
+    std::tuple<OmpClause::Copyin, OmpClause::Copyprivate, OmpClause::Exclusive,
+        OmpClause::Firstprivate, OmpClause::HasDeviceAddr, OmpClause::Inclusive,
+        OmpClause::IsDevicePtr, OmpClause::Link, OmpClause::Private,
+        OmpClause::Shared, OmpClause::UseDeviceAddr, OmpClause::UseDevicePtr>;
+
+// Clauses with flangClass = "OmpSomeClause", and OmpObjectList a
+// member of tuple OmpSomeClause::t.
+using TupleObjectListClauses = std::tuple<OmpClause::AdjustArgs,
+    OmpClause::Affinity, OmpClause::Aligned, OmpClause::Allocate,
+    OmpClause::Enter, OmpClause::From, OmpClause::InReduction,
+    OmpClause::Lastprivate, OmpClause::Linear, OmpClause::Map,
+    OmpClause::Reduction, OmpClause::TaskReduction, OmpClause::To>;
+
+// Does U have WrapperTrait (i.e. has a member 'v'), and if so, is T the
+// type of v?
+template <typename T, typename U, bool IsWrapper> struct WrappedInType {
+  static constexpr bool value{false};
+};
+
+template <typename T, typename U> struct WrappedInType<T, U, true> {
+  static constexpr bool value{std::is_same_v<T, decltype(U::v)>};
+};
+
+// Same as WrappedInType, but with a list of types Us. Satisfied if any
+// type U in Us satisfies WrappedInType<T, U>.
+template <typename...> struct WrappedInTypes;
+
+template <typename T> struct WrappedInTypes<T> {
+  static constexpr bool value{false};
+};
+
+template <typename T, typename U, typename... Us>
+struct WrappedInTypes<T, U, Us...> {
+  static constexpr bool value{WrappedInType<T, U, WrapperTrait<U>>::value ||
+      WrappedInTypes<T, Us...>::value};
+};
+
+// Same as WrappedInTypes, but takes type list in a form of a tuple or
+// a variant.
+template <typename...> struct WrappedInTupleOrVariant {
+  static constexpr bool value{false};
+};
+template <typename T, typename... Us>
+struct WrappedInTupleOrVariant<T, std::tuple<Us...>> {
+  static constexpr bool value{WrappedInTypes<T, Us...>::value};
+};
+template <typename T, typename... Us>
+struct WrappedInTupleOrVariant<T, std::variant<Us...>> {
+  static constexpr bool value{WrappedInTypes<T, Us...>::value};
+};
+template <typename T, typename U>
+constexpr bool WrappedInTupleOrVariantV{WrappedInTupleOrVariant<T, U>::value};
+} // namespace detail
+
+template <typename T> const OmpObjectList *GetOmpObjectList(const T &clause) {
+  using namespace detail;
+  static_assert(std::is_class_v<T>, "Unexpected argument type");
+
+  if constexpr (common::HasMember<T, decltype(OmpClause::u)>) {
+    if constexpr (common::HasMember<T, MemberObjectListClauses>) {
+      return &clause.v;
+    } else if constexpr (common::HasMember<T, TupleObjectListClauses>) {
+      return &std::get<OmpObjectList>(clause.v.t);
+    } else {
+      return nullptr;
+    }
+  } else if constexpr (WrappedInTupleOrVariantV<T, TupleObjectListClauses>) {
+    return &std::get<OmpObjectList>(clause.t);
+  } else if constexpr (WrappedInTupleOrVariantV<T, decltype(OmpClause::u)>) {
+    return nullptr;
+  } else {
+    // The condition should be type-dependent, but it should always be false.
+    static_assert(sizeof(T) < 0 && "Unexpected argument type");
+  }
+}
+
 const OmpObjectList *GetOmpObjectList(const OmpClause &clause);
+const OmpObjectList *GetOmpObjectList(const OmpClause::Depend &clause);
+const OmpObjectList *GetOmpObjectList(const OmpDependClause::TaskDep &x);
 
 template <typename T>
 const T *GetFirstArgument(const OmpDirectiveSpecification &spec) {
diff --git a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
index 187caa6043ac8..e7e71a9f0bcb3 100644
--- a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
+++ b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
@@ -126,6 +126,7 @@ struct IterWhileConversion : public mlir::OpRewritePattern<fir::IterWhileOp> {
 
     mlir::Value okInit = iterWhileOp.getIterateIn();
     mlir::ValueRange iterArgs = iterWhileOp.getInitArgs();
+    bool hasFinalValue = iterWhileOp.getFinalValue().has_value();
 
     mlir::SmallVector<mlir::Value> initVals;
     initVals.push_back(lowerBound);
@@ -164,17 +165,22 @@ struct IterWhileConversion : public mlir::OpRewritePattern<fir::IterWhileOp> {
 
     auto *afterBody = scfWhileOp.getAfterBody();
     auto resultOp = mlir::cast<fir::ResultOp>(afterBody->getTerminator());
-    mlir::SmallVector<mlir::Value> results(resultOp->getOperands());
-    mlir::Value ivInAfter = scfWhileOp.getAfterArguments()[0];
+    mlir::SmallVector<mlir::Value> results;
+    mlir::Value iv = scfWhileOp.getAfterArguments()[0];
 
     rewriter.setInsertionPointToStart(afterBody);
-    results[0] = mlir::arith::AddIOp::create(rewriter, loc, ivInAfter, step);
+    results.push_back(mlir::arith::AddIOp::create(rewriter, loc, iv, step));
+    llvm::append_range(results, hasFinalValue
+                                    ? resultOp->getOperands().drop_front()
+                                    : resultOp->getOperands());
 
     rewriter.setInsertionPointToEnd(afterBody);
     rewriter.replaceOpWithNewOp<mlir::scf::YieldOp>(resultOp, results);
 
     scfWhileOp->setAttrs(iterWhileOp->getAttrs());
-    rewriter.replaceOp(iterWhileOp, scfWhileOp);
+    rewriter.replaceOp(iterWhileOp,
+                       hasFinalValue ? scfWhileOp->getResults()
+                                     : scfWhileOp->getResults().drop_front());
     return mlir::success();
   }
 };
diff --git a/flang/lib/Parser/openmp-utils.cpp b/flang/lib/Parser/openmp-utils.cpp
index ab2ed0641f4c7..506442015aae2 100644
--- a/flang/lib/Parser/openmp-utils.cpp
+++ b/flang/lib/Parser/openmp-utils.cpp
@@ -117,43 +117,20 @@ std::optional<Label> GetFinalLabel(const OpenMPConstruct &x) {
 }
 
 const OmpObjectList *GetOmpObjectList(const OmpClause &clause) {
-  // Clauses with OmpObjectList as its data member
-  using MemberObjectListClauses = std::tuple<OmpClause::Copyin,
-      OmpClause::Copyprivate, OmpClause::Exclusive, OmpClause::Firstprivate,
-      OmpClause::HasDeviceAddr, OmpClause::Inclusive, OmpClause::IsDevicePtr,
-      OmpClause::Link, OmpClause::Private, OmpClause::Shared,
-      OmpClause::UseDeviceAddr, OmpClause::UseDevicePtr>;
-
-  // Clauses with OmpObjectList in the tuple
-  using TupleObjectListClauses = std::tuple<OmpClause::AdjustArgs,
-      OmpClause::Affinity, OmpClause::Aligned, OmpClause::Allocate,
-      OmpClause::Enter, OmpClause::From, OmpClause::InReduction,
-      OmpClause::Lastprivate, OmpClause::Linear, OmpClause::Map,
-      OmpClause::Reduction, OmpClause::TaskReduction, OmpClause::To>;
-
-  // TODO:: Generate the tuples using TableGen.
+  return common::visit([](auto &&s) { return GetOmpObjectList(s); }, clause.u);
+}
+
+const OmpObjectList *GetOmpObjectList(const OmpClause::Depend &clause) {
   return common::visit(
       common::visitors{
-          [&](const OmpClause::Depend &x) -> const OmpObjectList * {
-            if (auto *taskDep{std::get_if<OmpDependClause::TaskDep>(&x.v.u)}) {
-              return &std::get<OmpObjectList>(taskDep->t);
-            } else {
-              return nullptr;
-            }
-          },
-          [&](const auto &x) -> const OmpObjectList * {
-            using Ty = std::decay_t<decltype(x)>;
-            if constexpr (common::HasMember<Ty, MemberObjectListClauses>) {
-              return &x.v;
-            } else if constexpr (common::HasMember<Ty,
-                                     TupleObjectListClauses>) {
-              return &std::get<OmpObjectList>(x.v.t);
-            } else {
-              return nullptr;
-            }
-          },
+          [](const OmpDoacross &) { return nullptr; },
+          [](const OmpDependClause::TaskDep &x) { return GetOmpObjectList(x); },
       },
-      clause.u);
+      clause.v.u);
+}
+
+const OmpObjectList *GetOmpObjectList(const OmpDependClause::TaskDep &x) {
+  return &std::get<OmpObjectList>(x.t);
 }
 
 const BlockConstruct *GetFortranBlockConstruct(
diff --git a/flang/lib/Semantics/check-omp-loop.cpp b/flang/lib/Semantics/check-omp-loop.cpp
index 526e2d4658459..46ae3adaa6a68 100644
--- a/flang/lib/Semantics/check-omp-loop.cpp
+++ b/flang/lib/Semantics/check-omp-loop.cpp
@@ -480,9 +480,8 @@ void OmpStructureChecker::CheckDistLinear(
 
   // Collect symbols of all the variables from linear clauses
   for (auto &clause : clauses.v) {
-    if (auto *linearClause{std::get_if<parser::OmpClause::Linear>(&clause.u)}) {
-      auto &objects{std::get<parser::OmpObjectList>(linearClause->v.t)};
-      GetSymbolsInObjectList(objects, indexVars);
+    if (std::get_if<parser::OmpClause::Linear>(&clause.u)) {
+      GetSymbolsInObjectList(*parser::omp::GetOmpObjectList(clause), indexVars);
     }
   }
 
@@ -604,8 +603,6 @@ void OmpStructureChecker::Leave(const parser::OpenMPLoopConstruct &x) {
       auto *maybeModifier{OmpGetUniqueModifier<ReductionModifier>(modifiers)};
       if (maybeModifier &&
           maybeModifier->v == ReductionModifier::Value::Inscan) {
-        const auto &objectList{
-            std::get<parser::OmpObjectList>(reductionClause->v.t)};
         auto checkReductionSymbolInScan = [&](const parser::Name *name) {
           if (auto &symbol = name->symbol) {
             if (!symbol->test(Symbol::Flag::OmpInclusiveScan) &&
@@ -618,7 +615,7 @@ void OmpStructureChecker::Leave(const parser::OpenMPLoopConstruct &x) {
             }
           }
         };
-        for (const auto &ompObj : objectList.v) {
+        for (const auto &ompObj : parser::omp::GetOmpObjectList(clause)->v) {
           common::visit(
               common::visitors{
                   [&](const parser::Designator &designator) {
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 467e54719c283..dfba9e344a046 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -624,11 +624,9 @@ void OmpStructureChecker::CheckMultListItems() {
 
   // Linear clause
   for (auto [_, clause] : FindClauses(llvm::omp::Clause::OMPC_linear)) {
-    auto &linearClause{std::get<parser::OmpClause::Linear>(clause->u)};
     std::list<parser::Name> nameList;
     SymbolSourceMap symbols;
-    GetSymbolsInObjectList(
-        std::get<parser::OmpObjectList>(linearClause.v.t), symbols);
+    GetSymbolsInObjectList(*GetOmpObjectList(*clause), symbols);
     llvm::transform(symbols, std::back_inserter(nameList), [&](auto &&pair) {
       return parser::Name{pair.second, const_cast<Symbol *>(pair.first)};
     });
@@ -2117,29 +2115,29 @@ void OmpStructureChecker::Leave(const parser::OpenMPDeclareTargetConstruct &x) {
     }
   }
 
-  bool toClauseFound{false}, deviceTypeClauseFound{false},
-      enterClauseFound{false};
+  bool toClauseFound{false};
+  bool deviceTypeClauseFound{false};
+  bool enterClauseFound{false};
   for (const parser::OmpClause &clause : x.v.Clauses().v) {
     common::visit(
         common::visitors{
-            [&](const parser::OmpClause::To &toClause) {
-              toClauseFound = true;
-              auto &objList{std::get<parser::OmpObjectList>(toClause.v.t)};
-              CheckSymbolNames(dirName.source, objList);
-              CheckVarIsNotPartOfAnotherVar(dirName.source, objList);
-              CheckThreadprivateOrDeclareTargetVar(objList);
-            },
-            [&](const parser::OmpClause::Link &linkClause) {
-              CheckSymbolNames(dirName.source, linkClause.v);
-              CheckVarIsNotPartOfAnotherVar(dirName.source, linkClause.v);
-              CheckThreadprivateOrDeclareTargetVar(linkClause.v);
-            },
-            [&](const parser::OmpClause::Enter &enterClause) {
-              enterClauseFound = true;
-              auto &objList{std::get<parser::OmpObjectList>(enterClause.v.t)};
-              CheckSymbolNames(dirName.source, objList);
-              CheckVarIsNotPartOfAnotherVar(dirName.source, objList);
-              CheckThreadprivateOrDeclareTargetVar(objList);
+            [&](const auto &c) {
+              using TypeC = llvm::remove_cvref_t<decltype(c)>;
+              if constexpr ( //
+                  std::is_same_v<TypeC, parser::OmpClause::Enter> ||
+                  std::is_same_v<TypeC, parser::OmpClause::Link> ||
+                  std::is_same_v<TypeC, parser::OmpClause::To>) {
+                auto &objList{*GetOmpObjectList(c)};
+                CheckSymbolNames(dirName.source, objList);
+                CheckVarIsNotPartOfAnotherVar(dirName.source, objList);
+                CheckThreadprivateOrDeclareTargetVar(objList);
+              }
+              if constexpr (std::is_same_v<TypeC, parser::OmpClause::Enter>) {
+                enterClauseFound = true;
+              }
+              if constexpr (std::is_same_v<TypeC, parser::OmpClause::To>) {
+                toClauseFound = true;
+              }
             },
             [&](const parser::OmpClause::DeviceType &deviceTypeClause) {
               deviceTypeClauseFound = true;
@@ -2150,7 +2148,6 @@ void OmpStructureChecker::Leave(const parser::OpenMPDeclareTargetConstruct &x) {
                 deviceConstructFound_ = true;
               }
             },
-            [&](const auto &) {},
         },
         clause.u);
 
@@ -2440,12 +2437,8 @@ void OmpStructureChecker::CheckTargetUpdate() {
   }
   if (toWrapper && fromWrapper) {
     SymbolSourceMap toSymbols, fromSymbols;
-    auto &fromClause{std::get<parser::OmpClause::From>(fromWrapper->u).v};
-    auto &toClause{std::get<parser::OmpClause::To>(toWrapper->u).v};
-    GetSymbolsInObjectList(
-        std::get<parser::OmpObjectList>(fromClause.t), fromSymbols);
-    GetSymbolsInObjectList(
-        std::get<parser::OmpObjectList>(toClause.t), toSymbols);
+    GetSymbolsInObjectList(*GetOmpObjectList(*fromWrapper), fromSymbols);
+    GetSymbolsInObjectList(*GetOmpObjectList(*toWrapper), toSymbols);
 
     for (auto &[symbol, source] : toSymbols) {
       auto fromSymbol{fromSymbols.find(symbol)};
@@ -3285,7 +3278,7 @@ void OmpStructureChecker::Leave(const parser::OmpClauseList &) {
             const auto &irClause{
                 std::get<parser::OmpClause::InReduction>(dataEnvClause->u)};
             checkVarAppearsInDataEnvClause(
-                std::get<parser::OmpObjectList>(irClause.v.t), "IN_REDUCTION");
+                *GetOmpObjectList(irClause), "IN_REDUCTION");
           }
         }
       }
@@ -3437,7 +3430,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Destroy &x) {
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Reduction &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_reduction);
-  auto &objects{std::get<parser::OmpObjectList>(x.v.t)};
+  auto &objects{*GetOmpObjectList(x)};
 
   if (OmpVerifyModifiers(x.v, llvm::omp::OMPC_reduction,
           GetContext().clauseSource, context_)) {
@@ -3477,7 +3470,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Reduction &x) {
 
 void OmpStructureChecker::Enter(const parser::OmpClause::InReduction &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_in_reduction);
-  auto &objects{std::get<parser::OmpObjectList>(x.v.t)};
+  auto &objects{*GetOmpObjectList(x)};
 
   if (OmpVerifyModifiers(x.v, llvm::omp::OMPC_in_reduction,
           GetContext().clauseSource, context_)) {
@@ -3495,7 +3488,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::InReduction &x) {
 
 void OmpStructureChecker::Enter(const parser::OmpClause::TaskReduction &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_task_reduction);
-  auto &objects{std::get<parser::OmpObjectList>(x.v.t)};
+  auto &objects{*GetOmpObjectList(x)};
 
   if (OmpVerifyModifiers(x.v, llvm::omp::OMPC_task_reduction,
           GetContext().clauseSource, context_)) {
@@ -4348,8 +4341,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Map &x) {
     }};
 
     evaluate::ExpressionAnalyzer ea{context_};
-    const auto &objects{std::get<parser::OmpObjectList>(x.v.t)};
-    for (auto &object : objects.v) {
+    for (auto &object : GetOmpObjectList(x)->v) {
       if (const parser::Designator *d{GetDesignatorFromObj(object)}) {
         if (auto &&expr{ea.Analyze(*d)}) {
           if (hasBasePointer(*expr)) {
@@ -4502,7 +4494,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Depend &x) {
     }
   }
   if (taskDep) {
-    auto &objList{std::get<parser::OmpObjectList>(taskDep->t)};
+    auto &objList{*GetOmpObjectList(*taskDep)};
     if (dir == llvm::omp::OMPD_depobj) {
       // [5.0:255:13], [5.1:288:6], [5.2:322:26]
       // A depend clause on a depobj construct must only specify one locator.
@@ -4648,7 +4640,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Copyprivate &x) {
 void OmpStructureChecker::Enter(const parser::OmpClause::Lastprivate &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_lastprivate);
 
-  const auto &objectList{std::get<parser::OmpObjectList>(x.v.t)};
+  const auto &objectList{*GetOmpObjectList(x)};
   CheckVarIsNotPartOfAnotherVar(
       GetContext().clauseSource, objectList, "LASTPRIVATE");
   CheckCrayPointee(objectList, "LASTPRIVATE");
@@ -4890,9 +4882,8 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Enter &x) {
           x.v, llvm::omp::OMPC_enter, GetContext().clauseSource, context_)) {
     return;
   }
-  const parser::OmpObjectList &objList{std::get<parser::OmpObjectList>(x.v.t)};
   SymbolSourceMap symbols;
-  GetSymbolsInObjectList(objList, symbols);
+  GetSymbolsInObjectList(*GetOmpObjectList(x), symbols);
   for (const auto &[symbol, source] : symbols) {
     if (!IsExtendedListItem(*symbol)) {
       context_.SayWithDecl(*symbol, source,
@@ -4915,7 +4906,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::From &x) {
     CheckIteratorModifier(*iter);
   }
 
-  const auto &objList{std::get<parser::OmpObjectList>(x.v.t)};
+  const auto &objList{*GetOmpObjectList(x)};
   SymbolSourceMap symbols;
   GetSymbolsInObjectList(objList, symbols);
   CheckVariableListItem(symbols);
@@ -4955,7 +4946,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::To &x) {
     CheckIteratorModifier(*iter);
   }
 
-  const auto &objList{std::get<parser::OmpObjectList>(x.v.t)};
+  const auto &objList{*GetOmpObjectList(x)};
   SymbolSourceMap symbols;
   GetSymbolsInObjectList(objList, symbols);
   CheckVariableListItem(symbols);
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 8d08356dc6f14..5c9d03d9e5d67 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -717,8 +717,8 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     return false;
   }
   bool Pre(const parser::OmpAllocateClause &x) {
-    const auto &objectList{std::get<parser::OmpObjectList>(x.t)};
-    ResolveOmpObjectList(objectList, Symbol::Flag::OmpAllocate);
+    ResolveOmpObjectList(
+        *parser::omp::GetOmpObjectList(x), Symbol::Flag::OmpAllocate);
     return false;
   }
   bool Pre(const parser::OmpClause::Firstprivate &x) {
@@ -726,8 +726,8 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     return false;
   }
   bool Pre(const parser::OmpClause::Lastprivate &x) {
-    const auto &objList{std::get<parser::OmpObjectList>(x.v.t)};
-    ResolveOmpObjectList(objList, Symbol::Flag::OmpLastPrivate);
+    ResolveOmpObjectList(
+        *parser::omp::GetOmpObjectList(x), Symbol::Flag::OmpLastPrivate);
     return false;
   }
   bool Pre(const parser::OmpClause::Copyin &x) {
@@ -739,8 +739,8 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     return false;
   }
   bool Pre(const parser::OmpLinearClause &x) {
-    auto &objects{std::get<parser::OmpObjectList>(x.t)};
-    ResolveOmpObjectList(objects, Symbol::Flag::OmpLinear);
+    ResolveOmpObjectList(
+        *parser::omp::GetOmpObjectList(x), Symbol::Flag::OmpLinear);
     return false;
   }
 
@@ -750,13 +750,13 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
   }
 
   bool Pre(const parser::OmpInReductionClause &x) {
-    auto &objects{std::get<parser::OmpObjectList>(x.t)};
-    ResolveOmpObjectList(objects, Symbol::Flag::OmpInReduction);
+    ResolveOmpObjectList(
+        *parser::omp::GetOmpObjectList(x), Symbol::Flag::OmpInReduction);
     return false;
   }
 
   bool Pre(const parser::OmpClause::Reduction &x) {
-    const auto &objList{std::get<parser::OmpObjectList>(x.v.t)};
+    const auto &objList{*parser::omp::GetOmpObjectList(x)};
     ResolveOmpObjectList(objList, Symbol::Flag::OmpReduction);
 
     if (auto &modifiers{OmpGetModifiers(x.v)}) {
@@ -806,8 +806,8 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
   }
 
   bool Pre(const parser::OmpAlignedClause &x) {
-    const auto &alignedNameList{std::get<parser::OmpObjectList>(x.t)};
-    ResolveOmpObjectList(alignedNameList, Symbol::Flag::OmpAligned);
+    ResolveOmpObjectList(
+        *parser::omp::GetOmpObjectList(x), Symbol::Flag::OmpAligned);
     return false;
   }
 
@@ -920,7 +920,7 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
       }
     }
 
-    const auto &ompObjList{std::get<parser::OmpObjectList>(x.t)};
+    const auto &ompObjList{*parser::omp::GetOmpObjectList(x)};
     for (const auto &ompObj : ompObjList.v) {
       common::visit(
           common::visitors{
@@ -2566,9 +2566,8 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPAllocatorsConstruct &x) {
   PushContext(x.source, dirSpec.DirId());
 
   for (const auto &clause : dirSpec.Clauses().v) {
-    if (const auto *allocClause{
-            std::get_if<parser::OmpClause::Allocate>(&clause.u)}) {
-      ResolveOmpObjectList(std::get<parser::OmpObjectList>(allocClause->v.t),
+    if (std::get_if<parser::OmpClause::Allocate>(&clause.u)) {
+      ResolveOmpObjectList(*parser::omp::GetOmpObjectList(clause),
           Symbol::Flag::OmpExecutableAllocateDirective);
     }
   }
diff --git a/flang/test/Fir/FirToSCF/iter-while.fir b/flang/test/Fir/FirToSCF/iter-while.fir
index 0de7aabed120e..19cfaac1cc460 100644
--- a/flang/test/Fir/FirToSCF/iter-while.fir
+++ b/flang/test/Fir/FirToSCF/iter-while.fir
@@ -1,4 +1,4 @@
-// RUN: fir-opt %s --fir-to-scf | FileCheck %s
+// RUN: fir-opt %s --fir-to-scf --allow-unregistered-dialect | FileCheck %s
 
 // CHECK-LABEL:   func.func @test_simple_iterate_while_1() -> (index, i1, i16, i32) {
 // CHECK:           %[[VAL_0:.*]] = arith.constant 11 : index
@@ -97,3 +97,30 @@ func.func @test_zero_iterations() -> (index, i1, i8) {
 
   return %res#0, %res#1, %res#2 : index, i1, i8
 }
+
+// CHECK-LABEL:   func.func @test_without_final_value(
+// CHECK-SAME:      %[[ARG0:.*]]: index,
+// CHECK-SAME:      %[[ARG1:.*]]: index) -> i1 {
+// CHECK:           %[[CONSTANT_0:.*]] = arith.constant 1 : index
+// CHECK:           %[[CONSTANT_1:.*]] = arith.constant true
+// CHECK:           %[[WHILE_0:.*]]:2 = scf.while (%[[VAL_0:.*]] = %[[ARG0]], %[[VAL_1:.*]] = %[[CONSTANT_1]]) : (index, i1) -> (index, i1) {
+// CHECK:             %[[CMPI_0:.*]] = arith.cmpi sle, %[[VAL_0]], %[[ARG1]] : index
+// CHECK:             %[[ANDI_0:.*]] = arith.andi %[[CMPI_0]], %[[VAL_1]] : i1
+// CHECK:             scf.condition(%[[ANDI_0]]) %[[VAL_0]], %[[VAL_1]] : index, i1
+// CHECK:           } do {
+// CHECK:           ^bb0(%[[VAL_2:.*]]: index, %[[VAL_3:.*]]: i1):
+// CHECK:             %[[ADDI_0:.*]] = arith.addi %[[VAL_2]], %[[CONSTANT_0]] : index
+// CHECK:             %[[VAL_4:.*]] = "test.get_some_value"() : () -> i1
+// CHECK:             scf.yield %[[ADDI_0]], %[[VAL_4]] : index, i1
+// CHECK:           }
+// CHECK:           return %[[VAL_5:.*]]#1 : i1
+// CHECK:         }
+func.func @test_without_final_value(%lo : index, %up : index) -> i1 {
+  %c1 = arith.constant 1 : index
+  %ok1 = arith.constant true
+  %ok2 = fir.iterate_while (%i = %lo to %up step %c1) and (%j = %ok1) {
+    %ok = "test.get_some_value"() : () -> i1
+    fir.result %ok : i1
+  }
+  return %ok2 : i1
+}
diff --git a/libc/config/config.json b/libc/config/config.json
index cfbe9a43948ea..a7844e4fe2dd1 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -130,5 +130,11 @@
       "value": true,
       "doc": "Add nullptr checks in the library's implementations to some functions for which passing nullptr is undefined behavior."
     }
+  },
+  "wctype": {
+    "LIBC_CONF_WCTYPE_MODE": {
+      "value": "LIBC_WCTYPE_MODE_ASCII",
+      "doc": "The implementation used for wctype, acceptable values are LIBC_WCTYPE_MODE_ASCII and LIBC_WCTYPE_MODE_UTF8."
+    }
   }
 }
diff --git a/libc/docs/configure.rst b/libc/docs/configure.rst
index e23fc824ce7c8..362e293a4b714 100644
--- a/libc/docs/configure.rst
+++ b/libc/docs/configure.rst
@@ -64,3 +64,5 @@ to learn about the defaults for your platform and target.
     - ``LIBC_CONF_THREAD_MODE``: The implementation used for Mutex, acceptable values are LIBC_THREAD_MODE_PLATFORM, LIBC_THREAD_MODE_SINGLE, and LIBC_THREAD_MODE_EXTERNAL.
 * **"time" options**
     - ``LIBC_CONF_TIME_64BIT``: Force the size of time_t to 64 bits, even on platforms where compatibility considerations would otherwise make it 32-bit.
+* **"wctype" options**
+    - ``LIBC_CONF_WCTYPE_MODE``: The implementation used for wctype, acceptable values are LIBC_WCTYPE_MODE_ASCII and LIBC_WCTYPE_MODE_UTF8.
diff --git a/libc/shared/math.h b/libc/shared/math.h
index 282dd6243d6a7..2db1d5501b7b3 100644
--- a/libc/shared/math.h
+++ b/libc/shared/math.h
@@ -54,6 +54,7 @@
 #include "math/exp2m1f16.h"
 #include "math/expf.h"
 #include "math/expf16.h"
+#include "math/expm1.h"
 #include "math/frexpf.h"
 #include "math/frexpf128.h"
 #include "math/frexpf16.h"
diff --git a/libc/shared/math/expm1.h b/libc/shared/math/expm1.h
new file mode 100644
index 0000000000000..4c8dbdc013a11
--- /dev/null
+++ b/libc/shared/math/expm1.h
@@ -0,0 +1,23 @@
+//===-- Shared expm1 function -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_MATH_EXPM1_H
+#define LLVM_LIBC_SHARED_MATH_EXPM1_H
+
+#include "shared/libc_common.h"
+#include "src/__support/math/expm1.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace shared {
+
+using math::expm1;
+
+} // namespace shared
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SHARED_MATH_EXPM1_H
diff --git a/libc/src/__support/math/CMakeLists.txt b/libc/src/__support/math/CMakeLists.txt
index 13163ec0fd252..c36b0c0c216e9 100644
--- a/libc/src/__support/math/CMakeLists.txt
+++ b/libc/src/__support/math/CMakeLists.txt
@@ -871,6 +871,26 @@ add_header_library(
     libc.src.__support.macros.properties.cpu_features
 )
 
+add_header_library(
+  expm1
+  HDRS
+    expm1.h
+  DEPENDS
+    .common_constants
+    .exp_constants
+    libc.src.__support.CPP.bit
+    libc.src.__support.FPUtil.dyadic_float
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.FPUtil.polyeval
+    libc.src.__support.FPUtil.rounding_mode
+    libc.src.__support.FPUtil.triple_double
+    libc.src.__support.integer_literals
+    libc.src.__support.macros.optimization
+    libc.src.errno.errno
+)
+
 add_header_library(
   range_reduction_double
   HDRS
diff --git a/libc/src/__support/math/expm1.h b/libc/src/__support/math/expm1.h
new file mode 100644
index 0000000000000..4bbb20ffbf7a1
--- /dev/null
+++ b/libc/src/__support/math/expm1.h
@@ -0,0 +1,518 @@
+//===-- Implementation header for expm1 -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_EXPM1_H
+#define LLVM_LIBC_SRC___SUPPORT_MATH_EXPM1_H
+
+#include "common_constants.h" // Lookup tables EXP_M1 and EXP_M2.
+#include "exp_constants.h"
+#include "src/__support/CPP/bit.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/PolyEval.h"
+#include "src/__support/FPUtil/double_double.h"
+#include "src/__support/FPUtil/dyadic_float.h"
+#include "src/__support/FPUtil/except_value_utils.h"
+#include "src/__support/FPUtil/multiply_add.h"
+#include "src/__support/FPUtil/rounding_mode.h"
+#include "src/__support/FPUtil/triple_double.h"
+#include "src/__support/common.h"
+#include "src/__support/integer_literals.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace math {
+
+namespace expm1_internal {
+
+#if ((LIBC_MATH & LIBC_MATH_SKIP_ACCURATE_PASS) != 0)
+#define LIBC_MATH_EXPM1_SKIP_ACCURATE_PASS
+#endif
+
+using fputil::DoubleDouble;
+using fputil::TripleDouble;
+using Float128 = typename fputil::DyadicFloat<128>;
+
+using LIBC_NAMESPACE::operator""_u128;
+
+// log2(e)
+static constexpr double LOG2_E = 0x1.71547652b82fep+0;
+
+// Error bounds:
+// Errors when using double precision.
+// 0x1.8p-63;
+static constexpr uint64_t ERR_D = 0x3c08000000000000;
+// Errors when using double-double precision.
+// 0x1.0p-99
+[[maybe_unused]] static constexpr uint64_t ERR_DD = 0x39c0000000000000;
+
+// -2^-12 * log(2)
+// > a = -2^-12 * log(2);
+// > b = round(a, 30, RN);
+// > c = round(a - b, 30, RN);
+// > d = round(a - b - c, D, RN);
+// Errors < 1.5 * 2^-133
+static constexpr double MLOG_2_EXP2_M12_HI = -0x1.62e42ffp-13;
+static constexpr double MLOG_2_EXP2_M12_MID = 0x1.718432a1b0e26p-47;
+static constexpr double MLOG_2_EXP2_M12_MID_30 = 0x1.718432ap-47;
+static constexpr double MLOG_2_EXP2_M12_LO = 0x1.b0e2633fe0685p-79;
+
+using namespace common_constants_internal;
+
+// Polynomial approximations with double precision:
+// Return expm1(dx) / x ~ 1 + dx / 2 + dx^2 / 6 + dx^3 / 24.
+// For |dx| < 2^-13 + 2^-30:
+//   | output - expm1(dx) / dx | < 2^-51.
+LIBC_INLINE static double poly_approx_d(double dx) {
+  // dx^2
+  double dx2 = dx * dx;
+  // c0 = 1 + dx / 2
+  double c0 = fputil::multiply_add(dx, 0.5, 1.0);
+  // c1 = 1/6 + dx / 24
+  double c1 =
+      fputil::multiply_add(dx, 0x1.5555555555555p-5, 0x1.5555555555555p-3);
+  // p = dx^2 * c1 + c0 = 1 + dx / 2 + dx^2 / 6 + dx^3 / 24
+  double p = fputil::multiply_add(dx2, c1, c0);
+  return p;
+}
+
+// Polynomial approximation with double-double precision:
+// Return expm1(dx) / dx ~ 1 + dx / 2 + dx^2 / 6 + ... + dx^6 / 5040
+// For |dx| < 2^-13 + 2^-30:
+//   | output - expm1(dx) | < 2^-101
+LIBC_INLINE static constexpr DoubleDouble
+poly_approx_dd(const DoubleDouble &dx) {
+  // Taylor polynomial.
+  constexpr DoubleDouble COEFFS[] = {
+      {0, 0x1p0},                                      // 1
+      {0, 0x1p-1},                                     // 1/2
+      {0x1.5555555555555p-57, 0x1.5555555555555p-3},   // 1/6
+      {0x1.5555555555555p-59, 0x1.5555555555555p-5},   // 1/24
+      {0x1.1111111111111p-63, 0x1.1111111111111p-7},   // 1/120
+      {-0x1.f49f49f49f49fp-65, 0x1.6c16c16c16c17p-10}, // 1/720
+      {0x1.a01a01a01a01ap-73, 0x1.a01a01a01a01ap-13},  // 1/5040
+  };
+
+  DoubleDouble p = fputil::polyeval(dx, COEFFS[0], COEFFS[1], COEFFS[2],
+                                    COEFFS[3], COEFFS[4], COEFFS[5], COEFFS[6]);
+  return p;
+}
+
+// Polynomial approximation with 128-bit precision:
+// Return (exp(dx) - 1)/dx ~ 1 + dx / 2 + dx^2 / 6 + ... + dx^6 / 5040
+// For |dx| < 2^-13 + 2^-30:
+//   | output - exp(dx) | < 2^-126.
+[[maybe_unused]] LIBC_INLINE static constexpr Float128
+poly_approx_f128(const Float128 &dx) {
+  constexpr Float128 COEFFS_128[]{
+      {Sign::POS, -127, 0x80000000'00000000'00000000'00000000_u128}, // 1.0
+      {Sign::POS, -128, 0x80000000'00000000'00000000'00000000_u128}, // 0.5
+      {Sign::POS, -130, 0xaaaaaaaa'aaaaaaaa'aaaaaaaa'aaaaaaab_u128}, // 1/6
+      {Sign::POS, -132, 0xaaaaaaaa'aaaaaaaa'aaaaaaaa'aaaaaaab_u128}, // 1/24
+      {Sign::POS, -134, 0x88888888'88888888'88888888'88888889_u128}, // 1/120
+      {Sign::POS, -137, 0xb60b60b6'0b60b60b'60b60b60'b60b60b6_u128}, // 1/720
+      {Sign::POS, -140, 0xd00d00d0'0d00d00d'00d00d00'd00d00d0_u128}, // 1/5040
+  };
+
+  Float128 p = fputil::polyeval(dx, COEFFS_128[0], COEFFS_128[1], COEFFS_128[2],
+                                COEFFS_128[3], COEFFS_128[4], COEFFS_128[5],
+                                COEFFS_128[6]);
+  return p;
+}
+
+#ifdef DEBUGDEBUG
+std::ostream &operator<<(std::ostream &OS, const Float128 &r) {
+  OS << (r.sign == Sign::NEG ? "-(" : "(") << r.mantissa.val[0] << " + "
+     << r.mantissa.val[1] << " * 2^64) * 2^" << r.exponent << "\n";
+  return OS;
+}
+
+std::ostream &operator<<(std::ostream &OS, const DoubleDouble &r) {
+  OS << std::hexfloat << "(" << r.hi << " + " << r.lo << ")"
+     << std::defaultfloat << "\n";
+  return OS;
+}
+#endif
+
+// Compute exp(x) - 1 using 128-bit precision.
+// TODO(lntue): investigate triple-double precision implementation for this
+// step.
+[[maybe_unused]] LIBC_INLINE static Float128 expm1_f128(double x, double kd,
+                                                        int idx1, int idx2) {
+  // Recalculate dx:
+
+  double t1 = fputil::multiply_add(kd, MLOG_2_EXP2_M12_HI, x); // exact
+  double t2 = kd * MLOG_2_EXP2_M12_MID_30;                     // exact
+  double t3 = kd * MLOG_2_EXP2_M12_LO;                         // Error < 2^-133
+
+  Float128 dx = fputil::quick_add(
+      Float128(t1), fputil::quick_add(Float128(t2), Float128(t3)));
+
+  // TODO: Skip recalculating exp_mid1 and exp_mid2.
+  Float128 exp_mid1 =
+      fputil::quick_add(Float128(EXP2_MID1[idx1].hi),
+                        fputil::quick_add(Float128(EXP2_MID1[idx1].mid),
+                                          Float128(EXP2_MID1[idx1].lo)));
+
+  Float128 exp_mid2 =
+      fputil::quick_add(Float128(EXP2_MID2[idx2].hi),
+                        fputil::quick_add(Float128(EXP2_MID2[idx2].mid),
+                                          Float128(EXP2_MID2[idx2].lo)));
+
+  Float128 exp_mid = fputil::quick_mul(exp_mid1, exp_mid2);
+
+  int hi = static_cast<int>(kd) >> 12;
+  Float128 minus_one{Sign::NEG, -127 - hi,
+                     0x80000000'00000000'00000000'00000000_u128};
+
+  Float128 exp_mid_m1 = fputil::quick_add(exp_mid, minus_one);
+
+  Float128 p = poly_approx_f128(dx);
+
+  // r = exp_mid * (1 + dx * P) - 1
+  //   = (exp_mid - 1) + (dx * exp_mid) * P
+  Float128 r =
+      fputil::multiply_add(fputil::quick_mul(exp_mid, dx), p, exp_mid_m1);
+
+  r.exponent += hi;
+
+#ifdef DEBUGDEBUG
+  std::cout << "=== VERY SLOW PASS ===\n"
+            << "        kd: " << kd << "\n"
+            << "        hi: " << hi << "\n"
+            << " minus_one: " << minus_one << "        dx: " << dx
+            << "exp_mid_m1: " << exp_mid_m1 << "   exp_mid: " << exp_mid
+            << "         p: " << p << "         r: " << r << std::endl;
+#endif
+
+  return r;
+}
+
+// Compute exp(x) - 1 with double-double precision.
+LIBC_INLINE static DoubleDouble exp_double_double(double x, double kd,
+                                                  const DoubleDouble &exp_mid,
+                                                  const DoubleDouble &hi_part) {
+  // Recalculate dx:
+  //   dx = x - k * 2^-12 * log(2)
+  double t1 = fputil::multiply_add(kd, MLOG_2_EXP2_M12_HI, x); // exact
+  double t2 = kd * MLOG_2_EXP2_M12_MID_30;                     // exact
+  double t3 = kd * MLOG_2_EXP2_M12_LO;                         // Error < 2^-130
+
+  DoubleDouble dx = fputil::exact_add(t1, t2);
+  dx.lo += t3;
+
+  // Degree-6 Taylor polynomial approximation in double-double precision.
+  // | p - exp(x) | < 2^-100.
+  DoubleDouble p = poly_approx_dd(dx);
+
+  // Error bounds: 2^-99.
+  DoubleDouble r =
+      fputil::multiply_add(fputil::quick_mult(exp_mid, dx), p, hi_part);
+
+#ifdef DEBUGDEBUG
+  std::cout << "=== SLOW PASS ===\n"
+            << "   dx: " << dx << "    p: " << p << "    r: " << r << std::endl;
+#endif
+
+  return r;
+}
+
+// Check for exceptional cases when
+// |x| <= 2^-53 or x < log(2^-54) or x >= 0x1.6232bdd7abcd3p+9
+LIBC_INLINE static constexpr double set_exceptional(double x) {
+  using FPBits = typename fputil::FPBits<double>;
+  FPBits xbits(x);
+
+  uint64_t x_u = xbits.uintval();
+  uint64_t x_abs = xbits.abs().uintval();
+
+  // |x| <= 2^-53.
+  if (x_abs <= 0x3ca0'0000'0000'0000ULL) {
+    // expm1(x) ~ x.
+
+    if (LIBC_UNLIKELY(x_abs <= 0x0370'0000'0000'0000ULL)) {
+      if (LIBC_UNLIKELY(x_abs == 0))
+        return x;
+      // |x| <= 2^-968, need to scale up a bit before rounding, then scale it
+      // back down.
+      return 0x1.0p-200 * fputil::multiply_add(x, 0x1.0p+200, 0x1.0p-1022);
+    }
+
+    // 2^-968 < |x| <= 2^-53.
+    return fputil::round_result_slightly_up(x);
+  }
+
+  // x < log(2^-54) || x >= 0x1.6232bdd7abcd3p+9 or inf/nan.
+
+  // x < log(2^-54) or -inf/nan
+  if (x_u >= 0xc042'b708'8723'20e2ULL) {
+    // expm1(-Inf) = -1
+    if (xbits.is_inf())
+      return -1.0;
+
+    // exp(nan) = nan
+    if (xbits.is_nan())
+      return x;
+
+    return fputil::round_result_slightly_up(-1.0);
+  }
+
+  // x >= round(log(MAX_NORMAL), D, RU) = 0x1.62e42fefa39fp+9 or +inf/nan
+  // x is finite
+  if (x_u < 0x7ff0'0000'0000'0000ULL) {
+    int rounding = fputil::quick_get_round();
+    if (rounding == FE_DOWNWARD || rounding == FE_TOWARDZERO)
+      return FPBits::max_normal().get_val();
+
+    fputil::set_errno_if_required(ERANGE);
+    fputil::raise_except_if_required(FE_OVERFLOW);
+  }
+  // x is +inf or nan
+  return x + FPBits::inf().get_val();
+}
+
+} // namespace expm1_internal
+
+LIBC_INLINE static constexpr double expm1(double x) {
+  using namespace expm1_internal;
+
+  using FPBits = typename fputil::FPBits<double>;
+
+  FPBits xbits(x);
+
+  bool x_is_neg = xbits.is_neg();
+  uint64_t x_u = xbits.uintval();
+
+  // Upper bound: max normal number = 2^1023 * (2 - 2^-52)
+  // > round(log (2^1023 ( 2 - 2^-52 )), D, RU) = 0x1.62e42fefa39fp+9
+  // > round(log (2^1023 ( 2 - 2^-52 )), D, RD) = 0x1.62e42fefa39efp+9
+  // > round(log (2^1023 ( 2 - 2^-52 )), D, RN) = 0x1.62e42fefa39efp+9
+  // > round(exp(0x1.62e42fefa39fp+9), D, RN) = infty
+
+  // Lower bound: log(2^-54) = -0x1.2b708872320e2p5
+  // > round(log(2^-54), D, RN) = -0x1.2b708872320e2p5
+
+  // x < log(2^-54) or x >= 0x1.6232bdd7abcd3p+9 or |x| <= 2^-53.
+
+  if (LIBC_UNLIKELY(x_u >= 0xc042b708872320e2 ||
+                    (x_u <= 0xbca0000000000000 && x_u >= 0x40862e42fefa39f0) ||
+                    x_u <= 0x3ca0000000000000)) {
+    return set_exceptional(x);
+  }
+
+  // Now log(2^-54) <= x <= -2^-53 or 2^-53 <= x < log(2^1023 * (2 - 2^-52))
+
+  // Range reduction:
+  // Let x = log(2) * (hi + mid1 + mid2) + lo
+  // in which:
+  //   hi is an integer
+  //   mid1 * 2^6 is an integer
+  //   mid2 * 2^12 is an integer
+  // then:
+  //   exp(x) = 2^hi * 2^(mid1) * 2^(mid2) * exp(lo).
+  // With this formula:
+  //   - multiplying by 2^hi is exact and cheap, simply by adding the exponent
+  //     field.
+  //   - 2^(mid1) and 2^(mid2) are stored in 2 x 64-element tables.
+  //   - exp(lo) ~ 1 + lo + a0 * lo^2 + ...
+  //
+  // They can be defined by:
+  //   hi + mid1 + mid2 = 2^(-12) * round(2^12 * log_2(e) * x)
+  // If we store L2E = round(log2(e), D, RN), then:
+  //   log2(e) - L2E ~ 1.5 * 2^(-56)
+  // So the errors when computing in double precision is:
+  //   | x * 2^12 * log_2(e) - D(x * 2^12 * L2E) | <=
+  //  <= | x * 2^12 * log_2(e) - x * 2^12 * L2E | +
+  //     + | x * 2^12 * L2E - D(x * 2^12 * L2E) |
+  //  <= 2^12 * ( |x| * 1.5 * 2^-56 + eps(x))  for RN
+  //     2^12 * ( |x| * 1.5 * 2^-56 + 2*eps(x)) for other rounding modes.
+  // So if:
+  //   hi + mid1 + mid2 = 2^(-12) * round(x * 2^12 * L2E) is computed entirely
+  // in double precision, the reduced argument:
+  //   lo = x - log(2) * (hi + mid1 + mid2) is bounded by:
+  //   |lo| <= 2^-13 + (|x| * 1.5 * 2^-56 + 2*eps(x))
+  //         < 2^-13 + (1.5 * 2^9 * 1.5 * 2^-56 + 2*2^(9 - 52))
+  //         < 2^-13 + 2^-41
+  //
+
+  // The following trick computes the round(x * L2E) more efficiently
+  // than using the rounding instructions, with the tradeoff for less accuracy,
+  // and hence a slightly larger range for the reduced argument `lo`.
+  //
+  // To be precise, since |x| < |log(2^-1075)| < 1.5 * 2^9,
+  //   |x * 2^12 * L2E| < 1.5 * 2^9 * 1.5 < 2^23,
+  // So we can fit the rounded result round(x * 2^12 * L2E) in int32_t.
+  // Thus, the goal is to be able to use an additional addition and fixed width
+  // shift to get an int32_t representing round(x * 2^12 * L2E).
+  //
+  // Assuming int32_t using 2-complement representation, since the mantissa part
+  // of a double precision is unsigned with the leading bit hidden, if we add an
+  // extra constant C = 2^e1 + 2^e2 with e1 > e2 >= 2^25 to the product, the
+  // part that are < 2^e2 in resulted mantissa of (x*2^12*L2E + C) can be
+  // considered as a proper 2-complement representations of x*2^12*L2E.
+  //
+  // One small problem with this approach is that the sum (x*2^12*L2E + C) in
+  // double precision is rounded to the least significant bit of the dorminant
+  // factor C.  In order to minimize the rounding errors from this addition, we
+  // want to minimize e1.  Another constraint that we want is that after
+  // shifting the mantissa so that the least significant bit of int32_t
+  // corresponds to the unit bit of (x*2^12*L2E), the sign is correct without
+  // any adjustment.  So combining these 2 requirements, we can choose
+  //   C = 2^33 + 2^32, so that the sign bit corresponds to 2^31 bit, and hence
+  // after right shifting the mantissa, the resulting int32_t has correct sign.
+  // With this choice of C, the number of mantissa bits we need to shift to the
+  // right is: 52 - 33 = 19.
+  //
+  // Moreover, since the integer right shifts are equivalent to rounding down,
+  // we can add an extra 0.5 so that it will become round-to-nearest, tie-to-
+  // +infinity.  So in particular, we can compute:
+  //   hmm = x * 2^12 * L2E + C,
+  // where C = 2^33 + 2^32 + 2^-1, then if
+  //   k = int32_t(lower 51 bits of double(x * 2^12 * L2E + C) >> 19),
+  // the reduced argument:
+  //   lo = x - log(2) * 2^-12 * k is bounded by:
+  //   |lo| <= 2^-13 + 2^-41 + 2^-12*2^-19
+  //         = 2^-13 + 2^-31 + 2^-41.
+  //
+  // Finally, notice that k only uses the mantissa of x * 2^12 * L2E, so the
+  // exponent 2^12 is not needed.  So we can simply define
+  //   C = 2^(33 - 12) + 2^(32 - 12) + 2^(-13 - 12), and
+  //   k = int32_t(lower 51 bits of double(x * L2E + C) >> 19).
+
+  // Rounding errors <= 2^-31 + 2^-41.
+  double tmp = fputil::multiply_add(x, LOG2_E, 0x1.8000'0000'4p21);
+  int k = static_cast<int>(cpp::bit_cast<uint64_t>(tmp) >> 19);
+  double kd = static_cast<double>(k);
+
+  uint32_t idx1 = (k >> 6) & 0x3f;
+  uint32_t idx2 = k & 0x3f;
+  int hi = k >> 12;
+
+  DoubleDouble exp_mid1{EXP2_MID1[idx1].mid, EXP2_MID1[idx1].hi};
+  DoubleDouble exp_mid2{EXP2_MID2[idx2].mid, EXP2_MID2[idx2].hi};
+
+  DoubleDouble exp_mid = fputil::quick_mult(exp_mid1, exp_mid2);
+
+  // -2^(-hi)
+  double one_scaled =
+      FPBits::create_value(Sign::NEG, FPBits::EXP_BIAS - hi, 0).get_val();
+
+  // 2^(mid1 + mid2) - 2^(-hi)
+  DoubleDouble hi_part = x_is_neg ? fputil::exact_add(one_scaled, exp_mid.hi)
+                                  : fputil::exact_add(exp_mid.hi, one_scaled);
+
+  hi_part.lo += exp_mid.lo;
+
+  // |x - (hi + mid1 + mid2) * log(2) - dx| < 2^11 * eps(M_LOG_2_EXP2_M12.lo)
+  //                                        = 2^11 * 2^-13 * 2^-52
+  //                                        = 2^-54.
+  // |dx| < 2^-13 + 2^-30.
+  double lo_h = fputil::multiply_add(kd, MLOG_2_EXP2_M12_HI, x); // exact
+  double dx = fputil::multiply_add(kd, MLOG_2_EXP2_M12_MID, lo_h);
+
+  // We use the degree-4 Taylor polynomial to approximate exp(lo):
+  //   exp(lo) ~ 1 + lo + lo^2 / 2 + lo^3 / 6 + lo^4 / 24 = 1 + lo * P(lo)
+  // So that the errors are bounded by:
+  //   |P(lo) - expm1(lo)/lo| < |lo|^4 / 64 < 2^(-13 * 4) / 64 = 2^-58
+  // Let P_ be an evaluation of P where all intermediate computations are in
+  // double precision.  Using either Horner's or Estrin's schemes, the evaluated
+  // errors can be bounded by:
+  //      |P_(dx) - P(dx)| < 2^-51
+  //   => |dx * P_(dx) - expm1(lo) | < 1.5 * 2^-64
+  //   => 2^(mid1 + mid2) * |dx * P_(dx) - expm1(lo)| < 1.5 * 2^-63.
+  // Since we approximate
+  //   2^(mid1 + mid2) ~ exp_mid.hi + exp_mid.lo,
+  // We use the expression:
+  //    (exp_mid.hi + exp_mid.lo) * (1 + dx * P_(dx)) ~
+  //  ~ exp_mid.hi + (exp_mid.hi * dx * P_(dx) + exp_mid.lo)
+  // with errors bounded by 1.5 * 2^-63.
+
+  // Finally, we have the following approximation formula:
+  //   expm1(x) = 2^hi * 2^(mid1 + mid2) * exp(lo) - 1
+  //            = 2^hi * ( 2^(mid1 + mid2) * exp(lo) - 2^(-hi) )
+  //            ~ 2^hi * ( (exp_mid.hi - 2^-hi) +
+  //                       + (exp_mid.hi * dx * P_(dx) + exp_mid.lo))
+
+  double mid_lo = dx * exp_mid.hi;
+
+  // Approximate expm1(dx)/dx ~ 1 + dx / 2 + dx^2 / 6 + dx^3 / 24.
+  double p = poly_approx_d(dx);
+
+  double lo = fputil::multiply_add(p, mid_lo, hi_part.lo);
+
+  // TODO: The following line leaks encoding abstraction. Use FPBits methods
+  // instead.
+  uint64_t err = x_is_neg ? (static_cast<uint64_t>(-hi) << 52) : 0;
+
+  double err_d = cpp::bit_cast<double>(ERR_D + err);
+
+  double upper = hi_part.hi + (lo + err_d);
+  double lower = hi_part.hi + (lo - err_d);
+
+#ifdef DEBUGDEBUG
+  std::cout << "=== FAST PASS ===\n"
+            << "      x: " << std::hexfloat << x << std::defaultfloat << "\n"
+            << "      k: " << k << "\n"
+            << "   idx1: " << idx1 << "\n"
+            << "   idx2: " << idx2 << "\n"
+            << "     hi: " << hi << "\n"
+            << "     dx: " << std::hexfloat << dx << std::defaultfloat << "\n"
+            << "exp_mid: " << exp_mid << "hi_part: " << hi_part
+            << " mid_lo: " << std::hexfloat << mid_lo << std::defaultfloat
+            << "\n"
+            << "      p: " << std::hexfloat << p << std::defaultfloat << "\n"
+            << "     lo: " << std::hexfloat << lo << std::defaultfloat << "\n"
+            << "  upper: " << std::hexfloat << upper << std::defaultfloat
+            << "\n"
+            << "  lower: " << std::hexfloat << lower << std::defaultfloat
+            << "\n"
+            << std::endl;
+#endif
+
+  if (LIBC_LIKELY(upper == lower)) {
+    // to multiply by 2^hi, a fast way is to simply add hi to the exponent
+    // field.
+    int64_t exp_hi = static_cast<int64_t>(hi) << FPBits::FRACTION_LEN;
+    double r = cpp::bit_cast<double>(exp_hi + cpp::bit_cast<int64_t>(upper));
+    return r;
+  }
+
+  // Use double-double
+  DoubleDouble r_dd = exp_double_double(x, kd, exp_mid, hi_part);
+
+#ifdef LIBC_MATH_EXPM1_SKIP_ACCURATE_PASS
+  int64_t exp_hi = static_cast<int64_t>(hi) << FPBits::FRACTION_LEN;
+  double r =
+      cpp::bit_cast<double>(exp_hi + cpp::bit_cast<int64_t>(r_dd.hi + r_dd.lo));
+  return r;
+#else
+  double err_dd = cpp::bit_cast<double>(ERR_DD + err);
+
+  double upper_dd = r_dd.hi + (r_dd.lo + err_dd);
+  double lower_dd = r_dd.hi + (r_dd.lo - err_dd);
+
+  if (LIBC_LIKELY(upper_dd == lower_dd)) {
+    int64_t exp_hi = static_cast<int64_t>(hi) << FPBits::FRACTION_LEN;
+    double r = cpp::bit_cast<double>(exp_hi + cpp::bit_cast<int64_t>(upper_dd));
+    return r;
+  }
+
+  // Use 128-bit precision
+  Float128 r_f128 = expm1_f128(x, kd, idx1, idx2);
+
+  return static_cast<double>(r_f128);
+#endif // LIBC_MATH_EXPM1_SKIP_ACCURATE_PASS
+}
+
+} // namespace math
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_MATH_EXPM1_H
diff --git a/libc/src/__support/time/CMakeLists.txt b/libc/src/__support/time/CMakeLists.txt
index 3851037e4161f..dcb5d40ae7fb2 100644
--- a/libc/src/__support/time/CMakeLists.txt
+++ b/libc/src/__support/time/CMakeLists.txt
@@ -20,6 +20,15 @@ add_object_library(
     libc.src.__support.time.${LIBC_TARGET_OS}.clock_gettime
 )
 
+add_header_library(
+  clock_conversion
+  HDRS
+    clock_conversion.h
+  DEPENDS
+    .clock_gettime
+    libc.src.__support.time.units
+)
+
 if(TARGET libc.src.__support.time.${LIBC_TARGET_OS}.clock_settime)  
   add_object_library(
     clock_settime
diff --git a/libc/src/__support/time/clock_conversion.h b/libc/src/__support/time/clock_conversion.h
new file mode 100644
index 0000000000000..54ea4b6ca1430
--- /dev/null
+++ b/libc/src/__support/time/clock_conversion.h
@@ -0,0 +1,64 @@
+//===--- clock conversion implementation ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_TIME_CLOCK_CONVERSION_H
+#define LLVM_LIBC_SRC___SUPPORT_TIME_CLOCK_CONVERSION_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/time/clock_gettime.h"
+#include "src/__support/time/units.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace internal {
+
+// @brief Convert a timespec value from one clock domain to another.
+//
+// The function takes a timestamp that is expressed in terms of the clock
+// identified by param from and returns an equivalent timestamp expressed
+// in terms of the clock identified by param to.
+//
+// Internally it obtains the current time of both clocks with
+// clock_gettime, then subtracts the source clock’s value and
+// adds the target clock’s value. The result is normalised so that
+// the nanoseconds field is always in the range [0, 1 s).
+//
+// This is useful, for example, for converting a value obtained from
+// CLOCK_MONOTONIC to CLOCK_REALTIME (or vice‑versa) so that the
+// timestamp can be displayed to a user or stored in a format that
+// is independent of the original clock domain.
+//
+// @param input The timestamp to convert
+// @param from Clock ID of the original timestamp (e.g. CLOCK_MONOTONIC).
+// @param to Clock ID of the desired timestamp (e.g. CLOCK_REALTIME).
+// @return The converted timespec
+//
+LIBC_INLINE timespec convert_clock(timespec input, clockid_t from,
+                                   clockid_t to) {
+  using namespace time_units;
+  timespec from_time;
+  timespec to_time;
+  timespec output;
+  internal::clock_gettime(from, &from_time);
+  internal::clock_gettime(to, &to_time);
+  output.tv_sec = input.tv_sec - from_time.tv_sec + to_time.tv_sec;
+  output.tv_nsec = input.tv_nsec - from_time.tv_nsec + to_time.tv_nsec;
+
+  if (output.tv_nsec > 1_s_ns) {
+    output.tv_sec++;
+    output.tv_nsec -= 1_s_ns;
+  } else if (output.tv_nsec < 0) {
+    output.tv_sec--;
+    output.tv_nsec += 1_s_ns;
+  }
+  return output;
+}
+
+} // namespace internal
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_TIME_CLOCK_CONVERSION_H
diff --git a/libc/src/__support/time/linux/CMakeLists.txt b/libc/src/__support/time/linux/CMakeLists.txt
index 478529502b403..1d1c961bf1b2b 100644
--- a/libc/src/__support/time/linux/CMakeLists.txt
+++ b/libc/src/__support/time/linux/CMakeLists.txt
@@ -29,14 +29,6 @@ add_object_library(
     libc.src.__support.OSUtil.osutil
 )
 
-add_header_library(
-  clock_conversion
-  HDRS
-    clock_conversion.h
-  DEPENDS
-    .clock_gettime
-    libc.src.__support.time.units
-)
 
 add_header_library(
   abs_timeout
@@ -53,7 +45,7 @@ add_header_library(
   HDRS
     monotonicity.h
   DEPENDS
-    .clock_conversion
     .abs_timeout
     libc.hdr.time_macros
+    libc.src.__support.time.clock_conversion
 )
diff --git a/libc/src/__support/time/linux/clock_conversion.h b/libc/src/__support/time/linux/clock_conversion.h
deleted file mode 100644
index ac5357d308d7c..0000000000000
--- a/libc/src/__support/time/linux/clock_conversion.h
+++ /dev/null
@@ -1,43 +0,0 @@
-//===--- clock conversion linux implementation ------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_CONVERSION_H
-#define LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_CONVERSION_H
-
-#include "src/__support/macros/config.h"
-#include "src/__support/time/clock_gettime.h"
-#include "src/__support/time/units.h"
-
-namespace LIBC_NAMESPACE_DECL {
-namespace internal {
-
-LIBC_INLINE timespec convert_clock(timespec input, clockid_t from,
-                                   clockid_t to) {
-  using namespace time_units;
-  timespec from_time;
-  timespec to_time;
-  timespec output;
-  internal::clock_gettime(from, &from_time);
-  internal::clock_gettime(to, &to_time);
-  output.tv_sec = input.tv_sec - from_time.tv_sec + to_time.tv_sec;
-  output.tv_nsec = input.tv_nsec - from_time.tv_nsec + to_time.tv_nsec;
-
-  if (output.tv_nsec > 1_s_ns) {
-    output.tv_sec++;
-    output.tv_nsec -= 1_s_ns;
-  } else if (output.tv_nsec < 0) {
-    output.tv_sec--;
-    output.tv_nsec += 1_s_ns;
-  }
-  return output;
-}
-
-} // namespace internal
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_CONVERSION_H
diff --git a/libc/src/__support/time/linux/monotonicity.h b/libc/src/__support/time/linux/monotonicity.h
index c7234db2e64c4..2ee260ee44ba9 100644
--- a/libc/src/__support/time/linux/monotonicity.h
+++ b/libc/src/__support/time/linux/monotonicity.h
@@ -12,8 +12,8 @@
 #include "hdr/time_macros.h"
 #include "src/__support/libc_assert.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/time/clock_conversion.h"
 #include "src/__support/time/linux/abs_timeout.h"
-#include "src/__support/time/linux/clock_conversion.h"
 namespace LIBC_NAMESPACE_DECL {
 namespace internal {
 // This function is separated from abs_timeout.
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index e71300536616b..29ae77bcf938e 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -1561,18 +1561,7 @@ add_entrypoint_object(
   HDRS
     ../expm1.h
   DEPENDS
-    libc.src.__support.CPP.bit
-    libc.src.__support.FPUtil.dyadic_float
-    libc.src.__support.FPUtil.fenv_impl
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.FPUtil.multiply_add
-    libc.src.__support.FPUtil.polyeval
-    libc.src.__support.FPUtil.rounding_mode
-    libc.src.__support.FPUtil.triple_double
-    libc.src.__support.integer_literals
-    libc.src.__support.macros.optimization
-    libc.src.__support.math.common_constants
-    libc.src.errno.errno
+    libc.src.__support.math.expm1
 )
 
 add_entrypoint_object(
diff --git a/libc/src/math/generic/expm1.cpp b/libc/src/math/generic/expm1.cpp
index a3d0c1aa5261c..c410ae0a33a2a 100644
--- a/libc/src/math/generic/expm1.cpp
+++ b/libc/src/math/generic/expm1.cpp
@@ -7,498 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/expm1.h"
-#include "src/__support/CPP/bit.h"
-#include "src/__support/FPUtil/FEnvImpl.h"
-#include "src/__support/FPUtil/FPBits.h"
-#include "src/__support/FPUtil/PolyEval.h"
-#include "src/__support/FPUtil/double_double.h"
-#include "src/__support/FPUtil/dyadic_float.h"
-#include "src/__support/FPUtil/except_value_utils.h"
-#include "src/__support/FPUtil/multiply_add.h"
-#include "src/__support/FPUtil/rounding_mode.h"
-#include "src/__support/FPUtil/triple_double.h"
-#include "src/__support/common.h"
-#include "src/__support/integer_literals.h"
-#include "src/__support/macros/config.h"
-#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
-#include "src/__support/math/common_constants.h" // Lookup tables EXP_M1 and EXP_M2.
-#include "src/__support/math/exp_constants.h"
-
-#if ((LIBC_MATH & LIBC_MATH_SKIP_ACCURATE_PASS) != 0)
-#define LIBC_MATH_EXPM1_SKIP_ACCURATE_PASS
-#endif
+#include "src/__support/math/expm1.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-using fputil::DoubleDouble;
-using fputil::TripleDouble;
-using Float128 = typename fputil::DyadicFloat<128>;
-
-using LIBC_NAMESPACE::operator""_u128;
-
-// log2(e)
-constexpr double LOG2_E = 0x1.71547652b82fep+0;
-
-// Error bounds:
-// Errors when using double precision.
-// 0x1.8p-63;
-constexpr uint64_t ERR_D = 0x3c08000000000000;
-// Errors when using double-double precision.
-// 0x1.0p-99
-[[maybe_unused]] constexpr uint64_t ERR_DD = 0x39c0000000000000;
-
-// -2^-12 * log(2)
-// > a = -2^-12 * log(2);
-// > b = round(a, 30, RN);
-// > c = round(a - b, 30, RN);
-// > d = round(a - b - c, D, RN);
-// Errors < 1.5 * 2^-133
-constexpr double MLOG_2_EXP2_M12_HI = -0x1.62e42ffp-13;
-constexpr double MLOG_2_EXP2_M12_MID = 0x1.718432a1b0e26p-47;
-constexpr double MLOG_2_EXP2_M12_MID_30 = 0x1.718432ap-47;
-constexpr double MLOG_2_EXP2_M12_LO = 0x1.b0e2633fe0685p-79;
-
-namespace {
-
-using namespace common_constants_internal;
-
-// Polynomial approximations with double precision:
-// Return expm1(dx) / x ~ 1 + dx / 2 + dx^2 / 6 + dx^3 / 24.
-// For |dx| < 2^-13 + 2^-30:
-//   | output - expm1(dx) / dx | < 2^-51.
-LIBC_INLINE double poly_approx_d(double dx) {
-  // dx^2
-  double dx2 = dx * dx;
-  // c0 = 1 + dx / 2
-  double c0 = fputil::multiply_add(dx, 0.5, 1.0);
-  // c1 = 1/6 + dx / 24
-  double c1 =
-      fputil::multiply_add(dx, 0x1.5555555555555p-5, 0x1.5555555555555p-3);
-  // p = dx^2 * c1 + c0 = 1 + dx / 2 + dx^2 / 6 + dx^3 / 24
-  double p = fputil::multiply_add(dx2, c1, c0);
-  return p;
-}
-
-// Polynomial approximation with double-double precision:
-// Return expm1(dx) / dx ~ 1 + dx / 2 + dx^2 / 6 + ... + dx^6 / 5040
-// For |dx| < 2^-13 + 2^-30:
-//   | output - expm1(dx) | < 2^-101
-DoubleDouble poly_approx_dd(const DoubleDouble &dx) {
-  // Taylor polynomial.
-  constexpr DoubleDouble COEFFS[] = {
-      {0, 0x1p0},                                      // 1
-      {0, 0x1p-1},                                     // 1/2
-      {0x1.5555555555555p-57, 0x1.5555555555555p-3},   // 1/6
-      {0x1.5555555555555p-59, 0x1.5555555555555p-5},   // 1/24
-      {0x1.1111111111111p-63, 0x1.1111111111111p-7},   // 1/120
-      {-0x1.f49f49f49f49fp-65, 0x1.6c16c16c16c17p-10}, // 1/720
-      {0x1.a01a01a01a01ap-73, 0x1.a01a01a01a01ap-13},  // 1/5040
-  };
-
-  DoubleDouble p = fputil::polyeval(dx, COEFFS[0], COEFFS[1], COEFFS[2],
-                                    COEFFS[3], COEFFS[4], COEFFS[5], COEFFS[6]);
-  return p;
-}
-
-// Polynomial approximation with 128-bit precision:
-// Return (exp(dx) - 1)/dx ~ 1 + dx / 2 + dx^2 / 6 + ... + dx^6 / 5040
-// For |dx| < 2^-13 + 2^-30:
-//   | output - exp(dx) | < 2^-126.
-[[maybe_unused]] Float128 poly_approx_f128(const Float128 &dx) {
-  constexpr Float128 COEFFS_128[]{
-      {Sign::POS, -127, 0x80000000'00000000'00000000'00000000_u128}, // 1.0
-      {Sign::POS, -128, 0x80000000'00000000'00000000'00000000_u128}, // 0.5
-      {Sign::POS, -130, 0xaaaaaaaa'aaaaaaaa'aaaaaaaa'aaaaaaab_u128}, // 1/6
-      {Sign::POS, -132, 0xaaaaaaaa'aaaaaaaa'aaaaaaaa'aaaaaaab_u128}, // 1/24
-      {Sign::POS, -134, 0x88888888'88888888'88888888'88888889_u128}, // 1/120
-      {Sign::POS, -137, 0xb60b60b6'0b60b60b'60b60b60'b60b60b6_u128}, // 1/720
-      {Sign::POS, -140, 0xd00d00d0'0d00d00d'00d00d00'd00d00d0_u128}, // 1/5040
-  };
-
-  Float128 p = fputil::polyeval(dx, COEFFS_128[0], COEFFS_128[1], COEFFS_128[2],
-                                COEFFS_128[3], COEFFS_128[4], COEFFS_128[5],
-                                COEFFS_128[6]);
-  return p;
-}
-
-#ifdef DEBUGDEBUG
-std::ostream &operator<<(std::ostream &OS, const Float128 &r) {
-  OS << (r.sign == Sign::NEG ? "-(" : "(") << r.mantissa.val[0] << " + "
-     << r.mantissa.val[1] << " * 2^64) * 2^" << r.exponent << "\n";
-  return OS;
-}
-
-std::ostream &operator<<(std::ostream &OS, const DoubleDouble &r) {
-  OS << std::hexfloat << "(" << r.hi << " + " << r.lo << ")"
-     << std::defaultfloat << "\n";
-  return OS;
-}
-#endif
-
-// Compute exp(x) - 1 using 128-bit precision.
-// TODO(lntue): investigate triple-double precision implementation for this
-// step.
-[[maybe_unused]] Float128 expm1_f128(double x, double kd, int idx1, int idx2) {
-  // Recalculate dx:
-
-  double t1 = fputil::multiply_add(kd, MLOG_2_EXP2_M12_HI, x); // exact
-  double t2 = kd * MLOG_2_EXP2_M12_MID_30;                     // exact
-  double t3 = kd * MLOG_2_EXP2_M12_LO;                         // Error < 2^-133
-
-  Float128 dx = fputil::quick_add(
-      Float128(t1), fputil::quick_add(Float128(t2), Float128(t3)));
-
-  // TODO: Skip recalculating exp_mid1 and exp_mid2.
-  Float128 exp_mid1 =
-      fputil::quick_add(Float128(EXP2_MID1[idx1].hi),
-                        fputil::quick_add(Float128(EXP2_MID1[idx1].mid),
-                                          Float128(EXP2_MID1[idx1].lo)));
-
-  Float128 exp_mid2 =
-      fputil::quick_add(Float128(EXP2_MID2[idx2].hi),
-                        fputil::quick_add(Float128(EXP2_MID2[idx2].mid),
-                                          Float128(EXP2_MID2[idx2].lo)));
-
-  Float128 exp_mid = fputil::quick_mul(exp_mid1, exp_mid2);
-
-  int hi = static_cast<int>(kd) >> 12;
-  Float128 minus_one{Sign::NEG, -127 - hi,
-                     0x80000000'00000000'00000000'00000000_u128};
-
-  Float128 exp_mid_m1 = fputil::quick_add(exp_mid, minus_one);
-
-  Float128 p = poly_approx_f128(dx);
-
-  // r = exp_mid * (1 + dx * P) - 1
-  //   = (exp_mid - 1) + (dx * exp_mid) * P
-  Float128 r =
-      fputil::multiply_add(fputil::quick_mul(exp_mid, dx), p, exp_mid_m1);
-
-  r.exponent += hi;
-
-#ifdef DEBUGDEBUG
-  std::cout << "=== VERY SLOW PASS ===\n"
-            << "        kd: " << kd << "\n"
-            << "        hi: " << hi << "\n"
-            << " minus_one: " << minus_one << "        dx: " << dx
-            << "exp_mid_m1: " << exp_mid_m1 << "   exp_mid: " << exp_mid
-            << "         p: " << p << "         r: " << r << std::endl;
-#endif
-
-  return r;
-}
-
-// Compute exp(x) - 1 with double-double precision.
-DoubleDouble exp_double_double(double x, double kd, const DoubleDouble &exp_mid,
-                               const DoubleDouble &hi_part) {
-  // Recalculate dx:
-  //   dx = x - k * 2^-12 * log(2)
-  double t1 = fputil::multiply_add(kd, MLOG_2_EXP2_M12_HI, x); // exact
-  double t2 = kd * MLOG_2_EXP2_M12_MID_30;                     // exact
-  double t3 = kd * MLOG_2_EXP2_M12_LO;                         // Error < 2^-130
-
-  DoubleDouble dx = fputil::exact_add(t1, t2);
-  dx.lo += t3;
-
-  // Degree-6 Taylor polynomial approximation in double-double precision.
-  // | p - exp(x) | < 2^-100.
-  DoubleDouble p = poly_approx_dd(dx);
-
-  // Error bounds: 2^-99.
-  DoubleDouble r =
-      fputil::multiply_add(fputil::quick_mult(exp_mid, dx), p, hi_part);
-
-#ifdef DEBUGDEBUG
-  std::cout << "=== SLOW PASS ===\n"
-            << "   dx: " << dx << "    p: " << p << "    r: " << r << std::endl;
-#endif
-
-  return r;
-}
-
-// Check for exceptional cases when
-// |x| <= 2^-53 or x < log(2^-54) or x >= 0x1.6232bdd7abcd3p+9
-double set_exceptional(double x) {
-  using FPBits = typename fputil::FPBits<double>;
-  FPBits xbits(x);
-
-  uint64_t x_u = xbits.uintval();
-  uint64_t x_abs = xbits.abs().uintval();
-
-  // |x| <= 2^-53.
-  if (x_abs <= 0x3ca0'0000'0000'0000ULL) {
-    // expm1(x) ~ x.
-
-    if (LIBC_UNLIKELY(x_abs <= 0x0370'0000'0000'0000ULL)) {
-      if (LIBC_UNLIKELY(x_abs == 0))
-        return x;
-      // |x| <= 2^-968, need to scale up a bit before rounding, then scale it
-      // back down.
-      return 0x1.0p-200 * fputil::multiply_add(x, 0x1.0p+200, 0x1.0p-1022);
-    }
-
-    // 2^-968 < |x| <= 2^-53.
-    return fputil::round_result_slightly_up(x);
-  }
-
-  // x < log(2^-54) || x >= 0x1.6232bdd7abcd3p+9 or inf/nan.
-
-  // x < log(2^-54) or -inf/nan
-  if (x_u >= 0xc042'b708'8723'20e2ULL) {
-    // expm1(-Inf) = -1
-    if (xbits.is_inf())
-      return -1.0;
-
-    // exp(nan) = nan
-    if (xbits.is_nan())
-      return x;
-
-    return fputil::round_result_slightly_up(-1.0);
-  }
-
-  // x >= round(log(MAX_NORMAL), D, RU) = 0x1.62e42fefa39fp+9 or +inf/nan
-  // x is finite
-  if (x_u < 0x7ff0'0000'0000'0000ULL) {
-    int rounding = fputil::quick_get_round();
-    if (rounding == FE_DOWNWARD || rounding == FE_TOWARDZERO)
-      return FPBits::max_normal().get_val();
-
-    fputil::set_errno_if_required(ERANGE);
-    fputil::raise_except_if_required(FE_OVERFLOW);
-  }
-  // x is +inf or nan
-  return x + FPBits::inf().get_val();
-}
-
-} // namespace
-
-LLVM_LIBC_FUNCTION(double, expm1, (double x)) {
-  using FPBits = typename fputil::FPBits<double>;
-
-  FPBits xbits(x);
-
-  bool x_is_neg = xbits.is_neg();
-  uint64_t x_u = xbits.uintval();
-
-  // Upper bound: max normal number = 2^1023 * (2 - 2^-52)
-  // > round(log (2^1023 ( 2 - 2^-52 )), D, RU) = 0x1.62e42fefa39fp+9
-  // > round(log (2^1023 ( 2 - 2^-52 )), D, RD) = 0x1.62e42fefa39efp+9
-  // > round(log (2^1023 ( 2 - 2^-52 )), D, RN) = 0x1.62e42fefa39efp+9
-  // > round(exp(0x1.62e42fefa39fp+9), D, RN) = infty
-
-  // Lower bound: log(2^-54) = -0x1.2b708872320e2p5
-  // > round(log(2^-54), D, RN) = -0x1.2b708872320e2p5
-
-  // x < log(2^-54) or x >= 0x1.6232bdd7abcd3p+9 or |x| <= 2^-53.
-
-  if (LIBC_UNLIKELY(x_u >= 0xc042b708872320e2 ||
-                    (x_u <= 0xbca0000000000000 && x_u >= 0x40862e42fefa39f0) ||
-                    x_u <= 0x3ca0000000000000)) {
-    return set_exceptional(x);
-  }
-
-  // Now log(2^-54) <= x <= -2^-53 or 2^-53 <= x < log(2^1023 * (2 - 2^-52))
-
-  // Range reduction:
-  // Let x = log(2) * (hi + mid1 + mid2) + lo
-  // in which:
-  //   hi is an integer
-  //   mid1 * 2^6 is an integer
-  //   mid2 * 2^12 is an integer
-  // then:
-  //   exp(x) = 2^hi * 2^(mid1) * 2^(mid2) * exp(lo).
-  // With this formula:
-  //   - multiplying by 2^hi is exact and cheap, simply by adding the exponent
-  //     field.
-  //   - 2^(mid1) and 2^(mid2) are stored in 2 x 64-element tables.
-  //   - exp(lo) ~ 1 + lo + a0 * lo^2 + ...
-  //
-  // They can be defined by:
-  //   hi + mid1 + mid2 = 2^(-12) * round(2^12 * log_2(e) * x)
-  // If we store L2E = round(log2(e), D, RN), then:
-  //   log2(e) - L2E ~ 1.5 * 2^(-56)
-  // So the errors when computing in double precision is:
-  //   | x * 2^12 * log_2(e) - D(x * 2^12 * L2E) | <=
-  //  <= | x * 2^12 * log_2(e) - x * 2^12 * L2E | +
-  //     + | x * 2^12 * L2E - D(x * 2^12 * L2E) |
-  //  <= 2^12 * ( |x| * 1.5 * 2^-56 + eps(x))  for RN
-  //     2^12 * ( |x| * 1.5 * 2^-56 + 2*eps(x)) for other rounding modes.
-  // So if:
-  //   hi + mid1 + mid2 = 2^(-12) * round(x * 2^12 * L2E) is computed entirely
-  // in double precision, the reduced argument:
-  //   lo = x - log(2) * (hi + mid1 + mid2) is bounded by:
-  //   |lo| <= 2^-13 + (|x| * 1.5 * 2^-56 + 2*eps(x))
-  //         < 2^-13 + (1.5 * 2^9 * 1.5 * 2^-56 + 2*2^(9 - 52))
-  //         < 2^-13 + 2^-41
-  //
-
-  // The following trick computes the round(x * L2E) more efficiently
-  // than using the rounding instructions, with the tradeoff for less accuracy,
-  // and hence a slightly larger range for the reduced argument `lo`.
-  //
-  // To be precise, since |x| < |log(2^-1075)| < 1.5 * 2^9,
-  //   |x * 2^12 * L2E| < 1.5 * 2^9 * 1.5 < 2^23,
-  // So we can fit the rounded result round(x * 2^12 * L2E) in int32_t.
-  // Thus, the goal is to be able to use an additional addition and fixed width
-  // shift to get an int32_t representing round(x * 2^12 * L2E).
-  //
-  // Assuming int32_t using 2-complement representation, since the mantissa part
-  // of a double precision is unsigned with the leading bit hidden, if we add an
-  // extra constant C = 2^e1 + 2^e2 with e1 > e2 >= 2^25 to the product, the
-  // part that are < 2^e2 in resulted mantissa of (x*2^12*L2E + C) can be
-  // considered as a proper 2-complement representations of x*2^12*L2E.
-  //
-  // One small problem with this approach is that the sum (x*2^12*L2E + C) in
-  // double precision is rounded to the least significant bit of the dorminant
-  // factor C.  In order to minimize the rounding errors from this addition, we
-  // want to minimize e1.  Another constraint that we want is that after
-  // shifting the mantissa so that the least significant bit of int32_t
-  // corresponds to the unit bit of (x*2^12*L2E), the sign is correct without
-  // any adjustment.  So combining these 2 requirements, we can choose
-  //   C = 2^33 + 2^32, so that the sign bit corresponds to 2^31 bit, and hence
-  // after right shifting the mantissa, the resulting int32_t has correct sign.
-  // With this choice of C, the number of mantissa bits we need to shift to the
-  // right is: 52 - 33 = 19.
-  //
-  // Moreover, since the integer right shifts are equivalent to rounding down,
-  // we can add an extra 0.5 so that it will become round-to-nearest, tie-to-
-  // +infinity.  So in particular, we can compute:
-  //   hmm = x * 2^12 * L2E + C,
-  // where C = 2^33 + 2^32 + 2^-1, then if
-  //   k = int32_t(lower 51 bits of double(x * 2^12 * L2E + C) >> 19),
-  // the reduced argument:
-  //   lo = x - log(2) * 2^-12 * k is bounded by:
-  //   |lo| <= 2^-13 + 2^-41 + 2^-12*2^-19
-  //         = 2^-13 + 2^-31 + 2^-41.
-  //
-  // Finally, notice that k only uses the mantissa of x * 2^12 * L2E, so the
-  // exponent 2^12 is not needed.  So we can simply define
-  //   C = 2^(33 - 12) + 2^(32 - 12) + 2^(-13 - 12), and
-  //   k = int32_t(lower 51 bits of double(x * L2E + C) >> 19).
-
-  // Rounding errors <= 2^-31 + 2^-41.
-  double tmp = fputil::multiply_add(x, LOG2_E, 0x1.8000'0000'4p21);
-  int k = static_cast<int>(cpp::bit_cast<uint64_t>(tmp) >> 19);
-  double kd = static_cast<double>(k);
-
-  uint32_t idx1 = (k >> 6) & 0x3f;
-  uint32_t idx2 = k & 0x3f;
-  int hi = k >> 12;
-
-  DoubleDouble exp_mid1{EXP2_MID1[idx1].mid, EXP2_MID1[idx1].hi};
-  DoubleDouble exp_mid2{EXP2_MID2[idx2].mid, EXP2_MID2[idx2].hi};
-
-  DoubleDouble exp_mid = fputil::quick_mult(exp_mid1, exp_mid2);
-
-  // -2^(-hi)
-  double one_scaled =
-      FPBits::create_value(Sign::NEG, FPBits::EXP_BIAS - hi, 0).get_val();
-
-  // 2^(mid1 + mid2) - 2^(-hi)
-  DoubleDouble hi_part = x_is_neg ? fputil::exact_add(one_scaled, exp_mid.hi)
-                                  : fputil::exact_add(exp_mid.hi, one_scaled);
-
-  hi_part.lo += exp_mid.lo;
-
-  // |x - (hi + mid1 + mid2) * log(2) - dx| < 2^11 * eps(M_LOG_2_EXP2_M12.lo)
-  //                                        = 2^11 * 2^-13 * 2^-52
-  //                                        = 2^-54.
-  // |dx| < 2^-13 + 2^-30.
-  double lo_h = fputil::multiply_add(kd, MLOG_2_EXP2_M12_HI, x); // exact
-  double dx = fputil::multiply_add(kd, MLOG_2_EXP2_M12_MID, lo_h);
-
-  // We use the degree-4 Taylor polynomial to approximate exp(lo):
-  //   exp(lo) ~ 1 + lo + lo^2 / 2 + lo^3 / 6 + lo^4 / 24 = 1 + lo * P(lo)
-  // So that the errors are bounded by:
-  //   |P(lo) - expm1(lo)/lo| < |lo|^4 / 64 < 2^(-13 * 4) / 64 = 2^-58
-  // Let P_ be an evaluation of P where all intermediate computations are in
-  // double precision.  Using either Horner's or Estrin's schemes, the evaluated
-  // errors can be bounded by:
-  //      |P_(dx) - P(dx)| < 2^-51
-  //   => |dx * P_(dx) - expm1(lo) | < 1.5 * 2^-64
-  //   => 2^(mid1 + mid2) * |dx * P_(dx) - expm1(lo)| < 1.5 * 2^-63.
-  // Since we approximate
-  //   2^(mid1 + mid2) ~ exp_mid.hi + exp_mid.lo,
-  // We use the expression:
-  //    (exp_mid.hi + exp_mid.lo) * (1 + dx * P_(dx)) ~
-  //  ~ exp_mid.hi + (exp_mid.hi * dx * P_(dx) + exp_mid.lo)
-  // with errors bounded by 1.5 * 2^-63.
-
-  // Finally, we have the following approximation formula:
-  //   expm1(x) = 2^hi * 2^(mid1 + mid2) * exp(lo) - 1
-  //            = 2^hi * ( 2^(mid1 + mid2) * exp(lo) - 2^(-hi) )
-  //            ~ 2^hi * ( (exp_mid.hi - 2^-hi) +
-  //                       + (exp_mid.hi * dx * P_(dx) + exp_mid.lo))
-
-  double mid_lo = dx * exp_mid.hi;
-
-  // Approximate expm1(dx)/dx ~ 1 + dx / 2 + dx^2 / 6 + dx^3 / 24.
-  double p = poly_approx_d(dx);
-
-  double lo = fputil::multiply_add(p, mid_lo, hi_part.lo);
-
-  // TODO: The following line leaks encoding abstraction. Use FPBits methods
-  // instead.
-  uint64_t err = x_is_neg ? (static_cast<uint64_t>(-hi) << 52) : 0;
-
-  double err_d = cpp::bit_cast<double>(ERR_D + err);
-
-  double upper = hi_part.hi + (lo + err_d);
-  double lower = hi_part.hi + (lo - err_d);
-
-#ifdef DEBUGDEBUG
-  std::cout << "=== FAST PASS ===\n"
-            << "      x: " << std::hexfloat << x << std::defaultfloat << "\n"
-            << "      k: " << k << "\n"
-            << "   idx1: " << idx1 << "\n"
-            << "   idx2: " << idx2 << "\n"
-            << "     hi: " << hi << "\n"
-            << "     dx: " << std::hexfloat << dx << std::defaultfloat << "\n"
-            << "exp_mid: " << exp_mid << "hi_part: " << hi_part
-            << " mid_lo: " << std::hexfloat << mid_lo << std::defaultfloat
-            << "\n"
-            << "      p: " << std::hexfloat << p << std::defaultfloat << "\n"
-            << "     lo: " << std::hexfloat << lo << std::defaultfloat << "\n"
-            << "  upper: " << std::hexfloat << upper << std::defaultfloat
-            << "\n"
-            << "  lower: " << std::hexfloat << lower << std::defaultfloat
-            << "\n"
-            << std::endl;
-#endif
-
-  if (LIBC_LIKELY(upper == lower)) {
-    // to multiply by 2^hi, a fast way is to simply add hi to the exponent
-    // field.
-    int64_t exp_hi = static_cast<int64_t>(hi) << FPBits::FRACTION_LEN;
-    double r = cpp::bit_cast<double>(exp_hi + cpp::bit_cast<int64_t>(upper));
-    return r;
-  }
-
-  // Use double-double
-  DoubleDouble r_dd = exp_double_double(x, kd, exp_mid, hi_part);
-
-#ifdef LIBC_MATH_EXPM1_SKIP_ACCURATE_PASS
-  int64_t exp_hi = static_cast<int64_t>(hi) << FPBits::FRACTION_LEN;
-  double r =
-      cpp::bit_cast<double>(exp_hi + cpp::bit_cast<int64_t>(r_dd.hi + r_dd.lo));
-  return r;
-#else
-  double err_dd = cpp::bit_cast<double>(ERR_DD + err);
-
-  double upper_dd = r_dd.hi + (r_dd.lo + err_dd);
-  double lower_dd = r_dd.hi + (r_dd.lo - err_dd);
-
-  if (LIBC_LIKELY(upper_dd == lower_dd)) {
-    int64_t exp_hi = static_cast<int64_t>(hi) << FPBits::FRACTION_LEN;
-    double r = cpp::bit_cast<double>(exp_hi + cpp::bit_cast<int64_t>(upper_dd));
-    return r;
-  }
-
-  // Use 128-bit precision
-  Float128 r_f128 = expm1_f128(x, kd, idx1, idx2);
-
-  return static_cast<double>(r_f128);
-#endif // LIBC_MATH_EXPM1_SKIP_ACCURATE_PASS
-}
+LLVM_LIBC_FUNCTION(double, expm1, (double x)) { return math::expm1(x); }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/shared/CMakeLists.txt b/libc/test/shared/CMakeLists.txt
index 762b5b0417ef6..616772dbf3f8a 100644
--- a/libc/test/shared/CMakeLists.txt
+++ b/libc/test/shared/CMakeLists.txt
@@ -45,6 +45,7 @@ add_fp_unittest(
     libc.src.__support.math.exp2f16
     libc.src.__support.math.exp2m1f
     libc.src.__support.math.exp2m1f16
+    libc.src.__support.math.expm1
     libc.src.__support.math.exp10
     libc.src.__support.math.exp10f
     libc.src.__support.math.exp10f16
diff --git a/libc/test/shared/shared_math_test.cpp b/libc/test/shared/shared_math_test.cpp
index 5b409781a5b07..3b363244ba4b9 100644
--- a/libc/test/shared/shared_math_test.cpp
+++ b/libc/test/shared/shared_math_test.cpp
@@ -87,6 +87,7 @@ TEST(LlvmLibcSharedMathTest, AllDouble) {
   EXPECT_FP_EQ(0x1p+0, LIBC_NAMESPACE::shared::exp(0.0));
   EXPECT_FP_EQ(0x1p+0, LIBC_NAMESPACE::shared::exp2(0.0));
   EXPECT_FP_EQ(0x1p+0, LIBC_NAMESPACE::shared::exp10(0.0));
+  EXPECT_FP_EQ(0x0p+0, LIBC_NAMESPACE::shared::expm1(0.0));
 }
 
 #ifdef LIBC_TYPES_HAS_FLOAT128
diff --git a/lldb/source/Host/windows/ProcessLauncherWindows.cpp b/lldb/source/Host/windows/ProcessLauncherWindows.cpp
index 2f78ef80f385e..f22a5ffefc19a 100644
--- a/lldb/source/Host/windows/ProcessLauncherWindows.cpp
+++ b/lldb/source/Host/windows/ProcessLauncherWindows.cpp
@@ -198,7 +198,7 @@ llvm::ErrorOr<std::vector<HANDLE>> ProcessLauncherWindows::GetInheritedHandles(
     const ProcessLaunchInfo &launch_info, STARTUPINFOEXW &startupinfoex,
     HANDLE stdout_handle, HANDLE stderr_handle, HANDLE stdin_handle) {
   std::vector<HANDLE> inherited_handles;
-  STARTUPINFOW startupinfo = startupinfoex.StartupInfo;
+  STARTUPINFOW &startupinfo = startupinfoex.StartupInfo;
 
   startupinfo.hStdError =
       stderr_handle ? stderr_handle : GetStdHandle(STD_ERROR_HANDLE);
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index ed1361657e077..bd07ad0b3ff70 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -3096,13 +3096,25 @@ bool IRTranslator::translateCallBr(const User &U,
 
   // Update successor info.
   addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
-  // TODO: For most of the cases where there is an intrinsic callbr, we're
-  // having exactly one indirect target, which will be unreachable. As soon as
-  // this changes, we might need to enhance
-  // Target->setIsInlineAsmBrIndirectTarget or add something similar for
-  // intrinsic indirect branches.
+
+  // Add indirect targets as successors. For intrinsic callbr, these represent
+  // implicit control flow (e.g., the "kill" path for amdgcn.kill). We mark them
+  // with setIsInlineAsmBrIndirectTarget so the machine verifier accepts them as
+  // valid successors, even though they're not from inline asm.
+  for (BasicBlock *Dest : I.getIndirectDests()) {
+    MachineBasicBlock &Target = getMBB(*Dest);
+    Target.setIsInlineAsmBrIndirectTarget();
+    Target.setLabelMustBeEmitted();
+    // Don't add duplicate machine successors.
+    if (Dests.insert(Dest).second)
+      addSuccessorWithProb(CallBrMBB, &Target, BranchProbability::getZero());
+  }
+
   CallBrMBB->normalizeSuccProbs();
 
+  // Drop into default successor.
+  MIRBuilder.buildBr(*Return);
+
   return true;
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index dceab7cfecd19..2d8fab224212c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -5591,7 +5591,8 @@ SDValue DAGCombiner::visitMULHU(SDNode *N) {
     return DAG.getConstant(0, DL, VT);
 
   // fold (mulhu x, (1 << c)) -> x >> (bitwidth - c)
-  if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
+  if (isConstantOrConstantVector(N1, /*NoOpaques=*/true,
+                                 /*AllowTruncation=*/true) &&
       hasOperation(ISD::SRL, VT)) {
     if (SDValue LogBase2 = BuildLogBase2(N1, DL)) {
       unsigned NumEltBits = VT.getScalarSizeInBits();
@@ -29837,7 +29838,8 @@ static SDValue takeInexpensiveLog2(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
     return false;
   };
 
-  if (ISD::matchUnaryPredicate(Op, IsPowerOfTwo)) {
+  if (ISD::matchUnaryPredicate(Op, IsPowerOfTwo, /*AllowUndefs=*/false,
+                               /*AllowTruncation=*/true)) {
     if (!VT.isVector())
       return DAG.getConstant(Pow2Constants.back().logBase2(), DL, VT);
     // We need to create a build vector
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 1183f562c274d..b698ef4fcf0cc 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -6345,7 +6345,6 @@ static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
   SDValue Op0 = N->getOperand(0);
   SDValue Op1 = N->getOperand(1);
   EVT VT = N->getValueType(0);
-  EVT SVT = VT.getScalarType();
   EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
   EVT ShSVT = ShVT.getScalarType();
 
@@ -6355,6 +6354,8 @@ static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
   auto BuildSDIVPattern = [&](ConstantSDNode *C) {
     if (C->isZero())
       return false;
+
+    EVT CT = C->getValueType(0);
     APInt Divisor = C->getAPIntValue();
     unsigned Shift = Divisor.countr_zero();
     if (Shift) {
@@ -6363,12 +6364,13 @@ static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
     }
     APInt Factor = Divisor.multiplicativeInverse();
     Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
-    Factors.push_back(DAG.getConstant(Factor, dl, SVT));
+    Factors.push_back(DAG.getConstant(Factor, dl, CT));
     return true;
   };
 
   // Collect all magic values from the build vector.
-  if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
+  if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern, /*AllowUndefs=*/false,
+                                /*AllowTruncation=*/true))
     return SDValue();
 
   SDValue Shift, Factor;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 5be5c66ba17dc..ba80c0695115c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -1412,7 +1412,7 @@ SDValue AMDGPUTargetLowering::lowerUnhandledCall(CallLoweringInfo &CLI,
       InVals.push_back(DAG.getPOISON(Arg.VT));
   }
 
-  return DAG.getEntryNode();
+  return CLI.Chain;
 }
 
 SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI,
diff --git a/llvm/lib/Target/Hexagon/HexagonGenWideningVecInstr.cpp b/llvm/lib/Target/Hexagon/HexagonGenWideningVecInstr.cpp
index 8df22ae6ebb06..895de57561430 100644
--- a/llvm/lib/Target/Hexagon/HexagonGenWideningVecInstr.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonGenWideningVecInstr.cpp
@@ -349,7 +349,7 @@ Intrinsic::ID HexagonGenWideningVecInstr::getIntrinsic(
     llvm_unreachable("Incorrect input and output operand sizes");
 
   case OP_Mul:
-    assert(ResEltSize = 2 * InEltSize);
+    assert(ResEltSize == 2 * InEltSize);
     // Enter inside 'if' block when one of the operand is constant vector
     if (IsConstScalar) {
       // When inputs are of 8bit type and output is 16bit type, enter 'if' block
@@ -839,9 +839,9 @@ bool HexagonGenWideningVecInstr::replaceWithVmpaIntrinsic(Instruction *Inst,
   NewVOP2 = IRB.CreateBitCast(NewVOP2, InType);
   Value *VecOP = IRB.CreateCall(ExtF, {NewVOP1, NewVOP2});
 
-  Intrinsic::ID VmpaIntID =
-      (NewResEltSize == 16) ? VmpaIntID = Intrinsic::hexagon_V6_vmpabus_128B
-                            : VmpaIntID = Intrinsic::hexagon_V6_vmpauhb_128B;
+  Intrinsic::ID VmpaIntID = (NewResEltSize == 16)
+                                ? Intrinsic::hexagon_V6_vmpabus_128B
+                                : Intrinsic::hexagon_V6_vmpauhb_128B;
   ExtF = Intrinsic::getOrInsertDeclaration(M, VmpaIntID);
   auto *ResType =
       FixedVectorType::get(getElementTy(NewResEltSize, IRB), NumElts);
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 87ebee6a14eac..e5ba0201c0cc1 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -533,14 +533,11 @@ static Register buildBuiltinVariableLoad(
   return LoadedRegister;
 }
 
-/// Helper external function for inserting ASSIGN_TYPE instuction between \p Reg
-/// and its definition, set the new register as a destination of the definition,
-/// assign SPIRVType to both registers. If SpirvTy is provided, use it as
-/// SPIRVType in ASSIGN_TYPE, otherwise create it from \p Ty. Defined in
-/// SPIRVPreLegalizer.cpp.
-extern void insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpirvTy,
-                              SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
-                              MachineRegisterInfo &MRI);
+/// Helper external function for assigning SPIRVType to a register, ensuring the
+/// register class and type are set in MRI. Defined in SPIRVPreLegalizer.cpp.
+extern void updateRegType(Register Reg, Type *Ty, SPIRVType *SpirvTy,
+                          SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
+                          MachineRegisterInfo &MRI);
 
 // TODO: Move to TableGen.
 static SPIRV::MemorySemantics::MemorySemantics
@@ -860,8 +857,8 @@ static bool buildAtomicRMWInst(const SPIRV::IncomingCall *Call, unsigned Opcode,
       MIRBuilder.buildInstr(TargetOpcode::G_FNEG)
           .addDef(NegValueReg)
           .addUse(ValueReg);
-      insertAssignInstr(NegValueReg, nullptr, Call->ReturnType, GR, MIRBuilder,
-                        MIRBuilder.getMF().getRegInfo());
+      updateRegType(NegValueReg, nullptr, Call->ReturnType, GR, MIRBuilder,
+                    MIRBuilder.getMF().getRegInfo());
       ValueReg = NegValueReg;
     }
   }
@@ -1285,8 +1282,8 @@ static bool generateGroupInst(const SPIRV::IncomingCall *Call,
         MIRBuilder.buildICmp(
             CmpInst::ICMP_NE, Arg0, BoolReg,
             GR->buildConstantInt(0, MIRBuilder, BoolRegType, true));
-        insertAssignInstr(Arg0, nullptr, BoolType, GR, MIRBuilder,
-                          MIRBuilder.getMF().getRegInfo());
+        updateRegType(Arg0, nullptr, BoolType, GR, MIRBuilder,
+                      MIRBuilder.getMF().getRegInfo());
       } else if (BoolRegType->getOpcode() != SPIRV::OpTypeBool) {
         report_fatal_error("Expect a boolean argument");
       }
@@ -1337,8 +1334,8 @@ static bool generateGroupInst(const SPIRV::IncomingCall *Call,
       MIB.addUse(Call->Arguments[i]);
       setRegClassIfNull(Call->Arguments[i], MRI, GR);
     }
-    insertAssignInstr(VecReg, nullptr, VecType, GR, MIRBuilder,
-                      MIRBuilder.getMF().getRegInfo());
+    updateRegType(VecReg, nullptr, VecType, GR, MIRBuilder,
+                  MIRBuilder.getMF().getRegInfo());
   }
 
   // Build work/sub group instruction.
@@ -1603,8 +1600,7 @@ static bool genWorkgroupQuery(const SPIRV::IncomingCall *Call,
 
     // If the index is dynamic, need check if it's < 3, and then use a select.
     if (!IsConstantIndex) {
-      insertAssignInstr(Extracted, nullptr, PointerSizeType, GR, MIRBuilder,
-                        *MRI);
+      updateRegType(Extracted, nullptr, PointerSizeType, GR, MIRBuilder, *MRI);
 
       auto IndexType = GR->getSPIRVTypeForVReg(IndexRegister);
       auto BoolType = GR->getOrCreateSPIRVBoolType(MIRBuilder, true);
@@ -1992,8 +1988,8 @@ static bool generateImageSizeQueryInst(const SPIRV::IncomingCall *Call,
         .addUse(QueryResult)
         .addImm(ExtractedComposite);
     if (NewType != nullptr)
-      insertAssignInstr(Call->ReturnRegister, nullptr, NewType, GR, MIRBuilder,
-                        MIRBuilder.getMF().getRegInfo());
+      updateRegType(Call->ReturnRegister, nullptr, NewType, GR, MIRBuilder,
+                    MIRBuilder.getMF().getRegInfo());
   } else {
     // More than 1 component is expected, fill a new vector.
     auto MIB = MIRBuilder.buildInstr(SPIRV::OpVectorShuffle)
@@ -2766,8 +2762,8 @@ static bool generateAsyncCopy(const SPIRV::IncomingCall *Call,
                                : buildConstantIntReg32(1, MIRBuilder, GR))
                    .addUse(EventReg);
     if (NewType != nullptr)
-      insertAssignInstr(Call->ReturnRegister, nullptr, NewType, GR, MIRBuilder,
-                        MIRBuilder.getMF().getRegInfo());
+      updateRegType(Call->ReturnRegister, nullptr, NewType, GR, MIRBuilder,
+                    MIRBuilder.getMF().getRegInfo());
     return Res;
   }
   case SPIRV::OpGroupWaitEvents:
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index e5a1a2aa8d70f..eac6b4dc1de8a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -276,7 +276,7 @@ class SPIRVGlobalRegistry : public SPIRVIRMapping {
   }
 
   // Get or create a SPIR-V type corresponding the given LLVM IR type,
-  // and map it to the given VReg by creating an ASSIGN_TYPE instruction.
+  // and map it to the given VReg.
   SPIRVType *assignTypeToVReg(const Type *Type, Register VReg,
                               MachineIRBuilder &MIRBuilder,
                               SPIRV::AccessQualifier::AccessQualifier AQ,
@@ -290,7 +290,7 @@ class SPIRVGlobalRegistry : public SPIRVIRMapping {
                                   const SPIRVInstrInfo &TII);
 
   // In cases where the SPIR-V type is already known, this function can be
-  // used to map it to the given VReg via an ASSIGN_TYPE instruction.
+  // used to map it to the given VReg.
   void assignSPIRVTypeToVReg(SPIRVType *Type, Register VReg,
                              const MachineFunction &MF);
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index a2e29366dc4cc..df958d2c86b33 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -3353,10 +3353,11 @@ bool SPIRVInstructionSelector::selectGEP(Register ResVReg,
                  .addUse(GR.getSPIRVTypeID(ResType))
                  // Object to get a pointer to.
                  .addUse(I.getOperand(3).getReg());
-  assert(Opcode == SPIRV::OpPtrAccessChain ||
-         Opcode == SPIRV::OpInBoundsPtrAccessChain ||
-         (getImm(I.getOperand(4), MRI) && foldImm(I.getOperand(4), MRI) == 0) &&
-             "Cannot translate GEP to OpAccessChain. First index must be 0.");
+  assert(
+      (Opcode == SPIRV::OpPtrAccessChain ||
+       Opcode == SPIRV::OpInBoundsPtrAccessChain ||
+       (getImm(I.getOperand(4), MRI) && foldImm(I.getOperand(4), MRI) == 0)) &&
+      "Cannot translate GEP to OpAccessChain. First index must be 0.");
 
   // Adding indices.
   const unsigned StartingIndex =
diff --git a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
index 751ae0fe34d33..c90e6d8cfbfb4 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
@@ -1,6 +1,4 @@
-//===-- SPIRVPostLegalizer.cpp - ammend info after legalization -*- C++ -*-===//
-//
-// which may appear after the legalizer pass
+//===-- SPIRVPostLegalizer.cpp - amend info after legalization -*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -8,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// The pass partially apply pre-legalization logic to new instructions inserted
-// as a result of legalization:
+// The pass partially applies pre-legalization logic to new instructions
+// inserted as a result of legalization:
 // - assigns SPIR-V types to registers for new instructions.
+// - inserts ASSIGN_TYPE pseudo-instructions required for type folding.
 //
 //===----------------------------------------------------------------------===//
 
@@ -36,9 +35,9 @@ class SPIRVPostLegalizer : public MachineFunctionPass {
 
 namespace llvm {
 //  Defined in SPIRVPreLegalizer.cpp.
-extern void insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpirvTy,
-                              SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
-                              MachineRegisterInfo &MRI);
+extern void updateRegType(Register Reg, Type *Ty, SPIRVType *SpirvTy,
+                          SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
+                          MachineRegisterInfo &MRI);
 extern void processInstr(MachineInstr &MI, MachineIRBuilder &MIB,
                          MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR,
                          SPIRVType *KnownResType);
@@ -314,6 +313,54 @@ static void registerSpirvTypeForNewInstructions(MachineFunction &MF,
   }
 }
 
+static bool hasAssignType(Register Reg, MachineRegisterInfo &MRI) {
+  for (MachineInstr &UseInstr : MRI.use_nodbg_instructions(Reg)) {
+    if (UseInstr.getOpcode() == SPIRV::ASSIGN_TYPE) {
+      return true;
+    }
+  }
+  return false;
+}
+
+static void generateAssignType(MachineInstr &MI, Register ResultRegister,
+                               SPIRVType *ResultType, SPIRVGlobalRegistry *GR,
+                               MachineRegisterInfo &MRI) {
+  LLVM_DEBUG(dbgs() << "  Adding ASSIGN_TYPE for ResultRegister: "
+                    << printReg(ResultRegister, MRI.getTargetRegisterInfo())
+                    << " with type: " << *ResultType);
+  MachineIRBuilder MIB(MI);
+  updateRegType(ResultRegister, nullptr, ResultType, GR, MIB, MRI);
+
+  // Tablegen definition assumes SPIRV::ASSIGN_TYPE pseudo-instruction is
+  // present after each auto-folded instruction to take a type reference
+  // from.
+  Register NewReg =
+      MRI.createGenericVirtualRegister(MRI.getType(ResultRegister));
+  const auto *RegClass = GR->getRegClass(ResultType);
+  MRI.setRegClass(NewReg, RegClass);
+  MRI.setRegClass(ResultRegister, RegClass);
+
+  GR->assignSPIRVTypeToVReg(ResultType, ResultRegister, MIB.getMF());
+  // This is to make it convenient for Legalizer to get the SPIRVType
+  // when processing the actual MI (i.e. not pseudo one).
+  GR->assignSPIRVTypeToVReg(ResultType, NewReg, MIB.getMF());
+  // Copy MIFlags from Def to ASSIGN_TYPE instruction. It's required to
+  // keep the flags after instruction selection.
+  const uint32_t Flags = MI.getFlags();
+  MIB.buildInstr(SPIRV::ASSIGN_TYPE)
+      .addDef(ResultRegister)
+      .addUse(NewReg)
+      .addUse(GR->getSPIRVTypeID(ResultType))
+      .setMIFlags(Flags);
+  for (unsigned I = 0, E = MI.getNumDefs(); I != E; ++I) {
+    MachineOperand &MO = MI.getOperand(I);
+    if (MO.getReg() == ResultRegister) {
+      MO.setReg(NewReg);
+      break;
+    }
+  }
+}
+
 static void ensureAssignTypeForTypeFolding(MachineFunction &MF,
                                            SPIRVGlobalRegistry *GR) {
   LLVM_DEBUG(dbgs() << "Entering ensureAssignTypeForTypeFolding for function "
@@ -323,35 +370,18 @@ static void ensureAssignTypeForTypeFolding(MachineFunction &MF,
     for (MachineInstr &MI : MBB) {
       if (!isTypeFoldingSupported(MI.getOpcode()))
         continue;
-      if (MI.getNumOperands() == 1 || !MI.getOperand(1).isReg())
-        continue;
 
       LLVM_DEBUG(dbgs() << "Processing instruction: " << MI);
 
-      // Check uses of MI to see if it already has an use in SPIRV::ASSIGN_TYPE
-      bool HasAssignType = false;
       Register ResultRegister = MI.defs().begin()->getReg();
-      // All uses of Result register
-      for (MachineInstr &UseInstr :
-           MRI.use_nodbg_instructions(ResultRegister)) {
-        if (UseInstr.getOpcode() == SPIRV::ASSIGN_TYPE) {
-          HasAssignType = true;
-          LLVM_DEBUG(dbgs() << "  Instruction already has an ASSIGN_TYPE use: "
-                            << UseInstr);
-          break;
-        }
+      if (hasAssignType(ResultRegister, MRI)) {
+        LLVM_DEBUG(dbgs() << "  Instruction already has ASSIGN_TYPE\n");
+        continue;
       }
 
-      if (!HasAssignType) {
-        Register ResultRegister = MI.defs().begin()->getReg();
-        SPIRVType *ResultType = GR->getSPIRVTypeForVReg(ResultRegister);
-        LLVM_DEBUG(
-            dbgs() << "  Adding ASSIGN_TYPE for ResultRegister: "
-                   << printReg(ResultRegister, MRI.getTargetRegisterInfo())
-                   << " with type: " << *ResultType);
-        MachineIRBuilder MIB(MI);
-        insertAssignInstr(ResultRegister, nullptr, ResultType, GR, MIB, MRI);
-      }
+      SPIRVType *ResultType = GR->getSPIRVTypeForVReg(ResultRegister);
+      assert(ResultType);
+      generateAssignType(MI, ResultRegister, ResultType, GR, MRI);
     }
   }
 }
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index 7ca463460ffad..1e77b79ec496d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -425,52 +425,20 @@ static void setInsertPtAfterDef(MachineIRBuilder &MIB, MachineInstr *Def) {
 }
 
 namespace llvm {
-void insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpvType,
-                       SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
-                       MachineRegisterInfo &MRI) {
+void updateRegType(Register Reg, Type *Ty, SPIRVType *SpvType,
+                   SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
+                   MachineRegisterInfo &MRI) {
   assert((Ty || SpvType) && "Either LLVM or SPIRV type is expected.");
   MachineInstr *Def = MRI.getVRegDef(Reg);
   setInsertPtAfterDef(MIB, Def);
   if (!SpvType)
     SpvType = GR->getOrCreateSPIRVType(Ty, MIB,
                                        SPIRV::AccessQualifier::ReadWrite, true);
-
-  if (!isTypeFoldingSupported(Def->getOpcode())) {
-    // No need to generate SPIRV::ASSIGN_TYPE pseudo-instruction
-    if (!MRI.getRegClassOrNull(Reg))
-      MRI.setRegClass(Reg, GR->getRegClass(SpvType));
-    if (!MRI.getType(Reg).isValid())
-      MRI.setType(Reg, GR->getRegType(SpvType));
-    GR->assignSPIRVTypeToVReg(SpvType, Reg, MIB.getMF());
-    return;
-  }
-
-  // Tablegen definition assumes SPIRV::ASSIGN_TYPE pseudo-instruction is
-  // present after each auto-folded instruction to take a type reference from.
-  Register NewReg = MRI.createGenericVirtualRegister(MRI.getType(Reg));
-  const auto *RegClass = GR->getRegClass(SpvType);
-  MRI.setRegClass(NewReg, RegClass);
-  MRI.setRegClass(Reg, RegClass);
-
+  if (!MRI.getRegClassOrNull(Reg))
+    MRI.setRegClass(Reg, GR->getRegClass(SpvType));
+  if (!MRI.getType(Reg).isValid())
+    MRI.setType(Reg, GR->getRegType(SpvType));
   GR->assignSPIRVTypeToVReg(SpvType, Reg, MIB.getMF());
-  // This is to make it convenient for Legalizer to get the SPIRVType
-  // when processing the actual MI (i.e. not pseudo one).
-  GR->assignSPIRVTypeToVReg(SpvType, NewReg, MIB.getMF());
-  // Copy MIFlags from Def to ASSIGN_TYPE instruction. It's required to keep
-  // the flags after instruction selection.
-  const uint32_t Flags = Def->getFlags();
-  MIB.buildInstr(SPIRV::ASSIGN_TYPE)
-      .addDef(Reg)
-      .addUse(NewReg)
-      .addUse(GR->getSPIRVTypeID(SpvType))
-      .setMIFlags(Flags);
-  for (unsigned I = 0, E = Def->getNumDefs(); I != E; ++I) {
-    MachineOperand &MO = Def->getOperand(I);
-    if (MO.getReg() == Reg) {
-      MO.setReg(NewReg);
-      break;
-    }
-  }
 }
 
 void processInstr(MachineInstr &MI, MachineIRBuilder &MIB,
@@ -543,10 +511,9 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
         MachineInstr *Def = MRI.getVRegDef(Reg);
         assert(Def && "Expecting an instruction that defines the register");
         // G_GLOBAL_VALUE already has type info.
-        if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE &&
-            Def->getOpcode() != SPIRV::ASSIGN_TYPE)
-          insertAssignInstr(Reg, nullptr, AssignedPtrType, GR, MIB,
-                            MF.getRegInfo());
+        if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE)
+          updateRegType(Reg, nullptr, AssignedPtrType, GR, MIB,
+                        MF.getRegInfo());
         ToErase.push_back(&MI);
       } else if (isSpvIntrinsic(MI, Intrinsic::spv_assign_type)) {
         Register Reg = MI.getOperand(1).getReg();
@@ -554,9 +521,8 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
         MachineInstr *Def = MRI.getVRegDef(Reg);
         assert(Def && "Expecting an instruction that defines the register");
         // G_GLOBAL_VALUE already has type info.
-        if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE &&
-            Def->getOpcode() != SPIRV::ASSIGN_TYPE)
-          insertAssignInstr(Reg, Ty, nullptr, GR, MIB, MF.getRegInfo());
+        if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE)
+          updateRegType(Reg, Ty, nullptr, GR, MIB, MF.getRegInfo());
         ToErase.push_back(&MI);
       } else if (MIOp == TargetOpcode::FAKE_USE && MI.getNumOperands() > 0) {
         MachineInstr *MdMI = MI.getPrevNode();
@@ -581,20 +547,9 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
                  MIOp == TargetOpcode::G_FCONSTANT ||
                  MIOp == TargetOpcode::G_BUILD_VECTOR) {
         // %rc = G_CONSTANT ty Val
-        // ===>
-        // %cty = OpType* ty
-        // %rctmp = G_CONSTANT ty Val
-        // %rc = ASSIGN_TYPE %rctmp, %cty
+        // Ensure %rc has a valid SPIR-V type assigned in the Global Registry.
         Register Reg = MI.getOperand(0).getReg();
-        bool NeedAssignType = true;
-        if (MRI.hasOneUse(Reg)) {
-          MachineInstr &UseMI = *MRI.use_instr_begin(Reg);
-          if (isSpvIntrinsic(UseMI, Intrinsic::spv_assign_type) ||
-              isSpvIntrinsic(UseMI, Intrinsic::spv_assign_name))
-            continue;
-          if (UseMI.getOpcode() == SPIRV::ASSIGN_TYPE)
-            NeedAssignType = false;
-        }
+        bool NeedAssignType = GR->getSPIRVTypeForVReg(Reg) == nullptr;
         Type *Ty = nullptr;
         if (MIOp == TargetOpcode::G_CONSTANT) {
           auto TargetExtIt = TargetExtConstTypes.find(&MI);
@@ -639,13 +594,6 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
             if (const SPIRVType *ElemSpvType =
                     GR->getSPIRVTypeForVReg(MI.getOperand(1).getReg(), &MF))
               ElemTy = const_cast<Type *>(GR->getTypeForSPIRVType(ElemSpvType));
-            if (!ElemTy) {
-              // There may be a case when we already know Reg's type.
-              MachineInstr *NextMI = MI.getNextNode();
-              if (!NextMI || NextMI->getOpcode() != SPIRV::ASSIGN_TYPE ||
-                  NextMI->getOperand(1).getReg() != Reg)
-                llvm_unreachable("Unexpected opcode");
-            }
           }
           if (ElemTy)
             Ty = VectorType::get(
@@ -655,7 +603,7 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
             NeedAssignType = false;
         }
         if (NeedAssignType)
-          insertAssignInstr(Reg, Ty, nullptr, GR, MIB, MRI);
+          updateRegType(Reg, Ty, nullptr, GR, MIB, MRI);
       } else if (MIOp == TargetOpcode::G_GLOBAL_VALUE) {
         propagateSPIRVType(&MI, GR, MRI, MIB);
       }
diff --git a/llvm/lib/Transforms/Scalar/InferAlignment.cpp b/llvm/lib/Transforms/Scalar/InferAlignment.cpp
index 39751c04eba08..c0a147ed78231 100644
--- a/llvm/lib/Transforms/Scalar/InferAlignment.cpp
+++ b/llvm/lib/Transforms/Scalar/InferAlignment.cpp
@@ -12,15 +12,20 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Scalar/InferAlignment.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/PatternMatch.h"
 #include "llvm/Support/KnownBits.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/Local.h"
 
 using namespace llvm;
+using namespace llvm::PatternMatch;
 
 static bool tryToImproveAlign(
     const DataLayout &DL, Instruction *I,
@@ -37,6 +42,16 @@ static bool tryToImproveAlign(
     }
   }
 
+  Value *PtrOp;
+  const APInt *Const;
+  if (match(I, m_And(m_PtrToIntOrAddr(m_Value(PtrOp)), m_APInt(Const)))) {
+    Align ActualAlign = Fn(PtrOp, Align(1), Align(1));
+    if (Const->ult(ActualAlign.value())) {
+      I->replaceAllUsesWith(Constant::getNullValue(I->getType()));
+      return true;
+    }
+  }
+
   IntrinsicInst *II = dyn_cast<IntrinsicInst>(I);
   if (!II)
     return false;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 5ca89766dc837..91f58bdc0686d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5327,12 +5327,12 @@ class slpvectorizer::BoUpSLP {
         return false;
       SmallDenseMap<TreeEntry *, unsigned> PotentiallyReorderedEntriesCount;
       SmallDenseMap<const TreeEntry *, unsigned> OrderedEntriesCount;
+      ArrayRef<TreeEntry *> Entries = SLP.getTreeEntries(User);
+      if (Entries.empty())
+        return false;
       for (const Use &U : User->operands()) {
         if (U.get() != Op)
           continue;
-        ArrayRef<TreeEntry *> Entries = SLP.getTreeEntries(User);
-        if (Entries.empty())
-          return false;
         // Check all tree entries, if they have operands replaced by copyable
         // data.
         for (TreeEntry *TE : Entries) {
@@ -11316,44 +11316,91 @@ BoUpSLP::ScalarsVectorizationLegality BoUpSLP::getScalarsVectorizationLegality(
       VL, *this, TryCopyableElementsVectorization,
       /*WithProfitabilityCheck=*/true, TryCopyableElementsVectorization);
 
+  bool AreScatterAllGEPSameBlock = false;
+  if (!S) {
+    SmallVector<unsigned> SortedIndices;
+    BasicBlock *BB = nullptr;
+    bool IsScatterVectorizeUserTE =
+        UserTreeIdx.UserTE &&
+        UserTreeIdx.UserTE->State == TreeEntry::ScatterVectorize;
+    AreScatterAllGEPSameBlock =
+        (IsScatterVectorizeUserTE && VL.front()->getType()->isPointerTy() &&
+         VL.size() > 2 &&
+         all_of(VL,
+                [&BB](Value *V) {
+                  auto *I = dyn_cast<GetElementPtrInst>(V);
+                  if (!I)
+                    return doesNotNeedToBeScheduled(V);
+                  if (!BB)
+                    BB = I->getParent();
+                  return BB == I->getParent() && I->getNumOperands() == 2;
+                }) &&
+         BB &&
+         sortPtrAccesses(VL, UserTreeIdx.UserTE->getMainOp()->getType(), *DL,
+                         *SE, SortedIndices));
+    if (!AreScatterAllGEPSameBlock) {
+      LLVM_DEBUG(dbgs() << "SLP: Try split and if failed, gathering due to "
+                           "C,S,B,O, small shuffle. \n";
+                 dbgs() << "[";
+                 interleaveComma(VL, dbgs(), [&](Value *V) { dbgs() << *V; });
+                 dbgs() << "]\n");
+      return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
+                                          /*TryToFindDuplicates=*/true,
+                                          /*TrySplitVectorize=*/true);
+    }
+    // Reset S to make it GetElementPtr kind of node.
+    const auto *It = find_if(VL, IsaPred<GetElementPtrInst>);
+    assert(It != VL.end() && "Expected at least one GEP.");
+    S = getSameOpcode(*It, *TLI);
+  }
+  assert(S && "Must be valid.");
+
+  // Don't handle vectors.
+  if (!SLPReVec && getValueType(VL.front())->isVectorTy()) {
+    LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
+    // Do not try to pack to avoid extra instructions here.
+    return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
+                                        /*TryToFindDuplicates=*/false);
+  }
+
+  // Check that all of the users of the scalars that we want to vectorize are
+  // schedulable.
+  BasicBlock *BB = S.getMainOp()->getParent();
+
+  if (BB->isEHPad() || isa_and_nonnull<UnreachableInst>(BB->getTerminator()) ||
+      !DT->isReachableFromEntry(BB)) {
+    // Don't go into unreachable blocks. They may contain instructions with
+    // dependency cycles which confuse the final scheduling.
+    // Do not vectorize EH and non-returning blocks, not profitable in most
+    // cases.
+    LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
+    return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
+  }
+
   // Don't go into catchswitch blocks, which can happen with PHIs.
   // Such blocks can only have PHIs and the catchswitch.  There is no
   // place to insert a shuffle if we need to, so just avoid that issue.
-  if (S && isa<CatchSwitchInst>(S.getMainOp()->getParent()->getTerminator())) {
+  if (isa<CatchSwitchInst>(BB->getTerminator())) {
     LLVM_DEBUG(dbgs() << "SLP: bundle in catchswitch block.\n");
     // Do not try to pack to avoid extra instructions here.
     return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
                                         /*TryToFindDuplicates=*/false);
   }
 
-  // Check if this is a duplicate of another entry.
-  if (S) {
-    LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.getMainOp() << ".\n");
-    for (TreeEntry *E : getTreeEntries(S.getMainOp())) {
-      if (E->isSame(VL)) {
-        LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.getMainOp()
-                          << ".\n");
-        return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
-      }
-      SmallPtrSet<Value *, 8> Values(llvm::from_range, E->Scalars);
-      if (all_of(VL, [&](Value *V) {
-            return isa<PoisonValue>(V) || Values.contains(V) ||
-                   (S.getOpcode() == Instruction::PHI && isa<PHINode>(V) &&
-                    LI->getLoopFor(S.getMainOp()->getParent()) &&
-                    isVectorized(V));
-          })) {
-        LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n");
-        return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
-      }
-    }
+  // Don't handle scalable vectors
+  if (S.getOpcode() == Instruction::ExtractElement &&
+      isa<ScalableVectorType>(
+          cast<ExtractElementInst>(S.getMainOp())->getVectorOperandType())) {
+    LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
+    return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
   }
 
   // Gather if we hit the RecursionMaxDepth, unless this is a load (or z/sext of
   // a load), in which case peek through to include it in the tree, without
   // ballooning over-budget.
   if (Depth >= RecursionMaxDepth &&
-      !(S && !S.isAltShuffle() && VL.size() >= 4 &&
-        (match(S.getMainOp(), m_Load(m_Value())) ||
+      (S.isAltShuffle() || VL.size() < 4 ||
+       !(match(S.getMainOp(), m_Load(m_Value())) ||
          all_of(VL, [&S](const Value *I) {
            return match(I,
                         m_OneUse(m_ZExtOrSExt(m_OneUse(m_Load(m_Value()))))) &&
@@ -11363,20 +11410,24 @@ BoUpSLP::ScalarsVectorizationLegality BoUpSLP::getScalarsVectorizationLegality(
     return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
   }
 
-  // Don't handle scalable vectors
-  if (S && S.getOpcode() == Instruction::ExtractElement &&
-      isa<ScalableVectorType>(
-          cast<ExtractElementInst>(S.getMainOp())->getVectorOperandType())) {
-    LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
-    return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
-  }
-
-  // Don't handle vectors.
-  if (!SLPReVec && getValueType(VL.front())->isVectorTy()) {
-    LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
-    // Do not try to pack to avoid extra instructions here.
-    return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
-                                        /*TryToFindDuplicates=*/false);
+  // Check if this is a duplicate of another entry.
+  LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.getMainOp() << ".\n");
+  for (TreeEntry *E : getTreeEntries(S.getMainOp())) {
+    if (E->isSame(VL)) {
+      LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.getMainOp()
+                        << ".\n");
+      return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
+    }
+    SmallPtrSet<Value *, 8> Values(llvm::from_range, E->Scalars);
+    if (all_of(VL, [&](Value *V) {
+          return isa<PoisonValue>(V) || Values.contains(V) ||
+                 (S.getOpcode() == Instruction::PHI && isa<PHINode>(V) &&
+                  LI->getLoopFor(S.getMainOp()->getParent()) &&
+                  isVectorized(V));
+        })) {
+      LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n");
+      return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
+    }
   }
 
   // If all of the operands are identical or constant we have a simple solution.
@@ -11435,44 +11486,13 @@ BoUpSLP::ScalarsVectorizationLegality BoUpSLP::getScalarsVectorizationLegality(
     }
     return true;
   };
-  SmallVector<unsigned> SortedIndices;
-  BasicBlock *BB = nullptr;
-  bool IsScatterVectorizeUserTE =
-      UserTreeIdx.UserTE &&
-      UserTreeIdx.UserTE->State == TreeEntry::ScatterVectorize;
-  bool AreAllSameBlock = S.valid();
-  bool AreScatterAllGEPSameBlock =
-      (IsScatterVectorizeUserTE && VL.front()->getType()->isPointerTy() &&
-       VL.size() > 2 &&
-       all_of(VL,
-              [&BB](Value *V) {
-                auto *I = dyn_cast<GetElementPtrInst>(V);
-                if (!I)
-                  return doesNotNeedToBeScheduled(V);
-                if (!BB)
-                  BB = I->getParent();
-                return BB == I->getParent() && I->getNumOperands() == 2;
-              }) &&
-       BB &&
-       sortPtrAccesses(VL, UserTreeIdx.UserTE->getMainOp()->getType(), *DL, *SE,
-                       SortedIndices));
+  bool AreAllSameBlock = !AreScatterAllGEPSameBlock;
   bool AreAllSameInsts = AreAllSameBlock || AreScatterAllGEPSameBlock;
-  if (!AreAllSameInsts || (!S && allConstant(VL)) || isSplat(VL) ||
-      (S &&
-       isa<InsertElementInst, ExtractValueInst, ExtractElementInst>(
+  if (!AreAllSameInsts || isSplat(VL) ||
+      (isa<InsertElementInst, ExtractValueInst, ExtractElementInst>(
            S.getMainOp()) &&
        !all_of(VL, isVectorLikeInstWithConstOps)) ||
       NotProfitableForVectorization(VL)) {
-    if (!S) {
-      LLVM_DEBUG(dbgs() << "SLP: Try split and if failed, gathering due to "
-                           "C,S,B,O, small shuffle. \n";
-                 dbgs() << "[";
-                 interleaveComma(VL, dbgs(), [&](Value *V) { dbgs() << *V; });
-                 dbgs() << "]\n");
-      return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
-                                          /*TryToFindDuplicates=*/true,
-                                          /*TrySplitVectorize=*/true);
-    }
     LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O, small shuffle. \n";
                dbgs() << "[";
                interleaveComma(VL, dbgs(), [&](Value *V) { dbgs() << *V; });
@@ -11481,7 +11501,7 @@ BoUpSLP::ScalarsVectorizationLegality BoUpSLP::getScalarsVectorizationLegality(
   }
 
   // Don't vectorize ephemeral values.
-  if (S && !EphValues.empty()) {
+  if (!EphValues.empty()) {
     for (Value *V : VL) {
       if (EphValues.count(V)) {
         LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
@@ -11499,7 +11519,7 @@ BoUpSLP::ScalarsVectorizationLegality BoUpSLP::getScalarsVectorizationLegality(
   // Check that none of the instructions in the bundle are already in the tree
   // and the node may be not profitable for the vectorization as the small
   // alternate node.
-  if (S && S.isAltShuffle()) {
+  if (S.isAltShuffle()) {
     auto GetNumVectorizedExtracted = [&]() {
       APInt Extracted = APInt::getZero(VL.size());
       APInt Vectorized = APInt::getAllOnes(VL.size());
@@ -11551,33 +11571,6 @@ BoUpSLP::ScalarsVectorizationLegality BoUpSLP::getScalarsVectorizationLegality(
     }
   }
 
-  // Special processing for sorted pointers for ScatterVectorize node with
-  // constant indeces only.
-  if (!AreAllSameBlock && AreScatterAllGEPSameBlock) {
-    assert(VL.front()->getType()->isPointerTy() &&
-           count_if(VL, IsaPred<GetElementPtrInst>) >= 2 &&
-           "Expected pointers only.");
-    // Reset S to make it GetElementPtr kind of node.
-    const auto *It = find_if(VL, IsaPred<GetElementPtrInst>);
-    assert(It != VL.end() && "Expected at least one GEP.");
-    S = getSameOpcode(*It, *TLI);
-  }
-
-  // Check that all of the users of the scalars that we want to vectorize are
-  // schedulable.
-  Instruction *VL0 = S.getMainOp();
-  BB = VL0->getParent();
-
-  if (S &&
-      (BB->isEHPad() || isa_and_nonnull<UnreachableInst>(BB->getTerminator()) ||
-       !DT->isReachableFromEntry(BB))) {
-    // Don't go into unreachable blocks. They may contain instructions with
-    // dependency cycles which confuse the final scheduling.
-    // Do not vectorize EH and non-returning blocks, not profitable in most
-    // cases.
-    LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
-    return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
-  }
   return ScalarsVectorizationLegality(S, /*IsLegal=*/true);
 }
 
diff --git a/llvm/test/CodeGen/AArch64/mulhu-srl-promoted-ops.ll b/llvm/test/CodeGen/AArch64/mulhu-srl-promoted-ops.ll
new file mode 100644
index 0000000000000..46f994bfba9a2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/mulhu-srl-promoted-ops.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s
+
+define <8 x i16> @mulhu_v8i16_by_256(<8 x i16> %x) {
+; CHECK-LABEL: mulhu_v8i16_by_256:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushr v0.8h, v0.8h, #8
+; CHECK-NEXT:    ret
+  %x32 = zext <8 x i16> %x to <8 x i32>
+  %mul = mul <8 x i32> %x32, splat (i32 256)
+  %result = lshr <8 x i32> %mul, splat (i32 16)
+  %trunc = trunc <8 x i32> %result to <8 x i16>
+  ret <8 x i16> %trunc
+}
+
+define <16 x i16> @mulhu_v16i16_by_256(<16 x i16> %x) {
+; CHECK-LABEL: mulhu_v16i16_by_256:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushr v0.8h, v0.8h, #8
+; CHECK-NEXT:    ushr v1.8h, v1.8h, #8
+; CHECK-NEXT:    ret
+  %x32 = zext <16 x i16> %x to <16 x i32>
+  %mul = mul <16 x i32> %x32, splat (i32 256)
+  %result = lshr <16 x i32> %mul, splat (i32 16)
+  %trunc = trunc <16 x i32> %result to <16 x i16>
+  ret <16 x i16> %trunc
+}
diff --git a/llvm/test/CodeGen/AArch64/sdiv-by-const-promoted-ops.ll b/llvm/test/CodeGen/AArch64/sdiv-by-const-promoted-ops.ll
index 053cbc0616454..840501ec48e42 100644
--- a/llvm/test/CodeGen/AArch64/sdiv-by-const-promoted-ops.ll
+++ b/llvm/test/CodeGen/AArch64/sdiv-by-const-promoted-ops.ll
@@ -75,3 +75,24 @@ define <16 x i16> @srem_v16i16_by_7(<16 x i16> %x) {
   %rem = srem <16 x i16> %x, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
   ret <16 x i16> %rem
 }
+
+define <8 x i16> @sdiv_exact_v8i16_by_255(<8 x i16> %x) {
+; CHECK-LABEL: sdiv_exact_v8i16_by_255:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mvni v1.8h, #1, lsl #8
+; CHECK-NEXT:    mul v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    ret
+  %div = sdiv exact <8 x i16> %x, splat (i16 255)
+  ret <8 x i16> %div
+}
+
+define <16 x i16> @sdiv_exact_v16i16_by_255(<16 x i16> %x) {
+; CHECK-LABEL: sdiv_exact_v16i16_by_255:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mvni v2.8h, #1, lsl #8
+; CHECK-NEXT:    mul v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    mul v1.8h, v1.8h, v2.8h
+; CHECK-NEXT:    ret
+  %div = sdiv exact <16 x i16> %x, splat (i16 255)
+  ret <16 x i16> %div
+}
diff --git a/llvm/test/CodeGen/AMDGPU/callbr-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/callbr-intrinsics.ll
index ff1e23836fed2..8ac31b3c70ed7 100644
--- a/llvm/test/CodeGen/AMDGPU/callbr-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/callbr-intrinsics.ll
@@ -32,14 +32,21 @@ define void @test_kill(ptr %src, ptr %dst, i1 %c) {
 ; GISEL-NEXT:    s_mov_b64 s[4:5], exec
 ; GISEL-NEXT:    s_andn2_b64 s[6:7], exec, vcc
 ; GISEL-NEXT:    s_andn2_b64 s[4:5], s[4:5], s[6:7]
-; GISEL-NEXT:    s_cbranch_scc0 .LBB0_2
+; GISEL-NEXT:    s_cbranch_scc0 .LBB0_4
 ; GISEL-NEXT:  ; %bb.1:
 ; GISEL-NEXT:    s_and_b64 exec, exec, s[4:5]
+; GISEL-NEXT:  ; %bb.2: ; %cont
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    flat_store_dword v[2:3], v0
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
-; GISEL-NEXT:  .LBB0_2:
+; GISEL-NEXT:  .LBB0_3: ; Inline asm indirect target
+; GISEL-NEXT:    ; %kill
+; GISEL-NEXT:    ; Label of block must be emitted
+; GISEL-NEXT:    ; divergent unreachable
+; GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GISEL-NEXT:  .LBB0_4:
 ; GISEL-NEXT:    s_mov_b64 exec, 0
 ; GISEL-NEXT:    s_endpgm
   %a = load i32, ptr %src, align 4
@@ -81,14 +88,21 @@ define void @test_kill_block_order(ptr %src, ptr %dst, i1 %c) {
 ; GISEL-NEXT:    s_mov_b64 s[4:5], exec
 ; GISEL-NEXT:    s_andn2_b64 s[6:7], exec, vcc
 ; GISEL-NEXT:    s_andn2_b64 s[4:5], s[4:5], s[6:7]
-; GISEL-NEXT:    s_cbranch_scc0 .LBB1_2
+; GISEL-NEXT:    s_cbranch_scc0 .LBB1_4
 ; GISEL-NEXT:  ; %bb.1:
 ; GISEL-NEXT:    s_and_b64 exec, exec, s[4:5]
+; GISEL-NEXT:  ; %bb.2: ; %cont
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    flat_store_dword v[2:3], v0
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
-; GISEL-NEXT:  .LBB1_2:
+; GISEL-NEXT:  .LBB1_3: ; Inline asm indirect target
+; GISEL-NEXT:    ; %kill
+; GISEL-NEXT:    ; Label of block must be emitted
+; GISEL-NEXT:    ; divergent unreachable
+; GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GISEL-NEXT:  .LBB1_4:
 ; GISEL-NEXT:    s_mov_b64 exec, 0
 ; GISEL-NEXT:    s_endpgm
   %a = load i32, ptr %src, align 4
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmulhu-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vmulhu-sdnode.ll
index 3fd7f5be860cf..c0c9b1797f91f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmulhu-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmulhu-sdnode.ll
@@ -48,18 +48,11 @@ define <vscale x 1 x i32> @vmulhu_vi_nxv1i32_0(<vscale x 1 x i32> %va) {
 }
 
 define <vscale x 1 x i32> @vmulhu_vi_nxv1i32_1(<vscale x 1 x i32> %va) {
-; RV32-LABEL: vmulhu_vi_nxv1i32_1:
-; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v8, 28
-; RV32-NEXT:    ret
-;
-; RV64-LABEL: vmulhu_vi_nxv1i32_1:
-; RV64:       # %bb.0:
-; RV64-NEXT:    li a0, 16
-; RV64-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
-; RV64-NEXT:    vmulhu.vx v8, v8, a0
-; RV64-NEXT:    ret
+; CHECK-LABEL: vmulhu_vi_nxv1i32_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsrl.vi v8, v8, 28
+; CHECK-NEXT:    ret
   %vb = zext <vscale x 1 x i32> splat (i32 16) to <vscale x 1 x i64>
   %vc = zext <vscale x 1 x i32> %va to <vscale x 1 x i64>
   %vd = mul <vscale x 1 x i64> %vb, %vc
@@ -114,18 +107,11 @@ define <vscale x 2 x i32> @vmulhu_vi_nxv2i32_0(<vscale x 2 x i32> %va) {
 }
 
 define <vscale x 2 x i32> @vmulhu_vi_nxv2i32_1(<vscale x 2 x i32> %va) {
-; RV32-LABEL: vmulhu_vi_nxv2i32_1:
-; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v8, 28
-; RV32-NEXT:    ret
-;
-; RV64-LABEL: vmulhu_vi_nxv2i32_1:
-; RV64:       # %bb.0:
-; RV64-NEXT:    li a0, 16
-; RV64-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
-; RV64-NEXT:    vmulhu.vx v8, v8, a0
-; RV64-NEXT:    ret
+; CHECK-LABEL: vmulhu_vi_nxv2i32_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vsrl.vi v8, v8, 28
+; CHECK-NEXT:    ret
   %vb = zext <vscale x 2 x i32> splat (i32 16) to <vscale x 2 x i64>
   %vc = zext <vscale x 2 x i32> %va to <vscale x 2 x i64>
   %vd = mul <vscale x 2 x i64> %vb, %vc
@@ -180,18 +166,11 @@ define <vscale x 4 x i32> @vmulhu_vi_nxv4i32_0(<vscale x 4 x i32> %va) {
 }
 
 define <vscale x 4 x i32> @vmulhu_vi_nxv4i32_1(<vscale x 4 x i32> %va) {
-; RV32-LABEL: vmulhu_vi_nxv4i32_1:
-; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v8, 28
-; RV32-NEXT:    ret
-;
-; RV64-LABEL: vmulhu_vi_nxv4i32_1:
-; RV64:       # %bb.0:
-; RV64-NEXT:    li a0, 16
-; RV64-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; RV64-NEXT:    vmulhu.vx v8, v8, a0
-; RV64-NEXT:    ret
+; CHECK-LABEL: vmulhu_vi_nxv4i32_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsrl.vi v8, v8, 28
+; CHECK-NEXT:    ret
   %vb = zext <vscale x 4 x i32> splat (i32 16) to <vscale x 4 x i64>
   %vc = zext <vscale x 4 x i32> %va to <vscale x 4 x i64>
   %vd = mul <vscale x 4 x i64> %vb, %vc
@@ -246,18 +225,11 @@ define <vscale x 8 x i32> @vmulhu_vi_nxv8i32_0(<vscale x 8 x i32> %va) {
 }
 
 define <vscale x 8 x i32> @vmulhu_vi_nxv8i32_1(<vscale x 8 x i32> %va) {
-; RV32-LABEL: vmulhu_vi_nxv8i32_1:
-; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v8, 28
-; RV32-NEXT:    ret
-;
-; RV64-LABEL: vmulhu_vi_nxv8i32_1:
-; RV64:       # %bb.0:
-; RV64-NEXT:    li a0, 16
-; RV64-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; RV64-NEXT:    vmulhu.vx v8, v8, a0
-; RV64-NEXT:    ret
+; CHECK-LABEL: vmulhu_vi_nxv8i32_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vsrl.vi v8, v8, 28
+; CHECK-NEXT:    ret
   %vb = zext <vscale x 8 x i32> splat (i32 16) to <vscale x 8 x i64>
   %vc = zext <vscale x 8 x i32> %va to <vscale x 8 x i64>
   %vd = mul <vscale x 8 x i64> %vb, %vc
@@ -265,3 +237,6 @@ define <vscale x 8 x i32> @vmulhu_vi_nxv8i32_1(<vscale x 8 x i32> %va) {
   %vf = trunc <vscale x 8 x i64> %ve to <vscale x 8 x i32>
   ret <vscale x 8 x i32> %vf
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; RV32: {{.*}}
+; RV64: {{.*}}
diff --git a/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll b/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll
index c0962236f93dd..65b602801b365 100644
--- a/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll
+++ b/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll
@@ -1,36 +1,36 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE2,X64-NO-BMI2,X64-NO-SHLD-NO-BMI2,FALLBACK0
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE2,X64-NO-BMI2,X64-HAVE-SHLD-NO-BMI2,FALLBACK1
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE2,X64-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2,FALLBACK2
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE2,X64-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2,FALLBACK3
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE42,X64-NO-BMI2,X64-NO-SHLD-NO-BMI2,FALLBACK4
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE42,X64-NO-BMI2,X64-HAVE-SHLD-NO-BMI2,FALLBACK5
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE42,X64-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2,FALLBACK6
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE42,X64-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2,FALLBACK7
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX1,X64-NO-BMI2,X64-NO-SHLD-NO-BMI2,FALLBACK8
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX1,X64-NO-BMI2,X64-HAVE-SHLD-NO-BMI2,FALLBACK9
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX1,X64-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2,FALLBACK10
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX1,X64-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2,FALLBACK11
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX512,X64-NO-BMI2,X64-NO-SHLD-NO-BMI2,FALLBACK12
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX512,X64-NO-BMI2,X64-HAVE-SHLD-NO-BMI2,FALLBACK13
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX512,X64-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2,FALLBACK14
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX512,X64-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2,FALLBACK15
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE2,X86-NO-BMI2,X86-NO-SHLD-NO-BMI2,FALLBACK16
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE2,X86-NO-BMI2,X86-HAVE-SHLD-NO-BMI2,FALLBACK17
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE2,X86-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2,FALLBACK18
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE2,X86-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2,FALLBACK19
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse4.2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE42,X86-NO-BMI2,X86-NO-SHLD-NO-BMI2,FALLBACK20
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse4.2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE42,X86-NO-BMI2,X86-HAVE-SHLD-NO-BMI2,FALLBACK21
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse4.2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE42,X86-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2,FALLBACK22
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse4.2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE42,X86-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2,FALLBACK23
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX1,X86-NO-BMI2,X86-NO-SHLD-NO-BMI2,FALLBACK24
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX1,X86-NO-BMI2,X86-HAVE-SHLD-NO-BMI2,FALLBACK25
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX1,X86-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2,FALLBACK26
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX1,X86-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2,FALLBACK27
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx512vl,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX512,X86-NO-BMI2,X86-NO-SHLD-NO-BMI2,FALLBACK28
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx512vl,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX512,X86-NO-BMI2,X86-HAVE-SHLD-NO-BMI2,FALLBACK29
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx512vl,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX512,X86-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2,FALLBACK30
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx512vl,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX512,X86-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2,FALLBACK31
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE2,X64-NO-BMI2,X64-NO-SHLD-NO-BMI2,X64-NO-SHLD-NO-BMI2-SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE2,X64-NO-BMI2,X64-HAVE-SHLD-NO-BMI2,X64-HAVE-SHLD-NO-BMI2-SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE2,X64-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2-SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE2,X64-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2-SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE42,X64-NO-BMI2,X64-NO-SHLD-NO-BMI2,X64-NO-SHLD-NO-BMI2-SSE4
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE42,X64-NO-BMI2,X64-HAVE-SHLD-NO-BMI2,X64-HAVE-SHLD-NO-BMI2-SSE4
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE42,X64-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2-SSE4
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE42,X64-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2-SSE4
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX1,X64-NO-BMI2,X64-NO-SHLD-NO-BMI2,X64-NO-SHLD-NO-BMI2-AVX,X64-NO-SHLD-NO-BMI2-AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX1,X64-NO-BMI2,X64-HAVE-SHLD-NO-BMI2,X64-HAVE-SHLD-NO-BMI2-AVX,X64-HAVE-SHLD-NO-BMI2-AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX1,X64-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2-AVX,X64-NO-SHLD-HAVE-BMI2-AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX1,X64-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2-AVX,X64-HAVE-SHLD-HAVE-BMI2-AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX512,X64-NO-BMI2,X64-NO-SHLD-NO-BMI2,X64-NO-SHLD-NO-BMI2-AVX,X64-NO-SHLD-NO-BMI2-AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX512,X64-NO-BMI2,X64-HAVE-SHLD-NO-BMI2,X64-HAVE-SHLD-NO-BMI2-AVX,X64-HAVE-SHLD-NO-BMI2-AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX512,X64-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2-AVX,X64-NO-SHLD-HAVE-BMI2-AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX512,X64-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2-AVX,X64-HAVE-SHLD-HAVE-BMI2-AVX512
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE2,X86-NO-BMI2,X86-NO-SHLD-NO-BMI2,X86-NO-SHLD-NO-BMI2-SSE2
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE2,X86-NO-BMI2,X86-HAVE-SHLD-NO-BMI2,X86-HAVE-SHLD-NO-BMI2-SSE2
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE2,X86-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2-SSE2
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE2,X86-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2-SSE2
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse4.2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE42,X86-NO-BMI2,X86-NO-SHLD-NO-BMI2,X86-NO-SHLD-NO-BMI2-SSE4
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse4.2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE42,X86-NO-BMI2,X86-HAVE-SHLD-NO-BMI2,X86-HAVE-SHLD-NO-BMI2-SSE4
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse4.2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE42,X86-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2-SSE4
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse4.2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE42,X86-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2-SSE4
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX1,X86-NO-BMI2,X86-NO-SHLD-NO-BMI2,X86-NO-SHLD-NO-BMI2-AVX,X86-NO-SHLD-NO-BMI2-AVX1
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX1,X86-NO-BMI2,X86-HAVE-SHLD-NO-BMI2,X86-HAVE-SHLD-NO-BMI2-AVX,X86-HAVE-SHLD-NO-BMI2-AVX1
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX1,X86-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2-AVX,X86-NO-SHLD-HAVE-BMI2-AVX1
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX1,X86-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2-AVX,X86-HAVE-SHLD-HAVE-BMI2-AVX1
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx512vl,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX512,X86-NO-BMI2,X86-NO-SHLD-NO-BMI2,X86-NO-SHLD-NO-BMI2-AVX,X86-NO-SHLD-NO-BMI2-AVX512
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx512vl,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX512,X86-NO-BMI2,X86-HAVE-SHLD-NO-BMI2,X86-HAVE-SHLD-NO-BMI2-AVX,X86-HAVE-SHLD-NO-BMI2-AVX512
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx512vl,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX512,X86-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2-AVX,X86-NO-SHLD-HAVE-BMI2-AVX512
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx512vl,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX512,X86-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2-AVX,X86-HAVE-SHLD-HAVE-BMI2-AVX512
 
 define void @lshr_4bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; X64-NO-BMI2-LABEL: lshr_4bytes:
@@ -646,787 +646,596 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; X64-HAVE-SHLD-HAVE-BMI2-NEXT:    movq %rax, (%rdx)
 ; X64-HAVE-SHLD-HAVE-BMI2-NEXT:    retq
 ;
-; FALLBACK16-LABEL: lshr_16bytes:
-; FALLBACK16:       # %bb.0:
-; FALLBACK16-NEXT:    pushl %ebp
-; FALLBACK16-NEXT:    pushl %ebx
-; FALLBACK16-NEXT:    pushl %edi
-; FALLBACK16-NEXT:    pushl %esi
-; FALLBACK16-NEXT:    subl $60, %esp
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK16-NEXT:    movl (%ecx), %edx
-; FALLBACK16-NEXT:    movl 4(%ecx), %esi
-; FALLBACK16-NEXT:    movl 8(%ecx), %edi
-; FALLBACK16-NEXT:    movl 12(%ecx), %ecx
-; FALLBACK16-NEXT:    movb (%eax), %ah
-; FALLBACK16-NEXT:    movb %ah, %al
-; FALLBACK16-NEXT:    shlb $3, %al
-; FALLBACK16-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK16-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    andb $12, %ah
-; FALLBACK16-NEXT:    movzbl %ah, %ebp
-; FALLBACK16-NEXT:    movl 20(%esp,%ebp), %esi
-; FALLBACK16-NEXT:    movl %esi, %ebx
-; FALLBACK16-NEXT:    movl %eax, %ecx
-; FALLBACK16-NEXT:    shrl %cl, %ebx
-; FALLBACK16-NEXT:    movl %eax, %edx
-; FALLBACK16-NEXT:    notb %dl
-; FALLBACK16-NEXT:    movl 24(%esp,%ebp), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    leal (%ecx,%ecx), %edi
-; FALLBACK16-NEXT:    movl %edx, %ecx
-; FALLBACK16-NEXT:    shll %cl, %edi
-; FALLBACK16-NEXT:    orl %ebx, %edi
-; FALLBACK16-NEXT:    movl 16(%esp,%ebp), %ebx
-; FALLBACK16-NEXT:    movl %eax, %ecx
-; FALLBACK16-NEXT:    shrl %cl, %ebx
-; FALLBACK16-NEXT:    addl %esi, %esi
-; FALLBACK16-NEXT:    movl %edx, %ecx
-; FALLBACK16-NEXT:    shll %cl, %esi
-; FALLBACK16-NEXT:    orl %ebx, %esi
-; FALLBACK16-NEXT:    movl %eax, %ecx
-; FALLBACK16-NEXT:    shrl %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; FALLBACK16-NEXT:    movl 28(%esp,%ebp), %ebx
-; FALLBACK16-NEXT:    leal (%ebx,%ebx), %ebp
-; FALLBACK16-NEXT:    movl %edx, %ecx
-; FALLBACK16-NEXT:    shll %cl, %ebp
-; FALLBACK16-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK16-NEXT:    movl %eax, %ecx
-; FALLBACK16-NEXT:    shrl %cl, %ebx
-; FALLBACK16-NEXT:    movl %ebx, 12(%edx)
-; FALLBACK16-NEXT:    movl %ebp, 8(%edx)
-; FALLBACK16-NEXT:    movl %esi, (%edx)
-; FALLBACK16-NEXT:    movl %edi, 4(%edx)
-; FALLBACK16-NEXT:    addl $60, %esp
-; FALLBACK16-NEXT:    popl %esi
-; FALLBACK16-NEXT:    popl %edi
-; FALLBACK16-NEXT:    popl %ebx
-; FALLBACK16-NEXT:    popl %ebp
-; FALLBACK16-NEXT:    retl
-;
-; FALLBACK17-LABEL: lshr_16bytes:
-; FALLBACK17:       # %bb.0:
-; FALLBACK17-NEXT:    pushl %ebp
-; FALLBACK17-NEXT:    pushl %ebx
-; FALLBACK17-NEXT:    pushl %edi
-; FALLBACK17-NEXT:    pushl %esi
-; FALLBACK17-NEXT:    subl $44, %esp
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK17-NEXT:    movl (%edx), %esi
-; FALLBACK17-NEXT:    movl 4(%edx), %edi
-; FALLBACK17-NEXT:    movl 8(%edx), %ebx
-; FALLBACK17-NEXT:    movl 12(%edx), %edx
-; FALLBACK17-NEXT:    movb (%ecx), %ch
-; FALLBACK17-NEXT:    movb %ch, %cl
-; FALLBACK17-NEXT:    shlb $3, %cl
-; FALLBACK17-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK17-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %esi, (%esp)
-; FALLBACK17-NEXT:    andb $12, %ch
-; FALLBACK17-NEXT:    movzbl %ch, %ebx
-; FALLBACK17-NEXT:    movl 8(%esp,%ebx), %esi
-; FALLBACK17-NEXT:    movl (%esp,%ebx), %edx
-; FALLBACK17-NEXT:    movl 4(%esp,%ebx), %ebp
-; FALLBACK17-NEXT:    movl %ebp, %edi
-; FALLBACK17-NEXT:    shrdl %cl, %esi, %edi
-; FALLBACK17-NEXT:    movl 12(%esp,%ebx), %ebx
-; FALLBACK17-NEXT:    shrdl %cl, %ebx, %esi
-; FALLBACK17-NEXT:    shrdl %cl, %ebp, %edx
-; FALLBACK17-NEXT:    shrl %cl, %ebx
-; FALLBACK17-NEXT:    movl %esi, 8(%eax)
-; FALLBACK17-NEXT:    movl %ebx, 12(%eax)
-; FALLBACK17-NEXT:    movl %edx, (%eax)
-; FALLBACK17-NEXT:    movl %edi, 4(%eax)
-; FALLBACK17-NEXT:    addl $44, %esp
-; FALLBACK17-NEXT:    popl %esi
-; FALLBACK17-NEXT:    popl %edi
-; FALLBACK17-NEXT:    popl %ebx
-; FALLBACK17-NEXT:    popl %ebp
-; FALLBACK17-NEXT:    retl
-;
-; FALLBACK18-LABEL: lshr_16bytes:
-; FALLBACK18:       # %bb.0:
-; FALLBACK18-NEXT:    pushl %ebp
-; FALLBACK18-NEXT:    pushl %ebx
-; FALLBACK18-NEXT:    pushl %edi
-; FALLBACK18-NEXT:    pushl %esi
-; FALLBACK18-NEXT:    subl $44, %esp
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK18-NEXT:    movl (%ecx), %edx
-; FALLBACK18-NEXT:    movl 4(%ecx), %esi
-; FALLBACK18-NEXT:    movl 8(%ecx), %edi
-; FALLBACK18-NEXT:    movl 12(%ecx), %ecx
-; FALLBACK18-NEXT:    movzbl (%eax), %ebx
-; FALLBACK18-NEXT:    movl %ebx, %eax
-; FALLBACK18-NEXT:    shlb $3, %al
-; FALLBACK18-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK18-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %edx, (%esp)
-; FALLBACK18-NEXT:    movl %eax, %ecx
-; FALLBACK18-NEXT:    andb $12, %bl
-; FALLBACK18-NEXT:    movzbl %bl, %edi
-; FALLBACK18-NEXT:    movl 4(%esp,%edi), %ebx
-; FALLBACK18-NEXT:    movl 8(%esp,%edi), %esi
-; FALLBACK18-NEXT:    shrxl %ecx, %ebx, %ebp
-; FALLBACK18-NEXT:    notb %al
-; FALLBACK18-NEXT:    leal (%esi,%esi), %edx
-; FALLBACK18-NEXT:    shlxl %eax, %edx, %edx
-; FALLBACK18-NEXT:    orl %ebp, %edx
-; FALLBACK18-NEXT:    shrxl %ecx, (%esp,%edi), %ebp
-; FALLBACK18-NEXT:    addl %ebx, %ebx
-; FALLBACK18-NEXT:    shlxl %eax, %ebx, %ebx
-; FALLBACK18-NEXT:    orl %ebp, %ebx
-; FALLBACK18-NEXT:    movl 12(%esp,%edi), %edi
-; FALLBACK18-NEXT:    leal (%edi,%edi), %ebp
-; FALLBACK18-NEXT:    shlxl %eax, %ebp, %eax
-; FALLBACK18-NEXT:    shrxl %ecx, %esi, %esi
-; FALLBACK18-NEXT:    orl %esi, %eax
-; FALLBACK18-NEXT:    shrxl %ecx, %edi, %ecx
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; FALLBACK18-NEXT:    movl %ecx, 12(%esi)
-; FALLBACK18-NEXT:    movl %eax, 8(%esi)
-; FALLBACK18-NEXT:    movl %ebx, (%esi)
-; FALLBACK18-NEXT:    movl %edx, 4(%esi)
-; FALLBACK18-NEXT:    addl $44, %esp
-; FALLBACK18-NEXT:    popl %esi
-; FALLBACK18-NEXT:    popl %edi
-; FALLBACK18-NEXT:    popl %ebx
-; FALLBACK18-NEXT:    popl %ebp
-; FALLBACK18-NEXT:    retl
-;
-; FALLBACK19-LABEL: lshr_16bytes:
-; FALLBACK19:       # %bb.0:
-; FALLBACK19-NEXT:    pushl %ebp
-; FALLBACK19-NEXT:    pushl %ebx
-; FALLBACK19-NEXT:    pushl %edi
-; FALLBACK19-NEXT:    pushl %esi
-; FALLBACK19-NEXT:    subl $44, %esp
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK19-NEXT:    movl (%edx), %esi
-; FALLBACK19-NEXT:    movl 4(%edx), %edi
-; FALLBACK19-NEXT:    movl 8(%edx), %ebx
-; FALLBACK19-NEXT:    movl 12(%edx), %edx
-; FALLBACK19-NEXT:    movzbl (%ecx), %eax
-; FALLBACK19-NEXT:    movl %eax, %ecx
-; FALLBACK19-NEXT:    shlb $3, %cl
-; FALLBACK19-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK19-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %esi, (%esp)
-; FALLBACK19-NEXT:    andb $12, %al
-; FALLBACK19-NEXT:    movzbl %al, %eax
-; FALLBACK19-NEXT:    movl 8(%esp,%eax), %ebx
-; FALLBACK19-NEXT:    movl (%esp,%eax), %edx
-; FALLBACK19-NEXT:    movl 4(%esp,%eax), %esi
-; FALLBACK19-NEXT:    movl %esi, %edi
-; FALLBACK19-NEXT:    shrdl %cl, %ebx, %edi
-; FALLBACK19-NEXT:    movl 12(%esp,%eax), %eax
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK19-NEXT:    movl %ebx, 8(%ebp)
-; FALLBACK19-NEXT:    shrxl %ecx, %eax, %eax
-; FALLBACK19-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK19-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK19-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK19-NEXT:    movl %edx, (%ebp)
-; FALLBACK19-NEXT:    movl %edi, 4(%ebp)
-; FALLBACK19-NEXT:    addl $44, %esp
-; FALLBACK19-NEXT:    popl %esi
-; FALLBACK19-NEXT:    popl %edi
-; FALLBACK19-NEXT:    popl %ebx
-; FALLBACK19-NEXT:    popl %ebp
-; FALLBACK19-NEXT:    retl
-;
-; FALLBACK20-LABEL: lshr_16bytes:
-; FALLBACK20:       # %bb.0:
-; FALLBACK20-NEXT:    pushl %ebp
-; FALLBACK20-NEXT:    pushl %ebx
-; FALLBACK20-NEXT:    pushl %edi
-; FALLBACK20-NEXT:    pushl %esi
-; FALLBACK20-NEXT:    subl $60, %esp
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK20-NEXT:    movups (%ecx), %xmm0
-; FALLBACK20-NEXT:    movzbl (%eax), %ecx
-; FALLBACK20-NEXT:    movl %ecx, %eax
-; FALLBACK20-NEXT:    shlb $3, %al
-; FALLBACK20-NEXT:    xorps %xmm1, %xmm1
-; FALLBACK20-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    andb $12, %cl
-; FALLBACK20-NEXT:    movzbl %cl, %edi
-; FALLBACK20-NEXT:    movl 16(%esp,%edi), %ebx
-; FALLBACK20-NEXT:    movl 20(%esp,%edi), %esi
-; FALLBACK20-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %ebx
-; FALLBACK20-NEXT:    movl %eax, %edx
-; FALLBACK20-NEXT:    notb %dl
-; FALLBACK20-NEXT:    addl %esi, %esi
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shll %cl, %esi
-; FALLBACK20-NEXT:    orl %ebx, %esi
-; FALLBACK20-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 24(%esp,%edi), %ebx
-; FALLBACK20-NEXT:    movl %ebx, %esi
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %esi
-; FALLBACK20-NEXT:    movl 28(%esp,%edi), %edi
-; FALLBACK20-NEXT:    leal (%edi,%edi), %ebp
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shll %cl, %ebp
-; FALLBACK20-NEXT:    orl %esi, %ebp
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %cl, %esi
-; FALLBACK20-NEXT:    addl %ebx, %ebx
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    orl %esi, %ebx
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %edi
-; FALLBACK20-NEXT:    movl %edi, 12(%edx)
-; FALLBACK20-NEXT:    movl %ebx, 4(%edx)
-; FALLBACK20-NEXT:    movl %ebp, 8(%edx)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK20-NEXT:    movl %eax, (%edx)
-; FALLBACK20-NEXT:    addl $60, %esp
-; FALLBACK20-NEXT:    popl %esi
-; FALLBACK20-NEXT:    popl %edi
-; FALLBACK20-NEXT:    popl %ebx
-; FALLBACK20-NEXT:    popl %ebp
-; FALLBACK20-NEXT:    retl
-;
-; FALLBACK21-LABEL: lshr_16bytes:
-; FALLBACK21:       # %bb.0:
-; FALLBACK21-NEXT:    pushl %ebp
-; FALLBACK21-NEXT:    pushl %ebx
-; FALLBACK21-NEXT:    pushl %edi
-; FALLBACK21-NEXT:    pushl %esi
-; FALLBACK21-NEXT:    subl $44, %esp
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK21-NEXT:    movups (%edx), %xmm0
-; FALLBACK21-NEXT:    movzbl (%ecx), %edx
-; FALLBACK21-NEXT:    movl %edx, %ecx
-; FALLBACK21-NEXT:    shlb $3, %cl
-; FALLBACK21-NEXT:    xorps %xmm1, %xmm1
-; FALLBACK21-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm0, (%esp)
-; FALLBACK21-NEXT:    andb $12, %dl
-; FALLBACK21-NEXT:    movzbl %dl, %ebx
-; FALLBACK21-NEXT:    movl 12(%esp,%ebx), %edx
-; FALLBACK21-NEXT:    movl 8(%esp,%ebx), %ebp
-; FALLBACK21-NEXT:    movl %ebp, %edi
-; FALLBACK21-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK21-NEXT:    movl (%esp,%ebx), %esi
-; FALLBACK21-NEXT:    movl 4(%esp,%ebx), %eax
-; FALLBACK21-NEXT:    movl %eax, %ebx
-; FALLBACK21-NEXT:    shrdl %cl, %ebp, %ebx
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK21-NEXT:    movl %ebx, 4(%ebp)
-; FALLBACK21-NEXT:    movl %edi, 8(%ebp)
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK21-NEXT:    shrl %cl, %edx
-; FALLBACK21-NEXT:    movl %edx, 12(%ebp)
-; FALLBACK21-NEXT:    movl %esi, (%ebp)
-; FALLBACK21-NEXT:    addl $44, %esp
-; FALLBACK21-NEXT:    popl %esi
-; FALLBACK21-NEXT:    popl %edi
-; FALLBACK21-NEXT:    popl %ebx
-; FALLBACK21-NEXT:    popl %ebp
-; FALLBACK21-NEXT:    retl
-;
-; FALLBACK22-LABEL: lshr_16bytes:
-; FALLBACK22:       # %bb.0:
-; FALLBACK22-NEXT:    pushl %ebp
-; FALLBACK22-NEXT:    pushl %ebx
-; FALLBACK22-NEXT:    pushl %edi
-; FALLBACK22-NEXT:    pushl %esi
-; FALLBACK22-NEXT:    subl $60, %esp
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK22-NEXT:    movups (%ecx), %xmm0
-; FALLBACK22-NEXT:    movzbl (%eax), %edx
-; FALLBACK22-NEXT:    movl %edx, %eax
-; FALLBACK22-NEXT:    shlb $3, %al
-; FALLBACK22-NEXT:    xorps %xmm1, %xmm1
-; FALLBACK22-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %eax, %ecx
-; FALLBACK22-NEXT:    andb $12, %dl
-; FALLBACK22-NEXT:    movzbl %dl, %edi
-; FALLBACK22-NEXT:    shrxl %ecx, 16(%esp,%edi), %ebp
-; FALLBACK22-NEXT:    notb %al
-; FALLBACK22-NEXT:    movl 20(%esp,%edi), %edx
-; FALLBACK22-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 24(%esp,%edi), %ebx
-; FALLBACK22-NEXT:    addl %edx, %edx
-; FALLBACK22-NEXT:    shlxl %eax, %edx, %edx
-; FALLBACK22-NEXT:    orl %ebp, %edx
-; FALLBACK22-NEXT:    movl 28(%esp,%edi), %ebp
-; FALLBACK22-NEXT:    leal (%ebp,%ebp), %edi
-; FALLBACK22-NEXT:    shlxl %eax, %edi, %edi
-; FALLBACK22-NEXT:    shrxl %ecx, %ebx, %esi
-; FALLBACK22-NEXT:    orl %esi, %edi
-; FALLBACK22-NEXT:    addl %ebx, %ebx
-; FALLBACK22-NEXT:    shlxl %eax, %ebx, %eax
-; FALLBACK22-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; FALLBACK22-NEXT:    orl %esi, %eax
-; FALLBACK22-NEXT:    shrxl %ecx, %ebp, %ecx
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; FALLBACK22-NEXT:    movl %ecx, 12(%esi)
-; FALLBACK22-NEXT:    movl %eax, 4(%esi)
-; FALLBACK22-NEXT:    movl %edi, 8(%esi)
-; FALLBACK22-NEXT:    movl %edx, (%esi)
-; FALLBACK22-NEXT:    addl $60, %esp
-; FALLBACK22-NEXT:    popl %esi
-; FALLBACK22-NEXT:    popl %edi
-; FALLBACK22-NEXT:    popl %ebx
-; FALLBACK22-NEXT:    popl %ebp
-; FALLBACK22-NEXT:    retl
-;
-; FALLBACK23-LABEL: lshr_16bytes:
-; FALLBACK23:       # %bb.0:
-; FALLBACK23-NEXT:    pushl %ebp
-; FALLBACK23-NEXT:    pushl %ebx
-; FALLBACK23-NEXT:    pushl %edi
-; FALLBACK23-NEXT:    pushl %esi
-; FALLBACK23-NEXT:    subl $44, %esp
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK23-NEXT:    movups (%edx), %xmm0
-; FALLBACK23-NEXT:    movzbl (%ecx), %edx
-; FALLBACK23-NEXT:    movl %edx, %ecx
-; FALLBACK23-NEXT:    shlb $3, %cl
-; FALLBACK23-NEXT:    xorps %xmm1, %xmm1
-; FALLBACK23-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm0, (%esp)
-; FALLBACK23-NEXT:    andb $12, %dl
-; FALLBACK23-NEXT:    movzbl %dl, %ebx
-; FALLBACK23-NEXT:    movl 12(%esp,%ebx), %edx
-; FALLBACK23-NEXT:    movl 8(%esp,%ebx), %ebp
-; FALLBACK23-NEXT:    movl %ebp, %edi
-; FALLBACK23-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK23-NEXT:    movl (%esp,%ebx), %esi
-; FALLBACK23-NEXT:    movl 4(%esp,%ebx), %eax
-; FALLBACK23-NEXT:    movl %eax, %ebx
-; FALLBACK23-NEXT:    shrdl %cl, %ebp, %ebx
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK23-NEXT:    movl %ebx, 4(%ebp)
-; FALLBACK23-NEXT:    movl %edi, 8(%ebp)
-; FALLBACK23-NEXT:    shrxl %ecx, %edx, %edx
-; FALLBACK23-NEXT:    movl %edx, 12(%ebp)
-; FALLBACK23-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK23-NEXT:    movl %esi, (%ebp)
-; FALLBACK23-NEXT:    addl $44, %esp
-; FALLBACK23-NEXT:    popl %esi
-; FALLBACK23-NEXT:    popl %edi
-; FALLBACK23-NEXT:    popl %ebx
-; FALLBACK23-NEXT:    popl %ebp
-; FALLBACK23-NEXT:    retl
-;
-; FALLBACK24-LABEL: lshr_16bytes:
-; FALLBACK24:       # %bb.0:
-; FALLBACK24-NEXT:    pushl %ebp
-; FALLBACK24-NEXT:    pushl %ebx
-; FALLBACK24-NEXT:    pushl %edi
-; FALLBACK24-NEXT:    pushl %esi
-; FALLBACK24-NEXT:    subl $60, %esp
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK24-NEXT:    vmovups (%ecx), %xmm0
-; FALLBACK24-NEXT:    movzbl (%eax), %ecx
-; FALLBACK24-NEXT:    movl %ecx, %eax
-; FALLBACK24-NEXT:    shlb $3, %al
-; FALLBACK24-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK24-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    andb $12, %cl
-; FALLBACK24-NEXT:    movzbl %cl, %edi
-; FALLBACK24-NEXT:    movl 16(%esp,%edi), %ebx
-; FALLBACK24-NEXT:    movl 20(%esp,%edi), %esi
-; FALLBACK24-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %ebx
-; FALLBACK24-NEXT:    movl %eax, %edx
-; FALLBACK24-NEXT:    notb %dl
-; FALLBACK24-NEXT:    addl %esi, %esi
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shll %cl, %esi
-; FALLBACK24-NEXT:    orl %ebx, %esi
-; FALLBACK24-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 24(%esp,%edi), %ebx
-; FALLBACK24-NEXT:    movl %ebx, %esi
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %esi
-; FALLBACK24-NEXT:    movl 28(%esp,%edi), %edi
-; FALLBACK24-NEXT:    leal (%edi,%edi), %ebp
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shll %cl, %ebp
-; FALLBACK24-NEXT:    orl %esi, %ebp
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %cl, %esi
-; FALLBACK24-NEXT:    addl %ebx, %ebx
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    orl %esi, %ebx
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %edi
-; FALLBACK24-NEXT:    movl %edi, 12(%edx)
-; FALLBACK24-NEXT:    movl %ebx, 4(%edx)
-; FALLBACK24-NEXT:    movl %ebp, 8(%edx)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK24-NEXT:    movl %eax, (%edx)
-; FALLBACK24-NEXT:    addl $60, %esp
-; FALLBACK24-NEXT:    popl %esi
-; FALLBACK24-NEXT:    popl %edi
-; FALLBACK24-NEXT:    popl %ebx
-; FALLBACK24-NEXT:    popl %ebp
-; FALLBACK24-NEXT:    retl
-;
-; FALLBACK25-LABEL: lshr_16bytes:
-; FALLBACK25:       # %bb.0:
-; FALLBACK25-NEXT:    pushl %ebp
-; FALLBACK25-NEXT:    pushl %ebx
-; FALLBACK25-NEXT:    pushl %edi
-; FALLBACK25-NEXT:    pushl %esi
-; FALLBACK25-NEXT:    subl $44, %esp
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK25-NEXT:    vmovups (%edx), %xmm0
-; FALLBACK25-NEXT:    movzbl (%ecx), %edx
-; FALLBACK25-NEXT:    movl %edx, %ecx
-; FALLBACK25-NEXT:    shlb $3, %cl
-; FALLBACK25-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK25-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    vmovaps %xmm0, (%esp)
-; FALLBACK25-NEXT:    andb $12, %dl
-; FALLBACK25-NEXT:    movzbl %dl, %ebx
-; FALLBACK25-NEXT:    movl 12(%esp,%ebx), %edx
-; FALLBACK25-NEXT:    movl 8(%esp,%ebx), %ebp
-; FALLBACK25-NEXT:    movl %ebp, %edi
-; FALLBACK25-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK25-NEXT:    movl (%esp,%ebx), %esi
-; FALLBACK25-NEXT:    movl 4(%esp,%ebx), %eax
-; FALLBACK25-NEXT:    movl %eax, %ebx
-; FALLBACK25-NEXT:    shrdl %cl, %ebp, %ebx
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK25-NEXT:    movl %ebx, 4(%ebp)
-; FALLBACK25-NEXT:    movl %edi, 8(%ebp)
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK25-NEXT:    shrl %cl, %edx
-; FALLBACK25-NEXT:    movl %edx, 12(%ebp)
-; FALLBACK25-NEXT:    movl %esi, (%ebp)
-; FALLBACK25-NEXT:    addl $44, %esp
-; FALLBACK25-NEXT:    popl %esi
-; FALLBACK25-NEXT:    popl %edi
-; FALLBACK25-NEXT:    popl %ebx
-; FALLBACK25-NEXT:    popl %ebp
-; FALLBACK25-NEXT:    retl
-;
-; FALLBACK26-LABEL: lshr_16bytes:
-; FALLBACK26:       # %bb.0:
-; FALLBACK26-NEXT:    pushl %ebp
-; FALLBACK26-NEXT:    pushl %ebx
-; FALLBACK26-NEXT:    pushl %edi
-; FALLBACK26-NEXT:    pushl %esi
-; FALLBACK26-NEXT:    subl $60, %esp
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK26-NEXT:    vmovups (%ecx), %xmm0
-; FALLBACK26-NEXT:    movzbl (%eax), %edx
-; FALLBACK26-NEXT:    movl %edx, %eax
-; FALLBACK26-NEXT:    shlb $3, %al
-; FALLBACK26-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK26-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %eax, %ecx
-; FALLBACK26-NEXT:    andb $12, %dl
-; FALLBACK26-NEXT:    movzbl %dl, %edi
-; FALLBACK26-NEXT:    shrxl %ecx, 16(%esp,%edi), %ebp
-; FALLBACK26-NEXT:    notb %al
-; FALLBACK26-NEXT:    movl 20(%esp,%edi), %edx
-; FALLBACK26-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 24(%esp,%edi), %ebx
-; FALLBACK26-NEXT:    addl %edx, %edx
-; FALLBACK26-NEXT:    shlxl %eax, %edx, %edx
-; FALLBACK26-NEXT:    orl %ebp, %edx
-; FALLBACK26-NEXT:    movl 28(%esp,%edi), %ebp
-; FALLBACK26-NEXT:    leal (%ebp,%ebp), %edi
-; FALLBACK26-NEXT:    shlxl %eax, %edi, %edi
-; FALLBACK26-NEXT:    shrxl %ecx, %ebx, %esi
-; FALLBACK26-NEXT:    orl %esi, %edi
-; FALLBACK26-NEXT:    addl %ebx, %ebx
-; FALLBACK26-NEXT:    shlxl %eax, %ebx, %eax
-; FALLBACK26-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; FALLBACK26-NEXT:    orl %esi, %eax
-; FALLBACK26-NEXT:    shrxl %ecx, %ebp, %ecx
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; FALLBACK26-NEXT:    movl %ecx, 12(%esi)
-; FALLBACK26-NEXT:    movl %eax, 4(%esi)
-; FALLBACK26-NEXT:    movl %edi, 8(%esi)
-; FALLBACK26-NEXT:    movl %edx, (%esi)
-; FALLBACK26-NEXT:    addl $60, %esp
-; FALLBACK26-NEXT:    popl %esi
-; FALLBACK26-NEXT:    popl %edi
-; FALLBACK26-NEXT:    popl %ebx
-; FALLBACK26-NEXT:    popl %ebp
-; FALLBACK26-NEXT:    retl
-;
-; FALLBACK27-LABEL: lshr_16bytes:
-; FALLBACK27:       # %bb.0:
-; FALLBACK27-NEXT:    pushl %ebp
-; FALLBACK27-NEXT:    pushl %ebx
-; FALLBACK27-NEXT:    pushl %edi
-; FALLBACK27-NEXT:    pushl %esi
-; FALLBACK27-NEXT:    subl $44, %esp
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK27-NEXT:    vmovups (%edx), %xmm0
-; FALLBACK27-NEXT:    movzbl (%ecx), %edx
-; FALLBACK27-NEXT:    movl %edx, %ecx
-; FALLBACK27-NEXT:    shlb $3, %cl
-; FALLBACK27-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK27-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    vmovaps %xmm0, (%esp)
-; FALLBACK27-NEXT:    andb $12, %dl
-; FALLBACK27-NEXT:    movzbl %dl, %ebx
-; FALLBACK27-NEXT:    movl 12(%esp,%ebx), %edx
-; FALLBACK27-NEXT:    movl 8(%esp,%ebx), %ebp
-; FALLBACK27-NEXT:    movl %ebp, %edi
-; FALLBACK27-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK27-NEXT:    movl (%esp,%ebx), %esi
-; FALLBACK27-NEXT:    movl 4(%esp,%ebx), %eax
-; FALLBACK27-NEXT:    movl %eax, %ebx
-; FALLBACK27-NEXT:    shrdl %cl, %ebp, %ebx
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK27-NEXT:    movl %ebx, 4(%ebp)
-; FALLBACK27-NEXT:    movl %edi, 8(%ebp)
-; FALLBACK27-NEXT:    shrxl %ecx, %edx, %edx
-; FALLBACK27-NEXT:    movl %edx, 12(%ebp)
-; FALLBACK27-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK27-NEXT:    movl %esi, (%ebp)
-; FALLBACK27-NEXT:    addl $44, %esp
-; FALLBACK27-NEXT:    popl %esi
-; FALLBACK27-NEXT:    popl %edi
-; FALLBACK27-NEXT:    popl %ebx
-; FALLBACK27-NEXT:    popl %ebp
-; FALLBACK27-NEXT:    retl
-;
-; FALLBACK28-LABEL: lshr_16bytes:
-; FALLBACK28:       # %bb.0:
-; FALLBACK28-NEXT:    pushl %ebp
-; FALLBACK28-NEXT:    pushl %ebx
-; FALLBACK28-NEXT:    pushl %edi
-; FALLBACK28-NEXT:    pushl %esi
-; FALLBACK28-NEXT:    subl $60, %esp
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK28-NEXT:    vmovups (%ecx), %xmm0
-; FALLBACK28-NEXT:    movzbl (%eax), %ecx
-; FALLBACK28-NEXT:    movl %ecx, %eax
-; FALLBACK28-NEXT:    shlb $3, %al
-; FALLBACK28-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK28-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    andb $12, %cl
-; FALLBACK28-NEXT:    movzbl %cl, %edi
-; FALLBACK28-NEXT:    movl 16(%esp,%edi), %ebx
-; FALLBACK28-NEXT:    movl 20(%esp,%edi), %esi
-; FALLBACK28-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %ebx
-; FALLBACK28-NEXT:    movl %eax, %edx
-; FALLBACK28-NEXT:    notb %dl
-; FALLBACK28-NEXT:    addl %esi, %esi
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shll %cl, %esi
-; FALLBACK28-NEXT:    orl %ebx, %esi
-; FALLBACK28-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 24(%esp,%edi), %ebx
-; FALLBACK28-NEXT:    movl %ebx, %esi
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %esi
-; FALLBACK28-NEXT:    movl 28(%esp,%edi), %edi
-; FALLBACK28-NEXT:    leal (%edi,%edi), %ebp
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shll %cl, %ebp
-; FALLBACK28-NEXT:    orl %esi, %ebp
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %cl, %esi
-; FALLBACK28-NEXT:    addl %ebx, %ebx
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    orl %esi, %ebx
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %edi
-; FALLBACK28-NEXT:    movl %edi, 12(%edx)
-; FALLBACK28-NEXT:    movl %ebx, 4(%edx)
-; FALLBACK28-NEXT:    movl %ebp, 8(%edx)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK28-NEXT:    movl %eax, (%edx)
-; FALLBACK28-NEXT:    addl $60, %esp
-; FALLBACK28-NEXT:    popl %esi
-; FALLBACK28-NEXT:    popl %edi
-; FALLBACK28-NEXT:    popl %ebx
-; FALLBACK28-NEXT:    popl %ebp
-; FALLBACK28-NEXT:    retl
-;
-; FALLBACK29-LABEL: lshr_16bytes:
-; FALLBACK29:       # %bb.0:
-; FALLBACK29-NEXT:    pushl %ebp
-; FALLBACK29-NEXT:    pushl %ebx
-; FALLBACK29-NEXT:    pushl %edi
-; FALLBACK29-NEXT:    pushl %esi
-; FALLBACK29-NEXT:    subl $44, %esp
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK29-NEXT:    vmovups (%edx), %xmm0
-; FALLBACK29-NEXT:    movzbl (%ecx), %edx
-; FALLBACK29-NEXT:    movl %edx, %ecx
-; FALLBACK29-NEXT:    shlb $3, %cl
-; FALLBACK29-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK29-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    vmovaps %xmm0, (%esp)
-; FALLBACK29-NEXT:    andb $12, %dl
-; FALLBACK29-NEXT:    movzbl %dl, %ebx
-; FALLBACK29-NEXT:    movl 12(%esp,%ebx), %edx
-; FALLBACK29-NEXT:    movl 8(%esp,%ebx), %ebp
-; FALLBACK29-NEXT:    movl %ebp, %edi
-; FALLBACK29-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK29-NEXT:    movl (%esp,%ebx), %esi
-; FALLBACK29-NEXT:    movl 4(%esp,%ebx), %eax
-; FALLBACK29-NEXT:    movl %eax, %ebx
-; FALLBACK29-NEXT:    shrdl %cl, %ebp, %ebx
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK29-NEXT:    movl %ebx, 4(%ebp)
-; FALLBACK29-NEXT:    movl %edi, 8(%ebp)
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK29-NEXT:    shrl %cl, %edx
-; FALLBACK29-NEXT:    movl %edx, 12(%ebp)
-; FALLBACK29-NEXT:    movl %esi, (%ebp)
-; FALLBACK29-NEXT:    addl $44, %esp
-; FALLBACK29-NEXT:    popl %esi
-; FALLBACK29-NEXT:    popl %edi
-; FALLBACK29-NEXT:    popl %ebx
-; FALLBACK29-NEXT:    popl %ebp
-; FALLBACK29-NEXT:    retl
-;
-; FALLBACK30-LABEL: lshr_16bytes:
-; FALLBACK30:       # %bb.0:
-; FALLBACK30-NEXT:    pushl %ebp
-; FALLBACK30-NEXT:    pushl %ebx
-; FALLBACK30-NEXT:    pushl %edi
-; FALLBACK30-NEXT:    pushl %esi
-; FALLBACK30-NEXT:    subl $60, %esp
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK30-NEXT:    vmovups (%ecx), %xmm0
-; FALLBACK30-NEXT:    movzbl (%eax), %edx
-; FALLBACK30-NEXT:    movl %edx, %eax
-; FALLBACK30-NEXT:    shlb $3, %al
-; FALLBACK30-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK30-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %eax, %ecx
-; FALLBACK30-NEXT:    andb $12, %dl
-; FALLBACK30-NEXT:    movzbl %dl, %edi
-; FALLBACK30-NEXT:    shrxl %ecx, 16(%esp,%edi), %ebp
-; FALLBACK30-NEXT:    notb %al
-; FALLBACK30-NEXT:    movl 20(%esp,%edi), %edx
-; FALLBACK30-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 24(%esp,%edi), %ebx
-; FALLBACK30-NEXT:    addl %edx, %edx
-; FALLBACK30-NEXT:    shlxl %eax, %edx, %edx
-; FALLBACK30-NEXT:    orl %ebp, %edx
-; FALLBACK30-NEXT:    movl 28(%esp,%edi), %ebp
-; FALLBACK30-NEXT:    leal (%ebp,%ebp), %edi
-; FALLBACK30-NEXT:    shlxl %eax, %edi, %edi
-; FALLBACK30-NEXT:    shrxl %ecx, %ebx, %esi
-; FALLBACK30-NEXT:    orl %esi, %edi
-; FALLBACK30-NEXT:    addl %ebx, %ebx
-; FALLBACK30-NEXT:    shlxl %eax, %ebx, %eax
-; FALLBACK30-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; FALLBACK30-NEXT:    orl %esi, %eax
-; FALLBACK30-NEXT:    shrxl %ecx, %ebp, %ecx
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; FALLBACK30-NEXT:    movl %ecx, 12(%esi)
-; FALLBACK30-NEXT:    movl %eax, 4(%esi)
-; FALLBACK30-NEXT:    movl %edi, 8(%esi)
-; FALLBACK30-NEXT:    movl %edx, (%esi)
-; FALLBACK30-NEXT:    addl $60, %esp
-; FALLBACK30-NEXT:    popl %esi
-; FALLBACK30-NEXT:    popl %edi
-; FALLBACK30-NEXT:    popl %ebx
-; FALLBACK30-NEXT:    popl %ebp
-; FALLBACK30-NEXT:    retl
-;
-; FALLBACK31-LABEL: lshr_16bytes:
-; FALLBACK31:       # %bb.0:
-; FALLBACK31-NEXT:    pushl %ebp
-; FALLBACK31-NEXT:    pushl %ebx
-; FALLBACK31-NEXT:    pushl %edi
-; FALLBACK31-NEXT:    pushl %esi
-; FALLBACK31-NEXT:    subl $44, %esp
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK31-NEXT:    vmovups (%edx), %xmm0
-; FALLBACK31-NEXT:    movzbl (%ecx), %edx
-; FALLBACK31-NEXT:    movl %edx, %ecx
-; FALLBACK31-NEXT:    shlb $3, %cl
-; FALLBACK31-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK31-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    vmovaps %xmm0, (%esp)
-; FALLBACK31-NEXT:    andb $12, %dl
-; FALLBACK31-NEXT:    movzbl %dl, %ebx
-; FALLBACK31-NEXT:    movl 12(%esp,%ebx), %edx
-; FALLBACK31-NEXT:    movl 8(%esp,%ebx), %ebp
-; FALLBACK31-NEXT:    movl %ebp, %edi
-; FALLBACK31-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK31-NEXT:    movl (%esp,%ebx), %esi
-; FALLBACK31-NEXT:    movl 4(%esp,%ebx), %eax
-; FALLBACK31-NEXT:    movl %eax, %ebx
-; FALLBACK31-NEXT:    shrdl %cl, %ebp, %ebx
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK31-NEXT:    movl %ebx, 4(%ebp)
-; FALLBACK31-NEXT:    movl %edi, 8(%ebp)
-; FALLBACK31-NEXT:    shrxl %ecx, %edx, %edx
-; FALLBACK31-NEXT:    movl %edx, 12(%ebp)
-; FALLBACK31-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK31-NEXT:    movl %esi, (%ebp)
-; FALLBACK31-NEXT:    addl $44, %esp
-; FALLBACK31-NEXT:    popl %esi
-; FALLBACK31-NEXT:    popl %edi
-; FALLBACK31-NEXT:    popl %ebx
-; FALLBACK31-NEXT:    popl %ebp
-; FALLBACK31-NEXT:    retl
+; X86-NO-SHLD-NO-BMI2-SSE2-LABEL: lshr_16bytes:
+; X86-NO-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    subl $60, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl (%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%ecx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%ecx), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb (%eax), %ah
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ah, %al
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlb $3, %al
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    andb $12, %ah
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movzbl %ah, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    notb %dl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%esp,%ebp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%ecx,%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%esp,%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%esp,%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%ebx,%ebx), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, 12(%edx)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, 8(%edx)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, (%edx)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, 4(%edx)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl $60, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-SSE2-LABEL: lshr_16bytes:
+; X86-HAVE-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    subl $44, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%edx), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%edx), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%edx), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%edx), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movb (%ecx), %ch
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, (%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andb $12, %ch
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movzbl %ch, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%esp,%ebx), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%esp,%ebx), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%esp,%ebx), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %esi, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%esp,%ebx), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %ebx, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %ebp, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, 8(%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, 12(%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, (%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, 4(%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    addl $44, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-SSE2-LABEL: lshr_16bytes:
+; X86-NO-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    subl $44, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%ecx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%eax), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $3, %al
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, (%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $12, %bl
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl %bl, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%esp,%edi), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%esp,%edi), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, %ebx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    notb %al
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%esi,%esi), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %eax, %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, (%esp,%edi), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %eax, %ebx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%esp,%edi), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%edi,%edi), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %eax, %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, %edi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 12(%esi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 8(%esi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, (%esi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, 4(%esi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl $44, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: lshr_16bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    subl $44, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%edx), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%edx), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%edx), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%edx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, (%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $12, %al
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl %al, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%esp,%eax), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%esp,%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%esp,%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %ebx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%esp,%eax), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, 8(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, %eax, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, (%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, 4(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    addl $44, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-SSE4-LABEL: lshr_16bytes:
+; X86-NO-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    subl $60, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlb $3, %al
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm1, %xmm1
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    andb $12, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 16(%esp,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 20(%esp,%edi), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    notb %dl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 24(%esp,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 28(%esp,%edi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%edi,%edi), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %esi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, 12(%edx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, 4(%edx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, 8(%edx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, (%edx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl $60, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-SSE4-LABEL: lshr_16bytes:
+; X86-HAVE-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    subl $44, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups (%edx), %xmm0
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%ecx), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm1, %xmm1
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, (%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andb $12, %dl
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl %dl, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 12(%esp,%ebx), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 8(%esp,%ebx), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl (%esp,%ebx), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 4(%esp,%ebx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %ebp, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    addl $44, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-SSE4-LABEL: lshr_16bytes:
+; X86-NO-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    subl $60, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $3, %al
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm1, %xmm1
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $12, %dl
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl %dl, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, 16(%esp,%edi), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    notb %al
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 20(%esp,%edi), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 24(%esp,%edi), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %eax, %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 28(%esp,%edi), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%ebp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %eax, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %ebx, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %eax, %ebx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %ebp, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 12(%esi)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 4(%esi)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, 8(%esi)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, (%esi)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl $60, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: lshr_16bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    subl $44, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%edx), %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%ecx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm1, %xmm1
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, (%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $12, %dl
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl %dl, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 12(%esp,%ebx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 8(%esp,%ebx), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl (%esp,%ebx), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 4(%esp,%ebx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %ebp, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, 4(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, 8(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %edx, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, 12(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, (%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    addl $44, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-AVX-LABEL: lshr_16bytes:
+; X86-NO-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    subl $60, %esp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%ecx), %xmm0
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shlb $3, %al
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    andb $12, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 16(%esp,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 20(%esp,%edi), %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    notb %dl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 24(%esp,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 28(%esp,%edi), %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    leal (%edi,%edi), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %esi, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, 12(%edx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, 4(%edx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebp, 8(%edx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, (%edx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl $60, %esp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-AVX-LABEL: lshr_16bytes:
+; X86-HAVE-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    subl $44, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%edx), %xmm0
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%ecx), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm0, (%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    andb $12, %dl
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl %dl, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 12(%esp,%ebx), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 8(%esp,%ebx), %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %ebp, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl (%esp,%ebx), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 4(%esp,%ebx), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %ebp, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    addl $44, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-AVX-LABEL: lshr_16bytes:
+; X86-NO-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    subl $60, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%ecx), %xmm0
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $3, %al
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    andb $12, %dl
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl %dl, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ecx, 16(%esp,%edi), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    notb %al
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 20(%esp,%edi), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 24(%esp,%edi), %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %eax, %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %ebp, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 28(%esp,%edi), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (%ebp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %eax, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ecx, %ebx, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %eax, %ebx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ecx, %ebp, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 12(%esi)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 4(%esi)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, 8(%esi)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, (%esi)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl $60, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-LABEL: lshr_16bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    subl $44, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%edx), %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%ecx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm0, (%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    andb $12, %dl
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl %dl, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 12(%esp,%ebx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 8(%esp,%ebx), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebp, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl (%esp,%ebx), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 4(%esp,%ebx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %ebp, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebx, 4(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, 8(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ecx, %edx, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, 12(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, (%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    addl $44, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    retl
   %src = load i128, ptr %src.ptr, align 1
   %byteOff = load i128, ptr %byteOff.ptr, align 1
   %bitOff = shl i128 %byteOff, 3
@@ -1664,791 +1473,599 @@ define void @shl_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 ; X64-HAVE-SHLD-HAVE-BMI2-NEXT:    movq %rsi, (%rdx)
 ; X64-HAVE-SHLD-HAVE-BMI2-NEXT:    retq
 ;
-; FALLBACK16-LABEL: shl_16bytes:
-; FALLBACK16:       # %bb.0:
-; FALLBACK16-NEXT:    pushl %ebp
-; FALLBACK16-NEXT:    pushl %ebx
-; FALLBACK16-NEXT:    pushl %edi
-; FALLBACK16-NEXT:    pushl %esi
-; FALLBACK16-NEXT:    subl $60, %esp
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK16-NEXT:    movl (%ecx), %ebx
-; FALLBACK16-NEXT:    movl 4(%ecx), %esi
-; FALLBACK16-NEXT:    movl 8(%ecx), %edi
-; FALLBACK16-NEXT:    movl 12(%ecx), %ecx
-; FALLBACK16-NEXT:    movb (%eax), %ah
-; FALLBACK16-NEXT:    movb %ah, %dh
-; FALLBACK16-NEXT:    shlb $3, %dh
-; FALLBACK16-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK16-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    andb $12, %ah
-; FALLBACK16-NEXT:    negb %ah
-; FALLBACK16-NEXT:    movsbl %ah, %ebp
-; FALLBACK16-NEXT:    movl 32(%esp,%ebp), %ebx
-; FALLBACK16-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 36(%esp,%ebp), %esi
-; FALLBACK16-NEXT:    movl %esi, %edi
-; FALLBACK16-NEXT:    movb %dh, %cl
-; FALLBACK16-NEXT:    shll %cl, %edi
-; FALLBACK16-NEXT:    movb %dh, %dl
-; FALLBACK16-NEXT:    notb %dl
-; FALLBACK16-NEXT:    shrl %ebx
-; FALLBACK16-NEXT:    movl %edx, %ecx
-; FALLBACK16-NEXT:    shrl %cl, %ebx
-; FALLBACK16-NEXT:    orl %edi, %ebx
-; FALLBACK16-NEXT:    movl 44(%esp,%ebp), %eax
-; FALLBACK16-NEXT:    movb %dh, %cl
-; FALLBACK16-NEXT:    shll %cl, %eax
-; FALLBACK16-NEXT:    movl 40(%esp,%ebp), %edi
-; FALLBACK16-NEXT:    movl %edi, %ebp
-; FALLBACK16-NEXT:    shrl %ebp
-; FALLBACK16-NEXT:    movl %edx, %ecx
-; FALLBACK16-NEXT:    shrl %cl, %ebp
-; FALLBACK16-NEXT:    orl %eax, %ebp
-; FALLBACK16-NEXT:    movb %dh, %cl
-; FALLBACK16-NEXT:    shll %cl, %edi
-; FALLBACK16-NEXT:    shrl %esi
-; FALLBACK16-NEXT:    movl %edx, %ecx
-; FALLBACK16-NEXT:    shrl %cl, %esi
-; FALLBACK16-NEXT:    orl %edi, %esi
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK16-NEXT:    movb %dh, %cl
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK16-NEXT:    shll %cl, %edx
-; FALLBACK16-NEXT:    movl %edx, (%eax)
-; FALLBACK16-NEXT:    movl %esi, 8(%eax)
-; FALLBACK16-NEXT:    movl %ebp, 12(%eax)
-; FALLBACK16-NEXT:    movl %ebx, 4(%eax)
-; FALLBACK16-NEXT:    addl $60, %esp
-; FALLBACK16-NEXT:    popl %esi
-; FALLBACK16-NEXT:    popl %edi
-; FALLBACK16-NEXT:    popl %ebx
-; FALLBACK16-NEXT:    popl %ebp
-; FALLBACK16-NEXT:    retl
-;
-; FALLBACK17-LABEL: shl_16bytes:
-; FALLBACK17:       # %bb.0:
-; FALLBACK17-NEXT:    pushl %ebx
-; FALLBACK17-NEXT:    pushl %edi
-; FALLBACK17-NEXT:    pushl %esi
-; FALLBACK17-NEXT:    subl $32, %esp
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK17-NEXT:    movl (%edx), %esi
-; FALLBACK17-NEXT:    movl 4(%edx), %edi
-; FALLBACK17-NEXT:    movl 8(%edx), %ebx
-; FALLBACK17-NEXT:    movl 12(%edx), %edx
-; FALLBACK17-NEXT:    movb (%ecx), %ch
-; FALLBACK17-NEXT:    movb %ch, %cl
-; FALLBACK17-NEXT:    shlb $3, %cl
-; FALLBACK17-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK17-NEXT:    movaps %xmm0, (%esp)
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    andb $12, %ch
-; FALLBACK17-NEXT:    negb %ch
-; FALLBACK17-NEXT:    movsbl %ch, %edi
-; FALLBACK17-NEXT:    movl 24(%esp,%edi), %esi
-; FALLBACK17-NEXT:    movl 28(%esp,%edi), %edx
-; FALLBACK17-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK17-NEXT:    movl 16(%esp,%edi), %ebx
-; FALLBACK17-NEXT:    movl 20(%esp,%edi), %edi
-; FALLBACK17-NEXT:    shldl %cl, %edi, %esi
-; FALLBACK17-NEXT:    shldl %cl, %ebx, %edi
-; FALLBACK17-NEXT:    shll %cl, %ebx
-; FALLBACK17-NEXT:    movl %esi, 8(%eax)
-; FALLBACK17-NEXT:    movl %edx, 12(%eax)
-; FALLBACK17-NEXT:    movl %ebx, (%eax)
-; FALLBACK17-NEXT:    movl %edi, 4(%eax)
-; FALLBACK17-NEXT:    addl $32, %esp
-; FALLBACK17-NEXT:    popl %esi
-; FALLBACK17-NEXT:    popl %edi
-; FALLBACK17-NEXT:    popl %ebx
-; FALLBACK17-NEXT:    retl
-;
-; FALLBACK18-LABEL: shl_16bytes:
-; FALLBACK18:       # %bb.0:
-; FALLBACK18-NEXT:    pushl %ebp
-; FALLBACK18-NEXT:    pushl %ebx
-; FALLBACK18-NEXT:    pushl %edi
-; FALLBACK18-NEXT:    pushl %esi
-; FALLBACK18-NEXT:    subl $44, %esp
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK18-NEXT:    movl (%ecx), %edx
-; FALLBACK18-NEXT:    movl 4(%ecx), %esi
-; FALLBACK18-NEXT:    movl 8(%ecx), %edi
-; FALLBACK18-NEXT:    movl 12(%ecx), %ecx
-; FALLBACK18-NEXT:    movzbl (%eax), %ebx
-; FALLBACK18-NEXT:    movl %ebx, %eax
-; FALLBACK18-NEXT:    shlb $3, %al
-; FALLBACK18-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK18-NEXT:    movaps %xmm0, (%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %eax, %ecx
-; FALLBACK18-NEXT:    andb $12, %bl
-; FALLBACK18-NEXT:    negb %bl
-; FALLBACK18-NEXT:    movsbl %bl, %esi
-; FALLBACK18-NEXT:    movl 16(%esp,%esi), %ebx
-; FALLBACK18-NEXT:    movl 20(%esp,%esi), %edx
-; FALLBACK18-NEXT:    shlxl %ecx, %edx, %edi
-; FALLBACK18-NEXT:    notb %al
-; FALLBACK18-NEXT:    shlxl %ecx, %ebx, %ebp
-; FALLBACK18-NEXT:    shrl %ebx
-; FALLBACK18-NEXT:    shrxl %eax, %ebx, %ebx
-; FALLBACK18-NEXT:    orl %edi, %ebx
-; FALLBACK18-NEXT:    shlxl %ecx, 28(%esp,%esi), %edi
-; FALLBACK18-NEXT:    movl 24(%esp,%esi), %esi
-; FALLBACK18-NEXT:    shlxl %ecx, %esi, %ecx
-; FALLBACK18-NEXT:    shrl %esi
-; FALLBACK18-NEXT:    shrxl %eax, %esi, %esi
-; FALLBACK18-NEXT:    orl %edi, %esi
-; FALLBACK18-NEXT:    shrl %edx
-; FALLBACK18-NEXT:    shrxl %eax, %edx, %eax
-; FALLBACK18-NEXT:    orl %ecx, %eax
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK18-NEXT:    movl %ebp, (%ecx)
-; FALLBACK18-NEXT:    movl %eax, 8(%ecx)
-; FALLBACK18-NEXT:    movl %esi, 12(%ecx)
-; FALLBACK18-NEXT:    movl %ebx, 4(%ecx)
-; FALLBACK18-NEXT:    addl $44, %esp
-; FALLBACK18-NEXT:    popl %esi
-; FALLBACK18-NEXT:    popl %edi
-; FALLBACK18-NEXT:    popl %ebx
-; FALLBACK18-NEXT:    popl %ebp
-; FALLBACK18-NEXT:    retl
-;
-; FALLBACK19-LABEL: shl_16bytes:
-; FALLBACK19:       # %bb.0:
-; FALLBACK19-NEXT:    pushl %ebp
-; FALLBACK19-NEXT:    pushl %ebx
-; FALLBACK19-NEXT:    pushl %edi
-; FALLBACK19-NEXT:    pushl %esi
-; FALLBACK19-NEXT:    subl $44, %esp
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK19-NEXT:    movl (%edx), %esi
-; FALLBACK19-NEXT:    movl 4(%edx), %edi
-; FALLBACK19-NEXT:    movl 8(%edx), %ebx
-; FALLBACK19-NEXT:    movl 12(%edx), %edx
-; FALLBACK19-NEXT:    movzbl (%ecx), %eax
-; FALLBACK19-NEXT:    movl %eax, %ecx
-; FALLBACK19-NEXT:    shlb $3, %cl
-; FALLBACK19-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK19-NEXT:    movaps %xmm0, (%esp)
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    andb $12, %al
-; FALLBACK19-NEXT:    negb %al
-; FALLBACK19-NEXT:    movsbl %al, %eax
-; FALLBACK19-NEXT:    movl 24(%esp,%eax), %esi
-; FALLBACK19-NEXT:    movl 28(%esp,%eax), %edx
-; FALLBACK19-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK19-NEXT:    movl 16(%esp,%eax), %edi
-; FALLBACK19-NEXT:    movl 20(%esp,%eax), %eax
-; FALLBACK19-NEXT:    shldl %cl, %eax, %esi
-; FALLBACK19-NEXT:    shldl %cl, %edi, %eax
-; FALLBACK19-NEXT:    shlxl %ecx, %edi, %ecx
-; FALLBACK19-NEXT:    movl %esi, 8(%ebp)
-; FALLBACK19-NEXT:    movl %edx, 12(%ebp)
-; FALLBACK19-NEXT:    movl %ecx, (%ebp)
-; FALLBACK19-NEXT:    movl %eax, 4(%ebp)
-; FALLBACK19-NEXT:    addl $44, %esp
-; FALLBACK19-NEXT:    popl %esi
-; FALLBACK19-NEXT:    popl %edi
-; FALLBACK19-NEXT:    popl %ebx
-; FALLBACK19-NEXT:    popl %ebp
-; FALLBACK19-NEXT:    retl
-;
-; FALLBACK20-LABEL: shl_16bytes:
-; FALLBACK20:       # %bb.0:
-; FALLBACK20-NEXT:    pushl %ebp
-; FALLBACK20-NEXT:    pushl %ebx
-; FALLBACK20-NEXT:    pushl %edi
-; FALLBACK20-NEXT:    pushl %esi
-; FALLBACK20-NEXT:    subl $60, %esp
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK20-NEXT:    movups (%ecx), %xmm0
-; FALLBACK20-NEXT:    movzbl (%eax), %ecx
-; FALLBACK20-NEXT:    movl %ecx, %eax
-; FALLBACK20-NEXT:    shlb $3, %al
-; FALLBACK20-NEXT:    xorps %xmm1, %xmm1
-; FALLBACK20-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    andb $12, %cl
-; FALLBACK20-NEXT:    negb %cl
-; FALLBACK20-NEXT:    movsbl %cl, %edi
-; FALLBACK20-NEXT:    movl 44(%esp,%edi), %ebx
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    movl %eax, %edx
-; FALLBACK20-NEXT:    notb %dl
-; FALLBACK20-NEXT:    movl 40(%esp,%edi), %ebp
-; FALLBACK20-NEXT:    movl %ebp, %esi
-; FALLBACK20-NEXT:    shrl %esi
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %esi
-; FALLBACK20-NEXT:    orl %ebx, %esi
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shll %cl, %ebp
-; FALLBACK20-NEXT:    movl 32(%esp,%edi), %ecx
-; FALLBACK20-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 36(%esp,%edi), %ebx
-; FALLBACK20-NEXT:    movl %ebx, %edi
-; FALLBACK20-NEXT:    shrl %edi
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %edi
-; FALLBACK20-NEXT:    orl %ebp, %edi
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %ebp
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %ebp
-; FALLBACK20-NEXT:    orl %ebx, %ebp
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK20-NEXT:    shll %cl, %eax
-; FALLBACK20-NEXT:    movl %eax, (%edx)
-; FALLBACK20-NEXT:    movl %ebp, 4(%edx)
-; FALLBACK20-NEXT:    movl %edi, 8(%edx)
-; FALLBACK20-NEXT:    movl %esi, 12(%edx)
-; FALLBACK20-NEXT:    addl $60, %esp
-; FALLBACK20-NEXT:    popl %esi
-; FALLBACK20-NEXT:    popl %edi
-; FALLBACK20-NEXT:    popl %ebx
-; FALLBACK20-NEXT:    popl %ebp
-; FALLBACK20-NEXT:    retl
-;
-; FALLBACK21-LABEL: shl_16bytes:
-; FALLBACK21:       # %bb.0:
-; FALLBACK21-NEXT:    pushl %ebp
-; FALLBACK21-NEXT:    pushl %ebx
-; FALLBACK21-NEXT:    pushl %edi
-; FALLBACK21-NEXT:    pushl %esi
-; FALLBACK21-NEXT:    subl $44, %esp
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK21-NEXT:    movups (%edx), %xmm0
-; FALLBACK21-NEXT:    movzbl (%ecx), %edx
-; FALLBACK21-NEXT:    movl %edx, %ecx
-; FALLBACK21-NEXT:    shlb $3, %cl
-; FALLBACK21-NEXT:    xorps %xmm1, %xmm1
-; FALLBACK21-NEXT:    movaps %xmm1, (%esp)
-; FALLBACK21-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    andb $12, %dl
-; FALLBACK21-NEXT:    negb %dl
-; FALLBACK21-NEXT:    movsbl %dl, %edi
-; FALLBACK21-NEXT:    movl 24(%esp,%edi), %esi
-; FALLBACK21-NEXT:    movl 28(%esp,%edi), %edx
-; FALLBACK21-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK21-NEXT:    movl 16(%esp,%edi), %ebx
-; FALLBACK21-NEXT:    movl 20(%esp,%edi), %edi
-; FALLBACK21-NEXT:    shldl %cl, %edi, %esi
-; FALLBACK21-NEXT:    movl %ebx, %ebp
-; FALLBACK21-NEXT:    shll %cl, %ebp
-; FALLBACK21-NEXT:    shldl %cl, %ebx, %edi
-; FALLBACK21-NEXT:    movl %edi, 4(%eax)
-; FALLBACK21-NEXT:    movl %esi, 8(%eax)
-; FALLBACK21-NEXT:    movl %edx, 12(%eax)
-; FALLBACK21-NEXT:    movl %ebp, (%eax)
-; FALLBACK21-NEXT:    addl $44, %esp
-; FALLBACK21-NEXT:    popl %esi
-; FALLBACK21-NEXT:    popl %edi
-; FALLBACK21-NEXT:    popl %ebx
-; FALLBACK21-NEXT:    popl %ebp
-; FALLBACK21-NEXT:    retl
-;
-; FALLBACK22-LABEL: shl_16bytes:
-; FALLBACK22:       # %bb.0:
-; FALLBACK22-NEXT:    pushl %ebp
-; FALLBACK22-NEXT:    pushl %ebx
-; FALLBACK22-NEXT:    pushl %edi
-; FALLBACK22-NEXT:    pushl %esi
-; FALLBACK22-NEXT:    subl $44, %esp
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK22-NEXT:    movups (%ecx), %xmm0
-; FALLBACK22-NEXT:    movzbl (%eax), %edx
-; FALLBACK22-NEXT:    movl %edx, %eax
-; FALLBACK22-NEXT:    shlb $3, %al
-; FALLBACK22-NEXT:    xorps %xmm1, %xmm1
-; FALLBACK22-NEXT:    movaps %xmm1, (%esp)
-; FALLBACK22-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %eax, %ecx
-; FALLBACK22-NEXT:    andb $12, %dl
-; FALLBACK22-NEXT:    negb %dl
-; FALLBACK22-NEXT:    movsbl %dl, %edx
-; FALLBACK22-NEXT:    shlxl %ecx, 28(%esp,%edx), %edi
-; FALLBACK22-NEXT:    notb %al
-; FALLBACK22-NEXT:    movl 24(%esp,%edx), %esi
-; FALLBACK22-NEXT:    shlxl %ecx, %esi, %ebx
-; FALLBACK22-NEXT:    shrl %esi
-; FALLBACK22-NEXT:    shrxl %eax, %esi, %esi
-; FALLBACK22-NEXT:    orl %edi, %esi
-; FALLBACK22-NEXT:    movl 20(%esp,%edx), %edi
-; FALLBACK22-NEXT:    movl %edi, %ebp
-; FALLBACK22-NEXT:    shrl %ebp
-; FALLBACK22-NEXT:    shrxl %eax, %ebp, %ebp
-; FALLBACK22-NEXT:    orl %ebx, %ebp
-; FALLBACK22-NEXT:    shlxl %ecx, %edi, %edi
-; FALLBACK22-NEXT:    movl 16(%esp,%edx), %edx
-; FALLBACK22-NEXT:    shlxl %ecx, %edx, %ecx
-; FALLBACK22-NEXT:    shrl %edx
-; FALLBACK22-NEXT:    shrxl %eax, %edx, %eax
-; FALLBACK22-NEXT:    orl %edi, %eax
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK22-NEXT:    movl %ecx, (%edx)
-; FALLBACK22-NEXT:    movl %eax, 4(%edx)
-; FALLBACK22-NEXT:    movl %ebp, 8(%edx)
-; FALLBACK22-NEXT:    movl %esi, 12(%edx)
-; FALLBACK22-NEXT:    addl $44, %esp
-; FALLBACK22-NEXT:    popl %esi
-; FALLBACK22-NEXT:    popl %edi
-; FALLBACK22-NEXT:    popl %ebx
-; FALLBACK22-NEXT:    popl %ebp
-; FALLBACK22-NEXT:    retl
-;
-; FALLBACK23-LABEL: shl_16bytes:
-; FALLBACK23:       # %bb.0:
-; FALLBACK23-NEXT:    pushl %ebp
-; FALLBACK23-NEXT:    pushl %ebx
-; FALLBACK23-NEXT:    pushl %edi
-; FALLBACK23-NEXT:    pushl %esi
-; FALLBACK23-NEXT:    subl $44, %esp
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK23-NEXT:    movups (%edx), %xmm0
-; FALLBACK23-NEXT:    movzbl (%ecx), %edx
-; FALLBACK23-NEXT:    movl %edx, %ecx
-; FALLBACK23-NEXT:    shlb $3, %cl
-; FALLBACK23-NEXT:    xorps %xmm1, %xmm1
-; FALLBACK23-NEXT:    movaps %xmm1, (%esp)
-; FALLBACK23-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    andb $12, %dl
-; FALLBACK23-NEXT:    negb %dl
-; FALLBACK23-NEXT:    movsbl %dl, %edi
-; FALLBACK23-NEXT:    movl 24(%esp,%edi), %esi
-; FALLBACK23-NEXT:    movl 28(%esp,%edi), %edx
-; FALLBACK23-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK23-NEXT:    movl 16(%esp,%edi), %ebx
-; FALLBACK23-NEXT:    movl 20(%esp,%edi), %edi
-; FALLBACK23-NEXT:    shldl %cl, %edi, %esi
-; FALLBACK23-NEXT:    shlxl %ecx, %ebx, %ebp
-; FALLBACK23-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK23-NEXT:    shldl %cl, %ebx, %edi
-; FALLBACK23-NEXT:    movl %edi, 4(%eax)
-; FALLBACK23-NEXT:    movl %esi, 8(%eax)
-; FALLBACK23-NEXT:    movl %edx, 12(%eax)
-; FALLBACK23-NEXT:    movl %ebp, (%eax)
-; FALLBACK23-NEXT:    addl $44, %esp
-; FALLBACK23-NEXT:    popl %esi
-; FALLBACK23-NEXT:    popl %edi
-; FALLBACK23-NEXT:    popl %ebx
-; FALLBACK23-NEXT:    popl %ebp
-; FALLBACK23-NEXT:    retl
-;
-; FALLBACK24-LABEL: shl_16bytes:
-; FALLBACK24:       # %bb.0:
-; FALLBACK24-NEXT:    pushl %ebp
-; FALLBACK24-NEXT:    pushl %ebx
-; FALLBACK24-NEXT:    pushl %edi
-; FALLBACK24-NEXT:    pushl %esi
-; FALLBACK24-NEXT:    subl $60, %esp
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK24-NEXT:    vmovups (%ecx), %xmm0
-; FALLBACK24-NEXT:    movzbl (%eax), %ecx
-; FALLBACK24-NEXT:    movl %ecx, %eax
-; FALLBACK24-NEXT:    shlb $3, %al
-; FALLBACK24-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK24-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    andb $12, %cl
-; FALLBACK24-NEXT:    negb %cl
-; FALLBACK24-NEXT:    movsbl %cl, %edi
-; FALLBACK24-NEXT:    movl 44(%esp,%edi), %ebx
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    movl %eax, %edx
-; FALLBACK24-NEXT:    notb %dl
-; FALLBACK24-NEXT:    movl 40(%esp,%edi), %ebp
-; FALLBACK24-NEXT:    movl %ebp, %esi
-; FALLBACK24-NEXT:    shrl %esi
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %esi
-; FALLBACK24-NEXT:    orl %ebx, %esi
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shll %cl, %ebp
-; FALLBACK24-NEXT:    movl 32(%esp,%edi), %ecx
-; FALLBACK24-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 36(%esp,%edi), %ebx
-; FALLBACK24-NEXT:    movl %ebx, %edi
-; FALLBACK24-NEXT:    shrl %edi
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %edi
-; FALLBACK24-NEXT:    orl %ebp, %edi
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %ebp
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %ebp
-; FALLBACK24-NEXT:    orl %ebx, %ebp
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK24-NEXT:    shll %cl, %eax
-; FALLBACK24-NEXT:    movl %eax, (%edx)
-; FALLBACK24-NEXT:    movl %ebp, 4(%edx)
-; FALLBACK24-NEXT:    movl %edi, 8(%edx)
-; FALLBACK24-NEXT:    movl %esi, 12(%edx)
-; FALLBACK24-NEXT:    addl $60, %esp
-; FALLBACK24-NEXT:    popl %esi
-; FALLBACK24-NEXT:    popl %edi
-; FALLBACK24-NEXT:    popl %ebx
-; FALLBACK24-NEXT:    popl %ebp
-; FALLBACK24-NEXT:    retl
-;
-; FALLBACK25-LABEL: shl_16bytes:
-; FALLBACK25:       # %bb.0:
-; FALLBACK25-NEXT:    pushl %ebp
-; FALLBACK25-NEXT:    pushl %ebx
-; FALLBACK25-NEXT:    pushl %edi
-; FALLBACK25-NEXT:    pushl %esi
-; FALLBACK25-NEXT:    subl $44, %esp
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK25-NEXT:    vmovups (%edx), %xmm0
-; FALLBACK25-NEXT:    movzbl (%ecx), %edx
-; FALLBACK25-NEXT:    movl %edx, %ecx
-; FALLBACK25-NEXT:    shlb $3, %cl
-; FALLBACK25-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK25-NEXT:    vmovaps %xmm1, (%esp)
-; FALLBACK25-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    andb $12, %dl
-; FALLBACK25-NEXT:    negb %dl
-; FALLBACK25-NEXT:    movsbl %dl, %edi
-; FALLBACK25-NEXT:    movl 24(%esp,%edi), %esi
-; FALLBACK25-NEXT:    movl 28(%esp,%edi), %edx
-; FALLBACK25-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK25-NEXT:    movl 16(%esp,%edi), %ebx
-; FALLBACK25-NEXT:    movl 20(%esp,%edi), %edi
-; FALLBACK25-NEXT:    shldl %cl, %edi, %esi
-; FALLBACK25-NEXT:    movl %ebx, %ebp
-; FALLBACK25-NEXT:    shll %cl, %ebp
-; FALLBACK25-NEXT:    shldl %cl, %ebx, %edi
-; FALLBACK25-NEXT:    movl %edi, 4(%eax)
-; FALLBACK25-NEXT:    movl %esi, 8(%eax)
-; FALLBACK25-NEXT:    movl %edx, 12(%eax)
-; FALLBACK25-NEXT:    movl %ebp, (%eax)
-; FALLBACK25-NEXT:    addl $44, %esp
-; FALLBACK25-NEXT:    popl %esi
-; FALLBACK25-NEXT:    popl %edi
-; FALLBACK25-NEXT:    popl %ebx
-; FALLBACK25-NEXT:    popl %ebp
-; FALLBACK25-NEXT:    retl
-;
-; FALLBACK26-LABEL: shl_16bytes:
-; FALLBACK26:       # %bb.0:
-; FALLBACK26-NEXT:    pushl %ebp
-; FALLBACK26-NEXT:    pushl %ebx
-; FALLBACK26-NEXT:    pushl %edi
-; FALLBACK26-NEXT:    pushl %esi
-; FALLBACK26-NEXT:    subl $44, %esp
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK26-NEXT:    vmovups (%ecx), %xmm0
-; FALLBACK26-NEXT:    movzbl (%eax), %edx
-; FALLBACK26-NEXT:    movl %edx, %eax
-; FALLBACK26-NEXT:    shlb $3, %al
-; FALLBACK26-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK26-NEXT:    vmovaps %xmm1, (%esp)
-; FALLBACK26-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %eax, %ecx
-; FALLBACK26-NEXT:    andb $12, %dl
-; FALLBACK26-NEXT:    negb %dl
-; FALLBACK26-NEXT:    movsbl %dl, %edx
-; FALLBACK26-NEXT:    shlxl %ecx, 28(%esp,%edx), %edi
-; FALLBACK26-NEXT:    notb %al
-; FALLBACK26-NEXT:    movl 24(%esp,%edx), %esi
-; FALLBACK26-NEXT:    shlxl %ecx, %esi, %ebx
-; FALLBACK26-NEXT:    shrl %esi
-; FALLBACK26-NEXT:    shrxl %eax, %esi, %esi
-; FALLBACK26-NEXT:    orl %edi, %esi
-; FALLBACK26-NEXT:    movl 20(%esp,%edx), %edi
-; FALLBACK26-NEXT:    movl %edi, %ebp
-; FALLBACK26-NEXT:    shrl %ebp
-; FALLBACK26-NEXT:    shrxl %eax, %ebp, %ebp
-; FALLBACK26-NEXT:    orl %ebx, %ebp
-; FALLBACK26-NEXT:    shlxl %ecx, %edi, %edi
-; FALLBACK26-NEXT:    movl 16(%esp,%edx), %edx
-; FALLBACK26-NEXT:    shlxl %ecx, %edx, %ecx
-; FALLBACK26-NEXT:    shrl %edx
-; FALLBACK26-NEXT:    shrxl %eax, %edx, %eax
-; FALLBACK26-NEXT:    orl %edi, %eax
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK26-NEXT:    movl %ecx, (%edx)
-; FALLBACK26-NEXT:    movl %eax, 4(%edx)
-; FALLBACK26-NEXT:    movl %ebp, 8(%edx)
-; FALLBACK26-NEXT:    movl %esi, 12(%edx)
-; FALLBACK26-NEXT:    addl $44, %esp
-; FALLBACK26-NEXT:    popl %esi
-; FALLBACK26-NEXT:    popl %edi
-; FALLBACK26-NEXT:    popl %ebx
-; FALLBACK26-NEXT:    popl %ebp
-; FALLBACK26-NEXT:    retl
-;
-; FALLBACK27-LABEL: shl_16bytes:
-; FALLBACK27:       # %bb.0:
-; FALLBACK27-NEXT:    pushl %ebp
-; FALLBACK27-NEXT:    pushl %ebx
-; FALLBACK27-NEXT:    pushl %edi
-; FALLBACK27-NEXT:    pushl %esi
-; FALLBACK27-NEXT:    subl $44, %esp
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK27-NEXT:    vmovups (%edx), %xmm0
-; FALLBACK27-NEXT:    movzbl (%ecx), %edx
-; FALLBACK27-NEXT:    movl %edx, %ecx
-; FALLBACK27-NEXT:    shlb $3, %cl
-; FALLBACK27-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK27-NEXT:    vmovaps %xmm1, (%esp)
-; FALLBACK27-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    andb $12, %dl
-; FALLBACK27-NEXT:    negb %dl
-; FALLBACK27-NEXT:    movsbl %dl, %edi
-; FALLBACK27-NEXT:    movl 24(%esp,%edi), %esi
-; FALLBACK27-NEXT:    movl 28(%esp,%edi), %edx
-; FALLBACK27-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK27-NEXT:    movl 16(%esp,%edi), %ebx
-; FALLBACK27-NEXT:    movl 20(%esp,%edi), %edi
-; FALLBACK27-NEXT:    shldl %cl, %edi, %esi
-; FALLBACK27-NEXT:    shlxl %ecx, %ebx, %ebp
-; FALLBACK27-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK27-NEXT:    shldl %cl, %ebx, %edi
-; FALLBACK27-NEXT:    movl %edi, 4(%eax)
-; FALLBACK27-NEXT:    movl %esi, 8(%eax)
-; FALLBACK27-NEXT:    movl %edx, 12(%eax)
-; FALLBACK27-NEXT:    movl %ebp, (%eax)
-; FALLBACK27-NEXT:    addl $44, %esp
-; FALLBACK27-NEXT:    popl %esi
-; FALLBACK27-NEXT:    popl %edi
-; FALLBACK27-NEXT:    popl %ebx
-; FALLBACK27-NEXT:    popl %ebp
-; FALLBACK27-NEXT:    retl
-;
-; FALLBACK28-LABEL: shl_16bytes:
-; FALLBACK28:       # %bb.0:
-; FALLBACK28-NEXT:    pushl %ebp
-; FALLBACK28-NEXT:    pushl %ebx
-; FALLBACK28-NEXT:    pushl %edi
-; FALLBACK28-NEXT:    pushl %esi
-; FALLBACK28-NEXT:    subl $60, %esp
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK28-NEXT:    vmovups (%ecx), %xmm0
-; FALLBACK28-NEXT:    movzbl (%eax), %ecx
-; FALLBACK28-NEXT:    movl %ecx, %eax
-; FALLBACK28-NEXT:    shlb $3, %al
-; FALLBACK28-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK28-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    andb $12, %cl
-; FALLBACK28-NEXT:    negb %cl
-; FALLBACK28-NEXT:    movsbl %cl, %edi
-; FALLBACK28-NEXT:    movl 44(%esp,%edi), %ebx
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    movl %eax, %edx
-; FALLBACK28-NEXT:    notb %dl
-; FALLBACK28-NEXT:    movl 40(%esp,%edi), %ebp
-; FALLBACK28-NEXT:    movl %ebp, %esi
-; FALLBACK28-NEXT:    shrl %esi
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %esi
-; FALLBACK28-NEXT:    orl %ebx, %esi
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shll %cl, %ebp
-; FALLBACK28-NEXT:    movl 32(%esp,%edi), %ecx
-; FALLBACK28-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 36(%esp,%edi), %ebx
-; FALLBACK28-NEXT:    movl %ebx, %edi
-; FALLBACK28-NEXT:    shrl %edi
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %edi
-; FALLBACK28-NEXT:    orl %ebp, %edi
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %ebp
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %ebp
-; FALLBACK28-NEXT:    orl %ebx, %ebp
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK28-NEXT:    shll %cl, %eax
-; FALLBACK28-NEXT:    movl %eax, (%edx)
-; FALLBACK28-NEXT:    movl %ebp, 4(%edx)
-; FALLBACK28-NEXT:    movl %edi, 8(%edx)
-; FALLBACK28-NEXT:    movl %esi, 12(%edx)
-; FALLBACK28-NEXT:    addl $60, %esp
-; FALLBACK28-NEXT:    popl %esi
-; FALLBACK28-NEXT:    popl %edi
-; FALLBACK28-NEXT:    popl %ebx
-; FALLBACK28-NEXT:    popl %ebp
-; FALLBACK28-NEXT:    retl
-;
-; FALLBACK29-LABEL: shl_16bytes:
-; FALLBACK29:       # %bb.0:
-; FALLBACK29-NEXT:    pushl %ebp
-; FALLBACK29-NEXT:    pushl %ebx
-; FALLBACK29-NEXT:    pushl %edi
-; FALLBACK29-NEXT:    pushl %esi
-; FALLBACK29-NEXT:    subl $44, %esp
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK29-NEXT:    vmovups (%edx), %xmm0
-; FALLBACK29-NEXT:    movzbl (%ecx), %edx
-; FALLBACK29-NEXT:    movl %edx, %ecx
-; FALLBACK29-NEXT:    shlb $3, %cl
-; FALLBACK29-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK29-NEXT:    vmovaps %xmm1, (%esp)
-; FALLBACK29-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    andb $12, %dl
-; FALLBACK29-NEXT:    negb %dl
-; FALLBACK29-NEXT:    movsbl %dl, %edi
-; FALLBACK29-NEXT:    movl 24(%esp,%edi), %esi
-; FALLBACK29-NEXT:    movl 28(%esp,%edi), %edx
-; FALLBACK29-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK29-NEXT:    movl 16(%esp,%edi), %ebx
-; FALLBACK29-NEXT:    movl 20(%esp,%edi), %edi
-; FALLBACK29-NEXT:    shldl %cl, %edi, %esi
-; FALLBACK29-NEXT:    movl %ebx, %ebp
-; FALLBACK29-NEXT:    shll %cl, %ebp
-; FALLBACK29-NEXT:    shldl %cl, %ebx, %edi
-; FALLBACK29-NEXT:    movl %edi, 4(%eax)
-; FALLBACK29-NEXT:    movl %esi, 8(%eax)
-; FALLBACK29-NEXT:    movl %edx, 12(%eax)
-; FALLBACK29-NEXT:    movl %ebp, (%eax)
-; FALLBACK29-NEXT:    addl $44, %esp
-; FALLBACK29-NEXT:    popl %esi
-; FALLBACK29-NEXT:    popl %edi
-; FALLBACK29-NEXT:    popl %ebx
-; FALLBACK29-NEXT:    popl %ebp
-; FALLBACK29-NEXT:    retl
-;
-; FALLBACK30-LABEL: shl_16bytes:
-; FALLBACK30:       # %bb.0:
-; FALLBACK30-NEXT:    pushl %ebp
-; FALLBACK30-NEXT:    pushl %ebx
-; FALLBACK30-NEXT:    pushl %edi
-; FALLBACK30-NEXT:    pushl %esi
-; FALLBACK30-NEXT:    subl $44, %esp
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK30-NEXT:    vmovups (%ecx), %xmm0
-; FALLBACK30-NEXT:    movzbl (%eax), %edx
-; FALLBACK30-NEXT:    movl %edx, %eax
-; FALLBACK30-NEXT:    shlb $3, %al
-; FALLBACK30-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK30-NEXT:    vmovaps %xmm1, (%esp)
-; FALLBACK30-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %eax, %ecx
-; FALLBACK30-NEXT:    andb $12, %dl
-; FALLBACK30-NEXT:    negb %dl
-; FALLBACK30-NEXT:    movsbl %dl, %edx
-; FALLBACK30-NEXT:    shlxl %ecx, 28(%esp,%edx), %edi
-; FALLBACK30-NEXT:    notb %al
-; FALLBACK30-NEXT:    movl 24(%esp,%edx), %esi
-; FALLBACK30-NEXT:    shlxl %ecx, %esi, %ebx
-; FALLBACK30-NEXT:    shrl %esi
-; FALLBACK30-NEXT:    shrxl %eax, %esi, %esi
-; FALLBACK30-NEXT:    orl %edi, %esi
-; FALLBACK30-NEXT:    movl 20(%esp,%edx), %edi
-; FALLBACK30-NEXT:    movl %edi, %ebp
-; FALLBACK30-NEXT:    shrl %ebp
-; FALLBACK30-NEXT:    shrxl %eax, %ebp, %ebp
-; FALLBACK30-NEXT:    orl %ebx, %ebp
-; FALLBACK30-NEXT:    shlxl %ecx, %edi, %edi
-; FALLBACK30-NEXT:    movl 16(%esp,%edx), %edx
-; FALLBACK30-NEXT:    shlxl %ecx, %edx, %ecx
-; FALLBACK30-NEXT:    shrl %edx
-; FALLBACK30-NEXT:    shrxl %eax, %edx, %eax
-; FALLBACK30-NEXT:    orl %edi, %eax
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK30-NEXT:    movl %ecx, (%edx)
-; FALLBACK30-NEXT:    movl %eax, 4(%edx)
-; FALLBACK30-NEXT:    movl %ebp, 8(%edx)
-; FALLBACK30-NEXT:    movl %esi, 12(%edx)
-; FALLBACK30-NEXT:    addl $44, %esp
-; FALLBACK30-NEXT:    popl %esi
-; FALLBACK30-NEXT:    popl %edi
-; FALLBACK30-NEXT:    popl %ebx
-; FALLBACK30-NEXT:    popl %ebp
-; FALLBACK30-NEXT:    retl
-;
-; FALLBACK31-LABEL: shl_16bytes:
-; FALLBACK31:       # %bb.0:
-; FALLBACK31-NEXT:    pushl %ebp
-; FALLBACK31-NEXT:    pushl %ebx
-; FALLBACK31-NEXT:    pushl %edi
-; FALLBACK31-NEXT:    pushl %esi
-; FALLBACK31-NEXT:    subl $44, %esp
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK31-NEXT:    vmovups (%edx), %xmm0
-; FALLBACK31-NEXT:    movzbl (%ecx), %edx
-; FALLBACK31-NEXT:    movl %edx, %ecx
-; FALLBACK31-NEXT:    shlb $3, %cl
-; FALLBACK31-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK31-NEXT:    vmovaps %xmm1, (%esp)
-; FALLBACK31-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    andb $12, %dl
-; FALLBACK31-NEXT:    negb %dl
-; FALLBACK31-NEXT:    movsbl %dl, %edi
-; FALLBACK31-NEXT:    movl 24(%esp,%edi), %esi
-; FALLBACK31-NEXT:    movl 28(%esp,%edi), %edx
-; FALLBACK31-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK31-NEXT:    movl 16(%esp,%edi), %ebx
-; FALLBACK31-NEXT:    movl 20(%esp,%edi), %edi
-; FALLBACK31-NEXT:    shldl %cl, %edi, %esi
-; FALLBACK31-NEXT:    shlxl %ecx, %ebx, %ebp
-; FALLBACK31-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK31-NEXT:    shldl %cl, %ebx, %edi
-; FALLBACK31-NEXT:    movl %edi, 4(%eax)
-; FALLBACK31-NEXT:    movl %esi, 8(%eax)
-; FALLBACK31-NEXT:    movl %edx, 12(%eax)
-; FALLBACK31-NEXT:    movl %ebp, (%eax)
-; FALLBACK31-NEXT:    addl $44, %esp
-; FALLBACK31-NEXT:    popl %esi
-; FALLBACK31-NEXT:    popl %edi
-; FALLBACK31-NEXT:    popl %ebx
-; FALLBACK31-NEXT:    popl %ebp
-; FALLBACK31-NEXT:    retl
+; X86-NO-SHLD-NO-BMI2-SSE2-LABEL: shl_16bytes:
+; X86-NO-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    subl $60, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl (%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%ecx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%ecx), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb (%eax), %ah
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ah, %dh
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlb $3, %dh
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    andb $12, %ah
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    negb %ah
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movsbl %ah, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 32(%esp,%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 36(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dh, %dl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    notb %dl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 44(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 40(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %eax, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, (%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl $60, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-SSE2-LABEL: shl_16bytes:
+; X86-HAVE-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    subl $32, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%edx), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%edx), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%edx), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%edx), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movb (%ecx), %ch
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, (%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andb $12, %ch
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    negb %ch
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movsbl %ch, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%esp,%edi), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%esp,%edi), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%esp,%edi), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%esp,%edi), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %ebx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, 8(%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, 12(%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, (%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, 4(%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    addl $32, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-SSE2-LABEL: shl_16bytes:
+; X86-NO-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    subl $44, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%ecx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%eax), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $3, %al
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, (%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $12, %bl
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    negb %bl
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movsbl %bl, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%esp,%esi), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%esp,%esi), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    notb %al
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %ebx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %eax, %ebx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, 28(%esp,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%esp,%esi), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %esi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %eax, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %eax, %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, (%ecx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 8(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, 12(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, 4(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl $44, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: shl_16bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    subl $44, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%edx), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%edx), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%edx), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%edx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, (%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $12, %al
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    negb %al
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movsbl %al, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%esp,%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 28(%esp,%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%esp,%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%esp,%eax), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %edi, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %edi, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, 8(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, 12(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, (%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 4(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    addl $44, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-SSE4-LABEL: shl_16bytes:
+; X86-NO-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    subl $60, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlb $3, %al
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm1, %xmm1
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    andb $12, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    negb %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movsbl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 44(%esp,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    notb %dl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 40(%esp,%edi), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 32(%esp,%edi), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 36(%esp,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebp, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, (%edx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, 4(%edx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, 8(%edx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, 12(%edx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl $60, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-SSE4-LABEL: shl_16bytes:
+; X86-HAVE-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    subl $44, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups (%edx), %xmm0
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%ecx), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm1, %xmm1
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, (%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andb $12, %dl
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    negb %dl
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movsbl %dl, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 24(%esp,%edi), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 28(%esp,%edi), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 16(%esp,%edi), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 20(%esp,%edi), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %ebx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, 4(%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, 8(%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, 12(%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, (%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    addl $44, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-SSE4-LABEL: shl_16bytes:
+; X86-NO-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    subl $44, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $3, %al
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm1, %xmm1
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, (%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $12, %dl
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    negb %dl
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movsbl %dl, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ecx, 28(%esp,%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    notb %al
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 24(%esp,%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ecx, %esi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %eax, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 20(%esp,%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %eax, %ebp, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ecx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 16(%esp,%edx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ecx, %edx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %eax, %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, (%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 4(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, 8(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl $44, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: shl_16bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    subl $44, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%edx), %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%ecx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm1, %xmm1
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, (%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $12, %dl
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    negb %dl
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movsbl %dl, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 24(%esp,%edi), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 28(%esp,%edi), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 16(%esp,%edi), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 20(%esp,%edi), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %edi, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ecx, %ebx, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %ebx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, 4(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, 8(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, 12(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, (%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    addl $44, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-AVX-LABEL: shl_16bytes:
+; X86-NO-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    subl $60, %esp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%ecx), %xmm0
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shlb $3, %al
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    andb $12, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    negb %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movsbl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 44(%esp,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    notb %dl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 40(%esp,%edi), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebp, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 32(%esp,%edi), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 36(%esp,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebp, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, (%edx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebp, 4(%edx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, 8(%edx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, 12(%edx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl $60, %esp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-AVX-LABEL: shl_16bytes:
+; X86-HAVE-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    subl $44, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%edx), %xmm0
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%ecx), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm1, (%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    andb $12, %dl
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    negb %dl
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movsbl %dl, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 24(%esp,%edi), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 28(%esp,%edi), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 16(%esp,%edi), %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 20(%esp,%edi), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shldl %cl, %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shldl %cl, %ebx, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, 4(%eax)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, 8(%eax)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, 12(%eax)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %ebp, (%eax)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    addl $44, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-AVX-LABEL: shl_16bytes:
+; X86-NO-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    subl $44, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%ecx), %xmm0
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $3, %al
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm1, (%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    andb $12, %dl
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    negb %dl
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movsbl %dl, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ecx, 28(%esp,%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    notb %al
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 24(%esp,%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ecx, %esi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %eax, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 20(%esp,%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %eax, %ebp, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ecx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 16(%esp,%edx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ecx, %edx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrl %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %eax, %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, (%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 4(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebp, 8(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl $44, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-LABEL: shl_16bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    subl $44, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%edx), %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%ecx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm1, (%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    andb $12, %dl
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    negb %dl
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movsbl %dl, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 24(%esp,%edi), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 28(%esp,%edi), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 16(%esp,%edi), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 20(%esp,%edi), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shldl %cl, %edi, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ecx, %ebx, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shldl %cl, %ebx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, 4(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, 8(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, 12(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebp, (%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    addl $44, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    retl
   %src = load i128, ptr %src.ptr, align 1
   %byteOff = load i128, ptr %byteOff.ptr, align 1
   %bitOff = shl i128 %byteOff, 3
@@ -3111,1932 +2728,1477 @@ define void @ashr_16bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
 }
 
 define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
-; FALLBACK0-LABEL: lshr_32bytes:
-; FALLBACK0:       # %bb.0:
-; FALLBACK0-NEXT:    pushq %rbx
-; FALLBACK0-NEXT:    movq (%rdi), %rcx
-; FALLBACK0-NEXT:    movq 8(%rdi), %r8
-; FALLBACK0-NEXT:    movq 16(%rdi), %r9
-; FALLBACK0-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK0-NEXT:    movzbl (%rsi), %esi
-; FALLBACK0-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK0-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK0-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    andb $24, %sil
-; FALLBACK0-NEXT:    movzbl %sil, %r9d
-; FALLBACK0-NEXT:    movq -64(%rsp,%r9), %r10
-; FALLBACK0-NEXT:    movq -56(%rsp,%r9), %rdi
-; FALLBACK0-NEXT:    movq %rdi, %r11
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r11
-; FALLBACK0-NEXT:    movl %eax, %esi
-; FALLBACK0-NEXT:    notb %sil
-; FALLBACK0-NEXT:    movq -48(%rsp,%r9), %rbx
-; FALLBACK0-NEXT:    leaq (%rbx,%rbx), %r8
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r8
-; FALLBACK0-NEXT:    orq %r11, %r8
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r10
-; FALLBACK0-NEXT:    addq %rdi, %rdi
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %rdi
-; FALLBACK0-NEXT:    orq %r10, %rdi
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %rbx
-; FALLBACK0-NEXT:    movq -40(%rsp,%r9), %r9
-; FALLBACK0-NEXT:    leaq (%r9,%r9), %r10
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r10
-; FALLBACK0-NEXT:    orq %rbx, %r10
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r9
-; FALLBACK0-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK0-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK0-NEXT:    movq %rdi, (%rdx)
-; FALLBACK0-NEXT:    movq %r8, 8(%rdx)
-; FALLBACK0-NEXT:    popq %rbx
-; FALLBACK0-NEXT:    retq
-;
-; FALLBACK1-LABEL: lshr_32bytes:
-; FALLBACK1:       # %bb.0:
-; FALLBACK1-NEXT:    movq (%rdi), %rax
-; FALLBACK1-NEXT:    movq 8(%rdi), %r8
-; FALLBACK1-NEXT:    movq 16(%rdi), %r9
-; FALLBACK1-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK1-NEXT:    movzbl (%rsi), %esi
-; FALLBACK1-NEXT:    leal (,%rsi,8), %ecx
-; FALLBACK1-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK1-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    andb $24, %sil
-; FALLBACK1-NEXT:    movzbl %sil, %eax
-; FALLBACK1-NEXT:    movq -56(%rsp,%rax), %rsi
-; FALLBACK1-NEXT:    movq -72(%rsp,%rax), %rdi
-; FALLBACK1-NEXT:    movq -64(%rsp,%rax), %r8
-; FALLBACK1-NEXT:    movq %r8, %r9
-; FALLBACK1-NEXT:    shrdq %cl, %rsi, %r9
-; FALLBACK1-NEXT:    movq -48(%rsp,%rax), %rax
-; FALLBACK1-NEXT:    shrdq %cl, %rax, %rsi
-; FALLBACK1-NEXT:    shrdq %cl, %r8, %rdi
-; FALLBACK1-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK1-NEXT:    shrq %cl, %rax
-; FALLBACK1-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK1-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK1-NEXT:    movq %rdi, (%rdx)
-; FALLBACK1-NEXT:    movq %r9, 8(%rdx)
-; FALLBACK1-NEXT:    retq
-;
-; FALLBACK2-LABEL: lshr_32bytes:
-; FALLBACK2:       # %bb.0:
-; FALLBACK2-NEXT:    movq (%rdi), %rcx
-; FALLBACK2-NEXT:    movq 8(%rdi), %r8
-; FALLBACK2-NEXT:    movq 16(%rdi), %r9
-; FALLBACK2-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK2-NEXT:    movzbl (%rsi), %esi
-; FALLBACK2-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK2-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movl %eax, %ecx
-; FALLBACK2-NEXT:    andb $24, %sil
-; FALLBACK2-NEXT:    movzbl %sil, %esi
-; FALLBACK2-NEXT:    movq -64(%rsp,%rsi), %rdi
-; FALLBACK2-NEXT:    movq -56(%rsp,%rsi), %r8
-; FALLBACK2-NEXT:    shrxq %rcx, %rdi, %r9
-; FALLBACK2-NEXT:    notb %al
-; FALLBACK2-NEXT:    leaq (%r8,%r8), %r10
-; FALLBACK2-NEXT:    shlxq %rax, %r10, %r10
-; FALLBACK2-NEXT:    orq %r9, %r10
-; FALLBACK2-NEXT:    shrxq %rcx, -72(%rsp,%rsi), %r9
-; FALLBACK2-NEXT:    addq %rdi, %rdi
-; FALLBACK2-NEXT:    shlxq %rax, %rdi, %rdi
-; FALLBACK2-NEXT:    orq %r9, %rdi
-; FALLBACK2-NEXT:    shrxq %rcx, %r8, %r8
-; FALLBACK2-NEXT:    movq -48(%rsp,%rsi), %rsi
-; FALLBACK2-NEXT:    leaq (%rsi,%rsi), %r9
-; FALLBACK2-NEXT:    shlxq %rax, %r9, %rax
-; FALLBACK2-NEXT:    orq %r8, %rax
-; FALLBACK2-NEXT:    shrxq %rcx, %rsi, %rcx
-; FALLBACK2-NEXT:    movq %rcx, 24(%rdx)
-; FALLBACK2-NEXT:    movq %rax, 16(%rdx)
-; FALLBACK2-NEXT:    movq %rdi, (%rdx)
-; FALLBACK2-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK2-NEXT:    retq
-;
-; FALLBACK3-LABEL: lshr_32bytes:
-; FALLBACK3:       # %bb.0:
-; FALLBACK3-NEXT:    movq (%rdi), %rax
-; FALLBACK3-NEXT:    movq 8(%rdi), %r8
-; FALLBACK3-NEXT:    movq 16(%rdi), %r9
-; FALLBACK3-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK3-NEXT:    movzbl (%rsi), %esi
-; FALLBACK3-NEXT:    leal (,%rsi,8), %ecx
-; FALLBACK3-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    andb $24, %sil
-; FALLBACK3-NEXT:    movzbl %sil, %eax
-; FALLBACK3-NEXT:    movq -56(%rsp,%rax), %rsi
-; FALLBACK3-NEXT:    movq -72(%rsp,%rax), %rdi
-; FALLBACK3-NEXT:    movq -64(%rsp,%rax), %r8
-; FALLBACK3-NEXT:    movq %r8, %r9
-; FALLBACK3-NEXT:    shrdq %cl, %rsi, %r9
-; FALLBACK3-NEXT:    movq -48(%rsp,%rax), %rax
-; FALLBACK3-NEXT:    shrdq %cl, %rax, %rsi
-; FALLBACK3-NEXT:    shrdq %cl, %r8, %rdi
-; FALLBACK3-NEXT:    shrxq %rcx, %rax, %rax
-; FALLBACK3-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK3-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK3-NEXT:    movq %rdi, (%rdx)
-; FALLBACK3-NEXT:    movq %r9, 8(%rdx)
-; FALLBACK3-NEXT:    retq
-;
-; FALLBACK4-LABEL: lshr_32bytes:
-; FALLBACK4:       # %bb.0:
-; FALLBACK4-NEXT:    pushq %rbx
-; FALLBACK4-NEXT:    movups (%rdi), %xmm0
-; FALLBACK4-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK4-NEXT:    movzbl (%rsi), %ecx
-; FALLBACK4-NEXT:    leal (,%rcx,8), %eax
-; FALLBACK4-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    andb $24, %cl
-; FALLBACK4-NEXT:    movzbl %cl, %r9d
-; FALLBACK4-NEXT:    movq -64(%rsp,%r9), %r10
-; FALLBACK4-NEXT:    movq -56(%rsp,%r9), %r8
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r10
-; FALLBACK4-NEXT:    movl %eax, %esi
-; FALLBACK4-NEXT:    notb %sil
-; FALLBACK4-NEXT:    leaq (%r8,%r8), %rdi
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %rdi
-; FALLBACK4-NEXT:    orq %r10, %rdi
-; FALLBACK4-NEXT:    movq -48(%rsp,%r9), %r10
-; FALLBACK4-NEXT:    movq %r10, %r11
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r11
-; FALLBACK4-NEXT:    movq -40(%rsp,%r9), %r9
-; FALLBACK4-NEXT:    leaq (%r9,%r9), %rbx
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %rbx
-; FALLBACK4-NEXT:    orq %r11, %rbx
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r8
-; FALLBACK4-NEXT:    addq %r10, %r10
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r10
-; FALLBACK4-NEXT:    orq %r8, %r10
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r9
-; FALLBACK4-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK4-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK4-NEXT:    movq %rbx, 16(%rdx)
-; FALLBACK4-NEXT:    movq %rdi, (%rdx)
-; FALLBACK4-NEXT:    popq %rbx
-; FALLBACK4-NEXT:    retq
-;
-; FALLBACK5-LABEL: lshr_32bytes:
-; FALLBACK5:       # %bb.0:
-; FALLBACK5-NEXT:    movups (%rdi), %xmm0
-; FALLBACK5-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK5-NEXT:    movzbl (%rsi), %eax
-; FALLBACK5-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK5-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK5-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    andb $24, %al
-; FALLBACK5-NEXT:    movzbl %al, %eax
-; FALLBACK5-NEXT:    movq -48(%rsp,%rax), %rsi
-; FALLBACK5-NEXT:    movq -56(%rsp,%rax), %rdi
-; FALLBACK5-NEXT:    movq %rdi, %r8
-; FALLBACK5-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK5-NEXT:    movq -72(%rsp,%rax), %r9
-; FALLBACK5-NEXT:    movq -64(%rsp,%rax), %rax
-; FALLBACK5-NEXT:    movq %rax, %r10
-; FALLBACK5-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK5-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK5-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK5-NEXT:    shrq %cl, %rsi
-; FALLBACK5-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK5-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK5-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK5-NEXT:    movq %r9, (%rdx)
-; FALLBACK5-NEXT:    retq
-;
-; FALLBACK6-LABEL: lshr_32bytes:
-; FALLBACK6:       # %bb.0:
-; FALLBACK6-NEXT:    movups (%rdi), %xmm0
-; FALLBACK6-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK6-NEXT:    movzbl (%rsi), %ecx
-; FALLBACK6-NEXT:    leal (,%rcx,8), %eax
-; FALLBACK6-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK6-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movl %eax, %esi
-; FALLBACK6-NEXT:    andb $24, %cl
-; FALLBACK6-NEXT:    movzbl %cl, %ecx
-; FALLBACK6-NEXT:    shrxq %rsi, -72(%rsp,%rcx), %rdi
-; FALLBACK6-NEXT:    notb %al
-; FALLBACK6-NEXT:    movq -64(%rsp,%rcx), %r8
-; FALLBACK6-NEXT:    movq -56(%rsp,%rcx), %r9
-; FALLBACK6-NEXT:    leaq (%r8,%r8), %r10
-; FALLBACK6-NEXT:    shlxq %rax, %r10, %r10
-; FALLBACK6-NEXT:    orq %rdi, %r10
-; FALLBACK6-NEXT:    shrxq %rsi, %r9, %rdi
-; FALLBACK6-NEXT:    movq -48(%rsp,%rcx), %rcx
-; FALLBACK6-NEXT:    leaq (%rcx,%rcx), %r11
-; FALLBACK6-NEXT:    shlxq %rax, %r11, %r11
-; FALLBACK6-NEXT:    orq %rdi, %r11
-; FALLBACK6-NEXT:    shrxq %rsi, %r8, %rdi
-; FALLBACK6-NEXT:    addq %r9, %r9
-; FALLBACK6-NEXT:    shlxq %rax, %r9, %rax
-; FALLBACK6-NEXT:    orq %rdi, %rax
-; FALLBACK6-NEXT:    shrxq %rsi, %rcx, %rcx
-; FALLBACK6-NEXT:    movq %rcx, 24(%rdx)
-; FALLBACK6-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK6-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK6-NEXT:    movq %r10, (%rdx)
-; FALLBACK6-NEXT:    retq
-;
-; FALLBACK7-LABEL: lshr_32bytes:
-; FALLBACK7:       # %bb.0:
-; FALLBACK7-NEXT:    movups (%rdi), %xmm0
-; FALLBACK7-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK7-NEXT:    movzbl (%rsi), %eax
-; FALLBACK7-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK7-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK7-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    andb $24, %al
-; FALLBACK7-NEXT:    movzbl %al, %eax
-; FALLBACK7-NEXT:    movq -48(%rsp,%rax), %rsi
-; FALLBACK7-NEXT:    movq -56(%rsp,%rax), %rdi
-; FALLBACK7-NEXT:    movq %rdi, %r8
-; FALLBACK7-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK7-NEXT:    movq -72(%rsp,%rax), %r9
-; FALLBACK7-NEXT:    movq -64(%rsp,%rax), %rax
-; FALLBACK7-NEXT:    movq %rax, %r10
-; FALLBACK7-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK7-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK7-NEXT:    shrxq %rcx, %rsi, %rax
-; FALLBACK7-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK7-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK7-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK7-NEXT:    movq %r9, (%rdx)
-; FALLBACK7-NEXT:    retq
-;
-; FALLBACK8-LABEL: lshr_32bytes:
-; FALLBACK8:       # %bb.0:
-; FALLBACK8-NEXT:    pushq %rbx
-; FALLBACK8-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK8-NEXT:    movzbl (%rsi), %ecx
-; FALLBACK8-NEXT:    leal (,%rcx,8), %eax
-; FALLBACK8-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK8-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    andb $24, %cl
-; FALLBACK8-NEXT:    movzbl %cl, %r9d
-; FALLBACK8-NEXT:    movq -64(%rsp,%r9), %r10
-; FALLBACK8-NEXT:    movq -56(%rsp,%r9), %r8
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r10
-; FALLBACK8-NEXT:    movl %eax, %esi
-; FALLBACK8-NEXT:    notb %sil
-; FALLBACK8-NEXT:    leaq (%r8,%r8), %rdi
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %rdi
-; FALLBACK8-NEXT:    orq %r10, %rdi
-; FALLBACK8-NEXT:    movq -48(%rsp,%r9), %r10
-; FALLBACK8-NEXT:    movq %r10, %r11
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r11
-; FALLBACK8-NEXT:    movq -40(%rsp,%r9), %r9
-; FALLBACK8-NEXT:    leaq (%r9,%r9), %rbx
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %rbx
-; FALLBACK8-NEXT:    orq %r11, %rbx
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r8
-; FALLBACK8-NEXT:    addq %r10, %r10
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r10
-; FALLBACK8-NEXT:    orq %r8, %r10
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r9
-; FALLBACK8-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK8-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK8-NEXT:    movq %rbx, 16(%rdx)
-; FALLBACK8-NEXT:    movq %rdi, (%rdx)
-; FALLBACK8-NEXT:    popq %rbx
-; FALLBACK8-NEXT:    vzeroupper
-; FALLBACK8-NEXT:    retq
-;
-; FALLBACK9-LABEL: lshr_32bytes:
-; FALLBACK9:       # %bb.0:
-; FALLBACK9-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK9-NEXT:    movzbl (%rsi), %eax
-; FALLBACK9-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK9-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK9-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    andb $24, %al
-; FALLBACK9-NEXT:    movzbl %al, %eax
-; FALLBACK9-NEXT:    movq -48(%rsp,%rax), %rsi
-; FALLBACK9-NEXT:    movq -56(%rsp,%rax), %rdi
-; FALLBACK9-NEXT:    movq %rdi, %r8
-; FALLBACK9-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK9-NEXT:    movq -72(%rsp,%rax), %r9
-; FALLBACK9-NEXT:    movq -64(%rsp,%rax), %rax
-; FALLBACK9-NEXT:    movq %rax, %r10
-; FALLBACK9-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK9-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK9-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK9-NEXT:    shrq %cl, %rsi
-; FALLBACK9-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK9-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK9-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK9-NEXT:    movq %r9, (%rdx)
-; FALLBACK9-NEXT:    vzeroupper
-; FALLBACK9-NEXT:    retq
-;
-; FALLBACK10-LABEL: lshr_32bytes:
-; FALLBACK10:       # %bb.0:
-; FALLBACK10-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK10-NEXT:    movzbl (%rsi), %eax
-; FALLBACK10-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK10-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK10-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movl %ecx, %esi
-; FALLBACK10-NEXT:    andb $24, %al
-; FALLBACK10-NEXT:    movzbl %al, %eax
-; FALLBACK10-NEXT:    shrxq %rsi, -72(%rsp,%rax), %rdi
-; FALLBACK10-NEXT:    notb %cl
-; FALLBACK10-NEXT:    movq -64(%rsp,%rax), %r8
-; FALLBACK10-NEXT:    movq -56(%rsp,%rax), %r9
-; FALLBACK10-NEXT:    leaq (%r8,%r8), %r10
-; FALLBACK10-NEXT:    shlxq %rcx, %r10, %r10
-; FALLBACK10-NEXT:    orq %rdi, %r10
-; FALLBACK10-NEXT:    shrxq %rsi, %r9, %rdi
-; FALLBACK10-NEXT:    movq -48(%rsp,%rax), %rax
-; FALLBACK10-NEXT:    leaq (%rax,%rax), %r11
-; FALLBACK10-NEXT:    shlxq %rcx, %r11, %r11
-; FALLBACK10-NEXT:    orq %rdi, %r11
-; FALLBACK10-NEXT:    shrxq %rsi, %r8, %rdi
-; FALLBACK10-NEXT:    addq %r9, %r9
-; FALLBACK10-NEXT:    shlxq %rcx, %r9, %rcx
-; FALLBACK10-NEXT:    orq %rdi, %rcx
-; FALLBACK10-NEXT:    shrxq %rsi, %rax, %rax
-; FALLBACK10-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK10-NEXT:    movq %rcx, 8(%rdx)
-; FALLBACK10-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK10-NEXT:    movq %r10, (%rdx)
-; FALLBACK10-NEXT:    vzeroupper
-; FALLBACK10-NEXT:    retq
-;
-; FALLBACK11-LABEL: lshr_32bytes:
-; FALLBACK11:       # %bb.0:
-; FALLBACK11-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK11-NEXT:    movzbl (%rsi), %eax
-; FALLBACK11-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK11-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK11-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    andb $24, %al
-; FALLBACK11-NEXT:    movzbl %al, %eax
-; FALLBACK11-NEXT:    movq -48(%rsp,%rax), %rsi
-; FALLBACK11-NEXT:    movq -56(%rsp,%rax), %rdi
-; FALLBACK11-NEXT:    movq %rdi, %r8
-; FALLBACK11-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK11-NEXT:    movq -72(%rsp,%rax), %r9
-; FALLBACK11-NEXT:    movq -64(%rsp,%rax), %rax
-; FALLBACK11-NEXT:    movq %rax, %r10
-; FALLBACK11-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK11-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK11-NEXT:    shrxq %rcx, %rsi, %rax
-; FALLBACK11-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK11-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK11-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK11-NEXT:    movq %r9, (%rdx)
-; FALLBACK11-NEXT:    vzeroupper
-; FALLBACK11-NEXT:    retq
-;
-; FALLBACK12-LABEL: lshr_32bytes:
-; FALLBACK12:       # %bb.0:
-; FALLBACK12-NEXT:    pushq %rbx
-; FALLBACK12-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK12-NEXT:    movzbl (%rsi), %ecx
-; FALLBACK12-NEXT:    leal (,%rcx,8), %eax
-; FALLBACK12-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK12-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    andb $24, %cl
-; FALLBACK12-NEXT:    movzbl %cl, %r9d
-; FALLBACK12-NEXT:    movq -64(%rsp,%r9), %r10
-; FALLBACK12-NEXT:    movq -56(%rsp,%r9), %r8
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r10
-; FALLBACK12-NEXT:    movl %eax, %esi
-; FALLBACK12-NEXT:    notb %sil
-; FALLBACK12-NEXT:    leaq (%r8,%r8), %rdi
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %rdi
-; FALLBACK12-NEXT:    orq %r10, %rdi
-; FALLBACK12-NEXT:    movq -48(%rsp,%r9), %r10
-; FALLBACK12-NEXT:    movq %r10, %r11
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r11
-; FALLBACK12-NEXT:    movq -40(%rsp,%r9), %r9
-; FALLBACK12-NEXT:    leaq (%r9,%r9), %rbx
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %rbx
-; FALLBACK12-NEXT:    orq %r11, %rbx
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r8
-; FALLBACK12-NEXT:    addq %r10, %r10
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r10
-; FALLBACK12-NEXT:    orq %r8, %r10
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r9
-; FALLBACK12-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK12-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK12-NEXT:    movq %rbx, 16(%rdx)
-; FALLBACK12-NEXT:    movq %rdi, (%rdx)
-; FALLBACK12-NEXT:    popq %rbx
-; FALLBACK12-NEXT:    vzeroupper
-; FALLBACK12-NEXT:    retq
-;
-; FALLBACK13-LABEL: lshr_32bytes:
-; FALLBACK13:       # %bb.0:
-; FALLBACK13-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK13-NEXT:    movzbl (%rsi), %eax
-; FALLBACK13-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK13-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK13-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    andb $24, %al
-; FALLBACK13-NEXT:    movzbl %al, %eax
-; FALLBACK13-NEXT:    movq -48(%rsp,%rax), %rsi
-; FALLBACK13-NEXT:    movq -56(%rsp,%rax), %rdi
-; FALLBACK13-NEXT:    movq %rdi, %r8
-; FALLBACK13-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK13-NEXT:    movq -72(%rsp,%rax), %r9
-; FALLBACK13-NEXT:    movq -64(%rsp,%rax), %rax
-; FALLBACK13-NEXT:    movq %rax, %r10
-; FALLBACK13-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK13-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK13-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK13-NEXT:    shrq %cl, %rsi
-; FALLBACK13-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK13-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK13-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK13-NEXT:    movq %r9, (%rdx)
-; FALLBACK13-NEXT:    vzeroupper
-; FALLBACK13-NEXT:    retq
-;
-; FALLBACK14-LABEL: lshr_32bytes:
-; FALLBACK14:       # %bb.0:
-; FALLBACK14-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK14-NEXT:    movzbl (%rsi), %eax
-; FALLBACK14-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK14-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK14-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movl %ecx, %esi
-; FALLBACK14-NEXT:    andb $24, %al
-; FALLBACK14-NEXT:    movzbl %al, %eax
-; FALLBACK14-NEXT:    shrxq %rsi, -72(%rsp,%rax), %rdi
-; FALLBACK14-NEXT:    notb %cl
-; FALLBACK14-NEXT:    movq -64(%rsp,%rax), %r8
-; FALLBACK14-NEXT:    movq -56(%rsp,%rax), %r9
-; FALLBACK14-NEXT:    leaq (%r8,%r8), %r10
-; FALLBACK14-NEXT:    shlxq %rcx, %r10, %r10
-; FALLBACK14-NEXT:    orq %rdi, %r10
-; FALLBACK14-NEXT:    shrxq %rsi, %r9, %rdi
-; FALLBACK14-NEXT:    movq -48(%rsp,%rax), %rax
-; FALLBACK14-NEXT:    leaq (%rax,%rax), %r11
-; FALLBACK14-NEXT:    shlxq %rcx, %r11, %r11
-; FALLBACK14-NEXT:    orq %rdi, %r11
-; FALLBACK14-NEXT:    shrxq %rsi, %r8, %rdi
-; FALLBACK14-NEXT:    addq %r9, %r9
-; FALLBACK14-NEXT:    shlxq %rcx, %r9, %rcx
-; FALLBACK14-NEXT:    orq %rdi, %rcx
-; FALLBACK14-NEXT:    shrxq %rsi, %rax, %rax
-; FALLBACK14-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK14-NEXT:    movq %rcx, 8(%rdx)
-; FALLBACK14-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK14-NEXT:    movq %r10, (%rdx)
-; FALLBACK14-NEXT:    vzeroupper
-; FALLBACK14-NEXT:    retq
-;
-; FALLBACK15-LABEL: lshr_32bytes:
-; FALLBACK15:       # %bb.0:
-; FALLBACK15-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK15-NEXT:    movzbl (%rsi), %eax
-; FALLBACK15-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK15-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK15-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    andb $24, %al
-; FALLBACK15-NEXT:    movzbl %al, %eax
-; FALLBACK15-NEXT:    movq -48(%rsp,%rax), %rsi
-; FALLBACK15-NEXT:    movq -56(%rsp,%rax), %rdi
-; FALLBACK15-NEXT:    movq %rdi, %r8
-; FALLBACK15-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK15-NEXT:    movq -72(%rsp,%rax), %r9
-; FALLBACK15-NEXT:    movq -64(%rsp,%rax), %rax
-; FALLBACK15-NEXT:    movq %rax, %r10
-; FALLBACK15-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK15-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK15-NEXT:    shrxq %rcx, %rsi, %rax
-; FALLBACK15-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK15-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK15-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK15-NEXT:    movq %r9, (%rdx)
-; FALLBACK15-NEXT:    vzeroupper
-; FALLBACK15-NEXT:    retq
-;
-; FALLBACK16-LABEL: lshr_32bytes:
-; FALLBACK16:       # %bb.0:
-; FALLBACK16-NEXT:    pushl %ebp
-; FALLBACK16-NEXT:    pushl %ebx
-; FALLBACK16-NEXT:    pushl %edi
-; FALLBACK16-NEXT:    pushl %esi
-; FALLBACK16-NEXT:    subl $108, %esp
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK16-NEXT:    movl (%ebp), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 4(%ebp), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 8(%ebp), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 12(%ebp), %edi
-; FALLBACK16-NEXT:    movl 16(%ebp), %ebx
-; FALLBACK16-NEXT:    movb (%eax), %ah
-; FALLBACK16-NEXT:    movl 20(%ebp), %esi
-; FALLBACK16-NEXT:    movl 24(%ebp), %ecx
-; FALLBACK16-NEXT:    movl 28(%ebp), %ebp
-; FALLBACK16-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK16-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movb %ah, %dh
-; FALLBACK16-NEXT:    shlb $3, %dh
-; FALLBACK16-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    andb $28, %ah
-; FALLBACK16-NEXT:    movzbl %ah, %edi
-; FALLBACK16-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 32(%esp,%edi), %esi
-; FALLBACK16-NEXT:    movl 36(%esp,%edi), %eax
-; FALLBACK16-NEXT:    movl %eax, %ebx
-; FALLBACK16-NEXT:    movb %dh, %cl
-; FALLBACK16-NEXT:    shrl %cl, %ebx
-; FALLBACK16-NEXT:    movb %dh, %dl
-; FALLBACK16-NEXT:    notb %dl
-; FALLBACK16-NEXT:    movl 40(%esp,%edi), %edi
-; FALLBACK16-NEXT:    leal (%edi,%edi), %ebp
-; FALLBACK16-NEXT:    movl %edx, %ecx
-; FALLBACK16-NEXT:    shll %cl, %ebp
-; FALLBACK16-NEXT:    orl %ebx, %ebp
-; FALLBACK16-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %dh, %cl
-; FALLBACK16-NEXT:    shrl %cl, %esi
-; FALLBACK16-NEXT:    movl %eax, %ebx
-; FALLBACK16-NEXT:    addl %eax, %ebx
-; FALLBACK16-NEXT:    movl %edx, %ecx
-; FALLBACK16-NEXT:    shll %cl, %ebx
-; FALLBACK16-NEXT:    orl %esi, %ebx
-; FALLBACK16-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl 44(%esp,%eax), %ebp
-; FALLBACK16-NEXT:    movl %ebp, %esi
-; FALLBACK16-NEXT:    movb %dh, %cl
-; FALLBACK16-NEXT:    shrl %cl, %esi
-; FALLBACK16-NEXT:    movl 48(%esp,%eax), %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    leal (%eax,%eax), %ebx
-; FALLBACK16-NEXT:    movl %edx, %ecx
-; FALLBACK16-NEXT:    shll %cl, %ebx
-; FALLBACK16-NEXT:    orl %esi, %ebx
-; FALLBACK16-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %dh, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edi
-; FALLBACK16-NEXT:    addl %ebp, %ebp
-; FALLBACK16-NEXT:    movl %edx, %ecx
-; FALLBACK16-NEXT:    shll %cl, %ebp
-; FALLBACK16-NEXT:    orl %edi, %ebp
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl 52(%esp,%eax), %edi
-; FALLBACK16-NEXT:    movl %edi, %ebx
-; FALLBACK16-NEXT:    movb %dh, %cl
-; FALLBACK16-NEXT:    shrl %cl, %ebx
-; FALLBACK16-NEXT:    movl 56(%esp,%eax), %esi
-; FALLBACK16-NEXT:    leal (%esi,%esi), %eax
-; FALLBACK16-NEXT:    movl %edx, %ecx
-; FALLBACK16-NEXT:    shll %cl, %eax
-; FALLBACK16-NEXT:    orl %ebx, %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %dh, %cl
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK16-NEXT:    shrl %cl, %ebx
-; FALLBACK16-NEXT:    addl %edi, %edi
-; FALLBACK16-NEXT:    movl %edx, %ecx
-; FALLBACK16-NEXT:    shll %cl, %edi
-; FALLBACK16-NEXT:    orl %ebx, %edi
-; FALLBACK16-NEXT:    movb %dh, %cl
-; FALLBACK16-NEXT:    movl %esi, %eax
-; FALLBACK16-NEXT:    shrl %cl, %eax
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl 60(%esp,%ecx), %ebx
-; FALLBACK16-NEXT:    leal (%ebx,%ebx), %esi
-; FALLBACK16-NEXT:    movl %edx, %ecx
-; FALLBACK16-NEXT:    shll %cl, %esi
-; FALLBACK16-NEXT:    orl %eax, %esi
-; FALLBACK16-NEXT:    movb %dh, %cl
-; FALLBACK16-NEXT:    shrl %cl, %ebx
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK16-NEXT:    movl %ebx, 28(%eax)
-; FALLBACK16-NEXT:    movl %esi, 24(%eax)
-; FALLBACK16-NEXT:    movl %edi, 16(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK16-NEXT:    movl %ebp, 8(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, (%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK16-NEXT:    addl $108, %esp
-; FALLBACK16-NEXT:    popl %esi
-; FALLBACK16-NEXT:    popl %edi
-; FALLBACK16-NEXT:    popl %ebx
-; FALLBACK16-NEXT:    popl %ebp
-; FALLBACK16-NEXT:    retl
-;
-; FALLBACK17-LABEL: lshr_32bytes:
-; FALLBACK17:       # %bb.0:
-; FALLBACK17-NEXT:    pushl %ebp
-; FALLBACK17-NEXT:    pushl %ebx
-; FALLBACK17-NEXT:    pushl %edi
-; FALLBACK17-NEXT:    pushl %esi
-; FALLBACK17-NEXT:    subl $92, %esp
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK17-NEXT:    movl (%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 4(%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 8(%ebp), %esi
-; FALLBACK17-NEXT:    movl 12(%ebp), %edi
-; FALLBACK17-NEXT:    movl 16(%ebp), %ebx
-; FALLBACK17-NEXT:    movb (%ecx), %ch
-; FALLBACK17-NEXT:    movl 20(%ebp), %edx
-; FALLBACK17-NEXT:    movl 24(%ebp), %eax
-; FALLBACK17-NEXT:    movl 28(%ebp), %ebp
-; FALLBACK17-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK17-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movb %ch, %cl
-; FALLBACK17-NEXT:    shlb $3, %cl
-; FALLBACK17-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    andb $28, %ch
-; FALLBACK17-NEXT:    movzbl %ch, %ebp
-; FALLBACK17-NEXT:    movl 24(%esp,%ebp), %edx
-; FALLBACK17-NEXT:    movl 20(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; FALLBACK17-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 32(%esp,%ebp), %ebx
-; FALLBACK17-NEXT:    movl 28(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, %esi
-; FALLBACK17-NEXT:    shrdl %cl, %ebx, %esi
-; FALLBACK17-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK17-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 40(%esp,%ebp), %edx
-; FALLBACK17-NEXT:    movl 36(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, %edi
-; FALLBACK17-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK17-NEXT:    movl 16(%esp,%ebp), %esi
-; FALLBACK17-NEXT:    movl 44(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK17-NEXT:    movl %edx, 24(%ebp)
-; FALLBACK17-NEXT:    movl (%esp), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    shrdl %cl, %edx, %esi
-; FALLBACK17-NEXT:    shrl %cl, %eax
-; FALLBACK17-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK17-NEXT:    movl %ebx, 16(%ebp)
-; FALLBACK17-NEXT:    movl %edi, 20(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK17-NEXT:    movl %esi, (%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 4(%ebp)
-; FALLBACK17-NEXT:    addl $92, %esp
-; FALLBACK17-NEXT:    popl %esi
-; FALLBACK17-NEXT:    popl %edi
-; FALLBACK17-NEXT:    popl %ebx
-; FALLBACK17-NEXT:    popl %ebp
-; FALLBACK17-NEXT:    retl
-;
-; FALLBACK18-LABEL: lshr_32bytes:
-; FALLBACK18:       # %bb.0:
-; FALLBACK18-NEXT:    pushl %ebp
-; FALLBACK18-NEXT:    pushl %ebx
-; FALLBACK18-NEXT:    pushl %edi
-; FALLBACK18-NEXT:    pushl %esi
-; FALLBACK18-NEXT:    subl $108, %esp
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK18-NEXT:    movl (%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 4(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 8(%eax), %esi
-; FALLBACK18-NEXT:    movl 12(%eax), %edi
-; FALLBACK18-NEXT:    movl 16(%eax), %ebp
-; FALLBACK18-NEXT:    movzbl (%ebx), %ebx
-; FALLBACK18-NEXT:    movl 20(%eax), %edx
-; FALLBACK18-NEXT:    movl 24(%eax), %ecx
-; FALLBACK18-NEXT:    movl 28(%eax), %eax
-; FALLBACK18-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK18-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ebx, %ecx
-; FALLBACK18-NEXT:    shlb $3, %cl
-; FALLBACK18-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, %eax
-; FALLBACK18-NEXT:    andb $28, %bl
-; FALLBACK18-NEXT:    movzbl %bl, %esi
-; FALLBACK18-NEXT:    movl 36(%esp,%esi), %edx
-; FALLBACK18-NEXT:    movl 40(%esp,%esi), %ebp
-; FALLBACK18-NEXT:    shrxl %eax, %edx, %edi
-; FALLBACK18-NEXT:    notb %cl
-; FALLBACK18-NEXT:    leal (%ebp,%ebp), %ebx
-; FALLBACK18-NEXT:    shlxl %ecx, %ebx, %ebx
-; FALLBACK18-NEXT:    orl %edi, %ebx
-; FALLBACK18-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %eax, 32(%esp,%esi), %edi
-; FALLBACK18-NEXT:    addl %edx, %edx
-; FALLBACK18-NEXT:    shlxl %ecx, %edx, %edx
-; FALLBACK18-NEXT:    orl %edi, %edx
-; FALLBACK18-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 48(%esp,%esi), %edx
-; FALLBACK18-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    addl %edx, %edx
-; FALLBACK18-NEXT:    shlxl %ecx, %edx, %ebx
-; FALLBACK18-NEXT:    movl 44(%esp,%esi), %edx
-; FALLBACK18-NEXT:    shrxl %eax, %edx, %edi
-; FALLBACK18-NEXT:    orl %edi, %ebx
-; FALLBACK18-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %eax, %ebp, %edi
-; FALLBACK18-NEXT:    movl %eax, %ebp
-; FALLBACK18-NEXT:    addl %edx, %edx
-; FALLBACK18-NEXT:    shlxl %ecx, %edx, %eax
-; FALLBACK18-NEXT:    orl %edi, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 56(%esp,%esi), %edi
-; FALLBACK18-NEXT:    leal (%edi,%edi), %edx
-; FALLBACK18-NEXT:    shlxl %ecx, %edx, %edx
-; FALLBACK18-NEXT:    movl 52(%esp,%esi), %eax
-; FALLBACK18-NEXT:    shrxl %ebp, %eax, %ebx
-; FALLBACK18-NEXT:    orl %ebx, %edx
-; FALLBACK18-NEXT:    shrxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; FALLBACK18-NEXT:    addl %eax, %eax
-; FALLBACK18-NEXT:    shlxl %ecx, %eax, %eax
-; FALLBACK18-NEXT:    orl %ebx, %eax
-; FALLBACK18-NEXT:    movl 60(%esp,%esi), %esi
-; FALLBACK18-NEXT:    leal (%esi,%esi), %ebx
-; FALLBACK18-NEXT:    shlxl %ecx, %ebx, %ecx
-; FALLBACK18-NEXT:    shrxl %ebp, %edi, %edi
-; FALLBACK18-NEXT:    orl %edi, %ecx
-; FALLBACK18-NEXT:    shrxl %ebp, %esi, %esi
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; FALLBACK18-NEXT:    movl %esi, 28(%edi)
-; FALLBACK18-NEXT:    movl %ecx, 24(%edi)
-; FALLBACK18-NEXT:    movl %eax, 16(%edi)
-; FALLBACK18-NEXT:    movl %edx, 20(%edi)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 8(%edi)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 12(%edi)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, (%edi)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 4(%edi)
-; FALLBACK18-NEXT:    addl $108, %esp
-; FALLBACK18-NEXT:    popl %esi
-; FALLBACK18-NEXT:    popl %edi
-; FALLBACK18-NEXT:    popl %ebx
-; FALLBACK18-NEXT:    popl %ebp
-; FALLBACK18-NEXT:    retl
-;
-; FALLBACK19-LABEL: lshr_32bytes:
-; FALLBACK19:       # %bb.0:
-; FALLBACK19-NEXT:    pushl %ebp
-; FALLBACK19-NEXT:    pushl %ebx
-; FALLBACK19-NEXT:    pushl %edi
-; FALLBACK19-NEXT:    pushl %esi
-; FALLBACK19-NEXT:    subl $92, %esp
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK19-NEXT:    movl (%ecx), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 4(%ecx), %eax
-; FALLBACK19-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 8(%ecx), %esi
-; FALLBACK19-NEXT:    movl 12(%ecx), %edi
-; FALLBACK19-NEXT:    movl 16(%ecx), %ebp
-; FALLBACK19-NEXT:    movzbl (%ebx), %ebx
-; FALLBACK19-NEXT:    movl 20(%ecx), %edx
-; FALLBACK19-NEXT:    movl 24(%ecx), %eax
-; FALLBACK19-NEXT:    movl 28(%ecx), %ecx
-; FALLBACK19-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK19-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %ebx, %ecx
-; FALLBACK19-NEXT:    shlb $3, %cl
-; FALLBACK19-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    andb $28, %bl
-; FALLBACK19-NEXT:    movzbl %bl, %ebp
-; FALLBACK19-NEXT:    movl 24(%esp,%ebp), %esi
-; FALLBACK19-NEXT:    movl 20(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shrdl %cl, %esi, %eax
-; FALLBACK19-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 32(%esp,%ebp), %ebx
-; FALLBACK19-NEXT:    movl 28(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, %edx
-; FALLBACK19-NEXT:    shrdl %cl, %ebx, %edx
-; FALLBACK19-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK19-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 40(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl 36(%esp,%ebp), %edx
-; FALLBACK19-NEXT:    movl %edx, %esi
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK19-NEXT:    shrdl %cl, %edx, %ebx
-; FALLBACK19-NEXT:    movl 16(%esp,%ebp), %edx
-; FALLBACK19-NEXT:    movl 44(%esp,%ebp), %edi
-; FALLBACK19-NEXT:    shrdl %cl, %edi, %eax
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK19-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK19-NEXT:    shrxl %ecx, %edi, %eax
-; FALLBACK19-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK19-NEXT:    movl %ebx, 16(%ebp)
-; FALLBACK19-NEXT:    movl %esi, 20(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK19-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK19-NEXT:    movl %edx, (%ebp)
-; FALLBACK19-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 4(%ebp)
-; FALLBACK19-NEXT:    addl $92, %esp
-; FALLBACK19-NEXT:    popl %esi
-; FALLBACK19-NEXT:    popl %edi
-; FALLBACK19-NEXT:    popl %ebx
-; FALLBACK19-NEXT:    popl %ebp
-; FALLBACK19-NEXT:    retl
-;
-; FALLBACK20-LABEL: lshr_32bytes:
-; FALLBACK20:       # %bb.0:
-; FALLBACK20-NEXT:    pushl %ebp
-; FALLBACK20-NEXT:    pushl %ebx
-; FALLBACK20-NEXT:    pushl %edi
-; FALLBACK20-NEXT:    pushl %esi
-; FALLBACK20-NEXT:    subl $108, %esp
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK20-NEXT:    movups (%ecx), %xmm0
-; FALLBACK20-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK20-NEXT:    movzbl (%eax), %ecx
-; FALLBACK20-NEXT:    movl %ecx, %eax
-; FALLBACK20-NEXT:    shlb $3, %al
-; FALLBACK20-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK20-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    andb $28, %cl
-; FALLBACK20-NEXT:    movzbl %cl, %edi
-; FALLBACK20-NEXT:    movl 32(%esp,%edi), %esi
-; FALLBACK20-NEXT:    movl 36(%esp,%edi), %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %esi
-; FALLBACK20-NEXT:    movl %eax, %edx
-; FALLBACK20-NEXT:    notb %dl
-; FALLBACK20-NEXT:    addl %ebx, %ebx
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    orl %esi, %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 44(%esp,%edi), %ebp
-; FALLBACK20-NEXT:    movl %ebp, %esi
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %esi
-; FALLBACK20-NEXT:    movl 48(%esp,%edi), %ecx
-; FALLBACK20-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    orl %esi, %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 40(%esp,%edi), %esi
-; FALLBACK20-NEXT:    movl %esi, %ebx
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %ebx
-; FALLBACK20-NEXT:    addl %ebp, %ebp
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shll %cl, %ebp
-; FALLBACK20-NEXT:    orl %ebx, %ebp
-; FALLBACK20-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 52(%esp,%edi), %ebp
-; FALLBACK20-NEXT:    movl %ebp, %ebx
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %ebx
-; FALLBACK20-NEXT:    movl 56(%esp,%edi), %ecx
-; FALLBACK20-NEXT:    movl %ecx, (%esp) # 4-byte Spill
-; FALLBACK20-NEXT:    leal (%ecx,%ecx), %edi
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shll %cl, %edi
-; FALLBACK20-NEXT:    orl %ebx, %edi
-; FALLBACK20-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %cl, %edi
-; FALLBACK20-NEXT:    addl %ebp, %ebp
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shll %cl, %ebp
-; FALLBACK20-NEXT:    orl %edi, %ebp
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shrl %cl, (%esp) # 4-byte Folded Spill
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl 60(%esp,%ecx), %ebx
-; FALLBACK20-NEXT:    leal (%ebx,%ebx), %edi
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shll %cl, %edi
-; FALLBACK20-NEXT:    orl (%esp), %edi # 4-byte Folded Reload
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shrl %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; FALLBACK20-NEXT:    addl %esi, %esi
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shll %cl, %esi
-; FALLBACK20-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %ebx
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK20-NEXT:    movl %ebx, 28(%eax)
-; FALLBACK20-NEXT:    movl %esi, 4(%eax)
-; FALLBACK20-NEXT:    movl %edi, 24(%eax)
-; FALLBACK20-NEXT:    movl %ebp, 16(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, (%eax)
-; FALLBACK20-NEXT:    addl $108, %esp
-; FALLBACK20-NEXT:    popl %esi
-; FALLBACK20-NEXT:    popl %edi
-; FALLBACK20-NEXT:    popl %ebx
-; FALLBACK20-NEXT:    popl %ebp
-; FALLBACK20-NEXT:    retl
-;
-; FALLBACK21-LABEL: lshr_32bytes:
-; FALLBACK21:       # %bb.0:
-; FALLBACK21-NEXT:    pushl %ebp
-; FALLBACK21-NEXT:    pushl %ebx
-; FALLBACK21-NEXT:    pushl %edi
-; FALLBACK21-NEXT:    pushl %esi
-; FALLBACK21-NEXT:    subl $108, %esp
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK21-NEXT:    movups (%ecx), %xmm0
-; FALLBACK21-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK21-NEXT:    movzbl (%eax), %eax
-; FALLBACK21-NEXT:    movl %eax, %ecx
-; FALLBACK21-NEXT:    shlb $3, %cl
-; FALLBACK21-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK21-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    andb $28, %al
-; FALLBACK21-NEXT:    movzbl %al, %ebp
-; FALLBACK21-NEXT:    movl 48(%esp,%ebp), %esi
-; FALLBACK21-NEXT:    movl 44(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, %edx
-; FALLBACK21-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 40(%esp,%ebp), %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 56(%esp,%ebp), %ebx
-; FALLBACK21-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, %edx
-; FALLBACK21-NEXT:    shrdl %cl, %ebx, %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK21-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK21-NEXT:    movl 32(%esp,%ebp), %edx
-; FALLBACK21-NEXT:    movl 36(%esp,%ebp), %edi
-; FALLBACK21-NEXT:    movl %edi, %esi
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; FALLBACK21-NEXT:    shrdl %cl, %ebp, %esi
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK21-NEXT:    movl %esi, 4(%ebp)
-; FALLBACK21-NEXT:    movl %ebx, 24(%ebp)
-; FALLBACK21-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK21-NEXT:    shrl %cl, %eax
-; FALLBACK21-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK21-NEXT:    movl %edx, (%ebp)
-; FALLBACK21-NEXT:    addl $108, %esp
-; FALLBACK21-NEXT:    popl %esi
-; FALLBACK21-NEXT:    popl %edi
-; FALLBACK21-NEXT:    popl %ebx
-; FALLBACK21-NEXT:    popl %ebp
-; FALLBACK21-NEXT:    retl
-;
-; FALLBACK22-LABEL: lshr_32bytes:
-; FALLBACK22:       # %bb.0:
-; FALLBACK22-NEXT:    pushl %ebp
-; FALLBACK22-NEXT:    pushl %ebx
-; FALLBACK22-NEXT:    pushl %edi
-; FALLBACK22-NEXT:    pushl %esi
-; FALLBACK22-NEXT:    subl $108, %esp
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK22-NEXT:    movups (%ecx), %xmm0
-; FALLBACK22-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK22-NEXT:    movzbl (%eax), %edx
-; FALLBACK22-NEXT:    movl %edx, %ecx
-; FALLBACK22-NEXT:    shlb $3, %cl
-; FALLBACK22-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK22-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, %eax
-; FALLBACK22-NEXT:    andb $28, %dl
-; FALLBACK22-NEXT:    movzbl %dl, %ebx
-; FALLBACK22-NEXT:    shrxl %eax, 32(%esp,%ebx), %edx
-; FALLBACK22-NEXT:    movl %eax, %edi
-; FALLBACK22-NEXT:    notb %cl
-; FALLBACK22-NEXT:    movl 36(%esp,%ebx), %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    leal (%eax,%eax), %esi
-; FALLBACK22-NEXT:    shlxl %ecx, %esi, %eax
-; FALLBACK22-NEXT:    orl %edx, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 48(%esp,%ebx), %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    leal (%eax,%eax), %edx
-; FALLBACK22-NEXT:    shlxl %ecx, %edx, %eax
-; FALLBACK22-NEXT:    movl 44(%esp,%ebx), %edx
-; FALLBACK22-NEXT:    shrxl %edi, %edx, %esi
-; FALLBACK22-NEXT:    orl %esi, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    addl %edx, %edx
-; FALLBACK22-NEXT:    shlxl %ecx, %edx, %eax
-; FALLBACK22-NEXT:    movl 40(%esp,%ebx), %edx
-; FALLBACK22-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrxl %edi, %edx, %esi
-; FALLBACK22-NEXT:    movl %edi, %edx
-; FALLBACK22-NEXT:    orl %esi, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 56(%esp,%ebx), %esi
-; FALLBACK22-NEXT:    leal (%esi,%esi), %ebp
-; FALLBACK22-NEXT:    shlxl %ecx, %ebp, %ebp
-; FALLBACK22-NEXT:    movl 52(%esp,%ebx), %eax
-; FALLBACK22-NEXT:    shrxl %edi, %eax, %edi
-; FALLBACK22-NEXT:    orl %edi, %ebp
-; FALLBACK22-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK22-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    addl %eax, %eax
-; FALLBACK22-NEXT:    shlxl %ecx, %eax, %edi
-; FALLBACK22-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK22-NEXT:    shrxl %edx, %esi, %eax
-; FALLBACK22-NEXT:    movl 60(%esp,%ebx), %esi
-; FALLBACK22-NEXT:    leal (%esi,%esi), %ebx
-; FALLBACK22-NEXT:    shlxl %ecx, %ebx, %ebx
-; FALLBACK22-NEXT:    orl %eax, %ebx
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    addl %eax, %eax
-; FALLBACK22-NEXT:    shlxl %ecx, %eax, %eax
-; FALLBACK22-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; FALLBACK22-NEXT:    orl %ecx, %eax
-; FALLBACK22-NEXT:    shrxl %edx, %esi, %ecx
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK22-NEXT:    movl %ecx, 28(%edx)
-; FALLBACK22-NEXT:    movl %eax, 4(%edx)
-; FALLBACK22-NEXT:    movl %ebx, 24(%edx)
-; FALLBACK22-NEXT:    movl %edi, 16(%edx)
-; FALLBACK22-NEXT:    movl %ebp, 20(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 8(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 12(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, (%edx)
-; FALLBACK22-NEXT:    addl $108, %esp
-; FALLBACK22-NEXT:    popl %esi
-; FALLBACK22-NEXT:    popl %edi
-; FALLBACK22-NEXT:    popl %ebx
-; FALLBACK22-NEXT:    popl %ebp
-; FALLBACK22-NEXT:    retl
-;
-; FALLBACK23-LABEL: lshr_32bytes:
-; FALLBACK23:       # %bb.0:
-; FALLBACK23-NEXT:    pushl %ebp
-; FALLBACK23-NEXT:    pushl %ebx
-; FALLBACK23-NEXT:    pushl %edi
-; FALLBACK23-NEXT:    pushl %esi
-; FALLBACK23-NEXT:    subl $108, %esp
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK23-NEXT:    movups (%ecx), %xmm0
-; FALLBACK23-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK23-NEXT:    movzbl (%eax), %eax
-; FALLBACK23-NEXT:    movl %eax, %ecx
-; FALLBACK23-NEXT:    shlb $3, %cl
-; FALLBACK23-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK23-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    andb $28, %al
-; FALLBACK23-NEXT:    movzbl %al, %ebx
-; FALLBACK23-NEXT:    movl 48(%esp,%ebx), %esi
-; FALLBACK23-NEXT:    movl 44(%esp,%ebx), %eax
-; FALLBACK23-NEXT:    movl %eax, %edx
-; FALLBACK23-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 40(%esp,%ebx), %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 56(%esp,%ebx), %ebp
-; FALLBACK23-NEXT:    movl 52(%esp,%ebx), %eax
-; FALLBACK23-NEXT:    movl %eax, %edi
-; FALLBACK23-NEXT:    shrdl %cl, %ebp, %edi
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK23-NEXT:    movl 60(%esp,%ebx), %eax
-; FALLBACK23-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %ebp
-; FALLBACK23-NEXT:    movl 32(%esp,%ebx), %edx
-; FALLBACK23-NEXT:    movl 36(%esp,%ebx), %ebx
-; FALLBACK23-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK23-NEXT:    movl %ebx, 4(%eax)
-; FALLBACK23-NEXT:    movl %ebp, 24(%eax)
-; FALLBACK23-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; FALLBACK23-NEXT:    movl %ebx, 28(%eax)
-; FALLBACK23-NEXT:    movl %esi, 16(%eax)
-; FALLBACK23-NEXT:    movl %edi, 20(%eax)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK23-NEXT:    movl %esi, 8(%eax)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK23-NEXT:    movl %esi, 12(%eax)
-; FALLBACK23-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK23-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK23-NEXT:    movl %edx, (%eax)
-; FALLBACK23-NEXT:    addl $108, %esp
-; FALLBACK23-NEXT:    popl %esi
-; FALLBACK23-NEXT:    popl %edi
-; FALLBACK23-NEXT:    popl %ebx
-; FALLBACK23-NEXT:    popl %ebp
-; FALLBACK23-NEXT:    retl
-;
-; FALLBACK24-LABEL: lshr_32bytes:
-; FALLBACK24:       # %bb.0:
-; FALLBACK24-NEXT:    pushl %ebp
-; FALLBACK24-NEXT:    pushl %ebx
-; FALLBACK24-NEXT:    pushl %edi
-; FALLBACK24-NEXT:    pushl %esi
-; FALLBACK24-NEXT:    subl $108, %esp
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK24-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK24-NEXT:    movzbl (%eax), %ecx
-; FALLBACK24-NEXT:    movl %ecx, %eax
-; FALLBACK24-NEXT:    shlb $3, %al
-; FALLBACK24-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK24-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    andb $28, %cl
-; FALLBACK24-NEXT:    movzbl %cl, %edi
-; FALLBACK24-NEXT:    movl 32(%esp,%edi), %esi
-; FALLBACK24-NEXT:    movl 36(%esp,%edi), %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %esi
-; FALLBACK24-NEXT:    movl %eax, %edx
-; FALLBACK24-NEXT:    notb %dl
-; FALLBACK24-NEXT:    addl %ebx, %ebx
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    orl %esi, %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 44(%esp,%edi), %ebp
-; FALLBACK24-NEXT:    movl %ebp, %esi
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %esi
-; FALLBACK24-NEXT:    movl 48(%esp,%edi), %ecx
-; FALLBACK24-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    orl %esi, %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 40(%esp,%edi), %esi
-; FALLBACK24-NEXT:    movl %esi, %ebx
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %ebx
-; FALLBACK24-NEXT:    addl %ebp, %ebp
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shll %cl, %ebp
-; FALLBACK24-NEXT:    orl %ebx, %ebp
-; FALLBACK24-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 52(%esp,%edi), %ebp
-; FALLBACK24-NEXT:    movl %ebp, %ebx
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %ebx
-; FALLBACK24-NEXT:    movl 56(%esp,%edi), %ecx
-; FALLBACK24-NEXT:    movl %ecx, (%esp) # 4-byte Spill
-; FALLBACK24-NEXT:    leal (%ecx,%ecx), %edi
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shll %cl, %edi
-; FALLBACK24-NEXT:    orl %ebx, %edi
-; FALLBACK24-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %cl, %edi
-; FALLBACK24-NEXT:    addl %ebp, %ebp
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shll %cl, %ebp
-; FALLBACK24-NEXT:    orl %edi, %ebp
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shrl %cl, (%esp) # 4-byte Folded Spill
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl 60(%esp,%ecx), %ebx
-; FALLBACK24-NEXT:    leal (%ebx,%ebx), %edi
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shll %cl, %edi
-; FALLBACK24-NEXT:    orl (%esp), %edi # 4-byte Folded Reload
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shrl %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; FALLBACK24-NEXT:    addl %esi, %esi
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shll %cl, %esi
-; FALLBACK24-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %ebx
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK24-NEXT:    movl %ebx, 28(%eax)
-; FALLBACK24-NEXT:    movl %esi, 4(%eax)
-; FALLBACK24-NEXT:    movl %edi, 24(%eax)
-; FALLBACK24-NEXT:    movl %ebp, 16(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, (%eax)
-; FALLBACK24-NEXT:    addl $108, %esp
-; FALLBACK24-NEXT:    popl %esi
-; FALLBACK24-NEXT:    popl %edi
-; FALLBACK24-NEXT:    popl %ebx
-; FALLBACK24-NEXT:    popl %ebp
-; FALLBACK24-NEXT:    vzeroupper
-; FALLBACK24-NEXT:    retl
-;
-; FALLBACK25-LABEL: lshr_32bytes:
-; FALLBACK25:       # %bb.0:
-; FALLBACK25-NEXT:    pushl %ebp
-; FALLBACK25-NEXT:    pushl %ebx
-; FALLBACK25-NEXT:    pushl %edi
-; FALLBACK25-NEXT:    pushl %esi
-; FALLBACK25-NEXT:    subl $108, %esp
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK25-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK25-NEXT:    movzbl (%eax), %eax
-; FALLBACK25-NEXT:    movl %eax, %ecx
-; FALLBACK25-NEXT:    shlb $3, %cl
-; FALLBACK25-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK25-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    andb $28, %al
-; FALLBACK25-NEXT:    movzbl %al, %ebp
-; FALLBACK25-NEXT:    movl 48(%esp,%ebp), %esi
-; FALLBACK25-NEXT:    movl 44(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, %edx
-; FALLBACK25-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 40(%esp,%ebp), %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 56(%esp,%ebp), %ebx
-; FALLBACK25-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, %edx
-; FALLBACK25-NEXT:    shrdl %cl, %ebx, %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK25-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK25-NEXT:    movl 32(%esp,%ebp), %edx
-; FALLBACK25-NEXT:    movl 36(%esp,%ebp), %edi
-; FALLBACK25-NEXT:    movl %edi, %esi
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; FALLBACK25-NEXT:    shrdl %cl, %ebp, %esi
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK25-NEXT:    movl %esi, 4(%ebp)
-; FALLBACK25-NEXT:    movl %ebx, 24(%ebp)
-; FALLBACK25-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK25-NEXT:    shrl %cl, %eax
-; FALLBACK25-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK25-NEXT:    movl %edx, (%ebp)
-; FALLBACK25-NEXT:    addl $108, %esp
-; FALLBACK25-NEXT:    popl %esi
-; FALLBACK25-NEXT:    popl %edi
-; FALLBACK25-NEXT:    popl %ebx
-; FALLBACK25-NEXT:    popl %ebp
-; FALLBACK25-NEXT:    vzeroupper
-; FALLBACK25-NEXT:    retl
-;
-; FALLBACK26-LABEL: lshr_32bytes:
-; FALLBACK26:       # %bb.0:
-; FALLBACK26-NEXT:    pushl %ebp
-; FALLBACK26-NEXT:    pushl %ebx
-; FALLBACK26-NEXT:    pushl %edi
-; FALLBACK26-NEXT:    pushl %esi
-; FALLBACK26-NEXT:    subl $108, %esp
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK26-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK26-NEXT:    movzbl (%eax), %edx
-; FALLBACK26-NEXT:    movl %edx, %ecx
-; FALLBACK26-NEXT:    shlb $3, %cl
-; FALLBACK26-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK26-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, %eax
-; FALLBACK26-NEXT:    andb $28, %dl
-; FALLBACK26-NEXT:    movzbl %dl, %ebx
-; FALLBACK26-NEXT:    shrxl %eax, 32(%esp,%ebx), %edx
-; FALLBACK26-NEXT:    movl %eax, %edi
-; FALLBACK26-NEXT:    notb %cl
-; FALLBACK26-NEXT:    movl 36(%esp,%ebx), %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    leal (%eax,%eax), %esi
-; FALLBACK26-NEXT:    shlxl %ecx, %esi, %eax
-; FALLBACK26-NEXT:    orl %edx, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 48(%esp,%ebx), %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    leal (%eax,%eax), %edx
-; FALLBACK26-NEXT:    shlxl %ecx, %edx, %eax
-; FALLBACK26-NEXT:    movl 44(%esp,%ebx), %edx
-; FALLBACK26-NEXT:    shrxl %edi, %edx, %esi
-; FALLBACK26-NEXT:    orl %esi, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    addl %edx, %edx
-; FALLBACK26-NEXT:    shlxl %ecx, %edx, %eax
-; FALLBACK26-NEXT:    movl 40(%esp,%ebx), %edx
-; FALLBACK26-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrxl %edi, %edx, %esi
-; FALLBACK26-NEXT:    movl %edi, %edx
-; FALLBACK26-NEXT:    orl %esi, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 56(%esp,%ebx), %esi
-; FALLBACK26-NEXT:    leal (%esi,%esi), %ebp
-; FALLBACK26-NEXT:    shlxl %ecx, %ebp, %ebp
-; FALLBACK26-NEXT:    movl 52(%esp,%ebx), %eax
-; FALLBACK26-NEXT:    shrxl %edi, %eax, %edi
-; FALLBACK26-NEXT:    orl %edi, %ebp
-; FALLBACK26-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK26-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    addl %eax, %eax
-; FALLBACK26-NEXT:    shlxl %ecx, %eax, %edi
-; FALLBACK26-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK26-NEXT:    shrxl %edx, %esi, %eax
-; FALLBACK26-NEXT:    movl 60(%esp,%ebx), %esi
-; FALLBACK26-NEXT:    leal (%esi,%esi), %ebx
-; FALLBACK26-NEXT:    shlxl %ecx, %ebx, %ebx
-; FALLBACK26-NEXT:    orl %eax, %ebx
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    addl %eax, %eax
-; FALLBACK26-NEXT:    shlxl %ecx, %eax, %eax
-; FALLBACK26-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; FALLBACK26-NEXT:    orl %ecx, %eax
-; FALLBACK26-NEXT:    shrxl %edx, %esi, %ecx
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK26-NEXT:    movl %ecx, 28(%edx)
-; FALLBACK26-NEXT:    movl %eax, 4(%edx)
-; FALLBACK26-NEXT:    movl %ebx, 24(%edx)
-; FALLBACK26-NEXT:    movl %edi, 16(%edx)
-; FALLBACK26-NEXT:    movl %ebp, 20(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 8(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 12(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, (%edx)
-; FALLBACK26-NEXT:    addl $108, %esp
-; FALLBACK26-NEXT:    popl %esi
-; FALLBACK26-NEXT:    popl %edi
-; FALLBACK26-NEXT:    popl %ebx
-; FALLBACK26-NEXT:    popl %ebp
-; FALLBACK26-NEXT:    vzeroupper
-; FALLBACK26-NEXT:    retl
-;
-; FALLBACK27-LABEL: lshr_32bytes:
-; FALLBACK27:       # %bb.0:
-; FALLBACK27-NEXT:    pushl %ebp
-; FALLBACK27-NEXT:    pushl %ebx
-; FALLBACK27-NEXT:    pushl %edi
-; FALLBACK27-NEXT:    pushl %esi
-; FALLBACK27-NEXT:    subl $108, %esp
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK27-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK27-NEXT:    movzbl (%eax), %eax
-; FALLBACK27-NEXT:    movl %eax, %ecx
-; FALLBACK27-NEXT:    shlb $3, %cl
-; FALLBACK27-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK27-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    andb $28, %al
-; FALLBACK27-NEXT:    movzbl %al, %ebx
-; FALLBACK27-NEXT:    movl 48(%esp,%ebx), %esi
-; FALLBACK27-NEXT:    movl 44(%esp,%ebx), %eax
-; FALLBACK27-NEXT:    movl %eax, %edx
-; FALLBACK27-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 40(%esp,%ebx), %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 56(%esp,%ebx), %ebp
-; FALLBACK27-NEXT:    movl 52(%esp,%ebx), %eax
-; FALLBACK27-NEXT:    movl %eax, %edi
-; FALLBACK27-NEXT:    shrdl %cl, %ebp, %edi
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK27-NEXT:    movl 60(%esp,%ebx), %eax
-; FALLBACK27-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %ebp
-; FALLBACK27-NEXT:    movl 32(%esp,%ebx), %edx
-; FALLBACK27-NEXT:    movl 36(%esp,%ebx), %ebx
-; FALLBACK27-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK27-NEXT:    movl %ebx, 4(%eax)
-; FALLBACK27-NEXT:    movl %ebp, 24(%eax)
-; FALLBACK27-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; FALLBACK27-NEXT:    movl %ebx, 28(%eax)
-; FALLBACK27-NEXT:    movl %esi, 16(%eax)
-; FALLBACK27-NEXT:    movl %edi, 20(%eax)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK27-NEXT:    movl %esi, 8(%eax)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK27-NEXT:    movl %esi, 12(%eax)
-; FALLBACK27-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK27-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK27-NEXT:    movl %edx, (%eax)
-; FALLBACK27-NEXT:    addl $108, %esp
-; FALLBACK27-NEXT:    popl %esi
-; FALLBACK27-NEXT:    popl %edi
-; FALLBACK27-NEXT:    popl %ebx
-; FALLBACK27-NEXT:    popl %ebp
-; FALLBACK27-NEXT:    vzeroupper
-; FALLBACK27-NEXT:    retl
-;
-; FALLBACK28-LABEL: lshr_32bytes:
-; FALLBACK28:       # %bb.0:
-; FALLBACK28-NEXT:    pushl %ebp
-; FALLBACK28-NEXT:    pushl %ebx
-; FALLBACK28-NEXT:    pushl %edi
-; FALLBACK28-NEXT:    pushl %esi
-; FALLBACK28-NEXT:    subl $108, %esp
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK28-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK28-NEXT:    movzbl (%eax), %ecx
-; FALLBACK28-NEXT:    movl %ecx, %eax
-; FALLBACK28-NEXT:    shlb $3, %al
-; FALLBACK28-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK28-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    andb $28, %cl
-; FALLBACK28-NEXT:    movzbl %cl, %edi
-; FALLBACK28-NEXT:    movl 32(%esp,%edi), %esi
-; FALLBACK28-NEXT:    movl 36(%esp,%edi), %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %esi
-; FALLBACK28-NEXT:    movl %eax, %edx
-; FALLBACK28-NEXT:    notb %dl
-; FALLBACK28-NEXT:    addl %ebx, %ebx
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    orl %esi, %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 44(%esp,%edi), %ebp
-; FALLBACK28-NEXT:    movl %ebp, %esi
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %esi
-; FALLBACK28-NEXT:    movl 48(%esp,%edi), %ecx
-; FALLBACK28-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    orl %esi, %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 40(%esp,%edi), %esi
-; FALLBACK28-NEXT:    movl %esi, %ebx
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %ebx
-; FALLBACK28-NEXT:    addl %ebp, %ebp
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shll %cl, %ebp
-; FALLBACK28-NEXT:    orl %ebx, %ebp
-; FALLBACK28-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 52(%esp,%edi), %ebp
-; FALLBACK28-NEXT:    movl %ebp, %ebx
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %ebx
-; FALLBACK28-NEXT:    movl 56(%esp,%edi), %ecx
-; FALLBACK28-NEXT:    movl %ecx, (%esp) # 4-byte Spill
-; FALLBACK28-NEXT:    leal (%ecx,%ecx), %edi
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shll %cl, %edi
-; FALLBACK28-NEXT:    orl %ebx, %edi
-; FALLBACK28-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %cl, %edi
-; FALLBACK28-NEXT:    addl %ebp, %ebp
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shll %cl, %ebp
-; FALLBACK28-NEXT:    orl %edi, %ebp
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shrl %cl, (%esp) # 4-byte Folded Spill
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl 60(%esp,%ecx), %ebx
-; FALLBACK28-NEXT:    leal (%ebx,%ebx), %edi
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shll %cl, %edi
-; FALLBACK28-NEXT:    orl (%esp), %edi # 4-byte Folded Reload
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shrl %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; FALLBACK28-NEXT:    addl %esi, %esi
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shll %cl, %esi
-; FALLBACK28-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %ebx
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK28-NEXT:    movl %ebx, 28(%eax)
-; FALLBACK28-NEXT:    movl %esi, 4(%eax)
-; FALLBACK28-NEXT:    movl %edi, 24(%eax)
-; FALLBACK28-NEXT:    movl %ebp, 16(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, (%eax)
-; FALLBACK28-NEXT:    addl $108, %esp
-; FALLBACK28-NEXT:    popl %esi
-; FALLBACK28-NEXT:    popl %edi
-; FALLBACK28-NEXT:    popl %ebx
-; FALLBACK28-NEXT:    popl %ebp
-; FALLBACK28-NEXT:    vzeroupper
-; FALLBACK28-NEXT:    retl
-;
-; FALLBACK29-LABEL: lshr_32bytes:
-; FALLBACK29:       # %bb.0:
-; FALLBACK29-NEXT:    pushl %ebp
-; FALLBACK29-NEXT:    pushl %ebx
-; FALLBACK29-NEXT:    pushl %edi
-; FALLBACK29-NEXT:    pushl %esi
-; FALLBACK29-NEXT:    subl $108, %esp
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK29-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK29-NEXT:    movzbl (%eax), %eax
-; FALLBACK29-NEXT:    movl %eax, %ecx
-; FALLBACK29-NEXT:    shlb $3, %cl
-; FALLBACK29-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK29-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    andb $28, %al
-; FALLBACK29-NEXT:    movzbl %al, %ebp
-; FALLBACK29-NEXT:    movl 48(%esp,%ebp), %esi
-; FALLBACK29-NEXT:    movl 44(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, %edx
-; FALLBACK29-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 40(%esp,%ebp), %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 56(%esp,%ebp), %ebx
-; FALLBACK29-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, %edx
-; FALLBACK29-NEXT:    shrdl %cl, %ebx, %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK29-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK29-NEXT:    movl 32(%esp,%ebp), %edx
-; FALLBACK29-NEXT:    movl 36(%esp,%ebp), %edi
-; FALLBACK29-NEXT:    movl %edi, %esi
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; FALLBACK29-NEXT:    shrdl %cl, %ebp, %esi
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK29-NEXT:    movl %esi, 4(%ebp)
-; FALLBACK29-NEXT:    movl %ebx, 24(%ebp)
-; FALLBACK29-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK29-NEXT:    shrl %cl, %eax
-; FALLBACK29-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK29-NEXT:    movl %edx, (%ebp)
-; FALLBACK29-NEXT:    addl $108, %esp
-; FALLBACK29-NEXT:    popl %esi
-; FALLBACK29-NEXT:    popl %edi
-; FALLBACK29-NEXT:    popl %ebx
-; FALLBACK29-NEXT:    popl %ebp
-; FALLBACK29-NEXT:    vzeroupper
-; FALLBACK29-NEXT:    retl
-;
-; FALLBACK30-LABEL: lshr_32bytes:
-; FALLBACK30:       # %bb.0:
-; FALLBACK30-NEXT:    pushl %ebp
-; FALLBACK30-NEXT:    pushl %ebx
-; FALLBACK30-NEXT:    pushl %edi
-; FALLBACK30-NEXT:    pushl %esi
-; FALLBACK30-NEXT:    subl $108, %esp
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK30-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK30-NEXT:    movzbl (%eax), %edx
-; FALLBACK30-NEXT:    movl %edx, %ecx
-; FALLBACK30-NEXT:    shlb $3, %cl
-; FALLBACK30-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK30-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, %eax
-; FALLBACK30-NEXT:    andb $28, %dl
-; FALLBACK30-NEXT:    movzbl %dl, %ebx
-; FALLBACK30-NEXT:    shrxl %eax, 32(%esp,%ebx), %edx
-; FALLBACK30-NEXT:    movl %eax, %edi
-; FALLBACK30-NEXT:    notb %cl
-; FALLBACK30-NEXT:    movl 36(%esp,%ebx), %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    leal (%eax,%eax), %esi
-; FALLBACK30-NEXT:    shlxl %ecx, %esi, %eax
-; FALLBACK30-NEXT:    orl %edx, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 48(%esp,%ebx), %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    leal (%eax,%eax), %edx
-; FALLBACK30-NEXT:    shlxl %ecx, %edx, %eax
-; FALLBACK30-NEXT:    movl 44(%esp,%ebx), %edx
-; FALLBACK30-NEXT:    shrxl %edi, %edx, %esi
-; FALLBACK30-NEXT:    orl %esi, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    addl %edx, %edx
-; FALLBACK30-NEXT:    shlxl %ecx, %edx, %eax
-; FALLBACK30-NEXT:    movl 40(%esp,%ebx), %edx
-; FALLBACK30-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrxl %edi, %edx, %esi
-; FALLBACK30-NEXT:    movl %edi, %edx
-; FALLBACK30-NEXT:    orl %esi, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 56(%esp,%ebx), %esi
-; FALLBACK30-NEXT:    leal (%esi,%esi), %ebp
-; FALLBACK30-NEXT:    shlxl %ecx, %ebp, %ebp
-; FALLBACK30-NEXT:    movl 52(%esp,%ebx), %eax
-; FALLBACK30-NEXT:    shrxl %edi, %eax, %edi
-; FALLBACK30-NEXT:    orl %edi, %ebp
-; FALLBACK30-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK30-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    addl %eax, %eax
-; FALLBACK30-NEXT:    shlxl %ecx, %eax, %edi
-; FALLBACK30-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK30-NEXT:    shrxl %edx, %esi, %eax
-; FALLBACK30-NEXT:    movl 60(%esp,%ebx), %esi
-; FALLBACK30-NEXT:    leal (%esi,%esi), %ebx
-; FALLBACK30-NEXT:    shlxl %ecx, %ebx, %ebx
-; FALLBACK30-NEXT:    orl %eax, %ebx
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    addl %eax, %eax
-; FALLBACK30-NEXT:    shlxl %ecx, %eax, %eax
-; FALLBACK30-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; FALLBACK30-NEXT:    orl %ecx, %eax
-; FALLBACK30-NEXT:    shrxl %edx, %esi, %ecx
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK30-NEXT:    movl %ecx, 28(%edx)
-; FALLBACK30-NEXT:    movl %eax, 4(%edx)
-; FALLBACK30-NEXT:    movl %ebx, 24(%edx)
-; FALLBACK30-NEXT:    movl %edi, 16(%edx)
-; FALLBACK30-NEXT:    movl %ebp, 20(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 8(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 12(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, (%edx)
-; FALLBACK30-NEXT:    addl $108, %esp
-; FALLBACK30-NEXT:    popl %esi
-; FALLBACK30-NEXT:    popl %edi
-; FALLBACK30-NEXT:    popl %ebx
-; FALLBACK30-NEXT:    popl %ebp
-; FALLBACK30-NEXT:    vzeroupper
-; FALLBACK30-NEXT:    retl
-;
-; FALLBACK31-LABEL: lshr_32bytes:
-; FALLBACK31:       # %bb.0:
-; FALLBACK31-NEXT:    pushl %ebp
-; FALLBACK31-NEXT:    pushl %ebx
-; FALLBACK31-NEXT:    pushl %edi
-; FALLBACK31-NEXT:    pushl %esi
-; FALLBACK31-NEXT:    subl $108, %esp
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK31-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK31-NEXT:    movzbl (%eax), %eax
-; FALLBACK31-NEXT:    movl %eax, %ecx
-; FALLBACK31-NEXT:    shlb $3, %cl
-; FALLBACK31-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK31-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    andb $28, %al
-; FALLBACK31-NEXT:    movzbl %al, %ebx
-; FALLBACK31-NEXT:    movl 48(%esp,%ebx), %esi
-; FALLBACK31-NEXT:    movl 44(%esp,%ebx), %eax
-; FALLBACK31-NEXT:    movl %eax, %edx
-; FALLBACK31-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 40(%esp,%ebx), %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 56(%esp,%ebx), %ebp
-; FALLBACK31-NEXT:    movl 52(%esp,%ebx), %eax
-; FALLBACK31-NEXT:    movl %eax, %edi
-; FALLBACK31-NEXT:    shrdl %cl, %ebp, %edi
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK31-NEXT:    movl 60(%esp,%ebx), %eax
-; FALLBACK31-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %ebp
-; FALLBACK31-NEXT:    movl 32(%esp,%ebx), %edx
-; FALLBACK31-NEXT:    movl 36(%esp,%ebx), %ebx
-; FALLBACK31-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK31-NEXT:    movl %ebx, 4(%eax)
-; FALLBACK31-NEXT:    movl %ebp, 24(%eax)
-; FALLBACK31-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; FALLBACK31-NEXT:    movl %ebx, 28(%eax)
-; FALLBACK31-NEXT:    movl %esi, 16(%eax)
-; FALLBACK31-NEXT:    movl %edi, 20(%eax)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK31-NEXT:    movl %esi, 8(%eax)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK31-NEXT:    movl %esi, 12(%eax)
-; FALLBACK31-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK31-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK31-NEXT:    movl %edx, (%eax)
-; FALLBACK31-NEXT:    addl $108, %esp
-; FALLBACK31-NEXT:    popl %esi
-; FALLBACK31-NEXT:    popl %edi
-; FALLBACK31-NEXT:    popl %ebx
-; FALLBACK31-NEXT:    popl %ebp
-; FALLBACK31-NEXT:    vzeroupper
-; FALLBACK31-NEXT:    retl
+; X64-NO-SHLD-NO-BMI2-SSE2-LABEL: lshr_32bytes:
+; X64-NO-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    andb $24, %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movzbl %sil, %r9d
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -64(%rsp,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -56(%rsp,%r9), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -48(%rsp,%r9), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leaq (%rbx,%rbx), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r11, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    addq %rdi, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -40(%rsp,%r9), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leaq (%r9,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %rbx, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE2-LABEL: lshr_32bytes:
+; X64-HAVE-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    leal (,%rsi,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andb $24, %sil
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movzbl %sil, %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -56(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -72(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -64(%rsp,%rax), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %rsi, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -48(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %rax, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %r8, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE2-LABEL: lshr_32bytes:
+; X64-NO-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $24, %sil
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl %sil, %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -64(%rsp,%rsi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -56(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rcx, %rdi, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leaq (%r8,%r8), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rax, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r9, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rcx, -72(%rsp,%rsi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addq %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rax, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r9, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rcx, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -48(%rsp,%rsi), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leaq (%rsi,%rsi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rax, %r9, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r8, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rcx, %rsi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r10, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: lshr_32bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (,%rsi,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $24, %sil
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl %sil, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -56(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -72(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -64(%rsp,%rax), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %rsi, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -48(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %rax, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %r8, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rcx, %rax, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-SSE4-LABEL: lshr_32bytes:
+; X64-NO-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%rsi), %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (,%rcx,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    andb $24, %cl
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl %cl, %r9d
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -64(%rsp,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -56(%rsp,%r9), %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leaq (%r8,%r8), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -48(%rsp,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -40(%rsp,%r9), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leaq (%r9,%r9), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    addq %r10, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r8, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rbx, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE4-LABEL: lshr_32bytes:
+; X64-HAVE-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%rsi), %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andb $24, %al
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl %al, %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -48(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -56(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %rsi, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -72(%rsp,%rax), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -64(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rax, %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %rdi, %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %rax, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE4-LABEL: lshr_32bytes:
+; X64-NO-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%rsi), %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (,%rcx,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $24, %cl
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl %cl, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, -72(%rsp,%rcx), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -64(%rsp,%rcx), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -56(%rsp,%rcx), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leaq (%r8,%r8), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rax, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rdi, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %r9, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -48(%rsp,%rcx), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leaq (%rcx,%rcx), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rax, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rdi, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %r8, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addq %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rax, %r9, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rdi, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %rcx, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rcx, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: lshr_32bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $24, %al
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl %al, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -48(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -56(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %rsi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -72(%rsp,%rax), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -64(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %rdi, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %rax, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rcx, %rsi, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r10, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-AVX-LABEL: lshr_32bytes:
+; X64-NO-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%rsi), %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    leal (,%rcx,8), %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    andb $24, %cl
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl %cl, %r9d
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -64(%rsp,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -56(%rsp,%r9), %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    leaq (%r8,%r8), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -48(%rsp,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -40(%rsp,%r9), %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    leaq (%r9,%r9), %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    addq %r10, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r8, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rbx, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vzeroupper
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-AVX-LABEL: lshr_32bytes:
+; X64-HAVE-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%rsi), %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    andb $24, %al
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl %al, %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -48(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -56(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %rsi, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -72(%rsp,%rax), %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -64(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rax, %r10
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %rdi, %r10
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %rax, %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vzeroupper
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-AVX-LABEL: lshr_32bytes:
+; X64-NO-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%rsi), %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (,%rax,8), %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    andb $24, %al
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl %al, %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rsi, -72(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    notb %cl
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -64(%rsp,%rax), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -56(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leaq (%r8,%r8), %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %rdi, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rsi, %r9, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -48(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leaq (%rax,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %rdi, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rsi, %r8, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addq %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %r9, %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %rdi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rsi, %rax, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rcx, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vzeroupper
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-LABEL: lshr_32bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    andb $24, %al
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl %al, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -48(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -56(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %rsi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -72(%rsp,%rax), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -64(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %rdi, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %rax, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rcx, %rsi, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r10, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vzeroupper
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    retq
+;
+; X86-NO-SHLD-NO-BMI2-SSE2-LABEL: lshr_32bytes:
+; X86-NO-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    subl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl (%ebp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%ebp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%ebp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb (%eax), %ah
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%ebp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%ebp), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ah, %dh
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlb $3, %dh
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    andb $28, %ah
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movzbl %ah, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 32(%esp,%edi), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 36(%esp,%edi), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dh, %dl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    notb %dl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 40(%esp,%edi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%edi,%edi), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 44(%esp,%eax), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 48(%esp,%eax), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%eax,%eax), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %ebp, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 52(%esp,%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 56(%esp,%eax), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%esi,%esi), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 60(%esp,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%ebx,%ebx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, (%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-SSE2-LABEL: lshr_32bytes:
+; X86-HAVE-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    subl $92, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%ebp), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movb (%ecx), %ch
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%ebp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andb $28, %ch
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movzbl %ch, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 32(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %ebx, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 40(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 36(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 44(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, 24(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%esp), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, 16(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, 20(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    addl $92, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-SSE2-LABEL: lshr_32bytes:
+; X86-NO-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    subl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%eax), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%ebx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 28(%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $3, %cl
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $28, %bl
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl %bl, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 36(%esp,%esi), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 40(%esp,%esi), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %eax, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    notb %cl
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%ebp,%ebp), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %ebx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %eax, 32(%esp,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 48(%esp,%esi), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 44(%esp,%esi), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %eax, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %eax, %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 56(%esp,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%edi,%edi), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 52(%esp,%esi), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebp, %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 60(%esp,%esi), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%esi,%esi), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %ebx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebp, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebp, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, 28(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 24(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 16(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, 20(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 8(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 12(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, (%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 4(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: lshr_32bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    subl $92, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%ecx), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%ecx), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%ecx), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%ebx), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%ecx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 28(%ecx), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $28, %bl
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl %bl, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%esp,%ebp), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %esi, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 32(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 28(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %ebx, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 40(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 36(%esp,%ebp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%esp,%ebp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 44(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %edi, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, %edi, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, 16(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, 20(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, (%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 4(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    addl $92, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-SSE4-LABEL: lshr_32bytes:
+; X86-NO-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    subl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlb $3, %al
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    andb $28, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 32(%esp,%edi), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 36(%esp,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    notb %dl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 44(%esp,%edi), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 48(%esp,%edi), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%ecx,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 40(%esp,%edi), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %ebp, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 52(%esp,%edi), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 56(%esp,%edi), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%ecx,%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %ebp, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, (%esp) # 4-byte Folded Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 60(%esp,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%ebx,%ebx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl (%esp), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, (%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-SSE4-LABEL: lshr_32bytes:
+; X86-HAVE-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    subl $108, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%eax), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andb $28, %al
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl %al, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 48(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 44(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 40(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 56(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %ebx, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 32(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 36(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %ebp, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, 24(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    addl $108, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-SSE4-LABEL: lshr_32bytes:
+; X86-NO-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    subl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $3, %cl
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $28, %dl
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl %dl, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %eax, 32(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    notb %cl
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 36(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%eax,%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ecx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 48(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%eax,%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ecx, %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 44(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %edi, %edx, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ecx, %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 40(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %edi, %edx, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 56(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%esi,%esi), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ecx, %ebp, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 52(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %edi, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ecx, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 60(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%esi,%esi), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ecx, %ebx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ecx, %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %edx, %esi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 28(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 4(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, 24(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, 16(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, 20(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 8(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, (%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: lshr_32bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    subl $108, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%eax), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $28, %al
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl %al, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 48(%esp,%ebx), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 44(%esp,%ebx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 40(%esp,%ebx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 56(%esp,%ebx), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 52(%esp,%ebx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %ebp, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 60(%esp,%ebx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 32(%esp,%ebx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 36(%esp,%ebx), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, 4(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, 24(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, 28(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, 16(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, 20(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, 8(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, 12(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, (%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    addl $108, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-AVX-LABEL: lshr_32bytes:
+; X86-NO-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    subl $108, %esp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%ecx), %ymm0
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shlb $3, %al
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    andb $28, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 32(%esp,%edi), %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 36(%esp,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    notb %dl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 44(%esp,%edi), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebp, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 48(%esp,%edi), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    leal (%ecx,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 40(%esp,%edi), %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %ebp, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 52(%esp,%edi), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 56(%esp,%edi), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    leal (%ecx,%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %ebp, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, (%esp) # 4-byte Folded Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 60(%esp,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    leal (%ebx,%ebx), %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl (%esp), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebp, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, (%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl $108, %esp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vzeroupper
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-AVX-LABEL: lshr_32bytes:
+; X86-HAVE-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    subl $108, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%ecx), %ymm0
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%eax), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    andb $28, %al
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl %al, %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 48(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 44(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 40(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 56(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %ebx, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 32(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 36(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %ebp, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, 24(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    addl $108, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vzeroupper
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-AVX-LABEL: lshr_32bytes:
+; X86-NO-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    subl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%ecx), %ymm0
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $3, %cl
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    andb $28, %dl
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl %dl, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %eax, 32(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    notb %cl
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 36(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (%eax,%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ecx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 48(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (%eax,%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ecx, %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 44(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %edi, %edx, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ecx, %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 40(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %edi, %edx, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 56(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (%esi,%esi), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ecx, %ebp, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 52(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %edi, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ecx, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 60(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (%esi,%esi), %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ecx, %ebx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ecx, %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %edx, %esi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 28(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 4(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebx, 24(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, 16(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebp, 20(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 8(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, (%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vzeroupper
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-LABEL: lshr_32bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    subl $108, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%ecx), %ymm0
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%eax), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    andb $28, %al
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl %al, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 48(%esp,%ebx), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 44(%esp,%ebx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 40(%esp,%ebx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 56(%esp,%ebx), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 52(%esp,%ebx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %ebp, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 60(%esp,%ebx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %eax, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 32(%esp,%ebx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 36(%esp,%ebx), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebx, 4(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebp, 24(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebx, 28(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, 16(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, 20(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, 8(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, 12(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, (%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    addl $108, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vzeroupper
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    retl
   %src = load i256, ptr %src.ptr, align 1
   %byteOff = load i256, ptr %byteOff.ptr, align 1
   %bitOff = shl i256 %byteOff, 3
@@ -5046,591 +4208,452 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 }
 
 define void @lshr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nounwind {
-; FALLBACK0-LABEL: lshr_32bytes_dwordOff:
-; FALLBACK0:       # %bb.0:
-; FALLBACK0-NEXT:    pushq %rbx
-; FALLBACK0-NEXT:    movq (%rdi), %rcx
-; FALLBACK0-NEXT:    movq 8(%rdi), %r8
-; FALLBACK0-NEXT:    movq 16(%rdi), %r9
-; FALLBACK0-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK0-NEXT:    movzbl (%rsi), %esi
-; FALLBACK0-NEXT:    movl %esi, %eax
-; FALLBACK0-NEXT:    shlb $5, %al
-; FALLBACK0-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK0-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    andb $6, %sil
-; FALLBACK0-NEXT:    movzbl %sil, %r9d
-; FALLBACK0-NEXT:    movq -64(%rsp,%r9,4), %r10
-; FALLBACK0-NEXT:    movq -56(%rsp,%r9,4), %rdi
-; FALLBACK0-NEXT:    movq %rdi, %r11
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r11
-; FALLBACK0-NEXT:    movl %eax, %esi
-; FALLBACK0-NEXT:    notb %sil
-; FALLBACK0-NEXT:    movq -48(%rsp,%r9,4), %rbx
-; FALLBACK0-NEXT:    leaq (%rbx,%rbx), %r8
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r8
-; FALLBACK0-NEXT:    orq %r11, %r8
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r10
-; FALLBACK0-NEXT:    addq %rdi, %rdi
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %rdi
-; FALLBACK0-NEXT:    orq %r10, %rdi
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %rbx
-; FALLBACK0-NEXT:    movq -40(%rsp,%r9,4), %r9
-; FALLBACK0-NEXT:    leaq (%r9,%r9), %r10
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r10
-; FALLBACK0-NEXT:    orq %rbx, %r10
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r9
-; FALLBACK0-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK0-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK0-NEXT:    movq %rdi, (%rdx)
-; FALLBACK0-NEXT:    movq %r8, 8(%rdx)
-; FALLBACK0-NEXT:    popq %rbx
-; FALLBACK0-NEXT:    retq
-;
-; FALLBACK1-LABEL: lshr_32bytes_dwordOff:
-; FALLBACK1:       # %bb.0:
-; FALLBACK1-NEXT:    movq (%rdi), %rax
-; FALLBACK1-NEXT:    movq 8(%rdi), %r8
-; FALLBACK1-NEXT:    movq 16(%rdi), %r9
-; FALLBACK1-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK1-NEXT:    movzbl (%rsi), %esi
-; FALLBACK1-NEXT:    movl %esi, %ecx
-; FALLBACK1-NEXT:    shlb $5, %cl
-; FALLBACK1-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK1-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    andb $6, %sil
-; FALLBACK1-NEXT:    movzbl %sil, %eax
-; FALLBACK1-NEXT:    movq -56(%rsp,%rax,4), %rsi
-; FALLBACK1-NEXT:    movq -72(%rsp,%rax,4), %rdi
-; FALLBACK1-NEXT:    movq -64(%rsp,%rax,4), %r8
-; FALLBACK1-NEXT:    movq %r8, %r9
-; FALLBACK1-NEXT:    shrdq %cl, %rsi, %r9
-; FALLBACK1-NEXT:    movq -48(%rsp,%rax,4), %rax
-; FALLBACK1-NEXT:    shrdq %cl, %rax, %rsi
-; FALLBACK1-NEXT:    shrdq %cl, %r8, %rdi
-; FALLBACK1-NEXT:    shrq %cl, %rax
-; FALLBACK1-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK1-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK1-NEXT:    movq %rdi, (%rdx)
-; FALLBACK1-NEXT:    movq %r9, 8(%rdx)
-; FALLBACK1-NEXT:    retq
-;
-; FALLBACK2-LABEL: lshr_32bytes_dwordOff:
-; FALLBACK2:       # %bb.0:
-; FALLBACK2-NEXT:    movq (%rdi), %rcx
-; FALLBACK2-NEXT:    movq 8(%rdi), %r8
-; FALLBACK2-NEXT:    movq 16(%rdi), %r9
-; FALLBACK2-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK2-NEXT:    movzbl (%rsi), %esi
-; FALLBACK2-NEXT:    movl %esi, %eax
-; FALLBACK2-NEXT:    shlb $5, %al
-; FALLBACK2-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movl %eax, %ecx
-; FALLBACK2-NEXT:    andb $6, %sil
-; FALLBACK2-NEXT:    movzbl %sil, %esi
-; FALLBACK2-NEXT:    movq -64(%rsp,%rsi,4), %rdi
-; FALLBACK2-NEXT:    movq -56(%rsp,%rsi,4), %r8
-; FALLBACK2-NEXT:    shrxq %rcx, %rdi, %r9
-; FALLBACK2-NEXT:    notb %al
-; FALLBACK2-NEXT:    leaq (%r8,%r8), %r10
-; FALLBACK2-NEXT:    shlxq %rax, %r10, %r10
-; FALLBACK2-NEXT:    orq %r9, %r10
-; FALLBACK2-NEXT:    shrxq %rcx, -72(%rsp,%rsi,4), %r9
-; FALLBACK2-NEXT:    addq %rdi, %rdi
-; FALLBACK2-NEXT:    shlxq %rax, %rdi, %rdi
-; FALLBACK2-NEXT:    orq %r9, %rdi
-; FALLBACK2-NEXT:    shrxq %rcx, %r8, %r8
-; FALLBACK2-NEXT:    movq -48(%rsp,%rsi,4), %rsi
-; FALLBACK2-NEXT:    leaq (%rsi,%rsi), %r9
-; FALLBACK2-NEXT:    shlxq %rax, %r9, %rax
-; FALLBACK2-NEXT:    orq %r8, %rax
-; FALLBACK2-NEXT:    shrxq %rcx, %rsi, %rcx
-; FALLBACK2-NEXT:    movq %rcx, 24(%rdx)
-; FALLBACK2-NEXT:    movq %rax, 16(%rdx)
-; FALLBACK2-NEXT:    movq %rdi, (%rdx)
-; FALLBACK2-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK2-NEXT:    retq
-;
-; FALLBACK3-LABEL: lshr_32bytes_dwordOff:
-; FALLBACK3:       # %bb.0:
-; FALLBACK3-NEXT:    movq (%rdi), %rax
-; FALLBACK3-NEXT:    movq 8(%rdi), %r8
-; FALLBACK3-NEXT:    movq 16(%rdi), %r9
-; FALLBACK3-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK3-NEXT:    movzbl (%rsi), %esi
-; FALLBACK3-NEXT:    movl %esi, %ecx
-; FALLBACK3-NEXT:    shlb $5, %cl
-; FALLBACK3-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    andb $6, %sil
-; FALLBACK3-NEXT:    movzbl %sil, %eax
-; FALLBACK3-NEXT:    movq -56(%rsp,%rax,4), %rsi
-; FALLBACK3-NEXT:    movq -72(%rsp,%rax,4), %rdi
-; FALLBACK3-NEXT:    movq -64(%rsp,%rax,4), %r8
-; FALLBACK3-NEXT:    movq %r8, %r9
-; FALLBACK3-NEXT:    shrdq %cl, %rsi, %r9
-; FALLBACK3-NEXT:    movq -48(%rsp,%rax,4), %rax
-; FALLBACK3-NEXT:    shrdq %cl, %rax, %rsi
-; FALLBACK3-NEXT:    shrdq %cl, %r8, %rdi
-; FALLBACK3-NEXT:    shrxq %rcx, %rax, %rax
-; FALLBACK3-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK3-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK3-NEXT:    movq %rdi, (%rdx)
-; FALLBACK3-NEXT:    movq %r9, 8(%rdx)
-; FALLBACK3-NEXT:    retq
-;
-; FALLBACK4-LABEL: lshr_32bytes_dwordOff:
-; FALLBACK4:       # %bb.0:
-; FALLBACK4-NEXT:    pushq %rbx
-; FALLBACK4-NEXT:    movups (%rdi), %xmm0
-; FALLBACK4-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK4-NEXT:    movzbl (%rsi), %ecx
-; FALLBACK4-NEXT:    movl %ecx, %eax
-; FALLBACK4-NEXT:    shlb $5, %al
-; FALLBACK4-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    andb $6, %cl
-; FALLBACK4-NEXT:    movzbl %cl, %r9d
-; FALLBACK4-NEXT:    movq -64(%rsp,%r9,4), %r10
-; FALLBACK4-NEXT:    movq -56(%rsp,%r9,4), %r8
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r10
-; FALLBACK4-NEXT:    movl %eax, %esi
-; FALLBACK4-NEXT:    notb %sil
-; FALLBACK4-NEXT:    leaq (%r8,%r8), %rdi
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %rdi
-; FALLBACK4-NEXT:    orq %r10, %rdi
-; FALLBACK4-NEXT:    movq -48(%rsp,%r9,4), %r10
-; FALLBACK4-NEXT:    movq %r10, %r11
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r11
-; FALLBACK4-NEXT:    movq -40(%rsp,%r9,4), %r9
-; FALLBACK4-NEXT:    leaq (%r9,%r9), %rbx
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %rbx
-; FALLBACK4-NEXT:    orq %r11, %rbx
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r8
-; FALLBACK4-NEXT:    addq %r10, %r10
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r10
-; FALLBACK4-NEXT:    orq %r8, %r10
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r9
-; FALLBACK4-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK4-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK4-NEXT:    movq %rbx, 16(%rdx)
-; FALLBACK4-NEXT:    movq %rdi, (%rdx)
-; FALLBACK4-NEXT:    popq %rbx
-; FALLBACK4-NEXT:    retq
-;
-; FALLBACK5-LABEL: lshr_32bytes_dwordOff:
-; FALLBACK5:       # %bb.0:
-; FALLBACK5-NEXT:    movups (%rdi), %xmm0
-; FALLBACK5-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK5-NEXT:    movzbl (%rsi), %eax
-; FALLBACK5-NEXT:    movl %eax, %ecx
-; FALLBACK5-NEXT:    shlb $5, %cl
-; FALLBACK5-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK5-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    andb $6, %al
-; FALLBACK5-NEXT:    movzbl %al, %eax
-; FALLBACK5-NEXT:    movq -48(%rsp,%rax,4), %rsi
-; FALLBACK5-NEXT:    movq -56(%rsp,%rax,4), %rdi
-; FALLBACK5-NEXT:    movq %rdi, %r8
-; FALLBACK5-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK5-NEXT:    movq -72(%rsp,%rax,4), %r9
-; FALLBACK5-NEXT:    movq -64(%rsp,%rax,4), %rax
-; FALLBACK5-NEXT:    movq %rax, %r10
-; FALLBACK5-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK5-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK5-NEXT:    shrq %cl, %rsi
-; FALLBACK5-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK5-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK5-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK5-NEXT:    movq %r9, (%rdx)
-; FALLBACK5-NEXT:    retq
-;
-; FALLBACK6-LABEL: lshr_32bytes_dwordOff:
-; FALLBACK6:       # %bb.0:
-; FALLBACK6-NEXT:    movups (%rdi), %xmm0
-; FALLBACK6-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK6-NEXT:    movzbl (%rsi), %ecx
-; FALLBACK6-NEXT:    movl %ecx, %eax
-; FALLBACK6-NEXT:    shlb $5, %al
-; FALLBACK6-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK6-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movl %eax, %esi
-; FALLBACK6-NEXT:    andb $6, %cl
-; FALLBACK6-NEXT:    movzbl %cl, %ecx
-; FALLBACK6-NEXT:    shrxq %rsi, -72(%rsp,%rcx,4), %rdi
-; FALLBACK6-NEXT:    notb %al
-; FALLBACK6-NEXT:    movq -64(%rsp,%rcx,4), %r8
-; FALLBACK6-NEXT:    movq -56(%rsp,%rcx,4), %r9
-; FALLBACK6-NEXT:    leaq (%r8,%r8), %r10
-; FALLBACK6-NEXT:    shlxq %rax, %r10, %r10
-; FALLBACK6-NEXT:    orq %rdi, %r10
-; FALLBACK6-NEXT:    shrxq %rsi, %r9, %rdi
-; FALLBACK6-NEXT:    movq -48(%rsp,%rcx,4), %rcx
-; FALLBACK6-NEXT:    leaq (%rcx,%rcx), %r11
-; FALLBACK6-NEXT:    shlxq %rax, %r11, %r11
-; FALLBACK6-NEXT:    orq %rdi, %r11
-; FALLBACK6-NEXT:    shrxq %rsi, %r8, %rdi
-; FALLBACK6-NEXT:    addq %r9, %r9
-; FALLBACK6-NEXT:    shlxq %rax, %r9, %rax
-; FALLBACK6-NEXT:    orq %rdi, %rax
-; FALLBACK6-NEXT:    shrxq %rsi, %rcx, %rcx
-; FALLBACK6-NEXT:    movq %rcx, 24(%rdx)
-; FALLBACK6-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK6-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK6-NEXT:    movq %r10, (%rdx)
-; FALLBACK6-NEXT:    retq
-;
-; FALLBACK7-LABEL: lshr_32bytes_dwordOff:
-; FALLBACK7:       # %bb.0:
-; FALLBACK7-NEXT:    movups (%rdi), %xmm0
-; FALLBACK7-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK7-NEXT:    movzbl (%rsi), %eax
-; FALLBACK7-NEXT:    movl %eax, %ecx
-; FALLBACK7-NEXT:    shlb $5, %cl
-; FALLBACK7-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK7-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    andb $6, %al
-; FALLBACK7-NEXT:    movzbl %al, %eax
-; FALLBACK7-NEXT:    movq -48(%rsp,%rax,4), %rsi
-; FALLBACK7-NEXT:    movq -56(%rsp,%rax,4), %rdi
-; FALLBACK7-NEXT:    movq %rdi, %r8
-; FALLBACK7-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK7-NEXT:    movq -72(%rsp,%rax,4), %r9
-; FALLBACK7-NEXT:    movq -64(%rsp,%rax,4), %rax
-; FALLBACK7-NEXT:    movq %rax, %r10
-; FALLBACK7-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK7-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK7-NEXT:    shrxq %rcx, %rsi, %rax
-; FALLBACK7-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK7-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK7-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK7-NEXT:    movq %r9, (%rdx)
-; FALLBACK7-NEXT:    retq
-;
-; FALLBACK8-LABEL: lshr_32bytes_dwordOff:
-; FALLBACK8:       # %bb.0:
-; FALLBACK8-NEXT:    pushq %rbx
-; FALLBACK8-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK8-NEXT:    movzbl (%rsi), %ecx
-; FALLBACK8-NEXT:    movl %ecx, %eax
-; FALLBACK8-NEXT:    shlb $5, %al
-; FALLBACK8-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK8-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    andb $6, %cl
-; FALLBACK8-NEXT:    movzbl %cl, %r9d
-; FALLBACK8-NEXT:    movq -64(%rsp,%r9,4), %r10
-; FALLBACK8-NEXT:    movq -56(%rsp,%r9,4), %r8
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r10
-; FALLBACK8-NEXT:    movl %eax, %esi
-; FALLBACK8-NEXT:    notb %sil
-; FALLBACK8-NEXT:    leaq (%r8,%r8), %rdi
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %rdi
-; FALLBACK8-NEXT:    orq %r10, %rdi
-; FALLBACK8-NEXT:    movq -48(%rsp,%r9,4), %r10
-; FALLBACK8-NEXT:    movq %r10, %r11
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r11
-; FALLBACK8-NEXT:    movq -40(%rsp,%r9,4), %r9
-; FALLBACK8-NEXT:    leaq (%r9,%r9), %rbx
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %rbx
-; FALLBACK8-NEXT:    orq %r11, %rbx
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r8
-; FALLBACK8-NEXT:    addq %r10, %r10
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r10
-; FALLBACK8-NEXT:    orq %r8, %r10
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r9
-; FALLBACK8-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK8-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK8-NEXT:    movq %rbx, 16(%rdx)
-; FALLBACK8-NEXT:    movq %rdi, (%rdx)
-; FALLBACK8-NEXT:    popq %rbx
-; FALLBACK8-NEXT:    vzeroupper
-; FALLBACK8-NEXT:    retq
-;
-; FALLBACK9-LABEL: lshr_32bytes_dwordOff:
-; FALLBACK9:       # %bb.0:
-; FALLBACK9-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK9-NEXT:    movzbl (%rsi), %eax
-; FALLBACK9-NEXT:    movl %eax, %ecx
-; FALLBACK9-NEXT:    shlb $5, %cl
-; FALLBACK9-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK9-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    andb $6, %al
-; FALLBACK9-NEXT:    movzbl %al, %eax
-; FALLBACK9-NEXT:    movq -48(%rsp,%rax,4), %rsi
-; FALLBACK9-NEXT:    movq -56(%rsp,%rax,4), %rdi
-; FALLBACK9-NEXT:    movq %rdi, %r8
-; FALLBACK9-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK9-NEXT:    movq -72(%rsp,%rax,4), %r9
-; FALLBACK9-NEXT:    movq -64(%rsp,%rax,4), %rax
-; FALLBACK9-NEXT:    movq %rax, %r10
-; FALLBACK9-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK9-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK9-NEXT:    shrq %cl, %rsi
-; FALLBACK9-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK9-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK9-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK9-NEXT:    movq %r9, (%rdx)
-; FALLBACK9-NEXT:    vzeroupper
-; FALLBACK9-NEXT:    retq
-;
-; FALLBACK10-LABEL: lshr_32bytes_dwordOff:
-; FALLBACK10:       # %bb.0:
-; FALLBACK10-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK10-NEXT:    movzbl (%rsi), %eax
-; FALLBACK10-NEXT:    movl %eax, %ecx
-; FALLBACK10-NEXT:    shlb $5, %cl
-; FALLBACK10-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK10-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movl %ecx, %esi
-; FALLBACK10-NEXT:    andb $6, %al
-; FALLBACK10-NEXT:    movzbl %al, %eax
-; FALLBACK10-NEXT:    shrxq %rsi, -72(%rsp,%rax,4), %rdi
-; FALLBACK10-NEXT:    notb %cl
-; FALLBACK10-NEXT:    movq -64(%rsp,%rax,4), %r8
-; FALLBACK10-NEXT:    movq -56(%rsp,%rax,4), %r9
-; FALLBACK10-NEXT:    leaq (%r8,%r8), %r10
-; FALLBACK10-NEXT:    shlxq %rcx, %r10, %r10
-; FALLBACK10-NEXT:    orq %rdi, %r10
-; FALLBACK10-NEXT:    shrxq %rsi, %r9, %rdi
-; FALLBACK10-NEXT:    movq -48(%rsp,%rax,4), %rax
-; FALLBACK10-NEXT:    leaq (%rax,%rax), %r11
-; FALLBACK10-NEXT:    shlxq %rcx, %r11, %r11
-; FALLBACK10-NEXT:    orq %rdi, %r11
-; FALLBACK10-NEXT:    shrxq %rsi, %r8, %rdi
-; FALLBACK10-NEXT:    addq %r9, %r9
-; FALLBACK10-NEXT:    shlxq %rcx, %r9, %rcx
-; FALLBACK10-NEXT:    orq %rdi, %rcx
-; FALLBACK10-NEXT:    shrxq %rsi, %rax, %rax
-; FALLBACK10-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK10-NEXT:    movq %rcx, 8(%rdx)
-; FALLBACK10-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK10-NEXT:    movq %r10, (%rdx)
-; FALLBACK10-NEXT:    vzeroupper
-; FALLBACK10-NEXT:    retq
-;
-; FALLBACK11-LABEL: lshr_32bytes_dwordOff:
-; FALLBACK11:       # %bb.0:
-; FALLBACK11-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK11-NEXT:    movzbl (%rsi), %eax
-; FALLBACK11-NEXT:    movl %eax, %ecx
-; FALLBACK11-NEXT:    shlb $5, %cl
-; FALLBACK11-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK11-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    andb $6, %al
-; FALLBACK11-NEXT:    movzbl %al, %eax
-; FALLBACK11-NEXT:    movq -48(%rsp,%rax,4), %rsi
-; FALLBACK11-NEXT:    movq -56(%rsp,%rax,4), %rdi
-; FALLBACK11-NEXT:    movq %rdi, %r8
-; FALLBACK11-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK11-NEXT:    movq -72(%rsp,%rax,4), %r9
-; FALLBACK11-NEXT:    movq -64(%rsp,%rax,4), %rax
-; FALLBACK11-NEXT:    movq %rax, %r10
-; FALLBACK11-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK11-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK11-NEXT:    shrxq %rcx, %rsi, %rax
-; FALLBACK11-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK11-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK11-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK11-NEXT:    movq %r9, (%rdx)
-; FALLBACK11-NEXT:    vzeroupper
-; FALLBACK11-NEXT:    retq
-;
-; FALLBACK12-LABEL: lshr_32bytes_dwordOff:
-; FALLBACK12:       # %bb.0:
-; FALLBACK12-NEXT:    pushq %rbx
-; FALLBACK12-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK12-NEXT:    movzbl (%rsi), %ecx
-; FALLBACK12-NEXT:    movl %ecx, %eax
-; FALLBACK12-NEXT:    shlb $5, %al
-; FALLBACK12-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK12-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    andb $6, %cl
-; FALLBACK12-NEXT:    movzbl %cl, %r9d
-; FALLBACK12-NEXT:    movq -64(%rsp,%r9,4), %r10
-; FALLBACK12-NEXT:    movq -56(%rsp,%r9,4), %r8
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r10
-; FALLBACK12-NEXT:    movl %eax, %esi
-; FALLBACK12-NEXT:    notb %sil
-; FALLBACK12-NEXT:    leaq (%r8,%r8), %rdi
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %rdi
-; FALLBACK12-NEXT:    orq %r10, %rdi
-; FALLBACK12-NEXT:    movq -48(%rsp,%r9,4), %r10
-; FALLBACK12-NEXT:    movq %r10, %r11
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r11
-; FALLBACK12-NEXT:    movq -40(%rsp,%r9,4), %r9
-; FALLBACK12-NEXT:    leaq (%r9,%r9), %rbx
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %rbx
-; FALLBACK12-NEXT:    orq %r11, %rbx
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r8
-; FALLBACK12-NEXT:    addq %r10, %r10
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r10
-; FALLBACK12-NEXT:    orq %r8, %r10
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r9
-; FALLBACK12-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK12-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK12-NEXT:    movq %rbx, 16(%rdx)
-; FALLBACK12-NEXT:    movq %rdi, (%rdx)
-; FALLBACK12-NEXT:    popq %rbx
-; FALLBACK12-NEXT:    vzeroupper
-; FALLBACK12-NEXT:    retq
-;
-; FALLBACK13-LABEL: lshr_32bytes_dwordOff:
-; FALLBACK13:       # %bb.0:
-; FALLBACK13-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK13-NEXT:    movzbl (%rsi), %eax
-; FALLBACK13-NEXT:    movl %eax, %ecx
-; FALLBACK13-NEXT:    shlb $5, %cl
-; FALLBACK13-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK13-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    andb $6, %al
-; FALLBACK13-NEXT:    movzbl %al, %eax
-; FALLBACK13-NEXT:    movq -48(%rsp,%rax,4), %rsi
-; FALLBACK13-NEXT:    movq -56(%rsp,%rax,4), %rdi
-; FALLBACK13-NEXT:    movq %rdi, %r8
-; FALLBACK13-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK13-NEXT:    movq -72(%rsp,%rax,4), %r9
-; FALLBACK13-NEXT:    movq -64(%rsp,%rax,4), %rax
-; FALLBACK13-NEXT:    movq %rax, %r10
-; FALLBACK13-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK13-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK13-NEXT:    shrq %cl, %rsi
-; FALLBACK13-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK13-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK13-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK13-NEXT:    movq %r9, (%rdx)
-; FALLBACK13-NEXT:    vzeroupper
-; FALLBACK13-NEXT:    retq
-;
-; FALLBACK14-LABEL: lshr_32bytes_dwordOff:
-; FALLBACK14:       # %bb.0:
-; FALLBACK14-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK14-NEXT:    movzbl (%rsi), %eax
-; FALLBACK14-NEXT:    movl %eax, %ecx
-; FALLBACK14-NEXT:    shlb $5, %cl
-; FALLBACK14-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK14-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movl %ecx, %esi
-; FALLBACK14-NEXT:    andb $6, %al
-; FALLBACK14-NEXT:    movzbl %al, %eax
-; FALLBACK14-NEXT:    shrxq %rsi, -72(%rsp,%rax,4), %rdi
-; FALLBACK14-NEXT:    notb %cl
-; FALLBACK14-NEXT:    movq -64(%rsp,%rax,4), %r8
-; FALLBACK14-NEXT:    movq -56(%rsp,%rax,4), %r9
-; FALLBACK14-NEXT:    leaq (%r8,%r8), %r10
-; FALLBACK14-NEXT:    shlxq %rcx, %r10, %r10
-; FALLBACK14-NEXT:    orq %rdi, %r10
-; FALLBACK14-NEXT:    shrxq %rsi, %r9, %rdi
-; FALLBACK14-NEXT:    movq -48(%rsp,%rax,4), %rax
-; FALLBACK14-NEXT:    leaq (%rax,%rax), %r11
-; FALLBACK14-NEXT:    shlxq %rcx, %r11, %r11
-; FALLBACK14-NEXT:    orq %rdi, %r11
-; FALLBACK14-NEXT:    shrxq %rsi, %r8, %rdi
-; FALLBACK14-NEXT:    addq %r9, %r9
-; FALLBACK14-NEXT:    shlxq %rcx, %r9, %rcx
-; FALLBACK14-NEXT:    orq %rdi, %rcx
-; FALLBACK14-NEXT:    shrxq %rsi, %rax, %rax
-; FALLBACK14-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK14-NEXT:    movq %rcx, 8(%rdx)
-; FALLBACK14-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK14-NEXT:    movq %r10, (%rdx)
-; FALLBACK14-NEXT:    vzeroupper
-; FALLBACK14-NEXT:    retq
-;
-; FALLBACK15-LABEL: lshr_32bytes_dwordOff:
-; FALLBACK15:       # %bb.0:
-; FALLBACK15-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK15-NEXT:    movzbl (%rsi), %eax
-; FALLBACK15-NEXT:    movl %eax, %ecx
-; FALLBACK15-NEXT:    shlb $5, %cl
-; FALLBACK15-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK15-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    andb $6, %al
-; FALLBACK15-NEXT:    movzbl %al, %eax
-; FALLBACK15-NEXT:    movq -48(%rsp,%rax,4), %rsi
-; FALLBACK15-NEXT:    movq -56(%rsp,%rax,4), %rdi
-; FALLBACK15-NEXT:    movq %rdi, %r8
-; FALLBACK15-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK15-NEXT:    movq -72(%rsp,%rax,4), %r9
-; FALLBACK15-NEXT:    movq -64(%rsp,%rax,4), %rax
-; FALLBACK15-NEXT:    movq %rax, %r10
-; FALLBACK15-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK15-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK15-NEXT:    shrxq %rcx, %rsi, %rax
-; FALLBACK15-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK15-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK15-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK15-NEXT:    movq %r9, (%rdx)
-; FALLBACK15-NEXT:    vzeroupper
-; FALLBACK15-NEXT:    retq
+; X64-NO-SHLD-NO-BMI2-SSE2-LABEL: lshr_32bytes_dwordOff:
+; X64-NO-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    andb $6, %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movzbl %sil, %r9d
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -64(%rsp,%r9,4), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -56(%rsp,%r9,4), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -48(%rsp,%r9,4), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leaq (%rbx,%rbx), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r11, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    addq %rdi, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -40(%rsp,%r9,4), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leaq (%r9,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %rbx, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE2-LABEL: lshr_32bytes_dwordOff:
+; X64-HAVE-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andb $6, %sil
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movzbl %sil, %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -56(%rsp,%rax,4), %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -72(%rsp,%rax,4), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -64(%rsp,%rax,4), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %rsi, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -48(%rsp,%rax,4), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %rax, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %r8, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE2-LABEL: lshr_32bytes_dwordOff:
+; X64-NO-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $5, %al
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $6, %sil
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl %sil, %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -64(%rsp,%rsi,4), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -56(%rsp,%rsi,4), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rcx, %rdi, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leaq (%r8,%r8), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rax, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r9, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rcx, -72(%rsp,%rsi,4), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addq %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rax, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r9, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rcx, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -48(%rsp,%rsi,4), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leaq (%rsi,%rsi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rax, %r9, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r8, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rcx, %rsi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r10, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: lshr_32bytes_dwordOff:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $6, %sil
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl %sil, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -56(%rsp,%rax,4), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -72(%rsp,%rax,4), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -64(%rsp,%rax,4), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %rsi, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -48(%rsp,%rax,4), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %rax, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %r8, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rcx, %rax, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-SSE4-LABEL: lshr_32bytes_dwordOff:
+; X64-NO-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%rsi), %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    andb $6, %cl
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl %cl, %r9d
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -64(%rsp,%r9,4), %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -56(%rsp,%r9,4), %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leaq (%r8,%r8), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -48(%rsp,%r9,4), %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -40(%rsp,%r9,4), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leaq (%r9,%r9), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    addq %r10, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r8, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rbx, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE4-LABEL: lshr_32bytes_dwordOff:
+; X64-HAVE-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%rsi), %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andb $6, %al
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl %al, %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -48(%rsp,%rax,4), %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -56(%rsp,%rax,4), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %rsi, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -72(%rsp,%rax,4), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -64(%rsp,%rax,4), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rax, %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %rdi, %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %rax, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE4-LABEL: lshr_32bytes_dwordOff:
+; X64-NO-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%rsi), %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $5, %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $6, %cl
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl %cl, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, -72(%rsp,%rcx,4), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -64(%rsp,%rcx,4), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -56(%rsp,%rcx,4), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leaq (%r8,%r8), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rax, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rdi, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %r9, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -48(%rsp,%rcx,4), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leaq (%rcx,%rcx), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rax, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rdi, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %r8, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addq %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rax, %r9, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rdi, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %rcx, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rcx, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: lshr_32bytes_dwordOff:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $6, %al
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl %al, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -48(%rsp,%rax,4), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -56(%rsp,%rax,4), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %rsi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -72(%rsp,%rax,4), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -64(%rsp,%rax,4), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %rdi, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %rax, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rcx, %rsi, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r10, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-AVX-LABEL: lshr_32bytes_dwordOff:
+; X64-NO-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%rsi), %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    andb $6, %cl
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl %cl, %r9d
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -64(%rsp,%r9,4), %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -56(%rsp,%r9,4), %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    leaq (%r8,%r8), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -48(%rsp,%r9,4), %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -40(%rsp,%r9,4), %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    leaq (%r9,%r9), %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    addq %r10, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r8, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rbx, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vzeroupper
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-AVX-LABEL: lshr_32bytes_dwordOff:
+; X64-HAVE-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%rsi), %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    andb $6, %al
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl %al, %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -48(%rsp,%rax,4), %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -56(%rsp,%rax,4), %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %rsi, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -72(%rsp,%rax,4), %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -64(%rsp,%rax,4), %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rax, %r10
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %rdi, %r10
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %rax, %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vzeroupper
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-AVX-LABEL: lshr_32bytes_dwordOff:
+; X64-NO-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%rsi), %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $5, %cl
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    andb $6, %al
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl %al, %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rsi, -72(%rsp,%rax,4), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    notb %cl
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -64(%rsp,%rax,4), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -56(%rsp,%rax,4), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leaq (%r8,%r8), %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %rdi, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rsi, %r9, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -48(%rsp,%rax,4), %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leaq (%rax,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %rdi, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rsi, %r8, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addq %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %r9, %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %rdi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rsi, %rax, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rcx, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vzeroupper
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-LABEL: lshr_32bytes_dwordOff:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    andb $6, %al
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl %al, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -48(%rsp,%rax,4), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -56(%rsp,%rax,4), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %rsi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -72(%rsp,%rax,4), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -64(%rsp,%rax,4), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %rdi, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %rax, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rcx, %rsi, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r10, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vzeroupper
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    retq
 ;
 ; X86-SSE2-LABEL: lshr_32bytes_dwordOff:
 ; X86-SSE2:       # %bb.0:
@@ -5913,1953 +4936,1495 @@ define void @lshr_32bytes_qwordOff(ptr %src.ptr, ptr %qwordOff.ptr, ptr %dst) no
 }
 
 define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
-; FALLBACK0-LABEL: shl_32bytes:
-; FALLBACK0:       # %bb.0:
-; FALLBACK0-NEXT:    pushq %rbx
-; FALLBACK0-NEXT:    movq (%rdi), %rcx
-; FALLBACK0-NEXT:    movq 8(%rdi), %r8
-; FALLBACK0-NEXT:    movq 16(%rdi), %r9
-; FALLBACK0-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK0-NEXT:    movzbl (%rsi), %esi
-; FALLBACK0-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK0-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK0-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    andb $24, %sil
-; FALLBACK0-NEXT:    negb %sil
-; FALLBACK0-NEXT:    movsbq %sil, %r10
-; FALLBACK0-NEXT:    movq -32(%rsp,%r10), %r8
-; FALLBACK0-NEXT:    movq -24(%rsp,%r10), %rdi
-; FALLBACK0-NEXT:    movq %rdi, %r11
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r11
-; FALLBACK0-NEXT:    movl %eax, %esi
-; FALLBACK0-NEXT:    notb %sil
-; FALLBACK0-NEXT:    movq %r8, %r9
-; FALLBACK0-NEXT:    shrq %r9
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r9
-; FALLBACK0-NEXT:    orq %r11, %r9
-; FALLBACK0-NEXT:    movq -8(%rsp,%r10), %r11
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r11
-; FALLBACK0-NEXT:    movq -16(%rsp,%r10), %r10
-; FALLBACK0-NEXT:    movq %r10, %rbx
-; FALLBACK0-NEXT:    shrq %rbx
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %rbx
-; FALLBACK0-NEXT:    orq %r11, %rbx
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r10
-; FALLBACK0-NEXT:    shrq %rdi
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %rdi
-; FALLBACK0-NEXT:    orq %r10, %rdi
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r8
-; FALLBACK0-NEXT:    movq %r8, (%rdx)
-; FALLBACK0-NEXT:    movq %rdi, 16(%rdx)
-; FALLBACK0-NEXT:    movq %rbx, 24(%rdx)
-; FALLBACK0-NEXT:    movq %r9, 8(%rdx)
-; FALLBACK0-NEXT:    popq %rbx
-; FALLBACK0-NEXT:    retq
-;
-; FALLBACK1-LABEL: shl_32bytes:
-; FALLBACK1:       # %bb.0:
-; FALLBACK1-NEXT:    movq (%rdi), %rax
-; FALLBACK1-NEXT:    movq 8(%rdi), %r8
-; FALLBACK1-NEXT:    movq 16(%rdi), %r9
-; FALLBACK1-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK1-NEXT:    movzbl (%rsi), %esi
-; FALLBACK1-NEXT:    leal (,%rsi,8), %ecx
-; FALLBACK1-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK1-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    andb $24, %sil
-; FALLBACK1-NEXT:    negb %sil
-; FALLBACK1-NEXT:    movsbq %sil, %rax
-; FALLBACK1-NEXT:    movq -24(%rsp,%rax), %rsi
-; FALLBACK1-NEXT:    movq -16(%rsp,%rax), %rdi
-; FALLBACK1-NEXT:    shldq %cl, %rsi, %rdi
-; FALLBACK1-NEXT:    movq -40(%rsp,%rax), %r8
-; FALLBACK1-NEXT:    movq -32(%rsp,%rax), %rax
-; FALLBACK1-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK1-NEXT:    shldq %cl, %r8, %rax
-; FALLBACK1-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK1-NEXT:    shlq %cl, %r8
-; FALLBACK1-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK1-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK1-NEXT:    movq %r8, (%rdx)
-; FALLBACK1-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK1-NEXT:    retq
-;
-; FALLBACK2-LABEL: shl_32bytes:
-; FALLBACK2:       # %bb.0:
-; FALLBACK2-NEXT:    movq (%rdi), %rcx
-; FALLBACK2-NEXT:    movq 8(%rdi), %r8
-; FALLBACK2-NEXT:    movq 16(%rdi), %r9
-; FALLBACK2-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK2-NEXT:    movzbl (%rsi), %esi
-; FALLBACK2-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK2-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movl %eax, %ecx
-; FALLBACK2-NEXT:    andb $24, %sil
-; FALLBACK2-NEXT:    negb %sil
-; FALLBACK2-NEXT:    movsbq %sil, %rdi
-; FALLBACK2-NEXT:    movq -40(%rsp,%rdi), %r8
-; FALLBACK2-NEXT:    movq -32(%rsp,%rdi), %rsi
-; FALLBACK2-NEXT:    shlxq %rcx, %rsi, %r9
-; FALLBACK2-NEXT:    notb %al
-; FALLBACK2-NEXT:    shlxq %rcx, %r8, %r10
-; FALLBACK2-NEXT:    shrq %r8
-; FALLBACK2-NEXT:    shrxq %rax, %r8, %r8
-; FALLBACK2-NEXT:    orq %r9, %r8
-; FALLBACK2-NEXT:    shlxq %rcx, -16(%rsp,%rdi), %r9
-; FALLBACK2-NEXT:    movq -24(%rsp,%rdi), %rdi
-; FALLBACK2-NEXT:    shlxq %rcx, %rdi, %rcx
-; FALLBACK2-NEXT:    shrq %rdi
-; FALLBACK2-NEXT:    shrxq %rax, %rdi, %rdi
-; FALLBACK2-NEXT:    orq %r9, %rdi
-; FALLBACK2-NEXT:    shrq %rsi
-; FALLBACK2-NEXT:    shrxq %rax, %rsi, %rax
-; FALLBACK2-NEXT:    orq %rcx, %rax
-; FALLBACK2-NEXT:    movq %r10, (%rdx)
-; FALLBACK2-NEXT:    movq %rax, 16(%rdx)
-; FALLBACK2-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK2-NEXT:    movq %r8, 8(%rdx)
-; FALLBACK2-NEXT:    retq
-;
-; FALLBACK3-LABEL: shl_32bytes:
-; FALLBACK3:       # %bb.0:
-; FALLBACK3-NEXT:    movq (%rdi), %rax
-; FALLBACK3-NEXT:    movq 8(%rdi), %r8
-; FALLBACK3-NEXT:    movq 16(%rdi), %r9
-; FALLBACK3-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK3-NEXT:    movzbl (%rsi), %esi
-; FALLBACK3-NEXT:    leal (,%rsi,8), %ecx
-; FALLBACK3-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    andb $24, %sil
-; FALLBACK3-NEXT:    negb %sil
-; FALLBACK3-NEXT:    movsbq %sil, %rax
-; FALLBACK3-NEXT:    movq -24(%rsp,%rax), %rsi
-; FALLBACK3-NEXT:    movq -16(%rsp,%rax), %rdi
-; FALLBACK3-NEXT:    shldq %cl, %rsi, %rdi
-; FALLBACK3-NEXT:    movq -40(%rsp,%rax), %r8
-; FALLBACK3-NEXT:    movq -32(%rsp,%rax), %rax
-; FALLBACK3-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK3-NEXT:    shldq %cl, %r8, %rax
-; FALLBACK3-NEXT:    shlxq %rcx, %r8, %rcx
-; FALLBACK3-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK3-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK3-NEXT:    movq %rcx, (%rdx)
-; FALLBACK3-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK3-NEXT:    retq
-;
-; FALLBACK4-LABEL: shl_32bytes:
-; FALLBACK4:       # %bb.0:
-; FALLBACK4-NEXT:    movups (%rdi), %xmm0
-; FALLBACK4-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK4-NEXT:    movzbl (%rsi), %ecx
-; FALLBACK4-NEXT:    leal (,%rcx,8), %eax
-; FALLBACK4-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    andb $24, %cl
-; FALLBACK4-NEXT:    negb %cl
-; FALLBACK4-NEXT:    movsbq %cl, %r8
-; FALLBACK4-NEXT:    movq -16(%rsp,%r8), %r9
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r9
-; FALLBACK4-NEXT:    movl %eax, %esi
-; FALLBACK4-NEXT:    notb %sil
-; FALLBACK4-NEXT:    movq -24(%rsp,%r8), %r10
-; FALLBACK4-NEXT:    movq %r10, %rdi
-; FALLBACK4-NEXT:    shrq %rdi
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %rdi
-; FALLBACK4-NEXT:    orq %r9, %rdi
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r10
-; FALLBACK4-NEXT:    movq -40(%rsp,%r8), %r9
-; FALLBACK4-NEXT:    movq -32(%rsp,%r8), %r8
-; FALLBACK4-NEXT:    movq %r8, %r11
-; FALLBACK4-NEXT:    shrq %r11
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r11
-; FALLBACK4-NEXT:    orq %r10, %r11
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r8
-; FALLBACK4-NEXT:    movq %r9, %r10
-; FALLBACK4-NEXT:    shrq %r10
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r10
-; FALLBACK4-NEXT:    orq %r8, %r10
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r9
-; FALLBACK4-NEXT:    movq %r9, (%rdx)
-; FALLBACK4-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK4-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK4-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK4-NEXT:    retq
-;
-; FALLBACK5-LABEL: shl_32bytes:
-; FALLBACK5:       # %bb.0:
-; FALLBACK5-NEXT:    movups (%rdi), %xmm0
-; FALLBACK5-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK5-NEXT:    movzbl (%rsi), %eax
-; FALLBACK5-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK5-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK5-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    andb $24, %al
-; FALLBACK5-NEXT:    negb %al
-; FALLBACK5-NEXT:    movsbq %al, %rax
-; FALLBACK5-NEXT:    movq -24(%rsp,%rax), %rsi
-; FALLBACK5-NEXT:    movq -16(%rsp,%rax), %rdi
-; FALLBACK5-NEXT:    shldq %cl, %rsi, %rdi
-; FALLBACK5-NEXT:    movq -40(%rsp,%rax), %r8
-; FALLBACK5-NEXT:    movq -32(%rsp,%rax), %rax
-; FALLBACK5-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK5-NEXT:    movq %r8, %r9
-; FALLBACK5-NEXT:    shlq %cl, %r9
-; FALLBACK5-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK5-NEXT:    shldq %cl, %r8, %rax
-; FALLBACK5-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK5-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK5-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK5-NEXT:    movq %r9, (%rdx)
-; FALLBACK5-NEXT:    retq
-;
-; FALLBACK6-LABEL: shl_32bytes:
-; FALLBACK6:       # %bb.0:
-; FALLBACK6-NEXT:    movups (%rdi), %xmm0
-; FALLBACK6-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK6-NEXT:    movzbl (%rsi), %esi
-; FALLBACK6-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK6-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK6-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movl %eax, %ecx
-; FALLBACK6-NEXT:    andb $24, %sil
-; FALLBACK6-NEXT:    negb %sil
-; FALLBACK6-NEXT:    movsbq %sil, %rsi
-; FALLBACK6-NEXT:    shlxq %rcx, -16(%rsp,%rsi), %rdi
-; FALLBACK6-NEXT:    notb %al
-; FALLBACK6-NEXT:    movq -24(%rsp,%rsi), %r8
-; FALLBACK6-NEXT:    shlxq %rcx, %r8, %r9
-; FALLBACK6-NEXT:    shrq %r8
-; FALLBACK6-NEXT:    shrxq %rax, %r8, %r8
-; FALLBACK6-NEXT:    orq %rdi, %r8
-; FALLBACK6-NEXT:    movq -40(%rsp,%rsi), %rdi
-; FALLBACK6-NEXT:    movq -32(%rsp,%rsi), %rsi
-; FALLBACK6-NEXT:    shlxq %rcx, %rsi, %r10
-; FALLBACK6-NEXT:    shrq %rsi
-; FALLBACK6-NEXT:    shrxq %rax, %rsi, %rsi
-; FALLBACK6-NEXT:    orq %r9, %rsi
-; FALLBACK6-NEXT:    shlxq %rcx, %rdi, %rcx
-; FALLBACK6-NEXT:    shrq %rdi
-; FALLBACK6-NEXT:    shrxq %rax, %rdi, %rax
-; FALLBACK6-NEXT:    orq %r10, %rax
-; FALLBACK6-NEXT:    movq %rcx, (%rdx)
-; FALLBACK6-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK6-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK6-NEXT:    movq %r8, 24(%rdx)
-; FALLBACK6-NEXT:    retq
-;
-; FALLBACK7-LABEL: shl_32bytes:
-; FALLBACK7:       # %bb.0:
-; FALLBACK7-NEXT:    movups (%rdi), %xmm0
-; FALLBACK7-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK7-NEXT:    movzbl (%rsi), %eax
-; FALLBACK7-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK7-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK7-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    andb $24, %al
-; FALLBACK7-NEXT:    negb %al
-; FALLBACK7-NEXT:    movsbq %al, %rax
-; FALLBACK7-NEXT:    movq -24(%rsp,%rax), %rsi
-; FALLBACK7-NEXT:    movq -16(%rsp,%rax), %rdi
-; FALLBACK7-NEXT:    shldq %cl, %rsi, %rdi
-; FALLBACK7-NEXT:    movq -40(%rsp,%rax), %r8
-; FALLBACK7-NEXT:    movq -32(%rsp,%rax), %rax
-; FALLBACK7-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK7-NEXT:    shlxq %rcx, %r8, %r9
-; FALLBACK7-NEXT:    # kill: def $cl killed $cl killed $rcx
-; FALLBACK7-NEXT:    shldq %cl, %r8, %rax
-; FALLBACK7-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK7-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK7-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK7-NEXT:    movq %r9, (%rdx)
-; FALLBACK7-NEXT:    retq
-;
-; FALLBACK8-LABEL: shl_32bytes:
-; FALLBACK8:       # %bb.0:
-; FALLBACK8-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK8-NEXT:    movzbl (%rsi), %ecx
-; FALLBACK8-NEXT:    leal (,%rcx,8), %eax
-; FALLBACK8-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK8-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    andb $24, %cl
-; FALLBACK8-NEXT:    negb %cl
-; FALLBACK8-NEXT:    movsbq %cl, %r8
-; FALLBACK8-NEXT:    movq -16(%rsp,%r8), %r9
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r9
-; FALLBACK8-NEXT:    movl %eax, %esi
-; FALLBACK8-NEXT:    notb %sil
-; FALLBACK8-NEXT:    movq -24(%rsp,%r8), %r10
-; FALLBACK8-NEXT:    movq %r10, %rdi
-; FALLBACK8-NEXT:    shrq %rdi
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %rdi
-; FALLBACK8-NEXT:    orq %r9, %rdi
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r10
-; FALLBACK8-NEXT:    movq -40(%rsp,%r8), %r9
-; FALLBACK8-NEXT:    movq -32(%rsp,%r8), %r8
-; FALLBACK8-NEXT:    movq %r8, %r11
-; FALLBACK8-NEXT:    shrq %r11
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r11
-; FALLBACK8-NEXT:    orq %r10, %r11
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r8
-; FALLBACK8-NEXT:    movq %r9, %r10
-; FALLBACK8-NEXT:    shrq %r10
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r10
-; FALLBACK8-NEXT:    orq %r8, %r10
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r9
-; FALLBACK8-NEXT:    movq %r9, (%rdx)
-; FALLBACK8-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK8-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK8-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK8-NEXT:    vzeroupper
-; FALLBACK8-NEXT:    retq
-;
-; FALLBACK9-LABEL: shl_32bytes:
-; FALLBACK9:       # %bb.0:
-; FALLBACK9-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK9-NEXT:    movzbl (%rsi), %eax
-; FALLBACK9-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK9-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK9-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    andb $24, %al
-; FALLBACK9-NEXT:    negb %al
-; FALLBACK9-NEXT:    movsbq %al, %rax
-; FALLBACK9-NEXT:    movq -24(%rsp,%rax), %rsi
-; FALLBACK9-NEXT:    movq -16(%rsp,%rax), %rdi
-; FALLBACK9-NEXT:    shldq %cl, %rsi, %rdi
-; FALLBACK9-NEXT:    movq -40(%rsp,%rax), %r8
-; FALLBACK9-NEXT:    movq -32(%rsp,%rax), %rax
-; FALLBACK9-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK9-NEXT:    movq %r8, %r9
-; FALLBACK9-NEXT:    shlq %cl, %r9
-; FALLBACK9-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK9-NEXT:    shldq %cl, %r8, %rax
-; FALLBACK9-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK9-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK9-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK9-NEXT:    movq %r9, (%rdx)
-; FALLBACK9-NEXT:    vzeroupper
-; FALLBACK9-NEXT:    retq
-;
-; FALLBACK10-LABEL: shl_32bytes:
-; FALLBACK10:       # %bb.0:
-; FALLBACK10-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK10-NEXT:    movzbl (%rsi), %esi
-; FALLBACK10-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK10-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK10-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movl %eax, %ecx
-; FALLBACK10-NEXT:    andb $24, %sil
-; FALLBACK10-NEXT:    negb %sil
-; FALLBACK10-NEXT:    movsbq %sil, %rsi
-; FALLBACK10-NEXT:    shlxq %rcx, -16(%rsp,%rsi), %rdi
-; FALLBACK10-NEXT:    notb %al
-; FALLBACK10-NEXT:    movq -24(%rsp,%rsi), %r8
-; FALLBACK10-NEXT:    shlxq %rcx, %r8, %r9
-; FALLBACK10-NEXT:    shrq %r8
-; FALLBACK10-NEXT:    shrxq %rax, %r8, %r8
-; FALLBACK10-NEXT:    orq %rdi, %r8
-; FALLBACK10-NEXT:    movq -40(%rsp,%rsi), %rdi
-; FALLBACK10-NEXT:    movq -32(%rsp,%rsi), %rsi
-; FALLBACK10-NEXT:    shlxq %rcx, %rsi, %r10
-; FALLBACK10-NEXT:    shrq %rsi
-; FALLBACK10-NEXT:    shrxq %rax, %rsi, %rsi
-; FALLBACK10-NEXT:    orq %r9, %rsi
-; FALLBACK10-NEXT:    shlxq %rcx, %rdi, %rcx
-; FALLBACK10-NEXT:    shrq %rdi
-; FALLBACK10-NEXT:    shrxq %rax, %rdi, %rax
-; FALLBACK10-NEXT:    orq %r10, %rax
-; FALLBACK10-NEXT:    movq %rcx, (%rdx)
-; FALLBACK10-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK10-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK10-NEXT:    movq %r8, 24(%rdx)
-; FALLBACK10-NEXT:    vzeroupper
-; FALLBACK10-NEXT:    retq
-;
-; FALLBACK11-LABEL: shl_32bytes:
-; FALLBACK11:       # %bb.0:
-; FALLBACK11-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK11-NEXT:    movzbl (%rsi), %eax
-; FALLBACK11-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK11-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK11-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    andb $24, %al
-; FALLBACK11-NEXT:    negb %al
-; FALLBACK11-NEXT:    movsbq %al, %rax
-; FALLBACK11-NEXT:    movq -24(%rsp,%rax), %rsi
-; FALLBACK11-NEXT:    movq -16(%rsp,%rax), %rdi
-; FALLBACK11-NEXT:    shldq %cl, %rsi, %rdi
-; FALLBACK11-NEXT:    movq -40(%rsp,%rax), %r8
-; FALLBACK11-NEXT:    movq -32(%rsp,%rax), %rax
-; FALLBACK11-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK11-NEXT:    shlxq %rcx, %r8, %r9
-; FALLBACK11-NEXT:    # kill: def $cl killed $cl killed $rcx
-; FALLBACK11-NEXT:    shldq %cl, %r8, %rax
-; FALLBACK11-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK11-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK11-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK11-NEXT:    movq %r9, (%rdx)
-; FALLBACK11-NEXT:    vzeroupper
-; FALLBACK11-NEXT:    retq
-;
-; FALLBACK12-LABEL: shl_32bytes:
-; FALLBACK12:       # %bb.0:
-; FALLBACK12-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK12-NEXT:    movzbl (%rsi), %ecx
-; FALLBACK12-NEXT:    leal (,%rcx,8), %eax
-; FALLBACK12-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK12-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    andb $24, %cl
-; FALLBACK12-NEXT:    negb %cl
-; FALLBACK12-NEXT:    movsbq %cl, %r8
-; FALLBACK12-NEXT:    movq -16(%rsp,%r8), %r9
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r9
-; FALLBACK12-NEXT:    movl %eax, %esi
-; FALLBACK12-NEXT:    notb %sil
-; FALLBACK12-NEXT:    movq -24(%rsp,%r8), %r10
-; FALLBACK12-NEXT:    movq %r10, %rdi
-; FALLBACK12-NEXT:    shrq %rdi
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %rdi
-; FALLBACK12-NEXT:    orq %r9, %rdi
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r10
-; FALLBACK12-NEXT:    movq -40(%rsp,%r8), %r9
-; FALLBACK12-NEXT:    movq -32(%rsp,%r8), %r8
-; FALLBACK12-NEXT:    movq %r8, %r11
-; FALLBACK12-NEXT:    shrq %r11
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r11
-; FALLBACK12-NEXT:    orq %r10, %r11
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r8
-; FALLBACK12-NEXT:    movq %r9, %r10
-; FALLBACK12-NEXT:    shrq %r10
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r10
-; FALLBACK12-NEXT:    orq %r8, %r10
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r9
-; FALLBACK12-NEXT:    movq %r9, (%rdx)
-; FALLBACK12-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK12-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK12-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK12-NEXT:    vzeroupper
-; FALLBACK12-NEXT:    retq
-;
-; FALLBACK13-LABEL: shl_32bytes:
-; FALLBACK13:       # %bb.0:
-; FALLBACK13-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK13-NEXT:    movzbl (%rsi), %eax
-; FALLBACK13-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK13-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK13-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    andb $24, %al
-; FALLBACK13-NEXT:    negb %al
-; FALLBACK13-NEXT:    movsbq %al, %rax
-; FALLBACK13-NEXT:    movq -24(%rsp,%rax), %rsi
-; FALLBACK13-NEXT:    movq -16(%rsp,%rax), %rdi
-; FALLBACK13-NEXT:    shldq %cl, %rsi, %rdi
-; FALLBACK13-NEXT:    movq -40(%rsp,%rax), %r8
-; FALLBACK13-NEXT:    movq -32(%rsp,%rax), %rax
-; FALLBACK13-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK13-NEXT:    movq %r8, %r9
-; FALLBACK13-NEXT:    shlq %cl, %r9
-; FALLBACK13-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK13-NEXT:    shldq %cl, %r8, %rax
-; FALLBACK13-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK13-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK13-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK13-NEXT:    movq %r9, (%rdx)
-; FALLBACK13-NEXT:    vzeroupper
-; FALLBACK13-NEXT:    retq
-;
-; FALLBACK14-LABEL: shl_32bytes:
-; FALLBACK14:       # %bb.0:
-; FALLBACK14-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK14-NEXT:    movzbl (%rsi), %esi
-; FALLBACK14-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK14-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK14-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movl %eax, %ecx
-; FALLBACK14-NEXT:    andb $24, %sil
-; FALLBACK14-NEXT:    negb %sil
-; FALLBACK14-NEXT:    movsbq %sil, %rsi
-; FALLBACK14-NEXT:    shlxq %rcx, -16(%rsp,%rsi), %rdi
-; FALLBACK14-NEXT:    notb %al
-; FALLBACK14-NEXT:    movq -24(%rsp,%rsi), %r8
-; FALLBACK14-NEXT:    shlxq %rcx, %r8, %r9
-; FALLBACK14-NEXT:    shrq %r8
-; FALLBACK14-NEXT:    shrxq %rax, %r8, %r8
-; FALLBACK14-NEXT:    orq %rdi, %r8
-; FALLBACK14-NEXT:    movq -40(%rsp,%rsi), %rdi
-; FALLBACK14-NEXT:    movq -32(%rsp,%rsi), %rsi
-; FALLBACK14-NEXT:    shlxq %rcx, %rsi, %r10
-; FALLBACK14-NEXT:    shrq %rsi
-; FALLBACK14-NEXT:    shrxq %rax, %rsi, %rsi
-; FALLBACK14-NEXT:    orq %r9, %rsi
-; FALLBACK14-NEXT:    shlxq %rcx, %rdi, %rcx
-; FALLBACK14-NEXT:    shrq %rdi
-; FALLBACK14-NEXT:    shrxq %rax, %rdi, %rax
-; FALLBACK14-NEXT:    orq %r10, %rax
-; FALLBACK14-NEXT:    movq %rcx, (%rdx)
-; FALLBACK14-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK14-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK14-NEXT:    movq %r8, 24(%rdx)
-; FALLBACK14-NEXT:    vzeroupper
-; FALLBACK14-NEXT:    retq
-;
-; FALLBACK15-LABEL: shl_32bytes:
-; FALLBACK15:       # %bb.0:
-; FALLBACK15-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK15-NEXT:    movzbl (%rsi), %eax
-; FALLBACK15-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK15-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK15-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    andb $24, %al
-; FALLBACK15-NEXT:    negb %al
-; FALLBACK15-NEXT:    movsbq %al, %rax
-; FALLBACK15-NEXT:    movq -24(%rsp,%rax), %rsi
-; FALLBACK15-NEXT:    movq -16(%rsp,%rax), %rdi
-; FALLBACK15-NEXT:    shldq %cl, %rsi, %rdi
-; FALLBACK15-NEXT:    movq -40(%rsp,%rax), %r8
-; FALLBACK15-NEXT:    movq -32(%rsp,%rax), %rax
-; FALLBACK15-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK15-NEXT:    shlxq %rcx, %r8, %r9
-; FALLBACK15-NEXT:    # kill: def $cl killed $cl killed $rcx
-; FALLBACK15-NEXT:    shldq %cl, %r8, %rax
-; FALLBACK15-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK15-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK15-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK15-NEXT:    movq %r9, (%rdx)
-; FALLBACK15-NEXT:    vzeroupper
-; FALLBACK15-NEXT:    retq
-;
-; FALLBACK16-LABEL: shl_32bytes:
-; FALLBACK16:       # %bb.0:
-; FALLBACK16-NEXT:    pushl %ebp
-; FALLBACK16-NEXT:    pushl %ebx
-; FALLBACK16-NEXT:    pushl %edi
-; FALLBACK16-NEXT:    pushl %esi
-; FALLBACK16-NEXT:    subl $108, %esp
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK16-NEXT:    movl (%ecx), %edx
-; FALLBACK16-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 4(%ecx), %edx
-; FALLBACK16-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 8(%ecx), %esi
-; FALLBACK16-NEXT:    movl 12(%ecx), %edi
-; FALLBACK16-NEXT:    movl 16(%ecx), %ebx
-; FALLBACK16-NEXT:    movb (%eax), %ah
-; FALLBACK16-NEXT:    movl 20(%ecx), %ebp
-; FALLBACK16-NEXT:    movl 24(%ecx), %edx
-; FALLBACK16-NEXT:    movl 28(%ecx), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movb %ah, %ch
-; FALLBACK16-NEXT:    shlb $3, %ch
-; FALLBACK16-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK16-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    andb $28, %ah
-; FALLBACK16-NEXT:    negb %ah
-; FALLBACK16-NEXT:    movsbl %ah, %ebx
-; FALLBACK16-NEXT:    movl 64(%esp,%ebx), %edi
-; FALLBACK16-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 68(%esp,%ebx), %eax
-; FALLBACK16-NEXT:    movl %eax, %esi
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %esi
-; FALLBACK16-NEXT:    movb %ch, %dl
-; FALLBACK16-NEXT:    notb %dl
-; FALLBACK16-NEXT:    shrl %edi
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edi
-; FALLBACK16-NEXT:    orl %esi, %edi
-; FALLBACK16-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 76(%esp,%ebx), %edi
-; FALLBACK16-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %edi
-; FALLBACK16-NEXT:    movl 72(%esp,%ebx), %esi
-; FALLBACK16-NEXT:    movl %esi, %ebp
-; FALLBACK16-NEXT:    shrl %ebp
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %ebp
-; FALLBACK16-NEXT:    orl %edi, %ebp
-; FALLBACK16-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %esi
-; FALLBACK16-NEXT:    shrl %eax
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %eax
-; FALLBACK16-NEXT:    orl %esi, %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 84(%esp,%ebx), %esi
-; FALLBACK16-NEXT:    movl %esi, %eax
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %eax
-; FALLBACK16-NEXT:    movl 80(%esp,%ebx), %edi
-; FALLBACK16-NEXT:    movl %edi, %ebp
-; FALLBACK16-NEXT:    shrl %ebp
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %ebp
-; FALLBACK16-NEXT:    orl %eax, %ebp
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %edi
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    shrl %eax
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %eax
-; FALLBACK16-NEXT:    orl %edi, %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 92(%esp,%ebx), %eax
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %eax
-; FALLBACK16-NEXT:    movl 88(%esp,%ebx), %edi
-; FALLBACK16-NEXT:    movl %edi, %ebx
-; FALLBACK16-NEXT:    shrl %ebx
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %ebx
-; FALLBACK16-NEXT:    orl %eax, %ebx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %edi
-; FALLBACK16-NEXT:    shrl %esi
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %esi
-; FALLBACK16-NEXT:    orl %edi, %esi
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK16-NEXT:    shll %cl, %edx
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK16-NEXT:    movl %edx, (%eax)
-; FALLBACK16-NEXT:    movl %esi, 24(%eax)
-; FALLBACK16-NEXT:    movl %ebx, 28(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK16-NEXT:    movl %ebp, 20(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK16-NEXT:    addl $108, %esp
-; FALLBACK16-NEXT:    popl %esi
-; FALLBACK16-NEXT:    popl %edi
-; FALLBACK16-NEXT:    popl %ebx
-; FALLBACK16-NEXT:    popl %ebp
-; FALLBACK16-NEXT:    retl
-;
-; FALLBACK17-LABEL: shl_32bytes:
-; FALLBACK17:       # %bb.0:
-; FALLBACK17-NEXT:    pushl %ebp
-; FALLBACK17-NEXT:    pushl %ebx
-; FALLBACK17-NEXT:    pushl %edi
-; FALLBACK17-NEXT:    pushl %esi
-; FALLBACK17-NEXT:    subl $92, %esp
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK17-NEXT:    movl (%eax), %edx
-; FALLBACK17-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 4(%eax), %edx
-; FALLBACK17-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 8(%eax), %esi
-; FALLBACK17-NEXT:    movl 12(%eax), %edi
-; FALLBACK17-NEXT:    movl 16(%eax), %ebx
-; FALLBACK17-NEXT:    movb (%ecx), %ch
-; FALLBACK17-NEXT:    movl 20(%eax), %ebp
-; FALLBACK17-NEXT:    movl 24(%eax), %edx
-; FALLBACK17-NEXT:    movl 28(%eax), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movb %ch, %cl
-; FALLBACK17-NEXT:    shlb $3, %cl
-; FALLBACK17-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK17-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    andb $28, %ch
-; FALLBACK17-NEXT:    negb %ch
-; FALLBACK17-NEXT:    movsbl %ch, %eax
-; FALLBACK17-NEXT:    movl 56(%esp,%eax), %edx
-; FALLBACK17-NEXT:    movl 60(%esp,%eax), %ebx
-; FALLBACK17-NEXT:    movl %ebx, %esi
-; FALLBACK17-NEXT:    shldl %cl, %edx, %esi
-; FALLBACK17-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 52(%esp,%eax), %esi
-; FALLBACK17-NEXT:    movl %esi, (%esp) # 4-byte Spill
-; FALLBACK17-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK17-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 64(%esp,%eax), %edi
-; FALLBACK17-NEXT:    movl 68(%esp,%eax), %ebp
-; FALLBACK17-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shldl %cl, %edi, %ebp
-; FALLBACK17-NEXT:    shldl %cl, %ebx, %edi
-; FALLBACK17-NEXT:    movl 48(%esp,%eax), %ebx
-; FALLBACK17-NEXT:    movl 72(%esp,%eax), %edx
-; FALLBACK17-NEXT:    movl 76(%esp,%eax), %esi
-; FALLBACK17-NEXT:    shldl %cl, %edx, %esi
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    shldl %cl, %eax, %edx
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK17-NEXT:    movl %edx, 24(%eax)
-; FALLBACK17-NEXT:    movl %esi, 28(%eax)
-; FALLBACK17-NEXT:    movl %edi, 16(%eax)
-; FALLBACK17-NEXT:    movl %ebp, 20(%eax)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    movl %edx, 8(%eax)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    movl %edx, 12(%eax)
-; FALLBACK17-NEXT:    movl (%esp), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    shldl %cl, %ebx, %edx
-; FALLBACK17-NEXT:    shll %cl, %ebx
-; FALLBACK17-NEXT:    movl %ebx, (%eax)
-; FALLBACK17-NEXT:    movl %edx, 4(%eax)
-; FALLBACK17-NEXT:    addl $92, %esp
-; FALLBACK17-NEXT:    popl %esi
-; FALLBACK17-NEXT:    popl %edi
-; FALLBACK17-NEXT:    popl %ebx
-; FALLBACK17-NEXT:    popl %ebp
-; FALLBACK17-NEXT:    retl
-;
-; FALLBACK18-LABEL: shl_32bytes:
-; FALLBACK18:       # %bb.0:
-; FALLBACK18-NEXT:    pushl %ebp
-; FALLBACK18-NEXT:    pushl %ebx
-; FALLBACK18-NEXT:    pushl %edi
-; FALLBACK18-NEXT:    pushl %esi
-; FALLBACK18-NEXT:    subl $108, %esp
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK18-NEXT:    movl (%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 4(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 8(%eax), %esi
-; FALLBACK18-NEXT:    movl 12(%eax), %edi
-; FALLBACK18-NEXT:    movl 16(%eax), %ebp
-; FALLBACK18-NEXT:    movzbl (%ebx), %ebx
-; FALLBACK18-NEXT:    movl 20(%eax), %edx
-; FALLBACK18-NEXT:    movl 24(%eax), %ecx
-; FALLBACK18-NEXT:    movl 28(%eax), %eax
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ebx, %edx
-; FALLBACK18-NEXT:    shlb $3, %dl
-; FALLBACK18-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK18-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %edx, %eax
-; FALLBACK18-NEXT:    movl %eax, %ebp
-; FALLBACK18-NEXT:    andb $28, %bl
-; FALLBACK18-NEXT:    negb %bl
-; FALLBACK18-NEXT:    movsbl %bl, %esi
-; FALLBACK18-NEXT:    movl 64(%esp,%esi), %ebx
-; FALLBACK18-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 68(%esp,%esi), %ecx
-; FALLBACK18-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shlxl %eax, %ecx, %edi
-; FALLBACK18-NEXT:    notb %dl
-; FALLBACK18-NEXT:    shrl %ebx
-; FALLBACK18-NEXT:    shrxl %edx, %ebx, %ebx
-; FALLBACK18-NEXT:    orl %edi, %ebx
-; FALLBACK18-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 72(%esp,%esi), %ebx
-; FALLBACK18-NEXT:    movl %ebx, %edi
-; FALLBACK18-NEXT:    shrl %edi
-; FALLBACK18-NEXT:    shrxl %edx, %edi, %eax
-; FALLBACK18-NEXT:    movl 76(%esp,%esi), %edi
-; FALLBACK18-NEXT:    movl %ebp, %esi
-; FALLBACK18-NEXT:    shlxl %ebp, %edi, %ebp
-; FALLBACK18-NEXT:    orl %ebp, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shlxl %esi, %ebx, %ebx
-; FALLBACK18-NEXT:    shrl %ecx
-; FALLBACK18-NEXT:    shrxl %edx, %ecx, %ecx
-; FALLBACK18-NEXT:    orl %ebx, %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; FALLBACK18-NEXT:    movl 80(%esp,%ebp), %ecx
-; FALLBACK18-NEXT:    movl %ecx, %ebx
-; FALLBACK18-NEXT:    shrl %ebx
-; FALLBACK18-NEXT:    shrxl %edx, %ebx, %eax
-; FALLBACK18-NEXT:    movl 84(%esp,%ebp), %ebx
-; FALLBACK18-NEXT:    shlxl %esi, %ebx, %ebp
-; FALLBACK18-NEXT:    orl %ebp, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shlxl %esi, %ecx, %ecx
-; FALLBACK18-NEXT:    movl %esi, %eax
-; FALLBACK18-NEXT:    shrl %edi
-; FALLBACK18-NEXT:    shrxl %edx, %edi, %edi
-; FALLBACK18-NEXT:    orl %ecx, %edi
-; FALLBACK18-NEXT:    shlxl %esi, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    shlxl %esi, 92(%esp,%ecx), %ebp
-; FALLBACK18-NEXT:    movl 88(%esp,%ecx), %esi
-; FALLBACK18-NEXT:    shlxl %eax, %esi, %ecx
-; FALLBACK18-NEXT:    shrl %esi
-; FALLBACK18-NEXT:    shrxl %edx, %esi, %esi
-; FALLBACK18-NEXT:    orl %ebp, %esi
-; FALLBACK18-NEXT:    shrl %ebx
-; FALLBACK18-NEXT:    shrxl %edx, %ebx, %eax
-; FALLBACK18-NEXT:    orl %ecx, %eax
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, (%edx)
-; FALLBACK18-NEXT:    movl %eax, 24(%edx)
-; FALLBACK18-NEXT:    movl %esi, 28(%edx)
-; FALLBACK18-NEXT:    movl %edi, 16(%edx)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 20(%edx)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 8(%edx)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 12(%edx)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 4(%edx)
-; FALLBACK18-NEXT:    addl $108, %esp
-; FALLBACK18-NEXT:    popl %esi
-; FALLBACK18-NEXT:    popl %edi
-; FALLBACK18-NEXT:    popl %ebx
-; FALLBACK18-NEXT:    popl %ebp
-; FALLBACK18-NEXT:    retl
-;
-; FALLBACK19-LABEL: shl_32bytes:
-; FALLBACK19:       # %bb.0:
-; FALLBACK19-NEXT:    pushl %ebp
-; FALLBACK19-NEXT:    pushl %ebx
-; FALLBACK19-NEXT:    pushl %edi
-; FALLBACK19-NEXT:    pushl %esi
-; FALLBACK19-NEXT:    subl $92, %esp
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK19-NEXT:    movl (%ecx), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 4(%ecx), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 8(%ecx), %esi
-; FALLBACK19-NEXT:    movl 12(%ecx), %edi
-; FALLBACK19-NEXT:    movl 16(%ecx), %ebp
-; FALLBACK19-NEXT:    movzbl (%ebx), %ebx
-; FALLBACK19-NEXT:    movl 20(%ecx), %edx
-; FALLBACK19-NEXT:    movl 24(%ecx), %eax
-; FALLBACK19-NEXT:    movl 28(%ecx), %ecx
-; FALLBACK19-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %ebx, %ecx
-; FALLBACK19-NEXT:    shlb $3, %cl
-; FALLBACK19-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK19-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    andb $28, %bl
-; FALLBACK19-NEXT:    negb %bl
-; FALLBACK19-NEXT:    movsbl %bl, %eax
-; FALLBACK19-NEXT:    movl 56(%esp,%eax), %edx
-; FALLBACK19-NEXT:    movl 60(%esp,%eax), %esi
-; FALLBACK19-NEXT:    movl %esi, (%esp) # 4-byte Spill
-; FALLBACK19-NEXT:    shldl %cl, %edx, %esi
-; FALLBACK19-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 52(%esp,%eax), %ebx
-; FALLBACK19-NEXT:    shldl %cl, %ebx, %edx
-; FALLBACK19-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 64(%esp,%eax), %edi
-; FALLBACK19-NEXT:    movl 68(%esp,%eax), %ebp
-; FALLBACK19-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shldl %cl, %edi, %ebp
-; FALLBACK19-NEXT:    movl (%esp), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK19-NEXT:    movl 48(%esp,%eax), %edx
-; FALLBACK19-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 72(%esp,%eax), %edx
-; FALLBACK19-NEXT:    movl 76(%esp,%eax), %esi
-; FALLBACK19-NEXT:    shldl %cl, %edx, %esi
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    shldl %cl, %eax, %edx
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK19-NEXT:    movl %edx, 24(%eax)
-; FALLBACK19-NEXT:    movl %esi, 28(%eax)
-; FALLBACK19-NEXT:    movl %edi, 16(%eax)
-; FALLBACK19-NEXT:    movl %ebp, 20(%eax)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %edx, 8(%eax)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %edx, 12(%eax)
-; FALLBACK19-NEXT:    movl (%esp), %esi # 4-byte Reload
-; FALLBACK19-NEXT:    shlxl %ecx, %esi, %edx
-; FALLBACK19-NEXT:    movl %edx, (%eax)
-; FALLBACK19-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK19-NEXT:    shldl %cl, %esi, %ebx
-; FALLBACK19-NEXT:    movl %ebx, 4(%eax)
-; FALLBACK19-NEXT:    addl $92, %esp
-; FALLBACK19-NEXT:    popl %esi
-; FALLBACK19-NEXT:    popl %edi
-; FALLBACK19-NEXT:    popl %ebx
-; FALLBACK19-NEXT:    popl %ebp
-; FALLBACK19-NEXT:    retl
-;
-; FALLBACK20-LABEL: shl_32bytes:
-; FALLBACK20:       # %bb.0:
-; FALLBACK20-NEXT:    pushl %ebp
-; FALLBACK20-NEXT:    pushl %ebx
-; FALLBACK20-NEXT:    pushl %edi
-; FALLBACK20-NEXT:    pushl %esi
-; FALLBACK20-NEXT:    subl $108, %esp
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK20-NEXT:    movups (%ecx), %xmm0
-; FALLBACK20-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK20-NEXT:    movzbl (%eax), %ecx
-; FALLBACK20-NEXT:    movb %cl, %dh
-; FALLBACK20-NEXT:    shlb $3, %dh
-; FALLBACK20-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK20-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    andb $28, %cl
-; FALLBACK20-NEXT:    negb %cl
-; FALLBACK20-NEXT:    movsbl %cl, %ebx
-; FALLBACK20-NEXT:    movl 84(%esp,%ebx), %edi
-; FALLBACK20-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %dh, %cl
-; FALLBACK20-NEXT:    shll %cl, %edi
-; FALLBACK20-NEXT:    movb %dh, %dl
-; FALLBACK20-NEXT:    notb %dl
-; FALLBACK20-NEXT:    movl 80(%esp,%ebx), %esi
-; FALLBACK20-NEXT:    movl %esi, %eax
-; FALLBACK20-NEXT:    shrl %eax
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %eax
-; FALLBACK20-NEXT:    orl %edi, %eax
-; FALLBACK20-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %dh, %cl
-; FALLBACK20-NEXT:    shll %cl, %esi
-; FALLBACK20-NEXT:    movl %ebx, %edi
-; FALLBACK20-NEXT:    movl 76(%esp,%ebx), %ebp
-; FALLBACK20-NEXT:    movl %ebp, %eax
-; FALLBACK20-NEXT:    shrl %eax
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %eax
-; FALLBACK20-NEXT:    orl %esi, %eax
-; FALLBACK20-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %dh, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebp
-; FALLBACK20-NEXT:    movl 72(%esp,%ebx), %ebx
-; FALLBACK20-NEXT:    movl %ebx, %eax
-; FALLBACK20-NEXT:    shrl %eax
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %eax
-; FALLBACK20-NEXT:    orl %ebp, %eax
-; FALLBACK20-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %dh, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 68(%esp,%edi), %ebp
-; FALLBACK20-NEXT:    movl %ebp, %esi
-; FALLBACK20-NEXT:    shrl %esi
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %esi
-; FALLBACK20-NEXT:    orl %ebx, %esi
-; FALLBACK20-NEXT:    movb %dh, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebp
-; FALLBACK20-NEXT:    movl 64(%esp,%edi), %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    shrl %ebx
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %ebx
-; FALLBACK20-NEXT:    orl %ebp, %ebx
-; FALLBACK20-NEXT:    movl 88(%esp,%edi), %ebp
-; FALLBACK20-NEXT:    movl %ebp, %edi
-; FALLBACK20-NEXT:    movb %dh, %cl
-; FALLBACK20-NEXT:    shll %cl, %edi
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %eax
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %eax
-; FALLBACK20-NEXT:    orl %edi, %eax
-; FALLBACK20-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK20-NEXT:    movl 92(%esp,%eax), %edi
-; FALLBACK20-NEXT:    movb %dh, %cl
-; FALLBACK20-NEXT:    shll %cl, %edi
-; FALLBACK20-NEXT:    shrl %ebp
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %ebp
-; FALLBACK20-NEXT:    orl %edi, %ebp
-; FALLBACK20-NEXT:    movb %dh, %cl
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK20-NEXT:    shll %cl, %edx
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK20-NEXT:    movl %edx, (%eax)
-; FALLBACK20-NEXT:    movl %ebp, 28(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK20-NEXT:    movl %ebx, 4(%eax)
-; FALLBACK20-NEXT:    movl %esi, 8(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK20-NEXT:    addl $108, %esp
-; FALLBACK20-NEXT:    popl %esi
-; FALLBACK20-NEXT:    popl %edi
-; FALLBACK20-NEXT:    popl %ebx
-; FALLBACK20-NEXT:    popl %ebp
-; FALLBACK20-NEXT:    retl
-;
-; FALLBACK21-LABEL: shl_32bytes:
-; FALLBACK21:       # %bb.0:
-; FALLBACK21-NEXT:    pushl %ebp
-; FALLBACK21-NEXT:    pushl %ebx
-; FALLBACK21-NEXT:    pushl %edi
-; FALLBACK21-NEXT:    pushl %esi
-; FALLBACK21-NEXT:    subl $92, %esp
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK21-NEXT:    movups (%ecx), %xmm0
-; FALLBACK21-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK21-NEXT:    movzbl (%eax), %eax
-; FALLBACK21-NEXT:    movl %eax, %ecx
-; FALLBACK21-NEXT:    shlb $3, %cl
-; FALLBACK21-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK21-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    andb $28, %al
-; FALLBACK21-NEXT:    negb %al
-; FALLBACK21-NEXT:    movsbl %al, %ebp
-; FALLBACK21-NEXT:    movl 64(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl 68(%esp,%ebp), %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shldl %cl, %eax, %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 60(%esp,%ebp), %edx
-; FALLBACK21-NEXT:    shldl %cl, %edx, %eax
-; FALLBACK21-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 56(%esp,%ebp), %edi
-; FALLBACK21-NEXT:    shldl %cl, %edi, %edx
-; FALLBACK21-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 52(%esp,%ebp), %ebx
-; FALLBACK21-NEXT:    shldl %cl, %ebx, %edi
-; FALLBACK21-NEXT:    movl 72(%esp,%ebp), %edx
-; FALLBACK21-NEXT:    movl %edx, %eax
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK21-NEXT:    shldl %cl, %esi, %eax
-; FALLBACK21-NEXT:    movl 48(%esp,%ebp), %esi
-; FALLBACK21-NEXT:    movl 76(%esp,%ebp), %ebp
-; FALLBACK21-NEXT:    shldl %cl, %edx, %ebp
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK21-NEXT:    movl %ebp, 28(%edx)
-; FALLBACK21-NEXT:    movl %eax, 24(%edx)
-; FALLBACK21-NEXT:    movl %esi, %eax
-; FALLBACK21-NEXT:    shll %cl, %eax
-; FALLBACK21-NEXT:    shldl %cl, %esi, %ebx
-; FALLBACK21-NEXT:    movl %ebx, 4(%edx)
-; FALLBACK21-NEXT:    movl %edi, 8(%edx)
-; FALLBACK21-NEXT:    movl (%esp), %ecx # 4-byte Reload
-; FALLBACK21-NEXT:    movl %ecx, 12(%edx)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK21-NEXT:    movl %ecx, 16(%edx)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK21-NEXT:    movl %ecx, 20(%edx)
-; FALLBACK21-NEXT:    movl %eax, (%edx)
-; FALLBACK21-NEXT:    addl $92, %esp
-; FALLBACK21-NEXT:    popl %esi
-; FALLBACK21-NEXT:    popl %edi
-; FALLBACK21-NEXT:    popl %ebx
-; FALLBACK21-NEXT:    popl %ebp
-; FALLBACK21-NEXT:    retl
-;
-; FALLBACK22-LABEL: shl_32bytes:
-; FALLBACK22:       # %bb.0:
-; FALLBACK22-NEXT:    pushl %ebp
-; FALLBACK22-NEXT:    pushl %ebx
-; FALLBACK22-NEXT:    pushl %edi
-; FALLBACK22-NEXT:    pushl %esi
-; FALLBACK22-NEXT:    subl $108, %esp
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK22-NEXT:    movups (%ecx), %xmm0
-; FALLBACK22-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK22-NEXT:    movzbl (%eax), %edx
-; FALLBACK22-NEXT:    movl %edx, %ecx
-; FALLBACK22-NEXT:    shlb $3, %cl
-; FALLBACK22-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK22-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, %ebx
-; FALLBACK22-NEXT:    andb $28, %dl
-; FALLBACK22-NEXT:    negb %dl
-; FALLBACK22-NEXT:    movsbl %dl, %edx
-; FALLBACK22-NEXT:    movl 84(%esp,%edx), %eax
-; FALLBACK22-NEXT:    shlxl %ebx, %eax, %esi
-; FALLBACK22-NEXT:    notb %cl
-; FALLBACK22-NEXT:    movl 80(%esp,%edx), %edi
-; FALLBACK22-NEXT:    shlxl %ebx, %edi, %ebp
-; FALLBACK22-NEXT:    shrl %edi
-; FALLBACK22-NEXT:    shrxl %ecx, %edi, %edi
-; FALLBACK22-NEXT:    orl %esi, %edi
-; FALLBACK22-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 76(%esp,%edx), %esi
-; FALLBACK22-NEXT:    movl %esi, %edi
-; FALLBACK22-NEXT:    shrl %edi
-; FALLBACK22-NEXT:    shrxl %ecx, %edi, %edi
-; FALLBACK22-NEXT:    orl %ebp, %edi
-; FALLBACK22-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shlxl %ebx, %esi, %esi
-; FALLBACK22-NEXT:    movl 72(%esp,%edx), %edi
-; FALLBACK22-NEXT:    movl %edi, %ebp
-; FALLBACK22-NEXT:    shrl %ebp
-; FALLBACK22-NEXT:    shrxl %ecx, %ebp, %ebp
-; FALLBACK22-NEXT:    orl %esi, %ebp
-; FALLBACK22-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shlxl %ebx, %edi, %esi
-; FALLBACK22-NEXT:    movl 68(%esp,%edx), %ebp
-; FALLBACK22-NEXT:    movl %ebp, %edi
-; FALLBACK22-NEXT:    shrl %edi
-; FALLBACK22-NEXT:    shrxl %ecx, %edi, %edi
-; FALLBACK22-NEXT:    orl %esi, %edi
-; FALLBACK22-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shlxl %ebx, %ebp, %ebp
-; FALLBACK22-NEXT:    movl 64(%esp,%edx), %esi
-; FALLBACK22-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrl %esi
-; FALLBACK22-NEXT:    shrxl %ecx, %esi, %edi
-; FALLBACK22-NEXT:    orl %ebp, %edi
-; FALLBACK22-NEXT:    shrl %eax
-; FALLBACK22-NEXT:    shrxl %ecx, %eax, %esi
-; FALLBACK22-NEXT:    movl 88(%esp,%edx), %eax
-; FALLBACK22-NEXT:    shlxl %ebx, %eax, %ebp
-; FALLBACK22-NEXT:    orl %ebp, %esi
-; FALLBACK22-NEXT:    shlxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK22-NEXT:    shlxl %ebx, 92(%esp,%edx), %edx
-; FALLBACK22-NEXT:    shrl %eax
-; FALLBACK22-NEXT:    shrxl %ecx, %eax, %eax
-; FALLBACK22-NEXT:    orl %edx, %eax
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK22-NEXT:    movl %ebp, (%ecx)
-; FALLBACK22-NEXT:    movl %eax, 28(%ecx)
-; FALLBACK22-NEXT:    movl %esi, 24(%ecx)
-; FALLBACK22-NEXT:    movl %edi, 4(%ecx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 8(%ecx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 12(%ecx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 16(%ecx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 20(%ecx)
-; FALLBACK22-NEXT:    addl $108, %esp
-; FALLBACK22-NEXT:    popl %esi
-; FALLBACK22-NEXT:    popl %edi
-; FALLBACK22-NEXT:    popl %ebx
-; FALLBACK22-NEXT:    popl %ebp
-; FALLBACK22-NEXT:    retl
-;
-; FALLBACK23-LABEL: shl_32bytes:
-; FALLBACK23:       # %bb.0:
-; FALLBACK23-NEXT:    pushl %ebp
-; FALLBACK23-NEXT:    pushl %ebx
-; FALLBACK23-NEXT:    pushl %edi
-; FALLBACK23-NEXT:    pushl %esi
-; FALLBACK23-NEXT:    subl $92, %esp
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK23-NEXT:    movups (%ecx), %xmm0
-; FALLBACK23-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK23-NEXT:    movzbl (%eax), %eax
-; FALLBACK23-NEXT:    movl %eax, %ecx
-; FALLBACK23-NEXT:    shlb $3, %cl
-; FALLBACK23-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK23-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    andb $28, %al
-; FALLBACK23-NEXT:    negb %al
-; FALLBACK23-NEXT:    movsbl %al, %ebx
-; FALLBACK23-NEXT:    movl 64(%esp,%ebx), %eax
-; FALLBACK23-NEXT:    movl 68(%esp,%ebx), %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shldl %cl, %eax, %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 60(%esp,%ebx), %edx
-; FALLBACK23-NEXT:    shldl %cl, %edx, %eax
-; FALLBACK23-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 56(%esp,%ebx), %edi
-; FALLBACK23-NEXT:    shldl %cl, %edi, %edx
-; FALLBACK23-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 52(%esp,%ebx), %ebp
-; FALLBACK23-NEXT:    shldl %cl, %ebp, %edi
-; FALLBACK23-NEXT:    movl 72(%esp,%ebx), %edx
-; FALLBACK23-NEXT:    movl %edx, %eax
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK23-NEXT:    shldl %cl, %esi, %eax
-; FALLBACK23-NEXT:    movl 48(%esp,%ebx), %esi
-; FALLBACK23-NEXT:    movl 76(%esp,%ebx), %ebx
-; FALLBACK23-NEXT:    shldl %cl, %edx, %ebx
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK23-NEXT:    movl %ebx, 28(%edx)
-; FALLBACK23-NEXT:    movl %eax, 24(%edx)
-; FALLBACK23-NEXT:    shlxl %ecx, %esi, %eax
-; FALLBACK23-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK23-NEXT:    shldl %cl, %esi, %ebp
-; FALLBACK23-NEXT:    movl %ebp, 4(%edx)
-; FALLBACK23-NEXT:    movl %edi, 8(%edx)
-; FALLBACK23-NEXT:    movl (%esp), %ecx # 4-byte Reload
-; FALLBACK23-NEXT:    movl %ecx, 12(%edx)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK23-NEXT:    movl %ecx, 16(%edx)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK23-NEXT:    movl %ecx, 20(%edx)
-; FALLBACK23-NEXT:    movl %eax, (%edx)
-; FALLBACK23-NEXT:    addl $92, %esp
-; FALLBACK23-NEXT:    popl %esi
-; FALLBACK23-NEXT:    popl %edi
-; FALLBACK23-NEXT:    popl %ebx
-; FALLBACK23-NEXT:    popl %ebp
-; FALLBACK23-NEXT:    retl
-;
-; FALLBACK24-LABEL: shl_32bytes:
-; FALLBACK24:       # %bb.0:
-; FALLBACK24-NEXT:    pushl %ebp
-; FALLBACK24-NEXT:    pushl %ebx
-; FALLBACK24-NEXT:    pushl %edi
-; FALLBACK24-NEXT:    pushl %esi
-; FALLBACK24-NEXT:    subl $108, %esp
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK24-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK24-NEXT:    movzbl (%eax), %ecx
-; FALLBACK24-NEXT:    movb %cl, %dh
-; FALLBACK24-NEXT:    shlb $3, %dh
-; FALLBACK24-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK24-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    andb $28, %cl
-; FALLBACK24-NEXT:    negb %cl
-; FALLBACK24-NEXT:    movsbl %cl, %ebx
-; FALLBACK24-NEXT:    movl 84(%esp,%ebx), %edi
-; FALLBACK24-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %dh, %cl
-; FALLBACK24-NEXT:    shll %cl, %edi
-; FALLBACK24-NEXT:    movb %dh, %dl
-; FALLBACK24-NEXT:    notb %dl
-; FALLBACK24-NEXT:    movl 80(%esp,%ebx), %esi
-; FALLBACK24-NEXT:    movl %esi, %eax
-; FALLBACK24-NEXT:    shrl %eax
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %eax
-; FALLBACK24-NEXT:    orl %edi, %eax
-; FALLBACK24-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %dh, %cl
-; FALLBACK24-NEXT:    shll %cl, %esi
-; FALLBACK24-NEXT:    movl %ebx, %edi
-; FALLBACK24-NEXT:    movl 76(%esp,%ebx), %ebp
-; FALLBACK24-NEXT:    movl %ebp, %eax
-; FALLBACK24-NEXT:    shrl %eax
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %eax
-; FALLBACK24-NEXT:    orl %esi, %eax
-; FALLBACK24-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %dh, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebp
-; FALLBACK24-NEXT:    movl 72(%esp,%ebx), %ebx
-; FALLBACK24-NEXT:    movl %ebx, %eax
-; FALLBACK24-NEXT:    shrl %eax
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %eax
-; FALLBACK24-NEXT:    orl %ebp, %eax
-; FALLBACK24-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %dh, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 68(%esp,%edi), %ebp
-; FALLBACK24-NEXT:    movl %ebp, %esi
-; FALLBACK24-NEXT:    shrl %esi
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %esi
-; FALLBACK24-NEXT:    orl %ebx, %esi
-; FALLBACK24-NEXT:    movb %dh, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebp
-; FALLBACK24-NEXT:    movl 64(%esp,%edi), %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    shrl %ebx
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %ebx
-; FALLBACK24-NEXT:    orl %ebp, %ebx
-; FALLBACK24-NEXT:    movl 88(%esp,%edi), %ebp
-; FALLBACK24-NEXT:    movl %ebp, %edi
-; FALLBACK24-NEXT:    movb %dh, %cl
-; FALLBACK24-NEXT:    shll %cl, %edi
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %eax
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %eax
-; FALLBACK24-NEXT:    orl %edi, %eax
-; FALLBACK24-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK24-NEXT:    movl 92(%esp,%eax), %edi
-; FALLBACK24-NEXT:    movb %dh, %cl
-; FALLBACK24-NEXT:    shll %cl, %edi
-; FALLBACK24-NEXT:    shrl %ebp
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %ebp
-; FALLBACK24-NEXT:    orl %edi, %ebp
-; FALLBACK24-NEXT:    movb %dh, %cl
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK24-NEXT:    shll %cl, %edx
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK24-NEXT:    movl %edx, (%eax)
-; FALLBACK24-NEXT:    movl %ebp, 28(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK24-NEXT:    movl %ebx, 4(%eax)
-; FALLBACK24-NEXT:    movl %esi, 8(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK24-NEXT:    addl $108, %esp
-; FALLBACK24-NEXT:    popl %esi
-; FALLBACK24-NEXT:    popl %edi
-; FALLBACK24-NEXT:    popl %ebx
-; FALLBACK24-NEXT:    popl %ebp
-; FALLBACK24-NEXT:    vzeroupper
-; FALLBACK24-NEXT:    retl
-;
-; FALLBACK25-LABEL: shl_32bytes:
-; FALLBACK25:       # %bb.0:
-; FALLBACK25-NEXT:    pushl %ebp
-; FALLBACK25-NEXT:    pushl %ebx
-; FALLBACK25-NEXT:    pushl %edi
-; FALLBACK25-NEXT:    pushl %esi
-; FALLBACK25-NEXT:    subl $92, %esp
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK25-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK25-NEXT:    movzbl (%eax), %eax
-; FALLBACK25-NEXT:    movl %eax, %ecx
-; FALLBACK25-NEXT:    shlb $3, %cl
-; FALLBACK25-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK25-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    andb $28, %al
-; FALLBACK25-NEXT:    negb %al
-; FALLBACK25-NEXT:    movsbl %al, %ebp
-; FALLBACK25-NEXT:    movl 64(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl 68(%esp,%ebp), %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shldl %cl, %eax, %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 60(%esp,%ebp), %edx
-; FALLBACK25-NEXT:    shldl %cl, %edx, %eax
-; FALLBACK25-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 56(%esp,%ebp), %edi
-; FALLBACK25-NEXT:    shldl %cl, %edi, %edx
-; FALLBACK25-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 52(%esp,%ebp), %ebx
-; FALLBACK25-NEXT:    shldl %cl, %ebx, %edi
-; FALLBACK25-NEXT:    movl 72(%esp,%ebp), %edx
-; FALLBACK25-NEXT:    movl %edx, %eax
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK25-NEXT:    shldl %cl, %esi, %eax
-; FALLBACK25-NEXT:    movl 48(%esp,%ebp), %esi
-; FALLBACK25-NEXT:    movl 76(%esp,%ebp), %ebp
-; FALLBACK25-NEXT:    shldl %cl, %edx, %ebp
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK25-NEXT:    movl %ebp, 28(%edx)
-; FALLBACK25-NEXT:    movl %eax, 24(%edx)
-; FALLBACK25-NEXT:    movl %esi, %eax
-; FALLBACK25-NEXT:    shll %cl, %eax
-; FALLBACK25-NEXT:    shldl %cl, %esi, %ebx
-; FALLBACK25-NEXT:    movl %ebx, 4(%edx)
-; FALLBACK25-NEXT:    movl %edi, 8(%edx)
-; FALLBACK25-NEXT:    movl (%esp), %ecx # 4-byte Reload
-; FALLBACK25-NEXT:    movl %ecx, 12(%edx)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK25-NEXT:    movl %ecx, 16(%edx)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK25-NEXT:    movl %ecx, 20(%edx)
-; FALLBACK25-NEXT:    movl %eax, (%edx)
-; FALLBACK25-NEXT:    addl $92, %esp
-; FALLBACK25-NEXT:    popl %esi
-; FALLBACK25-NEXT:    popl %edi
-; FALLBACK25-NEXT:    popl %ebx
-; FALLBACK25-NEXT:    popl %ebp
-; FALLBACK25-NEXT:    vzeroupper
-; FALLBACK25-NEXT:    retl
-;
-; FALLBACK26-LABEL: shl_32bytes:
-; FALLBACK26:       # %bb.0:
-; FALLBACK26-NEXT:    pushl %ebp
-; FALLBACK26-NEXT:    pushl %ebx
-; FALLBACK26-NEXT:    pushl %edi
-; FALLBACK26-NEXT:    pushl %esi
-; FALLBACK26-NEXT:    subl $108, %esp
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK26-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK26-NEXT:    movzbl (%eax), %edx
-; FALLBACK26-NEXT:    movl %edx, %eax
-; FALLBACK26-NEXT:    shlb $3, %al
-; FALLBACK26-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK26-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %eax, %ebx
-; FALLBACK26-NEXT:    andb $28, %dl
-; FALLBACK26-NEXT:    negb %dl
-; FALLBACK26-NEXT:    movsbl %dl, %edx
-; FALLBACK26-NEXT:    movl 84(%esp,%edx), %ecx
-; FALLBACK26-NEXT:    shlxl %ebx, %ecx, %esi
-; FALLBACK26-NEXT:    notb %al
-; FALLBACK26-NEXT:    movl 80(%esp,%edx), %edi
-; FALLBACK26-NEXT:    shlxl %ebx, %edi, %ebp
-; FALLBACK26-NEXT:    shrl %edi
-; FALLBACK26-NEXT:    shrxl %eax, %edi, %edi
-; FALLBACK26-NEXT:    orl %esi, %edi
-; FALLBACK26-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 76(%esp,%edx), %esi
-; FALLBACK26-NEXT:    movl %esi, %edi
-; FALLBACK26-NEXT:    shrl %edi
-; FALLBACK26-NEXT:    shrxl %eax, %edi, %edi
-; FALLBACK26-NEXT:    orl %ebp, %edi
-; FALLBACK26-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shlxl %ebx, %esi, %esi
-; FALLBACK26-NEXT:    movl 72(%esp,%edx), %edi
-; FALLBACK26-NEXT:    movl %edi, %ebp
-; FALLBACK26-NEXT:    shrl %ebp
-; FALLBACK26-NEXT:    shrxl %eax, %ebp, %ebp
-; FALLBACK26-NEXT:    orl %esi, %ebp
-; FALLBACK26-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shlxl %ebx, %edi, %esi
-; FALLBACK26-NEXT:    movl 68(%esp,%edx), %ebp
-; FALLBACK26-NEXT:    movl %ebp, %edi
-; FALLBACK26-NEXT:    shrl %edi
-; FALLBACK26-NEXT:    shrxl %eax, %edi, %edi
-; FALLBACK26-NEXT:    orl %esi, %edi
-; FALLBACK26-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shlxl %ebx, %ebp, %ebp
-; FALLBACK26-NEXT:    movl 64(%esp,%edx), %esi
-; FALLBACK26-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrl %esi
-; FALLBACK26-NEXT:    shrxl %eax, %esi, %edi
-; FALLBACK26-NEXT:    orl %ebp, %edi
-; FALLBACK26-NEXT:    shrl %ecx
-; FALLBACK26-NEXT:    shrxl %eax, %ecx, %esi
-; FALLBACK26-NEXT:    movl 88(%esp,%edx), %ecx
-; FALLBACK26-NEXT:    shlxl %ebx, %ecx, %ebp
-; FALLBACK26-NEXT:    orl %ebp, %esi
-; FALLBACK26-NEXT:    shlxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK26-NEXT:    shlxl %ebx, 92(%esp,%edx), %edx
-; FALLBACK26-NEXT:    shrl %ecx
-; FALLBACK26-NEXT:    shrxl %eax, %ecx, %eax
-; FALLBACK26-NEXT:    orl %edx, %eax
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK26-NEXT:    movl %ebp, (%ecx)
-; FALLBACK26-NEXT:    movl %eax, 28(%ecx)
-; FALLBACK26-NEXT:    movl %esi, 24(%ecx)
-; FALLBACK26-NEXT:    movl %edi, 4(%ecx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 8(%ecx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 12(%ecx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 16(%ecx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 20(%ecx)
-; FALLBACK26-NEXT:    addl $108, %esp
-; FALLBACK26-NEXT:    popl %esi
-; FALLBACK26-NEXT:    popl %edi
-; FALLBACK26-NEXT:    popl %ebx
-; FALLBACK26-NEXT:    popl %ebp
-; FALLBACK26-NEXT:    vzeroupper
-; FALLBACK26-NEXT:    retl
-;
-; FALLBACK27-LABEL: shl_32bytes:
-; FALLBACK27:       # %bb.0:
-; FALLBACK27-NEXT:    pushl %ebp
-; FALLBACK27-NEXT:    pushl %ebx
-; FALLBACK27-NEXT:    pushl %edi
-; FALLBACK27-NEXT:    pushl %esi
-; FALLBACK27-NEXT:    subl $92, %esp
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK27-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK27-NEXT:    movzbl (%eax), %eax
-; FALLBACK27-NEXT:    movl %eax, %ecx
-; FALLBACK27-NEXT:    shlb $3, %cl
-; FALLBACK27-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK27-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    andb $28, %al
-; FALLBACK27-NEXT:    negb %al
-; FALLBACK27-NEXT:    movsbl %al, %ebx
-; FALLBACK27-NEXT:    movl 64(%esp,%ebx), %eax
-; FALLBACK27-NEXT:    movl 68(%esp,%ebx), %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shldl %cl, %eax, %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 60(%esp,%ebx), %edx
-; FALLBACK27-NEXT:    shldl %cl, %edx, %eax
-; FALLBACK27-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 56(%esp,%ebx), %edi
-; FALLBACK27-NEXT:    shldl %cl, %edi, %edx
-; FALLBACK27-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 52(%esp,%ebx), %ebp
-; FALLBACK27-NEXT:    shldl %cl, %ebp, %edi
-; FALLBACK27-NEXT:    movl 72(%esp,%ebx), %edx
-; FALLBACK27-NEXT:    movl %edx, %eax
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK27-NEXT:    shldl %cl, %esi, %eax
-; FALLBACK27-NEXT:    movl 48(%esp,%ebx), %esi
-; FALLBACK27-NEXT:    movl 76(%esp,%ebx), %ebx
-; FALLBACK27-NEXT:    shldl %cl, %edx, %ebx
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK27-NEXT:    movl %ebx, 28(%edx)
-; FALLBACK27-NEXT:    movl %eax, 24(%edx)
-; FALLBACK27-NEXT:    shlxl %ecx, %esi, %eax
-; FALLBACK27-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK27-NEXT:    shldl %cl, %esi, %ebp
-; FALLBACK27-NEXT:    movl %ebp, 4(%edx)
-; FALLBACK27-NEXT:    movl %edi, 8(%edx)
-; FALLBACK27-NEXT:    movl (%esp), %ecx # 4-byte Reload
-; FALLBACK27-NEXT:    movl %ecx, 12(%edx)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK27-NEXT:    movl %ecx, 16(%edx)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK27-NEXT:    movl %ecx, 20(%edx)
-; FALLBACK27-NEXT:    movl %eax, (%edx)
-; FALLBACK27-NEXT:    addl $92, %esp
-; FALLBACK27-NEXT:    popl %esi
-; FALLBACK27-NEXT:    popl %edi
-; FALLBACK27-NEXT:    popl %ebx
-; FALLBACK27-NEXT:    popl %ebp
-; FALLBACK27-NEXT:    vzeroupper
-; FALLBACK27-NEXT:    retl
-;
-; FALLBACK28-LABEL: shl_32bytes:
-; FALLBACK28:       # %bb.0:
-; FALLBACK28-NEXT:    pushl %ebp
-; FALLBACK28-NEXT:    pushl %ebx
-; FALLBACK28-NEXT:    pushl %edi
-; FALLBACK28-NEXT:    pushl %esi
-; FALLBACK28-NEXT:    subl $108, %esp
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK28-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK28-NEXT:    movzbl (%eax), %ecx
-; FALLBACK28-NEXT:    movb %cl, %dh
-; FALLBACK28-NEXT:    shlb $3, %dh
-; FALLBACK28-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK28-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    andb $28, %cl
-; FALLBACK28-NEXT:    negb %cl
-; FALLBACK28-NEXT:    movsbl %cl, %ebx
-; FALLBACK28-NEXT:    movl 84(%esp,%ebx), %edi
-; FALLBACK28-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %dh, %cl
-; FALLBACK28-NEXT:    shll %cl, %edi
-; FALLBACK28-NEXT:    movb %dh, %dl
-; FALLBACK28-NEXT:    notb %dl
-; FALLBACK28-NEXT:    movl 80(%esp,%ebx), %esi
-; FALLBACK28-NEXT:    movl %esi, %eax
-; FALLBACK28-NEXT:    shrl %eax
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %eax
-; FALLBACK28-NEXT:    orl %edi, %eax
-; FALLBACK28-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %dh, %cl
-; FALLBACK28-NEXT:    shll %cl, %esi
-; FALLBACK28-NEXT:    movl %ebx, %edi
-; FALLBACK28-NEXT:    movl 76(%esp,%ebx), %ebp
-; FALLBACK28-NEXT:    movl %ebp, %eax
-; FALLBACK28-NEXT:    shrl %eax
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %eax
-; FALLBACK28-NEXT:    orl %esi, %eax
-; FALLBACK28-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %dh, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebp
-; FALLBACK28-NEXT:    movl 72(%esp,%ebx), %ebx
-; FALLBACK28-NEXT:    movl %ebx, %eax
-; FALLBACK28-NEXT:    shrl %eax
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %eax
-; FALLBACK28-NEXT:    orl %ebp, %eax
-; FALLBACK28-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %dh, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 68(%esp,%edi), %ebp
-; FALLBACK28-NEXT:    movl %ebp, %esi
-; FALLBACK28-NEXT:    shrl %esi
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %esi
-; FALLBACK28-NEXT:    orl %ebx, %esi
-; FALLBACK28-NEXT:    movb %dh, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebp
-; FALLBACK28-NEXT:    movl 64(%esp,%edi), %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    shrl %ebx
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %ebx
-; FALLBACK28-NEXT:    orl %ebp, %ebx
-; FALLBACK28-NEXT:    movl 88(%esp,%edi), %ebp
-; FALLBACK28-NEXT:    movl %ebp, %edi
-; FALLBACK28-NEXT:    movb %dh, %cl
-; FALLBACK28-NEXT:    shll %cl, %edi
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %eax
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %eax
-; FALLBACK28-NEXT:    orl %edi, %eax
-; FALLBACK28-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK28-NEXT:    movl 92(%esp,%eax), %edi
-; FALLBACK28-NEXT:    movb %dh, %cl
-; FALLBACK28-NEXT:    shll %cl, %edi
-; FALLBACK28-NEXT:    shrl %ebp
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %ebp
-; FALLBACK28-NEXT:    orl %edi, %ebp
-; FALLBACK28-NEXT:    movb %dh, %cl
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK28-NEXT:    shll %cl, %edx
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK28-NEXT:    movl %edx, (%eax)
-; FALLBACK28-NEXT:    movl %ebp, 28(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK28-NEXT:    movl %ebx, 4(%eax)
-; FALLBACK28-NEXT:    movl %esi, 8(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK28-NEXT:    addl $108, %esp
-; FALLBACK28-NEXT:    popl %esi
-; FALLBACK28-NEXT:    popl %edi
-; FALLBACK28-NEXT:    popl %ebx
-; FALLBACK28-NEXT:    popl %ebp
-; FALLBACK28-NEXT:    vzeroupper
-; FALLBACK28-NEXT:    retl
-;
-; FALLBACK29-LABEL: shl_32bytes:
-; FALLBACK29:       # %bb.0:
-; FALLBACK29-NEXT:    pushl %ebp
-; FALLBACK29-NEXT:    pushl %ebx
-; FALLBACK29-NEXT:    pushl %edi
-; FALLBACK29-NEXT:    pushl %esi
-; FALLBACK29-NEXT:    subl $92, %esp
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK29-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK29-NEXT:    movzbl (%eax), %eax
-; FALLBACK29-NEXT:    movl %eax, %ecx
-; FALLBACK29-NEXT:    shlb $3, %cl
-; FALLBACK29-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK29-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    andb $28, %al
-; FALLBACK29-NEXT:    negb %al
-; FALLBACK29-NEXT:    movsbl %al, %ebp
-; FALLBACK29-NEXT:    movl 64(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl 68(%esp,%ebp), %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shldl %cl, %eax, %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 60(%esp,%ebp), %edx
-; FALLBACK29-NEXT:    shldl %cl, %edx, %eax
-; FALLBACK29-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 56(%esp,%ebp), %edi
-; FALLBACK29-NEXT:    shldl %cl, %edi, %edx
-; FALLBACK29-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 52(%esp,%ebp), %ebx
-; FALLBACK29-NEXT:    shldl %cl, %ebx, %edi
-; FALLBACK29-NEXT:    movl 72(%esp,%ebp), %edx
-; FALLBACK29-NEXT:    movl %edx, %eax
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK29-NEXT:    shldl %cl, %esi, %eax
-; FALLBACK29-NEXT:    movl 48(%esp,%ebp), %esi
-; FALLBACK29-NEXT:    movl 76(%esp,%ebp), %ebp
-; FALLBACK29-NEXT:    shldl %cl, %edx, %ebp
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK29-NEXT:    movl %ebp, 28(%edx)
-; FALLBACK29-NEXT:    movl %eax, 24(%edx)
-; FALLBACK29-NEXT:    movl %esi, %eax
-; FALLBACK29-NEXT:    shll %cl, %eax
-; FALLBACK29-NEXT:    shldl %cl, %esi, %ebx
-; FALLBACK29-NEXT:    movl %ebx, 4(%edx)
-; FALLBACK29-NEXT:    movl %edi, 8(%edx)
-; FALLBACK29-NEXT:    movl (%esp), %ecx # 4-byte Reload
-; FALLBACK29-NEXT:    movl %ecx, 12(%edx)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK29-NEXT:    movl %ecx, 16(%edx)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK29-NEXT:    movl %ecx, 20(%edx)
-; FALLBACK29-NEXT:    movl %eax, (%edx)
-; FALLBACK29-NEXT:    addl $92, %esp
-; FALLBACK29-NEXT:    popl %esi
-; FALLBACK29-NEXT:    popl %edi
-; FALLBACK29-NEXT:    popl %ebx
-; FALLBACK29-NEXT:    popl %ebp
-; FALLBACK29-NEXT:    vzeroupper
-; FALLBACK29-NEXT:    retl
-;
-; FALLBACK30-LABEL: shl_32bytes:
-; FALLBACK30:       # %bb.0:
-; FALLBACK30-NEXT:    pushl %ebp
-; FALLBACK30-NEXT:    pushl %ebx
-; FALLBACK30-NEXT:    pushl %edi
-; FALLBACK30-NEXT:    pushl %esi
-; FALLBACK30-NEXT:    subl $108, %esp
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK30-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK30-NEXT:    movzbl (%eax), %edx
-; FALLBACK30-NEXT:    movl %edx, %eax
-; FALLBACK30-NEXT:    shlb $3, %al
-; FALLBACK30-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK30-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %eax, %ebx
-; FALLBACK30-NEXT:    andb $28, %dl
-; FALLBACK30-NEXT:    negb %dl
-; FALLBACK30-NEXT:    movsbl %dl, %edx
-; FALLBACK30-NEXT:    movl 84(%esp,%edx), %ecx
-; FALLBACK30-NEXT:    shlxl %ebx, %ecx, %esi
-; FALLBACK30-NEXT:    notb %al
-; FALLBACK30-NEXT:    movl 80(%esp,%edx), %edi
-; FALLBACK30-NEXT:    shlxl %ebx, %edi, %ebp
-; FALLBACK30-NEXT:    shrl %edi
-; FALLBACK30-NEXT:    shrxl %eax, %edi, %edi
-; FALLBACK30-NEXT:    orl %esi, %edi
-; FALLBACK30-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 76(%esp,%edx), %esi
-; FALLBACK30-NEXT:    movl %esi, %edi
-; FALLBACK30-NEXT:    shrl %edi
-; FALLBACK30-NEXT:    shrxl %eax, %edi, %edi
-; FALLBACK30-NEXT:    orl %ebp, %edi
-; FALLBACK30-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shlxl %ebx, %esi, %esi
-; FALLBACK30-NEXT:    movl 72(%esp,%edx), %edi
-; FALLBACK30-NEXT:    movl %edi, %ebp
-; FALLBACK30-NEXT:    shrl %ebp
-; FALLBACK30-NEXT:    shrxl %eax, %ebp, %ebp
-; FALLBACK30-NEXT:    orl %esi, %ebp
-; FALLBACK30-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shlxl %ebx, %edi, %esi
-; FALLBACK30-NEXT:    movl 68(%esp,%edx), %ebp
-; FALLBACK30-NEXT:    movl %ebp, %edi
-; FALLBACK30-NEXT:    shrl %edi
-; FALLBACK30-NEXT:    shrxl %eax, %edi, %edi
-; FALLBACK30-NEXT:    orl %esi, %edi
-; FALLBACK30-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shlxl %ebx, %ebp, %ebp
-; FALLBACK30-NEXT:    movl 64(%esp,%edx), %esi
-; FALLBACK30-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrl %esi
-; FALLBACK30-NEXT:    shrxl %eax, %esi, %edi
-; FALLBACK30-NEXT:    orl %ebp, %edi
-; FALLBACK30-NEXT:    shrl %ecx
-; FALLBACK30-NEXT:    shrxl %eax, %ecx, %esi
-; FALLBACK30-NEXT:    movl 88(%esp,%edx), %ecx
-; FALLBACK30-NEXT:    shlxl %ebx, %ecx, %ebp
-; FALLBACK30-NEXT:    orl %ebp, %esi
-; FALLBACK30-NEXT:    shlxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK30-NEXT:    shlxl %ebx, 92(%esp,%edx), %edx
-; FALLBACK30-NEXT:    shrl %ecx
-; FALLBACK30-NEXT:    shrxl %eax, %ecx, %eax
-; FALLBACK30-NEXT:    orl %edx, %eax
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK30-NEXT:    movl %ebp, (%ecx)
-; FALLBACK30-NEXT:    movl %eax, 28(%ecx)
-; FALLBACK30-NEXT:    movl %esi, 24(%ecx)
-; FALLBACK30-NEXT:    movl %edi, 4(%ecx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 8(%ecx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 12(%ecx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 16(%ecx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 20(%ecx)
-; FALLBACK30-NEXT:    addl $108, %esp
-; FALLBACK30-NEXT:    popl %esi
-; FALLBACK30-NEXT:    popl %edi
-; FALLBACK30-NEXT:    popl %ebx
-; FALLBACK30-NEXT:    popl %ebp
-; FALLBACK30-NEXT:    vzeroupper
-; FALLBACK30-NEXT:    retl
-;
-; FALLBACK31-LABEL: shl_32bytes:
-; FALLBACK31:       # %bb.0:
-; FALLBACK31-NEXT:    pushl %ebp
-; FALLBACK31-NEXT:    pushl %ebx
-; FALLBACK31-NEXT:    pushl %edi
-; FALLBACK31-NEXT:    pushl %esi
-; FALLBACK31-NEXT:    subl $92, %esp
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK31-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK31-NEXT:    movzbl (%eax), %eax
-; FALLBACK31-NEXT:    movl %eax, %ecx
-; FALLBACK31-NEXT:    shlb $3, %cl
-; FALLBACK31-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK31-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    andb $28, %al
-; FALLBACK31-NEXT:    negb %al
-; FALLBACK31-NEXT:    movsbl %al, %ebx
-; FALLBACK31-NEXT:    movl 64(%esp,%ebx), %eax
-; FALLBACK31-NEXT:    movl 68(%esp,%ebx), %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shldl %cl, %eax, %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 60(%esp,%ebx), %edx
-; FALLBACK31-NEXT:    shldl %cl, %edx, %eax
-; FALLBACK31-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 56(%esp,%ebx), %edi
-; FALLBACK31-NEXT:    shldl %cl, %edi, %edx
-; FALLBACK31-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 52(%esp,%ebx), %ebp
-; FALLBACK31-NEXT:    shldl %cl, %ebp, %edi
-; FALLBACK31-NEXT:    movl 72(%esp,%ebx), %edx
-; FALLBACK31-NEXT:    movl %edx, %eax
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK31-NEXT:    shldl %cl, %esi, %eax
-; FALLBACK31-NEXT:    movl 48(%esp,%ebx), %esi
-; FALLBACK31-NEXT:    movl 76(%esp,%ebx), %ebx
-; FALLBACK31-NEXT:    shldl %cl, %edx, %ebx
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK31-NEXT:    movl %ebx, 28(%edx)
-; FALLBACK31-NEXT:    movl %eax, 24(%edx)
-; FALLBACK31-NEXT:    shlxl %ecx, %esi, %eax
-; FALLBACK31-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK31-NEXT:    shldl %cl, %esi, %ebp
-; FALLBACK31-NEXT:    movl %ebp, 4(%edx)
-; FALLBACK31-NEXT:    movl %edi, 8(%edx)
-; FALLBACK31-NEXT:    movl (%esp), %ecx # 4-byte Reload
-; FALLBACK31-NEXT:    movl %ecx, 12(%edx)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK31-NEXT:    movl %ecx, 16(%edx)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK31-NEXT:    movl %ecx, 20(%edx)
-; FALLBACK31-NEXT:    movl %eax, (%edx)
-; FALLBACK31-NEXT:    addl $92, %esp
-; FALLBACK31-NEXT:    popl %esi
-; FALLBACK31-NEXT:    popl %edi
-; FALLBACK31-NEXT:    popl %ebx
-; FALLBACK31-NEXT:    popl %ebp
-; FALLBACK31-NEXT:    vzeroupper
-; FALLBACK31-NEXT:    retl
+; X64-NO-SHLD-NO-BMI2-SSE2-LABEL: shl_32bytes:
+; X64-NO-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    andb $24, %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    negb %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movsbq %sil, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -32(%rsp,%r10), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -24(%rsp,%r10), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r11, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -8(%rsp,%r10), %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -16(%rsp,%r10), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rbx, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE2-LABEL: shl_32bytes:
+; X64-HAVE-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    leal (,%rsi,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andb $24, %sil
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    negb %sil
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movsbq %sil, %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -24(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -16(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldq %cl, %rsi, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -40(%rsp,%rax), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -32(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldq %cl, %r8, %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE2-LABEL: shl_32bytes:
+; X64-NO-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $24, %sil
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    negb %sil
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movsbq %sil, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -40(%rsp,%rdi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -32(%rsp,%rdi), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %rsi, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r8, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrq %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rax, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r9, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, -16(%rsp,%rdi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -24(%rsp,%rdi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %rdi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrq %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rax, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r9, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrq %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rax, %rsi, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %rcx, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: shl_32bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (,%rsi,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $24, %sil
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    negb %sil
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movsbq %sil, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -24(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -16(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldq %cl, %rsi, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -40(%rsp,%rax), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -32(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldq %cl, %r8, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r8, %rcx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-SSE4-LABEL: shl_32bytes:
+; X64-NO-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%rsi), %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (,%rcx,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    andb $24, %cl
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    negb %cl
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movsbq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -16(%rsp,%r8), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -24(%rsp,%r8), %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r9, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -40(%rsp,%r8), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -32(%rsp,%r8), %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r8, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r10, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r8, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE4-LABEL: shl_32bytes:
+; X64-HAVE-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%rsi), %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andb $24, %al
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    negb %al
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movsbq %al, %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -24(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -16(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldq %cl, %rsi, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -40(%rsp,%rax), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -32(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r8, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldq %cl, %r8, %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rax, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE4-LABEL: shl_32bytes:
+; X64-NO-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $24, %sil
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    negb %sil
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movsbq %sil, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, -16(%rsp,%rsi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -24(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r8, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrq %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rax, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rdi, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -40(%rsp,%rsi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -32(%rsp,%rsi), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %rsi, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrq %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rax, %rsi, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r9, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %rdi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrq %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rax, %rdi, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r10, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rcx, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rsi, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r8, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: shl_32bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $24, %al
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    negb %al
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movsbq %al, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -24(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -16(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldq %cl, %rsi, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -40(%rsp,%rax), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -32(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r8, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldq %cl, %r8, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-AVX-LABEL: shl_32bytes:
+; X64-NO-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%rsi), %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    leal (,%rcx,8), %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    andb $24, %cl
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    negb %cl
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movsbq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -16(%rsp,%r8), %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -24(%rsp,%r8), %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r9, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -40(%rsp,%r8), %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -32(%rsp,%r8), %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r8, %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r10, %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r9, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r8, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r9, (%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vzeroupper
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-AVX-LABEL: shl_32bytes:
+; X64-HAVE-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%rsi), %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    andb $24, %al
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    negb %al
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movsbq %al, %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -24(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -16(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shldq %cl, %rsi, %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -40(%rsp,%rax), %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -32(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r8, %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shldq %cl, %r8, %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rax, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vzeroupper
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-AVX-LABEL: shl_32bytes:
+; X64-NO-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    andb $24, %sil
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    negb %sil
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movsbq %sil, %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, -16(%rsp,%rsi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -24(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %r8, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrq %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rax, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %rdi, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -40(%rsp,%rsi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -32(%rsp,%rsi), %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %rsi, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrq %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rax, %rsi, %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %r9, %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %rdi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrq %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rax, %rdi, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %r10, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rcx, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rsi, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r8, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vzeroupper
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-LABEL: shl_32bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    andb $24, %al
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    negb %al
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movsbq %al, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -24(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -16(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shldq %cl, %rsi, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -40(%rsp,%rax), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -32(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %r8, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shldq %cl, %r8, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vzeroupper
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    retq
+;
+; X86-NO-SHLD-NO-BMI2-SSE2-LABEL: shl_32bytes:
+; X86-NO-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    subl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl (%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%ecx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb (%eax), %ah
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%ecx), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%ecx), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ah, %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlb $3, %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    andb $28, %ah
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    negb %ah
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movsbl %ah, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 64(%esp,%ebx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 68(%esp,%ebx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %dl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    notb %dl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 76(%esp,%ebx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 72(%esp,%ebx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 84(%esp,%ebx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 80(%esp,%ebx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %eax, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 92(%esp,%ebx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 88(%esp,%ebx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, (%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, 20(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-SSE2-LABEL: shl_32bytes:
+; X86-HAVE-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    subl $92, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%eax), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movb (%ecx), %ch
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%eax), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%eax), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andb $28, %ch
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    negb %ch
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movsbl %ch, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 56(%esp,%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 60(%esp,%eax), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %edx, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 52(%esp,%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 64(%esp,%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 68(%esp,%eax), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %edi, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %ebx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 48(%esp,%eax), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 72(%esp,%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 76(%esp,%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %edx, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, 24(%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, 28(%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, 16(%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, 20(%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, 8(%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, 12(%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%esp), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %ebx, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, (%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, 4(%eax)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    addl $92, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-SSE2-LABEL: shl_32bytes:
+; X86-NO-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    subl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%eax), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%ebx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 28(%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $3, %dl
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $28, %bl
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    negb %bl
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movsbl %bl, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 64(%esp,%esi), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 68(%esp,%esi), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %eax, %ecx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    notb %dl
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %edx, %ebx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 72(%esp,%esi), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 76(%esp,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ebp, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %esi, %ebx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %edx, %ecx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 80(%esp,%ebp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %edx, %ebx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 84(%esp,%ebp), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %esi, %ebx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %esi, %ecx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %edx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ecx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %esi, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %esi, 92(%esp,%ecx), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 88(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %eax, %esi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %edx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %edx, %ebx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, (%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 24(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, 28(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, 16(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 20(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 8(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 4(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: shl_32bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    subl $92, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%ecx), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%ecx), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%ecx), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%ebx), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%ecx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 28(%ecx), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $28, %bl
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    negb %bl
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movsbl %bl, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 56(%esp,%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 60(%esp,%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %edx, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 52(%esp,%eax), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %ebx, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 64(%esp,%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 68(%esp,%eax), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %edi, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%esp), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 48(%esp,%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 72(%esp,%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 76(%esp,%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %edx, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, 24(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, 28(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, 16(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, 20(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, 8(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, 12(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%esp), %esi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, (%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %esi, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, 4(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    addl $92, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-SSE4-LABEL: shl_32bytes:
+; X86-NO-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    subl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %cl, %dh
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlb $3, %dh
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    andb $28, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    negb %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movsbl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 84(%esp,%ebx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dh, %dl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    notb %dl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 80(%esp,%ebx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 76(%esp,%ebx), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 72(%esp,%ebx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 68(%esp,%edi), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 64(%esp,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 88(%esp,%edi), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 92(%esp,%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, (%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-SSE4-LABEL: shl_32bytes:
+; X86-HAVE-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    subl $92, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%eax), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andb $28, %al
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    negb %al
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movsbl %al, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 64(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 68(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 60(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %edx, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 56(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %edi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 52(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %ebx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 72(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %esi, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 48(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 76(%esp,%ebp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %edx, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, 28(%edx)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 24(%edx)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %esi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, 4(%edx)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, 8(%edx)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 12(%edx)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 16(%edx)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 20(%edx)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, (%edx)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    addl $92, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-SSE4-LABEL: shl_32bytes:
+; X86-NO-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    subl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $3, %cl
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $28, %dl
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    negb %dl
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movsbl %dl, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 84(%esp,%edx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebx, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    notb %cl
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 80(%esp,%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 76(%esp,%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 72(%esp,%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %ebp, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebx, %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 68(%esp,%edx), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebx, %ebp, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 64(%esp,%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 88(%esp,%edx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebx, %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebx, 92(%esp,%edx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, (%ecx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 28(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, 24(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, 4(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 8(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 12(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 16(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 20(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: shl_32bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    subl $92, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%eax), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $28, %al
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    negb %al
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movsbl %al, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 64(%esp,%ebx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 68(%esp,%ebx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 60(%esp,%ebx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %edx, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 56(%esp,%ebx), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %edi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 52(%esp,%ebx), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %ebp, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 72(%esp,%ebx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %esi, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 48(%esp,%ebx), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 76(%esp,%ebx), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %edx, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, 28(%edx)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 24(%edx)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ecx, %esi, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %esi, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, 4(%edx)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, 8(%edx)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 12(%edx)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 16(%edx)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 20(%edx)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, (%edx)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    addl $92, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-AVX-LABEL: shl_32bytes:
+; X86-NO-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    subl $108, %esp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%ecx), %ymm0
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %cl, %dh
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shlb $3, %dh
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    andb $28, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    negb %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movsbl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 84(%esp,%ebx), %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dh, %dl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    notb %dl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 80(%esp,%ebx), %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 76(%esp,%ebx), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 72(%esp,%ebx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 68(%esp,%edi), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebp, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 64(%esp,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 88(%esp,%edi), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebp, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 92(%esp,%eax), %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, (%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebp, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl $108, %esp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vzeroupper
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-AVX-LABEL: shl_32bytes:
+; X86-HAVE-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    subl $92, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%ecx), %ymm0
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%eax), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    andb $28, %al
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    negb %al
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movsbl %al, %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 64(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 68(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shldl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 60(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shldl %cl, %edx, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 56(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shldl %cl, %edi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 52(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shldl %cl, %ebx, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 72(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shldl %cl, %esi, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 48(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 76(%esp,%ebp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shldl %cl, %edx, %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %ebp, 28(%edx)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 24(%edx)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shldl %cl, %esi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, 4(%edx)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, 8(%edx)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 12(%edx)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 16(%edx)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 20(%edx)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, (%edx)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    addl $92, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vzeroupper
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-AVX-LABEL: shl_32bytes:
+; X86-NO-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    subl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%ecx), %ymm0
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $3, %al
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    andb $28, %dl
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    negb %dl
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movsbl %dl, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 84(%esp,%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ebx, %ecx, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    notb %al
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 80(%esp,%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %eax, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 76(%esp,%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %eax, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 72(%esp,%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %eax, %ebp, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ebx, %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 68(%esp,%edx), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %eax, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ebx, %ebp, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 64(%esp,%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %eax, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %eax, %ecx, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 88(%esp,%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ebx, %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %ebp, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ebx, 92(%esp,%edx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %eax, %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebp, (%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 28(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, 24(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, 4(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 8(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 12(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 16(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 20(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vzeroupper
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-LABEL: shl_32bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    subl $92, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%ecx), %ymm0
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%eax), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    andb $28, %al
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    negb %al
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movsbl %al, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 64(%esp,%ebx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 68(%esp,%ebx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shldl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 60(%esp,%ebx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shldl %cl, %edx, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 56(%esp,%ebx), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shldl %cl, %edi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 52(%esp,%ebx), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shldl %cl, %ebp, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 72(%esp,%ebx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shldl %cl, %esi, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 48(%esp,%ebx), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 76(%esp,%ebx), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shldl %cl, %edx, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebx, 28(%edx)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 24(%edx)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ecx, %esi, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shldl %cl, %esi, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebp, 4(%edx)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, 8(%edx)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 12(%edx)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 16(%edx)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 20(%edx)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, (%edx)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    addl $92, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vzeroupper
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    retl
   %src = load i256, ptr %src.ptr, align 1
   %byteOff = load i256, ptr %byteOff.ptr, align 1
   %bitOff = shl i256 %byteOff, 3
@@ -7869,617 +6434,472 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 }
 
 define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nounwind {
-; FALLBACK0-LABEL: shl_32bytes_dwordOff:
-; FALLBACK0:       # %bb.0:
-; FALLBACK0-NEXT:    pushq %rbx
-; FALLBACK0-NEXT:    movq (%rdi), %rcx
-; FALLBACK0-NEXT:    movq 8(%rdi), %r8
-; FALLBACK0-NEXT:    movq 16(%rdi), %r9
-; FALLBACK0-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK0-NEXT:    movzbl (%rsi), %esi
-; FALLBACK0-NEXT:    movl %esi, %eax
-; FALLBACK0-NEXT:    shlb $5, %al
-; FALLBACK0-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK0-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    shlb $2, %sil
-; FALLBACK0-NEXT:    andb $24, %sil
-; FALLBACK0-NEXT:    negb %sil
-; FALLBACK0-NEXT:    movsbq %sil, %r10
-; FALLBACK0-NEXT:    movq -32(%rsp,%r10), %r8
-; FALLBACK0-NEXT:    movq -24(%rsp,%r10), %rdi
-; FALLBACK0-NEXT:    movq %rdi, %r11
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r11
-; FALLBACK0-NEXT:    movl %eax, %esi
-; FALLBACK0-NEXT:    notb %sil
-; FALLBACK0-NEXT:    movq %r8, %r9
-; FALLBACK0-NEXT:    shrq %r9
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r9
-; FALLBACK0-NEXT:    orq %r11, %r9
-; FALLBACK0-NEXT:    movq -8(%rsp,%r10), %r11
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r11
-; FALLBACK0-NEXT:    movq -16(%rsp,%r10), %r10
-; FALLBACK0-NEXT:    movq %r10, %rbx
-; FALLBACK0-NEXT:    shrq %rbx
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %rbx
-; FALLBACK0-NEXT:    orq %r11, %rbx
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r10
-; FALLBACK0-NEXT:    shrq %rdi
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %rdi
-; FALLBACK0-NEXT:    orq %r10, %rdi
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r8
-; FALLBACK0-NEXT:    movq %r8, (%rdx)
-; FALLBACK0-NEXT:    movq %rdi, 16(%rdx)
-; FALLBACK0-NEXT:    movq %rbx, 24(%rdx)
-; FALLBACK0-NEXT:    movq %r9, 8(%rdx)
-; FALLBACK0-NEXT:    popq %rbx
-; FALLBACK0-NEXT:    retq
-;
-; FALLBACK1-LABEL: shl_32bytes_dwordOff:
-; FALLBACK1:       # %bb.0:
-; FALLBACK1-NEXT:    movq (%rdi), %rax
-; FALLBACK1-NEXT:    movq 8(%rdi), %r8
-; FALLBACK1-NEXT:    movq 16(%rdi), %r9
-; FALLBACK1-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK1-NEXT:    movzbl (%rsi), %esi
-; FALLBACK1-NEXT:    movl %esi, %ecx
-; FALLBACK1-NEXT:    shlb $5, %cl
-; FALLBACK1-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK1-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    shlb $2, %sil
-; FALLBACK1-NEXT:    andb $24, %sil
-; FALLBACK1-NEXT:    negb %sil
-; FALLBACK1-NEXT:    movsbq %sil, %rax
-; FALLBACK1-NEXT:    movq -24(%rsp,%rax), %rsi
-; FALLBACK1-NEXT:    movq -16(%rsp,%rax), %rdi
-; FALLBACK1-NEXT:    shldq %cl, %rsi, %rdi
-; FALLBACK1-NEXT:    movq -40(%rsp,%rax), %r8
-; FALLBACK1-NEXT:    movq -32(%rsp,%rax), %rax
-; FALLBACK1-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK1-NEXT:    shldq %cl, %r8, %rax
-; FALLBACK1-NEXT:    shlq %cl, %r8
-; FALLBACK1-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK1-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK1-NEXT:    movq %r8, (%rdx)
-; FALLBACK1-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK1-NEXT:    retq
-;
-; FALLBACK2-LABEL: shl_32bytes_dwordOff:
-; FALLBACK2:       # %bb.0:
-; FALLBACK2-NEXT:    movq (%rdi), %rcx
-; FALLBACK2-NEXT:    movq 8(%rdi), %r8
-; FALLBACK2-NEXT:    movq 16(%rdi), %r9
-; FALLBACK2-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK2-NEXT:    movzbl (%rsi), %esi
-; FALLBACK2-NEXT:    movl %esi, %eax
-; FALLBACK2-NEXT:    shlb $5, %al
-; FALLBACK2-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movl %eax, %ecx
-; FALLBACK2-NEXT:    shlb $2, %sil
-; FALLBACK2-NEXT:    andb $24, %sil
-; FALLBACK2-NEXT:    negb %sil
-; FALLBACK2-NEXT:    movsbq %sil, %rdi
-; FALLBACK2-NEXT:    movq -40(%rsp,%rdi), %r8
-; FALLBACK2-NEXT:    movq -32(%rsp,%rdi), %rsi
-; FALLBACK2-NEXT:    shlxq %rcx, %rsi, %r9
-; FALLBACK2-NEXT:    notb %al
-; FALLBACK2-NEXT:    shlxq %rcx, %r8, %r10
-; FALLBACK2-NEXT:    shrq %r8
-; FALLBACK2-NEXT:    shrxq %rax, %r8, %r8
-; FALLBACK2-NEXT:    orq %r9, %r8
-; FALLBACK2-NEXT:    shlxq %rcx, -16(%rsp,%rdi), %r9
-; FALLBACK2-NEXT:    movq -24(%rsp,%rdi), %rdi
-; FALLBACK2-NEXT:    shlxq %rcx, %rdi, %rcx
-; FALLBACK2-NEXT:    shrq %rdi
-; FALLBACK2-NEXT:    shrxq %rax, %rdi, %rdi
-; FALLBACK2-NEXT:    orq %r9, %rdi
-; FALLBACK2-NEXT:    shrq %rsi
-; FALLBACK2-NEXT:    shrxq %rax, %rsi, %rax
-; FALLBACK2-NEXT:    orq %rcx, %rax
-; FALLBACK2-NEXT:    movq %r10, (%rdx)
-; FALLBACK2-NEXT:    movq %rax, 16(%rdx)
-; FALLBACK2-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK2-NEXT:    movq %r8, 8(%rdx)
-; FALLBACK2-NEXT:    retq
-;
-; FALLBACK3-LABEL: shl_32bytes_dwordOff:
-; FALLBACK3:       # %bb.0:
-; FALLBACK3-NEXT:    movq (%rdi), %rax
-; FALLBACK3-NEXT:    movq 8(%rdi), %r8
-; FALLBACK3-NEXT:    movq 16(%rdi), %r9
-; FALLBACK3-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK3-NEXT:    movzbl (%rsi), %esi
-; FALLBACK3-NEXT:    movl %esi, %ecx
-; FALLBACK3-NEXT:    shlb $5, %cl
-; FALLBACK3-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    shlb $2, %sil
-; FALLBACK3-NEXT:    andb $24, %sil
-; FALLBACK3-NEXT:    negb %sil
-; FALLBACK3-NEXT:    movsbq %sil, %rax
-; FALLBACK3-NEXT:    movq -24(%rsp,%rax), %rsi
-; FALLBACK3-NEXT:    movq -16(%rsp,%rax), %rdi
-; FALLBACK3-NEXT:    shldq %cl, %rsi, %rdi
-; FALLBACK3-NEXT:    movq -40(%rsp,%rax), %r8
-; FALLBACK3-NEXT:    movq -32(%rsp,%rax), %rax
-; FALLBACK3-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK3-NEXT:    shldq %cl, %r8, %rax
-; FALLBACK3-NEXT:    shlxq %rcx, %r8, %rcx
-; FALLBACK3-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK3-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK3-NEXT:    movq %rcx, (%rdx)
-; FALLBACK3-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK3-NEXT:    retq
-;
-; FALLBACK4-LABEL: shl_32bytes_dwordOff:
-; FALLBACK4:       # %bb.0:
-; FALLBACK4-NEXT:    movups (%rdi), %xmm0
-; FALLBACK4-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK4-NEXT:    movzbl (%rsi), %ecx
-; FALLBACK4-NEXT:    movl %ecx, %eax
-; FALLBACK4-NEXT:    shlb $5, %al
-; FALLBACK4-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    shlb $2, %cl
-; FALLBACK4-NEXT:    andb $24, %cl
-; FALLBACK4-NEXT:    negb %cl
-; FALLBACK4-NEXT:    movsbq %cl, %r8
-; FALLBACK4-NEXT:    movq -16(%rsp,%r8), %r9
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r9
-; FALLBACK4-NEXT:    movl %eax, %esi
-; FALLBACK4-NEXT:    notb %sil
-; FALLBACK4-NEXT:    movq -24(%rsp,%r8), %r10
-; FALLBACK4-NEXT:    movq %r10, %rdi
-; FALLBACK4-NEXT:    shrq %rdi
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %rdi
-; FALLBACK4-NEXT:    orq %r9, %rdi
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r10
-; FALLBACK4-NEXT:    movq -40(%rsp,%r8), %r9
-; FALLBACK4-NEXT:    movq -32(%rsp,%r8), %r8
-; FALLBACK4-NEXT:    movq %r8, %r11
-; FALLBACK4-NEXT:    shrq %r11
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r11
-; FALLBACK4-NEXT:    orq %r10, %r11
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r8
-; FALLBACK4-NEXT:    movq %r9, %r10
-; FALLBACK4-NEXT:    shrq %r10
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r10
-; FALLBACK4-NEXT:    orq %r8, %r10
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r9
-; FALLBACK4-NEXT:    movq %r9, (%rdx)
-; FALLBACK4-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK4-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK4-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK4-NEXT:    retq
-;
-; FALLBACK5-LABEL: shl_32bytes_dwordOff:
-; FALLBACK5:       # %bb.0:
-; FALLBACK5-NEXT:    movups (%rdi), %xmm0
-; FALLBACK5-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK5-NEXT:    movzbl (%rsi), %eax
-; FALLBACK5-NEXT:    movl %eax, %ecx
-; FALLBACK5-NEXT:    shlb $5, %cl
-; FALLBACK5-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK5-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    shlb $2, %al
-; FALLBACK5-NEXT:    andb $24, %al
-; FALLBACK5-NEXT:    negb %al
-; FALLBACK5-NEXT:    movsbq %al, %rax
-; FALLBACK5-NEXT:    movq -24(%rsp,%rax), %rsi
-; FALLBACK5-NEXT:    movq -16(%rsp,%rax), %rdi
-; FALLBACK5-NEXT:    shldq %cl, %rsi, %rdi
-; FALLBACK5-NEXT:    movq -40(%rsp,%rax), %r8
-; FALLBACK5-NEXT:    movq -32(%rsp,%rax), %rax
-; FALLBACK5-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK5-NEXT:    movq %r8, %r9
-; FALLBACK5-NEXT:    shlq %cl, %r9
-; FALLBACK5-NEXT:    shldq %cl, %r8, %rax
-; FALLBACK5-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK5-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK5-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK5-NEXT:    movq %r9, (%rdx)
-; FALLBACK5-NEXT:    retq
-;
-; FALLBACK6-LABEL: shl_32bytes_dwordOff:
-; FALLBACK6:       # %bb.0:
-; FALLBACK6-NEXT:    movups (%rdi), %xmm0
-; FALLBACK6-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK6-NEXT:    movzbl (%rsi), %esi
-; FALLBACK6-NEXT:    movl %esi, %eax
-; FALLBACK6-NEXT:    shlb $5, %al
-; FALLBACK6-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK6-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movl %eax, %ecx
-; FALLBACK6-NEXT:    shlb $2, %sil
-; FALLBACK6-NEXT:    andb $24, %sil
-; FALLBACK6-NEXT:    negb %sil
-; FALLBACK6-NEXT:    movsbq %sil, %rsi
-; FALLBACK6-NEXT:    shlxq %rcx, -16(%rsp,%rsi), %rdi
-; FALLBACK6-NEXT:    notb %al
-; FALLBACK6-NEXT:    movq -24(%rsp,%rsi), %r8
-; FALLBACK6-NEXT:    shlxq %rcx, %r8, %r9
-; FALLBACK6-NEXT:    shrq %r8
-; FALLBACK6-NEXT:    shrxq %rax, %r8, %r8
-; FALLBACK6-NEXT:    orq %rdi, %r8
-; FALLBACK6-NEXT:    movq -40(%rsp,%rsi), %rdi
-; FALLBACK6-NEXT:    movq -32(%rsp,%rsi), %rsi
-; FALLBACK6-NEXT:    shlxq %rcx, %rsi, %r10
-; FALLBACK6-NEXT:    shrq %rsi
-; FALLBACK6-NEXT:    shrxq %rax, %rsi, %rsi
-; FALLBACK6-NEXT:    orq %r9, %rsi
-; FALLBACK6-NEXT:    shlxq %rcx, %rdi, %rcx
-; FALLBACK6-NEXT:    shrq %rdi
-; FALLBACK6-NEXT:    shrxq %rax, %rdi, %rax
-; FALLBACK6-NEXT:    orq %r10, %rax
-; FALLBACK6-NEXT:    movq %rcx, (%rdx)
-; FALLBACK6-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK6-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK6-NEXT:    movq %r8, 24(%rdx)
-; FALLBACK6-NEXT:    retq
-;
-; FALLBACK7-LABEL: shl_32bytes_dwordOff:
-; FALLBACK7:       # %bb.0:
-; FALLBACK7-NEXT:    movups (%rdi), %xmm0
-; FALLBACK7-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK7-NEXT:    movzbl (%rsi), %eax
-; FALLBACK7-NEXT:    movl %eax, %ecx
-; FALLBACK7-NEXT:    shlb $5, %cl
-; FALLBACK7-NEXT:    xorps %xmm2, %xmm2
-; FALLBACK7-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    shlb $2, %al
-; FALLBACK7-NEXT:    andb $24, %al
-; FALLBACK7-NEXT:    negb %al
-; FALLBACK7-NEXT:    movsbq %al, %rax
-; FALLBACK7-NEXT:    movq -24(%rsp,%rax), %rsi
-; FALLBACK7-NEXT:    movq -16(%rsp,%rax), %rdi
-; FALLBACK7-NEXT:    shldq %cl, %rsi, %rdi
-; FALLBACK7-NEXT:    movq -40(%rsp,%rax), %r8
-; FALLBACK7-NEXT:    movq -32(%rsp,%rax), %rax
-; FALLBACK7-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK7-NEXT:    shlxq %rcx, %r8, %r9
-; FALLBACK7-NEXT:    # kill: def $cl killed $cl killed $rcx
-; FALLBACK7-NEXT:    shldq %cl, %r8, %rax
-; FALLBACK7-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK7-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK7-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK7-NEXT:    movq %r9, (%rdx)
-; FALLBACK7-NEXT:    retq
-;
-; FALLBACK8-LABEL: shl_32bytes_dwordOff:
-; FALLBACK8:       # %bb.0:
-; FALLBACK8-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK8-NEXT:    movzbl (%rsi), %ecx
-; FALLBACK8-NEXT:    movl %ecx, %eax
-; FALLBACK8-NEXT:    shlb $5, %al
-; FALLBACK8-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK8-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    shlb $2, %cl
-; FALLBACK8-NEXT:    andb $24, %cl
-; FALLBACK8-NEXT:    negb %cl
-; FALLBACK8-NEXT:    movsbq %cl, %r8
-; FALLBACK8-NEXT:    movq -16(%rsp,%r8), %r9
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r9
-; FALLBACK8-NEXT:    movl %eax, %esi
-; FALLBACK8-NEXT:    notb %sil
-; FALLBACK8-NEXT:    movq -24(%rsp,%r8), %r10
-; FALLBACK8-NEXT:    movq %r10, %rdi
-; FALLBACK8-NEXT:    shrq %rdi
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %rdi
-; FALLBACK8-NEXT:    orq %r9, %rdi
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r10
-; FALLBACK8-NEXT:    movq -40(%rsp,%r8), %r9
-; FALLBACK8-NEXT:    movq -32(%rsp,%r8), %r8
-; FALLBACK8-NEXT:    movq %r8, %r11
-; FALLBACK8-NEXT:    shrq %r11
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r11
-; FALLBACK8-NEXT:    orq %r10, %r11
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r8
-; FALLBACK8-NEXT:    movq %r9, %r10
-; FALLBACK8-NEXT:    shrq %r10
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r10
-; FALLBACK8-NEXT:    orq %r8, %r10
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r9
-; FALLBACK8-NEXT:    movq %r9, (%rdx)
-; FALLBACK8-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK8-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK8-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK8-NEXT:    vzeroupper
-; FALLBACK8-NEXT:    retq
-;
-; FALLBACK9-LABEL: shl_32bytes_dwordOff:
-; FALLBACK9:       # %bb.0:
-; FALLBACK9-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK9-NEXT:    movzbl (%rsi), %eax
-; FALLBACK9-NEXT:    movl %eax, %ecx
-; FALLBACK9-NEXT:    shlb $5, %cl
-; FALLBACK9-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK9-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    shlb $2, %al
-; FALLBACK9-NEXT:    andb $24, %al
-; FALLBACK9-NEXT:    negb %al
-; FALLBACK9-NEXT:    movsbq %al, %rax
-; FALLBACK9-NEXT:    movq -24(%rsp,%rax), %rsi
-; FALLBACK9-NEXT:    movq -16(%rsp,%rax), %rdi
-; FALLBACK9-NEXT:    shldq %cl, %rsi, %rdi
-; FALLBACK9-NEXT:    movq -40(%rsp,%rax), %r8
-; FALLBACK9-NEXT:    movq -32(%rsp,%rax), %rax
-; FALLBACK9-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK9-NEXT:    movq %r8, %r9
-; FALLBACK9-NEXT:    shlq %cl, %r9
-; FALLBACK9-NEXT:    shldq %cl, %r8, %rax
-; FALLBACK9-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK9-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK9-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK9-NEXT:    movq %r9, (%rdx)
-; FALLBACK9-NEXT:    vzeroupper
-; FALLBACK9-NEXT:    retq
-;
-; FALLBACK10-LABEL: shl_32bytes_dwordOff:
-; FALLBACK10:       # %bb.0:
-; FALLBACK10-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK10-NEXT:    movzbl (%rsi), %esi
-; FALLBACK10-NEXT:    movl %esi, %eax
-; FALLBACK10-NEXT:    shlb $5, %al
-; FALLBACK10-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK10-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movl %eax, %ecx
-; FALLBACK10-NEXT:    shlb $2, %sil
-; FALLBACK10-NEXT:    andb $24, %sil
-; FALLBACK10-NEXT:    negb %sil
-; FALLBACK10-NEXT:    movsbq %sil, %rsi
-; FALLBACK10-NEXT:    shlxq %rcx, -16(%rsp,%rsi), %rdi
-; FALLBACK10-NEXT:    notb %al
-; FALLBACK10-NEXT:    movq -24(%rsp,%rsi), %r8
-; FALLBACK10-NEXT:    shlxq %rcx, %r8, %r9
-; FALLBACK10-NEXT:    shrq %r8
-; FALLBACK10-NEXT:    shrxq %rax, %r8, %r8
-; FALLBACK10-NEXT:    orq %rdi, %r8
-; FALLBACK10-NEXT:    movq -40(%rsp,%rsi), %rdi
-; FALLBACK10-NEXT:    movq -32(%rsp,%rsi), %rsi
-; FALLBACK10-NEXT:    shlxq %rcx, %rsi, %r10
-; FALLBACK10-NEXT:    shrq %rsi
-; FALLBACK10-NEXT:    shrxq %rax, %rsi, %rsi
-; FALLBACK10-NEXT:    orq %r9, %rsi
-; FALLBACK10-NEXT:    shlxq %rcx, %rdi, %rcx
-; FALLBACK10-NEXT:    shrq %rdi
-; FALLBACK10-NEXT:    shrxq %rax, %rdi, %rax
-; FALLBACK10-NEXT:    orq %r10, %rax
-; FALLBACK10-NEXT:    movq %rcx, (%rdx)
-; FALLBACK10-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK10-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK10-NEXT:    movq %r8, 24(%rdx)
-; FALLBACK10-NEXT:    vzeroupper
-; FALLBACK10-NEXT:    retq
-;
-; FALLBACK11-LABEL: shl_32bytes_dwordOff:
-; FALLBACK11:       # %bb.0:
-; FALLBACK11-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK11-NEXT:    movzbl (%rsi), %eax
-; FALLBACK11-NEXT:    movl %eax, %ecx
-; FALLBACK11-NEXT:    shlb $5, %cl
-; FALLBACK11-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK11-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    shlb $2, %al
-; FALLBACK11-NEXT:    andb $24, %al
-; FALLBACK11-NEXT:    negb %al
-; FALLBACK11-NEXT:    movsbq %al, %rax
-; FALLBACK11-NEXT:    movq -24(%rsp,%rax), %rsi
-; FALLBACK11-NEXT:    movq -16(%rsp,%rax), %rdi
-; FALLBACK11-NEXT:    shldq %cl, %rsi, %rdi
-; FALLBACK11-NEXT:    movq -40(%rsp,%rax), %r8
-; FALLBACK11-NEXT:    movq -32(%rsp,%rax), %rax
-; FALLBACK11-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK11-NEXT:    shlxq %rcx, %r8, %r9
-; FALLBACK11-NEXT:    # kill: def $cl killed $cl killed $rcx
-; FALLBACK11-NEXT:    shldq %cl, %r8, %rax
-; FALLBACK11-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK11-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK11-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK11-NEXT:    movq %r9, (%rdx)
-; FALLBACK11-NEXT:    vzeroupper
-; FALLBACK11-NEXT:    retq
-;
-; FALLBACK12-LABEL: shl_32bytes_dwordOff:
-; FALLBACK12:       # %bb.0:
-; FALLBACK12-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK12-NEXT:    movzbl (%rsi), %ecx
-; FALLBACK12-NEXT:    movl %ecx, %eax
-; FALLBACK12-NEXT:    shlb $5, %al
-; FALLBACK12-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK12-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    shlb $2, %cl
-; FALLBACK12-NEXT:    andb $24, %cl
-; FALLBACK12-NEXT:    negb %cl
-; FALLBACK12-NEXT:    movsbq %cl, %r8
-; FALLBACK12-NEXT:    movq -16(%rsp,%r8), %r9
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r9
-; FALLBACK12-NEXT:    movl %eax, %esi
-; FALLBACK12-NEXT:    notb %sil
-; FALLBACK12-NEXT:    movq -24(%rsp,%r8), %r10
-; FALLBACK12-NEXT:    movq %r10, %rdi
-; FALLBACK12-NEXT:    shrq %rdi
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %rdi
-; FALLBACK12-NEXT:    orq %r9, %rdi
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r10
-; FALLBACK12-NEXT:    movq -40(%rsp,%r8), %r9
-; FALLBACK12-NEXT:    movq -32(%rsp,%r8), %r8
-; FALLBACK12-NEXT:    movq %r8, %r11
-; FALLBACK12-NEXT:    shrq %r11
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r11
-; FALLBACK12-NEXT:    orq %r10, %r11
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r8
-; FALLBACK12-NEXT:    movq %r9, %r10
-; FALLBACK12-NEXT:    shrq %r10
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r10
-; FALLBACK12-NEXT:    orq %r8, %r10
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r9
-; FALLBACK12-NEXT:    movq %r9, (%rdx)
-; FALLBACK12-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK12-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK12-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK12-NEXT:    vzeroupper
-; FALLBACK12-NEXT:    retq
-;
-; FALLBACK13-LABEL: shl_32bytes_dwordOff:
-; FALLBACK13:       # %bb.0:
-; FALLBACK13-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK13-NEXT:    movzbl (%rsi), %eax
-; FALLBACK13-NEXT:    movl %eax, %ecx
-; FALLBACK13-NEXT:    shlb $5, %cl
-; FALLBACK13-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK13-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    shlb $2, %al
-; FALLBACK13-NEXT:    andb $24, %al
-; FALLBACK13-NEXT:    negb %al
-; FALLBACK13-NEXT:    movsbq %al, %rax
-; FALLBACK13-NEXT:    movq -24(%rsp,%rax), %rsi
-; FALLBACK13-NEXT:    movq -16(%rsp,%rax), %rdi
-; FALLBACK13-NEXT:    shldq %cl, %rsi, %rdi
-; FALLBACK13-NEXT:    movq -40(%rsp,%rax), %r8
-; FALLBACK13-NEXT:    movq -32(%rsp,%rax), %rax
-; FALLBACK13-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK13-NEXT:    movq %r8, %r9
-; FALLBACK13-NEXT:    shlq %cl, %r9
-; FALLBACK13-NEXT:    shldq %cl, %r8, %rax
-; FALLBACK13-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK13-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK13-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK13-NEXT:    movq %r9, (%rdx)
-; FALLBACK13-NEXT:    vzeroupper
-; FALLBACK13-NEXT:    retq
-;
-; FALLBACK14-LABEL: shl_32bytes_dwordOff:
-; FALLBACK14:       # %bb.0:
-; FALLBACK14-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK14-NEXT:    movzbl (%rsi), %esi
-; FALLBACK14-NEXT:    movl %esi, %eax
-; FALLBACK14-NEXT:    shlb $5, %al
-; FALLBACK14-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK14-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movl %eax, %ecx
-; FALLBACK14-NEXT:    shlb $2, %sil
-; FALLBACK14-NEXT:    andb $24, %sil
-; FALLBACK14-NEXT:    negb %sil
-; FALLBACK14-NEXT:    movsbq %sil, %rsi
-; FALLBACK14-NEXT:    shlxq %rcx, -16(%rsp,%rsi), %rdi
-; FALLBACK14-NEXT:    notb %al
-; FALLBACK14-NEXT:    movq -24(%rsp,%rsi), %r8
-; FALLBACK14-NEXT:    shlxq %rcx, %r8, %r9
-; FALLBACK14-NEXT:    shrq %r8
-; FALLBACK14-NEXT:    shrxq %rax, %r8, %r8
-; FALLBACK14-NEXT:    orq %rdi, %r8
-; FALLBACK14-NEXT:    movq -40(%rsp,%rsi), %rdi
-; FALLBACK14-NEXT:    movq -32(%rsp,%rsi), %rsi
-; FALLBACK14-NEXT:    shlxq %rcx, %rsi, %r10
-; FALLBACK14-NEXT:    shrq %rsi
-; FALLBACK14-NEXT:    shrxq %rax, %rsi, %rsi
-; FALLBACK14-NEXT:    orq %r9, %rsi
-; FALLBACK14-NEXT:    shlxq %rcx, %rdi, %rcx
-; FALLBACK14-NEXT:    shrq %rdi
-; FALLBACK14-NEXT:    shrxq %rax, %rdi, %rax
-; FALLBACK14-NEXT:    orq %r10, %rax
-; FALLBACK14-NEXT:    movq %rcx, (%rdx)
-; FALLBACK14-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK14-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK14-NEXT:    movq %r8, 24(%rdx)
-; FALLBACK14-NEXT:    vzeroupper
-; FALLBACK14-NEXT:    retq
-;
-; FALLBACK15-LABEL: shl_32bytes_dwordOff:
-; FALLBACK15:       # %bb.0:
-; FALLBACK15-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK15-NEXT:    movzbl (%rsi), %eax
-; FALLBACK15-NEXT:    movl %eax, %ecx
-; FALLBACK15-NEXT:    shlb $5, %cl
-; FALLBACK15-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK15-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    shlb $2, %al
-; FALLBACK15-NEXT:    andb $24, %al
-; FALLBACK15-NEXT:    negb %al
-; FALLBACK15-NEXT:    movsbq %al, %rax
-; FALLBACK15-NEXT:    movq -24(%rsp,%rax), %rsi
-; FALLBACK15-NEXT:    movq -16(%rsp,%rax), %rdi
-; FALLBACK15-NEXT:    shldq %cl, %rsi, %rdi
-; FALLBACK15-NEXT:    movq -40(%rsp,%rax), %r8
-; FALLBACK15-NEXT:    movq -32(%rsp,%rax), %rax
-; FALLBACK15-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK15-NEXT:    shlxq %rcx, %r8, %r9
-; FALLBACK15-NEXT:    # kill: def $cl killed $cl killed $rcx
-; FALLBACK15-NEXT:    shldq %cl, %r8, %rax
-; FALLBACK15-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK15-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK15-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK15-NEXT:    movq %r9, (%rdx)
-; FALLBACK15-NEXT:    vzeroupper
-; FALLBACK15-NEXT:    retq
+; X64-NO-SHLD-NO-BMI2-SSE2-LABEL: shl_32bytes_dwordOff:
+; X64-NO-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlb $2, %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    andb $24, %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    negb %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movsbq %sil, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -32(%rsp,%r10), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -24(%rsp,%r10), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r11, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -8(%rsp,%r10), %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -16(%rsp,%r10), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rbx, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE2-LABEL: shl_32bytes_dwordOff:
+; X64-HAVE-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shlb $2, %sil
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andb $24, %sil
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    negb %sil
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movsbq %sil, %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -24(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -16(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldq %cl, %rsi, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -40(%rsp,%rax), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -32(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldq %cl, %r8, %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE2-LABEL: shl_32bytes_dwordOff:
+; X64-NO-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $5, %al
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $2, %sil
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $24, %sil
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    negb %sil
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movsbq %sil, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -40(%rsp,%rdi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -32(%rsp,%rdi), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %rsi, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r8, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrq %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rax, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r9, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, -16(%rsp,%rdi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -24(%rsp,%rdi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %rdi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrq %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rax, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r9, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrq %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rax, %rsi, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %rcx, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: shl_32bytes_dwordOff:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $2, %sil
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $24, %sil
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    negb %sil
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movsbq %sil, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -24(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -16(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldq %cl, %rsi, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -40(%rsp,%rax), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -32(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldq %cl, %r8, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r8, %rcx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-SSE4-LABEL: shl_32bytes_dwordOff:
+; X64-NO-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%rsi), %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlb $2, %cl
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    andb $24, %cl
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    negb %cl
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movsbq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -16(%rsp,%r8), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -24(%rsp,%r8), %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r9, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -40(%rsp,%r8), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -32(%rsp,%r8), %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r8, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r10, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r8, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE4-LABEL: shl_32bytes_dwordOff:
+; X64-HAVE-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%rsi), %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shlb $2, %al
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andb $24, %al
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    negb %al
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movsbq %al, %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -24(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -16(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldq %cl, %rsi, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -40(%rsp,%rax), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -32(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r8, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldq %cl, %r8, %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rax, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE4-LABEL: shl_32bytes_dwordOff:
+; X64-NO-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $5, %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $2, %sil
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $24, %sil
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    negb %sil
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movsbq %sil, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, -16(%rsp,%rsi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -24(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r8, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrq %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rax, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rdi, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -40(%rsp,%rsi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -32(%rsp,%rsi), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %rsi, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrq %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rax, %rsi, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r9, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %rdi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrq %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rax, %rdi, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r10, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rcx, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rsi, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r8, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: shl_32bytes_dwordOff:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm2, %xmm2
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $2, %al
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $24, %al
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    negb %al
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movsbq %al, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -24(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -16(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldq %cl, %rsi, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -40(%rsp,%rax), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -32(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r8, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldq %cl, %r8, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-AVX-LABEL: shl_32bytes_dwordOff:
+; X64-NO-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%rsi), %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlb $2, %cl
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    andb $24, %cl
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    negb %cl
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movsbq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -16(%rsp,%r8), %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -24(%rsp,%r8), %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r9, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -40(%rsp,%r8), %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -32(%rsp,%r8), %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r8, %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r10, %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r9, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r8, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r9, (%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vzeroupper
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-AVX-LABEL: shl_32bytes_dwordOff:
+; X64-HAVE-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%rsi), %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shlb $2, %al
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    andb $24, %al
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    negb %al
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movsbq %al, %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -24(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -16(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shldq %cl, %rsi, %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -40(%rsp,%rax), %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -32(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r8, %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shldq %cl, %r8, %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rax, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vzeroupper
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-AVX-LABEL: shl_32bytes_dwordOff:
+; X64-NO-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $5, %al
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $2, %sil
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    andb $24, %sil
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    negb %sil
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movsbq %sil, %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, -16(%rsp,%rsi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -24(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %r8, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrq %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rax, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %rdi, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -40(%rsp,%rsi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -32(%rsp,%rsi), %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %rsi, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrq %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rax, %rsi, %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %r9, %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %rdi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrq %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rax, %rdi, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %r10, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rcx, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rsi, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r8, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vzeroupper
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-LABEL: shl_32bytes_dwordOff:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $2, %al
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    andb $24, %al
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    negb %al
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movsbq %al, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -24(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -16(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shldq %cl, %rsi, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -40(%rsp,%rax), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -32(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %r8, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shldq %cl, %r8, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vzeroupper
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    retq
 ;
 ; X86-SSE2-LABEL: shl_32bytes_dwordOff:
 ; X86-SSE2:       # %bb.0:
@@ -8789,2192 +7209,1656 @@ define void @shl_32bytes_qwordOff(ptr %src.ptr, ptr %qwordOff.ptr, ptr %dst) nou
 }
 
 define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
-; FALLBACK0-LABEL: ashr_32bytes:
-; FALLBACK0:       # %bb.0:
-; FALLBACK0-NEXT:    pushq %rbx
-; FALLBACK0-NEXT:    movq (%rdi), %rcx
-; FALLBACK0-NEXT:    movq 8(%rdi), %r8
-; FALLBACK0-NEXT:    movq 16(%rdi), %r9
-; FALLBACK0-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK0-NEXT:    movzbl (%rsi), %esi
-; FALLBACK0-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK0-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    sarq $63, %rdi
-; FALLBACK0-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    andb $24, %sil
-; FALLBACK0-NEXT:    movzbl %sil, %r9d
-; FALLBACK0-NEXT:    movq -64(%rsp,%r9), %r10
-; FALLBACK0-NEXT:    movq -56(%rsp,%r9), %rdi
-; FALLBACK0-NEXT:    movq %rdi, %r11
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r11
-; FALLBACK0-NEXT:    movl %eax, %esi
-; FALLBACK0-NEXT:    notb %sil
-; FALLBACK0-NEXT:    movq -48(%rsp,%r9), %rbx
-; FALLBACK0-NEXT:    leaq (%rbx,%rbx), %r8
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r8
-; FALLBACK0-NEXT:    orq %r11, %r8
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r10
-; FALLBACK0-NEXT:    addq %rdi, %rdi
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %rdi
-; FALLBACK0-NEXT:    orq %r10, %rdi
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %rbx
-; FALLBACK0-NEXT:    movq -40(%rsp,%r9), %r9
-; FALLBACK0-NEXT:    leaq (%r9,%r9), %r10
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r10
-; FALLBACK0-NEXT:    orq %rbx, %r10
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    sarq %cl, %r9
-; FALLBACK0-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK0-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK0-NEXT:    movq %rdi, (%rdx)
-; FALLBACK0-NEXT:    movq %r8, 8(%rdx)
-; FALLBACK0-NEXT:    popq %rbx
-; FALLBACK0-NEXT:    retq
-;
-; FALLBACK1-LABEL: ashr_32bytes:
-; FALLBACK1:       # %bb.0:
-; FALLBACK1-NEXT:    movq (%rdi), %rax
-; FALLBACK1-NEXT:    movq 8(%rdi), %r8
-; FALLBACK1-NEXT:    movq 16(%rdi), %r9
-; FALLBACK1-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK1-NEXT:    movzbl (%rsi), %esi
-; FALLBACK1-NEXT:    leal (,%rsi,8), %ecx
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    sarq $63, %rdi
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    andb $24, %sil
-; FALLBACK1-NEXT:    movzbl %sil, %eax
-; FALLBACK1-NEXT:    movq -56(%rsp,%rax), %rsi
-; FALLBACK1-NEXT:    movq -72(%rsp,%rax), %rdi
-; FALLBACK1-NEXT:    movq -64(%rsp,%rax), %r8
-; FALLBACK1-NEXT:    movq %r8, %r9
-; FALLBACK1-NEXT:    shrdq %cl, %rsi, %r9
-; FALLBACK1-NEXT:    movq -48(%rsp,%rax), %rax
-; FALLBACK1-NEXT:    shrdq %cl, %rax, %rsi
-; FALLBACK1-NEXT:    shrdq %cl, %r8, %rdi
-; FALLBACK1-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK1-NEXT:    sarq %cl, %rax
-; FALLBACK1-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK1-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK1-NEXT:    movq %rdi, (%rdx)
-; FALLBACK1-NEXT:    movq %r9, 8(%rdx)
-; FALLBACK1-NEXT:    retq
-;
-; FALLBACK2-LABEL: ashr_32bytes:
-; FALLBACK2:       # %bb.0:
-; FALLBACK2-NEXT:    movq (%rdi), %rcx
-; FALLBACK2-NEXT:    movq 8(%rdi), %r8
-; FALLBACK2-NEXT:    movq 16(%rdi), %r9
-; FALLBACK2-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK2-NEXT:    movzbl (%rsi), %esi
-; FALLBACK2-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    sarq $63, %rdi
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movl %eax, %ecx
-; FALLBACK2-NEXT:    andb $24, %sil
-; FALLBACK2-NEXT:    movzbl %sil, %esi
-; FALLBACK2-NEXT:    movq -64(%rsp,%rsi), %rdi
-; FALLBACK2-NEXT:    movq -56(%rsp,%rsi), %r8
-; FALLBACK2-NEXT:    shrxq %rcx, %rdi, %r9
-; FALLBACK2-NEXT:    notb %al
-; FALLBACK2-NEXT:    leaq (%r8,%r8), %r10
-; FALLBACK2-NEXT:    shlxq %rax, %r10, %r10
-; FALLBACK2-NEXT:    orq %r9, %r10
-; FALLBACK2-NEXT:    shrxq %rcx, -72(%rsp,%rsi), %r9
-; FALLBACK2-NEXT:    addq %rdi, %rdi
-; FALLBACK2-NEXT:    shlxq %rax, %rdi, %rdi
-; FALLBACK2-NEXT:    orq %r9, %rdi
-; FALLBACK2-NEXT:    shrxq %rcx, %r8, %r8
-; FALLBACK2-NEXT:    movq -48(%rsp,%rsi), %rsi
-; FALLBACK2-NEXT:    leaq (%rsi,%rsi), %r9
-; FALLBACK2-NEXT:    shlxq %rax, %r9, %rax
-; FALLBACK2-NEXT:    orq %r8, %rax
-; FALLBACK2-NEXT:    sarxq %rcx, %rsi, %rcx
-; FALLBACK2-NEXT:    movq %rcx, 24(%rdx)
-; FALLBACK2-NEXT:    movq %rax, 16(%rdx)
-; FALLBACK2-NEXT:    movq %rdi, (%rdx)
-; FALLBACK2-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK2-NEXT:    retq
-;
-; FALLBACK3-LABEL: ashr_32bytes:
-; FALLBACK3:       # %bb.0:
-; FALLBACK3-NEXT:    movq (%rdi), %rax
-; FALLBACK3-NEXT:    movq 8(%rdi), %r8
-; FALLBACK3-NEXT:    movq 16(%rdi), %r9
-; FALLBACK3-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK3-NEXT:    movzbl (%rsi), %esi
-; FALLBACK3-NEXT:    leal (,%rsi,8), %ecx
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    sarq $63, %rdi
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    andb $24, %sil
-; FALLBACK3-NEXT:    movzbl %sil, %eax
-; FALLBACK3-NEXT:    movq -56(%rsp,%rax), %rsi
-; FALLBACK3-NEXT:    movq -72(%rsp,%rax), %rdi
-; FALLBACK3-NEXT:    movq -64(%rsp,%rax), %r8
-; FALLBACK3-NEXT:    movq %r8, %r9
-; FALLBACK3-NEXT:    shrdq %cl, %rsi, %r9
-; FALLBACK3-NEXT:    movq -48(%rsp,%rax), %rax
-; FALLBACK3-NEXT:    shrdq %cl, %rax, %rsi
-; FALLBACK3-NEXT:    shrdq %cl, %r8, %rdi
-; FALLBACK3-NEXT:    sarxq %rcx, %rax, %rax
-; FALLBACK3-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK3-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK3-NEXT:    movq %rdi, (%rdx)
-; FALLBACK3-NEXT:    movq %r9, 8(%rdx)
-; FALLBACK3-NEXT:    retq
-;
-; FALLBACK4-LABEL: ashr_32bytes:
-; FALLBACK4:       # %bb.0:
-; FALLBACK4-NEXT:    pushq %rbx
-; FALLBACK4-NEXT:    movups (%rdi), %xmm0
-; FALLBACK4-NEXT:    movq 16(%rdi), %rcx
-; FALLBACK4-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK4-NEXT:    movzbl (%rsi), %esi
-; FALLBACK4-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    sarq $63, %rdi
-; FALLBACK4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    andb $24, %sil
-; FALLBACK4-NEXT:    movzbl %sil, %r9d
-; FALLBACK4-NEXT:    movq -64(%rsp,%r9), %r10
-; FALLBACK4-NEXT:    movq -56(%rsp,%r9), %r8
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r10
-; FALLBACK4-NEXT:    movl %eax, %esi
-; FALLBACK4-NEXT:    notb %sil
-; FALLBACK4-NEXT:    leaq (%r8,%r8), %rdi
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %rdi
-; FALLBACK4-NEXT:    orq %r10, %rdi
-; FALLBACK4-NEXT:    movq -48(%rsp,%r9), %r10
-; FALLBACK4-NEXT:    movq %r10, %r11
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r11
-; FALLBACK4-NEXT:    movq -40(%rsp,%r9), %r9
-; FALLBACK4-NEXT:    leaq (%r9,%r9), %rbx
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %rbx
-; FALLBACK4-NEXT:    orq %r11, %rbx
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r8
-; FALLBACK4-NEXT:    addq %r10, %r10
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r10
-; FALLBACK4-NEXT:    orq %r8, %r10
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    sarq %cl, %r9
-; FALLBACK4-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK4-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK4-NEXT:    movq %rbx, 16(%rdx)
-; FALLBACK4-NEXT:    movq %rdi, (%rdx)
-; FALLBACK4-NEXT:    popq %rbx
-; FALLBACK4-NEXT:    retq
-;
-; FALLBACK5-LABEL: ashr_32bytes:
-; FALLBACK5:       # %bb.0:
-; FALLBACK5-NEXT:    movups (%rdi), %xmm0
-; FALLBACK5-NEXT:    movq 16(%rdi), %rax
-; FALLBACK5-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK5-NEXT:    movzbl (%rsi), %esi
-; FALLBACK5-NEXT:    leal (,%rsi,8), %ecx
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    sarq $63, %rdi
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    andb $24, %sil
-; FALLBACK5-NEXT:    movzbl %sil, %eax
-; FALLBACK5-NEXT:    movq -48(%rsp,%rax), %rsi
-; FALLBACK5-NEXT:    movq -56(%rsp,%rax), %rdi
-; FALLBACK5-NEXT:    movq %rdi, %r8
-; FALLBACK5-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK5-NEXT:    movq -72(%rsp,%rax), %r9
-; FALLBACK5-NEXT:    movq -64(%rsp,%rax), %rax
-; FALLBACK5-NEXT:    movq %rax, %r10
-; FALLBACK5-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK5-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK5-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK5-NEXT:    sarq %cl, %rsi
-; FALLBACK5-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK5-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK5-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK5-NEXT:    movq %r9, (%rdx)
-; FALLBACK5-NEXT:    retq
-;
-; FALLBACK6-LABEL: ashr_32bytes:
-; FALLBACK6:       # %bb.0:
-; FALLBACK6-NEXT:    movups (%rdi), %xmm0
-; FALLBACK6-NEXT:    movq 16(%rdi), %rcx
-; FALLBACK6-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK6-NEXT:    movzbl (%rsi), %esi
-; FALLBACK6-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    sarq $63, %rdi
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movl %eax, %ecx
-; FALLBACK6-NEXT:    andb $24, %sil
-; FALLBACK6-NEXT:    movzbl %sil, %esi
-; FALLBACK6-NEXT:    shrxq %rcx, -72(%rsp,%rsi), %rdi
-; FALLBACK6-NEXT:    notb %al
-; FALLBACK6-NEXT:    movq -64(%rsp,%rsi), %r8
-; FALLBACK6-NEXT:    movq -56(%rsp,%rsi), %r9
-; FALLBACK6-NEXT:    leaq (%r8,%r8), %r10
-; FALLBACK6-NEXT:    shlxq %rax, %r10, %r10
-; FALLBACK6-NEXT:    orq %rdi, %r10
-; FALLBACK6-NEXT:    shrxq %rcx, %r9, %rdi
-; FALLBACK6-NEXT:    movq -48(%rsp,%rsi), %rsi
-; FALLBACK6-NEXT:    leaq (%rsi,%rsi), %r11
-; FALLBACK6-NEXT:    shlxq %rax, %r11, %r11
-; FALLBACK6-NEXT:    orq %rdi, %r11
-; FALLBACK6-NEXT:    shrxq %rcx, %r8, %rdi
-; FALLBACK6-NEXT:    addq %r9, %r9
-; FALLBACK6-NEXT:    shlxq %rax, %r9, %rax
-; FALLBACK6-NEXT:    orq %rdi, %rax
-; FALLBACK6-NEXT:    sarxq %rcx, %rsi, %rcx
-; FALLBACK6-NEXT:    movq %rcx, 24(%rdx)
-; FALLBACK6-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK6-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK6-NEXT:    movq %r10, (%rdx)
-; FALLBACK6-NEXT:    retq
-;
-; FALLBACK7-LABEL: ashr_32bytes:
-; FALLBACK7:       # %bb.0:
-; FALLBACK7-NEXT:    movups (%rdi), %xmm0
-; FALLBACK7-NEXT:    movq 16(%rdi), %rax
-; FALLBACK7-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK7-NEXT:    movzbl (%rsi), %esi
-; FALLBACK7-NEXT:    leal (,%rsi,8), %ecx
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    sarq $63, %rdi
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    andb $24, %sil
-; FALLBACK7-NEXT:    movzbl %sil, %eax
-; FALLBACK7-NEXT:    movq -48(%rsp,%rax), %rsi
-; FALLBACK7-NEXT:    movq -56(%rsp,%rax), %rdi
-; FALLBACK7-NEXT:    movq %rdi, %r8
-; FALLBACK7-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK7-NEXT:    movq -72(%rsp,%rax), %r9
-; FALLBACK7-NEXT:    movq -64(%rsp,%rax), %rax
-; FALLBACK7-NEXT:    movq %rax, %r10
-; FALLBACK7-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK7-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK7-NEXT:    sarxq %rcx, %rsi, %rax
-; FALLBACK7-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK7-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK7-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK7-NEXT:    movq %r9, (%rdx)
-; FALLBACK7-NEXT:    retq
-;
-; FALLBACK8-LABEL: ashr_32bytes:
-; FALLBACK8:       # %bb.0:
-; FALLBACK8-NEXT:    pushq %rbx
-; FALLBACK8-NEXT:    vmovups (%rdi), %xmm0
-; FALLBACK8-NEXT:    movq 16(%rdi), %rcx
-; FALLBACK8-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK8-NEXT:    movzbl (%rsi), %esi
-; FALLBACK8-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK8-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    sarq $63, %rdi
-; FALLBACK8-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    andb $24, %sil
-; FALLBACK8-NEXT:    movzbl %sil, %r9d
-; FALLBACK8-NEXT:    movq -64(%rsp,%r9), %r10
-; FALLBACK8-NEXT:    movq -56(%rsp,%r9), %r8
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r10
-; FALLBACK8-NEXT:    movl %eax, %esi
-; FALLBACK8-NEXT:    notb %sil
-; FALLBACK8-NEXT:    leaq (%r8,%r8), %rdi
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %rdi
-; FALLBACK8-NEXT:    orq %r10, %rdi
-; FALLBACK8-NEXT:    movq -48(%rsp,%r9), %r10
-; FALLBACK8-NEXT:    movq %r10, %r11
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r11
-; FALLBACK8-NEXT:    movq -40(%rsp,%r9), %r9
-; FALLBACK8-NEXT:    leaq (%r9,%r9), %rbx
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %rbx
-; FALLBACK8-NEXT:    orq %r11, %rbx
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r8
-; FALLBACK8-NEXT:    addq %r10, %r10
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r10
-; FALLBACK8-NEXT:    orq %r8, %r10
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    sarq %cl, %r9
-; FALLBACK8-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK8-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK8-NEXT:    movq %rbx, 16(%rdx)
-; FALLBACK8-NEXT:    movq %rdi, (%rdx)
-; FALLBACK8-NEXT:    popq %rbx
-; FALLBACK8-NEXT:    retq
-;
-; FALLBACK9-LABEL: ashr_32bytes:
-; FALLBACK9:       # %bb.0:
-; FALLBACK9-NEXT:    vmovups (%rdi), %xmm0
-; FALLBACK9-NEXT:    movq 16(%rdi), %rax
-; FALLBACK9-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK9-NEXT:    movzbl (%rsi), %esi
-; FALLBACK9-NEXT:    leal (,%rsi,8), %ecx
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    sarq $63, %rdi
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    andb $24, %sil
-; FALLBACK9-NEXT:    movzbl %sil, %eax
-; FALLBACK9-NEXT:    movq -48(%rsp,%rax), %rsi
-; FALLBACK9-NEXT:    movq -56(%rsp,%rax), %rdi
-; FALLBACK9-NEXT:    movq %rdi, %r8
-; FALLBACK9-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK9-NEXT:    movq -72(%rsp,%rax), %r9
-; FALLBACK9-NEXT:    movq -64(%rsp,%rax), %rax
-; FALLBACK9-NEXT:    movq %rax, %r10
-; FALLBACK9-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK9-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK9-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK9-NEXT:    sarq %cl, %rsi
-; FALLBACK9-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK9-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK9-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK9-NEXT:    movq %r9, (%rdx)
-; FALLBACK9-NEXT:    retq
-;
-; FALLBACK10-LABEL: ashr_32bytes:
-; FALLBACK10:       # %bb.0:
-; FALLBACK10-NEXT:    vmovups (%rdi), %xmm0
-; FALLBACK10-NEXT:    movq 16(%rdi), %rcx
-; FALLBACK10-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK10-NEXT:    movzbl (%rsi), %esi
-; FALLBACK10-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    sarq $63, %rdi
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movl %eax, %ecx
-; FALLBACK10-NEXT:    andb $24, %sil
-; FALLBACK10-NEXT:    movzbl %sil, %esi
-; FALLBACK10-NEXT:    shrxq %rcx, -72(%rsp,%rsi), %rdi
-; FALLBACK10-NEXT:    notb %al
-; FALLBACK10-NEXT:    movq -64(%rsp,%rsi), %r8
-; FALLBACK10-NEXT:    movq -56(%rsp,%rsi), %r9
-; FALLBACK10-NEXT:    leaq (%r8,%r8), %r10
-; FALLBACK10-NEXT:    shlxq %rax, %r10, %r10
-; FALLBACK10-NEXT:    orq %rdi, %r10
-; FALLBACK10-NEXT:    shrxq %rcx, %r9, %rdi
-; FALLBACK10-NEXT:    movq -48(%rsp,%rsi), %rsi
-; FALLBACK10-NEXT:    leaq (%rsi,%rsi), %r11
-; FALLBACK10-NEXT:    shlxq %rax, %r11, %r11
-; FALLBACK10-NEXT:    orq %rdi, %r11
-; FALLBACK10-NEXT:    shrxq %rcx, %r8, %rdi
-; FALLBACK10-NEXT:    addq %r9, %r9
-; FALLBACK10-NEXT:    shlxq %rax, %r9, %rax
-; FALLBACK10-NEXT:    orq %rdi, %rax
-; FALLBACK10-NEXT:    sarxq %rcx, %rsi, %rcx
-; FALLBACK10-NEXT:    movq %rcx, 24(%rdx)
-; FALLBACK10-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK10-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK10-NEXT:    movq %r10, (%rdx)
-; FALLBACK10-NEXT:    retq
-;
-; FALLBACK11-LABEL: ashr_32bytes:
-; FALLBACK11:       # %bb.0:
-; FALLBACK11-NEXT:    vmovups (%rdi), %xmm0
-; FALLBACK11-NEXT:    movq 16(%rdi), %rax
-; FALLBACK11-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK11-NEXT:    movzbl (%rsi), %esi
-; FALLBACK11-NEXT:    leal (,%rsi,8), %ecx
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    sarq $63, %rdi
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    andb $24, %sil
-; FALLBACK11-NEXT:    movzbl %sil, %eax
-; FALLBACK11-NEXT:    movq -48(%rsp,%rax), %rsi
-; FALLBACK11-NEXT:    movq -56(%rsp,%rax), %rdi
-; FALLBACK11-NEXT:    movq %rdi, %r8
-; FALLBACK11-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK11-NEXT:    movq -72(%rsp,%rax), %r9
-; FALLBACK11-NEXT:    movq -64(%rsp,%rax), %rax
-; FALLBACK11-NEXT:    movq %rax, %r10
-; FALLBACK11-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK11-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK11-NEXT:    sarxq %rcx, %rsi, %rax
-; FALLBACK11-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK11-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK11-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK11-NEXT:    movq %r9, (%rdx)
-; FALLBACK11-NEXT:    retq
-;
-; FALLBACK12-LABEL: ashr_32bytes:
-; FALLBACK12:       # %bb.0:
-; FALLBACK12-NEXT:    pushq %rbx
-; FALLBACK12-NEXT:    vmovups (%rdi), %xmm0
-; FALLBACK12-NEXT:    movq 16(%rdi), %rcx
-; FALLBACK12-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK12-NEXT:    movzbl (%rsi), %esi
-; FALLBACK12-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK12-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    sarq $63, %rdi
-; FALLBACK12-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    andb $24, %sil
-; FALLBACK12-NEXT:    movzbl %sil, %r9d
-; FALLBACK12-NEXT:    movq -64(%rsp,%r9), %r10
-; FALLBACK12-NEXT:    movq -56(%rsp,%r9), %r8
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r10
-; FALLBACK12-NEXT:    movl %eax, %esi
-; FALLBACK12-NEXT:    notb %sil
-; FALLBACK12-NEXT:    leaq (%r8,%r8), %rdi
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %rdi
-; FALLBACK12-NEXT:    orq %r10, %rdi
-; FALLBACK12-NEXT:    movq -48(%rsp,%r9), %r10
-; FALLBACK12-NEXT:    movq %r10, %r11
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r11
-; FALLBACK12-NEXT:    movq -40(%rsp,%r9), %r9
-; FALLBACK12-NEXT:    leaq (%r9,%r9), %rbx
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %rbx
-; FALLBACK12-NEXT:    orq %r11, %rbx
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r8
-; FALLBACK12-NEXT:    addq %r10, %r10
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r10
-; FALLBACK12-NEXT:    orq %r8, %r10
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    sarq %cl, %r9
-; FALLBACK12-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK12-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK12-NEXT:    movq %rbx, 16(%rdx)
-; FALLBACK12-NEXT:    movq %rdi, (%rdx)
-; FALLBACK12-NEXT:    popq %rbx
-; FALLBACK12-NEXT:    retq
-;
-; FALLBACK13-LABEL: ashr_32bytes:
-; FALLBACK13:       # %bb.0:
-; FALLBACK13-NEXT:    vmovups (%rdi), %xmm0
-; FALLBACK13-NEXT:    movq 16(%rdi), %rax
-; FALLBACK13-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK13-NEXT:    movzbl (%rsi), %esi
-; FALLBACK13-NEXT:    leal (,%rsi,8), %ecx
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    sarq $63, %rdi
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    andb $24, %sil
-; FALLBACK13-NEXT:    movzbl %sil, %eax
-; FALLBACK13-NEXT:    movq -48(%rsp,%rax), %rsi
-; FALLBACK13-NEXT:    movq -56(%rsp,%rax), %rdi
-; FALLBACK13-NEXT:    movq %rdi, %r8
-; FALLBACK13-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK13-NEXT:    movq -72(%rsp,%rax), %r9
-; FALLBACK13-NEXT:    movq -64(%rsp,%rax), %rax
-; FALLBACK13-NEXT:    movq %rax, %r10
-; FALLBACK13-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK13-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK13-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK13-NEXT:    sarq %cl, %rsi
-; FALLBACK13-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK13-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK13-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK13-NEXT:    movq %r9, (%rdx)
-; FALLBACK13-NEXT:    retq
-;
-; FALLBACK14-LABEL: ashr_32bytes:
-; FALLBACK14:       # %bb.0:
-; FALLBACK14-NEXT:    vmovups (%rdi), %xmm0
-; FALLBACK14-NEXT:    movq 16(%rdi), %rcx
-; FALLBACK14-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK14-NEXT:    movzbl (%rsi), %esi
-; FALLBACK14-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    sarq $63, %rdi
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movl %eax, %ecx
-; FALLBACK14-NEXT:    andb $24, %sil
-; FALLBACK14-NEXT:    movzbl %sil, %esi
-; FALLBACK14-NEXT:    shrxq %rcx, -72(%rsp,%rsi), %rdi
-; FALLBACK14-NEXT:    notb %al
-; FALLBACK14-NEXT:    movq -64(%rsp,%rsi), %r8
-; FALLBACK14-NEXT:    movq -56(%rsp,%rsi), %r9
-; FALLBACK14-NEXT:    leaq (%r8,%r8), %r10
-; FALLBACK14-NEXT:    shlxq %rax, %r10, %r10
-; FALLBACK14-NEXT:    orq %rdi, %r10
-; FALLBACK14-NEXT:    shrxq %rcx, %r9, %rdi
-; FALLBACK14-NEXT:    movq -48(%rsp,%rsi), %rsi
-; FALLBACK14-NEXT:    leaq (%rsi,%rsi), %r11
-; FALLBACK14-NEXT:    shlxq %rax, %r11, %r11
-; FALLBACK14-NEXT:    orq %rdi, %r11
-; FALLBACK14-NEXT:    shrxq %rcx, %r8, %rdi
-; FALLBACK14-NEXT:    addq %r9, %r9
-; FALLBACK14-NEXT:    shlxq %rax, %r9, %rax
-; FALLBACK14-NEXT:    orq %rdi, %rax
-; FALLBACK14-NEXT:    sarxq %rcx, %rsi, %rcx
-; FALLBACK14-NEXT:    movq %rcx, 24(%rdx)
-; FALLBACK14-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK14-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK14-NEXT:    movq %r10, (%rdx)
-; FALLBACK14-NEXT:    retq
-;
-; FALLBACK15-LABEL: ashr_32bytes:
-; FALLBACK15:       # %bb.0:
-; FALLBACK15-NEXT:    vmovups (%rdi), %xmm0
-; FALLBACK15-NEXT:    movq 16(%rdi), %rax
-; FALLBACK15-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK15-NEXT:    movzbl (%rsi), %esi
-; FALLBACK15-NEXT:    leal (,%rsi,8), %ecx
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    sarq $63, %rdi
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    andb $24, %sil
-; FALLBACK15-NEXT:    movzbl %sil, %eax
-; FALLBACK15-NEXT:    movq -48(%rsp,%rax), %rsi
-; FALLBACK15-NEXT:    movq -56(%rsp,%rax), %rdi
-; FALLBACK15-NEXT:    movq %rdi, %r8
-; FALLBACK15-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK15-NEXT:    movq -72(%rsp,%rax), %r9
-; FALLBACK15-NEXT:    movq -64(%rsp,%rax), %rax
-; FALLBACK15-NEXT:    movq %rax, %r10
-; FALLBACK15-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK15-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK15-NEXT:    sarxq %rcx, %rsi, %rax
-; FALLBACK15-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK15-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK15-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK15-NEXT:    movq %r9, (%rdx)
-; FALLBACK15-NEXT:    retq
-;
-; FALLBACK16-LABEL: ashr_32bytes:
-; FALLBACK16:       # %bb.0:
-; FALLBACK16-NEXT:    pushl %ebp
-; FALLBACK16-NEXT:    pushl %ebx
-; FALLBACK16-NEXT:    pushl %edi
-; FALLBACK16-NEXT:    pushl %esi
-; FALLBACK16-NEXT:    subl $108, %esp
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; FALLBACK16-NEXT:    movl (%esi), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 4(%esi), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 8(%esi), %ebx
-; FALLBACK16-NEXT:    movl 12(%esi), %ebp
-; FALLBACK16-NEXT:    movl 16(%esi), %edi
-; FALLBACK16-NEXT:    movzbl (%eax), %ecx
-; FALLBACK16-NEXT:    movl 20(%esi), %edx
-; FALLBACK16-NEXT:    movl 24(%esi), %eax
-; FALLBACK16-NEXT:    movl 28(%esi), %esi
-; FALLBACK16-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, %edx
-; FALLBACK16-NEXT:    shlb $3, %dl
-; FALLBACK16-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    sarl $31, %esi
-; FALLBACK16-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    andb $28, %cl
-; FALLBACK16-NEXT:    movzbl %cl, %edi
-; FALLBACK16-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 32(%esp,%edi), %esi
-; FALLBACK16-NEXT:    movl 36(%esp,%edi), %eax
-; FALLBACK16-NEXT:    movl %eax, %ebx
-; FALLBACK16-NEXT:    movl %edx, %ecx
-; FALLBACK16-NEXT:    shrl %cl, %ebx
-; FALLBACK16-NEXT:    movb %dl, %ch
-; FALLBACK16-NEXT:    notb %ch
-; FALLBACK16-NEXT:    movl 40(%esp,%edi), %edi
-; FALLBACK16-NEXT:    leal (%edi,%edi), %ebp
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebp
-; FALLBACK16-NEXT:    orl %ebx, %ebp
-; FALLBACK16-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %esi
-; FALLBACK16-NEXT:    addl %eax, %eax
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %eax
-; FALLBACK16-NEXT:    orl %esi, %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl 44(%esp,%eax), %ebp
-; FALLBACK16-NEXT:    movl %ebp, %esi
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    movl %edx, %ebx
-; FALLBACK16-NEXT:    shrl %cl, %esi
-; FALLBACK16-NEXT:    movl 48(%esp,%eax), %edx
-; FALLBACK16-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    leal (%edx,%edx), %eax
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %eax
-; FALLBACK16-NEXT:    orl %esi, %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl %ebx, %edx
-; FALLBACK16-NEXT:    movb %bl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edi
-; FALLBACK16-NEXT:    addl %ebp, %ebp
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebp
-; FALLBACK16-NEXT:    orl %edi, %ebp
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK16-NEXT:    movl 52(%esp,%esi), %edi
-; FALLBACK16-NEXT:    movl %edi, %eax
-; FALLBACK16-NEXT:    movb %bl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %eax
-; FALLBACK16-NEXT:    movl 56(%esp,%esi), %ebx
-; FALLBACK16-NEXT:    leal (%ebx,%ebx), %esi
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %esi
-; FALLBACK16-NEXT:    orl %eax, %esi
-; FALLBACK16-NEXT:    movb %dl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    shrl %cl, %eax
-; FALLBACK16-NEXT:    addl %edi, %edi
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %edi
-; FALLBACK16-NEXT:    orl %eax, %edi
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %ebx
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl 60(%esp,%eax), %eax
-; FALLBACK16-NEXT:    leal (%eax,%eax), %edx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %edx
-; FALLBACK16-NEXT:    orl %ebx, %edx
-; FALLBACK16-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; FALLBACK16-NEXT:    sarl %cl, %eax
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK16-NEXT:    movl %eax, 28(%ecx)
-; FALLBACK16-NEXT:    movl %edx, 24(%ecx)
-; FALLBACK16-NEXT:    movl %edi, 16(%ecx)
-; FALLBACK16-NEXT:    movl %esi, 20(%ecx)
-; FALLBACK16-NEXT:    movl %ebp, 8(%ecx)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl %eax, 12(%ecx)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl %eax, (%ecx)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl %eax, 4(%ecx)
-; FALLBACK16-NEXT:    addl $108, %esp
-; FALLBACK16-NEXT:    popl %esi
-; FALLBACK16-NEXT:    popl %edi
-; FALLBACK16-NEXT:    popl %ebx
-; FALLBACK16-NEXT:    popl %ebp
-; FALLBACK16-NEXT:    retl
-;
-; FALLBACK17-LABEL: ashr_32bytes:
-; FALLBACK17:       # %bb.0:
-; FALLBACK17-NEXT:    pushl %ebp
-; FALLBACK17-NEXT:    pushl %ebx
-; FALLBACK17-NEXT:    pushl %edi
-; FALLBACK17-NEXT:    pushl %esi
-; FALLBACK17-NEXT:    subl $92, %esp
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK17-NEXT:    movl (%ecx), %edx
-; FALLBACK17-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 4(%ecx), %edx
-; FALLBACK17-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 8(%ecx), %edx
-; FALLBACK17-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 12(%ecx), %ebp
-; FALLBACK17-NEXT:    movl 16(%ecx), %ebx
-; FALLBACK17-NEXT:    movzbl (%eax), %eax
-; FALLBACK17-NEXT:    movl 20(%ecx), %edi
-; FALLBACK17-NEXT:    movl 24(%ecx), %edx
-; FALLBACK17-NEXT:    movl 28(%ecx), %esi
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, %ecx
-; FALLBACK17-NEXT:    shlb $3, %cl
-; FALLBACK17-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl (%esp), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    sarl $31, %esi
-; FALLBACK17-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    andb $28, %al
-; FALLBACK17-NEXT:    movzbl %al, %ebp
-; FALLBACK17-NEXT:    movl 24(%esp,%ebp), %edx
-; FALLBACK17-NEXT:    movl 20(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 32(%esp,%ebp), %ebx
-; FALLBACK17-NEXT:    movl 28(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, %esi
-; FALLBACK17-NEXT:    shrdl %cl, %ebx, %esi
-; FALLBACK17-NEXT:    movl %esi, (%esp) # 4-byte Spill
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK17-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 40(%esp,%ebp), %edx
-; FALLBACK17-NEXT:    movl 36(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, %edi
-; FALLBACK17-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK17-NEXT:    movl 16(%esp,%ebp), %esi
-; FALLBACK17-NEXT:    movl 44(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK17-NEXT:    movl %edx, 24(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    shrdl %cl, %edx, %esi
-; FALLBACK17-NEXT:    sarl %cl, %eax
-; FALLBACK17-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK17-NEXT:    movl %ebx, 16(%ebp)
-; FALLBACK17-NEXT:    movl %edi, 20(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK17-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK17-NEXT:    movl %esi, (%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 4(%ebp)
-; FALLBACK17-NEXT:    addl $92, %esp
-; FALLBACK17-NEXT:    popl %esi
-; FALLBACK17-NEXT:    popl %edi
-; FALLBACK17-NEXT:    popl %ebx
-; FALLBACK17-NEXT:    popl %ebp
-; FALLBACK17-NEXT:    retl
-;
-; FALLBACK18-LABEL: ashr_32bytes:
-; FALLBACK18:       # %bb.0:
-; FALLBACK18-NEXT:    pushl %ebp
-; FALLBACK18-NEXT:    pushl %ebx
-; FALLBACK18-NEXT:    pushl %edi
-; FALLBACK18-NEXT:    pushl %esi
-; FALLBACK18-NEXT:    subl $108, %esp
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; FALLBACK18-NEXT:    movl (%esi), %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 4(%esi), %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 8(%esi), %ebx
-; FALLBACK18-NEXT:    movl 12(%esi), %ebp
-; FALLBACK18-NEXT:    movl 16(%esi), %edi
-; FALLBACK18-NEXT:    movzbl (%edx), %edx
-; FALLBACK18-NEXT:    movl 20(%esi), %ecx
-; FALLBACK18-NEXT:    movl 24(%esi), %eax
-; FALLBACK18-NEXT:    movl 28(%esi), %esi
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %edx, %ecx
-; FALLBACK18-NEXT:    shlb $3, %cl
-; FALLBACK18-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    sarl $31, %esi
-; FALLBACK18-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, %eax
-; FALLBACK18-NEXT:    andb $28, %dl
-; FALLBACK18-NEXT:    movzbl %dl, %esi
-; FALLBACK18-NEXT:    movl 36(%esp,%esi), %edx
-; FALLBACK18-NEXT:    movl 40(%esp,%esi), %ebp
-; FALLBACK18-NEXT:    shrxl %eax, %edx, %edi
-; FALLBACK18-NEXT:    notb %cl
-; FALLBACK18-NEXT:    leal (%ebp,%ebp), %ebx
-; FALLBACK18-NEXT:    shlxl %ecx, %ebx, %ebx
-; FALLBACK18-NEXT:    orl %edi, %ebx
-; FALLBACK18-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %eax, 32(%esp,%esi), %edi
-; FALLBACK18-NEXT:    addl %edx, %edx
-; FALLBACK18-NEXT:    shlxl %ecx, %edx, %edx
-; FALLBACK18-NEXT:    orl %edi, %edx
-; FALLBACK18-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 48(%esp,%esi), %edx
-; FALLBACK18-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    addl %edx, %edx
-; FALLBACK18-NEXT:    shlxl %ecx, %edx, %ebx
-; FALLBACK18-NEXT:    movl 44(%esp,%esi), %edx
-; FALLBACK18-NEXT:    shrxl %eax, %edx, %edi
-; FALLBACK18-NEXT:    orl %edi, %ebx
-; FALLBACK18-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %eax, %ebp, %edi
-; FALLBACK18-NEXT:    movl %eax, %ebp
-; FALLBACK18-NEXT:    addl %edx, %edx
-; FALLBACK18-NEXT:    shlxl %ecx, %edx, %eax
-; FALLBACK18-NEXT:    orl %edi, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 56(%esp,%esi), %edi
-; FALLBACK18-NEXT:    leal (%edi,%edi), %edx
-; FALLBACK18-NEXT:    shlxl %ecx, %edx, %edx
-; FALLBACK18-NEXT:    movl 52(%esp,%esi), %eax
-; FALLBACK18-NEXT:    shrxl %ebp, %eax, %ebx
-; FALLBACK18-NEXT:    orl %ebx, %edx
-; FALLBACK18-NEXT:    shrxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; FALLBACK18-NEXT:    addl %eax, %eax
-; FALLBACK18-NEXT:    shlxl %ecx, %eax, %eax
-; FALLBACK18-NEXT:    orl %ebx, %eax
-; FALLBACK18-NEXT:    movl 60(%esp,%esi), %esi
-; FALLBACK18-NEXT:    leal (%esi,%esi), %ebx
-; FALLBACK18-NEXT:    shlxl %ecx, %ebx, %ecx
-; FALLBACK18-NEXT:    shrxl %ebp, %edi, %edi
-; FALLBACK18-NEXT:    orl %edi, %ecx
-; FALLBACK18-NEXT:    sarxl %ebp, %esi, %esi
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; FALLBACK18-NEXT:    movl %esi, 28(%edi)
-; FALLBACK18-NEXT:    movl %ecx, 24(%edi)
-; FALLBACK18-NEXT:    movl %eax, 16(%edi)
-; FALLBACK18-NEXT:    movl %edx, 20(%edi)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 8(%edi)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 12(%edi)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, (%edi)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 4(%edi)
-; FALLBACK18-NEXT:    addl $108, %esp
-; FALLBACK18-NEXT:    popl %esi
-; FALLBACK18-NEXT:    popl %edi
-; FALLBACK18-NEXT:    popl %ebx
-; FALLBACK18-NEXT:    popl %ebp
-; FALLBACK18-NEXT:    retl
-;
-; FALLBACK19-LABEL: ashr_32bytes:
-; FALLBACK19:       # %bb.0:
-; FALLBACK19-NEXT:    pushl %ebp
-; FALLBACK19-NEXT:    pushl %ebx
-; FALLBACK19-NEXT:    pushl %edi
-; FALLBACK19-NEXT:    pushl %esi
-; FALLBACK19-NEXT:    subl $92, %esp
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK19-NEXT:    movl (%ecx), %edx
-; FALLBACK19-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 4(%ecx), %edx
-; FALLBACK19-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 8(%ecx), %edx
-; FALLBACK19-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 12(%ecx), %ebp
-; FALLBACK19-NEXT:    movl 16(%ecx), %ebx
-; FALLBACK19-NEXT:    movzbl (%eax), %eax
-; FALLBACK19-NEXT:    movl 20(%ecx), %edi
-; FALLBACK19-NEXT:    movl 24(%ecx), %edx
-; FALLBACK19-NEXT:    movl 28(%ecx), %esi
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, %ecx
-; FALLBACK19-NEXT:    shlb $3, %cl
-; FALLBACK19-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl (%esp), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    sarl $31, %esi
-; FALLBACK19-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    andb $28, %al
-; FALLBACK19-NEXT:    movzbl %al, %ebp
-; FALLBACK19-NEXT:    movl 24(%esp,%ebp), %esi
-; FALLBACK19-NEXT:    movl 20(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shrdl %cl, %esi, %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 32(%esp,%ebp), %ebx
-; FALLBACK19-NEXT:    movl 28(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, %edx
-; FALLBACK19-NEXT:    shrdl %cl, %ebx, %edx
-; FALLBACK19-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK19-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 40(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl 36(%esp,%ebp), %edx
-; FALLBACK19-NEXT:    movl %edx, %esi
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK19-NEXT:    shrdl %cl, %edx, %ebx
-; FALLBACK19-NEXT:    movl 16(%esp,%ebp), %edx
-; FALLBACK19-NEXT:    movl 44(%esp,%ebp), %edi
-; FALLBACK19-NEXT:    shrdl %cl, %edi, %eax
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK19-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK19-NEXT:    sarxl %ecx, %edi, %eax
-; FALLBACK19-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK19-NEXT:    movl %ebx, 16(%ebp)
-; FALLBACK19-NEXT:    movl %esi, 20(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK19-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK19-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK19-NEXT:    movl %edx, (%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 4(%ebp)
-; FALLBACK19-NEXT:    addl $92, %esp
-; FALLBACK19-NEXT:    popl %esi
-; FALLBACK19-NEXT:    popl %edi
-; FALLBACK19-NEXT:    popl %ebx
-; FALLBACK19-NEXT:    popl %ebp
-; FALLBACK19-NEXT:    retl
-;
-; FALLBACK20-LABEL: ashr_32bytes:
-; FALLBACK20:       # %bb.0:
-; FALLBACK20-NEXT:    pushl %ebp
-; FALLBACK20-NEXT:    pushl %ebx
-; FALLBACK20-NEXT:    pushl %edi
-; FALLBACK20-NEXT:    pushl %esi
-; FALLBACK20-NEXT:    subl $108, %esp
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK20-NEXT:    movups (%ecx), %xmm0
-; FALLBACK20-NEXT:    movl 16(%ecx), %esi
-; FALLBACK20-NEXT:    movl 20(%ecx), %edi
-; FALLBACK20-NEXT:    movl 24(%ecx), %ebx
-; FALLBACK20-NEXT:    movl 28(%ecx), %edx
-; FALLBACK20-NEXT:    movzbl (%eax), %eax
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shlb $3, %cl
-; FALLBACK20-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    sarl $31, %edx
-; FALLBACK20-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    andb $28, %al
-; FALLBACK20-NEXT:    movzbl %al, %edi
-; FALLBACK20-NEXT:    movl 32(%esp,%edi), %eax
-; FALLBACK20-NEXT:    movl 36(%esp,%edi), %esi
-; FALLBACK20-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    shrl %cl, %eax
-; FALLBACK20-NEXT:    movl %ecx, %edx
-; FALLBACK20-NEXT:    movb %cl, %dh
-; FALLBACK20-NEXT:    notb %dl
-; FALLBACK20-NEXT:    addl %esi, %esi
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shll %cl, %esi
-; FALLBACK20-NEXT:    orl %eax, %esi
-; FALLBACK20-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 44(%esp,%edi), %ebx
-; FALLBACK20-NEXT:    movl %ebx, %eax
-; FALLBACK20-NEXT:    movb %dh, %cl
-; FALLBACK20-NEXT:    shrl %cl, %eax
-; FALLBACK20-NEXT:    movl 48(%esp,%edi), %esi
-; FALLBACK20-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    addl %esi, %esi
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shll %cl, %esi
-; FALLBACK20-NEXT:    orl %eax, %esi
-; FALLBACK20-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 40(%esp,%edi), %esi
-; FALLBACK20-NEXT:    movl %esi, %eax
-; FALLBACK20-NEXT:    movb %dh, %cl
-; FALLBACK20-NEXT:    shrl %cl, %eax
-; FALLBACK20-NEXT:    addl %ebx, %ebx
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    orl %eax, %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 52(%esp,%edi), %ebp
-; FALLBACK20-NEXT:    movl %ebp, %eax
-; FALLBACK20-NEXT:    movb %dh, %cl
-; FALLBACK20-NEXT:    shrl %cl, %eax
-; FALLBACK20-NEXT:    movl 56(%esp,%edi), %ecx
-; FALLBACK20-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    orl %eax, %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %dh, %cl
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %cl, %eax
-; FALLBACK20-NEXT:    addl %ebp, %ebp
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shll %cl, %ebp
-; FALLBACK20-NEXT:    orl %eax, %ebp
-; FALLBACK20-NEXT:    movb %dh, %cl
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %cl, %ebx
-; FALLBACK20-NEXT:    movl 60(%esp,%edi), %eax
-; FALLBACK20-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shll %cl, %edi
-; FALLBACK20-NEXT:    orl %ebx, %edi
-; FALLBACK20-NEXT:    movb %dh, %cl
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %cl, %ebx
-; FALLBACK20-NEXT:    addl %esi, %esi
-; FALLBACK20-NEXT:    movl %edx, %ecx
-; FALLBACK20-NEXT:    shll %cl, %esi
-; FALLBACK20-NEXT:    orl %ebx, %esi
-; FALLBACK20-NEXT:    movb %dh, %cl
-; FALLBACK20-NEXT:    sarl %cl, %eax
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK20-NEXT:    movl %eax, 28(%ecx)
-; FALLBACK20-NEXT:    movl %esi, 4(%ecx)
-; FALLBACK20-NEXT:    movl %edi, 24(%ecx)
-; FALLBACK20-NEXT:    movl %ebp, 16(%ecx)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK20-NEXT:    movl %eax, 20(%ecx)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK20-NEXT:    movl %eax, 8(%ecx)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK20-NEXT:    movl %eax, 12(%ecx)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK20-NEXT:    movl %eax, (%ecx)
-; FALLBACK20-NEXT:    addl $108, %esp
-; FALLBACK20-NEXT:    popl %esi
-; FALLBACK20-NEXT:    popl %edi
-; FALLBACK20-NEXT:    popl %ebx
-; FALLBACK20-NEXT:    popl %ebp
-; FALLBACK20-NEXT:    retl
-;
-; FALLBACK21-LABEL: ashr_32bytes:
-; FALLBACK21:       # %bb.0:
-; FALLBACK21-NEXT:    pushl %ebp
-; FALLBACK21-NEXT:    pushl %ebx
-; FALLBACK21-NEXT:    pushl %edi
-; FALLBACK21-NEXT:    pushl %esi
-; FALLBACK21-NEXT:    subl $108, %esp
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK21-NEXT:    movups (%ecx), %xmm0
-; FALLBACK21-NEXT:    movl 16(%ecx), %esi
-; FALLBACK21-NEXT:    movl 20(%ecx), %edi
-; FALLBACK21-NEXT:    movl 24(%ecx), %ebx
-; FALLBACK21-NEXT:    movl 28(%ecx), %edx
-; FALLBACK21-NEXT:    movzbl (%eax), %eax
-; FALLBACK21-NEXT:    movl %eax, %ecx
-; FALLBACK21-NEXT:    shlb $3, %cl
-; FALLBACK21-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    sarl $31, %edx
-; FALLBACK21-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    andb $28, %al
-; FALLBACK21-NEXT:    movzbl %al, %ebp
-; FALLBACK21-NEXT:    movl 48(%esp,%ebp), %esi
-; FALLBACK21-NEXT:    movl 44(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, %edx
-; FALLBACK21-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 40(%esp,%ebp), %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 56(%esp,%ebp), %ebx
-; FALLBACK21-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, %edx
-; FALLBACK21-NEXT:    shrdl %cl, %ebx, %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK21-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK21-NEXT:    movl 32(%esp,%ebp), %edx
-; FALLBACK21-NEXT:    movl 36(%esp,%ebp), %edi
-; FALLBACK21-NEXT:    movl %edi, %esi
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; FALLBACK21-NEXT:    shrdl %cl, %ebp, %esi
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK21-NEXT:    movl %esi, 4(%ebp)
-; FALLBACK21-NEXT:    movl %ebx, 24(%ebp)
-; FALLBACK21-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK21-NEXT:    sarl %cl, %eax
-; FALLBACK21-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK21-NEXT:    movl %edx, (%ebp)
-; FALLBACK21-NEXT:    addl $108, %esp
-; FALLBACK21-NEXT:    popl %esi
-; FALLBACK21-NEXT:    popl %edi
-; FALLBACK21-NEXT:    popl %ebx
-; FALLBACK21-NEXT:    popl %ebp
-; FALLBACK21-NEXT:    retl
-;
-; FALLBACK22-LABEL: ashr_32bytes:
-; FALLBACK22:       # %bb.0:
-; FALLBACK22-NEXT:    pushl %ebp
-; FALLBACK22-NEXT:    pushl %ebx
-; FALLBACK22-NEXT:    pushl %edi
-; FALLBACK22-NEXT:    pushl %esi
-; FALLBACK22-NEXT:    subl $108, %esp
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK22-NEXT:    movups (%ecx), %xmm0
-; FALLBACK22-NEXT:    movl 16(%ecx), %esi
-; FALLBACK22-NEXT:    movl 20(%ecx), %edi
-; FALLBACK22-NEXT:    movl 24(%ecx), %ebp
-; FALLBACK22-NEXT:    movl 28(%ecx), %ecx
-; FALLBACK22-NEXT:    movzbl (%eax), %edx
-; FALLBACK22-NEXT:    movl %edx, %ebx
-; FALLBACK22-NEXT:    shlb $3, %bl
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    sarl $31, %ecx
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ebx, %eax
-; FALLBACK22-NEXT:    andb $28, %dl
-; FALLBACK22-NEXT:    movzbl %dl, %ecx
-; FALLBACK22-NEXT:    shrxl %eax, 32(%esp,%ecx), %edx
-; FALLBACK22-NEXT:    movl %eax, %ebp
-; FALLBACK22-NEXT:    notb %bl
-; FALLBACK22-NEXT:    movl 36(%esp,%ecx), %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    leal (%eax,%eax), %esi
-; FALLBACK22-NEXT:    shlxl %ebx, %esi, %eax
-; FALLBACK22-NEXT:    orl %edx, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 48(%esp,%ecx), %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    leal (%eax,%eax), %edx
-; FALLBACK22-NEXT:    shlxl %ebx, %edx, %edi
-; FALLBACK22-NEXT:    movl 44(%esp,%ecx), %edx
-; FALLBACK22-NEXT:    shrxl %ebp, %edx, %esi
-; FALLBACK22-NEXT:    orl %esi, %edi
-; FALLBACK22-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    addl %edx, %edx
-; FALLBACK22-NEXT:    shlxl %ebx, %edx, %edi
-; FALLBACK22-NEXT:    movl 40(%esp,%ecx), %edx
-; FALLBACK22-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrxl %ebp, %edx, %esi
-; FALLBACK22-NEXT:    movl %ebp, %edx
-; FALLBACK22-NEXT:    orl %esi, %edi
-; FALLBACK22-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 56(%esp,%ecx), %esi
-; FALLBACK22-NEXT:    leal (%esi,%esi), %ebp
-; FALLBACK22-NEXT:    shlxl %ebx, %ebp, %ebp
-; FALLBACK22-NEXT:    movl 52(%esp,%ecx), %eax
-; FALLBACK22-NEXT:    shrxl %edx, %eax, %edi
-; FALLBACK22-NEXT:    orl %edi, %ebp
-; FALLBACK22-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK22-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    addl %eax, %eax
-; FALLBACK22-NEXT:    shlxl %ebx, %eax, %edi
-; FALLBACK22-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK22-NEXT:    shrxl %edx, %esi, %eax
-; FALLBACK22-NEXT:    movl 60(%esp,%ecx), %ecx
-; FALLBACK22-NEXT:    leal (%ecx,%ecx), %esi
-; FALLBACK22-NEXT:    shlxl %ebx, %esi, %esi
-; FALLBACK22-NEXT:    orl %eax, %esi
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    addl %eax, %eax
-; FALLBACK22-NEXT:    shlxl %ebx, %eax, %eax
-; FALLBACK22-NEXT:    movl %edx, %ebx
-; FALLBACK22-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; FALLBACK22-NEXT:    orl %edx, %eax
-; FALLBACK22-NEXT:    sarxl %ebx, %ecx, %ecx
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK22-NEXT:    movl %ecx, 28(%edx)
-; FALLBACK22-NEXT:    movl %eax, 4(%edx)
-; FALLBACK22-NEXT:    movl %esi, 24(%edx)
-; FALLBACK22-NEXT:    movl %edi, 16(%edx)
-; FALLBACK22-NEXT:    movl %ebp, 20(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 8(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 12(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, (%edx)
-; FALLBACK22-NEXT:    addl $108, %esp
-; FALLBACK22-NEXT:    popl %esi
-; FALLBACK22-NEXT:    popl %edi
-; FALLBACK22-NEXT:    popl %ebx
-; FALLBACK22-NEXT:    popl %ebp
-; FALLBACK22-NEXT:    retl
-;
-; FALLBACK23-LABEL: ashr_32bytes:
-; FALLBACK23:       # %bb.0:
-; FALLBACK23-NEXT:    pushl %ebp
-; FALLBACK23-NEXT:    pushl %ebx
-; FALLBACK23-NEXT:    pushl %edi
-; FALLBACK23-NEXT:    pushl %esi
-; FALLBACK23-NEXT:    subl $108, %esp
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK23-NEXT:    movups (%ecx), %xmm0
-; FALLBACK23-NEXT:    movl 16(%ecx), %esi
-; FALLBACK23-NEXT:    movl 20(%ecx), %edi
-; FALLBACK23-NEXT:    movl 24(%ecx), %ebx
-; FALLBACK23-NEXT:    movl 28(%ecx), %edx
-; FALLBACK23-NEXT:    movzbl (%eax), %eax
-; FALLBACK23-NEXT:    movl %eax, %ecx
-; FALLBACK23-NEXT:    shlb $3, %cl
-; FALLBACK23-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    sarl $31, %edx
-; FALLBACK23-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    andb $28, %al
-; FALLBACK23-NEXT:    movzbl %al, %ebx
-; FALLBACK23-NEXT:    movl 48(%esp,%ebx), %esi
-; FALLBACK23-NEXT:    movl 44(%esp,%ebx), %eax
-; FALLBACK23-NEXT:    movl %eax, %edx
-; FALLBACK23-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 40(%esp,%ebx), %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 56(%esp,%ebx), %ebp
-; FALLBACK23-NEXT:    movl 52(%esp,%ebx), %eax
-; FALLBACK23-NEXT:    movl %eax, %edi
-; FALLBACK23-NEXT:    shrdl %cl, %ebp, %edi
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK23-NEXT:    movl 60(%esp,%ebx), %eax
-; FALLBACK23-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %ebp
-; FALLBACK23-NEXT:    movl 32(%esp,%ebx), %edx
-; FALLBACK23-NEXT:    movl 36(%esp,%ebx), %ebx
-; FALLBACK23-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK23-NEXT:    movl %ebx, 4(%eax)
-; FALLBACK23-NEXT:    movl %ebp, 24(%eax)
-; FALLBACK23-NEXT:    sarxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; FALLBACK23-NEXT:    movl %ebx, 28(%eax)
-; FALLBACK23-NEXT:    movl %esi, 16(%eax)
-; FALLBACK23-NEXT:    movl %edi, 20(%eax)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK23-NEXT:    movl %esi, 8(%eax)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK23-NEXT:    movl %esi, 12(%eax)
-; FALLBACK23-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK23-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK23-NEXT:    movl %edx, (%eax)
-; FALLBACK23-NEXT:    addl $108, %esp
-; FALLBACK23-NEXT:    popl %esi
-; FALLBACK23-NEXT:    popl %edi
-; FALLBACK23-NEXT:    popl %ebx
-; FALLBACK23-NEXT:    popl %ebp
-; FALLBACK23-NEXT:    retl
-;
-; FALLBACK24-LABEL: ashr_32bytes:
-; FALLBACK24:       # %bb.0:
-; FALLBACK24-NEXT:    pushl %ebp
-; FALLBACK24-NEXT:    pushl %ebx
-; FALLBACK24-NEXT:    pushl %edi
-; FALLBACK24-NEXT:    pushl %esi
-; FALLBACK24-NEXT:    subl $108, %esp
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK24-NEXT:    vmovups (%ecx), %xmm0
-; FALLBACK24-NEXT:    movl 16(%ecx), %esi
-; FALLBACK24-NEXT:    movl 20(%ecx), %edi
-; FALLBACK24-NEXT:    movl 24(%ecx), %ebx
-; FALLBACK24-NEXT:    movl 28(%ecx), %edx
-; FALLBACK24-NEXT:    movzbl (%eax), %eax
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shlb $3, %cl
-; FALLBACK24-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    sarl $31, %edx
-; FALLBACK24-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    andb $28, %al
-; FALLBACK24-NEXT:    movzbl %al, %edi
-; FALLBACK24-NEXT:    movl 32(%esp,%edi), %eax
-; FALLBACK24-NEXT:    movl 36(%esp,%edi), %esi
-; FALLBACK24-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    shrl %cl, %eax
-; FALLBACK24-NEXT:    movl %ecx, %edx
-; FALLBACK24-NEXT:    movb %cl, %dh
-; FALLBACK24-NEXT:    notb %dl
-; FALLBACK24-NEXT:    addl %esi, %esi
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shll %cl, %esi
-; FALLBACK24-NEXT:    orl %eax, %esi
-; FALLBACK24-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 44(%esp,%edi), %ebx
-; FALLBACK24-NEXT:    movl %ebx, %eax
-; FALLBACK24-NEXT:    movb %dh, %cl
-; FALLBACK24-NEXT:    shrl %cl, %eax
-; FALLBACK24-NEXT:    movl 48(%esp,%edi), %esi
-; FALLBACK24-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    addl %esi, %esi
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shll %cl, %esi
-; FALLBACK24-NEXT:    orl %eax, %esi
-; FALLBACK24-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 40(%esp,%edi), %esi
-; FALLBACK24-NEXT:    movl %esi, %eax
-; FALLBACK24-NEXT:    movb %dh, %cl
-; FALLBACK24-NEXT:    shrl %cl, %eax
-; FALLBACK24-NEXT:    addl %ebx, %ebx
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    orl %eax, %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 52(%esp,%edi), %ebp
-; FALLBACK24-NEXT:    movl %ebp, %eax
-; FALLBACK24-NEXT:    movb %dh, %cl
-; FALLBACK24-NEXT:    shrl %cl, %eax
-; FALLBACK24-NEXT:    movl 56(%esp,%edi), %ecx
-; FALLBACK24-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    orl %eax, %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %dh, %cl
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %cl, %eax
-; FALLBACK24-NEXT:    addl %ebp, %ebp
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shll %cl, %ebp
-; FALLBACK24-NEXT:    orl %eax, %ebp
-; FALLBACK24-NEXT:    movb %dh, %cl
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %cl, %ebx
-; FALLBACK24-NEXT:    movl 60(%esp,%edi), %eax
-; FALLBACK24-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shll %cl, %edi
-; FALLBACK24-NEXT:    orl %ebx, %edi
-; FALLBACK24-NEXT:    movb %dh, %cl
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %cl, %ebx
-; FALLBACK24-NEXT:    addl %esi, %esi
-; FALLBACK24-NEXT:    movl %edx, %ecx
-; FALLBACK24-NEXT:    shll %cl, %esi
-; FALLBACK24-NEXT:    orl %ebx, %esi
-; FALLBACK24-NEXT:    movb %dh, %cl
-; FALLBACK24-NEXT:    sarl %cl, %eax
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK24-NEXT:    movl %eax, 28(%ecx)
-; FALLBACK24-NEXT:    movl %esi, 4(%ecx)
-; FALLBACK24-NEXT:    movl %edi, 24(%ecx)
-; FALLBACK24-NEXT:    movl %ebp, 16(%ecx)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK24-NEXT:    movl %eax, 20(%ecx)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK24-NEXT:    movl %eax, 8(%ecx)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK24-NEXT:    movl %eax, 12(%ecx)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK24-NEXT:    movl %eax, (%ecx)
-; FALLBACK24-NEXT:    addl $108, %esp
-; FALLBACK24-NEXT:    popl %esi
-; FALLBACK24-NEXT:    popl %edi
-; FALLBACK24-NEXT:    popl %ebx
-; FALLBACK24-NEXT:    popl %ebp
-; FALLBACK24-NEXT:    retl
-;
-; FALLBACK25-LABEL: ashr_32bytes:
-; FALLBACK25:       # %bb.0:
-; FALLBACK25-NEXT:    pushl %ebp
-; FALLBACK25-NEXT:    pushl %ebx
-; FALLBACK25-NEXT:    pushl %edi
-; FALLBACK25-NEXT:    pushl %esi
-; FALLBACK25-NEXT:    subl $108, %esp
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK25-NEXT:    vmovups (%ecx), %xmm0
-; FALLBACK25-NEXT:    movl 16(%ecx), %esi
-; FALLBACK25-NEXT:    movl 20(%ecx), %edi
-; FALLBACK25-NEXT:    movl 24(%ecx), %ebx
-; FALLBACK25-NEXT:    movl 28(%ecx), %edx
-; FALLBACK25-NEXT:    movzbl (%eax), %eax
-; FALLBACK25-NEXT:    movl %eax, %ecx
-; FALLBACK25-NEXT:    shlb $3, %cl
-; FALLBACK25-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    sarl $31, %edx
-; FALLBACK25-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    andb $28, %al
-; FALLBACK25-NEXT:    movzbl %al, %ebp
-; FALLBACK25-NEXT:    movl 48(%esp,%ebp), %esi
-; FALLBACK25-NEXT:    movl 44(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, %edx
-; FALLBACK25-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 40(%esp,%ebp), %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 56(%esp,%ebp), %ebx
-; FALLBACK25-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, %edx
-; FALLBACK25-NEXT:    shrdl %cl, %ebx, %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK25-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK25-NEXT:    movl 32(%esp,%ebp), %edx
-; FALLBACK25-NEXT:    movl 36(%esp,%ebp), %edi
-; FALLBACK25-NEXT:    movl %edi, %esi
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; FALLBACK25-NEXT:    shrdl %cl, %ebp, %esi
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK25-NEXT:    movl %esi, 4(%ebp)
-; FALLBACK25-NEXT:    movl %ebx, 24(%ebp)
-; FALLBACK25-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK25-NEXT:    sarl %cl, %eax
-; FALLBACK25-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK25-NEXT:    movl %edx, (%ebp)
-; FALLBACK25-NEXT:    addl $108, %esp
-; FALLBACK25-NEXT:    popl %esi
-; FALLBACK25-NEXT:    popl %edi
-; FALLBACK25-NEXT:    popl %ebx
-; FALLBACK25-NEXT:    popl %ebp
-; FALLBACK25-NEXT:    retl
-;
-; FALLBACK26-LABEL: ashr_32bytes:
-; FALLBACK26:       # %bb.0:
-; FALLBACK26-NEXT:    pushl %ebp
-; FALLBACK26-NEXT:    pushl %ebx
-; FALLBACK26-NEXT:    pushl %edi
-; FALLBACK26-NEXT:    pushl %esi
-; FALLBACK26-NEXT:    subl $108, %esp
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK26-NEXT:    vmovups (%ecx), %xmm0
-; FALLBACK26-NEXT:    movl 16(%ecx), %esi
-; FALLBACK26-NEXT:    movl 20(%ecx), %edi
-; FALLBACK26-NEXT:    movl 24(%ecx), %ebp
-; FALLBACK26-NEXT:    movl 28(%ecx), %ecx
-; FALLBACK26-NEXT:    movzbl (%eax), %edx
-; FALLBACK26-NEXT:    movl %edx, %ebx
-; FALLBACK26-NEXT:    shlb $3, %bl
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    sarl $31, %ecx
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ebx, %eax
-; FALLBACK26-NEXT:    andb $28, %dl
-; FALLBACK26-NEXT:    movzbl %dl, %ecx
-; FALLBACK26-NEXT:    shrxl %eax, 32(%esp,%ecx), %edx
-; FALLBACK26-NEXT:    movl %eax, %ebp
-; FALLBACK26-NEXT:    notb %bl
-; FALLBACK26-NEXT:    movl 36(%esp,%ecx), %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    leal (%eax,%eax), %esi
-; FALLBACK26-NEXT:    shlxl %ebx, %esi, %eax
-; FALLBACK26-NEXT:    orl %edx, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 48(%esp,%ecx), %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    leal (%eax,%eax), %edx
-; FALLBACK26-NEXT:    shlxl %ebx, %edx, %edi
-; FALLBACK26-NEXT:    movl 44(%esp,%ecx), %edx
-; FALLBACK26-NEXT:    shrxl %ebp, %edx, %esi
-; FALLBACK26-NEXT:    orl %esi, %edi
-; FALLBACK26-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    addl %edx, %edx
-; FALLBACK26-NEXT:    shlxl %ebx, %edx, %edi
-; FALLBACK26-NEXT:    movl 40(%esp,%ecx), %edx
-; FALLBACK26-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrxl %ebp, %edx, %esi
-; FALLBACK26-NEXT:    movl %ebp, %edx
-; FALLBACK26-NEXT:    orl %esi, %edi
-; FALLBACK26-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 56(%esp,%ecx), %esi
-; FALLBACK26-NEXT:    leal (%esi,%esi), %ebp
-; FALLBACK26-NEXT:    shlxl %ebx, %ebp, %ebp
-; FALLBACK26-NEXT:    movl 52(%esp,%ecx), %eax
-; FALLBACK26-NEXT:    shrxl %edx, %eax, %edi
-; FALLBACK26-NEXT:    orl %edi, %ebp
-; FALLBACK26-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK26-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    addl %eax, %eax
-; FALLBACK26-NEXT:    shlxl %ebx, %eax, %edi
-; FALLBACK26-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK26-NEXT:    shrxl %edx, %esi, %eax
-; FALLBACK26-NEXT:    movl 60(%esp,%ecx), %ecx
-; FALLBACK26-NEXT:    leal (%ecx,%ecx), %esi
-; FALLBACK26-NEXT:    shlxl %ebx, %esi, %esi
-; FALLBACK26-NEXT:    orl %eax, %esi
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    addl %eax, %eax
-; FALLBACK26-NEXT:    shlxl %ebx, %eax, %eax
-; FALLBACK26-NEXT:    movl %edx, %ebx
-; FALLBACK26-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; FALLBACK26-NEXT:    orl %edx, %eax
-; FALLBACK26-NEXT:    sarxl %ebx, %ecx, %ecx
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK26-NEXT:    movl %ecx, 28(%edx)
-; FALLBACK26-NEXT:    movl %eax, 4(%edx)
-; FALLBACK26-NEXT:    movl %esi, 24(%edx)
-; FALLBACK26-NEXT:    movl %edi, 16(%edx)
-; FALLBACK26-NEXT:    movl %ebp, 20(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 8(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 12(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, (%edx)
-; FALLBACK26-NEXT:    addl $108, %esp
-; FALLBACK26-NEXT:    popl %esi
-; FALLBACK26-NEXT:    popl %edi
-; FALLBACK26-NEXT:    popl %ebx
-; FALLBACK26-NEXT:    popl %ebp
-; FALLBACK26-NEXT:    retl
-;
-; FALLBACK27-LABEL: ashr_32bytes:
-; FALLBACK27:       # %bb.0:
-; FALLBACK27-NEXT:    pushl %ebp
-; FALLBACK27-NEXT:    pushl %ebx
-; FALLBACK27-NEXT:    pushl %edi
-; FALLBACK27-NEXT:    pushl %esi
-; FALLBACK27-NEXT:    subl $108, %esp
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK27-NEXT:    vmovups (%ecx), %xmm0
-; FALLBACK27-NEXT:    movl 16(%ecx), %esi
-; FALLBACK27-NEXT:    movl 20(%ecx), %edi
-; FALLBACK27-NEXT:    movl 24(%ecx), %ebx
-; FALLBACK27-NEXT:    movl 28(%ecx), %edx
-; FALLBACK27-NEXT:    movzbl (%eax), %eax
-; FALLBACK27-NEXT:    movl %eax, %ecx
-; FALLBACK27-NEXT:    shlb $3, %cl
-; FALLBACK27-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    sarl $31, %edx
-; FALLBACK27-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    andb $28, %al
-; FALLBACK27-NEXT:    movzbl %al, %ebx
-; FALLBACK27-NEXT:    movl 48(%esp,%ebx), %esi
-; FALLBACK27-NEXT:    movl 44(%esp,%ebx), %eax
-; FALLBACK27-NEXT:    movl %eax, %edx
-; FALLBACK27-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 40(%esp,%ebx), %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 56(%esp,%ebx), %ebp
-; FALLBACK27-NEXT:    movl 52(%esp,%ebx), %eax
-; FALLBACK27-NEXT:    movl %eax, %edi
-; FALLBACK27-NEXT:    shrdl %cl, %ebp, %edi
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK27-NEXT:    movl 60(%esp,%ebx), %eax
-; FALLBACK27-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %ebp
-; FALLBACK27-NEXT:    movl 32(%esp,%ebx), %edx
-; FALLBACK27-NEXT:    movl 36(%esp,%ebx), %ebx
-; FALLBACK27-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK27-NEXT:    movl %ebx, 4(%eax)
-; FALLBACK27-NEXT:    movl %ebp, 24(%eax)
-; FALLBACK27-NEXT:    sarxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; FALLBACK27-NEXT:    movl %ebx, 28(%eax)
-; FALLBACK27-NEXT:    movl %esi, 16(%eax)
-; FALLBACK27-NEXT:    movl %edi, 20(%eax)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK27-NEXT:    movl %esi, 8(%eax)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK27-NEXT:    movl %esi, 12(%eax)
-; FALLBACK27-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK27-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK27-NEXT:    movl %edx, (%eax)
-; FALLBACK27-NEXT:    addl $108, %esp
-; FALLBACK27-NEXT:    popl %esi
-; FALLBACK27-NEXT:    popl %edi
-; FALLBACK27-NEXT:    popl %ebx
-; FALLBACK27-NEXT:    popl %ebp
-; FALLBACK27-NEXT:    retl
-;
-; FALLBACK28-LABEL: ashr_32bytes:
-; FALLBACK28:       # %bb.0:
-; FALLBACK28-NEXT:    pushl %ebp
-; FALLBACK28-NEXT:    pushl %ebx
-; FALLBACK28-NEXT:    pushl %edi
-; FALLBACK28-NEXT:    pushl %esi
-; FALLBACK28-NEXT:    subl $108, %esp
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK28-NEXT:    vmovups (%ecx), %xmm0
-; FALLBACK28-NEXT:    movl 16(%ecx), %esi
-; FALLBACK28-NEXT:    movl 20(%ecx), %edi
-; FALLBACK28-NEXT:    movl 24(%ecx), %ebx
-; FALLBACK28-NEXT:    movl 28(%ecx), %edx
-; FALLBACK28-NEXT:    movzbl (%eax), %eax
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shlb $3, %cl
-; FALLBACK28-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    sarl $31, %edx
-; FALLBACK28-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    andb $28, %al
-; FALLBACK28-NEXT:    movzbl %al, %edi
-; FALLBACK28-NEXT:    movl 32(%esp,%edi), %eax
-; FALLBACK28-NEXT:    movl 36(%esp,%edi), %esi
-; FALLBACK28-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    shrl %cl, %eax
-; FALLBACK28-NEXT:    movl %ecx, %edx
-; FALLBACK28-NEXT:    movb %cl, %dh
-; FALLBACK28-NEXT:    notb %dl
-; FALLBACK28-NEXT:    addl %esi, %esi
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shll %cl, %esi
-; FALLBACK28-NEXT:    orl %eax, %esi
-; FALLBACK28-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 44(%esp,%edi), %ebx
-; FALLBACK28-NEXT:    movl %ebx, %eax
-; FALLBACK28-NEXT:    movb %dh, %cl
-; FALLBACK28-NEXT:    shrl %cl, %eax
-; FALLBACK28-NEXT:    movl 48(%esp,%edi), %esi
-; FALLBACK28-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    addl %esi, %esi
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shll %cl, %esi
-; FALLBACK28-NEXT:    orl %eax, %esi
-; FALLBACK28-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 40(%esp,%edi), %esi
-; FALLBACK28-NEXT:    movl %esi, %eax
-; FALLBACK28-NEXT:    movb %dh, %cl
-; FALLBACK28-NEXT:    shrl %cl, %eax
-; FALLBACK28-NEXT:    addl %ebx, %ebx
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    orl %eax, %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 52(%esp,%edi), %ebp
-; FALLBACK28-NEXT:    movl %ebp, %eax
-; FALLBACK28-NEXT:    movb %dh, %cl
-; FALLBACK28-NEXT:    shrl %cl, %eax
-; FALLBACK28-NEXT:    movl 56(%esp,%edi), %ecx
-; FALLBACK28-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    orl %eax, %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %dh, %cl
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %cl, %eax
-; FALLBACK28-NEXT:    addl %ebp, %ebp
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shll %cl, %ebp
-; FALLBACK28-NEXT:    orl %eax, %ebp
-; FALLBACK28-NEXT:    movb %dh, %cl
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %cl, %ebx
-; FALLBACK28-NEXT:    movl 60(%esp,%edi), %eax
-; FALLBACK28-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shll %cl, %edi
-; FALLBACK28-NEXT:    orl %ebx, %edi
-; FALLBACK28-NEXT:    movb %dh, %cl
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %cl, %ebx
-; FALLBACK28-NEXT:    addl %esi, %esi
-; FALLBACK28-NEXT:    movl %edx, %ecx
-; FALLBACK28-NEXT:    shll %cl, %esi
-; FALLBACK28-NEXT:    orl %ebx, %esi
-; FALLBACK28-NEXT:    movb %dh, %cl
-; FALLBACK28-NEXT:    sarl %cl, %eax
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK28-NEXT:    movl %eax, 28(%ecx)
-; FALLBACK28-NEXT:    movl %esi, 4(%ecx)
-; FALLBACK28-NEXT:    movl %edi, 24(%ecx)
-; FALLBACK28-NEXT:    movl %ebp, 16(%ecx)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK28-NEXT:    movl %eax, 20(%ecx)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK28-NEXT:    movl %eax, 8(%ecx)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK28-NEXT:    movl %eax, 12(%ecx)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK28-NEXT:    movl %eax, (%ecx)
-; FALLBACK28-NEXT:    addl $108, %esp
-; FALLBACK28-NEXT:    popl %esi
-; FALLBACK28-NEXT:    popl %edi
-; FALLBACK28-NEXT:    popl %ebx
-; FALLBACK28-NEXT:    popl %ebp
-; FALLBACK28-NEXT:    retl
-;
-; FALLBACK29-LABEL: ashr_32bytes:
-; FALLBACK29:       # %bb.0:
-; FALLBACK29-NEXT:    pushl %ebp
-; FALLBACK29-NEXT:    pushl %ebx
-; FALLBACK29-NEXT:    pushl %edi
-; FALLBACK29-NEXT:    pushl %esi
-; FALLBACK29-NEXT:    subl $108, %esp
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK29-NEXT:    vmovups (%ecx), %xmm0
-; FALLBACK29-NEXT:    movl 16(%ecx), %esi
-; FALLBACK29-NEXT:    movl 20(%ecx), %edi
-; FALLBACK29-NEXT:    movl 24(%ecx), %ebx
-; FALLBACK29-NEXT:    movl 28(%ecx), %edx
-; FALLBACK29-NEXT:    movzbl (%eax), %eax
-; FALLBACK29-NEXT:    movl %eax, %ecx
-; FALLBACK29-NEXT:    shlb $3, %cl
-; FALLBACK29-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    sarl $31, %edx
-; FALLBACK29-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    andb $28, %al
-; FALLBACK29-NEXT:    movzbl %al, %ebp
-; FALLBACK29-NEXT:    movl 48(%esp,%ebp), %esi
-; FALLBACK29-NEXT:    movl 44(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, %edx
-; FALLBACK29-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 40(%esp,%ebp), %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 56(%esp,%ebp), %ebx
-; FALLBACK29-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, %edx
-; FALLBACK29-NEXT:    shrdl %cl, %ebx, %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK29-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK29-NEXT:    movl 32(%esp,%ebp), %edx
-; FALLBACK29-NEXT:    movl 36(%esp,%ebp), %edi
-; FALLBACK29-NEXT:    movl %edi, %esi
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; FALLBACK29-NEXT:    shrdl %cl, %ebp, %esi
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK29-NEXT:    movl %esi, 4(%ebp)
-; FALLBACK29-NEXT:    movl %ebx, 24(%ebp)
-; FALLBACK29-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK29-NEXT:    sarl %cl, %eax
-; FALLBACK29-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK29-NEXT:    movl %edx, (%ebp)
-; FALLBACK29-NEXT:    addl $108, %esp
-; FALLBACK29-NEXT:    popl %esi
-; FALLBACK29-NEXT:    popl %edi
-; FALLBACK29-NEXT:    popl %ebx
-; FALLBACK29-NEXT:    popl %ebp
-; FALLBACK29-NEXT:    retl
-;
-; FALLBACK30-LABEL: ashr_32bytes:
-; FALLBACK30:       # %bb.0:
-; FALLBACK30-NEXT:    pushl %ebp
-; FALLBACK30-NEXT:    pushl %ebx
-; FALLBACK30-NEXT:    pushl %edi
-; FALLBACK30-NEXT:    pushl %esi
-; FALLBACK30-NEXT:    subl $108, %esp
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK30-NEXT:    vmovups (%ecx), %xmm0
-; FALLBACK30-NEXT:    movl 16(%ecx), %esi
-; FALLBACK30-NEXT:    movl 20(%ecx), %edi
-; FALLBACK30-NEXT:    movl 24(%ecx), %ebp
-; FALLBACK30-NEXT:    movl 28(%ecx), %ecx
-; FALLBACK30-NEXT:    movzbl (%eax), %edx
-; FALLBACK30-NEXT:    movl %edx, %ebx
-; FALLBACK30-NEXT:    shlb $3, %bl
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    sarl $31, %ecx
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ebx, %eax
-; FALLBACK30-NEXT:    andb $28, %dl
-; FALLBACK30-NEXT:    movzbl %dl, %ecx
-; FALLBACK30-NEXT:    shrxl %eax, 32(%esp,%ecx), %edx
-; FALLBACK30-NEXT:    movl %eax, %ebp
-; FALLBACK30-NEXT:    notb %bl
-; FALLBACK30-NEXT:    movl 36(%esp,%ecx), %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    leal (%eax,%eax), %esi
-; FALLBACK30-NEXT:    shlxl %ebx, %esi, %eax
-; FALLBACK30-NEXT:    orl %edx, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 48(%esp,%ecx), %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    leal (%eax,%eax), %edx
-; FALLBACK30-NEXT:    shlxl %ebx, %edx, %edi
-; FALLBACK30-NEXT:    movl 44(%esp,%ecx), %edx
-; FALLBACK30-NEXT:    shrxl %ebp, %edx, %esi
-; FALLBACK30-NEXT:    orl %esi, %edi
-; FALLBACK30-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    addl %edx, %edx
-; FALLBACK30-NEXT:    shlxl %ebx, %edx, %edi
-; FALLBACK30-NEXT:    movl 40(%esp,%ecx), %edx
-; FALLBACK30-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrxl %ebp, %edx, %esi
-; FALLBACK30-NEXT:    movl %ebp, %edx
-; FALLBACK30-NEXT:    orl %esi, %edi
-; FALLBACK30-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 56(%esp,%ecx), %esi
-; FALLBACK30-NEXT:    leal (%esi,%esi), %ebp
-; FALLBACK30-NEXT:    shlxl %ebx, %ebp, %ebp
-; FALLBACK30-NEXT:    movl 52(%esp,%ecx), %eax
-; FALLBACK30-NEXT:    shrxl %edx, %eax, %edi
-; FALLBACK30-NEXT:    orl %edi, %ebp
-; FALLBACK30-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK30-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    addl %eax, %eax
-; FALLBACK30-NEXT:    shlxl %ebx, %eax, %edi
-; FALLBACK30-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK30-NEXT:    shrxl %edx, %esi, %eax
-; FALLBACK30-NEXT:    movl 60(%esp,%ecx), %ecx
-; FALLBACK30-NEXT:    leal (%ecx,%ecx), %esi
-; FALLBACK30-NEXT:    shlxl %ebx, %esi, %esi
-; FALLBACK30-NEXT:    orl %eax, %esi
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    addl %eax, %eax
-; FALLBACK30-NEXT:    shlxl %ebx, %eax, %eax
-; FALLBACK30-NEXT:    movl %edx, %ebx
-; FALLBACK30-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; FALLBACK30-NEXT:    orl %edx, %eax
-; FALLBACK30-NEXT:    sarxl %ebx, %ecx, %ecx
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK30-NEXT:    movl %ecx, 28(%edx)
-; FALLBACK30-NEXT:    movl %eax, 4(%edx)
-; FALLBACK30-NEXT:    movl %esi, 24(%edx)
-; FALLBACK30-NEXT:    movl %edi, 16(%edx)
-; FALLBACK30-NEXT:    movl %ebp, 20(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 8(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 12(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, (%edx)
-; FALLBACK30-NEXT:    addl $108, %esp
-; FALLBACK30-NEXT:    popl %esi
-; FALLBACK30-NEXT:    popl %edi
-; FALLBACK30-NEXT:    popl %ebx
-; FALLBACK30-NEXT:    popl %ebp
-; FALLBACK30-NEXT:    retl
-;
-; FALLBACK31-LABEL: ashr_32bytes:
-; FALLBACK31:       # %bb.0:
-; FALLBACK31-NEXT:    pushl %ebp
-; FALLBACK31-NEXT:    pushl %ebx
-; FALLBACK31-NEXT:    pushl %edi
-; FALLBACK31-NEXT:    pushl %esi
-; FALLBACK31-NEXT:    subl $108, %esp
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK31-NEXT:    vmovups (%ecx), %xmm0
-; FALLBACK31-NEXT:    movl 16(%ecx), %esi
-; FALLBACK31-NEXT:    movl 20(%ecx), %edi
-; FALLBACK31-NEXT:    movl 24(%ecx), %ebx
-; FALLBACK31-NEXT:    movl 28(%ecx), %edx
-; FALLBACK31-NEXT:    movzbl (%eax), %eax
-; FALLBACK31-NEXT:    movl %eax, %ecx
-; FALLBACK31-NEXT:    shlb $3, %cl
-; FALLBACK31-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    sarl $31, %edx
-; FALLBACK31-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    andb $28, %al
-; FALLBACK31-NEXT:    movzbl %al, %ebx
-; FALLBACK31-NEXT:    movl 48(%esp,%ebx), %esi
-; FALLBACK31-NEXT:    movl 44(%esp,%ebx), %eax
-; FALLBACK31-NEXT:    movl %eax, %edx
-; FALLBACK31-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 40(%esp,%ebx), %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 56(%esp,%ebx), %ebp
-; FALLBACK31-NEXT:    movl 52(%esp,%ebx), %eax
-; FALLBACK31-NEXT:    movl %eax, %edi
-; FALLBACK31-NEXT:    shrdl %cl, %ebp, %edi
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK31-NEXT:    movl 60(%esp,%ebx), %eax
-; FALLBACK31-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %ebp
-; FALLBACK31-NEXT:    movl 32(%esp,%ebx), %edx
-; FALLBACK31-NEXT:    movl 36(%esp,%ebx), %ebx
-; FALLBACK31-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK31-NEXT:    movl %ebx, 4(%eax)
-; FALLBACK31-NEXT:    movl %ebp, 24(%eax)
-; FALLBACK31-NEXT:    sarxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; FALLBACK31-NEXT:    movl %ebx, 28(%eax)
-; FALLBACK31-NEXT:    movl %esi, 16(%eax)
-; FALLBACK31-NEXT:    movl %edi, 20(%eax)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK31-NEXT:    movl %esi, 8(%eax)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK31-NEXT:    movl %esi, 12(%eax)
-; FALLBACK31-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; FALLBACK31-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK31-NEXT:    movl %edx, (%eax)
-; FALLBACK31-NEXT:    addl $108, %esp
-; FALLBACK31-NEXT:    popl %esi
-; FALLBACK31-NEXT:    popl %edi
-; FALLBACK31-NEXT:    popl %ebx
-; FALLBACK31-NEXT:    popl %ebp
-; FALLBACK31-NEXT:    retl
+; X64-NO-SHLD-NO-BMI2-SSE2-LABEL: ashr_32bytes:
+; X64-NO-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    sarq $63, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    andb $24, %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movzbl %sil, %r9d
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -64(%rsp,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -56(%rsp,%r9), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -48(%rsp,%r9), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leaq (%rbx,%rbx), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r11, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    addq %rdi, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -40(%rsp,%r9), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leaq (%r9,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %rbx, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    sarq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE2-LABEL: ashr_32bytes:
+; X64-HAVE-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    leal (,%rsi,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andb $24, %sil
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movzbl %sil, %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -56(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -72(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -64(%rsp,%rax), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %rsi, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -48(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %rax, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %r8, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    sarq %cl, %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE2-LABEL: ashr_32bytes:
+; X64-NO-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    sarq $63, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $24, %sil
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl %sil, %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -64(%rsp,%rsi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -56(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rcx, %rdi, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leaq (%r8,%r8), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rax, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r9, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rcx, -72(%rsp,%rsi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addq %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rax, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r9, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rcx, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -48(%rsp,%rsi), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leaq (%rsi,%rsi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rax, %r9, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r8, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    sarxq %rcx, %rsi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r10, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: ashr_32bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (,%rsi,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $24, %sil
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl %sil, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -56(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -72(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -64(%rsp,%rax), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %rsi, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -48(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %rax, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %r8, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    sarxq %rcx, %rax, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-SSE4-LABEL: ashr_32bytes:
+; X64-NO-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq 16(%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    sarq $63, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    andb $24, %sil
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl %sil, %r9d
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -64(%rsp,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -56(%rsp,%r9), %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leaq (%r8,%r8), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -48(%rsp,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -40(%rsp,%r9), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leaq (%r9,%r9), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    addq %r10, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r8, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    sarq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rbx, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE4-LABEL: ashr_32bytes:
+; X64-HAVE-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq 16(%rdi), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    leal (,%rsi,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andb $24, %sil
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl %sil, %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -48(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -56(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %rsi, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -72(%rsp,%rax), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -64(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rax, %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %rdi, %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %rax, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    sarq %cl, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE4-LABEL: ashr_32bytes:
+; X64-NO-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq 16(%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    sarq $63, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $24, %sil
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl %sil, %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rcx, -72(%rsp,%rsi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -64(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -56(%rsp,%rsi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leaq (%r8,%r8), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rax, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rdi, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rcx, %r9, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -48(%rsp,%rsi), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leaq (%rsi,%rsi), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rax, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rdi, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rcx, %r8, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addq %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rax, %r9, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rdi, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    sarxq %rcx, %rsi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rcx, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: ashr_32bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq 16(%rdi), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (,%rsi,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $24, %sil
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl %sil, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -48(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -56(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %rsi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -72(%rsp,%rax), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -64(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %rdi, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %rax, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    sarxq %rcx, %rsi, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r10, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-AVX-LABEL: ashr_32bytes:
+; X64-NO-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%rdi), %xmm0
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq 16(%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    sarq $63, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    andb $24, %sil
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl %sil, %r9d
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -64(%rsp,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -56(%rsp,%r9), %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    leaq (%r8,%r8), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -48(%rsp,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -40(%rsp,%r9), %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    leaq (%r9,%r9), %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    addq %r10, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r8, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    sarq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rbx, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-AVX-LABEL: ashr_32bytes:
+; X64-HAVE-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%rdi), %xmm0
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq 16(%rdi), %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    leal (,%rsi,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    andb $24, %sil
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl %sil, %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -48(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -56(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %rsi, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -72(%rsp,%rax), %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -64(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rax, %r10
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %rdi, %r10
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %rax, %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    sarq %cl, %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-AVX-LABEL: ashr_32bytes:
+; X64-NO-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%rdi), %xmm0
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq 16(%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    sarq $63, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    andb $24, %sil
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl %sil, %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rcx, -72(%rsp,%rsi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -64(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -56(%rsp,%rsi), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leaq (%r8,%r8), %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rax, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %rdi, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rcx, %r9, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -48(%rsp,%rsi), %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leaq (%rsi,%rsi), %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rax, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %rdi, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rcx, %r8, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addq %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rax, %r9, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %rdi, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    sarxq %rcx, %rsi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rcx, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-LABEL: ashr_32bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%rdi), %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq 16(%rdi), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    leal (,%rsi,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    andb $24, %sil
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl %sil, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -48(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -56(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %rsi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -72(%rsp,%rax), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -64(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %rdi, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %rax, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    sarxq %rcx, %rsi, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r10, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    retq
+;
+; X86-NO-SHLD-NO-BMI2-SSE2-LABEL: ashr_32bytes:
+; X86-NO-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    subl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl (%esi), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%esi), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%esi), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%esi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movzbl (%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%esi), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%esi), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%esi), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlb $3, %dl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    sarl $31, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    andb $28, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movzbl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 32(%esp,%edi), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 36(%esp,%edi), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    notb %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 40(%esp,%edi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%edi,%edi), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 44(%esp,%eax), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 48(%esp,%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%edx,%edx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %bl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %ebp, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 52(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %bl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 56(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%ebx,%ebx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 60(%esp,%eax), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%eax,%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    sarl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 28(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, 24(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, 16(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, 20(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, 8(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 12(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, (%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 4(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-SSE2-LABEL: ashr_32bytes:
+; X86-HAVE-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    subl $92, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%ecx), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%ecx), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%ecx), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%ecx), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%ecx), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movzbl (%eax), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%ecx), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%ecx), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%ecx), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%esp), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    sarl $31, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andb $28, %al
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movzbl %al, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 32(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %ebx, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 40(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 36(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 44(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, 24(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    sarl %cl, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, 16(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, 20(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    addl $92, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-SSE2-LABEL: ashr_32bytes:
+; X86-NO-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    subl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%esi), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%esi), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%esi), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%esi), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%edx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%esi), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%esi), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 28(%esi), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $3, %cl
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    sarl $31, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $28, %dl
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl %dl, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 36(%esp,%esi), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 40(%esp,%esi), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %eax, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    notb %cl
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%ebp,%ebp), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %ebx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %eax, 32(%esp,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 48(%esp,%esi), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 44(%esp,%esi), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %eax, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %eax, %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 56(%esp,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%edi,%edi), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 52(%esp,%esi), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebp, %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 60(%esp,%esi), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%esi,%esi), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %ebx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebp, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    sarxl %ebp, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, 28(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 24(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 16(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, 20(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 8(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 12(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, (%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 4(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: ashr_32bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    subl $92, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%ecx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%ecx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%ecx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%ecx), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%ecx), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%eax), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%ecx), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%ecx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 28(%ecx), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%esp), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    sarl $31, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $28, %al
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl %al, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%esp,%ebp), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %esi, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 32(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 28(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %ebx, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 40(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 36(%esp,%ebp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%esp,%ebp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 44(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %edi, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    sarxl %ecx, %edi, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, 16(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, 20(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, (%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 4(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    addl $92, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-SSE4-LABEL: ashr_32bytes:
+; X86-NO-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    subl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 16(%ecx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 20(%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 24(%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 28(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlb $3, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    sarl $31, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    andb $28, %al
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl %al, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 32(%esp,%edi), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 36(%esp,%edi), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %cl, %dh
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    notb %dl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 44(%esp,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 48(%esp,%edi), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 40(%esp,%edi), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 52(%esp,%edi), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 56(%esp,%edi), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%ecx,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %ebp, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %eax, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 60(%esp,%edi), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%eax,%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    sarl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 28(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, 4(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, 24(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, 16(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 20(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 8(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 12(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, (%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-SSE4-LABEL: ashr_32bytes:
+; X86-HAVE-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    subl $108, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 16(%ecx), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 20(%ecx), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 24(%ecx), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 28(%ecx), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%eax), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    sarl $31, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andb $28, %al
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl %al, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 48(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 44(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 40(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 56(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %ebx, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 32(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 36(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %ebp, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, 24(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    sarl %cl, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    addl $108, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-SSE4-LABEL: ashr_32bytes:
+; X86-NO-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    subl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 16(%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 20(%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 24(%ecx), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 28(%ecx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $3, %bl
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    sarl $31, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $28, %dl
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl %dl, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %eax, 32(%esp,%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    notb %bl
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 36(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%eax,%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 48(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%eax,%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebx, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 44(%esp,%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebp, %edx, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebx, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 40(%esp,%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebp, %edx, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 56(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%esi,%esi), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebx, %ebp, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 52(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %edx, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebx, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 60(%esp,%ecx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%ecx,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebx, %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    sarxl %ebx, %ecx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 28(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 4(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, 24(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, 16(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, 20(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 8(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, (%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: ashr_32bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    subl $108, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 16(%ecx), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 20(%ecx), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 24(%ecx), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 28(%ecx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%eax), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    sarl $31, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $28, %al
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl %al, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 48(%esp,%ebx), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 44(%esp,%ebx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 40(%esp,%ebx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 56(%esp,%ebx), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 52(%esp,%ebx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %ebp, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 60(%esp,%ebx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 32(%esp,%ebx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 36(%esp,%ebx), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, 4(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, 24(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    sarxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, 28(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, 16(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, 20(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, 8(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, 12(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, (%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    addl $108, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-AVX-LABEL: ashr_32bytes:
+; X86-NO-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    subl $108, %esp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%ecx), %xmm0
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 16(%ecx), %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 20(%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 24(%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 28(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shlb $3, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    sarl $31, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    andb $28, %al
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl %al, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 32(%esp,%edi), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 36(%esp,%edi), %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %cl, %dh
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    notb %dl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 44(%esp,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 48(%esp,%edi), %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 40(%esp,%edi), %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 52(%esp,%edi), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 56(%esp,%edi), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    leal (%ecx,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %ebp, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %eax, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 60(%esp,%edi), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    leal (%eax,%eax), %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    sarl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 28(%ecx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, 4(%ecx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, 24(%ecx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebp, 16(%ecx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 20(%ecx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 8(%ecx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 12(%ecx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, (%ecx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl $108, %esp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-AVX-LABEL: ashr_32bytes:
+; X86-HAVE-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    subl $108, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%ecx), %xmm0
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 16(%ecx), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 20(%ecx), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 24(%ecx), %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 28(%ecx), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%eax), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    sarl $31, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    andb $28, %al
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl %al, %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 48(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 44(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 40(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 56(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %ebx, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 32(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 36(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %ebp, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, 24(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    sarl %cl, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    addl $108, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-AVX-LABEL: ashr_32bytes:
+; X86-NO-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    subl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%ecx), %xmm0
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 16(%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 20(%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 24(%ecx), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 28(%ecx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $3, %bl
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    sarl $31, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    andb $28, %dl
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl %dl, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %eax, 32(%esp,%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    notb %bl
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 36(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (%eax,%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ebx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 48(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (%eax,%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ebx, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 44(%esp,%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ebp, %edx, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ebx, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 40(%esp,%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ebp, %edx, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebp, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 56(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (%esi,%esi), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ebx, %ebp, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 52(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %edx, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ebx, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 60(%esp,%ecx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (%ecx,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %ebx, %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    sarxl %ebx, %ecx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 28(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 4(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, 24(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, 16(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebp, 20(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 8(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, (%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-LABEL: ashr_32bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    subl $108, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%ecx), %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 16(%ecx), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 20(%ecx), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 24(%ecx), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 28(%ecx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%eax), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $3, %cl
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    sarl $31, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    andb $28, %al
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl %al, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 48(%esp,%ebx), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 44(%esp,%ebx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 40(%esp,%ebx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 56(%esp,%ebx), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 52(%esp,%ebx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %ebp, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 60(%esp,%ebx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %eax, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 32(%esp,%ebx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 36(%esp,%ebx), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebx, 4(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebp, 24(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    sarxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebx, 28(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, 16(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, 20(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, 8(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, 12(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, (%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    addl $108, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    retl
   %src = load i256, ptr %src.ptr, align 1
   %byteOff = load i256, ptr %byteOff.ptr, align 1
   %bitOff = shl i256 %byteOff, 3
@@ -10984,663 +8868,500 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
 }
 
 define void @ashr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nounwind {
-; FALLBACK0-LABEL: ashr_32bytes_dwordOff:
-; FALLBACK0:       # %bb.0:
-; FALLBACK0-NEXT:    pushq %rbx
-; FALLBACK0-NEXT:    movq (%rdi), %rcx
-; FALLBACK0-NEXT:    movq 8(%rdi), %r8
-; FALLBACK0-NEXT:    movq 16(%rdi), %r9
-; FALLBACK0-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK0-NEXT:    movzbl (%rsi), %esi
-; FALLBACK0-NEXT:    movl %esi, %eax
-; FALLBACK0-NEXT:    shlb $5, %al
-; FALLBACK0-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    sarq $63, %rdi
-; FALLBACK0-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    andb $6, %sil
-; FALLBACK0-NEXT:    movzbl %sil, %r9d
-; FALLBACK0-NEXT:    movq -64(%rsp,%r9,4), %r10
-; FALLBACK0-NEXT:    movq -56(%rsp,%r9,4), %rdi
-; FALLBACK0-NEXT:    movq %rdi, %r11
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r11
-; FALLBACK0-NEXT:    movl %eax, %esi
-; FALLBACK0-NEXT:    notb %sil
-; FALLBACK0-NEXT:    movq -48(%rsp,%r9,4), %rbx
-; FALLBACK0-NEXT:    leaq (%rbx,%rbx), %r8
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r8
-; FALLBACK0-NEXT:    orq %r11, %r8
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r10
-; FALLBACK0-NEXT:    addq %rdi, %rdi
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %rdi
-; FALLBACK0-NEXT:    orq %r10, %rdi
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %rbx
-; FALLBACK0-NEXT:    movq -40(%rsp,%r9,4), %r9
-; FALLBACK0-NEXT:    leaq (%r9,%r9), %r10
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r10
-; FALLBACK0-NEXT:    orq %rbx, %r10
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    sarq %cl, %r9
-; FALLBACK0-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK0-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK0-NEXT:    movq %rdi, (%rdx)
-; FALLBACK0-NEXT:    movq %r8, 8(%rdx)
-; FALLBACK0-NEXT:    popq %rbx
-; FALLBACK0-NEXT:    retq
-;
-; FALLBACK1-LABEL: ashr_32bytes_dwordOff:
-; FALLBACK1:       # %bb.0:
-; FALLBACK1-NEXT:    movq (%rdi), %rax
-; FALLBACK1-NEXT:    movq 8(%rdi), %r8
-; FALLBACK1-NEXT:    movq 16(%rdi), %r9
-; FALLBACK1-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK1-NEXT:    movzbl (%rsi), %esi
-; FALLBACK1-NEXT:    movl %esi, %ecx
-; FALLBACK1-NEXT:    shlb $5, %cl
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    sarq $63, %rdi
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    andb $6, %sil
-; FALLBACK1-NEXT:    movzbl %sil, %eax
-; FALLBACK1-NEXT:    movq -56(%rsp,%rax,4), %rsi
-; FALLBACK1-NEXT:    movq -72(%rsp,%rax,4), %rdi
-; FALLBACK1-NEXT:    movq -64(%rsp,%rax,4), %r8
-; FALLBACK1-NEXT:    movq %r8, %r9
-; FALLBACK1-NEXT:    shrdq %cl, %rsi, %r9
-; FALLBACK1-NEXT:    movq -48(%rsp,%rax,4), %rax
-; FALLBACK1-NEXT:    shrdq %cl, %rax, %rsi
-; FALLBACK1-NEXT:    shrdq %cl, %r8, %rdi
-; FALLBACK1-NEXT:    sarq %cl, %rax
-; FALLBACK1-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK1-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK1-NEXT:    movq %rdi, (%rdx)
-; FALLBACK1-NEXT:    movq %r9, 8(%rdx)
-; FALLBACK1-NEXT:    retq
-;
-; FALLBACK2-LABEL: ashr_32bytes_dwordOff:
-; FALLBACK2:       # %bb.0:
-; FALLBACK2-NEXT:    movq (%rdi), %rcx
-; FALLBACK2-NEXT:    movq 8(%rdi), %r8
-; FALLBACK2-NEXT:    movq 16(%rdi), %r9
-; FALLBACK2-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK2-NEXT:    movzbl (%rsi), %esi
-; FALLBACK2-NEXT:    movl %esi, %eax
-; FALLBACK2-NEXT:    shlb $5, %al
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    sarq $63, %rdi
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movl %eax, %ecx
-; FALLBACK2-NEXT:    andb $6, %sil
-; FALLBACK2-NEXT:    movzbl %sil, %esi
-; FALLBACK2-NEXT:    movq -64(%rsp,%rsi,4), %rdi
-; FALLBACK2-NEXT:    movq -56(%rsp,%rsi,4), %r8
-; FALLBACK2-NEXT:    shrxq %rcx, %rdi, %r9
-; FALLBACK2-NEXT:    notb %al
-; FALLBACK2-NEXT:    leaq (%r8,%r8), %r10
-; FALLBACK2-NEXT:    shlxq %rax, %r10, %r10
-; FALLBACK2-NEXT:    orq %r9, %r10
-; FALLBACK2-NEXT:    shrxq %rcx, -72(%rsp,%rsi,4), %r9
-; FALLBACK2-NEXT:    addq %rdi, %rdi
-; FALLBACK2-NEXT:    shlxq %rax, %rdi, %rdi
-; FALLBACK2-NEXT:    orq %r9, %rdi
-; FALLBACK2-NEXT:    shrxq %rcx, %r8, %r8
-; FALLBACK2-NEXT:    movq -48(%rsp,%rsi,4), %rsi
-; FALLBACK2-NEXT:    leaq (%rsi,%rsi), %r9
-; FALLBACK2-NEXT:    shlxq %rax, %r9, %rax
-; FALLBACK2-NEXT:    orq %r8, %rax
-; FALLBACK2-NEXT:    sarxq %rcx, %rsi, %rcx
-; FALLBACK2-NEXT:    movq %rcx, 24(%rdx)
-; FALLBACK2-NEXT:    movq %rax, 16(%rdx)
-; FALLBACK2-NEXT:    movq %rdi, (%rdx)
-; FALLBACK2-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK2-NEXT:    retq
-;
-; FALLBACK3-LABEL: ashr_32bytes_dwordOff:
-; FALLBACK3:       # %bb.0:
-; FALLBACK3-NEXT:    movq (%rdi), %rax
-; FALLBACK3-NEXT:    movq 8(%rdi), %r8
-; FALLBACK3-NEXT:    movq 16(%rdi), %r9
-; FALLBACK3-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK3-NEXT:    movzbl (%rsi), %esi
-; FALLBACK3-NEXT:    movl %esi, %ecx
-; FALLBACK3-NEXT:    shlb $5, %cl
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    sarq $63, %rdi
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    andb $6, %sil
-; FALLBACK3-NEXT:    movzbl %sil, %eax
-; FALLBACK3-NEXT:    movq -56(%rsp,%rax,4), %rsi
-; FALLBACK3-NEXT:    movq -72(%rsp,%rax,4), %rdi
-; FALLBACK3-NEXT:    movq -64(%rsp,%rax,4), %r8
-; FALLBACK3-NEXT:    movq %r8, %r9
-; FALLBACK3-NEXT:    shrdq %cl, %rsi, %r9
-; FALLBACK3-NEXT:    movq -48(%rsp,%rax,4), %rax
-; FALLBACK3-NEXT:    shrdq %cl, %rax, %rsi
-; FALLBACK3-NEXT:    shrdq %cl, %r8, %rdi
-; FALLBACK3-NEXT:    sarxq %rcx, %rax, %rax
-; FALLBACK3-NEXT:    movq %rsi, 16(%rdx)
-; FALLBACK3-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK3-NEXT:    movq %rdi, (%rdx)
-; FALLBACK3-NEXT:    movq %r9, 8(%rdx)
-; FALLBACK3-NEXT:    retq
-;
-; FALLBACK4-LABEL: ashr_32bytes_dwordOff:
-; FALLBACK4:       # %bb.0:
-; FALLBACK4-NEXT:    pushq %rbx
-; FALLBACK4-NEXT:    movups (%rdi), %xmm0
-; FALLBACK4-NEXT:    movq 16(%rdi), %rcx
-; FALLBACK4-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK4-NEXT:    movzbl (%rsi), %esi
-; FALLBACK4-NEXT:    movl %esi, %eax
-; FALLBACK4-NEXT:    shlb $5, %al
-; FALLBACK4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    sarq $63, %rdi
-; FALLBACK4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    andb $6, %sil
-; FALLBACK4-NEXT:    movzbl %sil, %r9d
-; FALLBACK4-NEXT:    movq -64(%rsp,%r9,4), %r10
-; FALLBACK4-NEXT:    movq -56(%rsp,%r9,4), %r8
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r10
-; FALLBACK4-NEXT:    movl %eax, %esi
-; FALLBACK4-NEXT:    notb %sil
-; FALLBACK4-NEXT:    leaq (%r8,%r8), %rdi
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %rdi
-; FALLBACK4-NEXT:    orq %r10, %rdi
-; FALLBACK4-NEXT:    movq -48(%rsp,%r9,4), %r10
-; FALLBACK4-NEXT:    movq %r10, %r11
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r11
-; FALLBACK4-NEXT:    movq -40(%rsp,%r9,4), %r9
-; FALLBACK4-NEXT:    leaq (%r9,%r9), %rbx
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %rbx
-; FALLBACK4-NEXT:    orq %r11, %rbx
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r8
-; FALLBACK4-NEXT:    addq %r10, %r10
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r10
-; FALLBACK4-NEXT:    orq %r8, %r10
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    sarq %cl, %r9
-; FALLBACK4-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK4-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK4-NEXT:    movq %rbx, 16(%rdx)
-; FALLBACK4-NEXT:    movq %rdi, (%rdx)
-; FALLBACK4-NEXT:    popq %rbx
-; FALLBACK4-NEXT:    retq
-;
-; FALLBACK5-LABEL: ashr_32bytes_dwordOff:
-; FALLBACK5:       # %bb.0:
-; FALLBACK5-NEXT:    movups (%rdi), %xmm0
-; FALLBACK5-NEXT:    movq 16(%rdi), %rax
-; FALLBACK5-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK5-NEXT:    movzbl (%rsi), %esi
-; FALLBACK5-NEXT:    movl %esi, %ecx
-; FALLBACK5-NEXT:    shlb $5, %cl
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    sarq $63, %rdi
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    andb $6, %sil
-; FALLBACK5-NEXT:    movzbl %sil, %eax
-; FALLBACK5-NEXT:    movq -48(%rsp,%rax,4), %rsi
-; FALLBACK5-NEXT:    movq -56(%rsp,%rax,4), %rdi
-; FALLBACK5-NEXT:    movq %rdi, %r8
-; FALLBACK5-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK5-NEXT:    movq -72(%rsp,%rax,4), %r9
-; FALLBACK5-NEXT:    movq -64(%rsp,%rax,4), %rax
-; FALLBACK5-NEXT:    movq %rax, %r10
-; FALLBACK5-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK5-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK5-NEXT:    sarq %cl, %rsi
-; FALLBACK5-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK5-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK5-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK5-NEXT:    movq %r9, (%rdx)
-; FALLBACK5-NEXT:    retq
-;
-; FALLBACK6-LABEL: ashr_32bytes_dwordOff:
-; FALLBACK6:       # %bb.0:
-; FALLBACK6-NEXT:    movups (%rdi), %xmm0
-; FALLBACK6-NEXT:    movq 16(%rdi), %rcx
-; FALLBACK6-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK6-NEXT:    movzbl (%rsi), %esi
-; FALLBACK6-NEXT:    movl %esi, %eax
-; FALLBACK6-NEXT:    shlb $5, %al
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    sarq $63, %rdi
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movl %eax, %ecx
-; FALLBACK6-NEXT:    andb $6, %sil
-; FALLBACK6-NEXT:    movzbl %sil, %esi
-; FALLBACK6-NEXT:    shrxq %rcx, -72(%rsp,%rsi,4), %rdi
-; FALLBACK6-NEXT:    notb %al
-; FALLBACK6-NEXT:    movq -64(%rsp,%rsi,4), %r8
-; FALLBACK6-NEXT:    movq -56(%rsp,%rsi,4), %r9
-; FALLBACK6-NEXT:    leaq (%r8,%r8), %r10
-; FALLBACK6-NEXT:    shlxq %rax, %r10, %r10
-; FALLBACK6-NEXT:    orq %rdi, %r10
-; FALLBACK6-NEXT:    shrxq %rcx, %r9, %rdi
-; FALLBACK6-NEXT:    movq -48(%rsp,%rsi,4), %rsi
-; FALLBACK6-NEXT:    leaq (%rsi,%rsi), %r11
-; FALLBACK6-NEXT:    shlxq %rax, %r11, %r11
-; FALLBACK6-NEXT:    orq %rdi, %r11
-; FALLBACK6-NEXT:    shrxq %rcx, %r8, %rdi
-; FALLBACK6-NEXT:    addq %r9, %r9
-; FALLBACK6-NEXT:    shlxq %rax, %r9, %rax
-; FALLBACK6-NEXT:    orq %rdi, %rax
-; FALLBACK6-NEXT:    sarxq %rcx, %rsi, %rcx
-; FALLBACK6-NEXT:    movq %rcx, 24(%rdx)
-; FALLBACK6-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK6-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK6-NEXT:    movq %r10, (%rdx)
-; FALLBACK6-NEXT:    retq
-;
-; FALLBACK7-LABEL: ashr_32bytes_dwordOff:
-; FALLBACK7:       # %bb.0:
-; FALLBACK7-NEXT:    movups (%rdi), %xmm0
-; FALLBACK7-NEXT:    movq 16(%rdi), %rax
-; FALLBACK7-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK7-NEXT:    movzbl (%rsi), %esi
-; FALLBACK7-NEXT:    movl %esi, %ecx
-; FALLBACK7-NEXT:    shlb $5, %cl
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    sarq $63, %rdi
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    andb $6, %sil
-; FALLBACK7-NEXT:    movzbl %sil, %eax
-; FALLBACK7-NEXT:    movq -48(%rsp,%rax,4), %rsi
-; FALLBACK7-NEXT:    movq -56(%rsp,%rax,4), %rdi
-; FALLBACK7-NEXT:    movq %rdi, %r8
-; FALLBACK7-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK7-NEXT:    movq -72(%rsp,%rax,4), %r9
-; FALLBACK7-NEXT:    movq -64(%rsp,%rax,4), %rax
-; FALLBACK7-NEXT:    movq %rax, %r10
-; FALLBACK7-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK7-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK7-NEXT:    sarxq %rcx, %rsi, %rax
-; FALLBACK7-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK7-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK7-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK7-NEXT:    movq %r9, (%rdx)
-; FALLBACK7-NEXT:    retq
-;
-; FALLBACK8-LABEL: ashr_32bytes_dwordOff:
-; FALLBACK8:       # %bb.0:
-; FALLBACK8-NEXT:    pushq %rbx
-; FALLBACK8-NEXT:    vmovups (%rdi), %xmm0
-; FALLBACK8-NEXT:    movq 16(%rdi), %rcx
-; FALLBACK8-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK8-NEXT:    movzbl (%rsi), %esi
-; FALLBACK8-NEXT:    movl %esi, %eax
-; FALLBACK8-NEXT:    shlb $5, %al
-; FALLBACK8-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    sarq $63, %rdi
-; FALLBACK8-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    andb $6, %sil
-; FALLBACK8-NEXT:    movzbl %sil, %r9d
-; FALLBACK8-NEXT:    movq -64(%rsp,%r9,4), %r10
-; FALLBACK8-NEXT:    movq -56(%rsp,%r9,4), %r8
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r10
-; FALLBACK8-NEXT:    movl %eax, %esi
-; FALLBACK8-NEXT:    notb %sil
-; FALLBACK8-NEXT:    leaq (%r8,%r8), %rdi
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %rdi
-; FALLBACK8-NEXT:    orq %r10, %rdi
-; FALLBACK8-NEXT:    movq -48(%rsp,%r9,4), %r10
-; FALLBACK8-NEXT:    movq %r10, %r11
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r11
-; FALLBACK8-NEXT:    movq -40(%rsp,%r9,4), %r9
-; FALLBACK8-NEXT:    leaq (%r9,%r9), %rbx
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %rbx
-; FALLBACK8-NEXT:    orq %r11, %rbx
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r8
-; FALLBACK8-NEXT:    addq %r10, %r10
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r10
-; FALLBACK8-NEXT:    orq %r8, %r10
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    sarq %cl, %r9
-; FALLBACK8-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK8-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK8-NEXT:    movq %rbx, 16(%rdx)
-; FALLBACK8-NEXT:    movq %rdi, (%rdx)
-; FALLBACK8-NEXT:    popq %rbx
-; FALLBACK8-NEXT:    retq
-;
-; FALLBACK9-LABEL: ashr_32bytes_dwordOff:
-; FALLBACK9:       # %bb.0:
-; FALLBACK9-NEXT:    vmovups (%rdi), %xmm0
-; FALLBACK9-NEXT:    movq 16(%rdi), %rax
-; FALLBACK9-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK9-NEXT:    movzbl (%rsi), %esi
-; FALLBACK9-NEXT:    movl %esi, %ecx
-; FALLBACK9-NEXT:    shlb $5, %cl
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    sarq $63, %rdi
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    andb $6, %sil
-; FALLBACK9-NEXT:    movzbl %sil, %eax
-; FALLBACK9-NEXT:    movq -48(%rsp,%rax,4), %rsi
-; FALLBACK9-NEXT:    movq -56(%rsp,%rax,4), %rdi
-; FALLBACK9-NEXT:    movq %rdi, %r8
-; FALLBACK9-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK9-NEXT:    movq -72(%rsp,%rax,4), %r9
-; FALLBACK9-NEXT:    movq -64(%rsp,%rax,4), %rax
-; FALLBACK9-NEXT:    movq %rax, %r10
-; FALLBACK9-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK9-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK9-NEXT:    sarq %cl, %rsi
-; FALLBACK9-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK9-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK9-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK9-NEXT:    movq %r9, (%rdx)
-; FALLBACK9-NEXT:    retq
-;
-; FALLBACK10-LABEL: ashr_32bytes_dwordOff:
-; FALLBACK10:       # %bb.0:
-; FALLBACK10-NEXT:    vmovups (%rdi), %xmm0
-; FALLBACK10-NEXT:    movq 16(%rdi), %rcx
-; FALLBACK10-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK10-NEXT:    movzbl (%rsi), %esi
-; FALLBACK10-NEXT:    movl %esi, %eax
-; FALLBACK10-NEXT:    shlb $5, %al
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    sarq $63, %rdi
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movl %eax, %ecx
-; FALLBACK10-NEXT:    andb $6, %sil
-; FALLBACK10-NEXT:    movzbl %sil, %esi
-; FALLBACK10-NEXT:    shrxq %rcx, -72(%rsp,%rsi,4), %rdi
-; FALLBACK10-NEXT:    notb %al
-; FALLBACK10-NEXT:    movq -64(%rsp,%rsi,4), %r8
-; FALLBACK10-NEXT:    movq -56(%rsp,%rsi,4), %r9
-; FALLBACK10-NEXT:    leaq (%r8,%r8), %r10
-; FALLBACK10-NEXT:    shlxq %rax, %r10, %r10
-; FALLBACK10-NEXT:    orq %rdi, %r10
-; FALLBACK10-NEXT:    shrxq %rcx, %r9, %rdi
-; FALLBACK10-NEXT:    movq -48(%rsp,%rsi,4), %rsi
-; FALLBACK10-NEXT:    leaq (%rsi,%rsi), %r11
-; FALLBACK10-NEXT:    shlxq %rax, %r11, %r11
-; FALLBACK10-NEXT:    orq %rdi, %r11
-; FALLBACK10-NEXT:    shrxq %rcx, %r8, %rdi
-; FALLBACK10-NEXT:    addq %r9, %r9
-; FALLBACK10-NEXT:    shlxq %rax, %r9, %rax
-; FALLBACK10-NEXT:    orq %rdi, %rax
-; FALLBACK10-NEXT:    sarxq %rcx, %rsi, %rcx
-; FALLBACK10-NEXT:    movq %rcx, 24(%rdx)
-; FALLBACK10-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK10-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK10-NEXT:    movq %r10, (%rdx)
-; FALLBACK10-NEXT:    retq
-;
-; FALLBACK11-LABEL: ashr_32bytes_dwordOff:
-; FALLBACK11:       # %bb.0:
-; FALLBACK11-NEXT:    vmovups (%rdi), %xmm0
-; FALLBACK11-NEXT:    movq 16(%rdi), %rax
-; FALLBACK11-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK11-NEXT:    movzbl (%rsi), %esi
-; FALLBACK11-NEXT:    movl %esi, %ecx
-; FALLBACK11-NEXT:    shlb $5, %cl
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    sarq $63, %rdi
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    andb $6, %sil
-; FALLBACK11-NEXT:    movzbl %sil, %eax
-; FALLBACK11-NEXT:    movq -48(%rsp,%rax,4), %rsi
-; FALLBACK11-NEXT:    movq -56(%rsp,%rax,4), %rdi
-; FALLBACK11-NEXT:    movq %rdi, %r8
-; FALLBACK11-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK11-NEXT:    movq -72(%rsp,%rax,4), %r9
-; FALLBACK11-NEXT:    movq -64(%rsp,%rax,4), %rax
-; FALLBACK11-NEXT:    movq %rax, %r10
-; FALLBACK11-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK11-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK11-NEXT:    sarxq %rcx, %rsi, %rax
-; FALLBACK11-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK11-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK11-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK11-NEXT:    movq %r9, (%rdx)
-; FALLBACK11-NEXT:    retq
-;
-; FALLBACK12-LABEL: ashr_32bytes_dwordOff:
-; FALLBACK12:       # %bb.0:
-; FALLBACK12-NEXT:    pushq %rbx
-; FALLBACK12-NEXT:    vmovups (%rdi), %xmm0
-; FALLBACK12-NEXT:    movq 16(%rdi), %rcx
-; FALLBACK12-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK12-NEXT:    movzbl (%rsi), %esi
-; FALLBACK12-NEXT:    movl %esi, %eax
-; FALLBACK12-NEXT:    shlb $5, %al
-; FALLBACK12-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    sarq $63, %rdi
-; FALLBACK12-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    andb $6, %sil
-; FALLBACK12-NEXT:    movzbl %sil, %r9d
-; FALLBACK12-NEXT:    movq -64(%rsp,%r9,4), %r10
-; FALLBACK12-NEXT:    movq -56(%rsp,%r9,4), %r8
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r10
-; FALLBACK12-NEXT:    movl %eax, %esi
-; FALLBACK12-NEXT:    notb %sil
-; FALLBACK12-NEXT:    leaq (%r8,%r8), %rdi
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %rdi
-; FALLBACK12-NEXT:    orq %r10, %rdi
-; FALLBACK12-NEXT:    movq -48(%rsp,%r9,4), %r10
-; FALLBACK12-NEXT:    movq %r10, %r11
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r11
-; FALLBACK12-NEXT:    movq -40(%rsp,%r9,4), %r9
-; FALLBACK12-NEXT:    leaq (%r9,%r9), %rbx
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %rbx
-; FALLBACK12-NEXT:    orq %r11, %rbx
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r8
-; FALLBACK12-NEXT:    addq %r10, %r10
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r10
-; FALLBACK12-NEXT:    orq %r8, %r10
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    sarq %cl, %r9
-; FALLBACK12-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK12-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK12-NEXT:    movq %rbx, 16(%rdx)
-; FALLBACK12-NEXT:    movq %rdi, (%rdx)
-; FALLBACK12-NEXT:    popq %rbx
-; FALLBACK12-NEXT:    retq
-;
-; FALLBACK13-LABEL: ashr_32bytes_dwordOff:
-; FALLBACK13:       # %bb.0:
-; FALLBACK13-NEXT:    vmovups (%rdi), %xmm0
-; FALLBACK13-NEXT:    movq 16(%rdi), %rax
-; FALLBACK13-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK13-NEXT:    movzbl (%rsi), %esi
-; FALLBACK13-NEXT:    movl %esi, %ecx
-; FALLBACK13-NEXT:    shlb $5, %cl
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    sarq $63, %rdi
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    andb $6, %sil
-; FALLBACK13-NEXT:    movzbl %sil, %eax
-; FALLBACK13-NEXT:    movq -48(%rsp,%rax,4), %rsi
-; FALLBACK13-NEXT:    movq -56(%rsp,%rax,4), %rdi
-; FALLBACK13-NEXT:    movq %rdi, %r8
-; FALLBACK13-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK13-NEXT:    movq -72(%rsp,%rax,4), %r9
-; FALLBACK13-NEXT:    movq -64(%rsp,%rax,4), %rax
-; FALLBACK13-NEXT:    movq %rax, %r10
-; FALLBACK13-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK13-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK13-NEXT:    sarq %cl, %rsi
-; FALLBACK13-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK13-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK13-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK13-NEXT:    movq %r9, (%rdx)
-; FALLBACK13-NEXT:    retq
-;
-; FALLBACK14-LABEL: ashr_32bytes_dwordOff:
-; FALLBACK14:       # %bb.0:
-; FALLBACK14-NEXT:    vmovups (%rdi), %xmm0
-; FALLBACK14-NEXT:    movq 16(%rdi), %rcx
-; FALLBACK14-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK14-NEXT:    movzbl (%rsi), %esi
-; FALLBACK14-NEXT:    movl %esi, %eax
-; FALLBACK14-NEXT:    shlb $5, %al
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    sarq $63, %rdi
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movl %eax, %ecx
-; FALLBACK14-NEXT:    andb $6, %sil
-; FALLBACK14-NEXT:    movzbl %sil, %esi
-; FALLBACK14-NEXT:    shrxq %rcx, -72(%rsp,%rsi,4), %rdi
-; FALLBACK14-NEXT:    notb %al
-; FALLBACK14-NEXT:    movq -64(%rsp,%rsi,4), %r8
-; FALLBACK14-NEXT:    movq -56(%rsp,%rsi,4), %r9
-; FALLBACK14-NEXT:    leaq (%r8,%r8), %r10
-; FALLBACK14-NEXT:    shlxq %rax, %r10, %r10
-; FALLBACK14-NEXT:    orq %rdi, %r10
-; FALLBACK14-NEXT:    shrxq %rcx, %r9, %rdi
-; FALLBACK14-NEXT:    movq -48(%rsp,%rsi,4), %rsi
-; FALLBACK14-NEXT:    leaq (%rsi,%rsi), %r11
-; FALLBACK14-NEXT:    shlxq %rax, %r11, %r11
-; FALLBACK14-NEXT:    orq %rdi, %r11
-; FALLBACK14-NEXT:    shrxq %rcx, %r8, %rdi
-; FALLBACK14-NEXT:    addq %r9, %r9
-; FALLBACK14-NEXT:    shlxq %rax, %r9, %rax
-; FALLBACK14-NEXT:    orq %rdi, %rax
-; FALLBACK14-NEXT:    sarxq %rcx, %rsi, %rcx
-; FALLBACK14-NEXT:    movq %rcx, 24(%rdx)
-; FALLBACK14-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK14-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK14-NEXT:    movq %r10, (%rdx)
-; FALLBACK14-NEXT:    retq
-;
-; FALLBACK15-LABEL: ashr_32bytes_dwordOff:
-; FALLBACK15:       # %bb.0:
-; FALLBACK15-NEXT:    vmovups (%rdi), %xmm0
-; FALLBACK15-NEXT:    movq 16(%rdi), %rax
-; FALLBACK15-NEXT:    movq 24(%rdi), %rdi
-; FALLBACK15-NEXT:    movzbl (%rsi), %esi
-; FALLBACK15-NEXT:    movl %esi, %ecx
-; FALLBACK15-NEXT:    shlb $5, %cl
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    sarq $63, %rdi
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    andb $6, %sil
-; FALLBACK15-NEXT:    movzbl %sil, %eax
-; FALLBACK15-NEXT:    movq -48(%rsp,%rax,4), %rsi
-; FALLBACK15-NEXT:    movq -56(%rsp,%rax,4), %rdi
-; FALLBACK15-NEXT:    movq %rdi, %r8
-; FALLBACK15-NEXT:    shrdq %cl, %rsi, %r8
-; FALLBACK15-NEXT:    movq -72(%rsp,%rax,4), %r9
-; FALLBACK15-NEXT:    movq -64(%rsp,%rax,4), %rax
-; FALLBACK15-NEXT:    movq %rax, %r10
-; FALLBACK15-NEXT:    shrdq %cl, %rdi, %r10
-; FALLBACK15-NEXT:    shrdq %cl, %rax, %r9
-; FALLBACK15-NEXT:    sarxq %rcx, %rsi, %rax
-; FALLBACK15-NEXT:    movq %r10, 8(%rdx)
-; FALLBACK15-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK15-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK15-NEXT:    movq %r9, (%rdx)
-; FALLBACK15-NEXT:    retq
+; X64-NO-SHLD-NO-BMI2-SSE2-LABEL: ashr_32bytes_dwordOff:
+; X64-NO-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    sarq $63, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    andb $6, %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movzbl %sil, %r9d
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -64(%rsp,%r9,4), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -56(%rsp,%r9,4), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -48(%rsp,%r9,4), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leaq (%rbx,%rbx), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r11, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    addq %rdi, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -40(%rsp,%r9,4), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leaq (%r9,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %rbx, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    sarq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE2-LABEL: ashr_32bytes_dwordOff:
+; X64-HAVE-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andb $6, %sil
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movzbl %sil, %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -56(%rsp,%rax,4), %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -72(%rsp,%rax,4), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -64(%rsp,%rax,4), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %rsi, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -48(%rsp,%rax,4), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %rax, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %r8, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    sarq %cl, %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE2-LABEL: ashr_32bytes_dwordOff:
+; X64-NO-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $5, %al
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    sarq $63, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $6, %sil
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl %sil, %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -64(%rsp,%rsi,4), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -56(%rsp,%rsi,4), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rcx, %rdi, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leaq (%r8,%r8), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rax, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r9, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rcx, -72(%rsp,%rsi,4), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addq %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rax, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r9, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rcx, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -48(%rsp,%rsi,4), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leaq (%rsi,%rsi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rax, %r9, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r8, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    sarxq %rcx, %rsi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r10, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: ashr_32bytes_dwordOff:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andb $6, %sil
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movzbl %sil, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -56(%rsp,%rax,4), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -72(%rsp,%rax,4), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -64(%rsp,%rax,4), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %rsi, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -48(%rsp,%rax,4), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %rax, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %r8, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    sarxq %rcx, %rax, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rsi, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-SSE4-LABEL: ashr_32bytes_dwordOff:
+; X64-NO-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq 16(%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    sarq $63, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    andb $6, %sil
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movzbl %sil, %r9d
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -64(%rsp,%r9,4), %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -56(%rsp,%r9,4), %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leaq (%r8,%r8), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -48(%rsp,%r9,4), %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -40(%rsp,%r9,4), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leaq (%r9,%r9), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    addq %r10, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r8, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    sarq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rbx, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE4-LABEL: ashr_32bytes_dwordOff:
+; X64-HAVE-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq 16(%rdi), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andb $6, %sil
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movzbl %sil, %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -48(%rsp,%rax,4), %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -56(%rsp,%rax,4), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %rsi, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -72(%rsp,%rax,4), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -64(%rsp,%rax,4), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rax, %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %rdi, %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %rax, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    sarq %cl, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE4-LABEL: ashr_32bytes_dwordOff:
+; X64-NO-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq 16(%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $5, %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    sarq $63, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $6, %sil
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl %sil, %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rcx, -72(%rsp,%rsi,4), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -64(%rsp,%rsi,4), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -56(%rsp,%rsi,4), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leaq (%r8,%r8), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rax, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rdi, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rcx, %r9, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -48(%rsp,%rsi,4), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leaq (%rsi,%rsi), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rax, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rdi, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rcx, %r8, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addq %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rax, %r9, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rdi, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    sarxq %rcx, %rsi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rcx, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: ashr_32bytes_dwordOff:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq 16(%rdi), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andb $6, %sil
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movzbl %sil, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -48(%rsp,%rax,4), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -56(%rsp,%rax,4), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %rsi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -72(%rsp,%rax,4), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -64(%rsp,%rax,4), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %rdi, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %rax, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    sarxq %rcx, %rsi, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r10, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-AVX-LABEL: ashr_32bytes_dwordOff:
+; X64-NO-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%rdi), %xmm0
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq 16(%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    sarq $63, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    andb $6, %sil
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movzbl %sil, %r9d
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -64(%rsp,%r9,4), %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -56(%rsp,%r9,4), %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    leaq (%r8,%r8), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -48(%rsp,%r9,4), %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -40(%rsp,%r9,4), %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    leaq (%r9,%r9), %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    addq %r10, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r8, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    sarq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rbx, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-AVX-LABEL: ashr_32bytes_dwordOff:
+; X64-HAVE-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%rdi), %xmm0
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq 16(%rdi), %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    andb $6, %sil
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movzbl %sil, %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -48(%rsp,%rax,4), %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -56(%rsp,%rax,4), %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %rsi, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -72(%rsp,%rax,4), %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -64(%rsp,%rax,4), %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rax, %r10
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %rdi, %r10
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %rax, %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    sarq %cl, %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-AVX-LABEL: ashr_32bytes_dwordOff:
+; X64-NO-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%rdi), %xmm0
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq 16(%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq 24(%rdi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $5, %al
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    sarq $63, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    andb $6, %sil
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl %sil, %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rcx, -72(%rsp,%rsi,4), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -64(%rsp,%rsi,4), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -56(%rsp,%rsi,4), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leaq (%r8,%r8), %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rax, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %rdi, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rcx, %r9, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -48(%rsp,%rsi,4), %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leaq (%rsi,%rsi), %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rax, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %rdi, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rcx, %r8, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addq %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rax, %r9, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %rdi, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    sarxq %rcx, %rsi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rcx, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-LABEL: ashr_32bytes_dwordOff:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%rdi), %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq 16(%rdi), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq 24(%rdi), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl (%rsi), %esi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shlb $5, %cl
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    andb $6, %sil
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movzbl %sil, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -48(%rsp,%rax,4), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -56(%rsp,%rax,4), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %rsi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -72(%rsp,%rax,4), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -64(%rsp,%rax,4), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %rdi, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %rax, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    sarxq %rcx, %rsi, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r10, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    retq
 ;
 ; X86-SSE2-LABEL: ashr_32bytes_dwordOff:
 ; X86-SSE2:       # %bb.0:
@@ -12023,3629 +9744,3629 @@ define void @ashr_32bytes_qwordOff(ptr %src.ptr, ptr %qwordOff.ptr, ptr %dst) no
 }
 
 define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
-; FALLBACK0-LABEL: lshr_64bytes:
-; FALLBACK0:       # %bb.0:
-; FALLBACK0-NEXT:    pushq %r15
-; FALLBACK0-NEXT:    pushq %r14
-; FALLBACK0-NEXT:    pushq %r13
-; FALLBACK0-NEXT:    pushq %r12
-; FALLBACK0-NEXT:    pushq %rbx
-; FALLBACK0-NEXT:    movq (%rdi), %rax
-; FALLBACK0-NEXT:    movq 8(%rdi), %rcx
-; FALLBACK0-NEXT:    movq 16(%rdi), %r8
-; FALLBACK0-NEXT:    movq 24(%rdi), %r9
-; FALLBACK0-NEXT:    movq 32(%rdi), %r10
-; FALLBACK0-NEXT:    movq 40(%rdi), %r11
-; FALLBACK0-NEXT:    movq 48(%rdi), %rbx
-; FALLBACK0-NEXT:    movq 56(%rdi), %r14
-; FALLBACK0-NEXT:    movl (%rsi), %edi
-; FALLBACK0-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK0-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    leal (,%rdi,8), %eax
-; FALLBACK0-NEXT:    andl $56, %eax
-; FALLBACK0-NEXT:    andl $56, %edi
-; FALLBACK0-NEXT:    movq -128(%rsp,%rdi), %r10
-; FALLBACK0-NEXT:    movq -120(%rsp,%rdi), %r8
-; FALLBACK0-NEXT:    movq %r8, %r11
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r11
-; FALLBACK0-NEXT:    movl %eax, %esi
-; FALLBACK0-NEXT:    notb %sil
-; FALLBACK0-NEXT:    movq -112(%rsp,%rdi), %rbx
-; FALLBACK0-NEXT:    leaq (%rbx,%rbx), %r9
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r9
-; FALLBACK0-NEXT:    orq %r11, %r9
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r10
-; FALLBACK0-NEXT:    addq %r8, %r8
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r8
-; FALLBACK0-NEXT:    orq %r10, %r8
-; FALLBACK0-NEXT:    movq -104(%rsp,%rdi), %r10
-; FALLBACK0-NEXT:    movq %r10, %r15
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r15
-; FALLBACK0-NEXT:    movq -96(%rsp,%rdi), %r14
-; FALLBACK0-NEXT:    leaq (%r14,%r14), %r11
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r11
-; FALLBACK0-NEXT:    orq %r15, %r11
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %rbx
-; FALLBACK0-NEXT:    addq %r10, %r10
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r10
-; FALLBACK0-NEXT:    orq %rbx, %r10
-; FALLBACK0-NEXT:    movq -88(%rsp,%rdi), %rbx
-; FALLBACK0-NEXT:    movq %rbx, %r12
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r12
-; FALLBACK0-NEXT:    movq -80(%rsp,%rdi), %r13
-; FALLBACK0-NEXT:    leaq (%r13,%r13), %r15
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r15
-; FALLBACK0-NEXT:    orq %r12, %r15
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r14
-; FALLBACK0-NEXT:    addq %rbx, %rbx
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %rbx
-; FALLBACK0-NEXT:    orq %r14, %rbx
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r13
-; FALLBACK0-NEXT:    movq -72(%rsp,%rdi), %rdi
-; FALLBACK0-NEXT:    leaq (%rdi,%rdi), %r14
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r14
-; FALLBACK0-NEXT:    orq %r13, %r14
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %rdi
-; FALLBACK0-NEXT:    movq %rdi, 56(%rdx)
-; FALLBACK0-NEXT:    movq %r14, 48(%rdx)
-; FALLBACK0-NEXT:    movq %rbx, 32(%rdx)
-; FALLBACK0-NEXT:    movq %r15, 40(%rdx)
-; FALLBACK0-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK0-NEXT:    movq %r11, 24(%rdx)
-; FALLBACK0-NEXT:    movq %r8, (%rdx)
-; FALLBACK0-NEXT:    movq %r9, 8(%rdx)
-; FALLBACK0-NEXT:    popq %rbx
-; FALLBACK0-NEXT:    popq %r12
-; FALLBACK0-NEXT:    popq %r13
-; FALLBACK0-NEXT:    popq %r14
-; FALLBACK0-NEXT:    popq %r15
-; FALLBACK0-NEXT:    retq
-;
-; FALLBACK1-LABEL: lshr_64bytes:
-; FALLBACK1:       # %bb.0:
-; FALLBACK1-NEXT:    pushq %r15
-; FALLBACK1-NEXT:    pushq %r14
-; FALLBACK1-NEXT:    pushq %rbx
-; FALLBACK1-NEXT:    movq (%rdi), %rcx
-; FALLBACK1-NEXT:    movq 8(%rdi), %r8
-; FALLBACK1-NEXT:    movq 16(%rdi), %r9
-; FALLBACK1-NEXT:    movq 24(%rdi), %r10
-; FALLBACK1-NEXT:    movq 32(%rdi), %r11
-; FALLBACK1-NEXT:    movq 40(%rdi), %rbx
-; FALLBACK1-NEXT:    movq 48(%rdi), %r14
-; FALLBACK1-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK1-NEXT:    movl (%rsi), %eax
-; FALLBACK1-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK1-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK1-NEXT:    andl $56, %ecx
-; FALLBACK1-NEXT:    andl $56, %eax
-; FALLBACK1-NEXT:    movq -112(%rsp,%rax), %rdi
-; FALLBACK1-NEXT:    movq -128(%rsp,%rax), %rsi
-; FALLBACK1-NEXT:    movq -120(%rsp,%rax), %r9
-; FALLBACK1-NEXT:    movq %r9, %r8
-; FALLBACK1-NEXT:    shrdq %cl, %rdi, %r8
-; FALLBACK1-NEXT:    movq -96(%rsp,%rax), %r10
-; FALLBACK1-NEXT:    movq -104(%rsp,%rax), %r11
-; FALLBACK1-NEXT:    movq %r11, %rbx
-; FALLBACK1-NEXT:    shrdq %cl, %r10, %rbx
-; FALLBACK1-NEXT:    shrdq %cl, %r11, %rdi
-; FALLBACK1-NEXT:    movq -80(%rsp,%rax), %r11
-; FALLBACK1-NEXT:    movq -88(%rsp,%rax), %r14
-; FALLBACK1-NEXT:    movq %r14, %r15
-; FALLBACK1-NEXT:    shrdq %cl, %r11, %r15
-; FALLBACK1-NEXT:    shrdq %cl, %r14, %r10
-; FALLBACK1-NEXT:    movq -72(%rsp,%rax), %rax
-; FALLBACK1-NEXT:    shrdq %cl, %rax, %r11
-; FALLBACK1-NEXT:    shrdq %cl, %r9, %rsi
-; FALLBACK1-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK1-NEXT:    shrq %cl, %rax
-; FALLBACK1-NEXT:    movq %r11, 48(%rdx)
-; FALLBACK1-NEXT:    movq %rax, 56(%rdx)
-; FALLBACK1-NEXT:    movq %r10, 32(%rdx)
-; FALLBACK1-NEXT:    movq %r15, 40(%rdx)
-; FALLBACK1-NEXT:    movq %rdi, 16(%rdx)
-; FALLBACK1-NEXT:    movq %rbx, 24(%rdx)
-; FALLBACK1-NEXT:    movq %rsi, (%rdx)
-; FALLBACK1-NEXT:    movq %r8, 8(%rdx)
-; FALLBACK1-NEXT:    popq %rbx
-; FALLBACK1-NEXT:    popq %r14
-; FALLBACK1-NEXT:    popq %r15
-; FALLBACK1-NEXT:    retq
-;
-; FALLBACK2-LABEL: lshr_64bytes:
-; FALLBACK2:       # %bb.0:
-; FALLBACK2-NEXT:    pushq %r15
-; FALLBACK2-NEXT:    pushq %r14
-; FALLBACK2-NEXT:    pushq %r12
-; FALLBACK2-NEXT:    pushq %rbx
-; FALLBACK2-NEXT:    pushq %rax
-; FALLBACK2-NEXT:    movq (%rdi), %rcx
-; FALLBACK2-NEXT:    movq 8(%rdi), %r8
-; FALLBACK2-NEXT:    movq 16(%rdi), %r9
-; FALLBACK2-NEXT:    movq 24(%rdi), %r10
-; FALLBACK2-NEXT:    movq 32(%rdi), %r11
-; FALLBACK2-NEXT:    movq 40(%rdi), %rbx
-; FALLBACK2-NEXT:    movq 48(%rdi), %r14
-; FALLBACK2-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK2-NEXT:    movl (%rsi), %eax
-; FALLBACK2-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK2-NEXT:    andl $56, %ecx
-; FALLBACK2-NEXT:    movl %ecx, %esi
-; FALLBACK2-NEXT:    andl $56, %eax
-; FALLBACK2-NEXT:    movq -120(%rsp,%rax), %r8
-; FALLBACK2-NEXT:    movq -112(%rsp,%rax), %r10
-; FALLBACK2-NEXT:    shrxq %rsi, %r8, %r9
-; FALLBACK2-NEXT:    notb %cl
-; FALLBACK2-NEXT:    leaq (%r10,%r10), %rdi
-; FALLBACK2-NEXT:    shlxq %rcx, %rdi, %rdi
-; FALLBACK2-NEXT:    orq %r9, %rdi
-; FALLBACK2-NEXT:    shrxq %rsi, -128(%rsp,%rax), %r9
-; FALLBACK2-NEXT:    addq %r8, %r8
-; FALLBACK2-NEXT:    shlxq %rcx, %r8, %r8
-; FALLBACK2-NEXT:    orq %r9, %r8
-; FALLBACK2-NEXT:    movq -104(%rsp,%rax), %r11
-; FALLBACK2-NEXT:    shrxq %rsi, %r11, %rbx
-; FALLBACK2-NEXT:    movq -96(%rsp,%rax), %r14
-; FALLBACK2-NEXT:    leaq (%r14,%r14), %r9
-; FALLBACK2-NEXT:    shlxq %rcx, %r9, %r9
-; FALLBACK2-NEXT:    orq %rbx, %r9
-; FALLBACK2-NEXT:    shrxq %rsi, %r10, %r10
-; FALLBACK2-NEXT:    addq %r11, %r11
-; FALLBACK2-NEXT:    shlxq %rcx, %r11, %r11
-; FALLBACK2-NEXT:    orq %r10, %r11
-; FALLBACK2-NEXT:    movq -88(%rsp,%rax), %r10
-; FALLBACK2-NEXT:    shrxq %rsi, %r10, %rbx
-; FALLBACK2-NEXT:    movq -80(%rsp,%rax), %r15
-; FALLBACK2-NEXT:    leaq (%r15,%r15), %r12
-; FALLBACK2-NEXT:    shlxq %rcx, %r12, %r12
-; FALLBACK2-NEXT:    orq %rbx, %r12
-; FALLBACK2-NEXT:    shrxq %rsi, %r14, %rbx
-; FALLBACK2-NEXT:    addq %r10, %r10
-; FALLBACK2-NEXT:    shlxq %rcx, %r10, %r10
-; FALLBACK2-NEXT:    orq %rbx, %r10
-; FALLBACK2-NEXT:    shrxq %rsi, %r15, %rbx
-; FALLBACK2-NEXT:    movq -72(%rsp,%rax), %rax
-; FALLBACK2-NEXT:    leaq (%rax,%rax), %r14
-; FALLBACK2-NEXT:    shlxq %rcx, %r14, %rcx
-; FALLBACK2-NEXT:    orq %rbx, %rcx
-; FALLBACK2-NEXT:    shrxq %rsi, %rax, %rax
-; FALLBACK2-NEXT:    movq %rax, 56(%rdx)
-; FALLBACK2-NEXT:    movq %rcx, 48(%rdx)
-; FALLBACK2-NEXT:    movq %r10, 32(%rdx)
-; FALLBACK2-NEXT:    movq %r12, 40(%rdx)
-; FALLBACK2-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK2-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK2-NEXT:    movq %r8, (%rdx)
-; FALLBACK2-NEXT:    movq %rdi, 8(%rdx)
-; FALLBACK2-NEXT:    addq $8, %rsp
-; FALLBACK2-NEXT:    popq %rbx
-; FALLBACK2-NEXT:    popq %r12
-; FALLBACK2-NEXT:    popq %r14
-; FALLBACK2-NEXT:    popq %r15
-; FALLBACK2-NEXT:    retq
-;
-; FALLBACK3-LABEL: lshr_64bytes:
-; FALLBACK3:       # %bb.0:
-; FALLBACK3-NEXT:    pushq %r15
-; FALLBACK3-NEXT:    pushq %r14
-; FALLBACK3-NEXT:    pushq %rbx
-; FALLBACK3-NEXT:    movq (%rdi), %rcx
-; FALLBACK3-NEXT:    movq 8(%rdi), %r8
-; FALLBACK3-NEXT:    movq 16(%rdi), %r9
-; FALLBACK3-NEXT:    movq 24(%rdi), %r10
-; FALLBACK3-NEXT:    movq 32(%rdi), %r11
-; FALLBACK3-NEXT:    movq 40(%rdi), %rbx
-; FALLBACK3-NEXT:    movq 48(%rdi), %r14
-; FALLBACK3-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK3-NEXT:    movl (%rsi), %eax
-; FALLBACK3-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK3-NEXT:    andl $56, %ecx
-; FALLBACK3-NEXT:    andl $56, %eax
-; FALLBACK3-NEXT:    movq -112(%rsp,%rax), %rdi
-; FALLBACK3-NEXT:    movq -128(%rsp,%rax), %rsi
-; FALLBACK3-NEXT:    movq -120(%rsp,%rax), %r9
-; FALLBACK3-NEXT:    movq %r9, %r8
-; FALLBACK3-NEXT:    shrdq %cl, %rdi, %r8
-; FALLBACK3-NEXT:    movq -96(%rsp,%rax), %r10
-; FALLBACK3-NEXT:    movq -104(%rsp,%rax), %r11
-; FALLBACK3-NEXT:    movq %r11, %rbx
-; FALLBACK3-NEXT:    shrdq %cl, %r10, %rbx
-; FALLBACK3-NEXT:    shrdq %cl, %r11, %rdi
-; FALLBACK3-NEXT:    movq -80(%rsp,%rax), %r11
-; FALLBACK3-NEXT:    movq -88(%rsp,%rax), %r14
-; FALLBACK3-NEXT:    movq %r14, %r15
-; FALLBACK3-NEXT:    shrdq %cl, %r11, %r15
-; FALLBACK3-NEXT:    shrdq %cl, %r14, %r10
-; FALLBACK3-NEXT:    movq -72(%rsp,%rax), %rax
-; FALLBACK3-NEXT:    shrdq %cl, %rax, %r11
-; FALLBACK3-NEXT:    shrxq %rcx, %rax, %rax
-; FALLBACK3-NEXT:    # kill: def $cl killed $cl killed $rcx
-; FALLBACK3-NEXT:    shrdq %cl, %r9, %rsi
-; FALLBACK3-NEXT:    movq %r11, 48(%rdx)
-; FALLBACK3-NEXT:    movq %r10, 32(%rdx)
-; FALLBACK3-NEXT:    movq %r15, 40(%rdx)
-; FALLBACK3-NEXT:    movq %rdi, 16(%rdx)
-; FALLBACK3-NEXT:    movq %rbx, 24(%rdx)
-; FALLBACK3-NEXT:    movq %rsi, (%rdx)
-; FALLBACK3-NEXT:    movq %r8, 8(%rdx)
-; FALLBACK3-NEXT:    movq %rax, 56(%rdx)
-; FALLBACK3-NEXT:    popq %rbx
-; FALLBACK3-NEXT:    popq %r14
-; FALLBACK3-NEXT:    popq %r15
-; FALLBACK3-NEXT:    retq
-;
-; FALLBACK4-LABEL: lshr_64bytes:
-; FALLBACK4:       # %bb.0:
-; FALLBACK4-NEXT:    pushq %rbp
-; FALLBACK4-NEXT:    pushq %r15
-; FALLBACK4-NEXT:    pushq %r14
-; FALLBACK4-NEXT:    pushq %r13
-; FALLBACK4-NEXT:    pushq %r12
-; FALLBACK4-NEXT:    pushq %rbx
-; FALLBACK4-NEXT:    pushq %rax
-; FALLBACK4-NEXT:    movups (%rdi), %xmm0
-; FALLBACK4-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK4-NEXT:    movups 32(%rdi), %xmm2
-; FALLBACK4-NEXT:    movups 48(%rdi), %xmm3
-; FALLBACK4-NEXT:    movl (%rsi), %r8d
-; FALLBACK4-NEXT:    xorps %xmm4, %xmm4
-; FALLBACK4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm3, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    leal (,%r8,8), %eax
-; FALLBACK4-NEXT:    andl $56, %eax
-; FALLBACK4-NEXT:    andl $56, %r8d
-; FALLBACK4-NEXT:    movq -128(%rsp,%r8), %r10
-; FALLBACK4-NEXT:    movq -120(%rsp,%r8), %r9
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r10
-; FALLBACK4-NEXT:    movl %eax, %esi
-; FALLBACK4-NEXT:    notb %sil
-; FALLBACK4-NEXT:    leaq (%r9,%r9), %rdi
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %rdi
-; FALLBACK4-NEXT:    orq %r10, %rdi
-; FALLBACK4-NEXT:    movq -104(%rsp,%r8), %r10
-; FALLBACK4-NEXT:    movq %r10, %rbx
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %rbx
-; FALLBACK4-NEXT:    movq -96(%rsp,%r8), %r12
-; FALLBACK4-NEXT:    leaq (%r12,%r12), %r11
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r11
-; FALLBACK4-NEXT:    orq %rbx, %r11
-; FALLBACK4-NEXT:    movq -112(%rsp,%r8), %rbx
-; FALLBACK4-NEXT:    movq %rbx, %r14
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r14
-; FALLBACK4-NEXT:    addq %r10, %r10
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r10
-; FALLBACK4-NEXT:    orq %r14, %r10
-; FALLBACK4-NEXT:    movq -88(%rsp,%r8), %r14
-; FALLBACK4-NEXT:    movq %r14, %r13
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r13
-; FALLBACK4-NEXT:    movq -80(%rsp,%r8), %rbp
-; FALLBACK4-NEXT:    leaq (%rbp,%rbp), %r15
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r15
-; FALLBACK4-NEXT:    orq %r13, %r15
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r12
-; FALLBACK4-NEXT:    addq %r14, %r14
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r14
-; FALLBACK4-NEXT:    orq %r12, %r14
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %rbp
-; FALLBACK4-NEXT:    movq -72(%rsp,%r8), %r8
-; FALLBACK4-NEXT:    leaq (%r8,%r8), %r12
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r12
-; FALLBACK4-NEXT:    orq %rbp, %r12
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r9
-; FALLBACK4-NEXT:    addq %rbx, %rbx
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %rbx
-; FALLBACK4-NEXT:    orq %r9, %rbx
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r8
-; FALLBACK4-NEXT:    movq %r8, 56(%rdx)
-; FALLBACK4-NEXT:    movq %rbx, 8(%rdx)
-; FALLBACK4-NEXT:    movq %r12, 48(%rdx)
-; FALLBACK4-NEXT:    movq %r14, 32(%rdx)
-; FALLBACK4-NEXT:    movq %r15, 40(%rdx)
-; FALLBACK4-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK4-NEXT:    movq %r11, 24(%rdx)
-; FALLBACK4-NEXT:    movq %rdi, (%rdx)
-; FALLBACK4-NEXT:    addq $8, %rsp
-; FALLBACK4-NEXT:    popq %rbx
-; FALLBACK4-NEXT:    popq %r12
-; FALLBACK4-NEXT:    popq %r13
-; FALLBACK4-NEXT:    popq %r14
-; FALLBACK4-NEXT:    popq %r15
-; FALLBACK4-NEXT:    popq %rbp
-; FALLBACK4-NEXT:    retq
-;
-; FALLBACK5-LABEL: lshr_64bytes:
-; FALLBACK5:       # %bb.0:
-; FALLBACK5-NEXT:    pushq %r15
-; FALLBACK5-NEXT:    pushq %r14
-; FALLBACK5-NEXT:    pushq %rbx
-; FALLBACK5-NEXT:    movups (%rdi), %xmm0
-; FALLBACK5-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK5-NEXT:    movups 32(%rdi), %xmm2
-; FALLBACK5-NEXT:    movups 48(%rdi), %xmm3
-; FALLBACK5-NEXT:    movl (%rsi), %eax
-; FALLBACK5-NEXT:    xorps %xmm4, %xmm4
-; FALLBACK5-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm3, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK5-NEXT:    andl $56, %ecx
-; FALLBACK5-NEXT:    andl $56, %eax
-; FALLBACK5-NEXT:    movq -96(%rsp,%rax), %rdi
-; FALLBACK5-NEXT:    movq -104(%rsp,%rax), %r9
-; FALLBACK5-NEXT:    movq %r9, %rsi
-; FALLBACK5-NEXT:    shrdq %cl, %rdi, %rsi
-; FALLBACK5-NEXT:    movq -112(%rsp,%rax), %r10
-; FALLBACK5-NEXT:    movq %r10, %r8
-; FALLBACK5-NEXT:    shrdq %cl, %r9, %r8
-; FALLBACK5-NEXT:    movq -80(%rsp,%rax), %r9
-; FALLBACK5-NEXT:    movq -88(%rsp,%rax), %r11
-; FALLBACK5-NEXT:    movq %r11, %rbx
-; FALLBACK5-NEXT:    shrdq %cl, %r9, %rbx
-; FALLBACK5-NEXT:    shrdq %cl, %r11, %rdi
-; FALLBACK5-NEXT:    movq -72(%rsp,%rax), %r11
-; FALLBACK5-NEXT:    shrdq %cl, %r11, %r9
-; FALLBACK5-NEXT:    movq -128(%rsp,%rax), %r14
-; FALLBACK5-NEXT:    movq -120(%rsp,%rax), %rax
-; FALLBACK5-NEXT:    movq %rax, %r15
-; FALLBACK5-NEXT:    shrdq %cl, %r10, %r15
-; FALLBACK5-NEXT:    shrdq %cl, %rax, %r14
-; FALLBACK5-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK5-NEXT:    shrq %cl, %r11
-; FALLBACK5-NEXT:    movq %r15, 8(%rdx)
-; FALLBACK5-NEXT:    movq %r9, 48(%rdx)
-; FALLBACK5-NEXT:    movq %r11, 56(%rdx)
-; FALLBACK5-NEXT:    movq %rdi, 32(%rdx)
-; FALLBACK5-NEXT:    movq %rbx, 40(%rdx)
-; FALLBACK5-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK5-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK5-NEXT:    movq %r14, (%rdx)
-; FALLBACK5-NEXT:    popq %rbx
-; FALLBACK5-NEXT:    popq %r14
-; FALLBACK5-NEXT:    popq %r15
-; FALLBACK5-NEXT:    retq
-;
-; FALLBACK6-LABEL: lshr_64bytes:
-; FALLBACK6:       # %bb.0:
-; FALLBACK6-NEXT:    pushq %r15
-; FALLBACK6-NEXT:    pushq %r14
-; FALLBACK6-NEXT:    pushq %r13
-; FALLBACK6-NEXT:    pushq %r12
-; FALLBACK6-NEXT:    pushq %rbx
-; FALLBACK6-NEXT:    movups (%rdi), %xmm0
-; FALLBACK6-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK6-NEXT:    movups 32(%rdi), %xmm2
-; FALLBACK6-NEXT:    movups 48(%rdi), %xmm3
-; FALLBACK6-NEXT:    movl (%rsi), %eax
-; FALLBACK6-NEXT:    xorps %xmm4, %xmm4
-; FALLBACK6-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm3, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK6-NEXT:    andl $56, %ecx
-; FALLBACK6-NEXT:    movl %ecx, %esi
-; FALLBACK6-NEXT:    andl $56, %eax
-; FALLBACK6-NEXT:    shrxq %rsi, -128(%rsp,%rax), %r8
-; FALLBACK6-NEXT:    notb %cl
-; FALLBACK6-NEXT:    movq -120(%rsp,%rax), %r10
-; FALLBACK6-NEXT:    movq -112(%rsp,%rax), %r9
-; FALLBACK6-NEXT:    leaq (%r10,%r10), %rdi
-; FALLBACK6-NEXT:    shlxq %rcx, %rdi, %rdi
-; FALLBACK6-NEXT:    orq %r8, %rdi
-; FALLBACK6-NEXT:    movq -104(%rsp,%rax), %r11
-; FALLBACK6-NEXT:    shrxq %rsi, %r11, %rbx
-; FALLBACK6-NEXT:    movq -96(%rsp,%rax), %r14
-; FALLBACK6-NEXT:    leaq (%r14,%r14), %r8
-; FALLBACK6-NEXT:    shlxq %rcx, %r8, %r8
-; FALLBACK6-NEXT:    orq %rbx, %r8
-; FALLBACK6-NEXT:    shrxq %rsi, %r9, %rbx
-; FALLBACK6-NEXT:    addq %r11, %r11
-; FALLBACK6-NEXT:    shlxq %rcx, %r11, %r11
-; FALLBACK6-NEXT:    orq %rbx, %r11
-; FALLBACK6-NEXT:    movq -88(%rsp,%rax), %rbx
-; FALLBACK6-NEXT:    shrxq %rsi, %rbx, %r15
-; FALLBACK6-NEXT:    movq -80(%rsp,%rax), %r12
-; FALLBACK6-NEXT:    leaq (%r12,%r12), %r13
-; FALLBACK6-NEXT:    shlxq %rcx, %r13, %r13
-; FALLBACK6-NEXT:    orq %r15, %r13
-; FALLBACK6-NEXT:    shrxq %rsi, %r14, %r14
-; FALLBACK6-NEXT:    addq %rbx, %rbx
-; FALLBACK6-NEXT:    shlxq %rcx, %rbx, %rbx
-; FALLBACK6-NEXT:    orq %r14, %rbx
-; FALLBACK6-NEXT:    shrxq %rsi, %r12, %r14
-; FALLBACK6-NEXT:    movq -72(%rsp,%rax), %rax
-; FALLBACK6-NEXT:    leaq (%rax,%rax), %r15
-; FALLBACK6-NEXT:    shlxq %rcx, %r15, %r15
-; FALLBACK6-NEXT:    orq %r14, %r15
-; FALLBACK6-NEXT:    shrxq %rsi, %r10, %r10
-; FALLBACK6-NEXT:    addq %r9, %r9
-; FALLBACK6-NEXT:    shlxq %rcx, %r9, %rcx
-; FALLBACK6-NEXT:    orq %r10, %rcx
-; FALLBACK6-NEXT:    shrxq %rsi, %rax, %rax
-; FALLBACK6-NEXT:    movq %rax, 56(%rdx)
-; FALLBACK6-NEXT:    movq %rcx, 8(%rdx)
-; FALLBACK6-NEXT:    movq %r15, 48(%rdx)
-; FALLBACK6-NEXT:    movq %rbx, 32(%rdx)
-; FALLBACK6-NEXT:    movq %r13, 40(%rdx)
-; FALLBACK6-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK6-NEXT:    movq %r8, 24(%rdx)
-; FALLBACK6-NEXT:    movq %rdi, (%rdx)
-; FALLBACK6-NEXT:    popq %rbx
-; FALLBACK6-NEXT:    popq %r12
-; FALLBACK6-NEXT:    popq %r13
-; FALLBACK6-NEXT:    popq %r14
-; FALLBACK6-NEXT:    popq %r15
-; FALLBACK6-NEXT:    retq
-;
-; FALLBACK7-LABEL: lshr_64bytes:
-; FALLBACK7:       # %bb.0:
-; FALLBACK7-NEXT:    pushq %r15
-; FALLBACK7-NEXT:    pushq %r14
-; FALLBACK7-NEXT:    pushq %rbx
-; FALLBACK7-NEXT:    movups (%rdi), %xmm0
-; FALLBACK7-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK7-NEXT:    movups 32(%rdi), %xmm2
-; FALLBACK7-NEXT:    movups 48(%rdi), %xmm3
-; FALLBACK7-NEXT:    movl (%rsi), %eax
-; FALLBACK7-NEXT:    xorps %xmm4, %xmm4
-; FALLBACK7-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm3, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK7-NEXT:    andl $56, %ecx
-; FALLBACK7-NEXT:    andl $56, %eax
-; FALLBACK7-NEXT:    movq -96(%rsp,%rax), %rdi
-; FALLBACK7-NEXT:    movq -104(%rsp,%rax), %r9
-; FALLBACK7-NEXT:    movq %r9, %rsi
-; FALLBACK7-NEXT:    shrdq %cl, %rdi, %rsi
-; FALLBACK7-NEXT:    movq -112(%rsp,%rax), %r10
-; FALLBACK7-NEXT:    movq %r10, %r8
-; FALLBACK7-NEXT:    shrdq %cl, %r9, %r8
-; FALLBACK7-NEXT:    movq -80(%rsp,%rax), %r9
-; FALLBACK7-NEXT:    movq -88(%rsp,%rax), %r11
-; FALLBACK7-NEXT:    movq %r11, %rbx
-; FALLBACK7-NEXT:    shrdq %cl, %r9, %rbx
-; FALLBACK7-NEXT:    shrdq %cl, %r11, %rdi
-; FALLBACK7-NEXT:    movq -72(%rsp,%rax), %r11
-; FALLBACK7-NEXT:    shrdq %cl, %r11, %r9
-; FALLBACK7-NEXT:    movq -128(%rsp,%rax), %r14
-; FALLBACK7-NEXT:    movq -120(%rsp,%rax), %rax
-; FALLBACK7-NEXT:    movq %rax, %r15
-; FALLBACK7-NEXT:    shrdq %cl, %r10, %r15
-; FALLBACK7-NEXT:    shrxq %rcx, %r11, %r10
-; FALLBACK7-NEXT:    # kill: def $cl killed $cl killed $rcx
-; FALLBACK7-NEXT:    shrdq %cl, %rax, %r14
-; FALLBACK7-NEXT:    movq %r15, 8(%rdx)
-; FALLBACK7-NEXT:    movq %r9, 48(%rdx)
-; FALLBACK7-NEXT:    movq %rdi, 32(%rdx)
-; FALLBACK7-NEXT:    movq %rbx, 40(%rdx)
-; FALLBACK7-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK7-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK7-NEXT:    movq %r14, (%rdx)
-; FALLBACK7-NEXT:    movq %r10, 56(%rdx)
-; FALLBACK7-NEXT:    popq %rbx
-; FALLBACK7-NEXT:    popq %r14
-; FALLBACK7-NEXT:    popq %r15
-; FALLBACK7-NEXT:    retq
-;
-; FALLBACK8-LABEL: lshr_64bytes:
-; FALLBACK8:       # %bb.0:
-; FALLBACK8-NEXT:    pushq %rbp
-; FALLBACK8-NEXT:    pushq %r15
-; FALLBACK8-NEXT:    pushq %r14
-; FALLBACK8-NEXT:    pushq %r13
-; FALLBACK8-NEXT:    pushq %r12
-; FALLBACK8-NEXT:    pushq %rbx
-; FALLBACK8-NEXT:    pushq %rax
-; FALLBACK8-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK8-NEXT:    vmovups 32(%rdi), %ymm1
-; FALLBACK8-NEXT:    movl (%rsi), %r9d
-; FALLBACK8-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; FALLBACK8-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    leal (,%r9,8), %eax
-; FALLBACK8-NEXT:    andl $56, %eax
-; FALLBACK8-NEXT:    andl $56, %r9d
-; FALLBACK8-NEXT:    movq -128(%rsp,%r9), %r10
-; FALLBACK8-NEXT:    movq -120(%rsp,%r9), %r8
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r10
-; FALLBACK8-NEXT:    movl %eax, %esi
-; FALLBACK8-NEXT:    notb %sil
-; FALLBACK8-NEXT:    leaq (%r8,%r8), %rdi
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %rdi
-; FALLBACK8-NEXT:    orq %r10, %rdi
-; FALLBACK8-NEXT:    movq -104(%rsp,%r9), %r10
-; FALLBACK8-NEXT:    movq %r10, %rbx
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %rbx
-; FALLBACK8-NEXT:    movq -96(%rsp,%r9), %r12
-; FALLBACK8-NEXT:    leaq (%r12,%r12), %r11
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r11
-; FALLBACK8-NEXT:    orq %rbx, %r11
-; FALLBACK8-NEXT:    movq -112(%rsp,%r9), %rbx
-; FALLBACK8-NEXT:    movq %rbx, %r14
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r14
-; FALLBACK8-NEXT:    addq %r10, %r10
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r10
-; FALLBACK8-NEXT:    orq %r14, %r10
-; FALLBACK8-NEXT:    movq -88(%rsp,%r9), %r14
-; FALLBACK8-NEXT:    movq %r14, %r13
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r13
-; FALLBACK8-NEXT:    movq -80(%rsp,%r9), %rbp
-; FALLBACK8-NEXT:    leaq (%rbp,%rbp), %r15
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r15
-; FALLBACK8-NEXT:    orq %r13, %r15
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r12
-; FALLBACK8-NEXT:    addq %r14, %r14
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r14
-; FALLBACK8-NEXT:    orq %r12, %r14
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %rbp
-; FALLBACK8-NEXT:    movq -72(%rsp,%r9), %r9
-; FALLBACK8-NEXT:    leaq (%r9,%r9), %r12
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r12
-; FALLBACK8-NEXT:    orq %rbp, %r12
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r8
-; FALLBACK8-NEXT:    addq %rbx, %rbx
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %rbx
-; FALLBACK8-NEXT:    orq %r8, %rbx
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r9
-; FALLBACK8-NEXT:    movq %r9, 56(%rdx)
-; FALLBACK8-NEXT:    movq %rbx, 8(%rdx)
-; FALLBACK8-NEXT:    movq %r12, 48(%rdx)
-; FALLBACK8-NEXT:    movq %r14, 32(%rdx)
-; FALLBACK8-NEXT:    movq %r15, 40(%rdx)
-; FALLBACK8-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK8-NEXT:    movq %r11, 24(%rdx)
-; FALLBACK8-NEXT:    movq %rdi, (%rdx)
-; FALLBACK8-NEXT:    addq $8, %rsp
-; FALLBACK8-NEXT:    popq %rbx
-; FALLBACK8-NEXT:    popq %r12
-; FALLBACK8-NEXT:    popq %r13
-; FALLBACK8-NEXT:    popq %r14
-; FALLBACK8-NEXT:    popq %r15
-; FALLBACK8-NEXT:    popq %rbp
-; FALLBACK8-NEXT:    vzeroupper
-; FALLBACK8-NEXT:    retq
-;
-; FALLBACK9-LABEL: lshr_64bytes:
-; FALLBACK9:       # %bb.0:
-; FALLBACK9-NEXT:    pushq %r15
-; FALLBACK9-NEXT:    pushq %r14
-; FALLBACK9-NEXT:    pushq %rbx
-; FALLBACK9-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK9-NEXT:    vmovups 32(%rdi), %ymm1
-; FALLBACK9-NEXT:    movl (%rsi), %eax
-; FALLBACK9-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; FALLBACK9-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK9-NEXT:    andl $56, %ecx
-; FALLBACK9-NEXT:    andl $56, %eax
-; FALLBACK9-NEXT:    movq -96(%rsp,%rax), %rdi
-; FALLBACK9-NEXT:    movq -104(%rsp,%rax), %r9
-; FALLBACK9-NEXT:    movq %r9, %rsi
-; FALLBACK9-NEXT:    shrdq %cl, %rdi, %rsi
-; FALLBACK9-NEXT:    movq -112(%rsp,%rax), %r10
-; FALLBACK9-NEXT:    movq %r10, %r8
-; FALLBACK9-NEXT:    shrdq %cl, %r9, %r8
-; FALLBACK9-NEXT:    movq -80(%rsp,%rax), %r9
-; FALLBACK9-NEXT:    movq -88(%rsp,%rax), %r11
-; FALLBACK9-NEXT:    movq %r11, %rbx
-; FALLBACK9-NEXT:    shrdq %cl, %r9, %rbx
-; FALLBACK9-NEXT:    shrdq %cl, %r11, %rdi
-; FALLBACK9-NEXT:    movq -72(%rsp,%rax), %r11
-; FALLBACK9-NEXT:    shrdq %cl, %r11, %r9
-; FALLBACK9-NEXT:    movq -128(%rsp,%rax), %r14
-; FALLBACK9-NEXT:    movq -120(%rsp,%rax), %rax
-; FALLBACK9-NEXT:    movq %rax, %r15
-; FALLBACK9-NEXT:    shrdq %cl, %r10, %r15
-; FALLBACK9-NEXT:    shrdq %cl, %rax, %r14
-; FALLBACK9-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK9-NEXT:    shrq %cl, %r11
-; FALLBACK9-NEXT:    movq %r15, 8(%rdx)
-; FALLBACK9-NEXT:    movq %r9, 48(%rdx)
-; FALLBACK9-NEXT:    movq %r11, 56(%rdx)
-; FALLBACK9-NEXT:    movq %rdi, 32(%rdx)
-; FALLBACK9-NEXT:    movq %rbx, 40(%rdx)
-; FALLBACK9-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK9-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK9-NEXT:    movq %r14, (%rdx)
-; FALLBACK9-NEXT:    popq %rbx
-; FALLBACK9-NEXT:    popq %r14
-; FALLBACK9-NEXT:    popq %r15
-; FALLBACK9-NEXT:    vzeroupper
-; FALLBACK9-NEXT:    retq
-;
-; FALLBACK10-LABEL: lshr_64bytes:
-; FALLBACK10:       # %bb.0:
-; FALLBACK10-NEXT:    pushq %r15
-; FALLBACK10-NEXT:    pushq %r14
-; FALLBACK10-NEXT:    pushq %r13
-; FALLBACK10-NEXT:    pushq %r12
-; FALLBACK10-NEXT:    pushq %rbx
-; FALLBACK10-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK10-NEXT:    vmovups 32(%rdi), %ymm1
-; FALLBACK10-NEXT:    movl (%rsi), %esi
-; FALLBACK10-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; FALLBACK10-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK10-NEXT:    andl $56, %eax
-; FALLBACK10-NEXT:    movl %eax, %ecx
-; FALLBACK10-NEXT:    andl $56, %esi
-; FALLBACK10-NEXT:    shrxq %rcx, -128(%rsp,%rsi), %r8
-; FALLBACK10-NEXT:    notb %al
-; FALLBACK10-NEXT:    movq -120(%rsp,%rsi), %r10
-; FALLBACK10-NEXT:    movq -112(%rsp,%rsi), %r9
-; FALLBACK10-NEXT:    leaq (%r10,%r10), %rdi
-; FALLBACK10-NEXT:    shlxq %rax, %rdi, %rdi
-; FALLBACK10-NEXT:    orq %r8, %rdi
-; FALLBACK10-NEXT:    movq -104(%rsp,%rsi), %r11
-; FALLBACK10-NEXT:    shrxq %rcx, %r11, %rbx
-; FALLBACK10-NEXT:    movq -96(%rsp,%rsi), %r14
-; FALLBACK10-NEXT:    leaq (%r14,%r14), %r8
-; FALLBACK10-NEXT:    shlxq %rax, %r8, %r8
-; FALLBACK10-NEXT:    orq %rbx, %r8
-; FALLBACK10-NEXT:    shrxq %rcx, %r9, %rbx
-; FALLBACK10-NEXT:    addq %r11, %r11
-; FALLBACK10-NEXT:    shlxq %rax, %r11, %r11
-; FALLBACK10-NEXT:    orq %rbx, %r11
-; FALLBACK10-NEXT:    movq -88(%rsp,%rsi), %rbx
-; FALLBACK10-NEXT:    shrxq %rcx, %rbx, %r15
-; FALLBACK10-NEXT:    movq -80(%rsp,%rsi), %r12
-; FALLBACK10-NEXT:    leaq (%r12,%r12), %r13
-; FALLBACK10-NEXT:    shlxq %rax, %r13, %r13
-; FALLBACK10-NEXT:    orq %r15, %r13
-; FALLBACK10-NEXT:    shrxq %rcx, %r14, %r14
-; FALLBACK10-NEXT:    addq %rbx, %rbx
-; FALLBACK10-NEXT:    shlxq %rax, %rbx, %rbx
-; FALLBACK10-NEXT:    orq %r14, %rbx
-; FALLBACK10-NEXT:    shrxq %rcx, %r12, %r14
-; FALLBACK10-NEXT:    movq -72(%rsp,%rsi), %rsi
-; FALLBACK10-NEXT:    leaq (%rsi,%rsi), %r15
-; FALLBACK10-NEXT:    shlxq %rax, %r15, %r15
-; FALLBACK10-NEXT:    orq %r14, %r15
-; FALLBACK10-NEXT:    shrxq %rcx, %r10, %r10
-; FALLBACK10-NEXT:    addq %r9, %r9
-; FALLBACK10-NEXT:    shlxq %rax, %r9, %rax
-; FALLBACK10-NEXT:    orq %r10, %rax
-; FALLBACK10-NEXT:    shrxq %rcx, %rsi, %rcx
-; FALLBACK10-NEXT:    movq %rcx, 56(%rdx)
-; FALLBACK10-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK10-NEXT:    movq %r15, 48(%rdx)
-; FALLBACK10-NEXT:    movq %rbx, 32(%rdx)
-; FALLBACK10-NEXT:    movq %r13, 40(%rdx)
-; FALLBACK10-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK10-NEXT:    movq %r8, 24(%rdx)
-; FALLBACK10-NEXT:    movq %rdi, (%rdx)
-; FALLBACK10-NEXT:    popq %rbx
-; FALLBACK10-NEXT:    popq %r12
-; FALLBACK10-NEXT:    popq %r13
-; FALLBACK10-NEXT:    popq %r14
-; FALLBACK10-NEXT:    popq %r15
-; FALLBACK10-NEXT:    vzeroupper
-; FALLBACK10-NEXT:    retq
-;
-; FALLBACK11-LABEL: lshr_64bytes:
-; FALLBACK11:       # %bb.0:
-; FALLBACK11-NEXT:    pushq %r15
-; FALLBACK11-NEXT:    pushq %r14
-; FALLBACK11-NEXT:    pushq %rbx
-; FALLBACK11-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK11-NEXT:    vmovups 32(%rdi), %ymm1
-; FALLBACK11-NEXT:    movl (%rsi), %eax
-; FALLBACK11-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; FALLBACK11-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK11-NEXT:    andl $56, %ecx
-; FALLBACK11-NEXT:    andl $56, %eax
-; FALLBACK11-NEXT:    movq -96(%rsp,%rax), %rdi
-; FALLBACK11-NEXT:    movq -104(%rsp,%rax), %r9
-; FALLBACK11-NEXT:    movq %r9, %rsi
-; FALLBACK11-NEXT:    shrdq %cl, %rdi, %rsi
-; FALLBACK11-NEXT:    movq -112(%rsp,%rax), %r10
-; FALLBACK11-NEXT:    movq %r10, %r8
-; FALLBACK11-NEXT:    shrdq %cl, %r9, %r8
-; FALLBACK11-NEXT:    movq -80(%rsp,%rax), %r9
-; FALLBACK11-NEXT:    movq -88(%rsp,%rax), %r11
-; FALLBACK11-NEXT:    movq %r11, %rbx
-; FALLBACK11-NEXT:    shrdq %cl, %r9, %rbx
-; FALLBACK11-NEXT:    shrdq %cl, %r11, %rdi
-; FALLBACK11-NEXT:    movq -72(%rsp,%rax), %r11
-; FALLBACK11-NEXT:    shrdq %cl, %r11, %r9
-; FALLBACK11-NEXT:    movq -128(%rsp,%rax), %r14
-; FALLBACK11-NEXT:    movq -120(%rsp,%rax), %rax
-; FALLBACK11-NEXT:    movq %rax, %r15
-; FALLBACK11-NEXT:    shrdq %cl, %r10, %r15
-; FALLBACK11-NEXT:    shrxq %rcx, %r11, %r10
-; FALLBACK11-NEXT:    # kill: def $cl killed $cl killed $rcx
-; FALLBACK11-NEXT:    shrdq %cl, %rax, %r14
-; FALLBACK11-NEXT:    movq %r15, 8(%rdx)
-; FALLBACK11-NEXT:    movq %r9, 48(%rdx)
-; FALLBACK11-NEXT:    movq %rdi, 32(%rdx)
-; FALLBACK11-NEXT:    movq %rbx, 40(%rdx)
-; FALLBACK11-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK11-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK11-NEXT:    movq %r14, (%rdx)
-; FALLBACK11-NEXT:    movq %r10, 56(%rdx)
-; FALLBACK11-NEXT:    popq %rbx
-; FALLBACK11-NEXT:    popq %r14
-; FALLBACK11-NEXT:    popq %r15
-; FALLBACK11-NEXT:    vzeroupper
-; FALLBACK11-NEXT:    retq
-;
-; FALLBACK12-LABEL: lshr_64bytes:
-; FALLBACK12:       # %bb.0:
-; FALLBACK12-NEXT:    pushq %rbp
-; FALLBACK12-NEXT:    pushq %r15
-; FALLBACK12-NEXT:    pushq %r14
-; FALLBACK12-NEXT:    pushq %r13
-; FALLBACK12-NEXT:    pushq %r12
-; FALLBACK12-NEXT:    pushq %rbx
-; FALLBACK12-NEXT:    pushq %rax
-; FALLBACK12-NEXT:    vmovups (%rdi), %zmm0
-; FALLBACK12-NEXT:    movl (%rsi), %r9d
-; FALLBACK12-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK12-NEXT:    vmovups %zmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    vmovups %zmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    leal (,%r9,8), %eax
-; FALLBACK12-NEXT:    andl $56, %eax
-; FALLBACK12-NEXT:    andl $56, %r9d
-; FALLBACK12-NEXT:    movq -128(%rsp,%r9), %r10
-; FALLBACK12-NEXT:    movq -120(%rsp,%r9), %r8
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r10
-; FALLBACK12-NEXT:    movl %eax, %esi
-; FALLBACK12-NEXT:    notb %sil
-; FALLBACK12-NEXT:    leaq (%r8,%r8), %rdi
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %rdi
-; FALLBACK12-NEXT:    orq %r10, %rdi
-; FALLBACK12-NEXT:    movq -104(%rsp,%r9), %r10
-; FALLBACK12-NEXT:    movq %r10, %rbx
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %rbx
-; FALLBACK12-NEXT:    movq -96(%rsp,%r9), %r12
-; FALLBACK12-NEXT:    leaq (%r12,%r12), %r11
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r11
-; FALLBACK12-NEXT:    orq %rbx, %r11
-; FALLBACK12-NEXT:    movq -112(%rsp,%r9), %rbx
-; FALLBACK12-NEXT:    movq %rbx, %r14
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r14
-; FALLBACK12-NEXT:    addq %r10, %r10
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r10
-; FALLBACK12-NEXT:    orq %r14, %r10
-; FALLBACK12-NEXT:    movq -88(%rsp,%r9), %r14
-; FALLBACK12-NEXT:    movq %r14, %r13
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r13
-; FALLBACK12-NEXT:    movq -80(%rsp,%r9), %rbp
-; FALLBACK12-NEXT:    leaq (%rbp,%rbp), %r15
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r15
-; FALLBACK12-NEXT:    orq %r13, %r15
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r12
-; FALLBACK12-NEXT:    addq %r14, %r14
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r14
-; FALLBACK12-NEXT:    orq %r12, %r14
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %rbp
-; FALLBACK12-NEXT:    movq -72(%rsp,%r9), %r9
-; FALLBACK12-NEXT:    leaq (%r9,%r9), %r12
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r12
-; FALLBACK12-NEXT:    orq %rbp, %r12
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r8
-; FALLBACK12-NEXT:    addq %rbx, %rbx
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %rbx
-; FALLBACK12-NEXT:    orq %r8, %rbx
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r9
-; FALLBACK12-NEXT:    movq %r9, 56(%rdx)
-; FALLBACK12-NEXT:    movq %rbx, 8(%rdx)
-; FALLBACK12-NEXT:    movq %r12, 48(%rdx)
-; FALLBACK12-NEXT:    movq %r14, 32(%rdx)
-; FALLBACK12-NEXT:    movq %r15, 40(%rdx)
-; FALLBACK12-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK12-NEXT:    movq %r11, 24(%rdx)
-; FALLBACK12-NEXT:    movq %rdi, (%rdx)
-; FALLBACK12-NEXT:    addq $8, %rsp
-; FALLBACK12-NEXT:    popq %rbx
-; FALLBACK12-NEXT:    popq %r12
-; FALLBACK12-NEXT:    popq %r13
-; FALLBACK12-NEXT:    popq %r14
-; FALLBACK12-NEXT:    popq %r15
-; FALLBACK12-NEXT:    popq %rbp
-; FALLBACK12-NEXT:    vzeroupper
-; FALLBACK12-NEXT:    retq
-;
-; FALLBACK13-LABEL: lshr_64bytes:
-; FALLBACK13:       # %bb.0:
-; FALLBACK13-NEXT:    pushq %r15
-; FALLBACK13-NEXT:    pushq %r14
-; FALLBACK13-NEXT:    pushq %rbx
-; FALLBACK13-NEXT:    vmovups (%rdi), %zmm0
-; FALLBACK13-NEXT:    movl (%rsi), %edi
-; FALLBACK13-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK13-NEXT:    vmovups %zmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    vmovups %zmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    leal (,%rdi,8), %ecx
-; FALLBACK13-NEXT:    andl $56, %ecx
-; FALLBACK13-NEXT:    andl $56, %edi
-; FALLBACK13-NEXT:    movq -96(%rsp,%rdi), %rsi
-; FALLBACK13-NEXT:    movq -104(%rsp,%rdi), %r9
-; FALLBACK13-NEXT:    movq %r9, %rax
-; FALLBACK13-NEXT:    shrdq %cl, %rsi, %rax
-; FALLBACK13-NEXT:    movq -112(%rsp,%rdi), %r10
-; FALLBACK13-NEXT:    movq %r10, %r8
-; FALLBACK13-NEXT:    shrdq %cl, %r9, %r8
-; FALLBACK13-NEXT:    movq -80(%rsp,%rdi), %r9
-; FALLBACK13-NEXT:    movq -88(%rsp,%rdi), %r11
-; FALLBACK13-NEXT:    movq %r11, %rbx
-; FALLBACK13-NEXT:    shrdq %cl, %r9, %rbx
-; FALLBACK13-NEXT:    shrdq %cl, %r11, %rsi
-; FALLBACK13-NEXT:    movq -72(%rsp,%rdi), %r11
-; FALLBACK13-NEXT:    shrdq %cl, %r11, %r9
-; FALLBACK13-NEXT:    movq -128(%rsp,%rdi), %r14
-; FALLBACK13-NEXT:    movq -120(%rsp,%rdi), %rdi
-; FALLBACK13-NEXT:    movq %rdi, %r15
-; FALLBACK13-NEXT:    shrdq %cl, %r10, %r15
-; FALLBACK13-NEXT:    shrdq %cl, %rdi, %r14
-; FALLBACK13-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK13-NEXT:    shrq %cl, %r11
-; FALLBACK13-NEXT:    movq %r15, 8(%rdx)
-; FALLBACK13-NEXT:    movq %r9, 48(%rdx)
-; FALLBACK13-NEXT:    movq %r11, 56(%rdx)
-; FALLBACK13-NEXT:    movq %rsi, 32(%rdx)
-; FALLBACK13-NEXT:    movq %rbx, 40(%rdx)
-; FALLBACK13-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK13-NEXT:    movq %rax, 24(%rdx)
-; FALLBACK13-NEXT:    movq %r14, (%rdx)
-; FALLBACK13-NEXT:    popq %rbx
-; FALLBACK13-NEXT:    popq %r14
-; FALLBACK13-NEXT:    popq %r15
-; FALLBACK13-NEXT:    vzeroupper
-; FALLBACK13-NEXT:    retq
-;
-; FALLBACK14-LABEL: lshr_64bytes:
-; FALLBACK14:       # %bb.0:
-; FALLBACK14-NEXT:    pushq %r15
-; FALLBACK14-NEXT:    pushq %r14
-; FALLBACK14-NEXT:    pushq %r13
-; FALLBACK14-NEXT:    pushq %r12
-; FALLBACK14-NEXT:    pushq %rbx
-; FALLBACK14-NEXT:    vmovups (%rdi), %zmm0
-; FALLBACK14-NEXT:    movl (%rsi), %esi
-; FALLBACK14-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK14-NEXT:    vmovups %zmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    vmovups %zmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK14-NEXT:    andl $56, %eax
-; FALLBACK14-NEXT:    movl %eax, %ecx
-; FALLBACK14-NEXT:    andl $56, %esi
-; FALLBACK14-NEXT:    shrxq %rcx, -128(%rsp,%rsi), %r8
-; FALLBACK14-NEXT:    notb %al
-; FALLBACK14-NEXT:    movq -120(%rsp,%rsi), %r10
-; FALLBACK14-NEXT:    movq -112(%rsp,%rsi), %r9
-; FALLBACK14-NEXT:    leaq (%r10,%r10), %rdi
-; FALLBACK14-NEXT:    shlxq %rax, %rdi, %rdi
-; FALLBACK14-NEXT:    orq %r8, %rdi
-; FALLBACK14-NEXT:    movq -104(%rsp,%rsi), %r11
-; FALLBACK14-NEXT:    shrxq %rcx, %r11, %rbx
-; FALLBACK14-NEXT:    movq -96(%rsp,%rsi), %r14
-; FALLBACK14-NEXT:    leaq (%r14,%r14), %r8
-; FALLBACK14-NEXT:    shlxq %rax, %r8, %r8
-; FALLBACK14-NEXT:    orq %rbx, %r8
-; FALLBACK14-NEXT:    shrxq %rcx, %r9, %rbx
-; FALLBACK14-NEXT:    addq %r11, %r11
-; FALLBACK14-NEXT:    shlxq %rax, %r11, %r11
-; FALLBACK14-NEXT:    orq %rbx, %r11
-; FALLBACK14-NEXT:    movq -88(%rsp,%rsi), %rbx
-; FALLBACK14-NEXT:    shrxq %rcx, %rbx, %r15
-; FALLBACK14-NEXT:    movq -80(%rsp,%rsi), %r12
-; FALLBACK14-NEXT:    leaq (%r12,%r12), %r13
-; FALLBACK14-NEXT:    shlxq %rax, %r13, %r13
-; FALLBACK14-NEXT:    orq %r15, %r13
-; FALLBACK14-NEXT:    shrxq %rcx, %r14, %r14
-; FALLBACK14-NEXT:    addq %rbx, %rbx
-; FALLBACK14-NEXT:    shlxq %rax, %rbx, %rbx
-; FALLBACK14-NEXT:    orq %r14, %rbx
-; FALLBACK14-NEXT:    shrxq %rcx, %r12, %r14
-; FALLBACK14-NEXT:    movq -72(%rsp,%rsi), %rsi
-; FALLBACK14-NEXT:    leaq (%rsi,%rsi), %r15
-; FALLBACK14-NEXT:    shlxq %rax, %r15, %r15
-; FALLBACK14-NEXT:    orq %r14, %r15
-; FALLBACK14-NEXT:    shrxq %rcx, %r10, %r10
-; FALLBACK14-NEXT:    addq %r9, %r9
-; FALLBACK14-NEXT:    shlxq %rax, %r9, %rax
-; FALLBACK14-NEXT:    orq %r10, %rax
-; FALLBACK14-NEXT:    shrxq %rcx, %rsi, %rcx
-; FALLBACK14-NEXT:    movq %rcx, 56(%rdx)
-; FALLBACK14-NEXT:    movq %rax, 8(%rdx)
-; FALLBACK14-NEXT:    movq %r15, 48(%rdx)
-; FALLBACK14-NEXT:    movq %rbx, 32(%rdx)
-; FALLBACK14-NEXT:    movq %r13, 40(%rdx)
-; FALLBACK14-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK14-NEXT:    movq %r8, 24(%rdx)
-; FALLBACK14-NEXT:    movq %rdi, (%rdx)
-; FALLBACK14-NEXT:    popq %rbx
-; FALLBACK14-NEXT:    popq %r12
-; FALLBACK14-NEXT:    popq %r13
-; FALLBACK14-NEXT:    popq %r14
-; FALLBACK14-NEXT:    popq %r15
-; FALLBACK14-NEXT:    vzeroupper
-; FALLBACK14-NEXT:    retq
-;
-; FALLBACK15-LABEL: lshr_64bytes:
-; FALLBACK15:       # %bb.0:
-; FALLBACK15-NEXT:    pushq %r15
-; FALLBACK15-NEXT:    pushq %r14
-; FALLBACK15-NEXT:    pushq %rbx
-; FALLBACK15-NEXT:    vmovups (%rdi), %zmm0
-; FALLBACK15-NEXT:    movl (%rsi), %eax
-; FALLBACK15-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK15-NEXT:    vmovups %zmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    vmovups %zmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK15-NEXT:    andl $56, %ecx
-; FALLBACK15-NEXT:    andl $56, %eax
-; FALLBACK15-NEXT:    movq -96(%rsp,%rax), %rdi
-; FALLBACK15-NEXT:    movq -104(%rsp,%rax), %r9
-; FALLBACK15-NEXT:    movq %r9, %rsi
-; FALLBACK15-NEXT:    shrdq %cl, %rdi, %rsi
-; FALLBACK15-NEXT:    movq -112(%rsp,%rax), %r10
-; FALLBACK15-NEXT:    movq %r10, %r8
-; FALLBACK15-NEXT:    shrdq %cl, %r9, %r8
-; FALLBACK15-NEXT:    movq -80(%rsp,%rax), %r9
-; FALLBACK15-NEXT:    movq -88(%rsp,%rax), %r11
-; FALLBACK15-NEXT:    movq %r11, %rbx
-; FALLBACK15-NEXT:    shrdq %cl, %r9, %rbx
-; FALLBACK15-NEXT:    shrdq %cl, %r11, %rdi
-; FALLBACK15-NEXT:    movq -72(%rsp,%rax), %r11
-; FALLBACK15-NEXT:    shrdq %cl, %r11, %r9
-; FALLBACK15-NEXT:    movq -128(%rsp,%rax), %r14
-; FALLBACK15-NEXT:    movq -120(%rsp,%rax), %rax
-; FALLBACK15-NEXT:    movq %rax, %r15
-; FALLBACK15-NEXT:    shrdq %cl, %r10, %r15
-; FALLBACK15-NEXT:    shrxq %rcx, %r11, %r10
-; FALLBACK15-NEXT:    # kill: def $cl killed $cl killed $rcx
-; FALLBACK15-NEXT:    shrdq %cl, %rax, %r14
-; FALLBACK15-NEXT:    movq %r15, 8(%rdx)
-; FALLBACK15-NEXT:    movq %r9, 48(%rdx)
-; FALLBACK15-NEXT:    movq %rdi, 32(%rdx)
-; FALLBACK15-NEXT:    movq %rbx, 40(%rdx)
-; FALLBACK15-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK15-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK15-NEXT:    movq %r14, (%rdx)
-; FALLBACK15-NEXT:    movq %r10, 56(%rdx)
-; FALLBACK15-NEXT:    popq %rbx
-; FALLBACK15-NEXT:    popq %r14
-; FALLBACK15-NEXT:    popq %r15
-; FALLBACK15-NEXT:    vzeroupper
-; FALLBACK15-NEXT:    retq
-;
-; FALLBACK16-LABEL: lshr_64bytes:
-; FALLBACK16:       # %bb.0:
-; FALLBACK16-NEXT:    pushl %ebp
-; FALLBACK16-NEXT:    pushl %ebx
-; FALLBACK16-NEXT:    pushl %edi
-; FALLBACK16-NEXT:    pushl %esi
-; FALLBACK16-NEXT:    subl $204, %esp
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK16-NEXT:    movl (%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 4(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 8(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 12(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 16(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 20(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 24(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 28(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 32(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 36(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 40(%eax), %ebp
-; FALLBACK16-NEXT:    movl 44(%eax), %ebx
-; FALLBACK16-NEXT:    movl 48(%eax), %edi
-; FALLBACK16-NEXT:    movl 52(%eax), %esi
-; FALLBACK16-NEXT:    movl 56(%eax), %edx
-; FALLBACK16-NEXT:    movl 60(%eax), %ecx
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK16-NEXT:    movl (%eax), %eax
-; FALLBACK16-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK16-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %eax, %esi
-; FALLBACK16-NEXT:    andl $60, %esi
-; FALLBACK16-NEXT:    movl 68(%esp,%esi), %edx
-; FALLBACK16-NEXT:    shll $3, %eax
-; FALLBACK16-NEXT:    andl $24, %eax
-; FALLBACK16-NEXT:    movl %edx, %edi
-; FALLBACK16-NEXT:    movl %eax, %ecx
-; FALLBACK16-NEXT:    shrl %cl, %edi
-; FALLBACK16-NEXT:    movl 72(%esp,%esi), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK16-NEXT:    movb %al, %ch
-; FALLBACK16-NEXT:    notb %ch
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebx
-; FALLBACK16-NEXT:    orl %edi, %ebx
-; FALLBACK16-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 64(%esp,%esi), %edi
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edi
-; FALLBACK16-NEXT:    addl %edx, %edx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %edx
-; FALLBACK16-NEXT:    orl %edi, %edx
-; FALLBACK16-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 76(%esp,%esi), %edx
-; FALLBACK16-NEXT:    movl %edx, %ebp
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shrl %cl, %ebp
-; FALLBACK16-NEXT:    movl 80(%esp,%esi), %edi
-; FALLBACK16-NEXT:    leal (%edi,%edi), %ebx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebx
-; FALLBACK16-NEXT:    orl %ebp, %ebx
-; FALLBACK16-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK16-NEXT:    shrl %cl, %ebx
-; FALLBACK16-NEXT:    addl %edx, %edx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %edx
-; FALLBACK16-NEXT:    orl %ebx, %edx
-; FALLBACK16-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 84(%esp,%esi), %ebx
-; FALLBACK16-NEXT:    movl %ebx, %ebp
-; FALLBACK16-NEXT:    movl %eax, %edx
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %ebp
-; FALLBACK16-NEXT:    movl 88(%esp,%esi), %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    addl %eax, %eax
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %eax
-; FALLBACK16-NEXT:    orl %ebp, %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edi
-; FALLBACK16-NEXT:    addl %ebx, %ebx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; FALLBACK16-NEXT:    shll %cl, %ebx
-; FALLBACK16-NEXT:    orl %edi, %ebx
-; FALLBACK16-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 92(%esp,%esi), %ebx
-; FALLBACK16-NEXT:    movl %ebx, %ebp
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %ebp
-; FALLBACK16-NEXT:    movl 96(%esp,%esi), %edi
-; FALLBACK16-NEXT:    leal (%edi,%edi), %eax
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %eax
-; FALLBACK16-NEXT:    orl %ebp, %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    shrl %cl, %eax
-; FALLBACK16-NEXT:    addl %ebx, %ebx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebx
-; FALLBACK16-NEXT:    orl %eax, %ebx
-; FALLBACK16-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 100(%esp,%esi), %ebx
-; FALLBACK16-NEXT:    movl %ebx, %ebp
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %ebp
-; FALLBACK16-NEXT:    movl 104(%esp,%esi), %edx
-; FALLBACK16-NEXT:    leal (%edx,%edx), %eax
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %eax
-; FALLBACK16-NEXT:    orl %ebp, %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edi
-; FALLBACK16-NEXT:    addl %ebx, %ebx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebx
-; FALLBACK16-NEXT:    orl %edi, %ebx
-; FALLBACK16-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 108(%esp,%esi), %edi
-; FALLBACK16-NEXT:    movl %edi, %ebp
-; FALLBACK16-NEXT:    movl %eax, %ecx
-; FALLBACK16-NEXT:    shrl %cl, %ebp
-; FALLBACK16-NEXT:    movl 112(%esp,%esi), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK16-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebx
-; FALLBACK16-NEXT:    orl %ebp, %ebx
-; FALLBACK16-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edx
-; FALLBACK16-NEXT:    addl %edi, %edi
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %edi
-; FALLBACK16-NEXT:    orl %edx, %edi
-; FALLBACK16-NEXT:    movl %esi, %edx
-; FALLBACK16-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 116(%esp,%esi), %esi
-; FALLBACK16-NEXT:    movl %esi, %ebx
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shrl %cl, %ebx
-; FALLBACK16-NEXT:    movl 120(%esp,%edx), %eax
-; FALLBACK16-NEXT:    leal (%eax,%eax), %ebp
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebp
-; FALLBACK16-NEXT:    orl %ebx, %ebp
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK16-NEXT:    shrl %cl, %ebx
-; FALLBACK16-NEXT:    addl %esi, %esi
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %esi
-; FALLBACK16-NEXT:    orl %ebx, %esi
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %eax
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK16-NEXT:    movl 124(%esp,%edx), %ebx
-; FALLBACK16-NEXT:    leal (%ebx,%ebx), %edx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %edx
-; FALLBACK16-NEXT:    orl %eax, %edx
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK16-NEXT:    shrl %cl, %ebx
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK16-NEXT:    movl %ebx, 60(%eax)
-; FALLBACK16-NEXT:    movl %edx, 56(%eax)
-; FALLBACK16-NEXT:    movl %esi, 48(%eax)
-; FALLBACK16-NEXT:    movl %ebp, 52(%eax)
-; FALLBACK16-NEXT:    movl %edi, 40(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, (%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK16-NEXT:    addl $204, %esp
-; FALLBACK16-NEXT:    popl %esi
-; FALLBACK16-NEXT:    popl %edi
-; FALLBACK16-NEXT:    popl %ebx
-; FALLBACK16-NEXT:    popl %ebp
-; FALLBACK16-NEXT:    retl
-;
-; FALLBACK17-LABEL: lshr_64bytes:
-; FALLBACK17:       # %bb.0:
-; FALLBACK17-NEXT:    pushl %ebp
-; FALLBACK17-NEXT:    pushl %ebx
-; FALLBACK17-NEXT:    pushl %edi
-; FALLBACK17-NEXT:    pushl %esi
-; FALLBACK17-NEXT:    subl $188, %esp
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK17-NEXT:    movl (%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 4(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 8(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 12(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 16(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 20(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 24(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 28(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 32(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 36(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 40(%ecx), %ebp
-; FALLBACK17-NEXT:    movl 44(%ecx), %ebx
-; FALLBACK17-NEXT:    movl 48(%ecx), %edi
-; FALLBACK17-NEXT:    movl 52(%ecx), %esi
-; FALLBACK17-NEXT:    movl 56(%ecx), %edx
-; FALLBACK17-NEXT:    movl 60(%ecx), %eax
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK17-NEXT:    movl (%ecx), %ecx
-; FALLBACK17-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK17-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %ecx, %ebp
-; FALLBACK17-NEXT:    andl $60, %ebp
-; FALLBACK17-NEXT:    movl 56(%esp,%ebp), %edx
-; FALLBACK17-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shll $3, %ecx
-; FALLBACK17-NEXT:    andl $24, %ecx
-; FALLBACK17-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 64(%esp,%ebp), %edi
-; FALLBACK17-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, %esi
-; FALLBACK17-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK17-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK17-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 72(%esp,%ebp), %esi
-; FALLBACK17-NEXT:    movl 68(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, %edx
-; FALLBACK17-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK17-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK17-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 80(%esp,%ebp), %edi
-; FALLBACK17-NEXT:    movl 76(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, %edx
-; FALLBACK17-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK17-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK17-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 88(%esp,%ebp), %esi
-; FALLBACK17-NEXT:    movl 84(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, %edx
-; FALLBACK17-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK17-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl %esi, %edx
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK17-NEXT:    movl %edi, (%esp) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 96(%esp,%ebp), %esi
-; FALLBACK17-NEXT:    movl 92(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, %edi
-; FALLBACK17-NEXT:    shrdl %cl, %esi, %edi
-; FALLBACK17-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK17-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 104(%esp,%ebp), %edx
-; FALLBACK17-NEXT:    movl 100(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, %edi
-; FALLBACK17-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK17-NEXT:    movl 48(%esp,%ebp), %ebx
-; FALLBACK17-NEXT:    movl 108(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK17-NEXT:    movl %edx, 56(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    shrdl %cl, %edx, %ebx
-; FALLBACK17-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK17-NEXT:    shrl %cl, %eax
-; FALLBACK17-NEXT:    movl %eax, 60(%ebp)
-; FALLBACK17-NEXT:    movl %esi, 48(%ebp)
-; FALLBACK17-NEXT:    movl %edi, 52(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 40(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK17-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK17-NEXT:    movl %ebx, (%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 4(%ebp)
-; FALLBACK17-NEXT:    addl $188, %esp
-; FALLBACK17-NEXT:    popl %esi
-; FALLBACK17-NEXT:    popl %edi
-; FALLBACK17-NEXT:    popl %ebx
-; FALLBACK17-NEXT:    popl %ebp
-; FALLBACK17-NEXT:    retl
-;
-; FALLBACK18-LABEL: lshr_64bytes:
-; FALLBACK18:       # %bb.0:
-; FALLBACK18-NEXT:    pushl %ebp
-; FALLBACK18-NEXT:    pushl %ebx
-; FALLBACK18-NEXT:    pushl %edi
-; FALLBACK18-NEXT:    pushl %esi
-; FALLBACK18-NEXT:    subl $204, %esp
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK18-NEXT:    movl (%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 4(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 8(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 12(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 16(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 20(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 24(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 28(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 32(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 36(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 40(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 44(%eax), %ebp
-; FALLBACK18-NEXT:    movl 48(%eax), %edi
-; FALLBACK18-NEXT:    movl 52(%eax), %esi
-; FALLBACK18-NEXT:    movl 56(%eax), %edx
-; FALLBACK18-NEXT:    movl 60(%eax), %ecx
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK18-NEXT:    movl (%eax), %ebx
-; FALLBACK18-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK18-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    leal (,%ebx,8), %edx
-; FALLBACK18-NEXT:    andl $24, %edx
-; FALLBACK18-NEXT:    movl %edx, %ecx
-; FALLBACK18-NEXT:    andl $60, %ebx
-; FALLBACK18-NEXT:    movl 68(%esp,%ebx), %esi
-; FALLBACK18-NEXT:    movl 72(%esp,%ebx), %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %ecx, %esi, %edi
-; FALLBACK18-NEXT:    notb %dl
-; FALLBACK18-NEXT:    leal (%eax,%eax), %ebp
-; FALLBACK18-NEXT:    shlxl %edx, %ebp, %eax
-; FALLBACK18-NEXT:    orl %edi, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %ecx, 64(%esp,%ebx), %edi
-; FALLBACK18-NEXT:    addl %esi, %esi
-; FALLBACK18-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK18-NEXT:    orl %edi, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 80(%esp,%ebx), %esi
-; FALLBACK18-NEXT:    leal (%esi,%esi), %edi
-; FALLBACK18-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK18-NEXT:    movl 76(%esp,%ebx), %edi
-; FALLBACK18-NEXT:    shrxl %ecx, %edi, %ebp
-; FALLBACK18-NEXT:    orl %ebp, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK18-NEXT:    addl %edi, %edi
-; FALLBACK18-NEXT:    shlxl %edx, %edi, %edi
-; FALLBACK18-NEXT:    orl %eax, %edi
-; FALLBACK18-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 88(%esp,%ebx), %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK18-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK18-NEXT:    movl 84(%esp,%ebx), %edi
-; FALLBACK18-NEXT:    shrxl %ecx, %edi, %ebp
-; FALLBACK18-NEXT:    orl %ebp, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %ecx, %esi, %esi
-; FALLBACK18-NEXT:    addl %edi, %edi
-; FALLBACK18-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK18-NEXT:    orl %esi, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 96(%esp,%ebx), %esi
-; FALLBACK18-NEXT:    leal (%esi,%esi), %edi
-; FALLBACK18-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK18-NEXT:    movl 92(%esp,%ebx), %edi
-; FALLBACK18-NEXT:    shrxl %ecx, %edi, %ebp
-; FALLBACK18-NEXT:    orl %ebp, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK18-NEXT:    addl %edi, %edi
-; FALLBACK18-NEXT:    shlxl %edx, %edi, %edi
-; FALLBACK18-NEXT:    orl %eax, %edi
-; FALLBACK18-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 104(%esp,%ebx), %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK18-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK18-NEXT:    movl 100(%esp,%ebx), %edi
-; FALLBACK18-NEXT:    shrxl %ecx, %edi, %ebp
-; FALLBACK18-NEXT:    orl %ebp, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %ecx, %esi, %esi
-; FALLBACK18-NEXT:    addl %edi, %edi
-; FALLBACK18-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK18-NEXT:    orl %esi, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 112(%esp,%ebx), %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    leal (%eax,%eax), %esi
-; FALLBACK18-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK18-NEXT:    movl 108(%esp,%ebx), %esi
-; FALLBACK18-NEXT:    shrxl %ecx, %esi, %edi
-; FALLBACK18-NEXT:    orl %edi, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK18-NEXT:    movl %ecx, %ebp
-; FALLBACK18-NEXT:    addl %esi, %esi
-; FALLBACK18-NEXT:    shlxl %edx, %esi, %ecx
-; FALLBACK18-NEXT:    orl %eax, %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 120(%esp,%ebx), %edi
-; FALLBACK18-NEXT:    leal (%edi,%edi), %ecx
-; FALLBACK18-NEXT:    shlxl %edx, %ecx, %esi
-; FALLBACK18-NEXT:    movl 116(%esp,%ebx), %eax
-; FALLBACK18-NEXT:    movl %ebp, %ecx
-; FALLBACK18-NEXT:    shrxl %ebp, %eax, %ebp
-; FALLBACK18-NEXT:    orl %ebp, %esi
-; FALLBACK18-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK18-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl %ecx, %ebp
-; FALLBACK18-NEXT:    addl %eax, %eax
-; FALLBACK18-NEXT:    shlxl %edx, %eax, %ecx
-; FALLBACK18-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; FALLBACK18-NEXT:    movl 124(%esp,%ebx), %eax
-; FALLBACK18-NEXT:    leal (%eax,%eax), %ebx
-; FALLBACK18-NEXT:    shlxl %edx, %ebx, %edx
-; FALLBACK18-NEXT:    shrxl %ebp, %edi, %edi
-; FALLBACK18-NEXT:    orl %edi, %edx
-; FALLBACK18-NEXT:    shrxl %ebp, %eax, %edi
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK18-NEXT:    movl %edi, 60(%eax)
-; FALLBACK18-NEXT:    movl %edx, 56(%eax)
-; FALLBACK18-NEXT:    movl %ecx, 48(%eax)
-; FALLBACK18-NEXT:    movl %esi, 52(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 40(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, (%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK18-NEXT:    addl $204, %esp
-; FALLBACK18-NEXT:    popl %esi
-; FALLBACK18-NEXT:    popl %edi
-; FALLBACK18-NEXT:    popl %ebx
-; FALLBACK18-NEXT:    popl %ebp
-; FALLBACK18-NEXT:    retl
-;
-; FALLBACK19-LABEL: lshr_64bytes:
-; FALLBACK19:       # %bb.0:
-; FALLBACK19-NEXT:    pushl %ebp
-; FALLBACK19-NEXT:    pushl %ebx
-; FALLBACK19-NEXT:    pushl %edi
-; FALLBACK19-NEXT:    pushl %esi
-; FALLBACK19-NEXT:    subl $188, %esp
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK19-NEXT:    movl (%ecx), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 4(%ecx), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 8(%ecx), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 12(%ecx), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 16(%ecx), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 20(%ecx), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 24(%ecx), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 28(%ecx), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 32(%ecx), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 36(%ecx), %eax
-; FALLBACK19-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 40(%ecx), %ebp
-; FALLBACK19-NEXT:    movl 44(%ecx), %ebx
-; FALLBACK19-NEXT:    movl 48(%ecx), %edi
-; FALLBACK19-NEXT:    movl 52(%ecx), %esi
-; FALLBACK19-NEXT:    movl 56(%ecx), %edx
-; FALLBACK19-NEXT:    movl 60(%ecx), %eax
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK19-NEXT:    movl (%ecx), %ecx
-; FALLBACK19-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK19-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %ecx, %ebp
-; FALLBACK19-NEXT:    andl $60, %ebp
-; FALLBACK19-NEXT:    movl 56(%esp,%ebp), %edx
-; FALLBACK19-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shll $3, %ecx
-; FALLBACK19-NEXT:    andl $24, %ecx
-; FALLBACK19-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 64(%esp,%ebp), %edi
-; FALLBACK19-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, %esi
-; FALLBACK19-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK19-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK19-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 72(%esp,%ebp), %esi
-; FALLBACK19-NEXT:    movl 68(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, %edx
-; FALLBACK19-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK19-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK19-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 80(%esp,%ebp), %edi
-; FALLBACK19-NEXT:    movl 76(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, %edx
-; FALLBACK19-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK19-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK19-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 88(%esp,%ebp), %ebx
-; FALLBACK19-NEXT:    movl 84(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, %edx
-; FALLBACK19-NEXT:    shrdl %cl, %ebx, %edx
-; FALLBACK19-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK19-NEXT:    movl %edi, (%esp) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 96(%esp,%ebp), %esi
-; FALLBACK19-NEXT:    movl 92(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, %edx
-; FALLBACK19-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK19-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK19-NEXT:    movl 104(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl 100(%esp,%ebp), %edi
-; FALLBACK19-NEXT:    movl %edi, %edx
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK19-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK19-NEXT:    movl 48(%esp,%ebp), %edi
-; FALLBACK19-NEXT:    movl 108(%esp,%ebp), %ebp
-; FALLBACK19-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shrdl %cl, %ebp, %eax
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK19-NEXT:    movl %eax, 56(%ebp)
-; FALLBACK19-NEXT:    movl %esi, 48(%ebp)
-; FALLBACK19-NEXT:    movl %edx, 52(%ebp)
-; FALLBACK19-NEXT:    movl %ebx, 40(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK19-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK19-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK19-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK19-NEXT:    movl %edi, (%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %ecx, 4(%ebp)
-; FALLBACK19-NEXT:    movl %eax, 60(%ebp)
-; FALLBACK19-NEXT:    addl $188, %esp
-; FALLBACK19-NEXT:    popl %esi
-; FALLBACK19-NEXT:    popl %edi
-; FALLBACK19-NEXT:    popl %ebx
-; FALLBACK19-NEXT:    popl %ebp
-; FALLBACK19-NEXT:    retl
-;
-; FALLBACK20-LABEL: lshr_64bytes:
-; FALLBACK20:       # %bb.0:
-; FALLBACK20-NEXT:    pushl %ebp
-; FALLBACK20-NEXT:    pushl %ebx
-; FALLBACK20-NEXT:    pushl %edi
-; FALLBACK20-NEXT:    pushl %esi
-; FALLBACK20-NEXT:    subl $204, %esp
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK20-NEXT:    movups (%ecx), %xmm0
-; FALLBACK20-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK20-NEXT:    movups 32(%ecx), %xmm2
-; FALLBACK20-NEXT:    movups 48(%ecx), %xmm3
-; FALLBACK20-NEXT:    movl (%eax), %eax
-; FALLBACK20-NEXT:    xorps %xmm4, %xmm4
-; FALLBACK20-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm3, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %eax, %esi
-; FALLBACK20-NEXT:    andl $60, %esi
-; FALLBACK20-NEXT:    movl 68(%esp,%esi), %edx
-; FALLBACK20-NEXT:    shll $3, %eax
-; FALLBACK20-NEXT:    andl $24, %eax
-; FALLBACK20-NEXT:    movl %edx, %edi
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %edi
-; FALLBACK20-NEXT:    movl 72(%esp,%esi), %ecx
-; FALLBACK20-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK20-NEXT:    movb %al, %ch
-; FALLBACK20-NEXT:    notb %ch
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    orl %edi, %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 64(%esp,%esi), %edi
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edi
-; FALLBACK20-NEXT:    addl %edx, %edx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %edx
-; FALLBACK20-NEXT:    orl %edi, %edx
-; FALLBACK20-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 76(%esp,%esi), %edx
-; FALLBACK20-NEXT:    movl %edx, %ebp
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shrl %cl, %ebp
-; FALLBACK20-NEXT:    movl 80(%esp,%esi), %edi
-; FALLBACK20-NEXT:    leal (%edi,%edi), %ebx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    orl %ebp, %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %cl, %ebx
-; FALLBACK20-NEXT:    addl %edx, %edx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %edx
-; FALLBACK20-NEXT:    orl %ebx, %edx
-; FALLBACK20-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 84(%esp,%esi), %ebx
-; FALLBACK20-NEXT:    movl %ebx, %ebp
-; FALLBACK20-NEXT:    movl %eax, %edx
-; FALLBACK20-NEXT:    movb %dl, %cl
-; FALLBACK20-NEXT:    shrl %cl, %ebp
-; FALLBACK20-NEXT:    movl 88(%esp,%esi), %eax
-; FALLBACK20-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    addl %eax, %eax
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %eax
-; FALLBACK20-NEXT:    orl %ebp, %eax
-; FALLBACK20-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %dl, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edi
-; FALLBACK20-NEXT:    addl %ebx, %ebx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    orl %edi, %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 92(%esp,%esi), %ebx
-; FALLBACK20-NEXT:    movl %ebx, %ebp
-; FALLBACK20-NEXT:    movb %dl, %cl
-; FALLBACK20-NEXT:    shrl %cl, %ebp
-; FALLBACK20-NEXT:    movl 96(%esp,%esi), %edi
-; FALLBACK20-NEXT:    leal (%edi,%edi), %eax
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %eax
-; FALLBACK20-NEXT:    orl %ebp, %eax
-; FALLBACK20-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %dl, %cl
-; FALLBACK20-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %cl, %eax
-; FALLBACK20-NEXT:    addl %ebx, %ebx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    orl %eax, %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 100(%esp,%esi), %ebx
-; FALLBACK20-NEXT:    movl %ebx, %ebp
-; FALLBACK20-NEXT:    movb %dl, %cl
-; FALLBACK20-NEXT:    shrl %cl, %ebp
-; FALLBACK20-NEXT:    movl 104(%esp,%esi), %edx
-; FALLBACK20-NEXT:    leal (%edx,%edx), %eax
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %eax
-; FALLBACK20-NEXT:    orl %ebp, %eax
-; FALLBACK20-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edi
-; FALLBACK20-NEXT:    addl %ebx, %ebx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    orl %edi, %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 108(%esp,%esi), %edi
-; FALLBACK20-NEXT:    movl %edi, %ebp
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %ebp
-; FALLBACK20-NEXT:    movl 112(%esp,%esi), %ecx
-; FALLBACK20-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK20-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    orl %ebp, %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edx
-; FALLBACK20-NEXT:    addl %edi, %edi
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %edi
-; FALLBACK20-NEXT:    orl %edx, %edi
-; FALLBACK20-NEXT:    movl %esi, %edx
-; FALLBACK20-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 116(%esp,%esi), %esi
-; FALLBACK20-NEXT:    movl %esi, %ebx
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shrl %cl, %ebx
-; FALLBACK20-NEXT:    movl 120(%esp,%edx), %eax
-; FALLBACK20-NEXT:    leal (%eax,%eax), %ebp
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebp
-; FALLBACK20-NEXT:    orl %ebx, %ebp
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK20-NEXT:    movb %dl, %cl
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %cl, %ebx
-; FALLBACK20-NEXT:    addl %esi, %esi
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %esi
-; FALLBACK20-NEXT:    orl %ebx, %esi
-; FALLBACK20-NEXT:    movb %dl, %cl
-; FALLBACK20-NEXT:    shrl %cl, %eax
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK20-NEXT:    movl 124(%esp,%edx), %ebx
-; FALLBACK20-NEXT:    leal (%ebx,%ebx), %edx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %edx
-; FALLBACK20-NEXT:    orl %eax, %edx
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK20-NEXT:    shrl %cl, %ebx
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK20-NEXT:    movl %ebx, 60(%eax)
-; FALLBACK20-NEXT:    movl %edx, 56(%eax)
-; FALLBACK20-NEXT:    movl %esi, 48(%eax)
-; FALLBACK20-NEXT:    movl %ebp, 52(%eax)
-; FALLBACK20-NEXT:    movl %edi, 40(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, (%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK20-NEXT:    addl $204, %esp
-; FALLBACK20-NEXT:    popl %esi
-; FALLBACK20-NEXT:    popl %edi
-; FALLBACK20-NEXT:    popl %ebx
-; FALLBACK20-NEXT:    popl %ebp
-; FALLBACK20-NEXT:    retl
-;
-; FALLBACK21-LABEL: lshr_64bytes:
-; FALLBACK21:       # %bb.0:
-; FALLBACK21-NEXT:    pushl %ebp
-; FALLBACK21-NEXT:    pushl %ebx
-; FALLBACK21-NEXT:    pushl %edi
-; FALLBACK21-NEXT:    pushl %esi
-; FALLBACK21-NEXT:    subl $188, %esp
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK21-NEXT:    movups (%ecx), %xmm0
-; FALLBACK21-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK21-NEXT:    movups 32(%ecx), %xmm2
-; FALLBACK21-NEXT:    movups 48(%ecx), %xmm3
-; FALLBACK21-NEXT:    movl (%eax), %ecx
-; FALLBACK21-NEXT:    xorps %xmm4, %xmm4
-; FALLBACK21-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm3, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %ecx, %ebp
-; FALLBACK21-NEXT:    andl $60, %ebp
-; FALLBACK21-NEXT:    movl 56(%esp,%ebp), %edx
-; FALLBACK21-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shll $3, %ecx
-; FALLBACK21-NEXT:    andl $24, %ecx
-; FALLBACK21-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK21-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 64(%esp,%ebp), %edi
-; FALLBACK21-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, %esi
-; FALLBACK21-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK21-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 72(%esp,%ebp), %esi
-; FALLBACK21-NEXT:    movl 68(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, %edx
-; FALLBACK21-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK21-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 80(%esp,%ebp), %edi
-; FALLBACK21-NEXT:    movl 76(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, %edx
-; FALLBACK21-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK21-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 88(%esp,%ebp), %esi
-; FALLBACK21-NEXT:    movl 84(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, %edx
-; FALLBACK21-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl %esi, %edx
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK21-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 96(%esp,%ebp), %esi
-; FALLBACK21-NEXT:    movl 92(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, %edi
-; FALLBACK21-NEXT:    shrdl %cl, %esi, %edi
-; FALLBACK21-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK21-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 104(%esp,%ebp), %edx
-; FALLBACK21-NEXT:    movl 100(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, %edi
-; FALLBACK21-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK21-NEXT:    movl 48(%esp,%ebp), %ebx
-; FALLBACK21-NEXT:    movl 108(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK21-NEXT:    movl %edx, 56(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK21-NEXT:    shrdl %cl, %edx, %ebx
-; FALLBACK21-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK21-NEXT:    shrl %cl, %eax
-; FALLBACK21-NEXT:    movl %eax, 60(%ebp)
-; FALLBACK21-NEXT:    movl %esi, 48(%ebp)
-; FALLBACK21-NEXT:    movl %edi, 52(%ebp)
-; FALLBACK21-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 40(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK21-NEXT:    movl %ebx, (%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 4(%ebp)
-; FALLBACK21-NEXT:    addl $188, %esp
-; FALLBACK21-NEXT:    popl %esi
-; FALLBACK21-NEXT:    popl %edi
-; FALLBACK21-NEXT:    popl %ebx
-; FALLBACK21-NEXT:    popl %ebp
-; FALLBACK21-NEXT:    retl
-;
-; FALLBACK22-LABEL: lshr_64bytes:
-; FALLBACK22:       # %bb.0:
-; FALLBACK22-NEXT:    pushl %ebp
-; FALLBACK22-NEXT:    pushl %ebx
-; FALLBACK22-NEXT:    pushl %edi
-; FALLBACK22-NEXT:    pushl %esi
-; FALLBACK22-NEXT:    subl $204, %esp
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK22-NEXT:    movups (%ecx), %xmm0
-; FALLBACK22-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK22-NEXT:    movups 32(%ecx), %xmm2
-; FALLBACK22-NEXT:    movups 48(%ecx), %xmm3
-; FALLBACK22-NEXT:    movl (%eax), %ebx
-; FALLBACK22-NEXT:    xorps %xmm4, %xmm4
-; FALLBACK22-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm3, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    leal (,%ebx,8), %edx
-; FALLBACK22-NEXT:    andl $24, %edx
-; FALLBACK22-NEXT:    movl %edx, %ecx
-; FALLBACK22-NEXT:    andl $60, %ebx
-; FALLBACK22-NEXT:    movl 68(%esp,%ebx), %esi
-; FALLBACK22-NEXT:    movl 72(%esp,%ebx), %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrxl %ecx, %esi, %edi
-; FALLBACK22-NEXT:    notb %dl
-; FALLBACK22-NEXT:    leal (%eax,%eax), %ebp
-; FALLBACK22-NEXT:    shlxl %edx, %ebp, %eax
-; FALLBACK22-NEXT:    orl %edi, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrxl %ecx, 64(%esp,%ebx), %edi
-; FALLBACK22-NEXT:    addl %esi, %esi
-; FALLBACK22-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK22-NEXT:    orl %edi, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 80(%esp,%ebx), %esi
-; FALLBACK22-NEXT:    leal (%esi,%esi), %edi
-; FALLBACK22-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK22-NEXT:    movl 76(%esp,%ebx), %edi
-; FALLBACK22-NEXT:    shrxl %ecx, %edi, %ebp
-; FALLBACK22-NEXT:    orl %ebp, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK22-NEXT:    addl %edi, %edi
-; FALLBACK22-NEXT:    shlxl %edx, %edi, %edi
-; FALLBACK22-NEXT:    orl %eax, %edi
-; FALLBACK22-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 88(%esp,%ebx), %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK22-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK22-NEXT:    movl 84(%esp,%ebx), %edi
-; FALLBACK22-NEXT:    shrxl %ecx, %edi, %ebp
-; FALLBACK22-NEXT:    orl %ebp, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrxl %ecx, %esi, %esi
-; FALLBACK22-NEXT:    addl %edi, %edi
-; FALLBACK22-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK22-NEXT:    orl %esi, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 96(%esp,%ebx), %esi
-; FALLBACK22-NEXT:    leal (%esi,%esi), %edi
-; FALLBACK22-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK22-NEXT:    movl 92(%esp,%ebx), %edi
-; FALLBACK22-NEXT:    shrxl %ecx, %edi, %ebp
-; FALLBACK22-NEXT:    orl %ebp, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK22-NEXT:    addl %edi, %edi
-; FALLBACK22-NEXT:    shlxl %edx, %edi, %edi
-; FALLBACK22-NEXT:    orl %eax, %edi
-; FALLBACK22-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 104(%esp,%ebx), %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK22-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK22-NEXT:    movl 100(%esp,%ebx), %edi
-; FALLBACK22-NEXT:    shrxl %ecx, %edi, %ebp
-; FALLBACK22-NEXT:    orl %ebp, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrxl %ecx, %esi, %esi
-; FALLBACK22-NEXT:    addl %edi, %edi
-; FALLBACK22-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK22-NEXT:    orl %esi, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 112(%esp,%ebx), %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    leal (%eax,%eax), %esi
-; FALLBACK22-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK22-NEXT:    movl 108(%esp,%ebx), %esi
-; FALLBACK22-NEXT:    shrxl %ecx, %esi, %edi
-; FALLBACK22-NEXT:    orl %edi, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK22-NEXT:    movl %ecx, %ebp
-; FALLBACK22-NEXT:    addl %esi, %esi
-; FALLBACK22-NEXT:    shlxl %edx, %esi, %ecx
-; FALLBACK22-NEXT:    orl %eax, %ecx
-; FALLBACK22-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 120(%esp,%ebx), %edi
-; FALLBACK22-NEXT:    leal (%edi,%edi), %ecx
-; FALLBACK22-NEXT:    shlxl %edx, %ecx, %esi
-; FALLBACK22-NEXT:    movl 116(%esp,%ebx), %eax
-; FALLBACK22-NEXT:    movl %ebp, %ecx
-; FALLBACK22-NEXT:    shrxl %ebp, %eax, %ebp
-; FALLBACK22-NEXT:    orl %ebp, %esi
-; FALLBACK22-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK22-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl %ecx, %ebp
-; FALLBACK22-NEXT:    addl %eax, %eax
-; FALLBACK22-NEXT:    shlxl %edx, %eax, %ecx
-; FALLBACK22-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; FALLBACK22-NEXT:    movl 124(%esp,%ebx), %eax
-; FALLBACK22-NEXT:    leal (%eax,%eax), %ebx
-; FALLBACK22-NEXT:    shlxl %edx, %ebx, %edx
-; FALLBACK22-NEXT:    shrxl %ebp, %edi, %edi
-; FALLBACK22-NEXT:    orl %edi, %edx
-; FALLBACK22-NEXT:    shrxl %ebp, %eax, %edi
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK22-NEXT:    movl %edi, 60(%eax)
-; FALLBACK22-NEXT:    movl %edx, 56(%eax)
-; FALLBACK22-NEXT:    movl %ecx, 48(%eax)
-; FALLBACK22-NEXT:    movl %esi, 52(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 40(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, (%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK22-NEXT:    addl $204, %esp
-; FALLBACK22-NEXT:    popl %esi
-; FALLBACK22-NEXT:    popl %edi
-; FALLBACK22-NEXT:    popl %ebx
-; FALLBACK22-NEXT:    popl %ebp
-; FALLBACK22-NEXT:    retl
-;
-; FALLBACK23-LABEL: lshr_64bytes:
-; FALLBACK23:       # %bb.0:
-; FALLBACK23-NEXT:    pushl %ebp
-; FALLBACK23-NEXT:    pushl %ebx
-; FALLBACK23-NEXT:    pushl %edi
-; FALLBACK23-NEXT:    pushl %esi
-; FALLBACK23-NEXT:    subl $188, %esp
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK23-NEXT:    movups (%ecx), %xmm0
-; FALLBACK23-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK23-NEXT:    movups 32(%ecx), %xmm2
-; FALLBACK23-NEXT:    movups 48(%ecx), %xmm3
-; FALLBACK23-NEXT:    movl (%eax), %ecx
-; FALLBACK23-NEXT:    xorps %xmm4, %xmm4
-; FALLBACK23-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm3, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %ecx, %ebp
-; FALLBACK23-NEXT:    andl $60, %ebp
-; FALLBACK23-NEXT:    movl 56(%esp,%ebp), %edx
-; FALLBACK23-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK23-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shll $3, %ecx
-; FALLBACK23-NEXT:    andl $24, %ecx
-; FALLBACK23-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK23-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 64(%esp,%ebp), %edi
-; FALLBACK23-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK23-NEXT:    movl %eax, %esi
-; FALLBACK23-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK23-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 72(%esp,%ebp), %esi
-; FALLBACK23-NEXT:    movl 68(%esp,%ebp), %eax
-; FALLBACK23-NEXT:    movl %eax, %edx
-; FALLBACK23-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK23-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 80(%esp,%ebp), %edi
-; FALLBACK23-NEXT:    movl 76(%esp,%ebp), %eax
-; FALLBACK23-NEXT:    movl %eax, %edx
-; FALLBACK23-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK23-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 88(%esp,%ebp), %ebx
-; FALLBACK23-NEXT:    movl 84(%esp,%ebp), %eax
-; FALLBACK23-NEXT:    movl %eax, %edx
-; FALLBACK23-NEXT:    shrdl %cl, %ebx, %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK23-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 96(%esp,%ebp), %esi
-; FALLBACK23-NEXT:    movl 92(%esp,%ebp), %eax
-; FALLBACK23-NEXT:    movl %eax, %edx
-; FALLBACK23-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK23-NEXT:    movl 104(%esp,%ebp), %eax
-; FALLBACK23-NEXT:    movl 100(%esp,%ebp), %edi
-; FALLBACK23-NEXT:    movl %edi, %edx
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK23-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK23-NEXT:    movl 48(%esp,%ebp), %edi
-; FALLBACK23-NEXT:    movl 108(%esp,%ebp), %ebp
-; FALLBACK23-NEXT:    movl %ebp, (%esp) # 4-byte Spill
-; FALLBACK23-NEXT:    shrdl %cl, %ebp, %eax
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK23-NEXT:    movl %eax, 56(%ebp)
-; FALLBACK23-NEXT:    movl %esi, 48(%ebp)
-; FALLBACK23-NEXT:    movl %edx, 52(%ebp)
-; FALLBACK23-NEXT:    movl %ebx, 40(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK23-NEXT:    shrxl %ecx, (%esp), %eax # 4-byte Folded Reload
-; FALLBACK23-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK23-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK23-NEXT:    movl %edi, (%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK23-NEXT:    movl %ecx, 4(%ebp)
-; FALLBACK23-NEXT:    movl %eax, 60(%ebp)
-; FALLBACK23-NEXT:    addl $188, %esp
-; FALLBACK23-NEXT:    popl %esi
-; FALLBACK23-NEXT:    popl %edi
-; FALLBACK23-NEXT:    popl %ebx
-; FALLBACK23-NEXT:    popl %ebp
-; FALLBACK23-NEXT:    retl
-;
-; FALLBACK24-LABEL: lshr_64bytes:
-; FALLBACK24:       # %bb.0:
-; FALLBACK24-NEXT:    pushl %ebp
-; FALLBACK24-NEXT:    pushl %ebx
-; FALLBACK24-NEXT:    pushl %edi
-; FALLBACK24-NEXT:    pushl %esi
-; FALLBACK24-NEXT:    subl $204, %esp
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK24-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK24-NEXT:    vmovups 32(%ecx), %ymm1
-; FALLBACK24-NEXT:    movl (%eax), %ecx
-; FALLBACK24-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; FALLBACK24-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %ecx, %esi
-; FALLBACK24-NEXT:    andl $60, %esi
-; FALLBACK24-NEXT:    movl 68(%esp,%esi), %edx
-; FALLBACK24-NEXT:    shll $3, %ecx
-; FALLBACK24-NEXT:    andl $24, %ecx
-; FALLBACK24-NEXT:    movl %edx, %edi
-; FALLBACK24-NEXT:    shrl %cl, %edi
-; FALLBACK24-NEXT:    movl 72(%esp,%esi), %eax
-; FALLBACK24-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    leal (%eax,%eax), %ebx
-; FALLBACK24-NEXT:    movl %ecx, %ebp
-; FALLBACK24-NEXT:    movb %cl, %ch
-; FALLBACK24-NEXT:    notb %ch
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    orl %edi, %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 64(%esp,%esi), %edi
-; FALLBACK24-NEXT:    movl %ebp, %eax
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edi
-; FALLBACK24-NEXT:    addl %edx, %edx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %edx
-; FALLBACK24-NEXT:    orl %edi, %edx
-; FALLBACK24-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 76(%esp,%esi), %edx
-; FALLBACK24-NEXT:    movl %edx, %ebp
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shrl %cl, %ebp
-; FALLBACK24-NEXT:    movl 80(%esp,%esi), %edi
-; FALLBACK24-NEXT:    leal (%edi,%edi), %ebx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    orl %ebp, %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %cl, %ebx
-; FALLBACK24-NEXT:    addl %edx, %edx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %edx
-; FALLBACK24-NEXT:    orl %ebx, %edx
-; FALLBACK24-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 84(%esp,%esi), %ebx
-; FALLBACK24-NEXT:    movl %ebx, %ebp
-; FALLBACK24-NEXT:    movl %eax, %edx
-; FALLBACK24-NEXT:    movb %dl, %cl
-; FALLBACK24-NEXT:    shrl %cl, %ebp
-; FALLBACK24-NEXT:    movl 88(%esp,%esi), %eax
-; FALLBACK24-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    addl %eax, %eax
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %eax
-; FALLBACK24-NEXT:    orl %ebp, %eax
-; FALLBACK24-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %dl, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edi
-; FALLBACK24-NEXT:    addl %ebx, %ebx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    orl %edi, %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 92(%esp,%esi), %ebx
-; FALLBACK24-NEXT:    movl %ebx, %ebp
-; FALLBACK24-NEXT:    movb %dl, %cl
-; FALLBACK24-NEXT:    shrl %cl, %ebp
-; FALLBACK24-NEXT:    movl 96(%esp,%esi), %edi
-; FALLBACK24-NEXT:    leal (%edi,%edi), %eax
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %eax
-; FALLBACK24-NEXT:    orl %ebp, %eax
-; FALLBACK24-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %dl, %cl
-; FALLBACK24-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %cl, %eax
-; FALLBACK24-NEXT:    addl %ebx, %ebx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    orl %eax, %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 100(%esp,%esi), %ebx
-; FALLBACK24-NEXT:    movl %ebx, %ebp
-; FALLBACK24-NEXT:    movb %dl, %cl
-; FALLBACK24-NEXT:    shrl %cl, %ebp
-; FALLBACK24-NEXT:    movl 104(%esp,%esi), %edx
-; FALLBACK24-NEXT:    leal (%edx,%edx), %eax
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %eax
-; FALLBACK24-NEXT:    orl %ebp, %eax
-; FALLBACK24-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edi
-; FALLBACK24-NEXT:    addl %ebx, %ebx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    orl %edi, %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 108(%esp,%esi), %edi
-; FALLBACK24-NEXT:    movl %edi, %ebp
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %ebp
-; FALLBACK24-NEXT:    movl 112(%esp,%esi), %ecx
-; FALLBACK24-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK24-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    orl %ebp, %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edx
-; FALLBACK24-NEXT:    addl %edi, %edi
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %edi
-; FALLBACK24-NEXT:    orl %edx, %edi
-; FALLBACK24-NEXT:    movl %esi, %edx
-; FALLBACK24-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 116(%esp,%esi), %esi
-; FALLBACK24-NEXT:    movl %esi, %ebx
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shrl %cl, %ebx
-; FALLBACK24-NEXT:    movl 120(%esp,%edx), %eax
-; FALLBACK24-NEXT:    leal (%eax,%eax), %ebp
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebp
-; FALLBACK24-NEXT:    orl %ebx, %ebp
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK24-NEXT:    movb %dl, %cl
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %cl, %ebx
-; FALLBACK24-NEXT:    addl %esi, %esi
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %esi
-; FALLBACK24-NEXT:    orl %ebx, %esi
-; FALLBACK24-NEXT:    movb %dl, %cl
-; FALLBACK24-NEXT:    shrl %cl, %eax
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK24-NEXT:    movl 124(%esp,%edx), %ebx
-; FALLBACK24-NEXT:    leal (%ebx,%ebx), %edx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %edx
-; FALLBACK24-NEXT:    orl %eax, %edx
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK24-NEXT:    shrl %cl, %ebx
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK24-NEXT:    movl %ebx, 60(%eax)
-; FALLBACK24-NEXT:    movl %edx, 56(%eax)
-; FALLBACK24-NEXT:    movl %esi, 48(%eax)
-; FALLBACK24-NEXT:    movl %ebp, 52(%eax)
-; FALLBACK24-NEXT:    movl %edi, 40(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, (%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK24-NEXT:    addl $204, %esp
-; FALLBACK24-NEXT:    popl %esi
-; FALLBACK24-NEXT:    popl %edi
-; FALLBACK24-NEXT:    popl %ebx
-; FALLBACK24-NEXT:    popl %ebp
-; FALLBACK24-NEXT:    vzeroupper
-; FALLBACK24-NEXT:    retl
-;
-; FALLBACK25-LABEL: lshr_64bytes:
-; FALLBACK25:       # %bb.0:
-; FALLBACK25-NEXT:    pushl %ebp
-; FALLBACK25-NEXT:    pushl %ebx
-; FALLBACK25-NEXT:    pushl %edi
-; FALLBACK25-NEXT:    pushl %esi
-; FALLBACK25-NEXT:    subl $188, %esp
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK25-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK25-NEXT:    vmovups 32(%ecx), %ymm1
-; FALLBACK25-NEXT:    movl (%eax), %ecx
-; FALLBACK25-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; FALLBACK25-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %ecx, %ebp
-; FALLBACK25-NEXT:    andl $60, %ebp
-; FALLBACK25-NEXT:    movl 56(%esp,%ebp), %edx
-; FALLBACK25-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shll $3, %ecx
-; FALLBACK25-NEXT:    andl $24, %ecx
-; FALLBACK25-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK25-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 64(%esp,%ebp), %edi
-; FALLBACK25-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, %esi
-; FALLBACK25-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK25-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 72(%esp,%ebp), %esi
-; FALLBACK25-NEXT:    movl 68(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, %edx
-; FALLBACK25-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK25-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 80(%esp,%ebp), %edi
-; FALLBACK25-NEXT:    movl 76(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, %edx
-; FALLBACK25-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK25-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 88(%esp,%ebp), %esi
-; FALLBACK25-NEXT:    movl 84(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, %edx
-; FALLBACK25-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl %esi, %edx
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK25-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 96(%esp,%ebp), %esi
-; FALLBACK25-NEXT:    movl 92(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, %edi
-; FALLBACK25-NEXT:    shrdl %cl, %esi, %edi
-; FALLBACK25-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK25-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 104(%esp,%ebp), %edx
-; FALLBACK25-NEXT:    movl 100(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, %edi
-; FALLBACK25-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK25-NEXT:    movl 48(%esp,%ebp), %ebx
-; FALLBACK25-NEXT:    movl 108(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK25-NEXT:    movl %edx, 56(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK25-NEXT:    shrdl %cl, %edx, %ebx
-; FALLBACK25-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK25-NEXT:    shrl %cl, %eax
-; FALLBACK25-NEXT:    movl %eax, 60(%ebp)
-; FALLBACK25-NEXT:    movl %esi, 48(%ebp)
-; FALLBACK25-NEXT:    movl %edi, 52(%ebp)
-; FALLBACK25-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 40(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK25-NEXT:    movl %ebx, (%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 4(%ebp)
-; FALLBACK25-NEXT:    addl $188, %esp
-; FALLBACK25-NEXT:    popl %esi
-; FALLBACK25-NEXT:    popl %edi
-; FALLBACK25-NEXT:    popl %ebx
-; FALLBACK25-NEXT:    popl %ebp
-; FALLBACK25-NEXT:    vzeroupper
-; FALLBACK25-NEXT:    retl
-;
-; FALLBACK26-LABEL: lshr_64bytes:
-; FALLBACK26:       # %bb.0:
-; FALLBACK26-NEXT:    pushl %ebp
-; FALLBACK26-NEXT:    pushl %ebx
-; FALLBACK26-NEXT:    pushl %edi
-; FALLBACK26-NEXT:    pushl %esi
-; FALLBACK26-NEXT:    subl $204, %esp
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK26-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK26-NEXT:    vmovups 32(%ecx), %ymm1
-; FALLBACK26-NEXT:    movl (%eax), %ecx
-; FALLBACK26-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; FALLBACK26-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    leal (,%ecx,8), %edx
-; FALLBACK26-NEXT:    andl $24, %edx
-; FALLBACK26-NEXT:    movl %edx, %ebx
-; FALLBACK26-NEXT:    andl $60, %ecx
-; FALLBACK26-NEXT:    movl 68(%esp,%ecx), %esi
-; FALLBACK26-NEXT:    movl 72(%esp,%ecx), %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrxl %ebx, %esi, %edi
-; FALLBACK26-NEXT:    notb %dl
-; FALLBACK26-NEXT:    leal (%eax,%eax), %ebp
-; FALLBACK26-NEXT:    shlxl %edx, %ebp, %ebp
-; FALLBACK26-NEXT:    orl %edi, %ebp
-; FALLBACK26-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrxl %ebx, 64(%esp,%ecx), %edi
-; FALLBACK26-NEXT:    addl %esi, %esi
-; FALLBACK26-NEXT:    shlxl %edx, %esi, %esi
-; FALLBACK26-NEXT:    orl %edi, %esi
-; FALLBACK26-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 80(%esp,%ecx), %esi
-; FALLBACK26-NEXT:    leal (%esi,%esi), %edi
-; FALLBACK26-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK26-NEXT:    movl 76(%esp,%ecx), %edi
-; FALLBACK26-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK26-NEXT:    orl %ebp, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK26-NEXT:    addl %edi, %edi
-; FALLBACK26-NEXT:    shlxl %edx, %edi, %edi
-; FALLBACK26-NEXT:    orl %eax, %edi
-; FALLBACK26-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 88(%esp,%ecx), %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK26-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK26-NEXT:    movl 84(%esp,%ecx), %edi
-; FALLBACK26-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK26-NEXT:    orl %ebp, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrxl %ebx, %esi, %esi
-; FALLBACK26-NEXT:    addl %edi, %edi
-; FALLBACK26-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK26-NEXT:    orl %esi, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 96(%esp,%ecx), %esi
-; FALLBACK26-NEXT:    leal (%esi,%esi), %edi
-; FALLBACK26-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK26-NEXT:    movl 92(%esp,%ecx), %edi
-; FALLBACK26-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK26-NEXT:    orl %ebp, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK26-NEXT:    addl %edi, %edi
-; FALLBACK26-NEXT:    shlxl %edx, %edi, %edi
-; FALLBACK26-NEXT:    orl %eax, %edi
-; FALLBACK26-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 104(%esp,%ecx), %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK26-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK26-NEXT:    movl 100(%esp,%ecx), %edi
-; FALLBACK26-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK26-NEXT:    orl %ebp, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrxl %ebx, %esi, %esi
-; FALLBACK26-NEXT:    addl %edi, %edi
-; FALLBACK26-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK26-NEXT:    orl %esi, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 112(%esp,%ecx), %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    leal (%eax,%eax), %esi
-; FALLBACK26-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK26-NEXT:    movl 108(%esp,%ecx), %esi
-; FALLBACK26-NEXT:    shrxl %ebx, %esi, %edi
-; FALLBACK26-NEXT:    orl %edi, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK26-NEXT:    addl %esi, %esi
-; FALLBACK26-NEXT:    shlxl %edx, %esi, %ebp
-; FALLBACK26-NEXT:    orl %eax, %ebp
-; FALLBACK26-NEXT:    movl 120(%esp,%ecx), %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    addl %eax, %eax
-; FALLBACK26-NEXT:    shlxl %edx, %eax, %esi
-; FALLBACK26-NEXT:    movl 116(%esp,%ecx), %eax
-; FALLBACK26-NEXT:    shrxl %ebx, %eax, %edi
-; FALLBACK26-NEXT:    orl %edi, %esi
-; FALLBACK26-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK26-NEXT:    addl %eax, %eax
-; FALLBACK26-NEXT:    shlxl %edx, %eax, %eax
-; FALLBACK26-NEXT:    orl %edi, %eax
-; FALLBACK26-NEXT:    movl 124(%esp,%ecx), %ecx
-; FALLBACK26-NEXT:    leal (%ecx,%ecx), %edi
-; FALLBACK26-NEXT:    shlxl %edx, %edi, %edx
-; FALLBACK26-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK26-NEXT:    orl %edi, %edx
-; FALLBACK26-NEXT:    shrxl %ebx, %ecx, %edi
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK26-NEXT:    movl %edi, 60(%ecx)
-; FALLBACK26-NEXT:    movl %edx, 56(%ecx)
-; FALLBACK26-NEXT:    movl %eax, 48(%ecx)
-; FALLBACK26-NEXT:    movl %esi, 52(%ecx)
-; FALLBACK26-NEXT:    movl %ebp, 40(%ecx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 44(%ecx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 32(%ecx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 36(%ecx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 24(%ecx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 28(%ecx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 16(%ecx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 20(%ecx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 8(%ecx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 12(%ecx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, (%ecx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 4(%ecx)
-; FALLBACK26-NEXT:    addl $204, %esp
-; FALLBACK26-NEXT:    popl %esi
-; FALLBACK26-NEXT:    popl %edi
-; FALLBACK26-NEXT:    popl %ebx
-; FALLBACK26-NEXT:    popl %ebp
-; FALLBACK26-NEXT:    vzeroupper
-; FALLBACK26-NEXT:    retl
-;
-; FALLBACK27-LABEL: lshr_64bytes:
-; FALLBACK27:       # %bb.0:
-; FALLBACK27-NEXT:    pushl %ebp
-; FALLBACK27-NEXT:    pushl %ebx
-; FALLBACK27-NEXT:    pushl %edi
-; FALLBACK27-NEXT:    pushl %esi
-; FALLBACK27-NEXT:    subl $188, %esp
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK27-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK27-NEXT:    vmovups 32(%ecx), %ymm1
-; FALLBACK27-NEXT:    movl (%eax), %ecx
-; FALLBACK27-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; FALLBACK27-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %ecx, %ebp
-; FALLBACK27-NEXT:    andl $60, %ebp
-; FALLBACK27-NEXT:    movl 56(%esp,%ebp), %edx
-; FALLBACK27-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK27-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shll $3, %ecx
-; FALLBACK27-NEXT:    andl $24, %ecx
-; FALLBACK27-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK27-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 64(%esp,%ebp), %edi
-; FALLBACK27-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK27-NEXT:    movl %eax, %esi
-; FALLBACK27-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK27-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 72(%esp,%ebp), %esi
-; FALLBACK27-NEXT:    movl 68(%esp,%ebp), %eax
-; FALLBACK27-NEXT:    movl %eax, %edx
-; FALLBACK27-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK27-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 80(%esp,%ebp), %edi
-; FALLBACK27-NEXT:    movl 76(%esp,%ebp), %eax
-; FALLBACK27-NEXT:    movl %eax, %edx
-; FALLBACK27-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK27-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 88(%esp,%ebp), %ebx
-; FALLBACK27-NEXT:    movl 84(%esp,%ebp), %eax
-; FALLBACK27-NEXT:    movl %eax, %edx
-; FALLBACK27-NEXT:    shrdl %cl, %ebx, %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK27-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 96(%esp,%ebp), %esi
-; FALLBACK27-NEXT:    movl 92(%esp,%ebp), %eax
-; FALLBACK27-NEXT:    movl %eax, %edx
-; FALLBACK27-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK27-NEXT:    movl 104(%esp,%ebp), %eax
-; FALLBACK27-NEXT:    movl 100(%esp,%ebp), %edi
-; FALLBACK27-NEXT:    movl %edi, %edx
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK27-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK27-NEXT:    movl 48(%esp,%ebp), %edi
-; FALLBACK27-NEXT:    movl 108(%esp,%ebp), %ebp
-; FALLBACK27-NEXT:    movl %ebp, (%esp) # 4-byte Spill
-; FALLBACK27-NEXT:    shrdl %cl, %ebp, %eax
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK27-NEXT:    movl %eax, 56(%ebp)
-; FALLBACK27-NEXT:    movl %esi, 48(%ebp)
-; FALLBACK27-NEXT:    movl %edx, 52(%ebp)
-; FALLBACK27-NEXT:    movl %ebx, 40(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK27-NEXT:    shrxl %ecx, (%esp), %eax # 4-byte Folded Reload
-; FALLBACK27-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK27-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK27-NEXT:    movl %edi, (%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK27-NEXT:    movl %ecx, 4(%ebp)
-; FALLBACK27-NEXT:    movl %eax, 60(%ebp)
-; FALLBACK27-NEXT:    addl $188, %esp
-; FALLBACK27-NEXT:    popl %esi
-; FALLBACK27-NEXT:    popl %edi
-; FALLBACK27-NEXT:    popl %ebx
-; FALLBACK27-NEXT:    popl %ebp
-; FALLBACK27-NEXT:    vzeroupper
-; FALLBACK27-NEXT:    retl
-;
-; FALLBACK28-LABEL: lshr_64bytes:
-; FALLBACK28:       # %bb.0:
-; FALLBACK28-NEXT:    pushl %ebp
-; FALLBACK28-NEXT:    pushl %ebx
-; FALLBACK28-NEXT:    pushl %edi
-; FALLBACK28-NEXT:    pushl %esi
-; FALLBACK28-NEXT:    subl $204, %esp
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK28-NEXT:    vmovups (%ecx), %zmm0
-; FALLBACK28-NEXT:    movl (%eax), %ecx
-; FALLBACK28-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK28-NEXT:    vmovups %zmm1, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    vmovups %zmm0, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %ecx, %esi
-; FALLBACK28-NEXT:    andl $60, %esi
-; FALLBACK28-NEXT:    movl 68(%esp,%esi), %edx
-; FALLBACK28-NEXT:    shll $3, %ecx
-; FALLBACK28-NEXT:    andl $24, %ecx
-; FALLBACK28-NEXT:    movl %edx, %edi
-; FALLBACK28-NEXT:    shrl %cl, %edi
-; FALLBACK28-NEXT:    movl 72(%esp,%esi), %eax
-; FALLBACK28-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    leal (%eax,%eax), %ebx
-; FALLBACK28-NEXT:    movl %ecx, %ebp
-; FALLBACK28-NEXT:    movb %cl, %ch
-; FALLBACK28-NEXT:    notb %ch
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    orl %edi, %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 64(%esp,%esi), %edi
-; FALLBACK28-NEXT:    movl %ebp, %eax
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edi
-; FALLBACK28-NEXT:    addl %edx, %edx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %edx
-; FALLBACK28-NEXT:    orl %edi, %edx
-; FALLBACK28-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 76(%esp,%esi), %edx
-; FALLBACK28-NEXT:    movl %edx, %ebp
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shrl %cl, %ebp
-; FALLBACK28-NEXT:    movl 80(%esp,%esi), %edi
-; FALLBACK28-NEXT:    leal (%edi,%edi), %ebx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    orl %ebp, %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %cl, %ebx
-; FALLBACK28-NEXT:    addl %edx, %edx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %edx
-; FALLBACK28-NEXT:    orl %ebx, %edx
-; FALLBACK28-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 84(%esp,%esi), %ebx
-; FALLBACK28-NEXT:    movl %ebx, %ebp
-; FALLBACK28-NEXT:    movl %eax, %edx
-; FALLBACK28-NEXT:    movb %dl, %cl
-; FALLBACK28-NEXT:    shrl %cl, %ebp
-; FALLBACK28-NEXT:    movl 88(%esp,%esi), %eax
-; FALLBACK28-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    addl %eax, %eax
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %eax
-; FALLBACK28-NEXT:    orl %ebp, %eax
-; FALLBACK28-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %dl, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edi
-; FALLBACK28-NEXT:    addl %ebx, %ebx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    orl %edi, %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 92(%esp,%esi), %ebx
-; FALLBACK28-NEXT:    movl %ebx, %ebp
-; FALLBACK28-NEXT:    movb %dl, %cl
-; FALLBACK28-NEXT:    shrl %cl, %ebp
-; FALLBACK28-NEXT:    movl 96(%esp,%esi), %edi
-; FALLBACK28-NEXT:    leal (%edi,%edi), %eax
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %eax
-; FALLBACK28-NEXT:    orl %ebp, %eax
-; FALLBACK28-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %dl, %cl
-; FALLBACK28-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %cl, %eax
-; FALLBACK28-NEXT:    addl %ebx, %ebx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    orl %eax, %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 100(%esp,%esi), %ebx
-; FALLBACK28-NEXT:    movl %ebx, %ebp
-; FALLBACK28-NEXT:    movb %dl, %cl
-; FALLBACK28-NEXT:    shrl %cl, %ebp
-; FALLBACK28-NEXT:    movl 104(%esp,%esi), %edx
-; FALLBACK28-NEXT:    leal (%edx,%edx), %eax
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %eax
-; FALLBACK28-NEXT:    orl %ebp, %eax
-; FALLBACK28-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edi
-; FALLBACK28-NEXT:    addl %ebx, %ebx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    orl %edi, %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 108(%esp,%esi), %edi
-; FALLBACK28-NEXT:    movl %edi, %ebp
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %ebp
-; FALLBACK28-NEXT:    movl 112(%esp,%esi), %ecx
-; FALLBACK28-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK28-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    orl %ebp, %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edx
-; FALLBACK28-NEXT:    addl %edi, %edi
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %edi
-; FALLBACK28-NEXT:    orl %edx, %edi
-; FALLBACK28-NEXT:    movl %esi, %edx
-; FALLBACK28-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 116(%esp,%esi), %esi
-; FALLBACK28-NEXT:    movl %esi, %ebx
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shrl %cl, %ebx
-; FALLBACK28-NEXT:    movl 120(%esp,%edx), %eax
-; FALLBACK28-NEXT:    leal (%eax,%eax), %ebp
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebp
-; FALLBACK28-NEXT:    orl %ebx, %ebp
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK28-NEXT:    movb %dl, %cl
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %cl, %ebx
-; FALLBACK28-NEXT:    addl %esi, %esi
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %esi
-; FALLBACK28-NEXT:    orl %ebx, %esi
-; FALLBACK28-NEXT:    movb %dl, %cl
-; FALLBACK28-NEXT:    shrl %cl, %eax
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK28-NEXT:    movl 124(%esp,%edx), %ebx
-; FALLBACK28-NEXT:    leal (%ebx,%ebx), %edx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %edx
-; FALLBACK28-NEXT:    orl %eax, %edx
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK28-NEXT:    shrl %cl, %ebx
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK28-NEXT:    movl %ebx, 60(%eax)
-; FALLBACK28-NEXT:    movl %edx, 56(%eax)
-; FALLBACK28-NEXT:    movl %esi, 48(%eax)
-; FALLBACK28-NEXT:    movl %ebp, 52(%eax)
-; FALLBACK28-NEXT:    movl %edi, 40(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, (%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK28-NEXT:    addl $204, %esp
-; FALLBACK28-NEXT:    popl %esi
-; FALLBACK28-NEXT:    popl %edi
-; FALLBACK28-NEXT:    popl %ebx
-; FALLBACK28-NEXT:    popl %ebp
-; FALLBACK28-NEXT:    vzeroupper
-; FALLBACK28-NEXT:    retl
-;
-; FALLBACK29-LABEL: lshr_64bytes:
-; FALLBACK29:       # %bb.0:
-; FALLBACK29-NEXT:    pushl %ebp
-; FALLBACK29-NEXT:    pushl %ebx
-; FALLBACK29-NEXT:    pushl %edi
-; FALLBACK29-NEXT:    pushl %esi
-; FALLBACK29-NEXT:    subl $188, %esp
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK29-NEXT:    vmovups (%ecx), %zmm0
-; FALLBACK29-NEXT:    movl (%eax), %ecx
-; FALLBACK29-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK29-NEXT:    vmovups %zmm1, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    vmovups %zmm0, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %ecx, %ebp
-; FALLBACK29-NEXT:    andl $60, %ebp
-; FALLBACK29-NEXT:    movl 56(%esp,%ebp), %edx
-; FALLBACK29-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shll $3, %ecx
-; FALLBACK29-NEXT:    andl $24, %ecx
-; FALLBACK29-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK29-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 64(%esp,%ebp), %edi
-; FALLBACK29-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, %esi
-; FALLBACK29-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK29-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 72(%esp,%ebp), %esi
-; FALLBACK29-NEXT:    movl 68(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, %edx
-; FALLBACK29-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK29-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 80(%esp,%ebp), %edi
-; FALLBACK29-NEXT:    movl 76(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, %edx
-; FALLBACK29-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK29-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 88(%esp,%ebp), %esi
-; FALLBACK29-NEXT:    movl 84(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, %edx
-; FALLBACK29-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl %esi, %edx
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK29-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 96(%esp,%ebp), %esi
-; FALLBACK29-NEXT:    movl 92(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, %edi
-; FALLBACK29-NEXT:    shrdl %cl, %esi, %edi
-; FALLBACK29-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK29-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 104(%esp,%ebp), %edx
-; FALLBACK29-NEXT:    movl 100(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, %edi
-; FALLBACK29-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK29-NEXT:    movl 48(%esp,%ebp), %ebx
-; FALLBACK29-NEXT:    movl 108(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK29-NEXT:    movl %edx, 56(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK29-NEXT:    shrdl %cl, %edx, %ebx
-; FALLBACK29-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK29-NEXT:    shrl %cl, %eax
-; FALLBACK29-NEXT:    movl %eax, 60(%ebp)
-; FALLBACK29-NEXT:    movl %esi, 48(%ebp)
-; FALLBACK29-NEXT:    movl %edi, 52(%ebp)
-; FALLBACK29-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 40(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK29-NEXT:    movl %ebx, (%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 4(%ebp)
-; FALLBACK29-NEXT:    addl $188, %esp
-; FALLBACK29-NEXT:    popl %esi
-; FALLBACK29-NEXT:    popl %edi
-; FALLBACK29-NEXT:    popl %ebx
-; FALLBACK29-NEXT:    popl %ebp
-; FALLBACK29-NEXT:    vzeroupper
-; FALLBACK29-NEXT:    retl
-;
-; FALLBACK30-LABEL: lshr_64bytes:
-; FALLBACK30:       # %bb.0:
-; FALLBACK30-NEXT:    pushl %ebp
-; FALLBACK30-NEXT:    pushl %ebx
-; FALLBACK30-NEXT:    pushl %edi
-; FALLBACK30-NEXT:    pushl %esi
-; FALLBACK30-NEXT:    subl $204, %esp
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK30-NEXT:    vmovups (%ecx), %zmm0
-; FALLBACK30-NEXT:    movl (%eax), %ecx
-; FALLBACK30-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK30-NEXT:    vmovups %zmm1, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    vmovups %zmm0, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    leal (,%ecx,8), %edx
-; FALLBACK30-NEXT:    andl $24, %edx
-; FALLBACK30-NEXT:    movl %edx, %ebx
-; FALLBACK30-NEXT:    andl $60, %ecx
-; FALLBACK30-NEXT:    movl 68(%esp,%ecx), %esi
-; FALLBACK30-NEXT:    movl 72(%esp,%ecx), %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrxl %ebx, %esi, %edi
-; FALLBACK30-NEXT:    notb %dl
-; FALLBACK30-NEXT:    leal (%eax,%eax), %ebp
-; FALLBACK30-NEXT:    shlxl %edx, %ebp, %ebp
-; FALLBACK30-NEXT:    orl %edi, %ebp
-; FALLBACK30-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrxl %ebx, 64(%esp,%ecx), %edi
-; FALLBACK30-NEXT:    addl %esi, %esi
-; FALLBACK30-NEXT:    shlxl %edx, %esi, %esi
-; FALLBACK30-NEXT:    orl %edi, %esi
-; FALLBACK30-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 80(%esp,%ecx), %esi
-; FALLBACK30-NEXT:    leal (%esi,%esi), %edi
-; FALLBACK30-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK30-NEXT:    movl 76(%esp,%ecx), %edi
-; FALLBACK30-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK30-NEXT:    orl %ebp, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK30-NEXT:    addl %edi, %edi
-; FALLBACK30-NEXT:    shlxl %edx, %edi, %edi
-; FALLBACK30-NEXT:    orl %eax, %edi
-; FALLBACK30-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 88(%esp,%ecx), %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK30-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK30-NEXT:    movl 84(%esp,%ecx), %edi
-; FALLBACK30-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK30-NEXT:    orl %ebp, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrxl %ebx, %esi, %esi
-; FALLBACK30-NEXT:    addl %edi, %edi
-; FALLBACK30-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK30-NEXT:    orl %esi, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 96(%esp,%ecx), %esi
-; FALLBACK30-NEXT:    leal (%esi,%esi), %edi
-; FALLBACK30-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK30-NEXT:    movl 92(%esp,%ecx), %edi
-; FALLBACK30-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK30-NEXT:    orl %ebp, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK30-NEXT:    addl %edi, %edi
-; FALLBACK30-NEXT:    shlxl %edx, %edi, %edi
-; FALLBACK30-NEXT:    orl %eax, %edi
-; FALLBACK30-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 104(%esp,%ecx), %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK30-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK30-NEXT:    movl 100(%esp,%ecx), %edi
-; FALLBACK30-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK30-NEXT:    orl %ebp, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrxl %ebx, %esi, %esi
-; FALLBACK30-NEXT:    addl %edi, %edi
-; FALLBACK30-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK30-NEXT:    orl %esi, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 112(%esp,%ecx), %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    leal (%eax,%eax), %esi
-; FALLBACK30-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK30-NEXT:    movl 108(%esp,%ecx), %esi
-; FALLBACK30-NEXT:    shrxl %ebx, %esi, %edi
-; FALLBACK30-NEXT:    orl %edi, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK30-NEXT:    addl %esi, %esi
-; FALLBACK30-NEXT:    shlxl %edx, %esi, %ebp
-; FALLBACK30-NEXT:    orl %eax, %ebp
-; FALLBACK30-NEXT:    movl 120(%esp,%ecx), %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    addl %eax, %eax
-; FALLBACK30-NEXT:    shlxl %edx, %eax, %esi
-; FALLBACK30-NEXT:    movl 116(%esp,%ecx), %eax
-; FALLBACK30-NEXT:    shrxl %ebx, %eax, %edi
-; FALLBACK30-NEXT:    orl %edi, %esi
-; FALLBACK30-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK30-NEXT:    addl %eax, %eax
-; FALLBACK30-NEXT:    shlxl %edx, %eax, %eax
-; FALLBACK30-NEXT:    orl %edi, %eax
-; FALLBACK30-NEXT:    movl 124(%esp,%ecx), %ecx
-; FALLBACK30-NEXT:    leal (%ecx,%ecx), %edi
-; FALLBACK30-NEXT:    shlxl %edx, %edi, %edx
-; FALLBACK30-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK30-NEXT:    orl %edi, %edx
-; FALLBACK30-NEXT:    shrxl %ebx, %ecx, %edi
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK30-NEXT:    movl %edi, 60(%ecx)
-; FALLBACK30-NEXT:    movl %edx, 56(%ecx)
-; FALLBACK30-NEXT:    movl %eax, 48(%ecx)
-; FALLBACK30-NEXT:    movl %esi, 52(%ecx)
-; FALLBACK30-NEXT:    movl %ebp, 40(%ecx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 44(%ecx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 32(%ecx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 36(%ecx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 24(%ecx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 28(%ecx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 16(%ecx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 20(%ecx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 8(%ecx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 12(%ecx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, (%ecx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 4(%ecx)
-; FALLBACK30-NEXT:    addl $204, %esp
-; FALLBACK30-NEXT:    popl %esi
-; FALLBACK30-NEXT:    popl %edi
-; FALLBACK30-NEXT:    popl %ebx
-; FALLBACK30-NEXT:    popl %ebp
-; FALLBACK30-NEXT:    vzeroupper
-; FALLBACK30-NEXT:    retl
-;
-; FALLBACK31-LABEL: lshr_64bytes:
-; FALLBACK31:       # %bb.0:
-; FALLBACK31-NEXT:    pushl %ebp
-; FALLBACK31-NEXT:    pushl %ebx
-; FALLBACK31-NEXT:    pushl %edi
-; FALLBACK31-NEXT:    pushl %esi
-; FALLBACK31-NEXT:    subl $188, %esp
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK31-NEXT:    vmovups (%ecx), %zmm0
-; FALLBACK31-NEXT:    movl (%eax), %ecx
-; FALLBACK31-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK31-NEXT:    vmovups %zmm1, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    vmovups %zmm0, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %ecx, %ebp
-; FALLBACK31-NEXT:    andl $60, %ebp
-; FALLBACK31-NEXT:    movl 56(%esp,%ebp), %edx
-; FALLBACK31-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK31-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shll $3, %ecx
-; FALLBACK31-NEXT:    andl $24, %ecx
-; FALLBACK31-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK31-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 64(%esp,%ebp), %edi
-; FALLBACK31-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK31-NEXT:    movl %eax, %esi
-; FALLBACK31-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK31-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 72(%esp,%ebp), %esi
-; FALLBACK31-NEXT:    movl 68(%esp,%ebp), %eax
-; FALLBACK31-NEXT:    movl %eax, %edx
-; FALLBACK31-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK31-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 80(%esp,%ebp), %edi
-; FALLBACK31-NEXT:    movl 76(%esp,%ebp), %eax
-; FALLBACK31-NEXT:    movl %eax, %edx
-; FALLBACK31-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK31-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 88(%esp,%ebp), %ebx
-; FALLBACK31-NEXT:    movl 84(%esp,%ebp), %eax
-; FALLBACK31-NEXT:    movl %eax, %edx
-; FALLBACK31-NEXT:    shrdl %cl, %ebx, %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK31-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 96(%esp,%ebp), %esi
-; FALLBACK31-NEXT:    movl 92(%esp,%ebp), %eax
-; FALLBACK31-NEXT:    movl %eax, %edx
-; FALLBACK31-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK31-NEXT:    movl 104(%esp,%ebp), %eax
-; FALLBACK31-NEXT:    movl 100(%esp,%ebp), %edi
-; FALLBACK31-NEXT:    movl %edi, %edx
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK31-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK31-NEXT:    movl 48(%esp,%ebp), %edi
-; FALLBACK31-NEXT:    movl 108(%esp,%ebp), %ebp
-; FALLBACK31-NEXT:    movl %ebp, (%esp) # 4-byte Spill
-; FALLBACK31-NEXT:    shrdl %cl, %ebp, %eax
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK31-NEXT:    movl %eax, 56(%ebp)
-; FALLBACK31-NEXT:    movl %esi, 48(%ebp)
-; FALLBACK31-NEXT:    movl %edx, 52(%ebp)
-; FALLBACK31-NEXT:    movl %ebx, 40(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK31-NEXT:    shrxl %ecx, (%esp), %eax # 4-byte Folded Reload
-; FALLBACK31-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK31-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK31-NEXT:    movl %edi, (%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK31-NEXT:    movl %ecx, 4(%ebp)
-; FALLBACK31-NEXT:    movl %eax, 60(%ebp)
-; FALLBACK31-NEXT:    addl $188, %esp
-; FALLBACK31-NEXT:    popl %esi
-; FALLBACK31-NEXT:    popl %edi
-; FALLBACK31-NEXT:    popl %ebx
-; FALLBACK31-NEXT:    popl %ebp
-; FALLBACK31-NEXT:    vzeroupper
-; FALLBACK31-NEXT:    retl
+; X64-NO-SHLD-NO-BMI2-SSE2-LABEL: lshr_64bytes:
+; X64-NO-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %r13
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %r12
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 32(%rdi), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 40(%rdi), %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 48(%rdi), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 56(%rdi), %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl (%rsi), %edi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (,%rdi,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    andl $56, %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    andl $56, %edi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -128(%rsp,%rdi), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -120(%rsp,%rdi), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -112(%rsp,%rdi), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leaq (%rbx,%rbx), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r11, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    addq %r8, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r10, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -104(%rsp,%rdi), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -96(%rsp,%rdi), %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leaq (%r14,%r14), %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r15, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    addq %r10, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %rbx, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -88(%rsp,%rdi), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rbx, %r12
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -80(%rsp,%rdi), %r13
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leaq (%r13,%r13), %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r12, %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    addq %rbx, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r14, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r13
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -72(%rsp,%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leaq (%rdi,%rdi), %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r13, %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rbx, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r15, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r11, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %r12
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %r13
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE2-LABEL: lshr_64bytes:
+; X64-HAVE-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushq %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushq %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 32(%rdi), %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 40(%rdi), %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 48(%rdi), %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 56(%rdi), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -112(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -128(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -120(%rsp,%rax), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %rdi, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -96(%rsp,%rax), %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -104(%rsp,%rax), %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r11, %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %r10, %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %r11, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -80(%rsp,%rax), %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -88(%rsp,%rax), %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %r11, %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %r14, %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -72(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %rax, %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %r9, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r11, 48(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, 56(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, 32(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r15, 40(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rbx, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rsi, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popq %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popq %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popq %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE2-LABEL: lshr_64bytes:
+; X64-NO-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 32(%rdi), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 40(%rdi), %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 48(%rdi), %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 56(%rdi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%rsi), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (,%rax,8), %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $56, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $56, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -120(%rsp,%rax), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -112(%rsp,%rax), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rsi, %r8, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    notb %cl
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leaq (%r10,%r10), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r9, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rsi, -128(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addq %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r9, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -104(%rsp,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rsi, %r11, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -96(%rsp,%rax), %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leaq (%r14,%r14), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %rbx, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rsi, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addq %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r10, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -88(%rsp,%rax), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rsi, %r10, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -80(%rsp,%rax), %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leaq (%r15,%r15), %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r12, %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %rbx, %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rsi, %r14, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addq %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %rbx, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rsi, %r15, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -72(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leaq (%rax,%rax), %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r14, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %rbx, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rsi, %rax, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 56(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r10, 32(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r12, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addq $8, %rsp
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: lshr_64bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 32(%rdi), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 40(%rdi), %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 48(%rdi), %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 56(%rdi), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -112(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -128(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -120(%rsp,%rax), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %rdi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -96(%rsp,%rax), %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -104(%rsp,%rax), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r11, %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %r10, %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %r11, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -80(%rsp,%rax), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -88(%rsp,%rax), %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r14, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %r11, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %r14, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -72(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %rax, %r11
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rcx, %rax, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %r9, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r11, 48(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r10, 32(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r15, 40(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rbx, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rsi, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 56(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-SSE4-LABEL: lshr_64bytes:
+; X64-NO-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %rbp
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r13
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %rax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 32(%rdi), %xmm2
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 48(%rdi), %xmm3
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl (%rsi), %r8d
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm4, %xmm4
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm3, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (,%r8,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    andl $56, %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    andl $56, %r8d
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -128(%rsp,%r8), %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -120(%rsp,%r8), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leaq (%r9,%r9), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -104(%rsp,%r8), %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -96(%rsp,%r8), %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leaq (%r12,%r12), %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %rbx, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -112(%rsp,%r8), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rbx, %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    addq %r10, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r14, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -88(%rsp,%r8), %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r14, %r13
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r13
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -80(%rsp,%r8), %rbp
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leaq (%rbp,%rbp), %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r13, %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    addq %r14, %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r12, %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %rbp
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -72(%rsp,%r8), %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leaq (%r8,%r8), %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %rbp, %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    addq %rbx, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r9, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r8, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rbx, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r12, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r14, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r15, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r11, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    addq $8, %rsp
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %r13
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %rbp
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE4-LABEL: lshr_64bytes:
+; X64-HAVE-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 32(%rdi), %xmm2
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 48(%rdi), %xmm3
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm4, %xmm4
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm3, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -96(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -104(%rsp,%rax), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %rdi, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -112(%rsp,%rax), %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %r9, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -80(%rsp,%rax), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -88(%rsp,%rax), %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r11, %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %r9, %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %r11, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -72(%rsp,%rax), %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %r11, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -128(%rsp,%rax), %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -120(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rax, %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %r10, %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %rax, %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r15, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, 48(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r11, 56(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, 32(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rbx, 40(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r14, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popq %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popq %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popq %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE4-LABEL: lshr_64bytes:
+; X64-NO-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %r13
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 32(%rdi), %xmm2
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 48(%rdi), %xmm3
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl (%rsi), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm4, %xmm4
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm3, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (,%rax,8), %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $56, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $56, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, -128(%rsp,%rax), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    notb %cl
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -120(%rsp,%rax), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -112(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leaq (%r10,%r10), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r8, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -104(%rsp,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %r11, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -96(%rsp,%rax), %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leaq (%r14,%r14), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rbx, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %r9, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addq %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rbx, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -88(%rsp,%rax), %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %rbx, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -80(%rsp,%rax), %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leaq (%r12,%r12), %r13
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r13, %r13
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r15, %r13
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %r14, %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addq %rbx, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %rbx, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r14, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %r12, %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -72(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leaq (%rax,%rax), %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r15, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r14, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addq %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r9, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r10, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %rax, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, 56(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rcx, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r15, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rbx, 32(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r13, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r8, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %r13
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: lshr_64bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 32(%rdi), %xmm2
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 48(%rdi), %xmm3
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm4, %xmm4
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm3, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -96(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -104(%rsp,%rax), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r9, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %rdi, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -112(%rsp,%rax), %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r10, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %r9, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -80(%rsp,%rax), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -88(%rsp,%rax), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r11, %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %r9, %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %r11, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -72(%rsp,%rax), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %r11, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -128(%rsp,%rax), %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -120(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %r10, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rcx, %r11, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %rax, %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r15, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r9, 48(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, 32(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rbx, 40(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r14, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r10, 56(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-AVX1-LABEL: lshr_64bytes:
+; X64-NO-SHLD-NO-BMI2-AVX1:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushq %rbp
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushq %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushq %r14
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushq %r13
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushq %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushq %rax
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl (%rsi), %r9d
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    leal (,%r9,8), %eax
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    andl $56, %eax
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    andl $56, %r9d
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq -128(%rsp,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq -120(%rsp,%r9), %r8
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    leaq (%r8,%r8), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shlq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    orq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq -104(%rsp,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r10, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq -96(%rsp,%r9), %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    leaq (%r12,%r12), %r11
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shlq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    orq %rbx, %r11
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq -112(%rsp,%r9), %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %rbx, %r14
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    addq %r10, %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    orq %r14, %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq -88(%rsp,%r9), %r14
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r14, %r13
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %cl, %r13
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq -80(%rsp,%r9), %rbp
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    leaq (%rbp,%rbp), %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shlq %cl, %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    orq %r13, %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    addq %r14, %r14
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shlq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    orq %r12, %r14
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %cl, %rbp
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq -72(%rsp,%r9), %r9
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    leaq (%r9,%r9), %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shlq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    orq %rbp, %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    addq %rbx, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    orq %r8, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r9, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %rbx, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r12, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r14, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r15, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r10, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r11, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    addq $8, %rsp
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    popq %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    popq %r13
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    popq %r14
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    popq %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    popq %rbp
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    vzeroupper
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-AVX1-LABEL: lshr_64bytes:
+; X64-HAVE-SHLD-NO-BMI2-AVX1:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    pushq %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    pushq %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq -96(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq -104(%rsp,%rax), %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %r9, %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdq %cl, %rdi, %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq -112(%rsp,%rax), %r10
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %r10, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdq %cl, %r9, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq -80(%rsp,%rax), %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq -88(%rsp,%rax), %r11
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %r11, %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdq %cl, %r9, %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdq %cl, %r11, %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq -72(%rsp,%rax), %r11
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdq %cl, %r11, %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq -128(%rsp,%rax), %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq -120(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %rax, %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdq %cl, %r10, %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdq %cl, %rax, %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrq %cl, %r11
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %r15, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %r9, 48(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %r11, 56(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %rdi, 32(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %rbx, 40(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %r14, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    popq %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    popq %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    popq %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vzeroupper
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-AVX1-LABEL: lshr_64bytes:
+; X64-NO-SHLD-HAVE-BMI2-AVX1:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushq %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushq %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushq %r13
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushq %r12
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushq %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    andl $56, %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    andl $56, %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxq %rcx, -128(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -120(%rsp,%rsi), %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -112(%rsp,%rsi), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    leaq (%r10,%r10), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxq %rax, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orq %r8, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -104(%rsp,%rsi), %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxq %rcx, %r11, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -96(%rsp,%rsi), %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    leaq (%r14,%r14), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxq %rax, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orq %rbx, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxq %rcx, %r9, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    addq %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxq %rax, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orq %rbx, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -88(%rsp,%rsi), %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxq %rcx, %rbx, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -80(%rsp,%rsi), %r12
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    leaq (%r12,%r12), %r13
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxq %rax, %r13, %r13
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orq %r15, %r13
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxq %rcx, %r14, %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    addq %rbx, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxq %rax, %rbx, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orq %r14, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxq %rcx, %r12, %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -72(%rsp,%rsi), %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    leaq (%rsi,%rsi), %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxq %rax, %r15, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orq %r14, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxq %rcx, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    addq %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxq %rax, %r9, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orq %r10, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxq %rcx, %rsi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %rcx, 56(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %rax, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r15, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %rbx, 32(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r13, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r8, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    popq %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    popq %r12
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    popq %r13
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    popq %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    popq %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vzeroupper
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-LABEL: lshr_64bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    pushq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    pushq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -96(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -104(%rsp,%rax), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r9, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdq %cl, %rdi, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -112(%rsp,%rax), %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r10, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdq %cl, %r9, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -80(%rsp,%rax), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -88(%rsp,%rax), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r11, %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdq %cl, %r9, %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdq %cl, %r11, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -72(%rsp,%rax), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdq %cl, %r11, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -128(%rsp,%rax), %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -120(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %rax, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdq %cl, %r10, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxq %rcx, %r11, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdq %cl, %rax, %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r15, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r9, 48(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %rdi, 32(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %rbx, 40(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r14, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r10, 56(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    popq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    popq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    popq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vzeroupper
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-AVX512-LABEL: lshr_64bytes:
+; X64-NO-SHLD-NO-BMI2-AVX512:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushq %rbp
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushq %r15
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushq %r14
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushq %r13
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushq %r12
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushq %rax
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    vmovups (%rdi), %zmm0
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl (%rsi), %r9d
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    vmovups %zmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    vmovups %zmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    leal (,%r9,8), %eax
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    andl $56, %eax
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    andl $56, %r9d
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq -128(%rsp,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq -120(%rsp,%r9), %r8
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    leaq (%r8,%r8), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shlq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    orq %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq -104(%rsp,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r10, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq -96(%rsp,%r9), %r12
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    leaq (%r12,%r12), %r11
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shlq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    orq %rbx, %r11
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq -112(%rsp,%r9), %rbx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %rbx, %r14
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    addq %r10, %r10
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    orq %r14, %r10
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq -88(%rsp,%r9), %r14
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r14, %r13
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %cl, %r13
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq -80(%rsp,%r9), %rbp
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    leaq (%rbp,%rbp), %r15
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shlq %cl, %r15
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    orq %r13, %r15
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    addq %r14, %r14
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shlq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    orq %r12, %r14
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %cl, %rbp
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq -72(%rsp,%r9), %r9
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    leaq (%r9,%r9), %r12
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shlq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    orq %rbp, %r12
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    addq %rbx, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    orq %r8, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r9, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %rbx, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r12, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r14, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r15, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r10, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r11, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    addq $8, %rsp
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    popq %r12
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    popq %r13
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    popq %r14
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    popq %r15
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    popq %rbp
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    vzeroupper
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-AVX512-LABEL: lshr_64bytes:
+; X64-HAVE-SHLD-NO-BMI2-AVX512:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    pushq %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    pushq %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vmovups (%rdi), %zmm0
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl (%rsi), %edi
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vmovups %zmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vmovups %zmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    leal (,%rdi,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    andl $56, %edi
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq -96(%rsp,%rdi), %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq -104(%rsp,%rdi), %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %r9, %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdq %cl, %rsi, %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq -112(%rsp,%rdi), %r10
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %r10, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdq %cl, %r9, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq -80(%rsp,%rdi), %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq -88(%rsp,%rdi), %r11
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %r11, %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdq %cl, %r9, %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdq %cl, %r11, %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq -72(%rsp,%rdi), %r11
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdq %cl, %r11, %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq -128(%rsp,%rdi), %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq -120(%rsp,%rdi), %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %rdi, %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdq %cl, %r10, %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdq %cl, %rdi, %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrq %cl, %r11
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %r15, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %r9, 48(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %r11, 56(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %rsi, 32(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %rbx, 40(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %rax, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %r14, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    popq %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    popq %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    popq %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vzeroupper
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-AVX512-LABEL: lshr_64bytes:
+; X64-NO-SHLD-HAVE-BMI2-AVX512:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushq %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushq %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushq %r13
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushq %r12
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushq %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups (%rdi), %zmm0
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups %zmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups %zmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    andl $56, %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    andl $56, %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxq %rcx, -128(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -120(%rsp,%rsi), %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -112(%rsp,%rsi), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    leaq (%r10,%r10), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxq %rax, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orq %r8, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -104(%rsp,%rsi), %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxq %rcx, %r11, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -96(%rsp,%rsi), %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    leaq (%r14,%r14), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxq %rax, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orq %rbx, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxq %rcx, %r9, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    addq %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxq %rax, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orq %rbx, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -88(%rsp,%rsi), %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxq %rcx, %rbx, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -80(%rsp,%rsi), %r12
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    leaq (%r12,%r12), %r13
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxq %rax, %r13, %r13
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orq %r15, %r13
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxq %rcx, %r14, %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    addq %rbx, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxq %rax, %rbx, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orq %r14, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxq %rcx, %r12, %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -72(%rsp,%rsi), %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    leaq (%rsi,%rsi), %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxq %rax, %r15, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orq %r14, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxq %rcx, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    addq %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxq %rax, %r9, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orq %r10, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxq %rcx, %rsi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %rcx, 56(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %rax, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r15, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %rbx, 32(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r13, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r8, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    popq %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    popq %r12
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    popq %r13
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    popq %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    popq %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vzeroupper
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-LABEL: lshr_64bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    pushq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    pushq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups (%rdi), %zmm0
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups %zmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups %zmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -96(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -104(%rsp,%rax), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r9, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdq %cl, %rdi, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -112(%rsp,%rax), %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r10, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdq %cl, %r9, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -80(%rsp,%rax), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -88(%rsp,%rax), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r11, %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdq %cl, %r9, %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdq %cl, %r11, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -72(%rsp,%rax), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdq %cl, %r11, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -128(%rsp,%rax), %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -120(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %rax, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdq %cl, %r10, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxq %rcx, %r11, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdq %cl, %rax, %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r15, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r9, 48(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %rdi, 32(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %rbx, 40(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r14, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r10, 56(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    popq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    popq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    popq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vzeroupper
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    retq
+;
+; X86-NO-SHLD-NO-BMI2-SSE2-LABEL: lshr_64bytes:
+; X86-NO-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    subl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl (%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 32(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 36(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 40(%eax), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 44(%eax), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 48(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 52(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 56(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 60(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    andl $60, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 68(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll $3, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    andl $24, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 72(%esp,%esi), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%ecx,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    notb %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 64(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 76(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 80(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%edi,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 84(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 88(%esp,%esi), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 92(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 96(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%edi,%edi), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 100(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 104(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%edx,%edx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 108(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 112(%esp,%esi), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%ecx,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 116(%esp,%esi), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 120(%esp,%edx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%eax,%eax), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 124(%esp,%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%ebx,%ebx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, 60(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, 56(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, 48(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, 52(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, 40(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 44(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 32(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 36(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, (%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-SSE2-LABEL: lshr_64bytes:
+; X86-HAVE-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 32(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 36(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 40(%ecx), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 44(%ecx), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 48(%ecx), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 52(%ecx), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 56(%ecx), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 60(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%ecx), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 56(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 64(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 72(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 68(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 80(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 76(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 88(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 84(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 96(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 92(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %esi, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 104(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 100(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 48(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 108(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, 56(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 60(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, 48(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, 52(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 40(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-SSE2-LABEL: lshr_64bytes:
+; X86-NO-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    subl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 28(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 32(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 36(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 40(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 44(%eax), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 48(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 52(%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 56(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 60(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%eax), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (,%ebx,8), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $24, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $60, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 68(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 72(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    notb %dl
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%eax,%eax), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, 64(%esp,%ebx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 80(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%esi,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 76(%esp,%ebx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 88(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%eax,%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 84(%esp,%ebx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 96(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%esi,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 92(%esp,%ebx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 104(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%eax,%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 100(%esp,%ebx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 112(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%eax,%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 108(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %esi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 120(%esp,%ebx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%edi,%edi), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %ecx, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 116(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebp, %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 124(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%eax,%eax), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %ebx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebp, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebp, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, 60(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, 56(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 48(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, 52(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 40(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 44(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 32(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 36(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 28(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, (%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: lshr_64bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 28(%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 32(%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 36(%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 40(%ecx), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 44(%ecx), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 48(%ecx), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 52(%ecx), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 56(%ecx), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 60(%ecx), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%ecx), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 56(%esp,%ebp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 64(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 72(%esp,%ebp), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 68(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 80(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 76(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 88(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 84(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %ebx, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 96(%esp,%ebp), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 92(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 104(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 100(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 48(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 108(%esp,%ebp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %ebp, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 56(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, 48(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, 52(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, 40(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, (%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 4(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 60(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-SSE4-LABEL: lshr_64bytes:
+; X86-NO-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    subl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 32(%ecx), %xmm2
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 48(%ecx), %xmm3
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm4, %xmm4
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm3, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    andl $60, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 68(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll $3, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    andl $24, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 72(%esp,%esi), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%ecx,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %ch
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    notb %ch
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 64(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %edi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 76(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 80(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%edi,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 84(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 88(%esp,%esi), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 92(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 96(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%edi,%edi), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 100(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 104(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%edx,%edx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 108(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 112(%esp,%esi), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%ecx,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 116(%esp,%esi), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 120(%esp,%edx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%eax,%eax), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 124(%esp,%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%ebx,%ebx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, 60(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, 56(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, 48(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, 52(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, 40(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 44(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 32(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 36(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, (%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-SSE4-LABEL: lshr_64bytes:
+; X86-HAVE-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 32(%ecx), %xmm2
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 48(%ecx), %xmm3
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl (%eax), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm4, %xmm4
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm3, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 56(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %edx, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 64(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 72(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 68(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 80(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 76(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 88(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 84(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 96(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 92(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %esi, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 104(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 100(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 48(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 108(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, 56(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %edx, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 60(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, 48(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, 52(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 40(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-SSE4-LABEL: lshr_64bytes:
+; X86-NO-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    subl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 32(%ecx), %xmm2
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 48(%ecx), %xmm3
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl (%eax), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm4, %xmm4
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm3, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (,%ebx,8), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $24, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $60, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 68(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 72(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    notb %dl
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%eax,%eax), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, 64(%esp,%ebx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 80(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%esi,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 76(%esp,%ebx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 88(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%eax,%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 84(%esp,%ebx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 96(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%esi,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 92(%esp,%ebx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 104(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%eax,%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 100(%esp,%ebx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 112(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%eax,%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 108(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %esi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 120(%esp,%ebx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%edi,%edi), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %ecx, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 116(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebp, %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 124(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%eax,%eax), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %ebx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebp, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebp, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, 60(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, 56(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 48(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, 52(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 40(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 44(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 32(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 36(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 28(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, (%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: lshr_64bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 32(%ecx), %xmm2
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 48(%ecx), %xmm3
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl (%eax), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm4, %xmm4
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm3, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 56(%esp,%ebp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %edx, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 64(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 72(%esp,%ebp), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 68(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 80(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 76(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 88(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 84(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %ebx, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 96(%esp,%ebp), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 92(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 104(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 100(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 48(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 108(%esp,%ebp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %ebp, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 56(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, 48(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, 52(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, 40(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ecx, (%esp), %eax # 4-byte Folded Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, (%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 4(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 60(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-AVX1-LABEL: lshr_64bytes:
+; X86-NO-SHLD-NO-BMI2-AVX1:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    subl $204, %esp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups (%ecx), %ymm0
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups 32(%ecx), %ymm1
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl (%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    andl $60, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 68(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll $3, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    andl $24, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 72(%esp,%esi), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    leal (%eax,%eax), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %cl, %ch
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    notb %ch
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 64(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %edi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 76(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 80(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    leal (%edi,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 84(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 88(%esp,%esi), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 92(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 96(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    leal (%edi,%edi), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 100(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 104(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    leal (%edx,%edx), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 108(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 112(%esp,%esi), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    leal (%ecx,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 116(%esp,%esi), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 120(%esp,%edx), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    leal (%eax,%eax), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 124(%esp,%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    leal (%ebx,%ebx), %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, 60(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, 56(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, 48(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebp, 52(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, 40(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 44(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 32(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 36(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, (%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    vzeroupper
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-AVX1-LABEL: lshr_64bytes:
+; X86-HAVE-SHLD-NO-BMI2-AVX1:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups (%ecx), %ymm0
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups 32(%ecx), %ymm1
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl (%eax), %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 56(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdl %cl, %edx, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 64(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 72(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 68(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 80(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 76(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 88(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 84(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 96(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 92(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdl %cl, %esi, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 104(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 100(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 48(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 108(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, 56(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrdl %cl, %edx, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 60(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, 48(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, 52(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 40(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vzeroupper
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-AVX1-LABEL: lshr_64bytes:
+; X86-NO-SHLD-HAVE-BMI2-AVX1:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    subl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups (%ecx), %ymm0
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups 32(%ecx), %ymm1
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl (%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    leal (,%ecx,8), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    andl $24, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    andl $60, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 68(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 72(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    notb %dl
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    leal (%eax,%eax), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edx, %ebp, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, 64(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 80(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    leal (%esi,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 76(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 88(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    leal (%eax,%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 84(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 96(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    leal (%esi,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 92(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 104(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    leal (%eax,%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 100(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 112(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    leal (%eax,%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 108(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edx, %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 120(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edx, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 116(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edx, %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 124(%esp,%ecx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    leal (%ecx,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edx, %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %ecx, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, 60(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edx, 56(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 48(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %esi, 52(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ebp, 40(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 44(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 32(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 36(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 24(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 28(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 16(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 20(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 8(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 12(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, (%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 4(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    addl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vzeroupper
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-LABEL: lshr_64bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups (%ecx), %ymm0
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups 32(%ecx), %ymm1
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl (%eax), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 56(%esp,%ebp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdl %cl, %edx, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 64(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 72(%esp,%ebp), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 68(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 80(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 76(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 88(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 84(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdl %cl, %ebx, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 96(%esp,%ebp), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 92(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 104(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 100(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 48(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 108(%esp,%ebp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ebp, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdl %cl, %ebp, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 56(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %esi, 48(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edx, 52(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ebx, 40(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ecx, (%esp), %eax # 4-byte Folded Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, (%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, 4(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 60(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vzeroupper
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-AVX512-LABEL: lshr_64bytes:
+; X86-NO-SHLD-NO-BMI2-AVX512:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    subl $204, %esp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    vmovups (%ecx), %zmm0
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl (%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    vmovups %zmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    vmovups %zmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    andl $60, %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 68(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll $3, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    andl $24, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 72(%esp,%esi), %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    leal (%eax,%eax), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %cl, %ch
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    notb %ch
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 64(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %edi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 76(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 80(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    leal (%edi,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 84(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 88(%esp,%esi), %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 92(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 96(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    leal (%edi,%edi), %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 100(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 104(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    leal (%edx,%edx), %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 108(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 112(%esp,%esi), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    leal (%ecx,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 116(%esp,%esi), %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 120(%esp,%edx), %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    leal (%eax,%eax), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 124(%esp,%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    leal (%ebx,%ebx), %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, 60(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, 56(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, 48(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebp, 52(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, 40(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 44(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 32(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 36(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, (%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    vzeroupper
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-AVX512-LABEL: lshr_64bytes:
+; X86-HAVE-SHLD-NO-BMI2-AVX512:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vmovups (%ecx), %zmm0
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl (%eax), %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vmovups %zmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vmovups %zmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 56(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdl %cl, %edx, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 64(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 72(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 68(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 80(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 76(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 88(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 84(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 96(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 92(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdl %cl, %esi, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 104(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 100(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 48(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 108(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, 56(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrdl %cl, %edx, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 60(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, 48(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, 52(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 40(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vzeroupper
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-AVX512-LABEL: lshr_64bytes:
+; X86-NO-SHLD-HAVE-BMI2-AVX512:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    subl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups (%ecx), %zmm0
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl (%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups %zmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups %zmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    leal (,%ecx,8), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    andl $24, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    andl $60, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 68(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 72(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    notb %dl
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    leal (%eax,%eax), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edx, %ebp, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, 64(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 80(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    leal (%esi,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 76(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 88(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    leal (%eax,%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 84(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 96(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    leal (%esi,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 92(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 104(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    leal (%eax,%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 100(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 112(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    leal (%eax,%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 108(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edx, %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 120(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edx, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 116(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edx, %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 124(%esp,%ecx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    leal (%ecx,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edx, %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %ecx, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, 60(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edx, 56(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 48(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %esi, 52(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ebp, 40(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 44(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 32(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 36(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 24(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 28(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 16(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 20(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 8(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 12(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, (%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 4(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    addl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vzeroupper
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-LABEL: lshr_64bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups (%ecx), %zmm0
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl (%eax), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups %zmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups %zmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 56(%esp,%ebp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdl %cl, %edx, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 64(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 72(%esp,%ebp), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 68(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 80(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 76(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 88(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 84(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdl %cl, %ebx, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 96(%esp,%ebp), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 92(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 104(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 100(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 48(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 108(%esp,%ebp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ebp, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdl %cl, %ebp, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 56(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %esi, 48(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edx, 52(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ebx, 40(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ecx, (%esp), %eax # 4-byte Folded Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, (%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, 4(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 60(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vzeroupper
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    retl
   %src = load i512, ptr %src.ptr, align 1
   %byteOff = load i512, ptr %byteOff.ptr, align 1
   %bitOff = shl i512 %byteOff, 3
@@ -15993,3774 +13714,3774 @@ define void @lshr_64bytes_qwordOff(ptr %src.ptr, ptr %qwordOff.ptr, ptr %dst) no
 }
 
 define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
-; FALLBACK0-LABEL: shl_64bytes:
-; FALLBACK0:       # %bb.0:
-; FALLBACK0-NEXT:    pushq %r15
-; FALLBACK0-NEXT:    pushq %r14
-; FALLBACK0-NEXT:    pushq %r13
-; FALLBACK0-NEXT:    pushq %r12
-; FALLBACK0-NEXT:    pushq %rbx
-; FALLBACK0-NEXT:    movq (%rdi), %rax
-; FALLBACK0-NEXT:    movq 8(%rdi), %rcx
-; FALLBACK0-NEXT:    movq 16(%rdi), %r8
-; FALLBACK0-NEXT:    movq 24(%rdi), %r9
-; FALLBACK0-NEXT:    movq 32(%rdi), %r10
-; FALLBACK0-NEXT:    movq 40(%rdi), %r11
-; FALLBACK0-NEXT:    movq 48(%rdi), %rbx
-; FALLBACK0-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK0-NEXT:    movl (%rsi), %esi
-; FALLBACK0-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK0-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK0-NEXT:    andl $56, %eax
-; FALLBACK0-NEXT:    andl $56, %esi
-; FALLBACK0-NEXT:    negl %esi
-; FALLBACK0-NEXT:    movslq %esi, %rbx
-; FALLBACK0-NEXT:    movq -64(%rsp,%rbx), %r8
-; FALLBACK0-NEXT:    movq -56(%rsp,%rbx), %rdi
-; FALLBACK0-NEXT:    movq %rdi, %r10
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r10
-; FALLBACK0-NEXT:    movl %eax, %esi
-; FALLBACK0-NEXT:    notb %sil
-; FALLBACK0-NEXT:    movq %r8, %r9
-; FALLBACK0-NEXT:    shrq %r9
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r9
-; FALLBACK0-NEXT:    orq %r10, %r9
-; FALLBACK0-NEXT:    movq -40(%rsp,%rbx), %r10
-; FALLBACK0-NEXT:    movq %r10, %r14
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r14
-; FALLBACK0-NEXT:    movq -48(%rsp,%rbx), %r15
-; FALLBACK0-NEXT:    movq %r15, %r11
-; FALLBACK0-NEXT:    shrq %r11
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r11
-; FALLBACK0-NEXT:    orq %r14, %r11
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r15
-; FALLBACK0-NEXT:    shrq %rdi
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %rdi
-; FALLBACK0-NEXT:    orq %r15, %rdi
-; FALLBACK0-NEXT:    movq -24(%rsp,%rbx), %r14
-; FALLBACK0-NEXT:    movq %r14, %r12
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r12
-; FALLBACK0-NEXT:    movq -32(%rsp,%rbx), %r13
-; FALLBACK0-NEXT:    movq %r13, %r15
-; FALLBACK0-NEXT:    shrq %r15
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r15
-; FALLBACK0-NEXT:    orq %r12, %r15
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r13
-; FALLBACK0-NEXT:    shrq %r10
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r10
-; FALLBACK0-NEXT:    orq %r13, %r10
-; FALLBACK0-NEXT:    movq -8(%rsp,%rbx), %r12
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r12
-; FALLBACK0-NEXT:    movq -16(%rsp,%rbx), %rbx
-; FALLBACK0-NEXT:    movq %rbx, %r13
-; FALLBACK0-NEXT:    shrq %r13
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r13
-; FALLBACK0-NEXT:    orq %r12, %r13
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %rbx
-; FALLBACK0-NEXT:    shrq %r14
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r14
-; FALLBACK0-NEXT:    orq %rbx, %r14
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r8
-; FALLBACK0-NEXT:    movq %r8, (%rdx)
-; FALLBACK0-NEXT:    movq %r14, 48(%rdx)
-; FALLBACK0-NEXT:    movq %r13, 56(%rdx)
-; FALLBACK0-NEXT:    movq %r10, 32(%rdx)
-; FALLBACK0-NEXT:    movq %r15, 40(%rdx)
-; FALLBACK0-NEXT:    movq %rdi, 16(%rdx)
-; FALLBACK0-NEXT:    movq %r11, 24(%rdx)
-; FALLBACK0-NEXT:    movq %r9, 8(%rdx)
-; FALLBACK0-NEXT:    popq %rbx
-; FALLBACK0-NEXT:    popq %r12
-; FALLBACK0-NEXT:    popq %r13
-; FALLBACK0-NEXT:    popq %r14
-; FALLBACK0-NEXT:    popq %r15
-; FALLBACK0-NEXT:    retq
-;
-; FALLBACK1-LABEL: shl_64bytes:
-; FALLBACK1:       # %bb.0:
-; FALLBACK1-NEXT:    pushq %r14
-; FALLBACK1-NEXT:    pushq %rbx
-; FALLBACK1-NEXT:    pushq %rax
-; FALLBACK1-NEXT:    movq (%rdi), %rax
-; FALLBACK1-NEXT:    movq 8(%rdi), %rcx
-; FALLBACK1-NEXT:    movq 16(%rdi), %r8
-; FALLBACK1-NEXT:    movq 24(%rdi), %r9
-; FALLBACK1-NEXT:    movq 32(%rdi), %r10
-; FALLBACK1-NEXT:    movq 40(%rdi), %r11
-; FALLBACK1-NEXT:    movq 48(%rdi), %rbx
-; FALLBACK1-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK1-NEXT:    movl (%rsi), %esi
-; FALLBACK1-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK1-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    leal (,%rsi,8), %ecx
-; FALLBACK1-NEXT:    andl $56, %ecx
-; FALLBACK1-NEXT:    andl $56, %esi
-; FALLBACK1-NEXT:    negl %esi
-; FALLBACK1-NEXT:    movslq %esi, %r9
-; FALLBACK1-NEXT:    movq -48(%rsp,%r9), %rax
-; FALLBACK1-NEXT:    movq -40(%rsp,%r9), %r10
-; FALLBACK1-NEXT:    movq %r10, %rsi
-; FALLBACK1-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK1-NEXT:    movq -64(%rsp,%r9), %r8
-; FALLBACK1-NEXT:    movq -56(%rsp,%r9), %rdi
-; FALLBACK1-NEXT:    shldq %cl, %rdi, %rax
-; FALLBACK1-NEXT:    movq -32(%rsp,%r9), %r11
-; FALLBACK1-NEXT:    movq -24(%rsp,%r9), %rbx
-; FALLBACK1-NEXT:    movq %rbx, %r14
-; FALLBACK1-NEXT:    shldq %cl, %r11, %r14
-; FALLBACK1-NEXT:    shldq %cl, %r10, %r11
-; FALLBACK1-NEXT:    movq -16(%rsp,%r9), %r10
-; FALLBACK1-NEXT:    movq -8(%rsp,%r9), %r9
-; FALLBACK1-NEXT:    shldq %cl, %r10, %r9
-; FALLBACK1-NEXT:    shldq %cl, %rbx, %r10
-; FALLBACK1-NEXT:    shldq %cl, %r8, %rdi
-; FALLBACK1-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK1-NEXT:    shlq %cl, %r8
-; FALLBACK1-NEXT:    movq %r10, 48(%rdx)
-; FALLBACK1-NEXT:    movq %r9, 56(%rdx)
-; FALLBACK1-NEXT:    movq %r11, 32(%rdx)
-; FALLBACK1-NEXT:    movq %r14, 40(%rdx)
-; FALLBACK1-NEXT:    movq %rax, 16(%rdx)
-; FALLBACK1-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK1-NEXT:    movq %r8, (%rdx)
-; FALLBACK1-NEXT:    movq %rdi, 8(%rdx)
-; FALLBACK1-NEXT:    addq $8, %rsp
-; FALLBACK1-NEXT:    popq %rbx
-; FALLBACK1-NEXT:    popq %r14
-; FALLBACK1-NEXT:    retq
-;
-; FALLBACK2-LABEL: shl_64bytes:
-; FALLBACK2:       # %bb.0:
-; FALLBACK2-NEXT:    pushq %r15
-; FALLBACK2-NEXT:    pushq %r14
-; FALLBACK2-NEXT:    pushq %r12
-; FALLBACK2-NEXT:    pushq %rbx
-; FALLBACK2-NEXT:    pushq %rax
-; FALLBACK2-NEXT:    movq (%rdi), %rax
-; FALLBACK2-NEXT:    movq 8(%rdi), %rcx
-; FALLBACK2-NEXT:    movq 16(%rdi), %r8
-; FALLBACK2-NEXT:    movq 24(%rdi), %r9
-; FALLBACK2-NEXT:    movq 32(%rdi), %r10
-; FALLBACK2-NEXT:    movq 40(%rdi), %r11
-; FALLBACK2-NEXT:    movq 48(%rdi), %rbx
-; FALLBACK2-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK2-NEXT:    movl (%rsi), %esi
-; FALLBACK2-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK2-NEXT:    andl $56, %eax
-; FALLBACK2-NEXT:    movl %eax, %ecx
-; FALLBACK2-NEXT:    andl $56, %esi
-; FALLBACK2-NEXT:    negl %esi
-; FALLBACK2-NEXT:    movslq %esi, %rsi
-; FALLBACK2-NEXT:    movq -64(%rsp,%rsi), %r9
-; FALLBACK2-NEXT:    movq -56(%rsp,%rsi), %rdi
-; FALLBACK2-NEXT:    shlxq %rcx, %rdi, %r8
-; FALLBACK2-NEXT:    notb %al
-; FALLBACK2-NEXT:    shlxq %rcx, %r9, %r10
-; FALLBACK2-NEXT:    shrq %r9
-; FALLBACK2-NEXT:    shrxq %rax, %r9, %r9
-; FALLBACK2-NEXT:    orq %r8, %r9
-; FALLBACK2-NEXT:    movq -40(%rsp,%rsi), %r11
-; FALLBACK2-NEXT:    shlxq %rcx, %r11, %rbx
-; FALLBACK2-NEXT:    movq -48(%rsp,%rsi), %r8
-; FALLBACK2-NEXT:    shlxq %rcx, %r8, %r14
-; FALLBACK2-NEXT:    shrq %r8
-; FALLBACK2-NEXT:    shrxq %rax, %r8, %r8
-; FALLBACK2-NEXT:    orq %rbx, %r8
-; FALLBACK2-NEXT:    shrq %rdi
-; FALLBACK2-NEXT:    shrxq %rax, %rdi, %rdi
-; FALLBACK2-NEXT:    orq %r14, %rdi
-; FALLBACK2-NEXT:    movq -24(%rsp,%rsi), %rbx
-; FALLBACK2-NEXT:    shlxq %rcx, %rbx, %r14
-; FALLBACK2-NEXT:    movq -32(%rsp,%rsi), %r15
-; FALLBACK2-NEXT:    shlxq %rcx, %r15, %r12
-; FALLBACK2-NEXT:    shrq %r15
-; FALLBACK2-NEXT:    shrxq %rax, %r15, %r15
-; FALLBACK2-NEXT:    orq %r14, %r15
-; FALLBACK2-NEXT:    shrq %r11
-; FALLBACK2-NEXT:    shrxq %rax, %r11, %r11
-; FALLBACK2-NEXT:    orq %r12, %r11
-; FALLBACK2-NEXT:    shlxq %rcx, -8(%rsp,%rsi), %r14
-; FALLBACK2-NEXT:    movq -16(%rsp,%rsi), %rsi
-; FALLBACK2-NEXT:    shlxq %rcx, %rsi, %rcx
-; FALLBACK2-NEXT:    shrq %rsi
-; FALLBACK2-NEXT:    shrxq %rax, %rsi, %rsi
-; FALLBACK2-NEXT:    orq %r14, %rsi
-; FALLBACK2-NEXT:    shrq %rbx
-; FALLBACK2-NEXT:    shrxq %rax, %rbx, %rax
-; FALLBACK2-NEXT:    orq %rcx, %rax
-; FALLBACK2-NEXT:    movq %r10, (%rdx)
-; FALLBACK2-NEXT:    movq %rax, 48(%rdx)
-; FALLBACK2-NEXT:    movq %rsi, 56(%rdx)
-; FALLBACK2-NEXT:    movq %r11, 32(%rdx)
-; FALLBACK2-NEXT:    movq %r15, 40(%rdx)
-; FALLBACK2-NEXT:    movq %rdi, 16(%rdx)
-; FALLBACK2-NEXT:    movq %r8, 24(%rdx)
-; FALLBACK2-NEXT:    movq %r9, 8(%rdx)
-; FALLBACK2-NEXT:    addq $8, %rsp
-; FALLBACK2-NEXT:    popq %rbx
-; FALLBACK2-NEXT:    popq %r12
-; FALLBACK2-NEXT:    popq %r14
-; FALLBACK2-NEXT:    popq %r15
-; FALLBACK2-NEXT:    retq
-;
-; FALLBACK3-LABEL: shl_64bytes:
-; FALLBACK3:       # %bb.0:
-; FALLBACK3-NEXT:    pushq %r14
-; FALLBACK3-NEXT:    pushq %rbx
-; FALLBACK3-NEXT:    pushq %rax
-; FALLBACK3-NEXT:    movq (%rdi), %rax
-; FALLBACK3-NEXT:    movq 8(%rdi), %rcx
-; FALLBACK3-NEXT:    movq 16(%rdi), %r8
-; FALLBACK3-NEXT:    movq 24(%rdi), %r9
-; FALLBACK3-NEXT:    movq 32(%rdi), %r10
-; FALLBACK3-NEXT:    movq 40(%rdi), %r11
-; FALLBACK3-NEXT:    movq 48(%rdi), %rbx
-; FALLBACK3-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK3-NEXT:    movl (%rsi), %esi
-; FALLBACK3-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    leal (,%rsi,8), %ecx
-; FALLBACK3-NEXT:    andl $56, %ecx
-; FALLBACK3-NEXT:    andl $56, %esi
-; FALLBACK3-NEXT:    negl %esi
-; FALLBACK3-NEXT:    movslq %esi, %r8
-; FALLBACK3-NEXT:    movq -48(%rsp,%r8), %rax
-; FALLBACK3-NEXT:    movq -40(%rsp,%r8), %r9
-; FALLBACK3-NEXT:    movq %r9, %rsi
-; FALLBACK3-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK3-NEXT:    movq -64(%rsp,%r8), %r10
-; FALLBACK3-NEXT:    movq -56(%rsp,%r8), %rdi
-; FALLBACK3-NEXT:    shldq %cl, %rdi, %rax
-; FALLBACK3-NEXT:    movq -32(%rsp,%r8), %r11
-; FALLBACK3-NEXT:    movq -24(%rsp,%r8), %rbx
-; FALLBACK3-NEXT:    movq %rbx, %r14
-; FALLBACK3-NEXT:    shldq %cl, %r11, %r14
-; FALLBACK3-NEXT:    shldq %cl, %r9, %r11
-; FALLBACK3-NEXT:    movq -16(%rsp,%r8), %r9
-; FALLBACK3-NEXT:    movq -8(%rsp,%r8), %r8
-; FALLBACK3-NEXT:    shldq %cl, %r9, %r8
-; FALLBACK3-NEXT:    shldq %cl, %rbx, %r9
-; FALLBACK3-NEXT:    shldq %cl, %r10, %rdi
-; FALLBACK3-NEXT:    shlxq %rcx, %r10, %rcx
-; FALLBACK3-NEXT:    movq %r9, 48(%rdx)
-; FALLBACK3-NEXT:    movq %r8, 56(%rdx)
-; FALLBACK3-NEXT:    movq %r11, 32(%rdx)
-; FALLBACK3-NEXT:    movq %r14, 40(%rdx)
-; FALLBACK3-NEXT:    movq %rax, 16(%rdx)
-; FALLBACK3-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK3-NEXT:    movq %rcx, (%rdx)
-; FALLBACK3-NEXT:    movq %rdi, 8(%rdx)
-; FALLBACK3-NEXT:    addq $8, %rsp
-; FALLBACK3-NEXT:    popq %rbx
-; FALLBACK3-NEXT:    popq %r14
-; FALLBACK3-NEXT:    retq
-;
-; FALLBACK4-LABEL: shl_64bytes:
-; FALLBACK4:       # %bb.0:
-; FALLBACK4-NEXT:    pushq %r15
-; FALLBACK4-NEXT:    pushq %r14
-; FALLBACK4-NEXT:    pushq %r13
-; FALLBACK4-NEXT:    pushq %r12
-; FALLBACK4-NEXT:    pushq %rbx
-; FALLBACK4-NEXT:    movups (%rdi), %xmm0
-; FALLBACK4-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK4-NEXT:    movups 32(%rdi), %xmm2
-; FALLBACK4-NEXT:    movups 48(%rdi), %xmm3
-; FALLBACK4-NEXT:    movl (%rsi), %ecx
-; FALLBACK4-NEXT:    xorps %xmm4, %xmm4
-; FALLBACK4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm3, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    leal (,%rcx,8), %eax
-; FALLBACK4-NEXT:    andl $56, %eax
-; FALLBACK4-NEXT:    andl $56, %ecx
-; FALLBACK4-NEXT:    negl %ecx
-; FALLBACK4-NEXT:    movslq %ecx, %r9
-; FALLBACK4-NEXT:    movq -24(%rsp,%r9), %rdi
-; FALLBACK4-NEXT:    movq %rdi, %r10
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r10
-; FALLBACK4-NEXT:    movl %eax, %esi
-; FALLBACK4-NEXT:    notb %sil
-; FALLBACK4-NEXT:    movq -32(%rsp,%r9), %r11
-; FALLBACK4-NEXT:    movq %r11, %r8
-; FALLBACK4-NEXT:    shrq %r8
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r8
-; FALLBACK4-NEXT:    orq %r10, %r8
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r11
-; FALLBACK4-NEXT:    movq -40(%rsp,%r9), %rbx
-; FALLBACK4-NEXT:    movq %rbx, %r10
-; FALLBACK4-NEXT:    shrq %r10
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r10
-; FALLBACK4-NEXT:    orq %r11, %r10
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %rbx
-; FALLBACK4-NEXT:    movq -48(%rsp,%r9), %r15
-; FALLBACK4-NEXT:    movq %r15, %r11
-; FALLBACK4-NEXT:    shrq %r11
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r11
-; FALLBACK4-NEXT:    orq %rbx, %r11
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r15
-; FALLBACK4-NEXT:    movq -64(%rsp,%r9), %r14
-; FALLBACK4-NEXT:    movq -56(%rsp,%r9), %r12
-; FALLBACK4-NEXT:    movq %r12, %rbx
-; FALLBACK4-NEXT:    shrq %rbx
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %rbx
-; FALLBACK4-NEXT:    orq %r15, %rbx
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r12
-; FALLBACK4-NEXT:    movq %r14, %r15
-; FALLBACK4-NEXT:    shrq %r15
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r15
-; FALLBACK4-NEXT:    orq %r12, %r15
-; FALLBACK4-NEXT:    movq -16(%rsp,%r9), %r12
-; FALLBACK4-NEXT:    movq %r12, %r13
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r13
-; FALLBACK4-NEXT:    shrq %rdi
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %rdi
-; FALLBACK4-NEXT:    orq %r13, %rdi
-; FALLBACK4-NEXT:    movq -8(%rsp,%r9), %r9
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r9
-; FALLBACK4-NEXT:    shrq %r12
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r12
-; FALLBACK4-NEXT:    orq %r9, %r12
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r14
-; FALLBACK4-NEXT:    movq %r14, (%rdx)
-; FALLBACK4-NEXT:    movq %r12, 56(%rdx)
-; FALLBACK4-NEXT:    movq %rdi, 48(%rdx)
-; FALLBACK4-NEXT:    movq %r15, 8(%rdx)
-; FALLBACK4-NEXT:    movq %rbx, 16(%rdx)
-; FALLBACK4-NEXT:    movq %r11, 24(%rdx)
-; FALLBACK4-NEXT:    movq %r10, 32(%rdx)
-; FALLBACK4-NEXT:    movq %r8, 40(%rdx)
-; FALLBACK4-NEXT:    popq %rbx
-; FALLBACK4-NEXT:    popq %r12
-; FALLBACK4-NEXT:    popq %r13
-; FALLBACK4-NEXT:    popq %r14
-; FALLBACK4-NEXT:    popq %r15
-; FALLBACK4-NEXT:    retq
-;
-; FALLBACK5-LABEL: shl_64bytes:
-; FALLBACK5:       # %bb.0:
-; FALLBACK5-NEXT:    pushq %r15
-; FALLBACK5-NEXT:    pushq %r14
-; FALLBACK5-NEXT:    pushq %rbx
-; FALLBACK5-NEXT:    movups (%rdi), %xmm0
-; FALLBACK5-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK5-NEXT:    movups 32(%rdi), %xmm2
-; FALLBACK5-NEXT:    movups 48(%rdi), %xmm3
-; FALLBACK5-NEXT:    movl (%rsi), %eax
-; FALLBACK5-NEXT:    xorps %xmm4, %xmm4
-; FALLBACK5-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm3, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK5-NEXT:    andl $56, %ecx
-; FALLBACK5-NEXT:    andl $56, %eax
-; FALLBACK5-NEXT:    negl %eax
-; FALLBACK5-NEXT:    movslq %eax, %r8
-; FALLBACK5-NEXT:    movq -32(%rsp,%r8), %rax
-; FALLBACK5-NEXT:    movq -24(%rsp,%r8), %r9
-; FALLBACK5-NEXT:    movq %r9, %rsi
-; FALLBACK5-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK5-NEXT:    movq -40(%rsp,%r8), %rdi
-; FALLBACK5-NEXT:    shldq %cl, %rdi, %rax
-; FALLBACK5-NEXT:    movq -48(%rsp,%r8), %r10
-; FALLBACK5-NEXT:    shldq %cl, %r10, %rdi
-; FALLBACK5-NEXT:    movq -64(%rsp,%r8), %r11
-; FALLBACK5-NEXT:    movq -56(%rsp,%r8), %rbx
-; FALLBACK5-NEXT:    shldq %cl, %rbx, %r10
-; FALLBACK5-NEXT:    movq -16(%rsp,%r8), %r14
-; FALLBACK5-NEXT:    movq %r14, %r15
-; FALLBACK5-NEXT:    shldq %cl, %r9, %r15
-; FALLBACK5-NEXT:    movq -8(%rsp,%r8), %r8
-; FALLBACK5-NEXT:    shldq %cl, %r14, %r8
-; FALLBACK5-NEXT:    movq %r11, %r9
-; FALLBACK5-NEXT:    shlq %cl, %r9
-; FALLBACK5-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK5-NEXT:    shldq %cl, %r11, %rbx
-; FALLBACK5-NEXT:    movq %r8, 56(%rdx)
-; FALLBACK5-NEXT:    movq %r15, 48(%rdx)
-; FALLBACK5-NEXT:    movq %rbx, 8(%rdx)
-; FALLBACK5-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK5-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK5-NEXT:    movq %rax, 32(%rdx)
-; FALLBACK5-NEXT:    movq %rsi, 40(%rdx)
-; FALLBACK5-NEXT:    movq %r9, (%rdx)
-; FALLBACK5-NEXT:    popq %rbx
-; FALLBACK5-NEXT:    popq %r14
-; FALLBACK5-NEXT:    popq %r15
-; FALLBACK5-NEXT:    retq
-;
-; FALLBACK6-LABEL: shl_64bytes:
-; FALLBACK6:       # %bb.0:
-; FALLBACK6-NEXT:    pushq %r15
-; FALLBACK6-NEXT:    pushq %r14
-; FALLBACK6-NEXT:    pushq %r12
-; FALLBACK6-NEXT:    pushq %rbx
-; FALLBACK6-NEXT:    pushq %rax
-; FALLBACK6-NEXT:    movups (%rdi), %xmm0
-; FALLBACK6-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK6-NEXT:    movups 32(%rdi), %xmm2
-; FALLBACK6-NEXT:    movups 48(%rdi), %xmm3
-; FALLBACK6-NEXT:    movl (%rsi), %esi
-; FALLBACK6-NEXT:    xorps %xmm4, %xmm4
-; FALLBACK6-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm3, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK6-NEXT:    andl $56, %eax
-; FALLBACK6-NEXT:    movl %eax, %ecx
-; FALLBACK6-NEXT:    andl $56, %esi
-; FALLBACK6-NEXT:    negl %esi
-; FALLBACK6-NEXT:    movslq %esi, %rsi
-; FALLBACK6-NEXT:    movq -24(%rsp,%rsi), %rdi
-; FALLBACK6-NEXT:    shlxq %rcx, %rdi, %r9
-; FALLBACK6-NEXT:    notb %al
-; FALLBACK6-NEXT:    movq -32(%rsp,%rsi), %r8
-; FALLBACK6-NEXT:    shlxq %rcx, %r8, %r10
-; FALLBACK6-NEXT:    shrq %r8
-; FALLBACK6-NEXT:    shrxq %rax, %r8, %r8
-; FALLBACK6-NEXT:    orq %r9, %r8
-; FALLBACK6-NEXT:    movq -40(%rsp,%rsi), %r9
-; FALLBACK6-NEXT:    shlxq %rcx, %r9, %r11
-; FALLBACK6-NEXT:    shrq %r9
-; FALLBACK6-NEXT:    shrxq %rax, %r9, %r9
-; FALLBACK6-NEXT:    orq %r10, %r9
-; FALLBACK6-NEXT:    movq -48(%rsp,%rsi), %r10
-; FALLBACK6-NEXT:    shlxq %rcx, %r10, %r14
-; FALLBACK6-NEXT:    shrq %r10
-; FALLBACK6-NEXT:    shrxq %rax, %r10, %r10
-; FALLBACK6-NEXT:    orq %r11, %r10
-; FALLBACK6-NEXT:    movq -64(%rsp,%rsi), %rbx
-; FALLBACK6-NEXT:    movq -56(%rsp,%rsi), %r11
-; FALLBACK6-NEXT:    shlxq %rcx, %r11, %r15
-; FALLBACK6-NEXT:    shrq %r11
-; FALLBACK6-NEXT:    shrxq %rax, %r11, %r11
-; FALLBACK6-NEXT:    orq %r14, %r11
-; FALLBACK6-NEXT:    shlxq %rcx, %rbx, %r14
-; FALLBACK6-NEXT:    shrq %rbx
-; FALLBACK6-NEXT:    shrxq %rax, %rbx, %rbx
-; FALLBACK6-NEXT:    orq %r15, %rbx
-; FALLBACK6-NEXT:    movq -16(%rsp,%rsi), %r15
-; FALLBACK6-NEXT:    shlxq %rcx, %r15, %r12
-; FALLBACK6-NEXT:    shrq %rdi
-; FALLBACK6-NEXT:    shrxq %rax, %rdi, %rdi
-; FALLBACK6-NEXT:    orq %r12, %rdi
-; FALLBACK6-NEXT:    shlxq %rcx, -8(%rsp,%rsi), %rcx
-; FALLBACK6-NEXT:    shrq %r15
-; FALLBACK6-NEXT:    shrxq %rax, %r15, %rax
-; FALLBACK6-NEXT:    orq %rcx, %rax
-; FALLBACK6-NEXT:    movq %r14, (%rdx)
-; FALLBACK6-NEXT:    movq %rax, 56(%rdx)
-; FALLBACK6-NEXT:    movq %rdi, 48(%rdx)
-; FALLBACK6-NEXT:    movq %rbx, 8(%rdx)
-; FALLBACK6-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK6-NEXT:    movq %r10, 24(%rdx)
-; FALLBACK6-NEXT:    movq %r9, 32(%rdx)
-; FALLBACK6-NEXT:    movq %r8, 40(%rdx)
-; FALLBACK6-NEXT:    addq $8, %rsp
-; FALLBACK6-NEXT:    popq %rbx
-; FALLBACK6-NEXT:    popq %r12
-; FALLBACK6-NEXT:    popq %r14
-; FALLBACK6-NEXT:    popq %r15
-; FALLBACK6-NEXT:    retq
-;
-; FALLBACK7-LABEL: shl_64bytes:
-; FALLBACK7:       # %bb.0:
-; FALLBACK7-NEXT:    pushq %r15
-; FALLBACK7-NEXT:    pushq %r14
-; FALLBACK7-NEXT:    pushq %rbx
-; FALLBACK7-NEXT:    movups (%rdi), %xmm0
-; FALLBACK7-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK7-NEXT:    movups 32(%rdi), %xmm2
-; FALLBACK7-NEXT:    movups 48(%rdi), %xmm3
-; FALLBACK7-NEXT:    movl (%rsi), %eax
-; FALLBACK7-NEXT:    xorps %xmm4, %xmm4
-; FALLBACK7-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm3, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK7-NEXT:    andl $56, %ecx
-; FALLBACK7-NEXT:    andl $56, %eax
-; FALLBACK7-NEXT:    negl %eax
-; FALLBACK7-NEXT:    movslq %eax, %r8
-; FALLBACK7-NEXT:    movq -32(%rsp,%r8), %rax
-; FALLBACK7-NEXT:    movq -24(%rsp,%r8), %r9
-; FALLBACK7-NEXT:    movq %r9, %rsi
-; FALLBACK7-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK7-NEXT:    movq -40(%rsp,%r8), %rdi
-; FALLBACK7-NEXT:    shldq %cl, %rdi, %rax
-; FALLBACK7-NEXT:    movq -48(%rsp,%r8), %r10
-; FALLBACK7-NEXT:    shldq %cl, %r10, %rdi
-; FALLBACK7-NEXT:    movq -64(%rsp,%r8), %r11
-; FALLBACK7-NEXT:    movq -56(%rsp,%r8), %rbx
-; FALLBACK7-NEXT:    shldq %cl, %rbx, %r10
-; FALLBACK7-NEXT:    movq -16(%rsp,%r8), %r14
-; FALLBACK7-NEXT:    movq %r14, %r15
-; FALLBACK7-NEXT:    shldq %cl, %r9, %r15
-; FALLBACK7-NEXT:    movq -8(%rsp,%r8), %r8
-; FALLBACK7-NEXT:    shldq %cl, %r14, %r8
-; FALLBACK7-NEXT:    shlxq %rcx, %r11, %r9
-; FALLBACK7-NEXT:    # kill: def $cl killed $cl killed $rcx
-; FALLBACK7-NEXT:    shldq %cl, %r11, %rbx
-; FALLBACK7-NEXT:    movq %r8, 56(%rdx)
-; FALLBACK7-NEXT:    movq %r15, 48(%rdx)
-; FALLBACK7-NEXT:    movq %rbx, 8(%rdx)
-; FALLBACK7-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK7-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK7-NEXT:    movq %rax, 32(%rdx)
-; FALLBACK7-NEXT:    movq %rsi, 40(%rdx)
-; FALLBACK7-NEXT:    movq %r9, (%rdx)
-; FALLBACK7-NEXT:    popq %rbx
-; FALLBACK7-NEXT:    popq %r14
-; FALLBACK7-NEXT:    popq %r15
-; FALLBACK7-NEXT:    retq
-;
-; FALLBACK8-LABEL: shl_64bytes:
-; FALLBACK8:       # %bb.0:
-; FALLBACK8-NEXT:    pushq %r15
-; FALLBACK8-NEXT:    pushq %r14
-; FALLBACK8-NEXT:    pushq %r13
-; FALLBACK8-NEXT:    pushq %r12
-; FALLBACK8-NEXT:    pushq %rbx
-; FALLBACK8-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK8-NEXT:    vmovups 32(%rdi), %ymm1
-; FALLBACK8-NEXT:    movl (%rsi), %ecx
-; FALLBACK8-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; FALLBACK8-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    leal (,%rcx,8), %eax
-; FALLBACK8-NEXT:    andl $56, %eax
-; FALLBACK8-NEXT:    andl $56, %ecx
-; FALLBACK8-NEXT:    negl %ecx
-; FALLBACK8-NEXT:    movslq %ecx, %r9
-; FALLBACK8-NEXT:    movq -24(%rsp,%r9), %rdi
-; FALLBACK8-NEXT:    movq %rdi, %r10
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r10
-; FALLBACK8-NEXT:    movl %eax, %esi
-; FALLBACK8-NEXT:    notb %sil
-; FALLBACK8-NEXT:    movq -32(%rsp,%r9), %r11
-; FALLBACK8-NEXT:    movq %r11, %r8
-; FALLBACK8-NEXT:    shrq %r8
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r8
-; FALLBACK8-NEXT:    orq %r10, %r8
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r11
-; FALLBACK8-NEXT:    movq -40(%rsp,%r9), %rbx
-; FALLBACK8-NEXT:    movq %rbx, %r10
-; FALLBACK8-NEXT:    shrq %r10
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r10
-; FALLBACK8-NEXT:    orq %r11, %r10
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %rbx
-; FALLBACK8-NEXT:    movq -48(%rsp,%r9), %r15
-; FALLBACK8-NEXT:    movq %r15, %r11
-; FALLBACK8-NEXT:    shrq %r11
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r11
-; FALLBACK8-NEXT:    orq %rbx, %r11
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r15
-; FALLBACK8-NEXT:    movq -64(%rsp,%r9), %r14
-; FALLBACK8-NEXT:    movq -56(%rsp,%r9), %r12
-; FALLBACK8-NEXT:    movq %r12, %rbx
-; FALLBACK8-NEXT:    shrq %rbx
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %rbx
-; FALLBACK8-NEXT:    orq %r15, %rbx
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r12
-; FALLBACK8-NEXT:    movq %r14, %r15
-; FALLBACK8-NEXT:    shrq %r15
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r15
-; FALLBACK8-NEXT:    orq %r12, %r15
-; FALLBACK8-NEXT:    movq -16(%rsp,%r9), %r12
-; FALLBACK8-NEXT:    movq %r12, %r13
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r13
-; FALLBACK8-NEXT:    shrq %rdi
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %rdi
-; FALLBACK8-NEXT:    orq %r13, %rdi
-; FALLBACK8-NEXT:    movq -8(%rsp,%r9), %r9
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r9
-; FALLBACK8-NEXT:    shrq %r12
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r12
-; FALLBACK8-NEXT:    orq %r9, %r12
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r14
-; FALLBACK8-NEXT:    movq %r14, (%rdx)
-; FALLBACK8-NEXT:    movq %r12, 56(%rdx)
-; FALLBACK8-NEXT:    movq %rdi, 48(%rdx)
-; FALLBACK8-NEXT:    movq %r15, 8(%rdx)
-; FALLBACK8-NEXT:    movq %rbx, 16(%rdx)
-; FALLBACK8-NEXT:    movq %r11, 24(%rdx)
-; FALLBACK8-NEXT:    movq %r10, 32(%rdx)
-; FALLBACK8-NEXT:    movq %r8, 40(%rdx)
-; FALLBACK8-NEXT:    popq %rbx
-; FALLBACK8-NEXT:    popq %r12
-; FALLBACK8-NEXT:    popq %r13
-; FALLBACK8-NEXT:    popq %r14
-; FALLBACK8-NEXT:    popq %r15
-; FALLBACK8-NEXT:    vzeroupper
-; FALLBACK8-NEXT:    retq
-;
-; FALLBACK9-LABEL: shl_64bytes:
-; FALLBACK9:       # %bb.0:
-; FALLBACK9-NEXT:    pushq %r15
-; FALLBACK9-NEXT:    pushq %r14
-; FALLBACK9-NEXT:    pushq %rbx
-; FALLBACK9-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK9-NEXT:    vmovups 32(%rdi), %ymm1
-; FALLBACK9-NEXT:    movl (%rsi), %eax
-; FALLBACK9-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; FALLBACK9-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK9-NEXT:    andl $56, %ecx
-; FALLBACK9-NEXT:    andl $56, %eax
-; FALLBACK9-NEXT:    negl %eax
-; FALLBACK9-NEXT:    movslq %eax, %r8
-; FALLBACK9-NEXT:    movq -32(%rsp,%r8), %rax
-; FALLBACK9-NEXT:    movq -24(%rsp,%r8), %r9
-; FALLBACK9-NEXT:    movq %r9, %rsi
-; FALLBACK9-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK9-NEXT:    movq -40(%rsp,%r8), %rdi
-; FALLBACK9-NEXT:    shldq %cl, %rdi, %rax
-; FALLBACK9-NEXT:    movq -48(%rsp,%r8), %r10
-; FALLBACK9-NEXT:    shldq %cl, %r10, %rdi
-; FALLBACK9-NEXT:    movq -64(%rsp,%r8), %r11
-; FALLBACK9-NEXT:    movq -56(%rsp,%r8), %rbx
-; FALLBACK9-NEXT:    shldq %cl, %rbx, %r10
-; FALLBACK9-NEXT:    movq -16(%rsp,%r8), %r14
-; FALLBACK9-NEXT:    movq %r14, %r15
-; FALLBACK9-NEXT:    shldq %cl, %r9, %r15
-; FALLBACK9-NEXT:    movq -8(%rsp,%r8), %r8
-; FALLBACK9-NEXT:    shldq %cl, %r14, %r8
-; FALLBACK9-NEXT:    movq %r11, %r9
-; FALLBACK9-NEXT:    shlq %cl, %r9
-; FALLBACK9-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK9-NEXT:    shldq %cl, %r11, %rbx
-; FALLBACK9-NEXT:    movq %r8, 56(%rdx)
-; FALLBACK9-NEXT:    movq %r15, 48(%rdx)
-; FALLBACK9-NEXT:    movq %rbx, 8(%rdx)
-; FALLBACK9-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK9-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK9-NEXT:    movq %rax, 32(%rdx)
-; FALLBACK9-NEXT:    movq %rsi, 40(%rdx)
-; FALLBACK9-NEXT:    movq %r9, (%rdx)
-; FALLBACK9-NEXT:    popq %rbx
-; FALLBACK9-NEXT:    popq %r14
-; FALLBACK9-NEXT:    popq %r15
-; FALLBACK9-NEXT:    vzeroupper
-; FALLBACK9-NEXT:    retq
-;
-; FALLBACK10-LABEL: shl_64bytes:
-; FALLBACK10:       # %bb.0:
-; FALLBACK10-NEXT:    pushq %r15
-; FALLBACK10-NEXT:    pushq %r14
-; FALLBACK10-NEXT:    pushq %r12
-; FALLBACK10-NEXT:    pushq %rbx
-; FALLBACK10-NEXT:    pushq %rax
-; FALLBACK10-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK10-NEXT:    vmovups 32(%rdi), %ymm1
-; FALLBACK10-NEXT:    movl (%rsi), %esi
-; FALLBACK10-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; FALLBACK10-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK10-NEXT:    andl $56, %eax
-; FALLBACK10-NEXT:    movl %eax, %ecx
-; FALLBACK10-NEXT:    andl $56, %esi
-; FALLBACK10-NEXT:    negl %esi
-; FALLBACK10-NEXT:    movslq %esi, %rsi
-; FALLBACK10-NEXT:    movq -24(%rsp,%rsi), %rdi
-; FALLBACK10-NEXT:    shlxq %rcx, %rdi, %r9
-; FALLBACK10-NEXT:    notb %al
-; FALLBACK10-NEXT:    movq -32(%rsp,%rsi), %r8
-; FALLBACK10-NEXT:    shlxq %rcx, %r8, %r10
-; FALLBACK10-NEXT:    shrq %r8
-; FALLBACK10-NEXT:    shrxq %rax, %r8, %r8
-; FALLBACK10-NEXT:    orq %r9, %r8
-; FALLBACK10-NEXT:    movq -40(%rsp,%rsi), %r9
-; FALLBACK10-NEXT:    shlxq %rcx, %r9, %r11
-; FALLBACK10-NEXT:    shrq %r9
-; FALLBACK10-NEXT:    shrxq %rax, %r9, %r9
-; FALLBACK10-NEXT:    orq %r10, %r9
-; FALLBACK10-NEXT:    movq -48(%rsp,%rsi), %r10
-; FALLBACK10-NEXT:    shlxq %rcx, %r10, %r14
-; FALLBACK10-NEXT:    shrq %r10
-; FALLBACK10-NEXT:    shrxq %rax, %r10, %r10
-; FALLBACK10-NEXT:    orq %r11, %r10
-; FALLBACK10-NEXT:    movq -64(%rsp,%rsi), %rbx
-; FALLBACK10-NEXT:    movq -56(%rsp,%rsi), %r11
-; FALLBACK10-NEXT:    shlxq %rcx, %r11, %r15
-; FALLBACK10-NEXT:    shrq %r11
-; FALLBACK10-NEXT:    shrxq %rax, %r11, %r11
-; FALLBACK10-NEXT:    orq %r14, %r11
-; FALLBACK10-NEXT:    shlxq %rcx, %rbx, %r14
-; FALLBACK10-NEXT:    shrq %rbx
-; FALLBACK10-NEXT:    shrxq %rax, %rbx, %rbx
-; FALLBACK10-NEXT:    orq %r15, %rbx
-; FALLBACK10-NEXT:    movq -16(%rsp,%rsi), %r15
-; FALLBACK10-NEXT:    shlxq %rcx, %r15, %r12
-; FALLBACK10-NEXT:    shrq %rdi
-; FALLBACK10-NEXT:    shrxq %rax, %rdi, %rdi
-; FALLBACK10-NEXT:    orq %r12, %rdi
-; FALLBACK10-NEXT:    shlxq %rcx, -8(%rsp,%rsi), %rcx
-; FALLBACK10-NEXT:    shrq %r15
-; FALLBACK10-NEXT:    shrxq %rax, %r15, %rax
-; FALLBACK10-NEXT:    orq %rcx, %rax
-; FALLBACK10-NEXT:    movq %r14, (%rdx)
-; FALLBACK10-NEXT:    movq %rax, 56(%rdx)
-; FALLBACK10-NEXT:    movq %rdi, 48(%rdx)
-; FALLBACK10-NEXT:    movq %rbx, 8(%rdx)
-; FALLBACK10-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK10-NEXT:    movq %r10, 24(%rdx)
-; FALLBACK10-NEXT:    movq %r9, 32(%rdx)
-; FALLBACK10-NEXT:    movq %r8, 40(%rdx)
-; FALLBACK10-NEXT:    addq $8, %rsp
-; FALLBACK10-NEXT:    popq %rbx
-; FALLBACK10-NEXT:    popq %r12
-; FALLBACK10-NEXT:    popq %r14
-; FALLBACK10-NEXT:    popq %r15
-; FALLBACK10-NEXT:    vzeroupper
-; FALLBACK10-NEXT:    retq
-;
-; FALLBACK11-LABEL: shl_64bytes:
-; FALLBACK11:       # %bb.0:
-; FALLBACK11-NEXT:    pushq %r15
-; FALLBACK11-NEXT:    pushq %r14
-; FALLBACK11-NEXT:    pushq %rbx
-; FALLBACK11-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK11-NEXT:    vmovups 32(%rdi), %ymm1
-; FALLBACK11-NEXT:    movl (%rsi), %eax
-; FALLBACK11-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; FALLBACK11-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK11-NEXT:    andl $56, %ecx
-; FALLBACK11-NEXT:    andl $56, %eax
-; FALLBACK11-NEXT:    negl %eax
-; FALLBACK11-NEXT:    movslq %eax, %r8
-; FALLBACK11-NEXT:    movq -32(%rsp,%r8), %rax
-; FALLBACK11-NEXT:    movq -24(%rsp,%r8), %r9
-; FALLBACK11-NEXT:    movq %r9, %rsi
-; FALLBACK11-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK11-NEXT:    movq -40(%rsp,%r8), %rdi
-; FALLBACK11-NEXT:    shldq %cl, %rdi, %rax
-; FALLBACK11-NEXT:    movq -48(%rsp,%r8), %r10
-; FALLBACK11-NEXT:    shldq %cl, %r10, %rdi
-; FALLBACK11-NEXT:    movq -64(%rsp,%r8), %r11
-; FALLBACK11-NEXT:    movq -56(%rsp,%r8), %rbx
-; FALLBACK11-NEXT:    shldq %cl, %rbx, %r10
-; FALLBACK11-NEXT:    movq -16(%rsp,%r8), %r14
-; FALLBACK11-NEXT:    movq %r14, %r15
-; FALLBACK11-NEXT:    shldq %cl, %r9, %r15
-; FALLBACK11-NEXT:    movq -8(%rsp,%r8), %r8
-; FALLBACK11-NEXT:    shldq %cl, %r14, %r8
-; FALLBACK11-NEXT:    shlxq %rcx, %r11, %r9
-; FALLBACK11-NEXT:    # kill: def $cl killed $cl killed $rcx
-; FALLBACK11-NEXT:    shldq %cl, %r11, %rbx
-; FALLBACK11-NEXT:    movq %r8, 56(%rdx)
-; FALLBACK11-NEXT:    movq %r15, 48(%rdx)
-; FALLBACK11-NEXT:    movq %rbx, 8(%rdx)
-; FALLBACK11-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK11-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK11-NEXT:    movq %rax, 32(%rdx)
-; FALLBACK11-NEXT:    movq %rsi, 40(%rdx)
-; FALLBACK11-NEXT:    movq %r9, (%rdx)
-; FALLBACK11-NEXT:    popq %rbx
-; FALLBACK11-NEXT:    popq %r14
-; FALLBACK11-NEXT:    popq %r15
-; FALLBACK11-NEXT:    vzeroupper
-; FALLBACK11-NEXT:    retq
-;
-; FALLBACK12-LABEL: shl_64bytes:
-; FALLBACK12:       # %bb.0:
-; FALLBACK12-NEXT:    pushq %r15
-; FALLBACK12-NEXT:    pushq %r14
-; FALLBACK12-NEXT:    pushq %r13
-; FALLBACK12-NEXT:    pushq %r12
-; FALLBACK12-NEXT:    pushq %rbx
-; FALLBACK12-NEXT:    vmovups (%rdi), %zmm0
-; FALLBACK12-NEXT:    movl (%rsi), %ecx
-; FALLBACK12-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK12-NEXT:    vmovups %zmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    vmovups %zmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    leal (,%rcx,8), %eax
-; FALLBACK12-NEXT:    andl $56, %eax
-; FALLBACK12-NEXT:    andl $56, %ecx
-; FALLBACK12-NEXT:    negl %ecx
-; FALLBACK12-NEXT:    movslq %ecx, %r9
-; FALLBACK12-NEXT:    movq -24(%rsp,%r9), %rdi
-; FALLBACK12-NEXT:    movq %rdi, %r10
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r10
-; FALLBACK12-NEXT:    movl %eax, %esi
-; FALLBACK12-NEXT:    notb %sil
-; FALLBACK12-NEXT:    movq -32(%rsp,%r9), %r11
-; FALLBACK12-NEXT:    movq %r11, %r8
-; FALLBACK12-NEXT:    shrq %r8
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r8
-; FALLBACK12-NEXT:    orq %r10, %r8
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r11
-; FALLBACK12-NEXT:    movq -40(%rsp,%r9), %rbx
-; FALLBACK12-NEXT:    movq %rbx, %r10
-; FALLBACK12-NEXT:    shrq %r10
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r10
-; FALLBACK12-NEXT:    orq %r11, %r10
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %rbx
-; FALLBACK12-NEXT:    movq -48(%rsp,%r9), %r15
-; FALLBACK12-NEXT:    movq %r15, %r11
-; FALLBACK12-NEXT:    shrq %r11
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r11
-; FALLBACK12-NEXT:    orq %rbx, %r11
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r15
-; FALLBACK12-NEXT:    movq -64(%rsp,%r9), %r14
-; FALLBACK12-NEXT:    movq -56(%rsp,%r9), %r12
-; FALLBACK12-NEXT:    movq %r12, %rbx
-; FALLBACK12-NEXT:    shrq %rbx
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %rbx
-; FALLBACK12-NEXT:    orq %r15, %rbx
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r12
-; FALLBACK12-NEXT:    movq %r14, %r15
-; FALLBACK12-NEXT:    shrq %r15
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r15
-; FALLBACK12-NEXT:    orq %r12, %r15
-; FALLBACK12-NEXT:    movq -16(%rsp,%r9), %r12
-; FALLBACK12-NEXT:    movq %r12, %r13
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r13
-; FALLBACK12-NEXT:    shrq %rdi
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %rdi
-; FALLBACK12-NEXT:    orq %r13, %rdi
-; FALLBACK12-NEXT:    movq -8(%rsp,%r9), %r9
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r9
-; FALLBACK12-NEXT:    shrq %r12
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r12
-; FALLBACK12-NEXT:    orq %r9, %r12
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r14
-; FALLBACK12-NEXT:    movq %r14, (%rdx)
-; FALLBACK12-NEXT:    movq %r12, 56(%rdx)
-; FALLBACK12-NEXT:    movq %rdi, 48(%rdx)
-; FALLBACK12-NEXT:    movq %r15, 8(%rdx)
-; FALLBACK12-NEXT:    movq %rbx, 16(%rdx)
-; FALLBACK12-NEXT:    movq %r11, 24(%rdx)
-; FALLBACK12-NEXT:    movq %r10, 32(%rdx)
-; FALLBACK12-NEXT:    movq %r8, 40(%rdx)
-; FALLBACK12-NEXT:    popq %rbx
-; FALLBACK12-NEXT:    popq %r12
-; FALLBACK12-NEXT:    popq %r13
-; FALLBACK12-NEXT:    popq %r14
-; FALLBACK12-NEXT:    popq %r15
-; FALLBACK12-NEXT:    vzeroupper
-; FALLBACK12-NEXT:    retq
-;
-; FALLBACK13-LABEL: shl_64bytes:
-; FALLBACK13:       # %bb.0:
-; FALLBACK13-NEXT:    pushq %r15
-; FALLBACK13-NEXT:    pushq %r14
-; FALLBACK13-NEXT:    pushq %rbx
-; FALLBACK13-NEXT:    vmovups (%rdi), %zmm0
-; FALLBACK13-NEXT:    movl (%rsi), %eax
-; FALLBACK13-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK13-NEXT:    vmovups %zmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    vmovups %zmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK13-NEXT:    andl $56, %ecx
-; FALLBACK13-NEXT:    andl $56, %eax
-; FALLBACK13-NEXT:    negl %eax
-; FALLBACK13-NEXT:    movslq %eax, %r8
-; FALLBACK13-NEXT:    movq -32(%rsp,%r8), %rax
-; FALLBACK13-NEXT:    movq -24(%rsp,%r8), %r9
-; FALLBACK13-NEXT:    movq %r9, %rsi
-; FALLBACK13-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK13-NEXT:    movq -40(%rsp,%r8), %rdi
-; FALLBACK13-NEXT:    shldq %cl, %rdi, %rax
-; FALLBACK13-NEXT:    movq -48(%rsp,%r8), %r10
-; FALLBACK13-NEXT:    shldq %cl, %r10, %rdi
-; FALLBACK13-NEXT:    movq -64(%rsp,%r8), %r11
-; FALLBACK13-NEXT:    movq -56(%rsp,%r8), %rbx
-; FALLBACK13-NEXT:    shldq %cl, %rbx, %r10
-; FALLBACK13-NEXT:    movq -16(%rsp,%r8), %r14
-; FALLBACK13-NEXT:    movq %r14, %r15
-; FALLBACK13-NEXT:    shldq %cl, %r9, %r15
-; FALLBACK13-NEXT:    movq -8(%rsp,%r8), %r8
-; FALLBACK13-NEXT:    shldq %cl, %r14, %r8
-; FALLBACK13-NEXT:    movq %r11, %r9
-; FALLBACK13-NEXT:    shlq %cl, %r9
-; FALLBACK13-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK13-NEXT:    shldq %cl, %r11, %rbx
-; FALLBACK13-NEXT:    movq %r8, 56(%rdx)
-; FALLBACK13-NEXT:    movq %r15, 48(%rdx)
-; FALLBACK13-NEXT:    movq %rbx, 8(%rdx)
-; FALLBACK13-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK13-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK13-NEXT:    movq %rax, 32(%rdx)
-; FALLBACK13-NEXT:    movq %rsi, 40(%rdx)
-; FALLBACK13-NEXT:    movq %r9, (%rdx)
-; FALLBACK13-NEXT:    popq %rbx
-; FALLBACK13-NEXT:    popq %r14
-; FALLBACK13-NEXT:    popq %r15
-; FALLBACK13-NEXT:    vzeroupper
-; FALLBACK13-NEXT:    retq
-;
-; FALLBACK14-LABEL: shl_64bytes:
-; FALLBACK14:       # %bb.0:
-; FALLBACK14-NEXT:    pushq %r15
-; FALLBACK14-NEXT:    pushq %r14
-; FALLBACK14-NEXT:    pushq %r12
-; FALLBACK14-NEXT:    pushq %rbx
-; FALLBACK14-NEXT:    pushq %rax
-; FALLBACK14-NEXT:    vmovups (%rdi), %zmm0
-; FALLBACK14-NEXT:    movl (%rsi), %esi
-; FALLBACK14-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK14-NEXT:    vmovups %zmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    vmovups %zmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    leal (,%rsi,8), %eax
-; FALLBACK14-NEXT:    andl $56, %eax
-; FALLBACK14-NEXT:    movl %eax, %ecx
-; FALLBACK14-NEXT:    andl $56, %esi
-; FALLBACK14-NEXT:    negl %esi
-; FALLBACK14-NEXT:    movslq %esi, %rsi
-; FALLBACK14-NEXT:    movq -24(%rsp,%rsi), %rdi
-; FALLBACK14-NEXT:    shlxq %rcx, %rdi, %r9
-; FALLBACK14-NEXT:    notb %al
-; FALLBACK14-NEXT:    movq -32(%rsp,%rsi), %r8
-; FALLBACK14-NEXT:    shlxq %rcx, %r8, %r10
-; FALLBACK14-NEXT:    shrq %r8
-; FALLBACK14-NEXT:    shrxq %rax, %r8, %r8
-; FALLBACK14-NEXT:    orq %r9, %r8
-; FALLBACK14-NEXT:    movq -40(%rsp,%rsi), %r9
-; FALLBACK14-NEXT:    shlxq %rcx, %r9, %r11
-; FALLBACK14-NEXT:    shrq %r9
-; FALLBACK14-NEXT:    shrxq %rax, %r9, %r9
-; FALLBACK14-NEXT:    orq %r10, %r9
-; FALLBACK14-NEXT:    movq -48(%rsp,%rsi), %r10
-; FALLBACK14-NEXT:    shlxq %rcx, %r10, %r14
-; FALLBACK14-NEXT:    shrq %r10
-; FALLBACK14-NEXT:    shrxq %rax, %r10, %r10
-; FALLBACK14-NEXT:    orq %r11, %r10
-; FALLBACK14-NEXT:    movq -64(%rsp,%rsi), %rbx
-; FALLBACK14-NEXT:    movq -56(%rsp,%rsi), %r11
-; FALLBACK14-NEXT:    shlxq %rcx, %r11, %r15
-; FALLBACK14-NEXT:    shrq %r11
-; FALLBACK14-NEXT:    shrxq %rax, %r11, %r11
-; FALLBACK14-NEXT:    orq %r14, %r11
-; FALLBACK14-NEXT:    shlxq %rcx, %rbx, %r14
-; FALLBACK14-NEXT:    shrq %rbx
-; FALLBACK14-NEXT:    shrxq %rax, %rbx, %rbx
-; FALLBACK14-NEXT:    orq %r15, %rbx
-; FALLBACK14-NEXT:    movq -16(%rsp,%rsi), %r15
-; FALLBACK14-NEXT:    shlxq %rcx, %r15, %r12
-; FALLBACK14-NEXT:    shrq %rdi
-; FALLBACK14-NEXT:    shrxq %rax, %rdi, %rdi
-; FALLBACK14-NEXT:    orq %r12, %rdi
-; FALLBACK14-NEXT:    shlxq %rcx, -8(%rsp,%rsi), %rcx
-; FALLBACK14-NEXT:    shrq %r15
-; FALLBACK14-NEXT:    shrxq %rax, %r15, %rax
-; FALLBACK14-NEXT:    orq %rcx, %rax
-; FALLBACK14-NEXT:    movq %r14, (%rdx)
-; FALLBACK14-NEXT:    movq %rax, 56(%rdx)
-; FALLBACK14-NEXT:    movq %rdi, 48(%rdx)
-; FALLBACK14-NEXT:    movq %rbx, 8(%rdx)
-; FALLBACK14-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK14-NEXT:    movq %r10, 24(%rdx)
-; FALLBACK14-NEXT:    movq %r9, 32(%rdx)
-; FALLBACK14-NEXT:    movq %r8, 40(%rdx)
-; FALLBACK14-NEXT:    addq $8, %rsp
-; FALLBACK14-NEXT:    popq %rbx
-; FALLBACK14-NEXT:    popq %r12
-; FALLBACK14-NEXT:    popq %r14
-; FALLBACK14-NEXT:    popq %r15
-; FALLBACK14-NEXT:    vzeroupper
-; FALLBACK14-NEXT:    retq
-;
-; FALLBACK15-LABEL: shl_64bytes:
-; FALLBACK15:       # %bb.0:
-; FALLBACK15-NEXT:    pushq %r15
-; FALLBACK15-NEXT:    pushq %r14
-; FALLBACK15-NEXT:    pushq %rbx
-; FALLBACK15-NEXT:    vmovups (%rdi), %zmm0
-; FALLBACK15-NEXT:    movl (%rsi), %eax
-; FALLBACK15-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK15-NEXT:    vmovups %zmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    vmovups %zmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK15-NEXT:    andl $56, %ecx
-; FALLBACK15-NEXT:    andl $56, %eax
-; FALLBACK15-NEXT:    negl %eax
-; FALLBACK15-NEXT:    movslq %eax, %r8
-; FALLBACK15-NEXT:    movq -32(%rsp,%r8), %rax
-; FALLBACK15-NEXT:    movq -24(%rsp,%r8), %r9
-; FALLBACK15-NEXT:    movq %r9, %rsi
-; FALLBACK15-NEXT:    shldq %cl, %rax, %rsi
-; FALLBACK15-NEXT:    movq -40(%rsp,%r8), %rdi
-; FALLBACK15-NEXT:    shldq %cl, %rdi, %rax
-; FALLBACK15-NEXT:    movq -48(%rsp,%r8), %r10
-; FALLBACK15-NEXT:    shldq %cl, %r10, %rdi
-; FALLBACK15-NEXT:    movq -64(%rsp,%r8), %r11
-; FALLBACK15-NEXT:    movq -56(%rsp,%r8), %rbx
-; FALLBACK15-NEXT:    shldq %cl, %rbx, %r10
-; FALLBACK15-NEXT:    movq -16(%rsp,%r8), %r14
-; FALLBACK15-NEXT:    movq %r14, %r15
-; FALLBACK15-NEXT:    shldq %cl, %r9, %r15
-; FALLBACK15-NEXT:    movq -8(%rsp,%r8), %r8
-; FALLBACK15-NEXT:    shldq %cl, %r14, %r8
-; FALLBACK15-NEXT:    shlxq %rcx, %r11, %r9
-; FALLBACK15-NEXT:    # kill: def $cl killed $cl killed $rcx
-; FALLBACK15-NEXT:    shldq %cl, %r11, %rbx
-; FALLBACK15-NEXT:    movq %r8, 56(%rdx)
-; FALLBACK15-NEXT:    movq %r15, 48(%rdx)
-; FALLBACK15-NEXT:    movq %rbx, 8(%rdx)
-; FALLBACK15-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK15-NEXT:    movq %rdi, 24(%rdx)
-; FALLBACK15-NEXT:    movq %rax, 32(%rdx)
-; FALLBACK15-NEXT:    movq %rsi, 40(%rdx)
-; FALLBACK15-NEXT:    movq %r9, (%rdx)
-; FALLBACK15-NEXT:    popq %rbx
-; FALLBACK15-NEXT:    popq %r14
-; FALLBACK15-NEXT:    popq %r15
-; FALLBACK15-NEXT:    vzeroupper
-; FALLBACK15-NEXT:    retq
-;
-; FALLBACK16-LABEL: shl_64bytes:
-; FALLBACK16:       # %bb.0:
-; FALLBACK16-NEXT:    pushl %ebp
-; FALLBACK16-NEXT:    pushl %ebx
-; FALLBACK16-NEXT:    pushl %edi
-; FALLBACK16-NEXT:    pushl %esi
-; FALLBACK16-NEXT:    subl $204, %esp
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK16-NEXT:    movl (%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 4(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 8(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 12(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 16(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 20(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 24(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 28(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 32(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 36(%eax), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 40(%eax), %ebp
-; FALLBACK16-NEXT:    movl 44(%eax), %ebx
-; FALLBACK16-NEXT:    movl 48(%eax), %edi
-; FALLBACK16-NEXT:    movl 52(%eax), %esi
-; FALLBACK16-NEXT:    movl 56(%eax), %edx
-; FALLBACK16-NEXT:    movl 60(%eax), %ecx
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK16-NEXT:    movl (%eax), %eax
-; FALLBACK16-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK16-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %eax, %edx
-; FALLBACK16-NEXT:    andl $60, %edx
-; FALLBACK16-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    leal {{[0-9]+}}(%esp), %ecx
-; FALLBACK16-NEXT:    subl %edx, %ecx
-; FALLBACK16-NEXT:    movl (%ecx), %edi
-; FALLBACK16-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 4(%ecx), %edx
-; FALLBACK16-NEXT:    movl %ecx, %ebp
-; FALLBACK16-NEXT:    shll $3, %eax
-; FALLBACK16-NEXT:    andl $24, %eax
-; FALLBACK16-NEXT:    movl %edx, %esi
-; FALLBACK16-NEXT:    movl %eax, %ecx
-; FALLBACK16-NEXT:    shll %cl, %esi
-; FALLBACK16-NEXT:    shrl %edi
-; FALLBACK16-NEXT:    movb %al, %ch
-; FALLBACK16-NEXT:    notb %ch
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edi
-; FALLBACK16-NEXT:    orl %esi, %edi
-; FALLBACK16-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 12(%ebp), %ebx
-; FALLBACK16-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebx
-; FALLBACK16-NEXT:    movl 8(%ebp), %esi
-; FALLBACK16-NEXT:    movl %ebp, %edi
-; FALLBACK16-NEXT:    movl %esi, %ebp
-; FALLBACK16-NEXT:    shrl %ebp
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shrl %cl, %ebp
-; FALLBACK16-NEXT:    orl %ebx, %ebp
-; FALLBACK16-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shll %cl, %esi
-; FALLBACK16-NEXT:    shrl %edx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edx
-; FALLBACK16-NEXT:    orl %esi, %edx
-; FALLBACK16-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl %edi, %ebp
-; FALLBACK16-NEXT:    movl 20(%edi), %ebx
-; FALLBACK16-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebx
-; FALLBACK16-NEXT:    movl 16(%edi), %esi
-; FALLBACK16-NEXT:    movl %esi, %edx
-; FALLBACK16-NEXT:    shrl %edx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edx
-; FALLBACK16-NEXT:    orl %ebx, %edx
-; FALLBACK16-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shll %cl, %esi
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK16-NEXT:    shrl %edi
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edi
-; FALLBACK16-NEXT:    orl %esi, %edi
-; FALLBACK16-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl %ebp, %edx
-; FALLBACK16-NEXT:    movl 28(%ebp), %ebx
-; FALLBACK16-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebx
-; FALLBACK16-NEXT:    movl 24(%ebp), %esi
-; FALLBACK16-NEXT:    movl %esi, %edi
-; FALLBACK16-NEXT:    shrl %edi
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edi
-; FALLBACK16-NEXT:    orl %ebx, %edi
-; FALLBACK16-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shll %cl, %esi
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; FALLBACK16-NEXT:    shrl %ebp
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shrl %cl, %ebp
-; FALLBACK16-NEXT:    orl %esi, %ebp
-; FALLBACK16-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 36(%edx), %ebx
-; FALLBACK16-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebx
-; FALLBACK16-NEXT:    movl 32(%edx), %esi
-; FALLBACK16-NEXT:    movl %edx, %ebp
-; FALLBACK16-NEXT:    movl %esi, %edi
-; FALLBACK16-NEXT:    shrl %edi
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edi
-; FALLBACK16-NEXT:    orl %ebx, %edi
-; FALLBACK16-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shll %cl, %esi
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK16-NEXT:    shrl %edx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edx
-; FALLBACK16-NEXT:    orl %esi, %edx
-; FALLBACK16-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 44(%ebp), %ebx
-; FALLBACK16-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebx
-; FALLBACK16-NEXT:    movl 40(%ebp), %esi
-; FALLBACK16-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl %esi, %edx
-; FALLBACK16-NEXT:    shrl %edx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edx
-; FALLBACK16-NEXT:    orl %ebx, %edx
-; FALLBACK16-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shll %cl, %esi
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK16-NEXT:    shrl %edx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edx
-; FALLBACK16-NEXT:    orl %esi, %edx
-; FALLBACK16-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 52(%ebp), %esi
-; FALLBACK16-NEXT:    movl %esi, %edi
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shll %cl, %edi
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK16-NEXT:    negl %edx
-; FALLBACK16-NEXT:    movl 176(%esp,%edx), %ebx
-; FALLBACK16-NEXT:    movl %ebx, %ebp
-; FALLBACK16-NEXT:    shrl %ebp
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shrl %cl, %ebp
-; FALLBACK16-NEXT:    orl %edi, %ebp
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebx
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK16-NEXT:    shrl %edx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edx
-; FALLBACK16-NEXT:    orl %ebx, %edx
-; FALLBACK16-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK16-NEXT:    movl 60(%edi), %edx
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shll %cl, %edx
-; FALLBACK16-NEXT:    movl 56(%edi), %ebx
-; FALLBACK16-NEXT:    movl %ebx, %edi
-; FALLBACK16-NEXT:    shrl %edi
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edi
-; FALLBACK16-NEXT:    orl %edx, %edi
-; FALLBACK16-NEXT:    movb %al, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebx
-; FALLBACK16-NEXT:    shrl %esi
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shrl %cl, %esi
-; FALLBACK16-NEXT:    orl %ebx, %esi
-; FALLBACK16-NEXT:    movl %eax, %ecx
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK16-NEXT:    shll %cl, %edx
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK16-NEXT:    movl %edx, (%eax)
-; FALLBACK16-NEXT:    movl %esi, 56(%eax)
-; FALLBACK16-NEXT:    movl %edi, 60(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 48(%eax)
-; FALLBACK16-NEXT:    movl %ebp, 52(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 40(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK16-NEXT:    addl $204, %esp
-; FALLBACK16-NEXT:    popl %esi
-; FALLBACK16-NEXT:    popl %edi
-; FALLBACK16-NEXT:    popl %ebx
-; FALLBACK16-NEXT:    popl %ebp
-; FALLBACK16-NEXT:    retl
-;
-; FALLBACK17-LABEL: shl_64bytes:
-; FALLBACK17:       # %bb.0:
-; FALLBACK17-NEXT:    pushl %ebp
-; FALLBACK17-NEXT:    pushl %ebx
-; FALLBACK17-NEXT:    pushl %edi
-; FALLBACK17-NEXT:    pushl %esi
-; FALLBACK17-NEXT:    subl $188, %esp
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK17-NEXT:    movl (%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 4(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 8(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 12(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 16(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 20(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 24(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 28(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 32(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 36(%ecx), %eax
-; FALLBACK17-NEXT:    movl %eax, (%esp) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 40(%ecx), %ebp
-; FALLBACK17-NEXT:    movl 44(%ecx), %ebx
-; FALLBACK17-NEXT:    movl 48(%ecx), %edi
-; FALLBACK17-NEXT:    movl 52(%ecx), %esi
-; FALLBACK17-NEXT:    movl 56(%ecx), %edx
-; FALLBACK17-NEXT:    movl 60(%ecx), %eax
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK17-NEXT:    movl (%ecx), %ecx
-; FALLBACK17-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK17-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %ecx, %ebp
-; FALLBACK17-NEXT:    andl $60, %ebp
-; FALLBACK17-NEXT:    leal {{[0-9]+}}(%esp), %eax
-; FALLBACK17-NEXT:    subl %ebp, %eax
-; FALLBACK17-NEXT:    movl 8(%eax), %esi
-; FALLBACK17-NEXT:    movl 12(%eax), %edx
-; FALLBACK17-NEXT:    shll $3, %ecx
-; FALLBACK17-NEXT:    andl $24, %ecx
-; FALLBACK17-NEXT:    movl %edx, %edi
-; FALLBACK17-NEXT:    shldl %cl, %esi, %edi
-; FALLBACK17-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 4(%eax), %edi
-; FALLBACK17-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shldl %cl, %edi, %esi
-; FALLBACK17-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 16(%eax), %edi
-; FALLBACK17-NEXT:    movl 20(%eax), %esi
-; FALLBACK17-NEXT:    movl %esi, %ebx
-; FALLBACK17-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK17-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK17-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 24(%eax), %edi
-; FALLBACK17-NEXT:    movl 28(%eax), %edx
-; FALLBACK17-NEXT:    movl %edx, %ebx
-; FALLBACK17-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK17-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shldl %cl, %esi, %edi
-; FALLBACK17-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 32(%eax), %edi
-; FALLBACK17-NEXT:    movl 36(%eax), %esi
-; FALLBACK17-NEXT:    movl %esi, %ebx
-; FALLBACK17-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK17-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK17-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 40(%eax), %edx
-; FALLBACK17-NEXT:    movl 44(%eax), %edi
-; FALLBACK17-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK17-NEXT:    movl %edi, (%esp) # 4-byte Spill
-; FALLBACK17-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK17-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 56(%eax), %edx
-; FALLBACK17-NEXT:    movl 60(%eax), %edi
-; FALLBACK17-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK17-NEXT:    movl (%eax), %ebx
-; FALLBACK17-NEXT:    movl 52(%eax), %esi
-; FALLBACK17-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK17-NEXT:    negl %ebp
-; FALLBACK17-NEXT:    movl 160(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK17-NEXT:    movl %edx, 56(%ebp)
-; FALLBACK17-NEXT:    movl %edi, 60(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    shldl %cl, %ebx, %edx
-; FALLBACK17-NEXT:    shll %cl, %ebx
-; FALLBACK17-NEXT:    shldl %cl, %eax, %esi
-; FALLBACK17-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK17-NEXT:    shldl %cl, %edi, %eax
-; FALLBACK17-NEXT:    movl %eax, 48(%ebp)
-; FALLBACK17-NEXT:    movl %esi, 52(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 40(%ebp)
-; FALLBACK17-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK17-NEXT:    movl %ebx, (%ebp)
-; FALLBACK17-NEXT:    movl %edx, 4(%ebp)
-; FALLBACK17-NEXT:    addl $188, %esp
-; FALLBACK17-NEXT:    popl %esi
-; FALLBACK17-NEXT:    popl %edi
-; FALLBACK17-NEXT:    popl %ebx
-; FALLBACK17-NEXT:    popl %ebp
-; FALLBACK17-NEXT:    retl
-;
-; FALLBACK18-LABEL: shl_64bytes:
-; FALLBACK18:       # %bb.0:
-; FALLBACK18-NEXT:    pushl %ebp
-; FALLBACK18-NEXT:    pushl %ebx
-; FALLBACK18-NEXT:    pushl %edi
-; FALLBACK18-NEXT:    pushl %esi
-; FALLBACK18-NEXT:    subl $204, %esp
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK18-NEXT:    movl (%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 4(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 8(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 12(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 16(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 20(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 24(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 28(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 32(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 36(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 40(%eax), %ebx
-; FALLBACK18-NEXT:    movl 44(%eax), %edi
-; FALLBACK18-NEXT:    movl 48(%eax), %esi
-; FALLBACK18-NEXT:    movl 52(%eax), %edx
-; FALLBACK18-NEXT:    movl 56(%eax), %ecx
-; FALLBACK18-NEXT:    movl 60(%eax), %eax
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK18-NEXT:    movl (%ebp), %ebp
-; FALLBACK18-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK18-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    leal (,%ebp,8), %ebx
-; FALLBACK18-NEXT:    andl $24, %ebx
-; FALLBACK18-NEXT:    movl %ebx, %eax
-; FALLBACK18-NEXT:    andl $60, %ebp
-; FALLBACK18-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    leal {{[0-9]+}}(%esp), %edx
-; FALLBACK18-NEXT:    subl %ebp, %edx
-; FALLBACK18-NEXT:    movl (%edx), %esi
-; FALLBACK18-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 4(%edx), %ecx
-; FALLBACK18-NEXT:    notb %bl
-; FALLBACK18-NEXT:    shrl %esi
-; FALLBACK18-NEXT:    shrxl %ebx, %esi, %edi
-; FALLBACK18-NEXT:    shlxl %eax, %ecx, %esi
-; FALLBACK18-NEXT:    movl %eax, %ebp
-; FALLBACK18-NEXT:    orl %esi, %edi
-; FALLBACK18-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 8(%edx), %esi
-; FALLBACK18-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrl %esi
-; FALLBACK18-NEXT:    shrxl %ebx, %esi, %eax
-; FALLBACK18-NEXT:    movl 12(%edx), %esi
-; FALLBACK18-NEXT:    movl %ebp, %edi
-; FALLBACK18-NEXT:    shlxl %ebp, %esi, %ebp
-; FALLBACK18-NEXT:    orl %ebp, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK18-NEXT:    shrl %ecx
-; FALLBACK18-NEXT:    shrxl %ebx, %ecx, %ecx
-; FALLBACK18-NEXT:    orl %eax, %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 16(%edx), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrl %ecx
-; FALLBACK18-NEXT:    shrxl %ebx, %ecx, %eax
-; FALLBACK18-NEXT:    movl 20(%edx), %ecx
-; FALLBACK18-NEXT:    shlxl %edi, %ecx, %ebp
-; FALLBACK18-NEXT:    orl %ebp, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK18-NEXT:    shrl %esi
-; FALLBACK18-NEXT:    shrxl %ebx, %esi, %eax
-; FALLBACK18-NEXT:    orl %ebp, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 24(%edx), %esi
-; FALLBACK18-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrl %esi
-; FALLBACK18-NEXT:    shrxl %ebx, %esi, %eax
-; FALLBACK18-NEXT:    movl 28(%edx), %esi
-; FALLBACK18-NEXT:    shlxl %edi, %esi, %ebp
-; FALLBACK18-NEXT:    orl %ebp, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK18-NEXT:    shrl %ecx
-; FALLBACK18-NEXT:    shrxl %ebx, %ecx, %eax
-; FALLBACK18-NEXT:    orl %ebp, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 32(%edx), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrl %ecx
-; FALLBACK18-NEXT:    shrxl %ebx, %ecx, %eax
-; FALLBACK18-NEXT:    movl 36(%edx), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shlxl %edi, %ecx, %ebp
-; FALLBACK18-NEXT:    orl %ebp, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK18-NEXT:    movl %edi, %eax
-; FALLBACK18-NEXT:    shrl %esi
-; FALLBACK18-NEXT:    shrxl %ebx, %esi, %esi
-; FALLBACK18-NEXT:    orl %ebp, %esi
-; FALLBACK18-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 40(%edx), %edi
-; FALLBACK18-NEXT:    movl %edi, %esi
-; FALLBACK18-NEXT:    shrl %esi
-; FALLBACK18-NEXT:    shrxl %ebx, %esi, %ecx
-; FALLBACK18-NEXT:    movl 44(%edx), %esi
-; FALLBACK18-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shlxl %eax, %esi, %ebp
-; FALLBACK18-NEXT:    orl %ebp, %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shlxl %eax, %edi, %edi
-; FALLBACK18-NEXT:    movl %eax, %esi
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    shrl %eax
-; FALLBACK18-NEXT:    shrxl %ebx, %eax, %eax
-; FALLBACK18-NEXT:    orl %edi, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 48(%edx), %ebp
-; FALLBACK18-NEXT:    movl %ebp, %edi
-; FALLBACK18-NEXT:    shrl %edi
-; FALLBACK18-NEXT:    shrxl %ebx, %edi, %eax
-; FALLBACK18-NEXT:    movl 52(%edx), %ecx
-; FALLBACK18-NEXT:    shlxl %esi, %ecx, %edi
-; FALLBACK18-NEXT:    orl %edi, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shlxl %esi, %ebp, %edi
-; FALLBACK18-NEXT:    movl %esi, %ebp
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    shrl %eax
-; FALLBACK18-NEXT:    shrxl %ebx, %eax, %esi
-; FALLBACK18-NEXT:    orl %edi, %esi
-; FALLBACK18-NEXT:    movl 56(%edx), %edi
-; FALLBACK18-NEXT:    shrl %ecx
-; FALLBACK18-NEXT:    shrxl %ebx, %ecx, %eax
-; FALLBACK18-NEXT:    shlxl %ebp, %edi, %ecx
-; FALLBACK18-NEXT:    orl %ecx, %eax
-; FALLBACK18-NEXT:    shrl %edi
-; FALLBACK18-NEXT:    shrxl %ebx, %edi, %ecx
-; FALLBACK18-NEXT:    shlxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK18-NEXT:    negl %ebx
-; FALLBACK18-NEXT:    shlxl %ebp, 188(%esp,%ebx), %ebx
-; FALLBACK18-NEXT:    orl %ecx, %ebx
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK18-NEXT:    movl %edi, (%edx)
-; FALLBACK18-NEXT:    movl %eax, 56(%edx)
-; FALLBACK18-NEXT:    movl %ebx, 60(%edx)
-; FALLBACK18-NEXT:    movl %esi, 48(%edx)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 52(%edx)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 40(%edx)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 44(%edx)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 32(%edx)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 36(%edx)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 24(%edx)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 28(%edx)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 16(%edx)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 20(%edx)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 8(%edx)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 12(%edx)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl %eax, 4(%edx)
-; FALLBACK18-NEXT:    addl $204, %esp
-; FALLBACK18-NEXT:    popl %esi
-; FALLBACK18-NEXT:    popl %edi
-; FALLBACK18-NEXT:    popl %ebx
-; FALLBACK18-NEXT:    popl %ebp
-; FALLBACK18-NEXT:    retl
-;
-; FALLBACK19-LABEL: shl_64bytes:
-; FALLBACK19:       # %bb.0:
-; FALLBACK19-NEXT:    pushl %ebp
-; FALLBACK19-NEXT:    pushl %ebx
-; FALLBACK19-NEXT:    pushl %edi
-; FALLBACK19-NEXT:    pushl %esi
-; FALLBACK19-NEXT:    subl $204, %esp
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK19-NEXT:    movl (%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 4(%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 8(%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 12(%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 16(%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 20(%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 24(%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 28(%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 32(%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 36(%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 40(%ebp), %ebx
-; FALLBACK19-NEXT:    movl 44(%ebp), %edi
-; FALLBACK19-NEXT:    movl 48(%ebp), %esi
-; FALLBACK19-NEXT:    movl 52(%ebp), %edx
-; FALLBACK19-NEXT:    movl 56(%ebp), %ecx
-; FALLBACK19-NEXT:    movl 60(%ebp), %eax
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK19-NEXT:    movl (%ebp), %ebp
-; FALLBACK19-NEXT:    xorps %xmm0, %xmm0
-; FALLBACK19-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    leal (,%ebp,8), %ecx
-; FALLBACK19-NEXT:    andl $24, %ecx
-; FALLBACK19-NEXT:    andl $60, %ebp
-; FALLBACK19-NEXT:    leal {{[0-9]+}}(%esp), %eax
-; FALLBACK19-NEXT:    subl %ebp, %eax
-; FALLBACK19-NEXT:    movl 4(%eax), %esi
-; FALLBACK19-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 8(%eax), %edi
-; FALLBACK19-NEXT:    movl 12(%eax), %edx
-; FALLBACK19-NEXT:    movl %edx, %ebx
-; FALLBACK19-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK19-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shldl %cl, %esi, %edi
-; FALLBACK19-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 16(%eax), %edi
-; FALLBACK19-NEXT:    movl 20(%eax), %esi
-; FALLBACK19-NEXT:    movl %esi, %ebx
-; FALLBACK19-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK19-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK19-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 24(%eax), %edi
-; FALLBACK19-NEXT:    movl 28(%eax), %edx
-; FALLBACK19-NEXT:    movl %edx, %ebx
-; FALLBACK19-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK19-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shldl %cl, %esi, %edi
-; FALLBACK19-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 32(%eax), %edi
-; FALLBACK19-NEXT:    movl 36(%eax), %esi
-; FALLBACK19-NEXT:    movl %esi, %ebx
-; FALLBACK19-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK19-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK19-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 40(%eax), %ebx
-; FALLBACK19-NEXT:    movl 44(%eax), %edx
-; FALLBACK19-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shldl %cl, %ebx, %edx
-; FALLBACK19-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shldl %cl, %esi, %ebx
-; FALLBACK19-NEXT:    movl 56(%eax), %edx
-; FALLBACK19-NEXT:    movl 60(%eax), %edi
-; FALLBACK19-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK19-NEXT:    movl (%eax), %esi
-; FALLBACK19-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 52(%eax), %esi
-; FALLBACK19-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK19-NEXT:    negl %ebp
-; FALLBACK19-NEXT:    movl 176(%esp,%ebp), %ebp
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK19-NEXT:    movl %edx, 56(%eax)
-; FALLBACK19-NEXT:    movl %edi, 60(%eax)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    shlxl %ecx, %edx, %edi
-; FALLBACK19-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK19-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK19-NEXT:    shldl %cl, %ebp, %esi
-; FALLBACK19-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    shldl %cl, %edx, %ebp
-; FALLBACK19-NEXT:    movl %ebp, 48(%eax)
-; FALLBACK19-NEXT:    movl %esi, 52(%eax)
-; FALLBACK19-NEXT:    movl %ebx, 40(%eax)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK19-NEXT:    movl %edi, 4(%eax)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %ecx, (%eax)
-; FALLBACK19-NEXT:    addl $204, %esp
-; FALLBACK19-NEXT:    popl %esi
-; FALLBACK19-NEXT:    popl %edi
-; FALLBACK19-NEXT:    popl %ebx
-; FALLBACK19-NEXT:    popl %ebp
-; FALLBACK19-NEXT:    retl
-;
-; FALLBACK20-LABEL: shl_64bytes:
-; FALLBACK20:       # %bb.0:
-; FALLBACK20-NEXT:    pushl %ebp
-; FALLBACK20-NEXT:    pushl %ebx
-; FALLBACK20-NEXT:    pushl %edi
-; FALLBACK20-NEXT:    pushl %esi
-; FALLBACK20-NEXT:    subl $204, %esp
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK20-NEXT:    movups (%ecx), %xmm0
-; FALLBACK20-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK20-NEXT:    movups 32(%ecx), %xmm2
-; FALLBACK20-NEXT:    movups 48(%ecx), %xmm3
-; FALLBACK20-NEXT:    movl (%eax), %eax
-; FALLBACK20-NEXT:    xorps %xmm4, %xmm4
-; FALLBACK20-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm3, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %eax, %edx
-; FALLBACK20-NEXT:    andl $60, %edx
-; FALLBACK20-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    leal {{[0-9]+}}(%esp), %ecx
-; FALLBACK20-NEXT:    subl %edx, %ecx
-; FALLBACK20-NEXT:    movl (%ecx), %edi
-; FALLBACK20-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 4(%ecx), %edx
-; FALLBACK20-NEXT:    movl %ecx, %ebp
-; FALLBACK20-NEXT:    shll $3, %eax
-; FALLBACK20-NEXT:    andl $24, %eax
-; FALLBACK20-NEXT:    movl %edx, %esi
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shll %cl, %esi
-; FALLBACK20-NEXT:    shrl %edi
-; FALLBACK20-NEXT:    movb %al, %ch
-; FALLBACK20-NEXT:    notb %ch
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edi
-; FALLBACK20-NEXT:    orl %esi, %edi
-; FALLBACK20-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 12(%ebp), %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    movl 8(%ebp), %esi
-; FALLBACK20-NEXT:    movl %ebp, %edi
-; FALLBACK20-NEXT:    movl %esi, %ebp
-; FALLBACK20-NEXT:    shrl %ebp
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shrl %cl, %ebp
-; FALLBACK20-NEXT:    orl %ebx, %ebp
-; FALLBACK20-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shll %cl, %esi
-; FALLBACK20-NEXT:    shrl %edx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edx
-; FALLBACK20-NEXT:    orl %esi, %edx
-; FALLBACK20-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl %edi, %ebp
-; FALLBACK20-NEXT:    movl 20(%edi), %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    movl 16(%edi), %esi
-; FALLBACK20-NEXT:    movl %esi, %edx
-; FALLBACK20-NEXT:    shrl %edx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edx
-; FALLBACK20-NEXT:    orl %ebx, %edx
-; FALLBACK20-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shll %cl, %esi
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %edi
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edi
-; FALLBACK20-NEXT:    orl %esi, %edi
-; FALLBACK20-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl %ebp, %edx
-; FALLBACK20-NEXT:    movl 28(%ebp), %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    movl 24(%ebp), %esi
-; FALLBACK20-NEXT:    movl %esi, %edi
-; FALLBACK20-NEXT:    shrl %edi
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edi
-; FALLBACK20-NEXT:    orl %ebx, %edi
-; FALLBACK20-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shll %cl, %esi
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %ebp
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shrl %cl, %ebp
-; FALLBACK20-NEXT:    orl %esi, %ebp
-; FALLBACK20-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 36(%edx), %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    movl 32(%edx), %esi
-; FALLBACK20-NEXT:    movl %edx, %ebp
-; FALLBACK20-NEXT:    movl %esi, %edi
-; FALLBACK20-NEXT:    shrl %edi
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edi
-; FALLBACK20-NEXT:    orl %ebx, %edi
-; FALLBACK20-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shll %cl, %esi
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %edx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edx
-; FALLBACK20-NEXT:    orl %esi, %edx
-; FALLBACK20-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 44(%ebp), %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    movl 40(%ebp), %esi
-; FALLBACK20-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl %esi, %edx
-; FALLBACK20-NEXT:    shrl %edx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edx
-; FALLBACK20-NEXT:    orl %ebx, %edx
-; FALLBACK20-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shll %cl, %esi
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %edx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edx
-; FALLBACK20-NEXT:    orl %esi, %edx
-; FALLBACK20-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 52(%ebp), %esi
-; FALLBACK20-NEXT:    movl %esi, %edi
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shll %cl, %edi
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK20-NEXT:    negl %edx
-; FALLBACK20-NEXT:    movl 176(%esp,%edx), %ebx
-; FALLBACK20-NEXT:    movl %ebx, %ebp
-; FALLBACK20-NEXT:    shrl %ebp
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shrl %cl, %ebp
-; FALLBACK20-NEXT:    orl %edi, %ebp
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %edx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edx
-; FALLBACK20-NEXT:    orl %ebx, %edx
-; FALLBACK20-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK20-NEXT:    movl 60(%edi), %edx
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shll %cl, %edx
-; FALLBACK20-NEXT:    movl 56(%edi), %ebx
-; FALLBACK20-NEXT:    movl %ebx, %edi
-; FALLBACK20-NEXT:    shrl %edi
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edi
-; FALLBACK20-NEXT:    orl %edx, %edi
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    shrl %esi
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shrl %cl, %esi
-; FALLBACK20-NEXT:    orl %ebx, %esi
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK20-NEXT:    shll %cl, %edx
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK20-NEXT:    movl %edx, (%eax)
-; FALLBACK20-NEXT:    movl %esi, 56(%eax)
-; FALLBACK20-NEXT:    movl %edi, 60(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 48(%eax)
-; FALLBACK20-NEXT:    movl %ebp, 52(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 40(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK20-NEXT:    addl $204, %esp
-; FALLBACK20-NEXT:    popl %esi
-; FALLBACK20-NEXT:    popl %edi
-; FALLBACK20-NEXT:    popl %ebx
-; FALLBACK20-NEXT:    popl %ebp
-; FALLBACK20-NEXT:    retl
-;
-; FALLBACK21-LABEL: shl_64bytes:
-; FALLBACK21:       # %bb.0:
-; FALLBACK21-NEXT:    pushl %ebp
-; FALLBACK21-NEXT:    pushl %ebx
-; FALLBACK21-NEXT:    pushl %edi
-; FALLBACK21-NEXT:    pushl %esi
-; FALLBACK21-NEXT:    subl $188, %esp
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK21-NEXT:    movups (%ecx), %xmm0
-; FALLBACK21-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK21-NEXT:    movups 32(%ecx), %xmm2
-; FALLBACK21-NEXT:    movups 48(%ecx), %xmm3
-; FALLBACK21-NEXT:    movl (%eax), %ecx
-; FALLBACK21-NEXT:    xorps %xmm4, %xmm4
-; FALLBACK21-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm3, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %ecx, %ebp
-; FALLBACK21-NEXT:    andl $60, %ebp
-; FALLBACK21-NEXT:    leal {{[0-9]+}}(%esp), %eax
-; FALLBACK21-NEXT:    subl %ebp, %eax
-; FALLBACK21-NEXT:    movl 8(%eax), %esi
-; FALLBACK21-NEXT:    movl 12(%eax), %edx
-; FALLBACK21-NEXT:    shll $3, %ecx
-; FALLBACK21-NEXT:    andl $24, %ecx
-; FALLBACK21-NEXT:    movl %edx, %edi
-; FALLBACK21-NEXT:    shldl %cl, %esi, %edi
-; FALLBACK21-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 4(%eax), %edi
-; FALLBACK21-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shldl %cl, %edi, %esi
-; FALLBACK21-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 16(%eax), %edi
-; FALLBACK21-NEXT:    movl 20(%eax), %esi
-; FALLBACK21-NEXT:    movl %esi, %ebx
-; FALLBACK21-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK21-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK21-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 24(%eax), %edi
-; FALLBACK21-NEXT:    movl 28(%eax), %edx
-; FALLBACK21-NEXT:    movl %edx, %ebx
-; FALLBACK21-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK21-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shldl %cl, %esi, %edi
-; FALLBACK21-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 32(%eax), %edi
-; FALLBACK21-NEXT:    movl 36(%eax), %esi
-; FALLBACK21-NEXT:    movl %esi, %ebx
-; FALLBACK21-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK21-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK21-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 40(%eax), %edx
-; FALLBACK21-NEXT:    movl 44(%eax), %edi
-; FALLBACK21-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK21-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK21-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 56(%eax), %edx
-; FALLBACK21-NEXT:    movl 60(%eax), %edi
-; FALLBACK21-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK21-NEXT:    movl (%eax), %ebx
-; FALLBACK21-NEXT:    movl 52(%eax), %esi
-; FALLBACK21-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK21-NEXT:    negl %ebp
-; FALLBACK21-NEXT:    movl 160(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK21-NEXT:    movl %edx, 56(%ebp)
-; FALLBACK21-NEXT:    movl %edi, 60(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK21-NEXT:    shldl %cl, %ebx, %edx
-; FALLBACK21-NEXT:    shll %cl, %ebx
-; FALLBACK21-NEXT:    shldl %cl, %eax, %esi
-; FALLBACK21-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK21-NEXT:    shldl %cl, %edi, %eax
-; FALLBACK21-NEXT:    movl %eax, 48(%ebp)
-; FALLBACK21-NEXT:    movl %esi, 52(%ebp)
-; FALLBACK21-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 40(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK21-NEXT:    movl %ebx, (%ebp)
-; FALLBACK21-NEXT:    movl %edx, 4(%ebp)
-; FALLBACK21-NEXT:    addl $188, %esp
-; FALLBACK21-NEXT:    popl %esi
-; FALLBACK21-NEXT:    popl %edi
-; FALLBACK21-NEXT:    popl %ebx
-; FALLBACK21-NEXT:    popl %ebp
-; FALLBACK21-NEXT:    retl
-;
-; FALLBACK22-LABEL: shl_64bytes:
-; FALLBACK22:       # %bb.0:
-; FALLBACK22-NEXT:    pushl %ebp
-; FALLBACK22-NEXT:    pushl %ebx
-; FALLBACK22-NEXT:    pushl %edi
-; FALLBACK22-NEXT:    pushl %esi
-; FALLBACK22-NEXT:    subl $204, %esp
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK22-NEXT:    movups (%ecx), %xmm0
-; FALLBACK22-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK22-NEXT:    movups 32(%ecx), %xmm2
-; FALLBACK22-NEXT:    movups 48(%ecx), %xmm3
-; FALLBACK22-NEXT:    movl (%eax), %eax
-; FALLBACK22-NEXT:    xorps %xmm4, %xmm4
-; FALLBACK22-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm3, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    leal (,%eax,8), %ebx
-; FALLBACK22-NEXT:    andl $24, %ebx
-; FALLBACK22-NEXT:    movl %ebx, %ecx
-; FALLBACK22-NEXT:    andl $60, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    leal {{[0-9]+}}(%esp), %edx
-; FALLBACK22-NEXT:    subl %eax, %edx
-; FALLBACK22-NEXT:    movl (%edx), %esi
-; FALLBACK22-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 4(%edx), %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    notb %bl
-; FALLBACK22-NEXT:    shrl %esi
-; FALLBACK22-NEXT:    shrxl %ebx, %esi, %edi
-; FALLBACK22-NEXT:    shlxl %ecx, %eax, %esi
-; FALLBACK22-NEXT:    orl %esi, %edi
-; FALLBACK22-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 8(%edx), %esi
-; FALLBACK22-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrl %esi
-; FALLBACK22-NEXT:    shrxl %ebx, %esi, %eax
-; FALLBACK22-NEXT:    movl 12(%edx), %esi
-; FALLBACK22-NEXT:    shlxl %ecx, %esi, %ebp
-; FALLBACK22-NEXT:    orl %ebp, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl %ecx, %edi
-; FALLBACK22-NEXT:    shlxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    shrl %ecx
-; FALLBACK22-NEXT:    shrxl %ebx, %ecx, %ecx
-; FALLBACK22-NEXT:    orl %eax, %ecx
-; FALLBACK22-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 16(%edx), %ecx
-; FALLBACK22-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrl %ecx
-; FALLBACK22-NEXT:    shrxl %ebx, %ecx, %eax
-; FALLBACK22-NEXT:    movl 20(%edx), %ecx
-; FALLBACK22-NEXT:    shlxl %edi, %ecx, %ebp
-; FALLBACK22-NEXT:    orl %ebp, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK22-NEXT:    shrl %esi
-; FALLBACK22-NEXT:    shrxl %ebx, %esi, %eax
-; FALLBACK22-NEXT:    orl %ebp, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 24(%edx), %esi
-; FALLBACK22-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrl %esi
-; FALLBACK22-NEXT:    shrxl %ebx, %esi, %eax
-; FALLBACK22-NEXT:    movl 28(%edx), %esi
-; FALLBACK22-NEXT:    shlxl %edi, %esi, %ebp
-; FALLBACK22-NEXT:    orl %ebp, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK22-NEXT:    shrl %ecx
-; FALLBACK22-NEXT:    shrxl %ebx, %ecx, %eax
-; FALLBACK22-NEXT:    orl %ebp, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 32(%edx), %ecx
-; FALLBACK22-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrl %ecx
-; FALLBACK22-NEXT:    shrxl %ebx, %ecx, %eax
-; FALLBACK22-NEXT:    movl 36(%edx), %ecx
-; FALLBACK22-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shlxl %edi, %ecx, %ebp
-; FALLBACK22-NEXT:    orl %ebp, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK22-NEXT:    movl %edi, %eax
-; FALLBACK22-NEXT:    shrl %esi
-; FALLBACK22-NEXT:    shrxl %ebx, %esi, %esi
-; FALLBACK22-NEXT:    orl %ebp, %esi
-; FALLBACK22-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 40(%edx), %edi
-; FALLBACK22-NEXT:    movl %edi, %esi
-; FALLBACK22-NEXT:    shrl %esi
-; FALLBACK22-NEXT:    shrxl %ebx, %esi, %ecx
-; FALLBACK22-NEXT:    movl 44(%edx), %esi
-; FALLBACK22-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shlxl %eax, %esi, %ebp
-; FALLBACK22-NEXT:    orl %ebp, %ecx
-; FALLBACK22-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shlxl %eax, %edi, %edi
-; FALLBACK22-NEXT:    movl %eax, %esi
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    shrl %eax
-; FALLBACK22-NEXT:    shrxl %ebx, %eax, %eax
-; FALLBACK22-NEXT:    orl %edi, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 48(%edx), %ebp
-; FALLBACK22-NEXT:    movl %ebp, %edi
-; FALLBACK22-NEXT:    shrl %edi
-; FALLBACK22-NEXT:    shrxl %ebx, %edi, %eax
-; FALLBACK22-NEXT:    movl 52(%edx), %ecx
-; FALLBACK22-NEXT:    shlxl %esi, %ecx, %edi
-; FALLBACK22-NEXT:    orl %edi, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shlxl %esi, %ebp, %edi
-; FALLBACK22-NEXT:    movl %esi, %ebp
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    shrl %eax
-; FALLBACK22-NEXT:    shrxl %ebx, %eax, %esi
-; FALLBACK22-NEXT:    orl %edi, %esi
-; FALLBACK22-NEXT:    movl 56(%edx), %edi
-; FALLBACK22-NEXT:    shrl %ecx
-; FALLBACK22-NEXT:    shrxl %ebx, %ecx, %eax
-; FALLBACK22-NEXT:    shlxl %ebp, %edi, %ecx
-; FALLBACK22-NEXT:    orl %ecx, %eax
-; FALLBACK22-NEXT:    shrl %edi
-; FALLBACK22-NEXT:    shrxl %ebx, %edi, %ecx
-; FALLBACK22-NEXT:    shlxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK22-NEXT:    negl %ebx
-; FALLBACK22-NEXT:    shlxl %ebp, 188(%esp,%ebx), %ebx
-; FALLBACK22-NEXT:    orl %ecx, %ebx
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK22-NEXT:    movl %edi, (%edx)
-; FALLBACK22-NEXT:    movl %eax, 56(%edx)
-; FALLBACK22-NEXT:    movl %ebx, 60(%edx)
-; FALLBACK22-NEXT:    movl %esi, 48(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 52(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 40(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 44(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 32(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 36(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 24(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 28(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 16(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 20(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 8(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 12(%edx)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl %eax, 4(%edx)
-; FALLBACK22-NEXT:    addl $204, %esp
-; FALLBACK22-NEXT:    popl %esi
-; FALLBACK22-NEXT:    popl %edi
-; FALLBACK22-NEXT:    popl %ebx
-; FALLBACK22-NEXT:    popl %ebp
-; FALLBACK22-NEXT:    retl
-;
-; FALLBACK23-LABEL: shl_64bytes:
-; FALLBACK23:       # %bb.0:
-; FALLBACK23-NEXT:    pushl %ebp
-; FALLBACK23-NEXT:    pushl %ebx
-; FALLBACK23-NEXT:    pushl %edi
-; FALLBACK23-NEXT:    pushl %esi
-; FALLBACK23-NEXT:    subl $204, %esp
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK23-NEXT:    movups (%ecx), %xmm0
-; FALLBACK23-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK23-NEXT:    movups 32(%ecx), %xmm2
-; FALLBACK23-NEXT:    movups 48(%ecx), %xmm3
-; FALLBACK23-NEXT:    movl (%eax), %ebp
-; FALLBACK23-NEXT:    xorps %xmm4, %xmm4
-; FALLBACK23-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm3, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    leal (,%ebp,8), %ecx
-; FALLBACK23-NEXT:    andl $24, %ecx
-; FALLBACK23-NEXT:    andl $60, %ebp
-; FALLBACK23-NEXT:    leal {{[0-9]+}}(%esp), %eax
-; FALLBACK23-NEXT:    subl %ebp, %eax
-; FALLBACK23-NEXT:    movl 4(%eax), %esi
-; FALLBACK23-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 8(%eax), %edi
-; FALLBACK23-NEXT:    movl 12(%eax), %edx
-; FALLBACK23-NEXT:    movl %edx, %ebx
-; FALLBACK23-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK23-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shldl %cl, %esi, %edi
-; FALLBACK23-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 16(%eax), %edi
-; FALLBACK23-NEXT:    movl 20(%eax), %esi
-; FALLBACK23-NEXT:    movl %esi, %ebx
-; FALLBACK23-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK23-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK23-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 24(%eax), %edi
-; FALLBACK23-NEXT:    movl 28(%eax), %edx
-; FALLBACK23-NEXT:    movl %edx, %ebx
-; FALLBACK23-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK23-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shldl %cl, %esi, %edi
-; FALLBACK23-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 32(%eax), %edi
-; FALLBACK23-NEXT:    movl 36(%eax), %esi
-; FALLBACK23-NEXT:    movl %esi, %ebx
-; FALLBACK23-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK23-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK23-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 40(%eax), %ebx
-; FALLBACK23-NEXT:    movl 44(%eax), %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shldl %cl, %ebx, %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shldl %cl, %esi, %ebx
-; FALLBACK23-NEXT:    movl 56(%eax), %edx
-; FALLBACK23-NEXT:    movl 60(%eax), %edi
-; FALLBACK23-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK23-NEXT:    movl (%eax), %esi
-; FALLBACK23-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 52(%eax), %esi
-; FALLBACK23-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK23-NEXT:    negl %ebp
-; FALLBACK23-NEXT:    movl 176(%esp,%ebp), %ebp
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK23-NEXT:    movl %edx, 56(%eax)
-; FALLBACK23-NEXT:    movl %edi, 60(%eax)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK23-NEXT:    shlxl %ecx, %edx, %edi
-; FALLBACK23-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK23-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK23-NEXT:    shldl %cl, %ebp, %esi
-; FALLBACK23-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK23-NEXT:    shldl %cl, %edx, %ebp
-; FALLBACK23-NEXT:    movl %ebp, 48(%eax)
-; FALLBACK23-NEXT:    movl %esi, 52(%eax)
-; FALLBACK23-NEXT:    movl %ebx, 40(%eax)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK23-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK23-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK23-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK23-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK23-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK23-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK23-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK23-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK23-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK23-NEXT:    movl %edi, 4(%eax)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK23-NEXT:    movl %ecx, (%eax)
-; FALLBACK23-NEXT:    addl $204, %esp
-; FALLBACK23-NEXT:    popl %esi
-; FALLBACK23-NEXT:    popl %edi
-; FALLBACK23-NEXT:    popl %ebx
-; FALLBACK23-NEXT:    popl %ebp
-; FALLBACK23-NEXT:    retl
-;
-; FALLBACK24-LABEL: shl_64bytes:
-; FALLBACK24:       # %bb.0:
-; FALLBACK24-NEXT:    pushl %ebp
-; FALLBACK24-NEXT:    pushl %ebx
-; FALLBACK24-NEXT:    pushl %edi
-; FALLBACK24-NEXT:    pushl %esi
-; FALLBACK24-NEXT:    subl $204, %esp
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK24-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK24-NEXT:    vmovups 32(%ecx), %ymm1
-; FALLBACK24-NEXT:    movl (%eax), %eax
-; FALLBACK24-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; FALLBACK24-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %eax, %edx
-; FALLBACK24-NEXT:    andl $60, %edx
-; FALLBACK24-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    leal {{[0-9]+}}(%esp), %ecx
-; FALLBACK24-NEXT:    subl %edx, %ecx
-; FALLBACK24-NEXT:    movl (%ecx), %edi
-; FALLBACK24-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 4(%ecx), %edx
-; FALLBACK24-NEXT:    movl %ecx, %ebp
-; FALLBACK24-NEXT:    shll $3, %eax
-; FALLBACK24-NEXT:    andl $24, %eax
-; FALLBACK24-NEXT:    movl %edx, %esi
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shll %cl, %esi
-; FALLBACK24-NEXT:    shrl %edi
-; FALLBACK24-NEXT:    movb %al, %ch
-; FALLBACK24-NEXT:    notb %ch
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edi
-; FALLBACK24-NEXT:    orl %esi, %edi
-; FALLBACK24-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 12(%ebp), %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    movl 8(%ebp), %esi
-; FALLBACK24-NEXT:    movl %ebp, %edi
-; FALLBACK24-NEXT:    movl %esi, %ebp
-; FALLBACK24-NEXT:    shrl %ebp
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shrl %cl, %ebp
-; FALLBACK24-NEXT:    orl %ebx, %ebp
-; FALLBACK24-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shll %cl, %esi
-; FALLBACK24-NEXT:    shrl %edx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edx
-; FALLBACK24-NEXT:    orl %esi, %edx
-; FALLBACK24-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl %edi, %ebp
-; FALLBACK24-NEXT:    movl 20(%edi), %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    movl 16(%edi), %esi
-; FALLBACK24-NEXT:    movl %esi, %edx
-; FALLBACK24-NEXT:    shrl %edx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edx
-; FALLBACK24-NEXT:    orl %ebx, %edx
-; FALLBACK24-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shll %cl, %esi
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %edi
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edi
-; FALLBACK24-NEXT:    orl %esi, %edi
-; FALLBACK24-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl %ebp, %edx
-; FALLBACK24-NEXT:    movl 28(%ebp), %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    movl 24(%ebp), %esi
-; FALLBACK24-NEXT:    movl %esi, %edi
-; FALLBACK24-NEXT:    shrl %edi
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edi
-; FALLBACK24-NEXT:    orl %ebx, %edi
-; FALLBACK24-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shll %cl, %esi
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %ebp
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shrl %cl, %ebp
-; FALLBACK24-NEXT:    orl %esi, %ebp
-; FALLBACK24-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 36(%edx), %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    movl 32(%edx), %esi
-; FALLBACK24-NEXT:    movl %edx, %ebp
-; FALLBACK24-NEXT:    movl %esi, %edi
-; FALLBACK24-NEXT:    shrl %edi
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edi
-; FALLBACK24-NEXT:    orl %ebx, %edi
-; FALLBACK24-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shll %cl, %esi
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %edx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edx
-; FALLBACK24-NEXT:    orl %esi, %edx
-; FALLBACK24-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 44(%ebp), %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    movl 40(%ebp), %esi
-; FALLBACK24-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl %esi, %edx
-; FALLBACK24-NEXT:    shrl %edx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edx
-; FALLBACK24-NEXT:    orl %ebx, %edx
-; FALLBACK24-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shll %cl, %esi
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %edx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edx
-; FALLBACK24-NEXT:    orl %esi, %edx
-; FALLBACK24-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 52(%ebp), %esi
-; FALLBACK24-NEXT:    movl %esi, %edi
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shll %cl, %edi
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK24-NEXT:    negl %edx
-; FALLBACK24-NEXT:    movl 176(%esp,%edx), %ebx
-; FALLBACK24-NEXT:    movl %ebx, %ebp
-; FALLBACK24-NEXT:    shrl %ebp
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shrl %cl, %ebp
-; FALLBACK24-NEXT:    orl %edi, %ebp
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %edx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edx
-; FALLBACK24-NEXT:    orl %ebx, %edx
-; FALLBACK24-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK24-NEXT:    movl 60(%edi), %edx
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shll %cl, %edx
-; FALLBACK24-NEXT:    movl 56(%edi), %ebx
-; FALLBACK24-NEXT:    movl %ebx, %edi
-; FALLBACK24-NEXT:    shrl %edi
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edi
-; FALLBACK24-NEXT:    orl %edx, %edi
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    shrl %esi
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shrl %cl, %esi
-; FALLBACK24-NEXT:    orl %ebx, %esi
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK24-NEXT:    shll %cl, %edx
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK24-NEXT:    movl %edx, (%eax)
-; FALLBACK24-NEXT:    movl %esi, 56(%eax)
-; FALLBACK24-NEXT:    movl %edi, 60(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 48(%eax)
-; FALLBACK24-NEXT:    movl %ebp, 52(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 40(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK24-NEXT:    addl $204, %esp
-; FALLBACK24-NEXT:    popl %esi
-; FALLBACK24-NEXT:    popl %edi
-; FALLBACK24-NEXT:    popl %ebx
-; FALLBACK24-NEXT:    popl %ebp
-; FALLBACK24-NEXT:    vzeroupper
-; FALLBACK24-NEXT:    retl
-;
-; FALLBACK25-LABEL: shl_64bytes:
-; FALLBACK25:       # %bb.0:
-; FALLBACK25-NEXT:    pushl %ebp
-; FALLBACK25-NEXT:    pushl %ebx
-; FALLBACK25-NEXT:    pushl %edi
-; FALLBACK25-NEXT:    pushl %esi
-; FALLBACK25-NEXT:    subl $188, %esp
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK25-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK25-NEXT:    vmovups 32(%ecx), %ymm1
-; FALLBACK25-NEXT:    movl (%eax), %ecx
-; FALLBACK25-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; FALLBACK25-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %ecx, %ebp
-; FALLBACK25-NEXT:    andl $60, %ebp
-; FALLBACK25-NEXT:    leal {{[0-9]+}}(%esp), %eax
-; FALLBACK25-NEXT:    subl %ebp, %eax
-; FALLBACK25-NEXT:    movl 8(%eax), %esi
-; FALLBACK25-NEXT:    movl 12(%eax), %edx
-; FALLBACK25-NEXT:    shll $3, %ecx
-; FALLBACK25-NEXT:    andl $24, %ecx
-; FALLBACK25-NEXT:    movl %edx, %edi
-; FALLBACK25-NEXT:    shldl %cl, %esi, %edi
-; FALLBACK25-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 4(%eax), %edi
-; FALLBACK25-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shldl %cl, %edi, %esi
-; FALLBACK25-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 16(%eax), %edi
-; FALLBACK25-NEXT:    movl 20(%eax), %esi
-; FALLBACK25-NEXT:    movl %esi, %ebx
-; FALLBACK25-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK25-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK25-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 24(%eax), %edi
-; FALLBACK25-NEXT:    movl 28(%eax), %edx
-; FALLBACK25-NEXT:    movl %edx, %ebx
-; FALLBACK25-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK25-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shldl %cl, %esi, %edi
-; FALLBACK25-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 32(%eax), %edi
-; FALLBACK25-NEXT:    movl 36(%eax), %esi
-; FALLBACK25-NEXT:    movl %esi, %ebx
-; FALLBACK25-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK25-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK25-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 40(%eax), %edx
-; FALLBACK25-NEXT:    movl 44(%eax), %edi
-; FALLBACK25-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK25-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK25-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 56(%eax), %edx
-; FALLBACK25-NEXT:    movl 60(%eax), %edi
-; FALLBACK25-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK25-NEXT:    movl (%eax), %ebx
-; FALLBACK25-NEXT:    movl 52(%eax), %esi
-; FALLBACK25-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK25-NEXT:    negl %ebp
-; FALLBACK25-NEXT:    movl 160(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK25-NEXT:    movl %edx, 56(%ebp)
-; FALLBACK25-NEXT:    movl %edi, 60(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK25-NEXT:    shldl %cl, %ebx, %edx
-; FALLBACK25-NEXT:    shll %cl, %ebx
-; FALLBACK25-NEXT:    shldl %cl, %eax, %esi
-; FALLBACK25-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK25-NEXT:    shldl %cl, %edi, %eax
-; FALLBACK25-NEXT:    movl %eax, 48(%ebp)
-; FALLBACK25-NEXT:    movl %esi, 52(%ebp)
-; FALLBACK25-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 40(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK25-NEXT:    movl %ebx, (%ebp)
-; FALLBACK25-NEXT:    movl %edx, 4(%ebp)
-; FALLBACK25-NEXT:    addl $188, %esp
-; FALLBACK25-NEXT:    popl %esi
-; FALLBACK25-NEXT:    popl %edi
-; FALLBACK25-NEXT:    popl %ebx
-; FALLBACK25-NEXT:    popl %ebp
-; FALLBACK25-NEXT:    vzeroupper
-; FALLBACK25-NEXT:    retl
-;
-; FALLBACK26-LABEL: shl_64bytes:
-; FALLBACK26:       # %bb.0:
-; FALLBACK26-NEXT:    pushl %ebp
-; FALLBACK26-NEXT:    pushl %ebx
-; FALLBACK26-NEXT:    pushl %edi
-; FALLBACK26-NEXT:    pushl %esi
-; FALLBACK26-NEXT:    subl $204, %esp
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK26-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK26-NEXT:    vmovups 32(%ecx), %ymm1
-; FALLBACK26-NEXT:    movl (%eax), %eax
-; FALLBACK26-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; FALLBACK26-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    leal (,%eax,8), %ebx
-; FALLBACK26-NEXT:    andl $24, %ebx
-; FALLBACK26-NEXT:    movl %ebx, %ecx
-; FALLBACK26-NEXT:    andl $60, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    leal {{[0-9]+}}(%esp), %edx
-; FALLBACK26-NEXT:    subl %eax, %edx
-; FALLBACK26-NEXT:    movl (%edx), %esi
-; FALLBACK26-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 4(%edx), %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    notb %bl
-; FALLBACK26-NEXT:    shrl %esi
-; FALLBACK26-NEXT:    shrxl %ebx, %esi, %edi
-; FALLBACK26-NEXT:    shlxl %ecx, %eax, %esi
-; FALLBACK26-NEXT:    orl %esi, %edi
-; FALLBACK26-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 8(%edx), %esi
-; FALLBACK26-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrl %esi
-; FALLBACK26-NEXT:    shrxl %ebx, %esi, %eax
-; FALLBACK26-NEXT:    movl 12(%edx), %esi
-; FALLBACK26-NEXT:    shlxl %ecx, %esi, %ebp
-; FALLBACK26-NEXT:    orl %ebp, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl %ecx, %edi
-; FALLBACK26-NEXT:    shlxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK26-NEXT:    shrl %ecx
-; FALLBACK26-NEXT:    shrxl %ebx, %ecx, %ecx
-; FALLBACK26-NEXT:    orl %eax, %ecx
-; FALLBACK26-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 16(%edx), %ecx
-; FALLBACK26-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrl %ecx
-; FALLBACK26-NEXT:    shrxl %ebx, %ecx, %eax
-; FALLBACK26-NEXT:    movl 20(%edx), %ecx
-; FALLBACK26-NEXT:    shlxl %edi, %ecx, %ebp
-; FALLBACK26-NEXT:    orl %ebp, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK26-NEXT:    shrl %esi
-; FALLBACK26-NEXT:    shrxl %ebx, %esi, %eax
-; FALLBACK26-NEXT:    orl %ebp, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 24(%edx), %esi
-; FALLBACK26-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrl %esi
-; FALLBACK26-NEXT:    shrxl %ebx, %esi, %eax
-; FALLBACK26-NEXT:    movl 28(%edx), %esi
-; FALLBACK26-NEXT:    shlxl %edi, %esi, %ebp
-; FALLBACK26-NEXT:    orl %ebp, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK26-NEXT:    shrl %ecx
-; FALLBACK26-NEXT:    shrxl %ebx, %ecx, %eax
-; FALLBACK26-NEXT:    orl %ebp, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 32(%edx), %ecx
-; FALLBACK26-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrl %ecx
-; FALLBACK26-NEXT:    shrxl %ebx, %ecx, %eax
-; FALLBACK26-NEXT:    movl 36(%edx), %ecx
-; FALLBACK26-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shlxl %edi, %ecx, %ebp
-; FALLBACK26-NEXT:    orl %ebp, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK26-NEXT:    movl %edi, %eax
-; FALLBACK26-NEXT:    shrl %esi
-; FALLBACK26-NEXT:    shrxl %ebx, %esi, %esi
-; FALLBACK26-NEXT:    orl %ebp, %esi
-; FALLBACK26-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 40(%edx), %edi
-; FALLBACK26-NEXT:    movl %edi, %esi
-; FALLBACK26-NEXT:    shrl %esi
-; FALLBACK26-NEXT:    shrxl %ebx, %esi, %ecx
-; FALLBACK26-NEXT:    movl 44(%edx), %esi
-; FALLBACK26-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shlxl %eax, %esi, %ebp
-; FALLBACK26-NEXT:    orl %ebp, %ecx
-; FALLBACK26-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shlxl %eax, %edi, %edi
-; FALLBACK26-NEXT:    movl %eax, %esi
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    shrl %eax
-; FALLBACK26-NEXT:    shrxl %ebx, %eax, %eax
-; FALLBACK26-NEXT:    orl %edi, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 48(%edx), %ebp
-; FALLBACK26-NEXT:    movl %ebp, %edi
-; FALLBACK26-NEXT:    shrl %edi
-; FALLBACK26-NEXT:    shrxl %ebx, %edi, %eax
-; FALLBACK26-NEXT:    movl 52(%edx), %ecx
-; FALLBACK26-NEXT:    shlxl %esi, %ecx, %edi
-; FALLBACK26-NEXT:    orl %edi, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shlxl %esi, %ebp, %edi
-; FALLBACK26-NEXT:    movl %esi, %ebp
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    shrl %eax
-; FALLBACK26-NEXT:    shrxl %ebx, %eax, %esi
-; FALLBACK26-NEXT:    orl %edi, %esi
-; FALLBACK26-NEXT:    movl 56(%edx), %edi
-; FALLBACK26-NEXT:    shrl %ecx
-; FALLBACK26-NEXT:    shrxl %ebx, %ecx, %eax
-; FALLBACK26-NEXT:    shlxl %ebp, %edi, %ecx
-; FALLBACK26-NEXT:    orl %ecx, %eax
-; FALLBACK26-NEXT:    shrl %edi
-; FALLBACK26-NEXT:    shrxl %ebx, %edi, %ecx
-; FALLBACK26-NEXT:    shlxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK26-NEXT:    negl %ebx
-; FALLBACK26-NEXT:    shlxl %ebp, 188(%esp,%ebx), %ebx
-; FALLBACK26-NEXT:    orl %ecx, %ebx
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK26-NEXT:    movl %edi, (%edx)
-; FALLBACK26-NEXT:    movl %eax, 56(%edx)
-; FALLBACK26-NEXT:    movl %ebx, 60(%edx)
-; FALLBACK26-NEXT:    movl %esi, 48(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 52(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 40(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 44(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 32(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 36(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 24(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 28(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 16(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 20(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 8(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 12(%edx)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl %eax, 4(%edx)
-; FALLBACK26-NEXT:    addl $204, %esp
-; FALLBACK26-NEXT:    popl %esi
-; FALLBACK26-NEXT:    popl %edi
-; FALLBACK26-NEXT:    popl %ebx
-; FALLBACK26-NEXT:    popl %ebp
-; FALLBACK26-NEXT:    vzeroupper
-; FALLBACK26-NEXT:    retl
-;
-; FALLBACK27-LABEL: shl_64bytes:
-; FALLBACK27:       # %bb.0:
-; FALLBACK27-NEXT:    pushl %ebp
-; FALLBACK27-NEXT:    pushl %ebx
-; FALLBACK27-NEXT:    pushl %edi
-; FALLBACK27-NEXT:    pushl %esi
-; FALLBACK27-NEXT:    subl $204, %esp
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK27-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK27-NEXT:    vmovups 32(%ecx), %ymm1
-; FALLBACK27-NEXT:    movl (%eax), %ebx
-; FALLBACK27-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; FALLBACK27-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    leal (,%ebx,8), %ecx
-; FALLBACK27-NEXT:    andl $24, %ecx
-; FALLBACK27-NEXT:    andl $60, %ebx
-; FALLBACK27-NEXT:    leal {{[0-9]+}}(%esp), %eax
-; FALLBACK27-NEXT:    subl %ebx, %eax
-; FALLBACK27-NEXT:    movl 4(%eax), %esi
-; FALLBACK27-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 8(%eax), %edi
-; FALLBACK27-NEXT:    movl 12(%eax), %edx
-; FALLBACK27-NEXT:    movl %edx, %ebp
-; FALLBACK27-NEXT:    shldl %cl, %edi, %ebp
-; FALLBACK27-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shldl %cl, %esi, %edi
-; FALLBACK27-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 16(%eax), %edi
-; FALLBACK27-NEXT:    movl 20(%eax), %esi
-; FALLBACK27-NEXT:    movl %esi, %ebp
-; FALLBACK27-NEXT:    shldl %cl, %edi, %ebp
-; FALLBACK27-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK27-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 24(%eax), %edi
-; FALLBACK27-NEXT:    movl 28(%eax), %edx
-; FALLBACK27-NEXT:    movl %edx, %ebp
-; FALLBACK27-NEXT:    shldl %cl, %edi, %ebp
-; FALLBACK27-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shldl %cl, %esi, %edi
-; FALLBACK27-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 32(%eax), %edi
-; FALLBACK27-NEXT:    movl 36(%eax), %esi
-; FALLBACK27-NEXT:    movl %esi, %ebp
-; FALLBACK27-NEXT:    shldl %cl, %edi, %ebp
-; FALLBACK27-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK27-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 40(%eax), %ebp
-; FALLBACK27-NEXT:    movl 44(%eax), %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shldl %cl, %ebp, %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shldl %cl, %esi, %ebp
-; FALLBACK27-NEXT:    movl 56(%eax), %edx
-; FALLBACK27-NEXT:    movl 60(%eax), %edi
-; FALLBACK27-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK27-NEXT:    movl (%eax), %esi
-; FALLBACK27-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 52(%eax), %esi
-; FALLBACK27-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK27-NEXT:    negl %ebx
-; FALLBACK27-NEXT:    movl 176(%esp,%ebx), %ebx
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK27-NEXT:    movl %edx, 56(%eax)
-; FALLBACK27-NEXT:    movl %edi, 60(%eax)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK27-NEXT:    shlxl %ecx, %edx, %edi
-; FALLBACK27-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK27-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK27-NEXT:    shldl %cl, %ebx, %esi
-; FALLBACK27-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK27-NEXT:    shldl %cl, %edx, %ebx
-; FALLBACK27-NEXT:    movl %ebx, 48(%eax)
-; FALLBACK27-NEXT:    movl %esi, 52(%eax)
-; FALLBACK27-NEXT:    movl %ebp, 40(%eax)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK27-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK27-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK27-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK27-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK27-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK27-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK27-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK27-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK27-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK27-NEXT:    movl %edi, 4(%eax)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK27-NEXT:    movl %ecx, (%eax)
-; FALLBACK27-NEXT:    addl $204, %esp
-; FALLBACK27-NEXT:    popl %esi
-; FALLBACK27-NEXT:    popl %edi
-; FALLBACK27-NEXT:    popl %ebx
-; FALLBACK27-NEXT:    popl %ebp
-; FALLBACK27-NEXT:    vzeroupper
-; FALLBACK27-NEXT:    retl
-;
-; FALLBACK28-LABEL: shl_64bytes:
-; FALLBACK28:       # %bb.0:
-; FALLBACK28-NEXT:    pushl %ebp
-; FALLBACK28-NEXT:    pushl %ebx
-; FALLBACK28-NEXT:    pushl %edi
-; FALLBACK28-NEXT:    pushl %esi
-; FALLBACK28-NEXT:    subl $204, %esp
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK28-NEXT:    vmovups (%ecx), %zmm0
-; FALLBACK28-NEXT:    movl (%eax), %eax
-; FALLBACK28-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK28-NEXT:    vmovups %zmm1, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    vmovups %zmm0, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %eax, %edx
-; FALLBACK28-NEXT:    andl $60, %edx
-; FALLBACK28-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    leal {{[0-9]+}}(%esp), %ecx
-; FALLBACK28-NEXT:    subl %edx, %ecx
-; FALLBACK28-NEXT:    movl (%ecx), %edi
-; FALLBACK28-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 4(%ecx), %edx
-; FALLBACK28-NEXT:    movl %ecx, %ebp
-; FALLBACK28-NEXT:    shll $3, %eax
-; FALLBACK28-NEXT:    andl $24, %eax
-; FALLBACK28-NEXT:    movl %edx, %esi
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shll %cl, %esi
-; FALLBACK28-NEXT:    shrl %edi
-; FALLBACK28-NEXT:    movb %al, %ch
-; FALLBACK28-NEXT:    notb %ch
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edi
-; FALLBACK28-NEXT:    orl %esi, %edi
-; FALLBACK28-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 12(%ebp), %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    movl 8(%ebp), %esi
-; FALLBACK28-NEXT:    movl %ebp, %edi
-; FALLBACK28-NEXT:    movl %esi, %ebp
-; FALLBACK28-NEXT:    shrl %ebp
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shrl %cl, %ebp
-; FALLBACK28-NEXT:    orl %ebx, %ebp
-; FALLBACK28-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shll %cl, %esi
-; FALLBACK28-NEXT:    shrl %edx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edx
-; FALLBACK28-NEXT:    orl %esi, %edx
-; FALLBACK28-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl %edi, %ebp
-; FALLBACK28-NEXT:    movl 20(%edi), %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    movl 16(%edi), %esi
-; FALLBACK28-NEXT:    movl %esi, %edx
-; FALLBACK28-NEXT:    shrl %edx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edx
-; FALLBACK28-NEXT:    orl %ebx, %edx
-; FALLBACK28-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shll %cl, %esi
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %edi
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edi
-; FALLBACK28-NEXT:    orl %esi, %edi
-; FALLBACK28-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl %ebp, %edx
-; FALLBACK28-NEXT:    movl 28(%ebp), %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    movl 24(%ebp), %esi
-; FALLBACK28-NEXT:    movl %esi, %edi
-; FALLBACK28-NEXT:    shrl %edi
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edi
-; FALLBACK28-NEXT:    orl %ebx, %edi
-; FALLBACK28-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shll %cl, %esi
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %ebp
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shrl %cl, %ebp
-; FALLBACK28-NEXT:    orl %esi, %ebp
-; FALLBACK28-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 36(%edx), %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    movl 32(%edx), %esi
-; FALLBACK28-NEXT:    movl %edx, %ebp
-; FALLBACK28-NEXT:    movl %esi, %edi
-; FALLBACK28-NEXT:    shrl %edi
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edi
-; FALLBACK28-NEXT:    orl %ebx, %edi
-; FALLBACK28-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shll %cl, %esi
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %edx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edx
-; FALLBACK28-NEXT:    orl %esi, %edx
-; FALLBACK28-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 44(%ebp), %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    movl 40(%ebp), %esi
-; FALLBACK28-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl %esi, %edx
-; FALLBACK28-NEXT:    shrl %edx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edx
-; FALLBACK28-NEXT:    orl %ebx, %edx
-; FALLBACK28-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shll %cl, %esi
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %edx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edx
-; FALLBACK28-NEXT:    orl %esi, %edx
-; FALLBACK28-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 52(%ebp), %esi
-; FALLBACK28-NEXT:    movl %esi, %edi
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shll %cl, %edi
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK28-NEXT:    negl %edx
-; FALLBACK28-NEXT:    movl 176(%esp,%edx), %ebx
-; FALLBACK28-NEXT:    movl %ebx, %ebp
-; FALLBACK28-NEXT:    shrl %ebp
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shrl %cl, %ebp
-; FALLBACK28-NEXT:    orl %edi, %ebp
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %edx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edx
-; FALLBACK28-NEXT:    orl %ebx, %edx
-; FALLBACK28-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK28-NEXT:    movl 60(%edi), %edx
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shll %cl, %edx
-; FALLBACK28-NEXT:    movl 56(%edi), %ebx
-; FALLBACK28-NEXT:    movl %ebx, %edi
-; FALLBACK28-NEXT:    shrl %edi
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edi
-; FALLBACK28-NEXT:    orl %edx, %edi
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    shrl %esi
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shrl %cl, %esi
-; FALLBACK28-NEXT:    orl %ebx, %esi
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK28-NEXT:    shll %cl, %edx
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK28-NEXT:    movl %edx, (%eax)
-; FALLBACK28-NEXT:    movl %esi, 56(%eax)
-; FALLBACK28-NEXT:    movl %edi, 60(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 48(%eax)
-; FALLBACK28-NEXT:    movl %ebp, 52(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 40(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK28-NEXT:    addl $204, %esp
-; FALLBACK28-NEXT:    popl %esi
-; FALLBACK28-NEXT:    popl %edi
-; FALLBACK28-NEXT:    popl %ebx
-; FALLBACK28-NEXT:    popl %ebp
-; FALLBACK28-NEXT:    vzeroupper
-; FALLBACK28-NEXT:    retl
-;
-; FALLBACK29-LABEL: shl_64bytes:
-; FALLBACK29:       # %bb.0:
-; FALLBACK29-NEXT:    pushl %ebp
-; FALLBACK29-NEXT:    pushl %ebx
-; FALLBACK29-NEXT:    pushl %edi
-; FALLBACK29-NEXT:    pushl %esi
-; FALLBACK29-NEXT:    subl $188, %esp
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK29-NEXT:    vmovups (%ecx), %zmm0
-; FALLBACK29-NEXT:    movl (%eax), %ecx
-; FALLBACK29-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK29-NEXT:    vmovups %zmm1, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    vmovups %zmm0, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %ecx, %ebp
-; FALLBACK29-NEXT:    andl $60, %ebp
-; FALLBACK29-NEXT:    leal {{[0-9]+}}(%esp), %eax
-; FALLBACK29-NEXT:    subl %ebp, %eax
-; FALLBACK29-NEXT:    movl 8(%eax), %esi
-; FALLBACK29-NEXT:    movl 12(%eax), %edx
-; FALLBACK29-NEXT:    shll $3, %ecx
-; FALLBACK29-NEXT:    andl $24, %ecx
-; FALLBACK29-NEXT:    movl %edx, %edi
-; FALLBACK29-NEXT:    shldl %cl, %esi, %edi
-; FALLBACK29-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 4(%eax), %edi
-; FALLBACK29-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shldl %cl, %edi, %esi
-; FALLBACK29-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 16(%eax), %edi
-; FALLBACK29-NEXT:    movl 20(%eax), %esi
-; FALLBACK29-NEXT:    movl %esi, %ebx
-; FALLBACK29-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK29-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK29-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 24(%eax), %edi
-; FALLBACK29-NEXT:    movl 28(%eax), %edx
-; FALLBACK29-NEXT:    movl %edx, %ebx
-; FALLBACK29-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK29-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shldl %cl, %esi, %edi
-; FALLBACK29-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 32(%eax), %edi
-; FALLBACK29-NEXT:    movl 36(%eax), %esi
-; FALLBACK29-NEXT:    movl %esi, %ebx
-; FALLBACK29-NEXT:    shldl %cl, %edi, %ebx
-; FALLBACK29-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK29-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 40(%eax), %edx
-; FALLBACK29-NEXT:    movl 44(%eax), %edi
-; FALLBACK29-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK29-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK29-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 56(%eax), %edx
-; FALLBACK29-NEXT:    movl 60(%eax), %edi
-; FALLBACK29-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK29-NEXT:    movl (%eax), %ebx
-; FALLBACK29-NEXT:    movl 52(%eax), %esi
-; FALLBACK29-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK29-NEXT:    negl %ebp
-; FALLBACK29-NEXT:    movl 160(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK29-NEXT:    movl %edx, 56(%ebp)
-; FALLBACK29-NEXT:    movl %edi, 60(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK29-NEXT:    shldl %cl, %ebx, %edx
-; FALLBACK29-NEXT:    shll %cl, %ebx
-; FALLBACK29-NEXT:    shldl %cl, %eax, %esi
-; FALLBACK29-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK29-NEXT:    shldl %cl, %edi, %eax
-; FALLBACK29-NEXT:    movl %eax, 48(%ebp)
-; FALLBACK29-NEXT:    movl %esi, 52(%ebp)
-; FALLBACK29-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 40(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK29-NEXT:    movl %ebx, (%ebp)
-; FALLBACK29-NEXT:    movl %edx, 4(%ebp)
-; FALLBACK29-NEXT:    addl $188, %esp
-; FALLBACK29-NEXT:    popl %esi
-; FALLBACK29-NEXT:    popl %edi
-; FALLBACK29-NEXT:    popl %ebx
-; FALLBACK29-NEXT:    popl %ebp
-; FALLBACK29-NEXT:    vzeroupper
-; FALLBACK29-NEXT:    retl
-;
-; FALLBACK30-LABEL: shl_64bytes:
-; FALLBACK30:       # %bb.0:
-; FALLBACK30-NEXT:    pushl %ebp
-; FALLBACK30-NEXT:    pushl %ebx
-; FALLBACK30-NEXT:    pushl %edi
-; FALLBACK30-NEXT:    pushl %esi
-; FALLBACK30-NEXT:    subl $204, %esp
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK30-NEXT:    vmovups (%ecx), %zmm0
-; FALLBACK30-NEXT:    movl (%eax), %eax
-; FALLBACK30-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK30-NEXT:    vmovups %zmm1, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    vmovups %zmm0, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    leal (,%eax,8), %ebx
-; FALLBACK30-NEXT:    andl $24, %ebx
-; FALLBACK30-NEXT:    movl %ebx, %ecx
-; FALLBACK30-NEXT:    andl $60, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    leal {{[0-9]+}}(%esp), %edx
-; FALLBACK30-NEXT:    subl %eax, %edx
-; FALLBACK30-NEXT:    movl (%edx), %esi
-; FALLBACK30-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 4(%edx), %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    notb %bl
-; FALLBACK30-NEXT:    shrl %esi
-; FALLBACK30-NEXT:    shrxl %ebx, %esi, %edi
-; FALLBACK30-NEXT:    shlxl %ecx, %eax, %esi
-; FALLBACK30-NEXT:    orl %esi, %edi
-; FALLBACK30-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 8(%edx), %esi
-; FALLBACK30-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrl %esi
-; FALLBACK30-NEXT:    shrxl %ebx, %esi, %eax
-; FALLBACK30-NEXT:    movl 12(%edx), %esi
-; FALLBACK30-NEXT:    shlxl %ecx, %esi, %ebp
-; FALLBACK30-NEXT:    orl %ebp, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl %ecx, %edi
-; FALLBACK30-NEXT:    shlxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK30-NEXT:    shrl %ecx
-; FALLBACK30-NEXT:    shrxl %ebx, %ecx, %ecx
-; FALLBACK30-NEXT:    orl %eax, %ecx
-; FALLBACK30-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 16(%edx), %ecx
-; FALLBACK30-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrl %ecx
-; FALLBACK30-NEXT:    shrxl %ebx, %ecx, %eax
-; FALLBACK30-NEXT:    movl 20(%edx), %ecx
-; FALLBACK30-NEXT:    shlxl %edi, %ecx, %ebp
-; FALLBACK30-NEXT:    orl %ebp, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK30-NEXT:    shrl %esi
-; FALLBACK30-NEXT:    shrxl %ebx, %esi, %eax
-; FALLBACK30-NEXT:    orl %ebp, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 24(%edx), %esi
-; FALLBACK30-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrl %esi
-; FALLBACK30-NEXT:    shrxl %ebx, %esi, %eax
-; FALLBACK30-NEXT:    movl 28(%edx), %esi
-; FALLBACK30-NEXT:    shlxl %edi, %esi, %ebp
-; FALLBACK30-NEXT:    orl %ebp, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK30-NEXT:    shrl %ecx
-; FALLBACK30-NEXT:    shrxl %ebx, %ecx, %eax
-; FALLBACK30-NEXT:    orl %ebp, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 32(%edx), %ecx
-; FALLBACK30-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrl %ecx
-; FALLBACK30-NEXT:    shrxl %ebx, %ecx, %eax
-; FALLBACK30-NEXT:    movl 36(%edx), %ecx
-; FALLBACK30-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shlxl %edi, %ecx, %ebp
-; FALLBACK30-NEXT:    orl %ebp, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK30-NEXT:    movl %edi, %eax
-; FALLBACK30-NEXT:    shrl %esi
-; FALLBACK30-NEXT:    shrxl %ebx, %esi, %esi
-; FALLBACK30-NEXT:    orl %ebp, %esi
-; FALLBACK30-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 40(%edx), %edi
-; FALLBACK30-NEXT:    movl %edi, %esi
-; FALLBACK30-NEXT:    shrl %esi
-; FALLBACK30-NEXT:    shrxl %ebx, %esi, %ecx
-; FALLBACK30-NEXT:    movl 44(%edx), %esi
-; FALLBACK30-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shlxl %eax, %esi, %ebp
-; FALLBACK30-NEXT:    orl %ebp, %ecx
-; FALLBACK30-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shlxl %eax, %edi, %edi
-; FALLBACK30-NEXT:    movl %eax, %esi
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    shrl %eax
-; FALLBACK30-NEXT:    shrxl %ebx, %eax, %eax
-; FALLBACK30-NEXT:    orl %edi, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 48(%edx), %ebp
-; FALLBACK30-NEXT:    movl %ebp, %edi
-; FALLBACK30-NEXT:    shrl %edi
-; FALLBACK30-NEXT:    shrxl %ebx, %edi, %eax
-; FALLBACK30-NEXT:    movl 52(%edx), %ecx
-; FALLBACK30-NEXT:    shlxl %esi, %ecx, %edi
-; FALLBACK30-NEXT:    orl %edi, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shlxl %esi, %ebp, %edi
-; FALLBACK30-NEXT:    movl %esi, %ebp
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    shrl %eax
-; FALLBACK30-NEXT:    shrxl %ebx, %eax, %esi
-; FALLBACK30-NEXT:    orl %edi, %esi
-; FALLBACK30-NEXT:    movl 56(%edx), %edi
-; FALLBACK30-NEXT:    shrl %ecx
-; FALLBACK30-NEXT:    shrxl %ebx, %ecx, %eax
-; FALLBACK30-NEXT:    shlxl %ebp, %edi, %ecx
-; FALLBACK30-NEXT:    orl %ecx, %eax
-; FALLBACK30-NEXT:    shrl %edi
-; FALLBACK30-NEXT:    shrxl %ebx, %edi, %ecx
-; FALLBACK30-NEXT:    shlxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK30-NEXT:    negl %ebx
-; FALLBACK30-NEXT:    shlxl %ebp, 188(%esp,%ebx), %ebx
-; FALLBACK30-NEXT:    orl %ecx, %ebx
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FALLBACK30-NEXT:    movl %edi, (%edx)
-; FALLBACK30-NEXT:    movl %eax, 56(%edx)
-; FALLBACK30-NEXT:    movl %ebx, 60(%edx)
-; FALLBACK30-NEXT:    movl %esi, 48(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 52(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 40(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 44(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 32(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 36(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 24(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 28(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 16(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 20(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 8(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 12(%edx)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl %eax, 4(%edx)
-; FALLBACK30-NEXT:    addl $204, %esp
-; FALLBACK30-NEXT:    popl %esi
-; FALLBACK30-NEXT:    popl %edi
-; FALLBACK30-NEXT:    popl %ebx
-; FALLBACK30-NEXT:    popl %ebp
-; FALLBACK30-NEXT:    vzeroupper
-; FALLBACK30-NEXT:    retl
-;
-; FALLBACK31-LABEL: shl_64bytes:
-; FALLBACK31:       # %bb.0:
-; FALLBACK31-NEXT:    pushl %ebp
-; FALLBACK31-NEXT:    pushl %ebx
-; FALLBACK31-NEXT:    pushl %edi
-; FALLBACK31-NEXT:    pushl %esi
-; FALLBACK31-NEXT:    subl $204, %esp
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK31-NEXT:    vmovups (%ecx), %zmm0
-; FALLBACK31-NEXT:    movl (%eax), %ebx
-; FALLBACK31-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; FALLBACK31-NEXT:    vmovups %zmm1, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    vmovups %zmm0, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    leal (,%ebx,8), %ecx
-; FALLBACK31-NEXT:    andl $24, %ecx
-; FALLBACK31-NEXT:    andl $60, %ebx
-; FALLBACK31-NEXT:    leal {{[0-9]+}}(%esp), %eax
-; FALLBACK31-NEXT:    subl %ebx, %eax
-; FALLBACK31-NEXT:    movl 4(%eax), %esi
-; FALLBACK31-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 8(%eax), %edi
-; FALLBACK31-NEXT:    movl 12(%eax), %edx
-; FALLBACK31-NEXT:    movl %edx, %ebp
-; FALLBACK31-NEXT:    shldl %cl, %edi, %ebp
-; FALLBACK31-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shldl %cl, %esi, %edi
-; FALLBACK31-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 16(%eax), %edi
-; FALLBACK31-NEXT:    movl 20(%eax), %esi
-; FALLBACK31-NEXT:    movl %esi, %ebp
-; FALLBACK31-NEXT:    shldl %cl, %edi, %ebp
-; FALLBACK31-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK31-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 24(%eax), %edi
-; FALLBACK31-NEXT:    movl 28(%eax), %edx
-; FALLBACK31-NEXT:    movl %edx, %ebp
-; FALLBACK31-NEXT:    shldl %cl, %edi, %ebp
-; FALLBACK31-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shldl %cl, %esi, %edi
-; FALLBACK31-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 32(%eax), %edi
-; FALLBACK31-NEXT:    movl 36(%eax), %esi
-; FALLBACK31-NEXT:    movl %esi, %ebp
-; FALLBACK31-NEXT:    shldl %cl, %edi, %ebp
-; FALLBACK31-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK31-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 40(%eax), %ebp
-; FALLBACK31-NEXT:    movl 44(%eax), %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shldl %cl, %ebp, %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shldl %cl, %esi, %ebp
-; FALLBACK31-NEXT:    movl 56(%eax), %edx
-; FALLBACK31-NEXT:    movl 60(%eax), %edi
-; FALLBACK31-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK31-NEXT:    movl (%eax), %esi
-; FALLBACK31-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 52(%eax), %esi
-; FALLBACK31-NEXT:    shldl %cl, %esi, %edx
-; FALLBACK31-NEXT:    negl %ebx
-; FALLBACK31-NEXT:    movl 176(%esp,%ebx), %ebx
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK31-NEXT:    movl %edx, 56(%eax)
-; FALLBACK31-NEXT:    movl %edi, 60(%eax)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK31-NEXT:    shlxl %ecx, %edx, %edi
-; FALLBACK31-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; FALLBACK31-NEXT:    shldl %cl, %edx, %edi
-; FALLBACK31-NEXT:    shldl %cl, %ebx, %esi
-; FALLBACK31-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK31-NEXT:    shldl %cl, %edx, %ebx
-; FALLBACK31-NEXT:    movl %ebx, 48(%eax)
-; FALLBACK31-NEXT:    movl %esi, 52(%eax)
-; FALLBACK31-NEXT:    movl %ebp, 40(%eax)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK31-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK31-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK31-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK31-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK31-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK31-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK31-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK31-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK31-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK31-NEXT:    movl %edi, 4(%eax)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK31-NEXT:    movl %ecx, (%eax)
-; FALLBACK31-NEXT:    addl $204, %esp
-; FALLBACK31-NEXT:    popl %esi
-; FALLBACK31-NEXT:    popl %edi
-; FALLBACK31-NEXT:    popl %ebx
-; FALLBACK31-NEXT:    popl %ebp
-; FALLBACK31-NEXT:    vzeroupper
-; FALLBACK31-NEXT:    retl
+; X64-NO-SHLD-NO-BMI2-SSE2-LABEL: shl_64bytes:
+; X64-NO-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %r13
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %r12
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 32(%rdi), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 40(%rdi), %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 48(%rdi), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 56(%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl (%rsi), %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    andl $56, %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    andl $56, %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    negl %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movslq %esi, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -64(%rsp,%rbx), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -56(%rsp,%rbx), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r10, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -40(%rsp,%rbx), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -48(%rsp,%rbx), %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r15, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r14, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r15, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -24(%rsp,%rbx), %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, %r12
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -32(%rsp,%rbx), %r13
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r13, %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r12, %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r13
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r13, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -8(%rsp,%rbx), %r12
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -16(%rsp,%rbx), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rbx, %r13
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %r13
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r13
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r12, %r13
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %rbx, %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r13, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r15, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r11, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %r12
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %r13
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE2-LABEL: shl_64bytes:
+; X64-HAVE-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushq %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushq %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %rcx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 32(%rdi), %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 40(%rdi), %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 48(%rdi), %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 56(%rdi), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%rsi), %esi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    leal (,%rsi,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andl $56, %esi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    negl %esi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movslq %esi, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -48(%rsp,%r9), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -40(%rsp,%r9), %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -64(%rsp,%r9), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -56(%rsp,%r9), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldq %cl, %rdi, %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -32(%rsp,%r9), %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -24(%rsp,%r9), %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rbx, %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldq %cl, %r11, %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldq %cl, %r10, %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -16(%rsp,%r9), %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -8(%rsp,%r9), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldq %cl, %r10, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldq %cl, %rbx, %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldq %cl, %r8, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, 48(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, 56(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r11, 32(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, 40(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    addq $8, %rsp
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popq %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popq %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE2-LABEL: shl_64bytes:
+; X64-NO-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 32(%rdi), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 40(%rdi), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 48(%rdi), %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 56(%rdi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $56, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $56, %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    negl %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movslq %esi, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -64(%rsp,%rsi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -56(%rsp,%rsi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %rdi, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r9, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrq %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rax, %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r8, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -40(%rsp,%rsi), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r11, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -48(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r8, %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrq %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rax, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %rbx, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrq %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rax, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r14, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -24(%rsp,%rsi), %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %rbx, %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -32(%rsp,%rsi), %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r15, %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrq %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rax, %r15, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r14, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrq %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rax, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r12, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, -8(%rsp,%rsi), %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -16(%rsp,%rsi), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %rsi, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrq %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rax, %rsi, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r14, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrq %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rax, %rbx, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %rcx, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rsi, 56(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r11, 32(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r15, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addq $8, %rsp
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: shl_64bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %rcx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 32(%rdi), %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 40(%rdi), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 48(%rdi), %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 56(%rdi), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%rsi), %esi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (,%rsi,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $56, %esi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    negl %esi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movslq %esi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -48(%rsp,%r8), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -40(%rsp,%r8), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -64(%rsp,%r8), %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -56(%rsp,%r8), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldq %cl, %rdi, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -32(%rsp,%r8), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -24(%rsp,%r8), %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rbx, %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldq %cl, %r11, %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldq %cl, %r9, %r11
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -16(%rsp,%r8), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -8(%rsp,%r8), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldq %cl, %r9, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldq %cl, %rbx, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldq %cl, %r10, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r10, %rcx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, 48(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, 56(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r11, 32(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r14, 40(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    addq $8, %rsp
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-SSE4-LABEL: shl_64bytes:
+; X64-NO-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r13
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 32(%rdi), %xmm2
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 48(%rdi), %xmm3
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl (%rsi), %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm4, %xmm4
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm3, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (,%rcx,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    andl $56, %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    andl $56, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    negl %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movslq %ecx, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -24(%rsp,%r9), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -32(%rsp,%r9), %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r11, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r10, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -40(%rsp,%r9), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rbx, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r11, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -48(%rsp,%r9), %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r15, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %rbx, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -64(%rsp,%r9), %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -56(%rsp,%r9), %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r12, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r15, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r14, %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r12, %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -16(%rsp,%r9), %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r12, %r13
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r13
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r13, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -8(%rsp,%r9), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r9, %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r14, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r12, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r15, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rbx, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r11, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r8, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %r13
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE4-LABEL: shl_64bytes:
+; X64-HAVE-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 32(%rdi), %xmm2
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 48(%rdi), %xmm3
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm4, %xmm4
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm3, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    negl %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movslq %eax, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -32(%rsp,%r8), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -24(%rsp,%r8), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -40(%rsp,%r8), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldq %cl, %rdi, %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -48(%rsp,%r8), %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldq %cl, %r10, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -64(%rsp,%r8), %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -56(%rsp,%r8), %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldq %cl, %rbx, %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -16(%rsp,%r8), %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r14, %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldq %cl, %r9, %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -8(%rsp,%r8), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldq %cl, %r14, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r11, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldq %cl, %r11, %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r8, 56(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r15, 48(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rbx, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rax, 32(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rsi, 40(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popq %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popq %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popq %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE4-LABEL: shl_64bytes:
+; X64-NO-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 32(%rdi), %xmm2
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 48(%rdi), %xmm3
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm4, %xmm4
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm3, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $56, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $56, %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    negl %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movslq %esi, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -24(%rsp,%rsi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %rdi, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -32(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r8, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrq %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rax, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r9, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -40(%rsp,%rsi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r9, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrq %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rax, %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r10, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -48(%rsp,%rsi), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r10, %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrq %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rax, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r11, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -64(%rsp,%rsi), %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -56(%rsp,%rsi), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r11, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrq %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rax, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r14, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %rbx, %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrq %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rax, %rbx, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r15, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -16(%rsp,%rsi), %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r15, %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrq %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rax, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r12, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, -8(%rsp,%rsi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrq %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rax, %r15, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rcx, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r14, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, 56(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rbx, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r10, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r9, 32(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r8, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addq $8, %rsp
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: shl_64bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 32(%rdi), %xmm2
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 48(%rdi), %xmm3
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm4, %xmm4
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm3, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    negl %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movslq %eax, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -32(%rsp,%r8), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -24(%rsp,%r8), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r9, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -40(%rsp,%r8), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldq %cl, %rdi, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -48(%rsp,%r8), %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldq %cl, %r10, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -64(%rsp,%r8), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -56(%rsp,%r8), %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldq %cl, %rbx, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -16(%rsp,%r8), %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r14, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldq %cl, %r9, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -8(%rsp,%r8), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldq %cl, %r14, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r11, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldq %cl, %r11, %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r8, 56(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r15, 48(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rbx, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r10, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, 32(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rsi, 40(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-AVX1-LABEL: shl_64bytes:
+; X64-NO-SHLD-NO-BMI2-AVX1:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushq %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushq %r14
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushq %r13
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushq %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl (%rsi), %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    leal (,%rcx,8), %eax
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    andl $56, %eax
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    andl $56, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    negl %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movslq %ecx, %r9
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq -24(%rsp,%r9), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %rdi, %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq -32(%rsp,%r9), %r11
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r11, %r8
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %r8
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    orq %r10, %r8
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shlq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq -40(%rsp,%r9), %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %rbx, %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    orq %r11, %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq -48(%rsp,%r9), %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r15, %r11
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %r11
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    orq %rbx, %r11
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shlq %cl, %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq -64(%rsp,%r9), %r14
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq -56(%rsp,%r9), %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r12, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    orq %r15, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shlq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r14, %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %cl, %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    orq %r12, %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq -16(%rsp,%r9), %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r12, %r13
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shlq %cl, %r13
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %rdi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    orq %r13, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq -8(%rsp,%r9), %r9
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shlq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    orq %r9, %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    shlq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r14, (%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r12, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %rdi, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r15, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %rbx, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r11, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r10, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    movq %r8, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    popq %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    popq %r13
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    popq %r14
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    popq %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    vzeroupper
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-AVX1-LABEL: shl_64bytes:
+; X64-HAVE-SHLD-NO-BMI2-AVX1:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    pushq %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    pushq %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    negl %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movslq %eax, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq -32(%rsp,%r8), %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq -24(%rsp,%r8), %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %r9, %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq -40(%rsp,%r8), %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldq %cl, %rdi, %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq -48(%rsp,%r8), %r10
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldq %cl, %r10, %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq -64(%rsp,%r8), %r11
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq -56(%rsp,%r8), %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldq %cl, %rbx, %r10
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq -16(%rsp,%r8), %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %r14, %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldq %cl, %r9, %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq -8(%rsp,%r8), %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldq %cl, %r14, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %r11, %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shlq %cl, %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldq %cl, %r11, %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %r8, 56(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %r15, 48(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %rbx, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %r10, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %rax, 32(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %rsi, 40(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    popq %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    popq %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    popq %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vzeroupper
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-AVX1-LABEL: shl_64bytes:
+; X64-NO-SHLD-HAVE-BMI2-AVX1:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushq %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushq %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushq %r12
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushq %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushq %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    andl $56, %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    andl $56, %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    negl %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movslq %esi, %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -24(%rsp,%rsi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxq %rcx, %rdi, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -32(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxq %rcx, %r8, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrq %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxq %rax, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orq %r9, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -40(%rsp,%rsi), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxq %rcx, %r9, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrq %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxq %rax, %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orq %r10, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -48(%rsp,%rsi), %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxq %rcx, %r10, %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrq %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxq %rax, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orq %r11, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -64(%rsp,%rsi), %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -56(%rsp,%rsi), %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxq %rcx, %r11, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrq %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxq %rax, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orq %r14, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxq %rcx, %rbx, %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrq %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxq %rax, %rbx, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orq %r15, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -16(%rsp,%rsi), %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxq %rcx, %r15, %r12
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrq %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxq %rax, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orq %r12, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxq %rcx, -8(%rsp,%rsi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrq %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxq %rax, %r15, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orq %rcx, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r14, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %rax, 56(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %rdi, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %rbx, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r10, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r9, 32(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r8, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    addq $8, %rsp
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    popq %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    popq %r12
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    popq %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    popq %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vzeroupper
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-LABEL: shl_64bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    pushq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    pushq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups (%rdi), %ymm0
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups 32(%rdi), %ymm1
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    negl %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movslq %eax, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -32(%rsp,%r8), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -24(%rsp,%r8), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r9, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -40(%rsp,%r8), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldq %cl, %rdi, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -48(%rsp,%r8), %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldq %cl, %r10, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -64(%rsp,%r8), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -56(%rsp,%r8), %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldq %cl, %rbx, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -16(%rsp,%r8), %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r14, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldq %cl, %r9, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq -8(%rsp,%r8), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldq %cl, %r14, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxq %rcx, %r11, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldq %cl, %r11, %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r8, 56(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r15, 48(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %rbx, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r10, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %rax, 32(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %rsi, 40(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    popq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    popq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    popq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vzeroupper
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-AVX512-LABEL: shl_64bytes:
+; X64-NO-SHLD-NO-BMI2-AVX512:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushq %r15
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushq %r14
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushq %r13
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushq %r12
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    vmovups (%rdi), %zmm0
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl (%rsi), %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    vmovups %zmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    vmovups %zmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    leal (,%rcx,8), %eax
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    andl $56, %eax
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    andl $56, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    negl %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movslq %ecx, %r9
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq -24(%rsp,%r9), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %rdi, %r10
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq -32(%rsp,%r9), %r11
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r11, %r8
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %r8
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    orq %r10, %r8
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shlq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq -40(%rsp,%r9), %rbx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %rbx, %r10
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %r10
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    orq %r11, %r10
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq -48(%rsp,%r9), %r15
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r15, %r11
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %r11
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    orq %rbx, %r11
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shlq %cl, %r15
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq -64(%rsp,%r9), %r14
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq -56(%rsp,%r9), %r12
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r12, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    orq %r15, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shlq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r14, %r15
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %r15
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %cl, %r15
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    orq %r12, %r15
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq -16(%rsp,%r9), %r12
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r12, %r13
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shlq %cl, %r13
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %rdi
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    orq %r13, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq -8(%rsp,%r9), %r9
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shlq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %r12
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    orq %r9, %r12
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    shlq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r14, (%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r12, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %rdi, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r15, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %rbx, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r11, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r10, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    movq %r8, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    popq %r12
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    popq %r13
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    popq %r14
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    popq %r15
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    vzeroupper
+; X64-NO-SHLD-NO-BMI2-AVX512-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-AVX512-LABEL: shl_64bytes:
+; X64-HAVE-SHLD-NO-BMI2-AVX512:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    pushq %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    pushq %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vmovups (%rdi), %zmm0
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vmovups %zmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vmovups %zmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    negl %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movslq %eax, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq -32(%rsp,%r8), %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq -24(%rsp,%r8), %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %r9, %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq -40(%rsp,%r8), %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldq %cl, %rdi, %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq -48(%rsp,%r8), %r10
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldq %cl, %r10, %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq -64(%rsp,%r8), %r11
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq -56(%rsp,%r8), %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldq %cl, %rbx, %r10
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq -16(%rsp,%r8), %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %r14, %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldq %cl, %r9, %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq -8(%rsp,%r8), %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldq %cl, %r14, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %r11, %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shlq %cl, %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldq %cl, %r11, %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %r8, 56(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %r15, 48(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %rbx, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %r10, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %rax, 32(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %rsi, 40(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    popq %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    popq %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    popq %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vzeroupper
+; X64-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-AVX512-LABEL: shl_64bytes:
+; X64-NO-SHLD-HAVE-BMI2-AVX512:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushq %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushq %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushq %r12
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushq %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushq %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups (%rdi), %zmm0
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups %zmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups %zmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    andl $56, %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    andl $56, %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    negl %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movslq %esi, %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -24(%rsp,%rsi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxq %rcx, %rdi, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    notb %al
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -32(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxq %rcx, %r8, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrq %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxq %rax, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orq %r9, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -40(%rsp,%rsi), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxq %rcx, %r9, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrq %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxq %rax, %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orq %r10, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -48(%rsp,%rsi), %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxq %rcx, %r10, %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrq %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxq %rax, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orq %r11, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -64(%rsp,%rsi), %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -56(%rsp,%rsi), %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxq %rcx, %r11, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrq %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxq %rax, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orq %r14, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxq %rcx, %rbx, %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrq %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxq %rax, %rbx, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orq %r15, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -16(%rsp,%rsi), %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxq %rcx, %r15, %r12
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrq %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxq %rax, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orq %r12, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxq %rcx, -8(%rsp,%rsi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrq %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxq %rax, %r15, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orq %rcx, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r14, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %rax, 56(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %rdi, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %rbx, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r10, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r9, 32(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r8, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    addq $8, %rsp
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    popq %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    popq %r12
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    popq %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    popq %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vzeroupper
+; X64-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-LABEL: shl_64bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    pushq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    pushq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups (%rdi), %zmm0
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups %zmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups %zmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    negl %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movslq %eax, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -32(%rsp,%r8), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -24(%rsp,%r8), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r9, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldq %cl, %rax, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -40(%rsp,%r8), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldq %cl, %rdi, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -48(%rsp,%r8), %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldq %cl, %r10, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -64(%rsp,%r8), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -56(%rsp,%r8), %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldq %cl, %rbx, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -16(%rsp,%r8), %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r14, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldq %cl, %r9, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq -8(%rsp,%r8), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldq %cl, %r14, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxq %rcx, %r11, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldq %cl, %r11, %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r8, 56(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r15, 48(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %rbx, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r10, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %rdi, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %rax, 32(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %rsi, 40(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movq %r9, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    popq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    popq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    popq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vzeroupper
+; X64-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    retq
+;
+; X86-NO-SHLD-NO-BMI2-SSE2-LABEL: shl_64bytes:
+; X86-NO-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    subl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl (%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 32(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 36(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 40(%eax), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 44(%eax), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 48(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 52(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 56(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 60(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    andl $60, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    subl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl (%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll $3, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    andl $24, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    notb %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%edi), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %esi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 36(%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 32(%edx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 44(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 40(%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 52(%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    negl %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 176(%esp,%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 60(%edi), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 56(%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, (%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, 56(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, 60(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 48(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, 52(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 40(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 44(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 32(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 36(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-SSE2-LABEL: shl_64bytes:
+; X86-HAVE-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 32(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 36(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 40(%ecx), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 44(%ecx), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 48(%ecx), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 52(%ecx), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 56(%ecx), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 60(%ecx), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%ecx), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    subl %ebp, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %esi, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %esi, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 32(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 36(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 40(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 44(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 56(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 60(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%eax), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 52(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    negl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 160(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, 56(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, 60(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %ebx, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shldl %cl, %edi, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 48(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, 52(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 40(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-SSE2-LABEL: shl_64bytes:
+; X86-NO-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    subl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 28(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 32(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 36(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 40(%eax), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 44(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 48(%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 52(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 56(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 60(%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%ebp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (,%ebp,8), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $24, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $60, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    subl %ebp, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    notb %bl
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %eax, %ecx, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ebp, %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %ecx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edi, %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 28(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edi, %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 32(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 36(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edi, %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 40(%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %esi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 44(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %eax, %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %eax, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 48(%edx), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 52(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %esi, %ecx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %esi, %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 56(%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ebp, %edi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %edi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    negl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ebp, 188(%esp,%ebx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ecx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, (%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 56(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, 60(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, 48(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 52(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 40(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 44(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 32(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 36(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 24(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 28(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 16(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 20(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 8(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 4(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: shl_64bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    subl $204, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 28(%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 32(%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 36(%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 40(%ebp), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 44(%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 48(%ebp), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 52(%ebp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 56(%ebp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 60(%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%ebp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    xorps %xmm0, %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (,%ebp,8), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    subl %ebp, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %esi, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 28(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %esi, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 32(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 36(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 40(%eax), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 44(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %ebx, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %esi, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 56(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 60(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 52(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    negl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 176(%esp,%ebp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, 56(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, 60(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %ecx, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %ebp, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shldl %cl, %edx, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, 48(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, 52(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, 40(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 44(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 32(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 36(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 24(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 28(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 16(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 20(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 8(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 12(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, 4(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, (%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    addl $204, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-SSE4-LABEL: shl_64bytes:
+; X86-NO-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    subl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 32(%ecx), %xmm2
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 48(%ecx), %xmm3
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm4, %xmm4
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm3, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    andl $60, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    subl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl (%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 4(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll $3, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    andl $24, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %ch
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    notb %ch
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 12(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 8(%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 20(%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 16(%edi), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 28(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 24(%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %esi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 36(%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 32(%edx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 44(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 40(%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 52(%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    negl %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 176(%esp,%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 60(%edi), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 56(%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, (%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, 56(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, 60(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 48(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, 52(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 40(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 44(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 32(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 36(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-SSE4-LABEL: shl_64bytes:
+; X86-HAVE-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 32(%ecx), %xmm2
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 48(%ecx), %xmm3
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl (%eax), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    xorps %xmm4, %xmm4
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm3, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    subl %ebp, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 8(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 12(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %esi, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 4(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 16(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 20(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 24(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 28(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %esi, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 32(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 36(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 40(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 44(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 56(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 60(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl (%eax), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 52(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    negl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 160(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, 56(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, 60(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %ebx, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shldl %cl, %edi, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 48(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, 52(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 40(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-SSE4-LABEL: shl_64bytes:
+; X86-NO-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    subl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 32(%ecx), %xmm2
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 48(%ecx), %xmm3
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl (%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm4, %xmm4
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm3, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (,%eax,8), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $24, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $60, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    subl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl (%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 4(%edx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    notb %bl
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ecx, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 8(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 12(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ecx, %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %ecx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 16(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 20(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edi, %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 24(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 28(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edi, %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 32(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 36(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edi, %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 40(%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %esi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 44(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %eax, %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %eax, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 48(%edx), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 52(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %esi, %ecx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %esi, %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 56(%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebp, %edi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %edi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    negl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ebp, 188(%esp,%ebx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ecx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, (%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 56(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, 60(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, 48(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 52(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 40(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 44(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 32(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 36(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 24(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 28(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 16(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 20(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 8(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 4(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: shl_64bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    subl $204, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 32(%ecx), %xmm2
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 48(%ecx), %xmm3
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl (%eax), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    xorps %xmm4, %xmm4
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm3, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (,%ebp,8), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    subl %ebp, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 4(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 8(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 12(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %esi, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 16(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 20(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 24(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 28(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %esi, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 32(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 36(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 40(%eax), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 44(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %ebx, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %esi, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 56(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 60(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl (%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 52(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    negl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 176(%esp,%ebp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, 56(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, 60(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %ecx, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %ebp, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shldl %cl, %edx, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, 48(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, 52(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, 40(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 44(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 32(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 36(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 24(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 28(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 16(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 20(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 8(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 12(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, 4(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, (%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    addl $204, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-AVX1-LABEL: shl_64bytes:
+; X86-NO-SHLD-NO-BMI2-AVX1:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    subl $204, %esp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups (%ecx), %ymm0
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups 32(%ecx), %ymm1
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    andl $60, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    subl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl (%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 4(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll $3, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    andl $24, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %ch
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    notb %ch
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 12(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 8(%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebp, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 20(%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 16(%edi), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebp, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 28(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 24(%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %esi, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 36(%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 32(%edx), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 44(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 40(%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 52(%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    negl %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 176(%esp,%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 60(%edi), %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl 56(%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, (%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, 56(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, 60(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 48(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebp, 52(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 40(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 44(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 32(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 36(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    vzeroupper
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-AVX1-LABEL: shl_64bytes:
+; X86-HAVE-SHLD-NO-BMI2-AVX1:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups (%ecx), %ymm0
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups 32(%ecx), %ymm1
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl (%eax), %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    subl %ebp, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 8(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 12(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldl %cl, %esi, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 4(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldl %cl, %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 16(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 20(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 24(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 28(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldl %cl, %esi, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 32(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 36(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 40(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 44(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 56(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 60(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl (%eax), %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 52(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    negl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl 160(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, 56(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edi, 60(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldl %cl, %ebx, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shll %cl, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    shldl %cl, %edi, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 48(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %esi, 52(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 40(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %ebx, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    movl %edx, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    vzeroupper
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-AVX1-LABEL: shl_64bytes:
+; X86-NO-SHLD-HAVE-BMI2-AVX1:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    subl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups (%ecx), %ymm0
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups 32(%ecx), %ymm1
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl (%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    leal (,%eax,8), %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    andl $24, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ebx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    andl $60, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    leal {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    subl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl (%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 4(%edx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    notb %bl
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %ecx, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 8(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 12(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %ecx, %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %ecx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 16(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 20(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edi, %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 24(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 28(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edi, %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 32(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 36(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edi, %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %ebp, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 40(%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %esi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 44(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %eax, %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %ebp, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %eax, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrl %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 48(%edx), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 52(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %esi, %ecx, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %esi, %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrl %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 56(%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %ebp, %edi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shrxl %ebx, %edi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    negl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %ebp, 188(%esp,%ebx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    orl %ecx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, (%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 56(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ebx, 60(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %esi, 48(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 52(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 40(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 44(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 32(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 36(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 24(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 28(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 16(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 20(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 8(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %eax, 4(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    addl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    vzeroupper
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-LABEL: shl_64bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    subl $204, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups (%ecx), %ymm0
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups 32(%ecx), %ymm1
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl (%eax), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    leal (,%ebx,8), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    andl $60, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    subl %ebx, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 4(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 8(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 12(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edx, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldl %cl, %edi, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldl %cl, %esi, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 16(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 20(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %esi, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldl %cl, %edi, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 24(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 28(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edx, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldl %cl, %edi, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldl %cl, %esi, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 32(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 36(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %esi, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldl %cl, %edi, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 40(%eax), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 44(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldl %cl, %ebp, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldl %cl, %esi, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 56(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 60(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl (%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 52(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    negl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl 176(%esp,%ebx), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edx, 56(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, 60(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shlxl %ecx, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldl %cl, %ebx, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    shldl %cl, %edx, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ebx, 48(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %esi, 52(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ebp, 40(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, 44(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, 32(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, 36(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, 24(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, 28(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, 16(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, 20(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, 8(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, 12(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %edi, 4(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    movl %ecx, (%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    addl $204, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    vzeroupper
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-AVX512-LABEL: shl_64bytes:
+; X86-NO-SHLD-NO-BMI2-AVX512:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    subl $204, %esp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    vmovups (%ecx), %zmm0
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    vmovups %zmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    vmovups %zmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    andl $60, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    subl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl (%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 4(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll $3, %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    andl $24, %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %ch
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    notb %ch
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 12(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 8(%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebp, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 20(%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 16(%edi), %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebp, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 28(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 24(%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %esi, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 36(%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 32(%edx), %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 44(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 40(%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 52(%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    negl %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 176(%esp,%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 60(%edi), %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl 56(%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, (%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, 56(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, 60(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 48(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebp, 52(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 40(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 44(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 32(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 36(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    vzeroupper
+; X86-NO-SHLD-NO-BMI2-AVX512-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-AVX512-LABEL: shl_64bytes:
+; X86-HAVE-SHLD-NO-BMI2-AVX512:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vmovups (%ecx), %zmm0
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl (%eax), %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vmovups %zmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vmovups %zmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    subl %ebp, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 8(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 12(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldl %cl, %esi, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 4(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldl %cl, %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 16(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 20(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 24(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 28(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldl %cl, %esi, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 32(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 36(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldl %cl, %edi, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 40(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 44(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 56(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 60(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl (%eax), %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 52(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    negl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl 160(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, 56(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edi, 60(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldl %cl, %ebx, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shll %cl, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    shldl %cl, %edi, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 48(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %esi, 52(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 40(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %ebx, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    movl %edx, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    vzeroupper
+; X86-HAVE-SHLD-NO-BMI2-AVX512-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-AVX512-LABEL: shl_64bytes:
+; X86-NO-SHLD-HAVE-BMI2-AVX512:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    subl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups (%ecx), %zmm0
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl (%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups %zmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups %zmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    leal (,%eax,8), %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    andl $24, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ebx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    andl $60, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    leal {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    subl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl (%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 4(%edx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    notb %bl
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %ecx, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 8(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 12(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %ecx, %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %ecx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 16(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 20(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edi, %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 24(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 28(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edi, %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 32(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 36(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edi, %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %ebp, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 40(%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %esi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 44(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %eax, %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %ebp, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %eax, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrl %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 48(%edx), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 52(%edx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %esi, %ecx, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %esi, %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrl %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 56(%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrl %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %ebp, %edi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shrxl %ebx, %edi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    negl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %ebp, 188(%esp,%ebx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    orl %ecx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, (%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 56(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ebx, 60(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %esi, 48(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 52(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 40(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 44(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 32(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 36(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 24(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 28(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 16(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 20(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 8(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %eax, 4(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    addl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    vzeroupper
+; X86-NO-SHLD-HAVE-BMI2-AVX512-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-LABEL: shl_64bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    subl $204, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups (%ecx), %zmm0
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl (%eax), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups %zmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vmovups %zmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    leal (,%ebx,8), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    andl $60, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    subl %ebx, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 4(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 8(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 12(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edx, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldl %cl, %edi, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldl %cl, %esi, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 16(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 20(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %esi, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldl %cl, %edi, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 24(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 28(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edx, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldl %cl, %edi, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldl %cl, %esi, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 32(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 36(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %esi, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldl %cl, %edi, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 40(%eax), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 44(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldl %cl, %ebp, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldl %cl, %esi, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 56(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 60(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl (%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 52(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    negl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl 176(%esp,%ebx), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edx, 56(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, 60(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shlxl %ecx, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldl %cl, %ebx, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    shldl %cl, %edx, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ebx, 48(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %esi, 52(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ebp, 40(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, 44(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, 32(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, 36(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, 24(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, 28(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, 16(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, 20(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, 8(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, 12(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %edi, 4(%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    movl %ecx, (%eax)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    addl $204, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    vzeroupper
+; X86-HAVE-SHLD-HAVE-BMI2-AVX512-NEXT:    retl
   %src = load i512, ptr %src.ptr, align 1
   %byteOff = load i512, ptr %byteOff.ptr, align 1
   %bitOff = shl i512 %byteOff, 3
@@ -20136,4087 +17857,3115 @@ define void @shl_64bytes_qwordOff(ptr %src.ptr, ptr %qwordOff.ptr, ptr %dst) nou
 }
 
 define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
-; FALLBACK0-LABEL: ashr_64bytes:
-; FALLBACK0:       # %bb.0:
-; FALLBACK0-NEXT:    pushq %r15
-; FALLBACK0-NEXT:    pushq %r14
-; FALLBACK0-NEXT:    pushq %r13
-; FALLBACK0-NEXT:    pushq %r12
-; FALLBACK0-NEXT:    pushq %rbx
-; FALLBACK0-NEXT:    movq (%rdi), %rax
-; FALLBACK0-NEXT:    movq 8(%rdi), %rcx
-; FALLBACK0-NEXT:    movq 16(%rdi), %r8
-; FALLBACK0-NEXT:    movq 24(%rdi), %r9
-; FALLBACK0-NEXT:    movq 32(%rdi), %r10
-; FALLBACK0-NEXT:    movq 40(%rdi), %r11
-; FALLBACK0-NEXT:    movq 48(%rdi), %rbx
-; FALLBACK0-NEXT:    movq 56(%rdi), %r14
-; FALLBACK0-NEXT:    movl (%rsi), %edi
-; FALLBACK0-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    sarq $63, %r14
-; FALLBACK0-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
-; FALLBACK0-NEXT:    leal (,%rdi,8), %eax
-; FALLBACK0-NEXT:    andl $56, %eax
-; FALLBACK0-NEXT:    andl $56, %edi
-; FALLBACK0-NEXT:    movq -128(%rsp,%rdi), %r10
-; FALLBACK0-NEXT:    movq -120(%rsp,%rdi), %r8
-; FALLBACK0-NEXT:    movq %r8, %r11
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r11
-; FALLBACK0-NEXT:    movl %eax, %esi
-; FALLBACK0-NEXT:    notb %sil
-; FALLBACK0-NEXT:    movq -112(%rsp,%rdi), %rbx
-; FALLBACK0-NEXT:    leaq (%rbx,%rbx), %r9
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r9
-; FALLBACK0-NEXT:    orq %r11, %r9
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r10
-; FALLBACK0-NEXT:    addq %r8, %r8
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r8
-; FALLBACK0-NEXT:    orq %r10, %r8
-; FALLBACK0-NEXT:    movq -104(%rsp,%rdi), %r10
-; FALLBACK0-NEXT:    movq %r10, %r15
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r15
-; FALLBACK0-NEXT:    movq -96(%rsp,%rdi), %r14
-; FALLBACK0-NEXT:    leaq (%r14,%r14), %r11
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r11
-; FALLBACK0-NEXT:    orq %r15, %r11
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %rbx
-; FALLBACK0-NEXT:    addq %r10, %r10
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r10
-; FALLBACK0-NEXT:    orq %rbx, %r10
-; FALLBACK0-NEXT:    movq -88(%rsp,%rdi), %rbx
-; FALLBACK0-NEXT:    movq %rbx, %r12
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r12
-; FALLBACK0-NEXT:    movq -80(%rsp,%rdi), %r13
-; FALLBACK0-NEXT:    leaq (%r13,%r13), %r15
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r15
-; FALLBACK0-NEXT:    orq %r12, %r15
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r14
-; FALLBACK0-NEXT:    addq %rbx, %rbx
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %rbx
-; FALLBACK0-NEXT:    orq %r14, %rbx
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    shrq %cl, %r13
-; FALLBACK0-NEXT:    movq -72(%rsp,%rdi), %rdi
-; FALLBACK0-NEXT:    leaq (%rdi,%rdi), %r14
-; FALLBACK0-NEXT:    movl %esi, %ecx
-; FALLBACK0-NEXT:    shlq %cl, %r14
-; FALLBACK0-NEXT:    orq %r13, %r14
-; FALLBACK0-NEXT:    movl %eax, %ecx
-; FALLBACK0-NEXT:    sarq %cl, %rdi
-; FALLBACK0-NEXT:    movq %rdi, 56(%rdx)
-; FALLBACK0-NEXT:    movq %r14, 48(%rdx)
-; FALLBACK0-NEXT:    movq %rbx, 32(%rdx)
-; FALLBACK0-NEXT:    movq %r15, 40(%rdx)
-; FALLBACK0-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK0-NEXT:    movq %r11, 24(%rdx)
-; FALLBACK0-NEXT:    movq %r8, (%rdx)
-; FALLBACK0-NEXT:    movq %r9, 8(%rdx)
-; FALLBACK0-NEXT:    popq %rbx
-; FALLBACK0-NEXT:    popq %r12
-; FALLBACK0-NEXT:    popq %r13
-; FALLBACK0-NEXT:    popq %r14
-; FALLBACK0-NEXT:    popq %r15
-; FALLBACK0-NEXT:    retq
-;
-; FALLBACK1-LABEL: ashr_64bytes:
-; FALLBACK1:       # %bb.0:
-; FALLBACK1-NEXT:    pushq %r15
-; FALLBACK1-NEXT:    pushq %r14
-; FALLBACK1-NEXT:    pushq %rbx
-; FALLBACK1-NEXT:    movq (%rdi), %rcx
-; FALLBACK1-NEXT:    movq 8(%rdi), %r8
-; FALLBACK1-NEXT:    movq 16(%rdi), %r9
-; FALLBACK1-NEXT:    movq 24(%rdi), %r10
-; FALLBACK1-NEXT:    movq 32(%rdi), %r11
-; FALLBACK1-NEXT:    movq 40(%rdi), %rbx
-; FALLBACK1-NEXT:    movq 48(%rdi), %r14
-; FALLBACK1-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK1-NEXT:    movl (%rsi), %eax
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    sarq $63, %rdi
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK1-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK1-NEXT:    andl $56, %ecx
-; FALLBACK1-NEXT:    andl $56, %eax
-; FALLBACK1-NEXT:    movq -112(%rsp,%rax), %rdi
-; FALLBACK1-NEXT:    movq -128(%rsp,%rax), %rsi
-; FALLBACK1-NEXT:    movq -120(%rsp,%rax), %r9
-; FALLBACK1-NEXT:    movq %r9, %r8
-; FALLBACK1-NEXT:    shrdq %cl, %rdi, %r8
-; FALLBACK1-NEXT:    movq -96(%rsp,%rax), %r10
-; FALLBACK1-NEXT:    movq -104(%rsp,%rax), %r11
-; FALLBACK1-NEXT:    movq %r11, %rbx
-; FALLBACK1-NEXT:    shrdq %cl, %r10, %rbx
-; FALLBACK1-NEXT:    shrdq %cl, %r11, %rdi
-; FALLBACK1-NEXT:    movq -80(%rsp,%rax), %r11
-; FALLBACK1-NEXT:    movq -88(%rsp,%rax), %r14
-; FALLBACK1-NEXT:    movq %r14, %r15
-; FALLBACK1-NEXT:    shrdq %cl, %r11, %r15
-; FALLBACK1-NEXT:    shrdq %cl, %r14, %r10
-; FALLBACK1-NEXT:    movq -72(%rsp,%rax), %rax
-; FALLBACK1-NEXT:    shrdq %cl, %rax, %r11
-; FALLBACK1-NEXT:    shrdq %cl, %r9, %rsi
-; FALLBACK1-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK1-NEXT:    sarq %cl, %rax
-; FALLBACK1-NEXT:    movq %r11, 48(%rdx)
-; FALLBACK1-NEXT:    movq %rax, 56(%rdx)
-; FALLBACK1-NEXT:    movq %r10, 32(%rdx)
-; FALLBACK1-NEXT:    movq %r15, 40(%rdx)
-; FALLBACK1-NEXT:    movq %rdi, 16(%rdx)
-; FALLBACK1-NEXT:    movq %rbx, 24(%rdx)
-; FALLBACK1-NEXT:    movq %rsi, (%rdx)
-; FALLBACK1-NEXT:    movq %r8, 8(%rdx)
-; FALLBACK1-NEXT:    popq %rbx
-; FALLBACK1-NEXT:    popq %r14
-; FALLBACK1-NEXT:    popq %r15
-; FALLBACK1-NEXT:    retq
-;
-; FALLBACK2-LABEL: ashr_64bytes:
-; FALLBACK2:       # %bb.0:
-; FALLBACK2-NEXT:    pushq %r15
-; FALLBACK2-NEXT:    pushq %r14
-; FALLBACK2-NEXT:    pushq %r12
-; FALLBACK2-NEXT:    pushq %rbx
-; FALLBACK2-NEXT:    pushq %rax
-; FALLBACK2-NEXT:    movq (%rdi), %rcx
-; FALLBACK2-NEXT:    movq 8(%rdi), %r8
-; FALLBACK2-NEXT:    movq 16(%rdi), %r9
-; FALLBACK2-NEXT:    movq 24(%rdi), %r10
-; FALLBACK2-NEXT:    movq 32(%rdi), %r11
-; FALLBACK2-NEXT:    movq 40(%rdi), %rbx
-; FALLBACK2-NEXT:    movq 48(%rdi), %r14
-; FALLBACK2-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK2-NEXT:    movl (%rsi), %eax
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    sarq $63, %rdi
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK2-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK2-NEXT:    andl $56, %ecx
-; FALLBACK2-NEXT:    movl %ecx, %esi
-; FALLBACK2-NEXT:    andl $56, %eax
-; FALLBACK2-NEXT:    movq -120(%rsp,%rax), %r8
-; FALLBACK2-NEXT:    movq -112(%rsp,%rax), %r10
-; FALLBACK2-NEXT:    shrxq %rsi, %r8, %r9
-; FALLBACK2-NEXT:    notb %cl
-; FALLBACK2-NEXT:    leaq (%r10,%r10), %rdi
-; FALLBACK2-NEXT:    shlxq %rcx, %rdi, %rdi
-; FALLBACK2-NEXT:    orq %r9, %rdi
-; FALLBACK2-NEXT:    shrxq %rsi, -128(%rsp,%rax), %r9
-; FALLBACK2-NEXT:    addq %r8, %r8
-; FALLBACK2-NEXT:    shlxq %rcx, %r8, %r8
-; FALLBACK2-NEXT:    orq %r9, %r8
-; FALLBACK2-NEXT:    movq -104(%rsp,%rax), %r11
-; FALLBACK2-NEXT:    shrxq %rsi, %r11, %rbx
-; FALLBACK2-NEXT:    movq -96(%rsp,%rax), %r14
-; FALLBACK2-NEXT:    leaq (%r14,%r14), %r9
-; FALLBACK2-NEXT:    shlxq %rcx, %r9, %r9
-; FALLBACK2-NEXT:    orq %rbx, %r9
-; FALLBACK2-NEXT:    shrxq %rsi, %r10, %r10
-; FALLBACK2-NEXT:    addq %r11, %r11
-; FALLBACK2-NEXT:    shlxq %rcx, %r11, %r11
-; FALLBACK2-NEXT:    orq %r10, %r11
-; FALLBACK2-NEXT:    movq -88(%rsp,%rax), %r10
-; FALLBACK2-NEXT:    shrxq %rsi, %r10, %rbx
-; FALLBACK2-NEXT:    movq -80(%rsp,%rax), %r15
-; FALLBACK2-NEXT:    leaq (%r15,%r15), %r12
-; FALLBACK2-NEXT:    shlxq %rcx, %r12, %r12
-; FALLBACK2-NEXT:    orq %rbx, %r12
-; FALLBACK2-NEXT:    shrxq %rsi, %r14, %rbx
-; FALLBACK2-NEXT:    addq %r10, %r10
-; FALLBACK2-NEXT:    shlxq %rcx, %r10, %r10
-; FALLBACK2-NEXT:    orq %rbx, %r10
-; FALLBACK2-NEXT:    shrxq %rsi, %r15, %rbx
-; FALLBACK2-NEXT:    movq -72(%rsp,%rax), %rax
-; FALLBACK2-NEXT:    leaq (%rax,%rax), %r14
-; FALLBACK2-NEXT:    shlxq %rcx, %r14, %rcx
-; FALLBACK2-NEXT:    orq %rbx, %rcx
-; FALLBACK2-NEXT:    sarxq %rsi, %rax, %rax
-; FALLBACK2-NEXT:    movq %rax, 56(%rdx)
-; FALLBACK2-NEXT:    movq %rcx, 48(%rdx)
-; FALLBACK2-NEXT:    movq %r10, 32(%rdx)
-; FALLBACK2-NEXT:    movq %r12, 40(%rdx)
-; FALLBACK2-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK2-NEXT:    movq %r9, 24(%rdx)
-; FALLBACK2-NEXT:    movq %r8, (%rdx)
-; FALLBACK2-NEXT:    movq %rdi, 8(%rdx)
-; FALLBACK2-NEXT:    addq $8, %rsp
-; FALLBACK2-NEXT:    popq %rbx
-; FALLBACK2-NEXT:    popq %r12
-; FALLBACK2-NEXT:    popq %r14
-; FALLBACK2-NEXT:    popq %r15
-; FALLBACK2-NEXT:    retq
-;
-; FALLBACK3-LABEL: ashr_64bytes:
-; FALLBACK3:       # %bb.0:
-; FALLBACK3-NEXT:    pushq %r15
-; FALLBACK3-NEXT:    pushq %r14
-; FALLBACK3-NEXT:    pushq %rbx
-; FALLBACK3-NEXT:    movq (%rdi), %rcx
-; FALLBACK3-NEXT:    movq 8(%rdi), %r8
-; FALLBACK3-NEXT:    movq 16(%rdi), %r9
-; FALLBACK3-NEXT:    movq 24(%rdi), %r10
-; FALLBACK3-NEXT:    movq 32(%rdi), %r11
-; FALLBACK3-NEXT:    movq 40(%rdi), %rbx
-; FALLBACK3-NEXT:    movq 48(%rdi), %r14
-; FALLBACK3-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK3-NEXT:    movl (%rsi), %eax
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    sarq $63, %rdi
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK3-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK3-NEXT:    andl $56, %ecx
-; FALLBACK3-NEXT:    andl $56, %eax
-; FALLBACK3-NEXT:    movq -112(%rsp,%rax), %rdi
-; FALLBACK3-NEXT:    movq -128(%rsp,%rax), %rsi
-; FALLBACK3-NEXT:    movq -120(%rsp,%rax), %r9
-; FALLBACK3-NEXT:    movq %r9, %r8
-; FALLBACK3-NEXT:    shrdq %cl, %rdi, %r8
-; FALLBACK3-NEXT:    movq -96(%rsp,%rax), %r10
-; FALLBACK3-NEXT:    movq -104(%rsp,%rax), %r11
-; FALLBACK3-NEXT:    movq %r11, %rbx
-; FALLBACK3-NEXT:    shrdq %cl, %r10, %rbx
-; FALLBACK3-NEXT:    shrdq %cl, %r11, %rdi
-; FALLBACK3-NEXT:    movq -80(%rsp,%rax), %r11
-; FALLBACK3-NEXT:    movq -88(%rsp,%rax), %r14
-; FALLBACK3-NEXT:    movq %r14, %r15
-; FALLBACK3-NEXT:    shrdq %cl, %r11, %r15
-; FALLBACK3-NEXT:    shrdq %cl, %r14, %r10
-; FALLBACK3-NEXT:    movq -72(%rsp,%rax), %rax
-; FALLBACK3-NEXT:    shrdq %cl, %rax, %r11
-; FALLBACK3-NEXT:    sarxq %rcx, %rax, %rax
-; FALLBACK3-NEXT:    # kill: def $cl killed $cl killed $rcx
-; FALLBACK3-NEXT:    shrdq %cl, %r9, %rsi
-; FALLBACK3-NEXT:    movq %r11, 48(%rdx)
-; FALLBACK3-NEXT:    movq %r10, 32(%rdx)
-; FALLBACK3-NEXT:    movq %r15, 40(%rdx)
-; FALLBACK3-NEXT:    movq %rdi, 16(%rdx)
-; FALLBACK3-NEXT:    movq %rbx, 24(%rdx)
-; FALLBACK3-NEXT:    movq %rsi, (%rdx)
-; FALLBACK3-NEXT:    movq %r8, 8(%rdx)
-; FALLBACK3-NEXT:    movq %rax, 56(%rdx)
-; FALLBACK3-NEXT:    popq %rbx
-; FALLBACK3-NEXT:    popq %r14
-; FALLBACK3-NEXT:    popq %r15
-; FALLBACK3-NEXT:    retq
-;
-; FALLBACK4-LABEL: ashr_64bytes:
-; FALLBACK4:       # %bb.0:
-; FALLBACK4-NEXT:    pushq %rbp
-; FALLBACK4-NEXT:    pushq %r15
-; FALLBACK4-NEXT:    pushq %r14
-; FALLBACK4-NEXT:    pushq %r13
-; FALLBACK4-NEXT:    pushq %r12
-; FALLBACK4-NEXT:    pushq %rbx
-; FALLBACK4-NEXT:    pushq %rax
-; FALLBACK4-NEXT:    movups (%rdi), %xmm0
-; FALLBACK4-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK4-NEXT:    movups 32(%rdi), %xmm2
-; FALLBACK4-NEXT:    movq 48(%rdi), %rax
-; FALLBACK4-NEXT:    movq 56(%rdi), %rcx
-; FALLBACK4-NEXT:    movl (%rsi), %edi
-; FALLBACK4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    sarq $63, %rcx
-; FALLBACK4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK4-NEXT:    leal (,%rdi,8), %eax
-; FALLBACK4-NEXT:    andl $56, %eax
-; FALLBACK4-NEXT:    andl $56, %edi
-; FALLBACK4-NEXT:    movq -128(%rsp,%rdi), %r10
-; FALLBACK4-NEXT:    movq -120(%rsp,%rdi), %r9
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r10
-; FALLBACK4-NEXT:    movl %eax, %esi
-; FALLBACK4-NEXT:    notb %sil
-; FALLBACK4-NEXT:    leaq (%r9,%r9), %r8
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r8
-; FALLBACK4-NEXT:    orq %r10, %r8
-; FALLBACK4-NEXT:    movq -104(%rsp,%rdi), %r10
-; FALLBACK4-NEXT:    movq %r10, %rbx
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %rbx
-; FALLBACK4-NEXT:    movq -96(%rsp,%rdi), %r12
-; FALLBACK4-NEXT:    leaq (%r12,%r12), %r11
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r11
-; FALLBACK4-NEXT:    orq %rbx, %r11
-; FALLBACK4-NEXT:    movq -112(%rsp,%rdi), %rbx
-; FALLBACK4-NEXT:    movq %rbx, %r14
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r14
-; FALLBACK4-NEXT:    addq %r10, %r10
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r10
-; FALLBACK4-NEXT:    orq %r14, %r10
-; FALLBACK4-NEXT:    movq -88(%rsp,%rdi), %r14
-; FALLBACK4-NEXT:    movq %r14, %r13
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r13
-; FALLBACK4-NEXT:    movq -80(%rsp,%rdi), %rbp
-; FALLBACK4-NEXT:    leaq (%rbp,%rbp), %r15
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r15
-; FALLBACK4-NEXT:    orq %r13, %r15
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r12
-; FALLBACK4-NEXT:    addq %r14, %r14
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r14
-; FALLBACK4-NEXT:    orq %r12, %r14
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %rbp
-; FALLBACK4-NEXT:    movq -72(%rsp,%rdi), %rdi
-; FALLBACK4-NEXT:    leaq (%rdi,%rdi), %r12
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %r12
-; FALLBACK4-NEXT:    orq %rbp, %r12
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    shrq %cl, %r9
-; FALLBACK4-NEXT:    addq %rbx, %rbx
-; FALLBACK4-NEXT:    movl %esi, %ecx
-; FALLBACK4-NEXT:    shlq %cl, %rbx
-; FALLBACK4-NEXT:    orq %r9, %rbx
-; FALLBACK4-NEXT:    movl %eax, %ecx
-; FALLBACK4-NEXT:    sarq %cl, %rdi
-; FALLBACK4-NEXT:    movq %rdi, 56(%rdx)
-; FALLBACK4-NEXT:    movq %rbx, 8(%rdx)
-; FALLBACK4-NEXT:    movq %r12, 48(%rdx)
-; FALLBACK4-NEXT:    movq %r14, 32(%rdx)
-; FALLBACK4-NEXT:    movq %r15, 40(%rdx)
-; FALLBACK4-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK4-NEXT:    movq %r11, 24(%rdx)
-; FALLBACK4-NEXT:    movq %r8, (%rdx)
-; FALLBACK4-NEXT:    addq $8, %rsp
-; FALLBACK4-NEXT:    popq %rbx
-; FALLBACK4-NEXT:    popq %r12
-; FALLBACK4-NEXT:    popq %r13
-; FALLBACK4-NEXT:    popq %r14
-; FALLBACK4-NEXT:    popq %r15
-; FALLBACK4-NEXT:    popq %rbp
-; FALLBACK4-NEXT:    retq
-;
-; FALLBACK5-LABEL: ashr_64bytes:
-; FALLBACK5:       # %bb.0:
-; FALLBACK5-NEXT:    pushq %r15
-; FALLBACK5-NEXT:    pushq %r14
-; FALLBACK5-NEXT:    pushq %rbx
-; FALLBACK5-NEXT:    movups (%rdi), %xmm0
-; FALLBACK5-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK5-NEXT:    movups 32(%rdi), %xmm2
-; FALLBACK5-NEXT:    movq 48(%rdi), %rcx
-; FALLBACK5-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK5-NEXT:    movl (%rsi), %eax
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    sarq $63, %rdi
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK5-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK5-NEXT:    andl $56, %ecx
-; FALLBACK5-NEXT:    andl $56, %eax
-; FALLBACK5-NEXT:    movq -96(%rsp,%rax), %rdi
-; FALLBACK5-NEXT:    movq -104(%rsp,%rax), %r9
-; FALLBACK5-NEXT:    movq %r9, %rsi
-; FALLBACK5-NEXT:    shrdq %cl, %rdi, %rsi
-; FALLBACK5-NEXT:    movq -112(%rsp,%rax), %r10
-; FALLBACK5-NEXT:    movq %r10, %r8
-; FALLBACK5-NEXT:    shrdq %cl, %r9, %r8
-; FALLBACK5-NEXT:    movq -80(%rsp,%rax), %r9
-; FALLBACK5-NEXT:    movq -88(%rsp,%rax), %r11
-; FALLBACK5-NEXT:    movq %r11, %rbx
-; FALLBACK5-NEXT:    shrdq %cl, %r9, %rbx
-; FALLBACK5-NEXT:    shrdq %cl, %r11, %rdi
-; FALLBACK5-NEXT:    movq -72(%rsp,%rax), %r11
-; FALLBACK5-NEXT:    shrdq %cl, %r11, %r9
-; FALLBACK5-NEXT:    movq -128(%rsp,%rax), %r14
-; FALLBACK5-NEXT:    movq -120(%rsp,%rax), %rax
-; FALLBACK5-NEXT:    movq %rax, %r15
-; FALLBACK5-NEXT:    shrdq %cl, %r10, %r15
-; FALLBACK5-NEXT:    shrdq %cl, %rax, %r14
-; FALLBACK5-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK5-NEXT:    sarq %cl, %r11
-; FALLBACK5-NEXT:    movq %r15, 8(%rdx)
-; FALLBACK5-NEXT:    movq %r9, 48(%rdx)
-; FALLBACK5-NEXT:    movq %r11, 56(%rdx)
-; FALLBACK5-NEXT:    movq %rdi, 32(%rdx)
-; FALLBACK5-NEXT:    movq %rbx, 40(%rdx)
-; FALLBACK5-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK5-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK5-NEXT:    movq %r14, (%rdx)
-; FALLBACK5-NEXT:    popq %rbx
-; FALLBACK5-NEXT:    popq %r14
-; FALLBACK5-NEXT:    popq %r15
-; FALLBACK5-NEXT:    retq
-;
-; FALLBACK6-LABEL: ashr_64bytes:
-; FALLBACK6:       # %bb.0:
-; FALLBACK6-NEXT:    pushq %r15
-; FALLBACK6-NEXT:    pushq %r14
-; FALLBACK6-NEXT:    pushq %r13
-; FALLBACK6-NEXT:    pushq %r12
-; FALLBACK6-NEXT:    pushq %rbx
-; FALLBACK6-NEXT:    movups (%rdi), %xmm0
-; FALLBACK6-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK6-NEXT:    movups 32(%rdi), %xmm2
-; FALLBACK6-NEXT:    movq 48(%rdi), %rcx
-; FALLBACK6-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK6-NEXT:    movl (%rsi), %eax
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    sarq $63, %rdi
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK6-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK6-NEXT:    andl $56, %ecx
-; FALLBACK6-NEXT:    movl %ecx, %esi
-; FALLBACK6-NEXT:    andl $56, %eax
-; FALLBACK6-NEXT:    shrxq %rsi, -128(%rsp,%rax), %r8
-; FALLBACK6-NEXT:    notb %cl
-; FALLBACK6-NEXT:    movq -120(%rsp,%rax), %r10
-; FALLBACK6-NEXT:    movq -112(%rsp,%rax), %r9
-; FALLBACK6-NEXT:    leaq (%r10,%r10), %rdi
-; FALLBACK6-NEXT:    shlxq %rcx, %rdi, %rdi
-; FALLBACK6-NEXT:    orq %r8, %rdi
-; FALLBACK6-NEXT:    movq -104(%rsp,%rax), %r11
-; FALLBACK6-NEXT:    shrxq %rsi, %r11, %rbx
-; FALLBACK6-NEXT:    movq -96(%rsp,%rax), %r14
-; FALLBACK6-NEXT:    leaq (%r14,%r14), %r8
-; FALLBACK6-NEXT:    shlxq %rcx, %r8, %r8
-; FALLBACK6-NEXT:    orq %rbx, %r8
-; FALLBACK6-NEXT:    shrxq %rsi, %r9, %rbx
-; FALLBACK6-NEXT:    addq %r11, %r11
-; FALLBACK6-NEXT:    shlxq %rcx, %r11, %r11
-; FALLBACK6-NEXT:    orq %rbx, %r11
-; FALLBACK6-NEXT:    movq -88(%rsp,%rax), %rbx
-; FALLBACK6-NEXT:    shrxq %rsi, %rbx, %r15
-; FALLBACK6-NEXT:    movq -80(%rsp,%rax), %r12
-; FALLBACK6-NEXT:    leaq (%r12,%r12), %r13
-; FALLBACK6-NEXT:    shlxq %rcx, %r13, %r13
-; FALLBACK6-NEXT:    orq %r15, %r13
-; FALLBACK6-NEXT:    shrxq %rsi, %r14, %r14
-; FALLBACK6-NEXT:    addq %rbx, %rbx
-; FALLBACK6-NEXT:    shlxq %rcx, %rbx, %rbx
-; FALLBACK6-NEXT:    orq %r14, %rbx
-; FALLBACK6-NEXT:    shrxq %rsi, %r12, %r14
-; FALLBACK6-NEXT:    movq -72(%rsp,%rax), %rax
-; FALLBACK6-NEXT:    leaq (%rax,%rax), %r15
-; FALLBACK6-NEXT:    shlxq %rcx, %r15, %r15
-; FALLBACK6-NEXT:    orq %r14, %r15
-; FALLBACK6-NEXT:    shrxq %rsi, %r10, %r10
-; FALLBACK6-NEXT:    addq %r9, %r9
-; FALLBACK6-NEXT:    shlxq %rcx, %r9, %rcx
-; FALLBACK6-NEXT:    orq %r10, %rcx
-; FALLBACK6-NEXT:    sarxq %rsi, %rax, %rax
-; FALLBACK6-NEXT:    movq %rax, 56(%rdx)
-; FALLBACK6-NEXT:    movq %rcx, 8(%rdx)
-; FALLBACK6-NEXT:    movq %r15, 48(%rdx)
-; FALLBACK6-NEXT:    movq %rbx, 32(%rdx)
-; FALLBACK6-NEXT:    movq %r13, 40(%rdx)
-; FALLBACK6-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK6-NEXT:    movq %r8, 24(%rdx)
-; FALLBACK6-NEXT:    movq %rdi, (%rdx)
-; FALLBACK6-NEXT:    popq %rbx
-; FALLBACK6-NEXT:    popq %r12
-; FALLBACK6-NEXT:    popq %r13
-; FALLBACK6-NEXT:    popq %r14
-; FALLBACK6-NEXT:    popq %r15
-; FALLBACK6-NEXT:    retq
-;
-; FALLBACK7-LABEL: ashr_64bytes:
-; FALLBACK7:       # %bb.0:
-; FALLBACK7-NEXT:    pushq %r15
-; FALLBACK7-NEXT:    pushq %r14
-; FALLBACK7-NEXT:    pushq %rbx
-; FALLBACK7-NEXT:    movups (%rdi), %xmm0
-; FALLBACK7-NEXT:    movups 16(%rdi), %xmm1
-; FALLBACK7-NEXT:    movups 32(%rdi), %xmm2
-; FALLBACK7-NEXT:    movq 48(%rdi), %rcx
-; FALLBACK7-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK7-NEXT:    movl (%rsi), %eax
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    sarq $63, %rdi
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK7-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK7-NEXT:    andl $56, %ecx
-; FALLBACK7-NEXT:    andl $56, %eax
-; FALLBACK7-NEXT:    movq -96(%rsp,%rax), %rdi
-; FALLBACK7-NEXT:    movq -104(%rsp,%rax), %r9
-; FALLBACK7-NEXT:    movq %r9, %rsi
-; FALLBACK7-NEXT:    shrdq %cl, %rdi, %rsi
-; FALLBACK7-NEXT:    movq -112(%rsp,%rax), %r10
-; FALLBACK7-NEXT:    movq %r10, %r8
-; FALLBACK7-NEXT:    shrdq %cl, %r9, %r8
-; FALLBACK7-NEXT:    movq -80(%rsp,%rax), %r9
-; FALLBACK7-NEXT:    movq -88(%rsp,%rax), %r11
-; FALLBACK7-NEXT:    movq %r11, %rbx
-; FALLBACK7-NEXT:    shrdq %cl, %r9, %rbx
-; FALLBACK7-NEXT:    shrdq %cl, %r11, %rdi
-; FALLBACK7-NEXT:    movq -72(%rsp,%rax), %r11
-; FALLBACK7-NEXT:    shrdq %cl, %r11, %r9
-; FALLBACK7-NEXT:    movq -128(%rsp,%rax), %r14
-; FALLBACK7-NEXT:    movq -120(%rsp,%rax), %rax
-; FALLBACK7-NEXT:    movq %rax, %r15
-; FALLBACK7-NEXT:    shrdq %cl, %r10, %r15
-; FALLBACK7-NEXT:    sarxq %rcx, %r11, %r10
-; FALLBACK7-NEXT:    # kill: def $cl killed $cl killed $rcx
-; FALLBACK7-NEXT:    shrdq %cl, %rax, %r14
-; FALLBACK7-NEXT:    movq %r15, 8(%rdx)
-; FALLBACK7-NEXT:    movq %r9, 48(%rdx)
-; FALLBACK7-NEXT:    movq %rdi, 32(%rdx)
-; FALLBACK7-NEXT:    movq %rbx, 40(%rdx)
-; FALLBACK7-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK7-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK7-NEXT:    movq %r14, (%rdx)
-; FALLBACK7-NEXT:    movq %r10, 56(%rdx)
-; FALLBACK7-NEXT:    popq %rbx
-; FALLBACK7-NEXT:    popq %r14
-; FALLBACK7-NEXT:    popq %r15
-; FALLBACK7-NEXT:    retq
-;
-; FALLBACK8-LABEL: ashr_64bytes:
-; FALLBACK8:       # %bb.0:
-; FALLBACK8-NEXT:    pushq %rbp
-; FALLBACK8-NEXT:    pushq %r15
-; FALLBACK8-NEXT:    pushq %r14
-; FALLBACK8-NEXT:    pushq %r13
-; FALLBACK8-NEXT:    pushq %r12
-; FALLBACK8-NEXT:    pushq %rbx
-; FALLBACK8-NEXT:    pushq %rax
-; FALLBACK8-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK8-NEXT:    vmovups 32(%rdi), %xmm1
-; FALLBACK8-NEXT:    movq 48(%rdi), %rax
-; FALLBACK8-NEXT:    movq 56(%rdi), %rcx
-; FALLBACK8-NEXT:    movl (%rsi), %edi
-; FALLBACK8-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    vmovaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    sarq $63, %rcx
-; FALLBACK8-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK8-NEXT:    leal (,%rdi,8), %eax
-; FALLBACK8-NEXT:    andl $56, %eax
-; FALLBACK8-NEXT:    andl $56, %edi
-; FALLBACK8-NEXT:    movq -128(%rsp,%rdi), %r10
-; FALLBACK8-NEXT:    movq -120(%rsp,%rdi), %r9
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r10
-; FALLBACK8-NEXT:    movl %eax, %esi
-; FALLBACK8-NEXT:    notb %sil
-; FALLBACK8-NEXT:    leaq (%r9,%r9), %r8
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r8
-; FALLBACK8-NEXT:    orq %r10, %r8
-; FALLBACK8-NEXT:    movq -104(%rsp,%rdi), %r10
-; FALLBACK8-NEXT:    movq %r10, %rbx
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %rbx
-; FALLBACK8-NEXT:    movq -96(%rsp,%rdi), %r12
-; FALLBACK8-NEXT:    leaq (%r12,%r12), %r11
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r11
-; FALLBACK8-NEXT:    orq %rbx, %r11
-; FALLBACK8-NEXT:    movq -112(%rsp,%rdi), %rbx
-; FALLBACK8-NEXT:    movq %rbx, %r14
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r14
-; FALLBACK8-NEXT:    addq %r10, %r10
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r10
-; FALLBACK8-NEXT:    orq %r14, %r10
-; FALLBACK8-NEXT:    movq -88(%rsp,%rdi), %r14
-; FALLBACK8-NEXT:    movq %r14, %r13
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r13
-; FALLBACK8-NEXT:    movq -80(%rsp,%rdi), %rbp
-; FALLBACK8-NEXT:    leaq (%rbp,%rbp), %r15
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r15
-; FALLBACK8-NEXT:    orq %r13, %r15
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r12
-; FALLBACK8-NEXT:    addq %r14, %r14
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r14
-; FALLBACK8-NEXT:    orq %r12, %r14
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %rbp
-; FALLBACK8-NEXT:    movq -72(%rsp,%rdi), %rdi
-; FALLBACK8-NEXT:    leaq (%rdi,%rdi), %r12
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %r12
-; FALLBACK8-NEXT:    orq %rbp, %r12
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    shrq %cl, %r9
-; FALLBACK8-NEXT:    addq %rbx, %rbx
-; FALLBACK8-NEXT:    movl %esi, %ecx
-; FALLBACK8-NEXT:    shlq %cl, %rbx
-; FALLBACK8-NEXT:    orq %r9, %rbx
-; FALLBACK8-NEXT:    movl %eax, %ecx
-; FALLBACK8-NEXT:    sarq %cl, %rdi
-; FALLBACK8-NEXT:    movq %rdi, 56(%rdx)
-; FALLBACK8-NEXT:    movq %rbx, 8(%rdx)
-; FALLBACK8-NEXT:    movq %r12, 48(%rdx)
-; FALLBACK8-NEXT:    movq %r14, 32(%rdx)
-; FALLBACK8-NEXT:    movq %r15, 40(%rdx)
-; FALLBACK8-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK8-NEXT:    movq %r11, 24(%rdx)
-; FALLBACK8-NEXT:    movq %r8, (%rdx)
-; FALLBACK8-NEXT:    addq $8, %rsp
-; FALLBACK8-NEXT:    popq %rbx
-; FALLBACK8-NEXT:    popq %r12
-; FALLBACK8-NEXT:    popq %r13
-; FALLBACK8-NEXT:    popq %r14
-; FALLBACK8-NEXT:    popq %r15
-; FALLBACK8-NEXT:    popq %rbp
-; FALLBACK8-NEXT:    vzeroupper
-; FALLBACK8-NEXT:    retq
-;
-; FALLBACK9-LABEL: ashr_64bytes:
-; FALLBACK9:       # %bb.0:
-; FALLBACK9-NEXT:    pushq %r15
-; FALLBACK9-NEXT:    pushq %r14
-; FALLBACK9-NEXT:    pushq %rbx
-; FALLBACK9-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK9-NEXT:    vmovups 32(%rdi), %xmm1
-; FALLBACK9-NEXT:    movq 48(%rdi), %rcx
-; FALLBACK9-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK9-NEXT:    movl (%rsi), %eax
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    vmovaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    sarq $63, %rdi
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK9-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK9-NEXT:    andl $56, %ecx
-; FALLBACK9-NEXT:    andl $56, %eax
-; FALLBACK9-NEXT:    movq -96(%rsp,%rax), %rdi
-; FALLBACK9-NEXT:    movq -104(%rsp,%rax), %r9
-; FALLBACK9-NEXT:    movq %r9, %rsi
-; FALLBACK9-NEXT:    shrdq %cl, %rdi, %rsi
-; FALLBACK9-NEXT:    movq -112(%rsp,%rax), %r10
-; FALLBACK9-NEXT:    movq %r10, %r8
-; FALLBACK9-NEXT:    shrdq %cl, %r9, %r8
-; FALLBACK9-NEXT:    movq -80(%rsp,%rax), %r9
-; FALLBACK9-NEXT:    movq -88(%rsp,%rax), %r11
-; FALLBACK9-NEXT:    movq %r11, %rbx
-; FALLBACK9-NEXT:    shrdq %cl, %r9, %rbx
-; FALLBACK9-NEXT:    shrdq %cl, %r11, %rdi
-; FALLBACK9-NEXT:    movq -72(%rsp,%rax), %r11
-; FALLBACK9-NEXT:    shrdq %cl, %r11, %r9
-; FALLBACK9-NEXT:    movq -128(%rsp,%rax), %r14
-; FALLBACK9-NEXT:    movq -120(%rsp,%rax), %rax
-; FALLBACK9-NEXT:    movq %rax, %r15
-; FALLBACK9-NEXT:    shrdq %cl, %r10, %r15
-; FALLBACK9-NEXT:    shrdq %cl, %rax, %r14
-; FALLBACK9-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK9-NEXT:    sarq %cl, %r11
-; FALLBACK9-NEXT:    movq %r15, 8(%rdx)
-; FALLBACK9-NEXT:    movq %r9, 48(%rdx)
-; FALLBACK9-NEXT:    movq %r11, 56(%rdx)
-; FALLBACK9-NEXT:    movq %rdi, 32(%rdx)
-; FALLBACK9-NEXT:    movq %rbx, 40(%rdx)
-; FALLBACK9-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK9-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK9-NEXT:    movq %r14, (%rdx)
-; FALLBACK9-NEXT:    popq %rbx
-; FALLBACK9-NEXT:    popq %r14
-; FALLBACK9-NEXT:    popq %r15
-; FALLBACK9-NEXT:    vzeroupper
-; FALLBACK9-NEXT:    retq
-;
-; FALLBACK10-LABEL: ashr_64bytes:
-; FALLBACK10:       # %bb.0:
-; FALLBACK10-NEXT:    pushq %r15
-; FALLBACK10-NEXT:    pushq %r14
-; FALLBACK10-NEXT:    pushq %r13
-; FALLBACK10-NEXT:    pushq %r12
-; FALLBACK10-NEXT:    pushq %rbx
-; FALLBACK10-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK10-NEXT:    vmovups 32(%rdi), %xmm1
-; FALLBACK10-NEXT:    movq 48(%rdi), %rcx
-; FALLBACK10-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK10-NEXT:    movl (%rsi), %eax
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    vmovaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    sarq $63, %rdi
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK10-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK10-NEXT:    andl $56, %ecx
-; FALLBACK10-NEXT:    movl %ecx, %esi
-; FALLBACK10-NEXT:    andl $56, %eax
-; FALLBACK10-NEXT:    shrxq %rsi, -128(%rsp,%rax), %r8
-; FALLBACK10-NEXT:    notb %cl
-; FALLBACK10-NEXT:    movq -120(%rsp,%rax), %r10
-; FALLBACK10-NEXT:    movq -112(%rsp,%rax), %r9
-; FALLBACK10-NEXT:    leaq (%r10,%r10), %rdi
-; FALLBACK10-NEXT:    shlxq %rcx, %rdi, %rdi
-; FALLBACK10-NEXT:    orq %r8, %rdi
-; FALLBACK10-NEXT:    movq -104(%rsp,%rax), %r11
-; FALLBACK10-NEXT:    shrxq %rsi, %r11, %rbx
-; FALLBACK10-NEXT:    movq -96(%rsp,%rax), %r14
-; FALLBACK10-NEXT:    leaq (%r14,%r14), %r8
-; FALLBACK10-NEXT:    shlxq %rcx, %r8, %r8
-; FALLBACK10-NEXT:    orq %rbx, %r8
-; FALLBACK10-NEXT:    shrxq %rsi, %r9, %rbx
-; FALLBACK10-NEXT:    addq %r11, %r11
-; FALLBACK10-NEXT:    shlxq %rcx, %r11, %r11
-; FALLBACK10-NEXT:    orq %rbx, %r11
-; FALLBACK10-NEXT:    movq -88(%rsp,%rax), %rbx
-; FALLBACK10-NEXT:    shrxq %rsi, %rbx, %r15
-; FALLBACK10-NEXT:    movq -80(%rsp,%rax), %r12
-; FALLBACK10-NEXT:    leaq (%r12,%r12), %r13
-; FALLBACK10-NEXT:    shlxq %rcx, %r13, %r13
-; FALLBACK10-NEXT:    orq %r15, %r13
-; FALLBACK10-NEXT:    shrxq %rsi, %r14, %r14
-; FALLBACK10-NEXT:    addq %rbx, %rbx
-; FALLBACK10-NEXT:    shlxq %rcx, %rbx, %rbx
-; FALLBACK10-NEXT:    orq %r14, %rbx
-; FALLBACK10-NEXT:    shrxq %rsi, %r12, %r14
-; FALLBACK10-NEXT:    movq -72(%rsp,%rax), %rax
-; FALLBACK10-NEXT:    leaq (%rax,%rax), %r15
-; FALLBACK10-NEXT:    shlxq %rcx, %r15, %r15
-; FALLBACK10-NEXT:    orq %r14, %r15
-; FALLBACK10-NEXT:    shrxq %rsi, %r10, %r10
-; FALLBACK10-NEXT:    addq %r9, %r9
-; FALLBACK10-NEXT:    shlxq %rcx, %r9, %rcx
-; FALLBACK10-NEXT:    orq %r10, %rcx
-; FALLBACK10-NEXT:    sarxq %rsi, %rax, %rax
-; FALLBACK10-NEXT:    movq %rax, 56(%rdx)
-; FALLBACK10-NEXT:    movq %rcx, 8(%rdx)
-; FALLBACK10-NEXT:    movq %r15, 48(%rdx)
-; FALLBACK10-NEXT:    movq %rbx, 32(%rdx)
-; FALLBACK10-NEXT:    movq %r13, 40(%rdx)
-; FALLBACK10-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK10-NEXT:    movq %r8, 24(%rdx)
-; FALLBACK10-NEXT:    movq %rdi, (%rdx)
-; FALLBACK10-NEXT:    popq %rbx
-; FALLBACK10-NEXT:    popq %r12
-; FALLBACK10-NEXT:    popq %r13
-; FALLBACK10-NEXT:    popq %r14
-; FALLBACK10-NEXT:    popq %r15
-; FALLBACK10-NEXT:    vzeroupper
-; FALLBACK10-NEXT:    retq
-;
-; FALLBACK11-LABEL: ashr_64bytes:
-; FALLBACK11:       # %bb.0:
-; FALLBACK11-NEXT:    pushq %r15
-; FALLBACK11-NEXT:    pushq %r14
-; FALLBACK11-NEXT:    pushq %rbx
-; FALLBACK11-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK11-NEXT:    vmovups 32(%rdi), %xmm1
-; FALLBACK11-NEXT:    movq 48(%rdi), %rcx
-; FALLBACK11-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK11-NEXT:    movl (%rsi), %eax
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    vmovaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    sarq $63, %rdi
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK11-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK11-NEXT:    andl $56, %ecx
-; FALLBACK11-NEXT:    andl $56, %eax
-; FALLBACK11-NEXT:    movq -96(%rsp,%rax), %rdi
-; FALLBACK11-NEXT:    movq -104(%rsp,%rax), %r9
-; FALLBACK11-NEXT:    movq %r9, %rsi
-; FALLBACK11-NEXT:    shrdq %cl, %rdi, %rsi
-; FALLBACK11-NEXT:    movq -112(%rsp,%rax), %r10
-; FALLBACK11-NEXT:    movq %r10, %r8
-; FALLBACK11-NEXT:    shrdq %cl, %r9, %r8
-; FALLBACK11-NEXT:    movq -80(%rsp,%rax), %r9
-; FALLBACK11-NEXT:    movq -88(%rsp,%rax), %r11
-; FALLBACK11-NEXT:    movq %r11, %rbx
-; FALLBACK11-NEXT:    shrdq %cl, %r9, %rbx
-; FALLBACK11-NEXT:    shrdq %cl, %r11, %rdi
-; FALLBACK11-NEXT:    movq -72(%rsp,%rax), %r11
-; FALLBACK11-NEXT:    shrdq %cl, %r11, %r9
-; FALLBACK11-NEXT:    movq -128(%rsp,%rax), %r14
-; FALLBACK11-NEXT:    movq -120(%rsp,%rax), %rax
-; FALLBACK11-NEXT:    movq %rax, %r15
-; FALLBACK11-NEXT:    shrdq %cl, %r10, %r15
-; FALLBACK11-NEXT:    sarxq %rcx, %r11, %r10
-; FALLBACK11-NEXT:    # kill: def $cl killed $cl killed $rcx
-; FALLBACK11-NEXT:    shrdq %cl, %rax, %r14
-; FALLBACK11-NEXT:    movq %r15, 8(%rdx)
-; FALLBACK11-NEXT:    movq %r9, 48(%rdx)
-; FALLBACK11-NEXT:    movq %rdi, 32(%rdx)
-; FALLBACK11-NEXT:    movq %rbx, 40(%rdx)
-; FALLBACK11-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK11-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK11-NEXT:    movq %r14, (%rdx)
-; FALLBACK11-NEXT:    movq %r10, 56(%rdx)
-; FALLBACK11-NEXT:    popq %rbx
-; FALLBACK11-NEXT:    popq %r14
-; FALLBACK11-NEXT:    popq %r15
-; FALLBACK11-NEXT:    vzeroupper
-; FALLBACK11-NEXT:    retq
-;
-; FALLBACK12-LABEL: ashr_64bytes:
-; FALLBACK12:       # %bb.0:
-; FALLBACK12-NEXT:    pushq %rbp
-; FALLBACK12-NEXT:    pushq %r15
-; FALLBACK12-NEXT:    pushq %r14
-; FALLBACK12-NEXT:    pushq %r13
-; FALLBACK12-NEXT:    pushq %r12
-; FALLBACK12-NEXT:    pushq %rbx
-; FALLBACK12-NEXT:    pushq %rax
-; FALLBACK12-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK12-NEXT:    vmovups 32(%rdi), %xmm1
-; FALLBACK12-NEXT:    movq 48(%rdi), %rax
-; FALLBACK12-NEXT:    movq 56(%rdi), %rcx
-; FALLBACK12-NEXT:    movl (%rsi), %edi
-; FALLBACK12-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    vmovaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    sarq $63, %rcx
-; FALLBACK12-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK12-NEXT:    leal (,%rdi,8), %eax
-; FALLBACK12-NEXT:    andl $56, %eax
-; FALLBACK12-NEXT:    andl $56, %edi
-; FALLBACK12-NEXT:    movq -128(%rsp,%rdi), %r10
-; FALLBACK12-NEXT:    movq -120(%rsp,%rdi), %r9
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r10
-; FALLBACK12-NEXT:    movl %eax, %esi
-; FALLBACK12-NEXT:    notb %sil
-; FALLBACK12-NEXT:    leaq (%r9,%r9), %r8
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r8
-; FALLBACK12-NEXT:    orq %r10, %r8
-; FALLBACK12-NEXT:    movq -104(%rsp,%rdi), %r10
-; FALLBACK12-NEXT:    movq %r10, %rbx
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %rbx
-; FALLBACK12-NEXT:    movq -96(%rsp,%rdi), %r12
-; FALLBACK12-NEXT:    leaq (%r12,%r12), %r11
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r11
-; FALLBACK12-NEXT:    orq %rbx, %r11
-; FALLBACK12-NEXT:    movq -112(%rsp,%rdi), %rbx
-; FALLBACK12-NEXT:    movq %rbx, %r14
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r14
-; FALLBACK12-NEXT:    addq %r10, %r10
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r10
-; FALLBACK12-NEXT:    orq %r14, %r10
-; FALLBACK12-NEXT:    movq -88(%rsp,%rdi), %r14
-; FALLBACK12-NEXT:    movq %r14, %r13
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r13
-; FALLBACK12-NEXT:    movq -80(%rsp,%rdi), %rbp
-; FALLBACK12-NEXT:    leaq (%rbp,%rbp), %r15
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r15
-; FALLBACK12-NEXT:    orq %r13, %r15
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r12
-; FALLBACK12-NEXT:    addq %r14, %r14
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r14
-; FALLBACK12-NEXT:    orq %r12, %r14
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %rbp
-; FALLBACK12-NEXT:    movq -72(%rsp,%rdi), %rdi
-; FALLBACK12-NEXT:    leaq (%rdi,%rdi), %r12
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %r12
-; FALLBACK12-NEXT:    orq %rbp, %r12
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    shrq %cl, %r9
-; FALLBACK12-NEXT:    addq %rbx, %rbx
-; FALLBACK12-NEXT:    movl %esi, %ecx
-; FALLBACK12-NEXT:    shlq %cl, %rbx
-; FALLBACK12-NEXT:    orq %r9, %rbx
-; FALLBACK12-NEXT:    movl %eax, %ecx
-; FALLBACK12-NEXT:    sarq %cl, %rdi
-; FALLBACK12-NEXT:    movq %rdi, 56(%rdx)
-; FALLBACK12-NEXT:    movq %rbx, 8(%rdx)
-; FALLBACK12-NEXT:    movq %r12, 48(%rdx)
-; FALLBACK12-NEXT:    movq %r14, 32(%rdx)
-; FALLBACK12-NEXT:    movq %r15, 40(%rdx)
-; FALLBACK12-NEXT:    movq %r10, 16(%rdx)
-; FALLBACK12-NEXT:    movq %r11, 24(%rdx)
-; FALLBACK12-NEXT:    movq %r8, (%rdx)
-; FALLBACK12-NEXT:    addq $8, %rsp
-; FALLBACK12-NEXT:    popq %rbx
-; FALLBACK12-NEXT:    popq %r12
-; FALLBACK12-NEXT:    popq %r13
-; FALLBACK12-NEXT:    popq %r14
-; FALLBACK12-NEXT:    popq %r15
-; FALLBACK12-NEXT:    popq %rbp
-; FALLBACK12-NEXT:    vzeroupper
-; FALLBACK12-NEXT:    retq
-;
-; FALLBACK13-LABEL: ashr_64bytes:
-; FALLBACK13:       # %bb.0:
-; FALLBACK13-NEXT:    pushq %r15
-; FALLBACK13-NEXT:    pushq %r14
-; FALLBACK13-NEXT:    pushq %rbx
-; FALLBACK13-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK13-NEXT:    vmovups 32(%rdi), %xmm1
-; FALLBACK13-NEXT:    movq 48(%rdi), %rcx
-; FALLBACK13-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK13-NEXT:    movl (%rsi), %eax
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    vmovaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    sarq $63, %rdi
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK13-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK13-NEXT:    andl $56, %ecx
-; FALLBACK13-NEXT:    andl $56, %eax
-; FALLBACK13-NEXT:    movq -96(%rsp,%rax), %rdi
-; FALLBACK13-NEXT:    movq -104(%rsp,%rax), %r9
-; FALLBACK13-NEXT:    movq %r9, %rsi
-; FALLBACK13-NEXT:    shrdq %cl, %rdi, %rsi
-; FALLBACK13-NEXT:    movq -112(%rsp,%rax), %r10
-; FALLBACK13-NEXT:    movq %r10, %r8
-; FALLBACK13-NEXT:    shrdq %cl, %r9, %r8
-; FALLBACK13-NEXT:    movq -80(%rsp,%rax), %r9
-; FALLBACK13-NEXT:    movq -88(%rsp,%rax), %r11
-; FALLBACK13-NEXT:    movq %r11, %rbx
-; FALLBACK13-NEXT:    shrdq %cl, %r9, %rbx
-; FALLBACK13-NEXT:    shrdq %cl, %r11, %rdi
-; FALLBACK13-NEXT:    movq -72(%rsp,%rax), %r11
-; FALLBACK13-NEXT:    shrdq %cl, %r11, %r9
-; FALLBACK13-NEXT:    movq -128(%rsp,%rax), %r14
-; FALLBACK13-NEXT:    movq -120(%rsp,%rax), %rax
-; FALLBACK13-NEXT:    movq %rax, %r15
-; FALLBACK13-NEXT:    shrdq %cl, %r10, %r15
-; FALLBACK13-NEXT:    shrdq %cl, %rax, %r14
-; FALLBACK13-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK13-NEXT:    sarq %cl, %r11
-; FALLBACK13-NEXT:    movq %r15, 8(%rdx)
-; FALLBACK13-NEXT:    movq %r9, 48(%rdx)
-; FALLBACK13-NEXT:    movq %r11, 56(%rdx)
-; FALLBACK13-NEXT:    movq %rdi, 32(%rdx)
-; FALLBACK13-NEXT:    movq %rbx, 40(%rdx)
-; FALLBACK13-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK13-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK13-NEXT:    movq %r14, (%rdx)
-; FALLBACK13-NEXT:    popq %rbx
-; FALLBACK13-NEXT:    popq %r14
-; FALLBACK13-NEXT:    popq %r15
-; FALLBACK13-NEXT:    vzeroupper
-; FALLBACK13-NEXT:    retq
-;
-; FALLBACK14-LABEL: ashr_64bytes:
-; FALLBACK14:       # %bb.0:
-; FALLBACK14-NEXT:    pushq %r15
-; FALLBACK14-NEXT:    pushq %r14
-; FALLBACK14-NEXT:    pushq %r13
-; FALLBACK14-NEXT:    pushq %r12
-; FALLBACK14-NEXT:    pushq %rbx
-; FALLBACK14-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK14-NEXT:    vmovups 32(%rdi), %xmm1
-; FALLBACK14-NEXT:    movq 48(%rdi), %rcx
-; FALLBACK14-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK14-NEXT:    movl (%rsi), %eax
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    vmovaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    sarq $63, %rdi
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK14-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK14-NEXT:    andl $56, %ecx
-; FALLBACK14-NEXT:    movl %ecx, %esi
-; FALLBACK14-NEXT:    andl $56, %eax
-; FALLBACK14-NEXT:    shrxq %rsi, -128(%rsp,%rax), %r8
-; FALLBACK14-NEXT:    notb %cl
-; FALLBACK14-NEXT:    movq -120(%rsp,%rax), %r10
-; FALLBACK14-NEXT:    movq -112(%rsp,%rax), %r9
-; FALLBACK14-NEXT:    leaq (%r10,%r10), %rdi
-; FALLBACK14-NEXT:    shlxq %rcx, %rdi, %rdi
-; FALLBACK14-NEXT:    orq %r8, %rdi
-; FALLBACK14-NEXT:    movq -104(%rsp,%rax), %r11
-; FALLBACK14-NEXT:    shrxq %rsi, %r11, %rbx
-; FALLBACK14-NEXT:    movq -96(%rsp,%rax), %r14
-; FALLBACK14-NEXT:    leaq (%r14,%r14), %r8
-; FALLBACK14-NEXT:    shlxq %rcx, %r8, %r8
-; FALLBACK14-NEXT:    orq %rbx, %r8
-; FALLBACK14-NEXT:    shrxq %rsi, %r9, %rbx
-; FALLBACK14-NEXT:    addq %r11, %r11
-; FALLBACK14-NEXT:    shlxq %rcx, %r11, %r11
-; FALLBACK14-NEXT:    orq %rbx, %r11
-; FALLBACK14-NEXT:    movq -88(%rsp,%rax), %rbx
-; FALLBACK14-NEXT:    shrxq %rsi, %rbx, %r15
-; FALLBACK14-NEXT:    movq -80(%rsp,%rax), %r12
-; FALLBACK14-NEXT:    leaq (%r12,%r12), %r13
-; FALLBACK14-NEXT:    shlxq %rcx, %r13, %r13
-; FALLBACK14-NEXT:    orq %r15, %r13
-; FALLBACK14-NEXT:    shrxq %rsi, %r14, %r14
-; FALLBACK14-NEXT:    addq %rbx, %rbx
-; FALLBACK14-NEXT:    shlxq %rcx, %rbx, %rbx
-; FALLBACK14-NEXT:    orq %r14, %rbx
-; FALLBACK14-NEXT:    shrxq %rsi, %r12, %r14
-; FALLBACK14-NEXT:    movq -72(%rsp,%rax), %rax
-; FALLBACK14-NEXT:    leaq (%rax,%rax), %r15
-; FALLBACK14-NEXT:    shlxq %rcx, %r15, %r15
-; FALLBACK14-NEXT:    orq %r14, %r15
-; FALLBACK14-NEXT:    shrxq %rsi, %r10, %r10
-; FALLBACK14-NEXT:    addq %r9, %r9
-; FALLBACK14-NEXT:    shlxq %rcx, %r9, %rcx
-; FALLBACK14-NEXT:    orq %r10, %rcx
-; FALLBACK14-NEXT:    sarxq %rsi, %rax, %rax
-; FALLBACK14-NEXT:    movq %rax, 56(%rdx)
-; FALLBACK14-NEXT:    movq %rcx, 8(%rdx)
-; FALLBACK14-NEXT:    movq %r15, 48(%rdx)
-; FALLBACK14-NEXT:    movq %rbx, 32(%rdx)
-; FALLBACK14-NEXT:    movq %r13, 40(%rdx)
-; FALLBACK14-NEXT:    movq %r11, 16(%rdx)
-; FALLBACK14-NEXT:    movq %r8, 24(%rdx)
-; FALLBACK14-NEXT:    movq %rdi, (%rdx)
-; FALLBACK14-NEXT:    popq %rbx
-; FALLBACK14-NEXT:    popq %r12
-; FALLBACK14-NEXT:    popq %r13
-; FALLBACK14-NEXT:    popq %r14
-; FALLBACK14-NEXT:    popq %r15
-; FALLBACK14-NEXT:    vzeroupper
-; FALLBACK14-NEXT:    retq
-;
-; FALLBACK15-LABEL: ashr_64bytes:
-; FALLBACK15:       # %bb.0:
-; FALLBACK15-NEXT:    pushq %r15
-; FALLBACK15-NEXT:    pushq %r14
-; FALLBACK15-NEXT:    pushq %rbx
-; FALLBACK15-NEXT:    vmovups (%rdi), %ymm0
-; FALLBACK15-NEXT:    vmovups 32(%rdi), %xmm1
-; FALLBACK15-NEXT:    movq 48(%rdi), %rcx
-; FALLBACK15-NEXT:    movq 56(%rdi), %rdi
-; FALLBACK15-NEXT:    movl (%rsi), %eax
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    vmovaps %xmm1, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    sarq $63, %rdi
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
-; FALLBACK15-NEXT:    leal (,%rax,8), %ecx
-; FALLBACK15-NEXT:    andl $56, %ecx
-; FALLBACK15-NEXT:    andl $56, %eax
-; FALLBACK15-NEXT:    movq -96(%rsp,%rax), %rdi
-; FALLBACK15-NEXT:    movq -104(%rsp,%rax), %r9
-; FALLBACK15-NEXT:    movq %r9, %rsi
-; FALLBACK15-NEXT:    shrdq %cl, %rdi, %rsi
-; FALLBACK15-NEXT:    movq -112(%rsp,%rax), %r10
-; FALLBACK15-NEXT:    movq %r10, %r8
-; FALLBACK15-NEXT:    shrdq %cl, %r9, %r8
-; FALLBACK15-NEXT:    movq -80(%rsp,%rax), %r9
-; FALLBACK15-NEXT:    movq -88(%rsp,%rax), %r11
-; FALLBACK15-NEXT:    movq %r11, %rbx
-; FALLBACK15-NEXT:    shrdq %cl, %r9, %rbx
-; FALLBACK15-NEXT:    shrdq %cl, %r11, %rdi
-; FALLBACK15-NEXT:    movq -72(%rsp,%rax), %r11
-; FALLBACK15-NEXT:    shrdq %cl, %r11, %r9
-; FALLBACK15-NEXT:    movq -128(%rsp,%rax), %r14
-; FALLBACK15-NEXT:    movq -120(%rsp,%rax), %rax
-; FALLBACK15-NEXT:    movq %rax, %r15
-; FALLBACK15-NEXT:    shrdq %cl, %r10, %r15
-; FALLBACK15-NEXT:    sarxq %rcx, %r11, %r10
-; FALLBACK15-NEXT:    # kill: def $cl killed $cl killed $rcx
-; FALLBACK15-NEXT:    shrdq %cl, %rax, %r14
-; FALLBACK15-NEXT:    movq %r15, 8(%rdx)
-; FALLBACK15-NEXT:    movq %r9, 48(%rdx)
-; FALLBACK15-NEXT:    movq %rdi, 32(%rdx)
-; FALLBACK15-NEXT:    movq %rbx, 40(%rdx)
-; FALLBACK15-NEXT:    movq %r8, 16(%rdx)
-; FALLBACK15-NEXT:    movq %rsi, 24(%rdx)
-; FALLBACK15-NEXT:    movq %r14, (%rdx)
-; FALLBACK15-NEXT:    movq %r10, 56(%rdx)
-; FALLBACK15-NEXT:    popq %rbx
-; FALLBACK15-NEXT:    popq %r14
-; FALLBACK15-NEXT:    popq %r15
-; FALLBACK15-NEXT:    vzeroupper
-; FALLBACK15-NEXT:    retq
-;
-; FALLBACK16-LABEL: ashr_64bytes:
-; FALLBACK16:       # %bb.0:
-; FALLBACK16-NEXT:    pushl %ebp
-; FALLBACK16-NEXT:    pushl %ebx
-; FALLBACK16-NEXT:    pushl %edi
-; FALLBACK16-NEXT:    pushl %esi
-; FALLBACK16-NEXT:    subl $204, %esp
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK16-NEXT:    movl (%ecx), %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 4(%ecx), %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 8(%ecx), %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 12(%ecx), %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 16(%ecx), %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 20(%ecx), %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 24(%ecx), %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 28(%ecx), %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 32(%ecx), %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 36(%ecx), %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 40(%ecx), %ebx
-; FALLBACK16-NEXT:    movl 44(%ecx), %edi
-; FALLBACK16-NEXT:    movl 48(%ecx), %esi
-; FALLBACK16-NEXT:    movl 52(%ecx), %edx
-; FALLBACK16-NEXT:    movl 56(%ecx), %eax
-; FALLBACK16-NEXT:    movl 60(%ecx), %ecx
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK16-NEXT:    movl (%ebp), %ebp
-; FALLBACK16-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    sarl $31, %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK16-NEXT:    movl %ebp, %ecx
-; FALLBACK16-NEXT:    movl %ebp, %esi
-; FALLBACK16-NEXT:    andl $60, %esi
-; FALLBACK16-NEXT:    movl 68(%esp,%esi), %edx
-; FALLBACK16-NEXT:    shll $3, %ecx
-; FALLBACK16-NEXT:    andl $24, %ecx
-; FALLBACK16-NEXT:    movl %edx, %eax
-; FALLBACK16-NEXT:    shrl %cl, %eax
-; FALLBACK16-NEXT:    movl 72(%esp,%esi), %edi
-; FALLBACK16-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    addl %edi, %edi
-; FALLBACK16-NEXT:    movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; FALLBACK16-NEXT:    movl %ecx, %ebx
-; FALLBACK16-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; FALLBACK16-NEXT:    notb %ch
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; FALLBACK16-NEXT:    shll %cl, %edi
-; FALLBACK16-NEXT:    orl %eax, %edi
-; FALLBACK16-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 64(%esp,%esi), %eax
-; FALLBACK16-NEXT:    movb %bl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %eax
-; FALLBACK16-NEXT:    addl %edx, %edx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %edx
-; FALLBACK16-NEXT:    orl %eax, %edx
-; FALLBACK16-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 76(%esp,%esi), %ebp
-; FALLBACK16-NEXT:    movl %ebp, %edx
-; FALLBACK16-NEXT:    movb %bl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edx
-; FALLBACK16-NEXT:    movl 80(%esp,%esi), %edi
-; FALLBACK16-NEXT:    leal (%edi,%edi), %eax
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %eax
-; FALLBACK16-NEXT:    orl %edx, %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %bl, %cl
-; FALLBACK16-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    shrl %cl, %eax
-; FALLBACK16-NEXT:    addl %ebp, %ebp
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebp
-; FALLBACK16-NEXT:    orl %eax, %ebp
-; FALLBACK16-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl %esi, %edx
-; FALLBACK16-NEXT:    movl 84(%esp,%esi), %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %bl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %eax
-; FALLBACK16-NEXT:    movl 88(%esp,%esi), %esi
-; FALLBACK16-NEXT:    leal (%esi,%esi), %ebp
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebp
-; FALLBACK16-NEXT:    orl %eax, %ebp
-; FALLBACK16-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %bl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edi
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK16-NEXT:    addl %ebx, %ebx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebx
-; FALLBACK16-NEXT:    orl %edi, %ebx
-; FALLBACK16-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl %edx, %eax
-; FALLBACK16-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl 92(%esp,%edx), %ebp
-; FALLBACK16-NEXT:    movl %ebp, %edx
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK16-NEXT:    movb %bl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edx
-; FALLBACK16-NEXT:    movl 96(%esp,%eax), %edi
-; FALLBACK16-NEXT:    leal (%edi,%edi), %eax
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %eax
-; FALLBACK16-NEXT:    orl %edx, %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %bl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %esi
-; FALLBACK16-NEXT:    addl %ebp, %ebp
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebp
-; FALLBACK16-NEXT:    orl %esi, %ebp
-; FALLBACK16-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK16-NEXT:    movl 100(%esp,%edx), %eax
-; FALLBACK16-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %bl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %eax
-; FALLBACK16-NEXT:    movl 104(%esp,%edx), %esi
-; FALLBACK16-NEXT:    leal (%esi,%esi), %ebp
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebp
-; FALLBACK16-NEXT:    orl %eax, %ebp
-; FALLBACK16-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl %ebx, %edx
-; FALLBACK16-NEXT:    movb %dl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %edi
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK16-NEXT:    addl %ebx, %ebx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebx
-; FALLBACK16-NEXT:    orl %edi, %ebx
-; FALLBACK16-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; FALLBACK16-NEXT:    movl 108(%esp,%ebp), %edi
-; FALLBACK16-NEXT:    movl %edi, %eax
-; FALLBACK16-NEXT:    movl %edx, %ebx
-; FALLBACK16-NEXT:    movl %ebx, %ecx
-; FALLBACK16-NEXT:    shrl %cl, %eax
-; FALLBACK16-NEXT:    movl 112(%esp,%ebp), %ecx
-; FALLBACK16-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movl %ebp, %edx
-; FALLBACK16-NEXT:    leal (%ecx,%ecx), %ebp
-; FALLBACK16-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebp
-; FALLBACK16-NEXT:    orl %eax, %ebp
-; FALLBACK16-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK16-NEXT:    movb %bl, %cl
-; FALLBACK16-NEXT:    shrl %cl, %esi
-; FALLBACK16-NEXT:    addl %edi, %edi
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %edi
-; FALLBACK16-NEXT:    orl %esi, %edi
-; FALLBACK16-NEXT:    movl 116(%esp,%edx), %esi
-; FALLBACK16-NEXT:    movl %esi, %eax
-; FALLBACK16-NEXT:    movl %ebx, %ecx
-; FALLBACK16-NEXT:    shrl %cl, %eax
-; FALLBACK16-NEXT:    movl 120(%esp,%edx), %edx
-; FALLBACK16-NEXT:    leal (%edx,%edx), %ebp
-; FALLBACK16-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %ebp
-; FALLBACK16-NEXT:    orl %eax, %ebp
-; FALLBACK16-NEXT:    movb %bl, %cl
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK16-NEXT:    shrl %cl, %eax
-; FALLBACK16-NEXT:    addl %esi, %esi
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %esi
-; FALLBACK16-NEXT:    orl %eax, %esi
-; FALLBACK16-NEXT:    movb %bl, %cl
-; FALLBACK16-NEXT:    movl %edx, %eax
-; FALLBACK16-NEXT:    shrl %cl, %eax
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK16-NEXT:    movl 124(%esp,%edx), %ebx
-; FALLBACK16-NEXT:    leal (%ebx,%ebx), %edx
-; FALLBACK16-NEXT:    movb %ch, %cl
-; FALLBACK16-NEXT:    shll %cl, %edx
-; FALLBACK16-NEXT:    orl %eax, %edx
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK16-NEXT:    sarl %cl, %ebx
-; FALLBACK16-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK16-NEXT:    movl %ebx, 60(%eax)
-; FALLBACK16-NEXT:    movl %edx, 56(%eax)
-; FALLBACK16-NEXT:    movl %esi, 48(%eax)
-; FALLBACK16-NEXT:    movl %ebp, 52(%eax)
-; FALLBACK16-NEXT:    movl %edi, 40(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, (%eax)
-; FALLBACK16-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK16-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK16-NEXT:    addl $204, %esp
-; FALLBACK16-NEXT:    popl %esi
-; FALLBACK16-NEXT:    popl %edi
-; FALLBACK16-NEXT:    popl %ebx
-; FALLBACK16-NEXT:    popl %ebp
-; FALLBACK16-NEXT:    retl
-;
-; FALLBACK17-LABEL: ashr_64bytes:
-; FALLBACK17:       # %bb.0:
-; FALLBACK17-NEXT:    pushl %ebp
-; FALLBACK17-NEXT:    pushl %ebx
-; FALLBACK17-NEXT:    pushl %edi
-; FALLBACK17-NEXT:    pushl %esi
-; FALLBACK17-NEXT:    subl $188, %esp
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK17-NEXT:    movl (%eax), %ecx
-; FALLBACK17-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 4(%eax), %ecx
-; FALLBACK17-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 8(%eax), %ecx
-; FALLBACK17-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 12(%eax), %ecx
-; FALLBACK17-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 16(%eax), %ecx
-; FALLBACK17-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 20(%eax), %ecx
-; FALLBACK17-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 24(%eax), %ecx
-; FALLBACK17-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 28(%eax), %ecx
-; FALLBACK17-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 32(%eax), %ecx
-; FALLBACK17-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 36(%eax), %ecx
-; FALLBACK17-NEXT:    movl %ecx, (%esp) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 40(%eax), %ebp
-; FALLBACK17-NEXT:    movl 44(%eax), %ebx
-; FALLBACK17-NEXT:    movl 48(%eax), %edi
-; FALLBACK17-NEXT:    movl 52(%eax), %esi
-; FALLBACK17-NEXT:    movl 56(%eax), %edx
-; FALLBACK17-NEXT:    movl 60(%eax), %eax
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK17-NEXT:    movl (%ecx), %ecx
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl (%esp), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    sarl $31, %eax
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK17-NEXT:    movl %ecx, %ebp
-; FALLBACK17-NEXT:    andl $60, %ebp
-; FALLBACK17-NEXT:    movl 56(%esp,%ebp), %edx
-; FALLBACK17-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shll $3, %ecx
-; FALLBACK17-NEXT:    andl $24, %ecx
-; FALLBACK17-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK17-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 64(%esp,%ebp), %edi
-; FALLBACK17-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, %esi
-; FALLBACK17-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK17-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK17-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 72(%esp,%ebp), %esi
-; FALLBACK17-NEXT:    movl 68(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, %edx
-; FALLBACK17-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK17-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK17-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 80(%esp,%ebp), %edi
-; FALLBACK17-NEXT:    movl 76(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, %edx
-; FALLBACK17-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK17-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK17-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 88(%esp,%ebp), %esi
-; FALLBACK17-NEXT:    movl 84(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, %edx
-; FALLBACK17-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK17-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl %esi, %edx
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK17-NEXT:    movl %edi, (%esp) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 96(%esp,%ebp), %esi
-; FALLBACK17-NEXT:    movl 92(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, %edi
-; FALLBACK17-NEXT:    shrdl %cl, %esi, %edi
-; FALLBACK17-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK17-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK17-NEXT:    movl 104(%esp,%ebp), %edx
-; FALLBACK17-NEXT:    movl 100(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    movl %eax, %edi
-; FALLBACK17-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK17-NEXT:    movl 48(%esp,%ebp), %ebx
-; FALLBACK17-NEXT:    movl 108(%esp,%ebp), %eax
-; FALLBACK17-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK17-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK17-NEXT:    movl %edx, 56(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK17-NEXT:    shrdl %cl, %edx, %ebx
-; FALLBACK17-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK17-NEXT:    sarl %cl, %eax
-; FALLBACK17-NEXT:    movl %eax, 60(%ebp)
-; FALLBACK17-NEXT:    movl %esi, 48(%ebp)
-; FALLBACK17-NEXT:    movl %edi, 52(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 40(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK17-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK17-NEXT:    movl %ebx, (%ebp)
-; FALLBACK17-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK17-NEXT:    movl %eax, 4(%ebp)
-; FALLBACK17-NEXT:    addl $188, %esp
-; FALLBACK17-NEXT:    popl %esi
-; FALLBACK17-NEXT:    popl %edi
-; FALLBACK17-NEXT:    popl %ebx
-; FALLBACK17-NEXT:    popl %ebp
-; FALLBACK17-NEXT:    retl
-;
-; FALLBACK18-LABEL: ashr_64bytes:
-; FALLBACK18:       # %bb.0:
-; FALLBACK18-NEXT:    pushl %ebp
-; FALLBACK18-NEXT:    pushl %ebx
-; FALLBACK18-NEXT:    pushl %edi
-; FALLBACK18-NEXT:    pushl %esi
-; FALLBACK18-NEXT:    subl $204, %esp
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK18-NEXT:    movl (%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 4(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 8(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 12(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 16(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 20(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 24(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 28(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 32(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 36(%eax), %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 40(%eax), %ebp
-; FALLBACK18-NEXT:    movl 44(%eax), %ebx
-; FALLBACK18-NEXT:    movl 48(%eax), %edi
-; FALLBACK18-NEXT:    movl 52(%eax), %esi
-; FALLBACK18-NEXT:    movl 56(%eax), %edx
-; FALLBACK18-NEXT:    movl 60(%eax), %ecx
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK18-NEXT:    movl (%eax), %eax
-; FALLBACK18-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    sarl $31, %ecx
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK18-NEXT:    movl %eax, %ecx
-; FALLBACK18-NEXT:    leal (,%eax,8), %edx
-; FALLBACK18-NEXT:    andl $24, %edx
-; FALLBACK18-NEXT:    movl %edx, %ebx
-; FALLBACK18-NEXT:    andl $60, %ecx
-; FALLBACK18-NEXT:    movl 68(%esp,%ecx), %esi
-; FALLBACK18-NEXT:    movl 72(%esp,%ecx), %edi
-; FALLBACK18-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %ebx, %esi, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    notb %dl
-; FALLBACK18-NEXT:    leal (%edi,%edi), %ebp
-; FALLBACK18-NEXT:    shlxl %edx, %ebp, %eax
-; FALLBACK18-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %ebx, 64(%esp,%ecx), %edi
-; FALLBACK18-NEXT:    addl %esi, %esi
-; FALLBACK18-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK18-NEXT:    orl %edi, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 80(%esp,%ecx), %esi
-; FALLBACK18-NEXT:    leal (%esi,%esi), %edi
-; FALLBACK18-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK18-NEXT:    movl 76(%esp,%ecx), %edi
-; FALLBACK18-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK18-NEXT:    orl %ebp, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK18-NEXT:    addl %edi, %edi
-; FALLBACK18-NEXT:    shlxl %edx, %edi, %edi
-; FALLBACK18-NEXT:    orl %eax, %edi
-; FALLBACK18-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 88(%esp,%ecx), %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK18-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK18-NEXT:    movl 84(%esp,%ecx), %edi
-; FALLBACK18-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK18-NEXT:    orl %ebp, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %ebx, %esi, %esi
-; FALLBACK18-NEXT:    addl %edi, %edi
-; FALLBACK18-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK18-NEXT:    orl %esi, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 96(%esp,%ecx), %esi
-; FALLBACK18-NEXT:    leal (%esi,%esi), %edi
-; FALLBACK18-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK18-NEXT:    movl 92(%esp,%ecx), %edi
-; FALLBACK18-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK18-NEXT:    orl %ebp, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK18-NEXT:    addl %edi, %edi
-; FALLBACK18-NEXT:    shlxl %edx, %edi, %edi
-; FALLBACK18-NEXT:    orl %eax, %edi
-; FALLBACK18-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 104(%esp,%ecx), %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK18-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK18-NEXT:    movl 100(%esp,%ecx), %edi
-; FALLBACK18-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK18-NEXT:    orl %ebp, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %ebx, %esi, %esi
-; FALLBACK18-NEXT:    addl %edi, %edi
-; FALLBACK18-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK18-NEXT:    orl %esi, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl %ecx, %ebp
-; FALLBACK18-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 112(%esp,%ecx), %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    leal (%eax,%eax), %esi
-; FALLBACK18-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK18-NEXT:    movl 108(%esp,%ecx), %esi
-; FALLBACK18-NEXT:    shrxl %ebx, %esi, %edi
-; FALLBACK18-NEXT:    orl %edi, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; FALLBACK18-NEXT:    addl %esi, %esi
-; FALLBACK18-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK18-NEXT:    orl %ecx, %eax
-; FALLBACK18-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK18-NEXT:    movl 120(%esp,%ebp), %edi
-; FALLBACK18-NEXT:    leal (%edi,%edi), %ecx
-; FALLBACK18-NEXT:    shlxl %edx, %ecx, %esi
-; FALLBACK18-NEXT:    movl 116(%esp,%ebp), %eax
-; FALLBACK18-NEXT:    shrxl %ebx, %eax, %ebp
-; FALLBACK18-NEXT:    orl %ebp, %esi
-; FALLBACK18-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK18-NEXT:    addl %eax, %eax
-; FALLBACK18-NEXT:    shlxl %edx, %eax, %ecx
-; FALLBACK18-NEXT:    orl %ebp, %ecx
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK18-NEXT:    movl 124(%esp,%eax), %eax
-; FALLBACK18-NEXT:    leal (%eax,%eax), %ebp
-; FALLBACK18-NEXT:    shlxl %edx, %ebp, %edx
-; FALLBACK18-NEXT:    shrxl %ebx, %edi, %edi
-; FALLBACK18-NEXT:    orl %edi, %edx
-; FALLBACK18-NEXT:    sarxl %ebx, %eax, %edi
-; FALLBACK18-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK18-NEXT:    movl %edi, 60(%eax)
-; FALLBACK18-NEXT:    movl %edx, 56(%eax)
-; FALLBACK18-NEXT:    movl %ecx, 48(%eax)
-; FALLBACK18-NEXT:    movl %esi, 52(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 40(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, (%eax)
-; FALLBACK18-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK18-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK18-NEXT:    addl $204, %esp
-; FALLBACK18-NEXT:    popl %esi
-; FALLBACK18-NEXT:    popl %edi
-; FALLBACK18-NEXT:    popl %ebx
-; FALLBACK18-NEXT:    popl %ebp
-; FALLBACK18-NEXT:    retl
-;
-; FALLBACK19-LABEL: ashr_64bytes:
-; FALLBACK19:       # %bb.0:
-; FALLBACK19-NEXT:    pushl %ebp
-; FALLBACK19-NEXT:    pushl %ebx
-; FALLBACK19-NEXT:    pushl %edi
-; FALLBACK19-NEXT:    pushl %esi
-; FALLBACK19-NEXT:    subl $188, %esp
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK19-NEXT:    movl (%eax), %ecx
-; FALLBACK19-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 4(%eax), %ecx
-; FALLBACK19-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 8(%eax), %ecx
-; FALLBACK19-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 12(%eax), %ecx
-; FALLBACK19-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 16(%eax), %ecx
-; FALLBACK19-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 20(%eax), %ecx
-; FALLBACK19-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 24(%eax), %ecx
-; FALLBACK19-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 28(%eax), %ecx
-; FALLBACK19-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 32(%eax), %ecx
-; FALLBACK19-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 36(%eax), %ecx
-; FALLBACK19-NEXT:    movl %ecx, (%esp) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 40(%eax), %ebp
-; FALLBACK19-NEXT:    movl 44(%eax), %ebx
-; FALLBACK19-NEXT:    movl 48(%eax), %edi
-; FALLBACK19-NEXT:    movl 52(%eax), %esi
-; FALLBACK19-NEXT:    movl 56(%eax), %edx
-; FALLBACK19-NEXT:    movl 60(%eax), %eax
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK19-NEXT:    movl (%ecx), %ecx
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl (%esp), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    sarl $31, %eax
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK19-NEXT:    movl %ecx, %ebp
-; FALLBACK19-NEXT:    andl $60, %ebp
-; FALLBACK19-NEXT:    movl 56(%esp,%ebp), %edx
-; FALLBACK19-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shll $3, %ecx
-; FALLBACK19-NEXT:    andl $24, %ecx
-; FALLBACK19-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK19-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 64(%esp,%ebp), %edi
-; FALLBACK19-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, %esi
-; FALLBACK19-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK19-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK19-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 72(%esp,%ebp), %esi
-; FALLBACK19-NEXT:    movl 68(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, %edx
-; FALLBACK19-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK19-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK19-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 80(%esp,%ebp), %edi
-; FALLBACK19-NEXT:    movl 76(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, %edx
-; FALLBACK19-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK19-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK19-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 88(%esp,%ebp), %ebx
-; FALLBACK19-NEXT:    movl 84(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, %edx
-; FALLBACK19-NEXT:    shrdl %cl, %ebx, %edx
-; FALLBACK19-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK19-NEXT:    movl %edi, (%esp) # 4-byte Spill
-; FALLBACK19-NEXT:    movl 96(%esp,%ebp), %esi
-; FALLBACK19-NEXT:    movl 92(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl %eax, %edx
-; FALLBACK19-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK19-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK19-NEXT:    movl 104(%esp,%ebp), %eax
-; FALLBACK19-NEXT:    movl 100(%esp,%ebp), %edi
-; FALLBACK19-NEXT:    movl %edi, %edx
-; FALLBACK19-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK19-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK19-NEXT:    movl 48(%esp,%ebp), %edi
-; FALLBACK19-NEXT:    movl 108(%esp,%ebp), %ebp
-; FALLBACK19-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK19-NEXT:    shrdl %cl, %ebp, %eax
-; FALLBACK19-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK19-NEXT:    movl %eax, 56(%ebp)
-; FALLBACK19-NEXT:    movl %esi, 48(%ebp)
-; FALLBACK19-NEXT:    movl %edx, 52(%ebp)
-; FALLBACK19-NEXT:    movl %ebx, 40(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK19-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK19-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK19-NEXT:    sarxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK19-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK19-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK19-NEXT:    movl %edi, (%ebp)
-; FALLBACK19-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK19-NEXT:    movl %ecx, 4(%ebp)
-; FALLBACK19-NEXT:    movl %eax, 60(%ebp)
-; FALLBACK19-NEXT:    addl $188, %esp
-; FALLBACK19-NEXT:    popl %esi
-; FALLBACK19-NEXT:    popl %edi
-; FALLBACK19-NEXT:    popl %ebx
-; FALLBACK19-NEXT:    popl %ebp
-; FALLBACK19-NEXT:    retl
-;
-; FALLBACK20-LABEL: ashr_64bytes:
-; FALLBACK20:       # %bb.0:
-; FALLBACK20-NEXT:    pushl %ebp
-; FALLBACK20-NEXT:    pushl %ebx
-; FALLBACK20-NEXT:    pushl %edi
-; FALLBACK20-NEXT:    pushl %esi
-; FALLBACK20-NEXT:    subl $204, %esp
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK20-NEXT:    movups (%ecx), %xmm0
-; FALLBACK20-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK20-NEXT:    movups 32(%ecx), %xmm2
-; FALLBACK20-NEXT:    movl 48(%ecx), %edx
-; FALLBACK20-NEXT:    movl 52(%ecx), %esi
-; FALLBACK20-NEXT:    movl 56(%ecx), %edi
-; FALLBACK20-NEXT:    movl 60(%ecx), %ecx
-; FALLBACK20-NEXT:    movl (%eax), %eax
-; FALLBACK20-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    sarl $31, %ecx
-; FALLBACK20-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK20-NEXT:    movl %eax, %esi
-; FALLBACK20-NEXT:    andl $60, %esi
-; FALLBACK20-NEXT:    movl 68(%esp,%esi), %edx
-; FALLBACK20-NEXT:    shll $3, %eax
-; FALLBACK20-NEXT:    andl $24, %eax
-; FALLBACK20-NEXT:    movl %edx, %edi
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %edi
-; FALLBACK20-NEXT:    movl 72(%esp,%esi), %ecx
-; FALLBACK20-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK20-NEXT:    movb %al, %ch
-; FALLBACK20-NEXT:    notb %ch
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    orl %edi, %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 64(%esp,%esi), %edi
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edi
-; FALLBACK20-NEXT:    addl %edx, %edx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %edx
-; FALLBACK20-NEXT:    orl %edi, %edx
-; FALLBACK20-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 76(%esp,%esi), %edx
-; FALLBACK20-NEXT:    movl %edx, %ebp
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shrl %cl, %ebp
-; FALLBACK20-NEXT:    movl 80(%esp,%esi), %edi
-; FALLBACK20-NEXT:    leal (%edi,%edi), %ebx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    orl %ebp, %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %cl, %ebx
-; FALLBACK20-NEXT:    addl %edx, %edx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %edx
-; FALLBACK20-NEXT:    orl %ebx, %edx
-; FALLBACK20-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 84(%esp,%esi), %ebx
-; FALLBACK20-NEXT:    movl %ebx, %ebp
-; FALLBACK20-NEXT:    movl %eax, %edx
-; FALLBACK20-NEXT:    movb %dl, %cl
-; FALLBACK20-NEXT:    shrl %cl, %ebp
-; FALLBACK20-NEXT:    movl 88(%esp,%esi), %eax
-; FALLBACK20-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    addl %eax, %eax
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %eax
-; FALLBACK20-NEXT:    orl %ebp, %eax
-; FALLBACK20-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %dl, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edi
-; FALLBACK20-NEXT:    addl %ebx, %ebx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    orl %edi, %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 92(%esp,%esi), %ebx
-; FALLBACK20-NEXT:    movl %ebx, %ebp
-; FALLBACK20-NEXT:    movb %dl, %cl
-; FALLBACK20-NEXT:    shrl %cl, %ebp
-; FALLBACK20-NEXT:    movl 96(%esp,%esi), %edi
-; FALLBACK20-NEXT:    leal (%edi,%edi), %eax
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %eax
-; FALLBACK20-NEXT:    orl %ebp, %eax
-; FALLBACK20-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %dl, %cl
-; FALLBACK20-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %cl, %eax
-; FALLBACK20-NEXT:    addl %ebx, %ebx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    orl %eax, %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 100(%esp,%esi), %ebx
-; FALLBACK20-NEXT:    movl %ebx, %ebp
-; FALLBACK20-NEXT:    movb %dl, %cl
-; FALLBACK20-NEXT:    shrl %cl, %ebp
-; FALLBACK20-NEXT:    movl 104(%esp,%esi), %edx
-; FALLBACK20-NEXT:    leal (%edx,%edx), %eax
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %eax
-; FALLBACK20-NEXT:    orl %ebp, %eax
-; FALLBACK20-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edi
-; FALLBACK20-NEXT:    addl %ebx, %ebx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    orl %edi, %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 108(%esp,%esi), %edi
-; FALLBACK20-NEXT:    movl %edi, %ebp
-; FALLBACK20-NEXT:    movl %eax, %ecx
-; FALLBACK20-NEXT:    shrl %cl, %ebp
-; FALLBACK20-NEXT:    movl 112(%esp,%esi), %ecx
-; FALLBACK20-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK20-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebx
-; FALLBACK20-NEXT:    orl %ebp, %ebx
-; FALLBACK20-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shrl %cl, %edx
-; FALLBACK20-NEXT:    addl %edi, %edi
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %edi
-; FALLBACK20-NEXT:    orl %edx, %edi
-; FALLBACK20-NEXT:    movl %esi, %edx
-; FALLBACK20-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK20-NEXT:    movl 116(%esp,%esi), %esi
-; FALLBACK20-NEXT:    movl %esi, %ebx
-; FALLBACK20-NEXT:    movb %al, %cl
-; FALLBACK20-NEXT:    shrl %cl, %ebx
-; FALLBACK20-NEXT:    movl 120(%esp,%edx), %eax
-; FALLBACK20-NEXT:    leal (%eax,%eax), %ebp
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %ebp
-; FALLBACK20-NEXT:    orl %ebx, %ebp
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK20-NEXT:    movb %dl, %cl
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK20-NEXT:    shrl %cl, %ebx
-; FALLBACK20-NEXT:    addl %esi, %esi
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %esi
-; FALLBACK20-NEXT:    orl %ebx, %esi
-; FALLBACK20-NEXT:    movb %dl, %cl
-; FALLBACK20-NEXT:    shrl %cl, %eax
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK20-NEXT:    movl 124(%esp,%edx), %ebx
-; FALLBACK20-NEXT:    leal (%ebx,%ebx), %edx
-; FALLBACK20-NEXT:    movb %ch, %cl
-; FALLBACK20-NEXT:    shll %cl, %edx
-; FALLBACK20-NEXT:    orl %eax, %edx
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK20-NEXT:    sarl %cl, %ebx
-; FALLBACK20-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK20-NEXT:    movl %ebx, 60(%eax)
-; FALLBACK20-NEXT:    movl %edx, 56(%eax)
-; FALLBACK20-NEXT:    movl %esi, 48(%eax)
-; FALLBACK20-NEXT:    movl %ebp, 52(%eax)
-; FALLBACK20-NEXT:    movl %edi, 40(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, (%eax)
-; FALLBACK20-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK20-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK20-NEXT:    addl $204, %esp
-; FALLBACK20-NEXT:    popl %esi
-; FALLBACK20-NEXT:    popl %edi
-; FALLBACK20-NEXT:    popl %ebx
-; FALLBACK20-NEXT:    popl %ebp
-; FALLBACK20-NEXT:    retl
-;
-; FALLBACK21-LABEL: ashr_64bytes:
-; FALLBACK21:       # %bb.0:
-; FALLBACK21-NEXT:    pushl %ebp
-; FALLBACK21-NEXT:    pushl %ebx
-; FALLBACK21-NEXT:    pushl %edi
-; FALLBACK21-NEXT:    pushl %esi
-; FALLBACK21-NEXT:    subl $188, %esp
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK21-NEXT:    movups (%eax), %xmm0
-; FALLBACK21-NEXT:    movups 16(%eax), %xmm1
-; FALLBACK21-NEXT:    movups 32(%eax), %xmm2
-; FALLBACK21-NEXT:    movl 48(%eax), %edx
-; FALLBACK21-NEXT:    movl 52(%eax), %esi
-; FALLBACK21-NEXT:    movl 56(%eax), %edi
-; FALLBACK21-NEXT:    movl 60(%eax), %eax
-; FALLBACK21-NEXT:    movl (%ecx), %ecx
-; FALLBACK21-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    sarl $31, %eax
-; FALLBACK21-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK21-NEXT:    movl %ecx, %ebp
-; FALLBACK21-NEXT:    andl $60, %ebp
-; FALLBACK21-NEXT:    movl 56(%esp,%ebp), %edx
-; FALLBACK21-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shll $3, %ecx
-; FALLBACK21-NEXT:    andl $24, %ecx
-; FALLBACK21-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK21-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 64(%esp,%ebp), %edi
-; FALLBACK21-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, %esi
-; FALLBACK21-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK21-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 72(%esp,%ebp), %esi
-; FALLBACK21-NEXT:    movl 68(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, %edx
-; FALLBACK21-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK21-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 80(%esp,%ebp), %edi
-; FALLBACK21-NEXT:    movl 76(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, %edx
-; FALLBACK21-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK21-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 88(%esp,%ebp), %esi
-; FALLBACK21-NEXT:    movl 84(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, %edx
-; FALLBACK21-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK21-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl %esi, %edx
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK21-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 96(%esp,%ebp), %esi
-; FALLBACK21-NEXT:    movl 92(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, %edi
-; FALLBACK21-NEXT:    shrdl %cl, %esi, %edi
-; FALLBACK21-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK21-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK21-NEXT:    movl 104(%esp,%ebp), %edx
-; FALLBACK21-NEXT:    movl 100(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    movl %eax, %edi
-; FALLBACK21-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK21-NEXT:    movl 48(%esp,%ebp), %ebx
-; FALLBACK21-NEXT:    movl 108(%esp,%ebp), %eax
-; FALLBACK21-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK21-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK21-NEXT:    movl %edx, 56(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK21-NEXT:    shrdl %cl, %edx, %ebx
-; FALLBACK21-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK21-NEXT:    sarl %cl, %eax
-; FALLBACK21-NEXT:    movl %eax, 60(%ebp)
-; FALLBACK21-NEXT:    movl %esi, 48(%ebp)
-; FALLBACK21-NEXT:    movl %edi, 52(%ebp)
-; FALLBACK21-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 40(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK21-NEXT:    movl %ebx, (%ebp)
-; FALLBACK21-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK21-NEXT:    movl %eax, 4(%ebp)
-; FALLBACK21-NEXT:    addl $188, %esp
-; FALLBACK21-NEXT:    popl %esi
-; FALLBACK21-NEXT:    popl %edi
-; FALLBACK21-NEXT:    popl %ebx
-; FALLBACK21-NEXT:    popl %ebp
-; FALLBACK21-NEXT:    retl
-;
-; FALLBACK22-LABEL: ashr_64bytes:
-; FALLBACK22:       # %bb.0:
-; FALLBACK22-NEXT:    pushl %ebp
-; FALLBACK22-NEXT:    pushl %ebx
-; FALLBACK22-NEXT:    pushl %edi
-; FALLBACK22-NEXT:    pushl %esi
-; FALLBACK22-NEXT:    subl $204, %esp
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK22-NEXT:    movups (%ecx), %xmm0
-; FALLBACK22-NEXT:    movups 16(%ecx), %xmm1
-; FALLBACK22-NEXT:    movups 32(%ecx), %xmm2
-; FALLBACK22-NEXT:    movl 48(%ecx), %edx
-; FALLBACK22-NEXT:    movl 52(%ecx), %esi
-; FALLBACK22-NEXT:    movl 56(%ecx), %edi
-; FALLBACK22-NEXT:    movl 60(%ecx), %ecx
-; FALLBACK22-NEXT:    movl (%eax), %eax
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    sarl $31, %ecx
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK22-NEXT:    movl %eax, %ecx
-; FALLBACK22-NEXT:    leal (,%eax,8), %edx
-; FALLBACK22-NEXT:    andl $24, %edx
-; FALLBACK22-NEXT:    movl %edx, %ebx
-; FALLBACK22-NEXT:    andl $60, %ecx
-; FALLBACK22-NEXT:    movl 68(%esp,%ecx), %esi
-; FALLBACK22-NEXT:    movl 72(%esp,%ecx), %edi
-; FALLBACK22-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrxl %ebx, %esi, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    notb %dl
-; FALLBACK22-NEXT:    leal (%edi,%edi), %ebp
-; FALLBACK22-NEXT:    shlxl %edx, %ebp, %eax
-; FALLBACK22-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrxl %ebx, 64(%esp,%ecx), %edi
-; FALLBACK22-NEXT:    addl %esi, %esi
-; FALLBACK22-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK22-NEXT:    orl %edi, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 80(%esp,%ecx), %esi
-; FALLBACK22-NEXT:    leal (%esi,%esi), %edi
-; FALLBACK22-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK22-NEXT:    movl 76(%esp,%ecx), %edi
-; FALLBACK22-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK22-NEXT:    orl %ebp, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK22-NEXT:    addl %edi, %edi
-; FALLBACK22-NEXT:    shlxl %edx, %edi, %edi
-; FALLBACK22-NEXT:    orl %eax, %edi
-; FALLBACK22-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 88(%esp,%ecx), %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK22-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK22-NEXT:    movl 84(%esp,%ecx), %edi
-; FALLBACK22-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK22-NEXT:    orl %ebp, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrxl %ebx, %esi, %esi
-; FALLBACK22-NEXT:    addl %edi, %edi
-; FALLBACK22-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK22-NEXT:    orl %esi, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 96(%esp,%ecx), %esi
-; FALLBACK22-NEXT:    leal (%esi,%esi), %edi
-; FALLBACK22-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK22-NEXT:    movl 92(%esp,%ecx), %edi
-; FALLBACK22-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK22-NEXT:    orl %ebp, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK22-NEXT:    addl %edi, %edi
-; FALLBACK22-NEXT:    shlxl %edx, %edi, %edi
-; FALLBACK22-NEXT:    orl %eax, %edi
-; FALLBACK22-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 104(%esp,%ecx), %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK22-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK22-NEXT:    movl 100(%esp,%ecx), %edi
-; FALLBACK22-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK22-NEXT:    orl %ebp, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrxl %ebx, %esi, %esi
-; FALLBACK22-NEXT:    addl %edi, %edi
-; FALLBACK22-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK22-NEXT:    orl %esi, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl %ecx, %ebp
-; FALLBACK22-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 112(%esp,%ecx), %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    leal (%eax,%eax), %esi
-; FALLBACK22-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK22-NEXT:    movl 108(%esp,%ecx), %esi
-; FALLBACK22-NEXT:    shrxl %ebx, %esi, %edi
-; FALLBACK22-NEXT:    orl %edi, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; FALLBACK22-NEXT:    addl %esi, %esi
-; FALLBACK22-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK22-NEXT:    orl %ecx, %eax
-; FALLBACK22-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK22-NEXT:    movl 120(%esp,%ebp), %edi
-; FALLBACK22-NEXT:    leal (%edi,%edi), %ecx
-; FALLBACK22-NEXT:    shlxl %edx, %ecx, %esi
-; FALLBACK22-NEXT:    movl 116(%esp,%ebp), %eax
-; FALLBACK22-NEXT:    shrxl %ebx, %eax, %ebp
-; FALLBACK22-NEXT:    orl %ebp, %esi
-; FALLBACK22-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK22-NEXT:    addl %eax, %eax
-; FALLBACK22-NEXT:    shlxl %edx, %eax, %ecx
-; FALLBACK22-NEXT:    orl %ebp, %ecx
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK22-NEXT:    movl 124(%esp,%eax), %eax
-; FALLBACK22-NEXT:    leal (%eax,%eax), %ebp
-; FALLBACK22-NEXT:    shlxl %edx, %ebp, %edx
-; FALLBACK22-NEXT:    shrxl %ebx, %edi, %edi
-; FALLBACK22-NEXT:    orl %edi, %edx
-; FALLBACK22-NEXT:    sarxl %ebx, %eax, %edi
-; FALLBACK22-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK22-NEXT:    movl %edi, 60(%eax)
-; FALLBACK22-NEXT:    movl %edx, 56(%eax)
-; FALLBACK22-NEXT:    movl %ecx, 48(%eax)
-; FALLBACK22-NEXT:    movl %esi, 52(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 40(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, (%eax)
-; FALLBACK22-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK22-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK22-NEXT:    addl $204, %esp
-; FALLBACK22-NEXT:    popl %esi
-; FALLBACK22-NEXT:    popl %edi
-; FALLBACK22-NEXT:    popl %ebx
-; FALLBACK22-NEXT:    popl %ebp
-; FALLBACK22-NEXT:    retl
-;
-; FALLBACK23-LABEL: ashr_64bytes:
-; FALLBACK23:       # %bb.0:
-; FALLBACK23-NEXT:    pushl %ebp
-; FALLBACK23-NEXT:    pushl %ebx
-; FALLBACK23-NEXT:    pushl %edi
-; FALLBACK23-NEXT:    pushl %esi
-; FALLBACK23-NEXT:    subl $188, %esp
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK23-NEXT:    movups (%eax), %xmm0
-; FALLBACK23-NEXT:    movups 16(%eax), %xmm1
-; FALLBACK23-NEXT:    movups 32(%eax), %xmm2
-; FALLBACK23-NEXT:    movl 48(%eax), %edx
-; FALLBACK23-NEXT:    movl 52(%eax), %esi
-; FALLBACK23-NEXT:    movl 56(%eax), %edi
-; FALLBACK23-NEXT:    movl 60(%eax), %eax
-; FALLBACK23-NEXT:    movl (%ecx), %ecx
-; FALLBACK23-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    sarl $31, %eax
-; FALLBACK23-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK23-NEXT:    movl %ecx, %ebp
-; FALLBACK23-NEXT:    andl $60, %ebp
-; FALLBACK23-NEXT:    movl 56(%esp,%ebp), %edx
-; FALLBACK23-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK23-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shll $3, %ecx
-; FALLBACK23-NEXT:    andl $24, %ecx
-; FALLBACK23-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK23-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 64(%esp,%ebp), %edi
-; FALLBACK23-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK23-NEXT:    movl %eax, %esi
-; FALLBACK23-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK23-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 72(%esp,%ebp), %esi
-; FALLBACK23-NEXT:    movl 68(%esp,%ebp), %eax
-; FALLBACK23-NEXT:    movl %eax, %edx
-; FALLBACK23-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK23-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 80(%esp,%ebp), %edi
-; FALLBACK23-NEXT:    movl 76(%esp,%ebp), %eax
-; FALLBACK23-NEXT:    movl %eax, %edx
-; FALLBACK23-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK23-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 88(%esp,%ebp), %ebx
-; FALLBACK23-NEXT:    movl 84(%esp,%ebp), %eax
-; FALLBACK23-NEXT:    movl %eax, %edx
-; FALLBACK23-NEXT:    shrdl %cl, %ebx, %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK23-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    movl 96(%esp,%ebp), %esi
-; FALLBACK23-NEXT:    movl 92(%esp,%ebp), %eax
-; FALLBACK23-NEXT:    movl %eax, %edx
-; FALLBACK23-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK23-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK23-NEXT:    movl 104(%esp,%ebp), %eax
-; FALLBACK23-NEXT:    movl 100(%esp,%ebp), %edi
-; FALLBACK23-NEXT:    movl %edi, %edx
-; FALLBACK23-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK23-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK23-NEXT:    movl 48(%esp,%ebp), %edi
-; FALLBACK23-NEXT:    movl 108(%esp,%ebp), %ebp
-; FALLBACK23-NEXT:    movl %ebp, (%esp) # 4-byte Spill
-; FALLBACK23-NEXT:    shrdl %cl, %ebp, %eax
-; FALLBACK23-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK23-NEXT:    movl %eax, 56(%ebp)
-; FALLBACK23-NEXT:    movl %esi, 48(%ebp)
-; FALLBACK23-NEXT:    movl %edx, 52(%ebp)
-; FALLBACK23-NEXT:    movl %ebx, 40(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK23-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK23-NEXT:    sarxl %ecx, (%esp), %eax # 4-byte Folded Reload
-; FALLBACK23-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK23-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK23-NEXT:    movl %edi, (%ebp)
-; FALLBACK23-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK23-NEXT:    movl %ecx, 4(%ebp)
-; FALLBACK23-NEXT:    movl %eax, 60(%ebp)
-; FALLBACK23-NEXT:    addl $188, %esp
-; FALLBACK23-NEXT:    popl %esi
-; FALLBACK23-NEXT:    popl %edi
-; FALLBACK23-NEXT:    popl %ebx
-; FALLBACK23-NEXT:    popl %ebp
-; FALLBACK23-NEXT:    retl
-;
-; FALLBACK24-LABEL: ashr_64bytes:
-; FALLBACK24:       # %bb.0:
-; FALLBACK24-NEXT:    pushl %ebp
-; FALLBACK24-NEXT:    pushl %ebx
-; FALLBACK24-NEXT:    pushl %edi
-; FALLBACK24-NEXT:    pushl %esi
-; FALLBACK24-NEXT:    subl $204, %esp
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK24-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK24-NEXT:    vmovups 32(%ecx), %xmm1
-; FALLBACK24-NEXT:    movl 48(%ecx), %edx
-; FALLBACK24-NEXT:    movl 52(%ecx), %esi
-; FALLBACK24-NEXT:    movl 56(%ecx), %edi
-; FALLBACK24-NEXT:    movl 60(%ecx), %ecx
-; FALLBACK24-NEXT:    movl (%eax), %eax
-; FALLBACK24-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    sarl $31, %ecx
-; FALLBACK24-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK24-NEXT:    movl %eax, %esi
-; FALLBACK24-NEXT:    andl $60, %esi
-; FALLBACK24-NEXT:    movl 68(%esp,%esi), %edx
-; FALLBACK24-NEXT:    shll $3, %eax
-; FALLBACK24-NEXT:    andl $24, %eax
-; FALLBACK24-NEXT:    movl %edx, %edi
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %edi
-; FALLBACK24-NEXT:    movl 72(%esp,%esi), %ecx
-; FALLBACK24-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK24-NEXT:    movb %al, %ch
-; FALLBACK24-NEXT:    notb %ch
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    orl %edi, %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 64(%esp,%esi), %edi
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edi
-; FALLBACK24-NEXT:    addl %edx, %edx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %edx
-; FALLBACK24-NEXT:    orl %edi, %edx
-; FALLBACK24-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 76(%esp,%esi), %edx
-; FALLBACK24-NEXT:    movl %edx, %ebp
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shrl %cl, %ebp
-; FALLBACK24-NEXT:    movl 80(%esp,%esi), %edi
-; FALLBACK24-NEXT:    leal (%edi,%edi), %ebx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    orl %ebp, %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %cl, %ebx
-; FALLBACK24-NEXT:    addl %edx, %edx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %edx
-; FALLBACK24-NEXT:    orl %ebx, %edx
-; FALLBACK24-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 84(%esp,%esi), %ebx
-; FALLBACK24-NEXT:    movl %ebx, %ebp
-; FALLBACK24-NEXT:    movl %eax, %edx
-; FALLBACK24-NEXT:    movb %dl, %cl
-; FALLBACK24-NEXT:    shrl %cl, %ebp
-; FALLBACK24-NEXT:    movl 88(%esp,%esi), %eax
-; FALLBACK24-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    addl %eax, %eax
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %eax
-; FALLBACK24-NEXT:    orl %ebp, %eax
-; FALLBACK24-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %dl, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edi
-; FALLBACK24-NEXT:    addl %ebx, %ebx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    orl %edi, %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 92(%esp,%esi), %ebx
-; FALLBACK24-NEXT:    movl %ebx, %ebp
-; FALLBACK24-NEXT:    movb %dl, %cl
-; FALLBACK24-NEXT:    shrl %cl, %ebp
-; FALLBACK24-NEXT:    movl 96(%esp,%esi), %edi
-; FALLBACK24-NEXT:    leal (%edi,%edi), %eax
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %eax
-; FALLBACK24-NEXT:    orl %ebp, %eax
-; FALLBACK24-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %dl, %cl
-; FALLBACK24-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %cl, %eax
-; FALLBACK24-NEXT:    addl %ebx, %ebx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    orl %eax, %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 100(%esp,%esi), %ebx
-; FALLBACK24-NEXT:    movl %ebx, %ebp
-; FALLBACK24-NEXT:    movb %dl, %cl
-; FALLBACK24-NEXT:    shrl %cl, %ebp
-; FALLBACK24-NEXT:    movl 104(%esp,%esi), %edx
-; FALLBACK24-NEXT:    leal (%edx,%edx), %eax
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %eax
-; FALLBACK24-NEXT:    orl %ebp, %eax
-; FALLBACK24-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edi
-; FALLBACK24-NEXT:    addl %ebx, %ebx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    orl %edi, %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 108(%esp,%esi), %edi
-; FALLBACK24-NEXT:    movl %edi, %ebp
-; FALLBACK24-NEXT:    movl %eax, %ecx
-; FALLBACK24-NEXT:    shrl %cl, %ebp
-; FALLBACK24-NEXT:    movl 112(%esp,%esi), %ecx
-; FALLBACK24-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK24-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebx
-; FALLBACK24-NEXT:    orl %ebp, %ebx
-; FALLBACK24-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shrl %cl, %edx
-; FALLBACK24-NEXT:    addl %edi, %edi
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %edi
-; FALLBACK24-NEXT:    orl %edx, %edi
-; FALLBACK24-NEXT:    movl %esi, %edx
-; FALLBACK24-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK24-NEXT:    movl 116(%esp,%esi), %esi
-; FALLBACK24-NEXT:    movl %esi, %ebx
-; FALLBACK24-NEXT:    movb %al, %cl
-; FALLBACK24-NEXT:    shrl %cl, %ebx
-; FALLBACK24-NEXT:    movl 120(%esp,%edx), %eax
-; FALLBACK24-NEXT:    leal (%eax,%eax), %ebp
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %ebp
-; FALLBACK24-NEXT:    orl %ebx, %ebp
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK24-NEXT:    movb %dl, %cl
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK24-NEXT:    shrl %cl, %ebx
-; FALLBACK24-NEXT:    addl %esi, %esi
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %esi
-; FALLBACK24-NEXT:    orl %ebx, %esi
-; FALLBACK24-NEXT:    movb %dl, %cl
-; FALLBACK24-NEXT:    shrl %cl, %eax
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK24-NEXT:    movl 124(%esp,%edx), %ebx
-; FALLBACK24-NEXT:    leal (%ebx,%ebx), %edx
-; FALLBACK24-NEXT:    movb %ch, %cl
-; FALLBACK24-NEXT:    shll %cl, %edx
-; FALLBACK24-NEXT:    orl %eax, %edx
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK24-NEXT:    sarl %cl, %ebx
-; FALLBACK24-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK24-NEXT:    movl %ebx, 60(%eax)
-; FALLBACK24-NEXT:    movl %edx, 56(%eax)
-; FALLBACK24-NEXT:    movl %esi, 48(%eax)
-; FALLBACK24-NEXT:    movl %ebp, 52(%eax)
-; FALLBACK24-NEXT:    movl %edi, 40(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, (%eax)
-; FALLBACK24-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK24-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK24-NEXT:    addl $204, %esp
-; FALLBACK24-NEXT:    popl %esi
-; FALLBACK24-NEXT:    popl %edi
-; FALLBACK24-NEXT:    popl %ebx
-; FALLBACK24-NEXT:    popl %ebp
-; FALLBACK24-NEXT:    vzeroupper
-; FALLBACK24-NEXT:    retl
-;
-; FALLBACK25-LABEL: ashr_64bytes:
-; FALLBACK25:       # %bb.0:
-; FALLBACK25-NEXT:    pushl %ebp
-; FALLBACK25-NEXT:    pushl %ebx
-; FALLBACK25-NEXT:    pushl %edi
-; FALLBACK25-NEXT:    pushl %esi
-; FALLBACK25-NEXT:    subl $188, %esp
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK25-NEXT:    vmovups (%eax), %ymm0
-; FALLBACK25-NEXT:    vmovups 32(%eax), %xmm1
-; FALLBACK25-NEXT:    movl 48(%eax), %edx
-; FALLBACK25-NEXT:    movl 52(%eax), %esi
-; FALLBACK25-NEXT:    movl 56(%eax), %edi
-; FALLBACK25-NEXT:    movl 60(%eax), %eax
-; FALLBACK25-NEXT:    movl (%ecx), %ecx
-; FALLBACK25-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    sarl $31, %eax
-; FALLBACK25-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK25-NEXT:    movl %ecx, %ebp
-; FALLBACK25-NEXT:    andl $60, %ebp
-; FALLBACK25-NEXT:    movl 56(%esp,%ebp), %edx
-; FALLBACK25-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shll $3, %ecx
-; FALLBACK25-NEXT:    andl $24, %ecx
-; FALLBACK25-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK25-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 64(%esp,%ebp), %edi
-; FALLBACK25-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, %esi
-; FALLBACK25-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK25-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 72(%esp,%ebp), %esi
-; FALLBACK25-NEXT:    movl 68(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, %edx
-; FALLBACK25-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK25-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 80(%esp,%ebp), %edi
-; FALLBACK25-NEXT:    movl 76(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, %edx
-; FALLBACK25-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK25-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 88(%esp,%ebp), %esi
-; FALLBACK25-NEXT:    movl 84(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, %edx
-; FALLBACK25-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK25-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl %esi, %edx
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK25-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 96(%esp,%ebp), %esi
-; FALLBACK25-NEXT:    movl 92(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, %edi
-; FALLBACK25-NEXT:    shrdl %cl, %esi, %edi
-; FALLBACK25-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK25-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK25-NEXT:    movl 104(%esp,%ebp), %edx
-; FALLBACK25-NEXT:    movl 100(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    movl %eax, %edi
-; FALLBACK25-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK25-NEXT:    movl 48(%esp,%ebp), %ebx
-; FALLBACK25-NEXT:    movl 108(%esp,%ebp), %eax
-; FALLBACK25-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK25-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK25-NEXT:    movl %edx, 56(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK25-NEXT:    shrdl %cl, %edx, %ebx
-; FALLBACK25-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK25-NEXT:    sarl %cl, %eax
-; FALLBACK25-NEXT:    movl %eax, 60(%ebp)
-; FALLBACK25-NEXT:    movl %esi, 48(%ebp)
-; FALLBACK25-NEXT:    movl %edi, 52(%ebp)
-; FALLBACK25-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 40(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK25-NEXT:    movl %ebx, (%ebp)
-; FALLBACK25-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK25-NEXT:    movl %eax, 4(%ebp)
-; FALLBACK25-NEXT:    addl $188, %esp
-; FALLBACK25-NEXT:    popl %esi
-; FALLBACK25-NEXT:    popl %edi
-; FALLBACK25-NEXT:    popl %ebx
-; FALLBACK25-NEXT:    popl %ebp
-; FALLBACK25-NEXT:    vzeroupper
-; FALLBACK25-NEXT:    retl
-;
-; FALLBACK26-LABEL: ashr_64bytes:
-; FALLBACK26:       # %bb.0:
-; FALLBACK26-NEXT:    pushl %ebp
-; FALLBACK26-NEXT:    pushl %ebx
-; FALLBACK26-NEXT:    pushl %edi
-; FALLBACK26-NEXT:    pushl %esi
-; FALLBACK26-NEXT:    subl $204, %esp
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK26-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK26-NEXT:    vmovups 32(%ecx), %xmm1
-; FALLBACK26-NEXT:    movl 48(%ecx), %edx
-; FALLBACK26-NEXT:    movl 52(%ecx), %esi
-; FALLBACK26-NEXT:    movl 56(%ecx), %edi
-; FALLBACK26-NEXT:    movl 60(%ecx), %ecx
-; FALLBACK26-NEXT:    movl (%eax), %eax
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    sarl $31, %ecx
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK26-NEXT:    movl %eax, %ecx
-; FALLBACK26-NEXT:    leal (,%eax,8), %edx
-; FALLBACK26-NEXT:    andl $24, %edx
-; FALLBACK26-NEXT:    movl %edx, %ebx
-; FALLBACK26-NEXT:    andl $60, %ecx
-; FALLBACK26-NEXT:    movl 68(%esp,%ecx), %esi
-; FALLBACK26-NEXT:    movl 72(%esp,%ecx), %edi
-; FALLBACK26-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrxl %ebx, %esi, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    notb %dl
-; FALLBACK26-NEXT:    leal (%edi,%edi), %ebp
-; FALLBACK26-NEXT:    shlxl %edx, %ebp, %eax
-; FALLBACK26-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrxl %ebx, 64(%esp,%ecx), %edi
-; FALLBACK26-NEXT:    addl %esi, %esi
-; FALLBACK26-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK26-NEXT:    orl %edi, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 80(%esp,%ecx), %esi
-; FALLBACK26-NEXT:    leal (%esi,%esi), %edi
-; FALLBACK26-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK26-NEXT:    movl 76(%esp,%ecx), %edi
-; FALLBACK26-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK26-NEXT:    orl %ebp, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK26-NEXT:    addl %edi, %edi
-; FALLBACK26-NEXT:    shlxl %edx, %edi, %edi
-; FALLBACK26-NEXT:    orl %eax, %edi
-; FALLBACK26-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 88(%esp,%ecx), %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK26-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK26-NEXT:    movl 84(%esp,%ecx), %edi
-; FALLBACK26-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK26-NEXT:    orl %ebp, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrxl %ebx, %esi, %esi
-; FALLBACK26-NEXT:    addl %edi, %edi
-; FALLBACK26-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK26-NEXT:    orl %esi, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 96(%esp,%ecx), %esi
-; FALLBACK26-NEXT:    leal (%esi,%esi), %edi
-; FALLBACK26-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK26-NEXT:    movl 92(%esp,%ecx), %edi
-; FALLBACK26-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK26-NEXT:    orl %ebp, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK26-NEXT:    addl %edi, %edi
-; FALLBACK26-NEXT:    shlxl %edx, %edi, %edi
-; FALLBACK26-NEXT:    orl %eax, %edi
-; FALLBACK26-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 104(%esp,%ecx), %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK26-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK26-NEXT:    movl 100(%esp,%ecx), %edi
-; FALLBACK26-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK26-NEXT:    orl %ebp, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrxl %ebx, %esi, %esi
-; FALLBACK26-NEXT:    addl %edi, %edi
-; FALLBACK26-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK26-NEXT:    orl %esi, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl %ecx, %ebp
-; FALLBACK26-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 112(%esp,%ecx), %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    leal (%eax,%eax), %esi
-; FALLBACK26-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK26-NEXT:    movl 108(%esp,%ecx), %esi
-; FALLBACK26-NEXT:    shrxl %ebx, %esi, %edi
-; FALLBACK26-NEXT:    orl %edi, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; FALLBACK26-NEXT:    addl %esi, %esi
-; FALLBACK26-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK26-NEXT:    orl %ecx, %eax
-; FALLBACK26-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK26-NEXT:    movl 120(%esp,%ebp), %edi
-; FALLBACK26-NEXT:    leal (%edi,%edi), %ecx
-; FALLBACK26-NEXT:    shlxl %edx, %ecx, %esi
-; FALLBACK26-NEXT:    movl 116(%esp,%ebp), %eax
-; FALLBACK26-NEXT:    shrxl %ebx, %eax, %ebp
-; FALLBACK26-NEXT:    orl %ebp, %esi
-; FALLBACK26-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK26-NEXT:    addl %eax, %eax
-; FALLBACK26-NEXT:    shlxl %edx, %eax, %ecx
-; FALLBACK26-NEXT:    orl %ebp, %ecx
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK26-NEXT:    movl 124(%esp,%eax), %eax
-; FALLBACK26-NEXT:    leal (%eax,%eax), %ebp
-; FALLBACK26-NEXT:    shlxl %edx, %ebp, %edx
-; FALLBACK26-NEXT:    shrxl %ebx, %edi, %edi
-; FALLBACK26-NEXT:    orl %edi, %edx
-; FALLBACK26-NEXT:    sarxl %ebx, %eax, %edi
-; FALLBACK26-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK26-NEXT:    movl %edi, 60(%eax)
-; FALLBACK26-NEXT:    movl %edx, 56(%eax)
-; FALLBACK26-NEXT:    movl %ecx, 48(%eax)
-; FALLBACK26-NEXT:    movl %esi, 52(%eax)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK26-NEXT:    movl %ecx, 40(%eax)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK26-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK26-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK26-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK26-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK26-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK26-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK26-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK26-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK26-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK26-NEXT:    movl %ecx, (%eax)
-; FALLBACK26-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK26-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK26-NEXT:    addl $204, %esp
-; FALLBACK26-NEXT:    popl %esi
-; FALLBACK26-NEXT:    popl %edi
-; FALLBACK26-NEXT:    popl %ebx
-; FALLBACK26-NEXT:    popl %ebp
-; FALLBACK26-NEXT:    vzeroupper
-; FALLBACK26-NEXT:    retl
-;
-; FALLBACK27-LABEL: ashr_64bytes:
-; FALLBACK27:       # %bb.0:
-; FALLBACK27-NEXT:    pushl %ebp
-; FALLBACK27-NEXT:    pushl %ebx
-; FALLBACK27-NEXT:    pushl %edi
-; FALLBACK27-NEXT:    pushl %esi
-; FALLBACK27-NEXT:    subl $188, %esp
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK27-NEXT:    vmovups (%eax), %ymm0
-; FALLBACK27-NEXT:    vmovups 32(%eax), %xmm1
-; FALLBACK27-NEXT:    movl 48(%eax), %edx
-; FALLBACK27-NEXT:    movl 52(%eax), %esi
-; FALLBACK27-NEXT:    movl 56(%eax), %edi
-; FALLBACK27-NEXT:    movl 60(%eax), %eax
-; FALLBACK27-NEXT:    movl (%ecx), %ecx
-; FALLBACK27-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    sarl $31, %eax
-; FALLBACK27-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK27-NEXT:    movl %ecx, %ebp
-; FALLBACK27-NEXT:    andl $60, %ebp
-; FALLBACK27-NEXT:    movl 56(%esp,%ebp), %edx
-; FALLBACK27-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK27-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shll $3, %ecx
-; FALLBACK27-NEXT:    andl $24, %ecx
-; FALLBACK27-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK27-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 64(%esp,%ebp), %edi
-; FALLBACK27-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK27-NEXT:    movl %eax, %esi
-; FALLBACK27-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK27-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 72(%esp,%ebp), %esi
-; FALLBACK27-NEXT:    movl 68(%esp,%ebp), %eax
-; FALLBACK27-NEXT:    movl %eax, %edx
-; FALLBACK27-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK27-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 80(%esp,%ebp), %edi
-; FALLBACK27-NEXT:    movl 76(%esp,%ebp), %eax
-; FALLBACK27-NEXT:    movl %eax, %edx
-; FALLBACK27-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK27-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 88(%esp,%ebp), %ebx
-; FALLBACK27-NEXT:    movl 84(%esp,%ebp), %eax
-; FALLBACK27-NEXT:    movl %eax, %edx
-; FALLBACK27-NEXT:    shrdl %cl, %ebx, %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK27-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    movl 96(%esp,%ebp), %esi
-; FALLBACK27-NEXT:    movl 92(%esp,%ebp), %eax
-; FALLBACK27-NEXT:    movl %eax, %edx
-; FALLBACK27-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK27-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK27-NEXT:    movl 104(%esp,%ebp), %eax
-; FALLBACK27-NEXT:    movl 100(%esp,%ebp), %edi
-; FALLBACK27-NEXT:    movl %edi, %edx
-; FALLBACK27-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK27-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK27-NEXT:    movl 48(%esp,%ebp), %edi
-; FALLBACK27-NEXT:    movl 108(%esp,%ebp), %ebp
-; FALLBACK27-NEXT:    movl %ebp, (%esp) # 4-byte Spill
-; FALLBACK27-NEXT:    shrdl %cl, %ebp, %eax
-; FALLBACK27-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK27-NEXT:    movl %eax, 56(%ebp)
-; FALLBACK27-NEXT:    movl %esi, 48(%ebp)
-; FALLBACK27-NEXT:    movl %edx, 52(%ebp)
-; FALLBACK27-NEXT:    movl %ebx, 40(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK27-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK27-NEXT:    sarxl %ecx, (%esp), %eax # 4-byte Folded Reload
-; FALLBACK27-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK27-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK27-NEXT:    movl %edi, (%ebp)
-; FALLBACK27-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK27-NEXT:    movl %ecx, 4(%ebp)
-; FALLBACK27-NEXT:    movl %eax, 60(%ebp)
-; FALLBACK27-NEXT:    addl $188, %esp
-; FALLBACK27-NEXT:    popl %esi
-; FALLBACK27-NEXT:    popl %edi
-; FALLBACK27-NEXT:    popl %ebx
-; FALLBACK27-NEXT:    popl %ebp
-; FALLBACK27-NEXT:    vzeroupper
-; FALLBACK27-NEXT:    retl
-;
-; FALLBACK28-LABEL: ashr_64bytes:
-; FALLBACK28:       # %bb.0:
-; FALLBACK28-NEXT:    pushl %ebp
-; FALLBACK28-NEXT:    pushl %ebx
-; FALLBACK28-NEXT:    pushl %edi
-; FALLBACK28-NEXT:    pushl %esi
-; FALLBACK28-NEXT:    subl $204, %esp
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK28-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK28-NEXT:    vmovups 32(%ecx), %xmm1
-; FALLBACK28-NEXT:    movl 48(%ecx), %edx
-; FALLBACK28-NEXT:    movl 52(%ecx), %esi
-; FALLBACK28-NEXT:    movl 56(%ecx), %edi
-; FALLBACK28-NEXT:    movl 60(%ecx), %ecx
-; FALLBACK28-NEXT:    movl (%eax), %eax
-; FALLBACK28-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    sarl $31, %ecx
-; FALLBACK28-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK28-NEXT:    movl %eax, %esi
-; FALLBACK28-NEXT:    andl $60, %esi
-; FALLBACK28-NEXT:    movl 68(%esp,%esi), %edx
-; FALLBACK28-NEXT:    shll $3, %eax
-; FALLBACK28-NEXT:    andl $24, %eax
-; FALLBACK28-NEXT:    movl %edx, %edi
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %edi
-; FALLBACK28-NEXT:    movl 72(%esp,%esi), %ecx
-; FALLBACK28-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK28-NEXT:    movb %al, %ch
-; FALLBACK28-NEXT:    notb %ch
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    orl %edi, %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 64(%esp,%esi), %edi
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edi
-; FALLBACK28-NEXT:    addl %edx, %edx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %edx
-; FALLBACK28-NEXT:    orl %edi, %edx
-; FALLBACK28-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 76(%esp,%esi), %edx
-; FALLBACK28-NEXT:    movl %edx, %ebp
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shrl %cl, %ebp
-; FALLBACK28-NEXT:    movl 80(%esp,%esi), %edi
-; FALLBACK28-NEXT:    leal (%edi,%edi), %ebx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    orl %ebp, %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %cl, %ebx
-; FALLBACK28-NEXT:    addl %edx, %edx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %edx
-; FALLBACK28-NEXT:    orl %ebx, %edx
-; FALLBACK28-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 84(%esp,%esi), %ebx
-; FALLBACK28-NEXT:    movl %ebx, %ebp
-; FALLBACK28-NEXT:    movl %eax, %edx
-; FALLBACK28-NEXT:    movb %dl, %cl
-; FALLBACK28-NEXT:    shrl %cl, %ebp
-; FALLBACK28-NEXT:    movl 88(%esp,%esi), %eax
-; FALLBACK28-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    addl %eax, %eax
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %eax
-; FALLBACK28-NEXT:    orl %ebp, %eax
-; FALLBACK28-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %dl, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edi
-; FALLBACK28-NEXT:    addl %ebx, %ebx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    orl %edi, %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 92(%esp,%esi), %ebx
-; FALLBACK28-NEXT:    movl %ebx, %ebp
-; FALLBACK28-NEXT:    movb %dl, %cl
-; FALLBACK28-NEXT:    shrl %cl, %ebp
-; FALLBACK28-NEXT:    movl 96(%esp,%esi), %edi
-; FALLBACK28-NEXT:    leal (%edi,%edi), %eax
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %eax
-; FALLBACK28-NEXT:    orl %ebp, %eax
-; FALLBACK28-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %dl, %cl
-; FALLBACK28-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %cl, %eax
-; FALLBACK28-NEXT:    addl %ebx, %ebx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    orl %eax, %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 100(%esp,%esi), %ebx
-; FALLBACK28-NEXT:    movl %ebx, %ebp
-; FALLBACK28-NEXT:    movb %dl, %cl
-; FALLBACK28-NEXT:    shrl %cl, %ebp
-; FALLBACK28-NEXT:    movl 104(%esp,%esi), %edx
-; FALLBACK28-NEXT:    leal (%edx,%edx), %eax
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %eax
-; FALLBACK28-NEXT:    orl %ebp, %eax
-; FALLBACK28-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edi
-; FALLBACK28-NEXT:    addl %ebx, %ebx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    orl %edi, %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 108(%esp,%esi), %edi
-; FALLBACK28-NEXT:    movl %edi, %ebp
-; FALLBACK28-NEXT:    movl %eax, %ecx
-; FALLBACK28-NEXT:    shrl %cl, %ebp
-; FALLBACK28-NEXT:    movl 112(%esp,%esi), %ecx
-; FALLBACK28-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    leal (%ecx,%ecx), %ebx
-; FALLBACK28-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebx
-; FALLBACK28-NEXT:    orl %ebp, %ebx
-; FALLBACK28-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shrl %cl, %edx
-; FALLBACK28-NEXT:    addl %edi, %edi
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %edi
-; FALLBACK28-NEXT:    orl %edx, %edi
-; FALLBACK28-NEXT:    movl %esi, %edx
-; FALLBACK28-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK28-NEXT:    movl 116(%esp,%esi), %esi
-; FALLBACK28-NEXT:    movl %esi, %ebx
-; FALLBACK28-NEXT:    movb %al, %cl
-; FALLBACK28-NEXT:    shrl %cl, %ebx
-; FALLBACK28-NEXT:    movl 120(%esp,%edx), %eax
-; FALLBACK28-NEXT:    leal (%eax,%eax), %ebp
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %ebp
-; FALLBACK28-NEXT:    orl %ebx, %ebp
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK28-NEXT:    movb %dl, %cl
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; FALLBACK28-NEXT:    shrl %cl, %ebx
-; FALLBACK28-NEXT:    addl %esi, %esi
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %esi
-; FALLBACK28-NEXT:    orl %ebx, %esi
-; FALLBACK28-NEXT:    movb %dl, %cl
-; FALLBACK28-NEXT:    shrl %cl, %eax
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK28-NEXT:    movl 124(%esp,%edx), %ebx
-; FALLBACK28-NEXT:    leal (%ebx,%ebx), %edx
-; FALLBACK28-NEXT:    movb %ch, %cl
-; FALLBACK28-NEXT:    shll %cl, %edx
-; FALLBACK28-NEXT:    orl %eax, %edx
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK28-NEXT:    sarl %cl, %ebx
-; FALLBACK28-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK28-NEXT:    movl %ebx, 60(%eax)
-; FALLBACK28-NEXT:    movl %edx, 56(%eax)
-; FALLBACK28-NEXT:    movl %esi, 48(%eax)
-; FALLBACK28-NEXT:    movl %ebp, 52(%eax)
-; FALLBACK28-NEXT:    movl %edi, 40(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, (%eax)
-; FALLBACK28-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK28-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK28-NEXT:    addl $204, %esp
-; FALLBACK28-NEXT:    popl %esi
-; FALLBACK28-NEXT:    popl %edi
-; FALLBACK28-NEXT:    popl %ebx
-; FALLBACK28-NEXT:    popl %ebp
-; FALLBACK28-NEXT:    vzeroupper
-; FALLBACK28-NEXT:    retl
-;
-; FALLBACK29-LABEL: ashr_64bytes:
-; FALLBACK29:       # %bb.0:
-; FALLBACK29-NEXT:    pushl %ebp
-; FALLBACK29-NEXT:    pushl %ebx
-; FALLBACK29-NEXT:    pushl %edi
-; FALLBACK29-NEXT:    pushl %esi
-; FALLBACK29-NEXT:    subl $188, %esp
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK29-NEXT:    vmovups (%eax), %ymm0
-; FALLBACK29-NEXT:    vmovups 32(%eax), %xmm1
-; FALLBACK29-NEXT:    movl 48(%eax), %edx
-; FALLBACK29-NEXT:    movl 52(%eax), %esi
-; FALLBACK29-NEXT:    movl 56(%eax), %edi
-; FALLBACK29-NEXT:    movl 60(%eax), %eax
-; FALLBACK29-NEXT:    movl (%ecx), %ecx
-; FALLBACK29-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    sarl $31, %eax
-; FALLBACK29-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK29-NEXT:    movl %ecx, %ebp
-; FALLBACK29-NEXT:    andl $60, %ebp
-; FALLBACK29-NEXT:    movl 56(%esp,%ebp), %edx
-; FALLBACK29-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shll $3, %ecx
-; FALLBACK29-NEXT:    andl $24, %ecx
-; FALLBACK29-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK29-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 64(%esp,%ebp), %edi
-; FALLBACK29-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, %esi
-; FALLBACK29-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK29-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 72(%esp,%ebp), %esi
-; FALLBACK29-NEXT:    movl 68(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, %edx
-; FALLBACK29-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK29-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 80(%esp,%ebp), %edi
-; FALLBACK29-NEXT:    movl 76(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, %edx
-; FALLBACK29-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK29-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 88(%esp,%ebp), %esi
-; FALLBACK29-NEXT:    movl 84(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, %edx
-; FALLBACK29-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK29-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl %esi, %edx
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK29-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 96(%esp,%ebp), %esi
-; FALLBACK29-NEXT:    movl 92(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, %edi
-; FALLBACK29-NEXT:    shrdl %cl, %esi, %edi
-; FALLBACK29-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK29-NEXT:    movl %edx, (%esp) # 4-byte Spill
-; FALLBACK29-NEXT:    movl 104(%esp,%ebp), %edx
-; FALLBACK29-NEXT:    movl 100(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    movl %eax, %edi
-; FALLBACK29-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK29-NEXT:    movl 48(%esp,%ebp), %ebx
-; FALLBACK29-NEXT:    movl 108(%esp,%ebp), %eax
-; FALLBACK29-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK29-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK29-NEXT:    movl %edx, 56(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK29-NEXT:    shrdl %cl, %edx, %ebx
-; FALLBACK29-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK29-NEXT:    sarl %cl, %eax
-; FALLBACK29-NEXT:    movl %eax, 60(%ebp)
-; FALLBACK29-NEXT:    movl %esi, 48(%ebp)
-; FALLBACK29-NEXT:    movl %edi, 52(%ebp)
-; FALLBACK29-NEXT:    movl (%esp), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 40(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK29-NEXT:    movl %ebx, (%ebp)
-; FALLBACK29-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK29-NEXT:    movl %eax, 4(%ebp)
-; FALLBACK29-NEXT:    addl $188, %esp
-; FALLBACK29-NEXT:    popl %esi
-; FALLBACK29-NEXT:    popl %edi
-; FALLBACK29-NEXT:    popl %ebx
-; FALLBACK29-NEXT:    popl %ebp
-; FALLBACK29-NEXT:    vzeroupper
-; FALLBACK29-NEXT:    retl
-;
-; FALLBACK30-LABEL: ashr_64bytes:
-; FALLBACK30:       # %bb.0:
-; FALLBACK30-NEXT:    pushl %ebp
-; FALLBACK30-NEXT:    pushl %ebx
-; FALLBACK30-NEXT:    pushl %edi
-; FALLBACK30-NEXT:    pushl %esi
-; FALLBACK30-NEXT:    subl $204, %esp
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK30-NEXT:    vmovups (%ecx), %ymm0
-; FALLBACK30-NEXT:    vmovups 32(%ecx), %xmm1
-; FALLBACK30-NEXT:    movl 48(%ecx), %edx
-; FALLBACK30-NEXT:    movl 52(%ecx), %esi
-; FALLBACK30-NEXT:    movl 56(%ecx), %edi
-; FALLBACK30-NEXT:    movl 60(%ecx), %ecx
-; FALLBACK30-NEXT:    movl (%eax), %eax
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    sarl $31, %ecx
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; FALLBACK30-NEXT:    movl %eax, %ecx
-; FALLBACK30-NEXT:    leal (,%eax,8), %edx
-; FALLBACK30-NEXT:    andl $24, %edx
-; FALLBACK30-NEXT:    movl %edx, %ebx
-; FALLBACK30-NEXT:    andl $60, %ecx
-; FALLBACK30-NEXT:    movl 68(%esp,%ecx), %esi
-; FALLBACK30-NEXT:    movl 72(%esp,%ecx), %edi
-; FALLBACK30-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrxl %ebx, %esi, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    notb %dl
-; FALLBACK30-NEXT:    leal (%edi,%edi), %ebp
-; FALLBACK30-NEXT:    shlxl %edx, %ebp, %eax
-; FALLBACK30-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrxl %ebx, 64(%esp,%ecx), %edi
-; FALLBACK30-NEXT:    addl %esi, %esi
-; FALLBACK30-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK30-NEXT:    orl %edi, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 80(%esp,%ecx), %esi
-; FALLBACK30-NEXT:    leal (%esi,%esi), %edi
-; FALLBACK30-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK30-NEXT:    movl 76(%esp,%ecx), %edi
-; FALLBACK30-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK30-NEXT:    orl %ebp, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK30-NEXT:    addl %edi, %edi
-; FALLBACK30-NEXT:    shlxl %edx, %edi, %edi
-; FALLBACK30-NEXT:    orl %eax, %edi
-; FALLBACK30-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 88(%esp,%ecx), %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK30-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK30-NEXT:    movl 84(%esp,%ecx), %edi
-; FALLBACK30-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK30-NEXT:    orl %ebp, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrxl %ebx, %esi, %esi
-; FALLBACK30-NEXT:    addl %edi, %edi
-; FALLBACK30-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK30-NEXT:    orl %esi, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 96(%esp,%ecx), %esi
-; FALLBACK30-NEXT:    leal (%esi,%esi), %edi
-; FALLBACK30-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK30-NEXT:    movl 92(%esp,%ecx), %edi
-; FALLBACK30-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK30-NEXT:    orl %ebp, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; FALLBACK30-NEXT:    addl %edi, %edi
-; FALLBACK30-NEXT:    shlxl %edx, %edi, %edi
-; FALLBACK30-NEXT:    orl %eax, %edi
-; FALLBACK30-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 104(%esp,%ecx), %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    leal (%eax,%eax), %edi
-; FALLBACK30-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK30-NEXT:    movl 100(%esp,%ecx), %edi
-; FALLBACK30-NEXT:    shrxl %ebx, %edi, %ebp
-; FALLBACK30-NEXT:    orl %ebp, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrxl %ebx, %esi, %esi
-; FALLBACK30-NEXT:    addl %edi, %edi
-; FALLBACK30-NEXT:    shlxl %edx, %edi, %eax
-; FALLBACK30-NEXT:    orl %esi, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl %ecx, %ebp
-; FALLBACK30-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 112(%esp,%ecx), %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    leal (%eax,%eax), %esi
-; FALLBACK30-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK30-NEXT:    movl 108(%esp,%ecx), %esi
-; FALLBACK30-NEXT:    shrxl %ebx, %esi, %edi
-; FALLBACK30-NEXT:    orl %edi, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; FALLBACK30-NEXT:    addl %esi, %esi
-; FALLBACK30-NEXT:    shlxl %edx, %esi, %eax
-; FALLBACK30-NEXT:    orl %ecx, %eax
-; FALLBACK30-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK30-NEXT:    movl 120(%esp,%ebp), %edi
-; FALLBACK30-NEXT:    leal (%edi,%edi), %ecx
-; FALLBACK30-NEXT:    shlxl %edx, %ecx, %esi
-; FALLBACK30-NEXT:    movl 116(%esp,%ebp), %eax
-; FALLBACK30-NEXT:    shrxl %ebx, %eax, %ebp
-; FALLBACK30-NEXT:    orl %ebp, %esi
-; FALLBACK30-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; FALLBACK30-NEXT:    addl %eax, %eax
-; FALLBACK30-NEXT:    shlxl %edx, %eax, %ecx
-; FALLBACK30-NEXT:    orl %ebp, %ecx
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK30-NEXT:    movl 124(%esp,%eax), %eax
-; FALLBACK30-NEXT:    leal (%eax,%eax), %ebp
-; FALLBACK30-NEXT:    shlxl %edx, %ebp, %edx
-; FALLBACK30-NEXT:    shrxl %ebx, %edi, %edi
-; FALLBACK30-NEXT:    orl %edi, %edx
-; FALLBACK30-NEXT:    sarxl %ebx, %eax, %edi
-; FALLBACK30-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK30-NEXT:    movl %edi, 60(%eax)
-; FALLBACK30-NEXT:    movl %edx, 56(%eax)
-; FALLBACK30-NEXT:    movl %ecx, 48(%eax)
-; FALLBACK30-NEXT:    movl %esi, 52(%eax)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK30-NEXT:    movl %ecx, 40(%eax)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK30-NEXT:    movl %ecx, 44(%eax)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK30-NEXT:    movl %ecx, 32(%eax)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK30-NEXT:    movl %ecx, 36(%eax)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK30-NEXT:    movl %ecx, 24(%eax)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK30-NEXT:    movl %ecx, 28(%eax)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK30-NEXT:    movl %ecx, 16(%eax)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK30-NEXT:    movl %ecx, 20(%eax)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK30-NEXT:    movl %ecx, 8(%eax)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK30-NEXT:    movl %ecx, 12(%eax)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK30-NEXT:    movl %ecx, (%eax)
-; FALLBACK30-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK30-NEXT:    movl %ecx, 4(%eax)
-; FALLBACK30-NEXT:    addl $204, %esp
-; FALLBACK30-NEXT:    popl %esi
-; FALLBACK30-NEXT:    popl %edi
-; FALLBACK30-NEXT:    popl %ebx
-; FALLBACK30-NEXT:    popl %ebp
-; FALLBACK30-NEXT:    vzeroupper
-; FALLBACK30-NEXT:    retl
-;
-; FALLBACK31-LABEL: ashr_64bytes:
-; FALLBACK31:       # %bb.0:
-; FALLBACK31-NEXT:    pushl %ebp
-; FALLBACK31-NEXT:    pushl %ebx
-; FALLBACK31-NEXT:    pushl %edi
-; FALLBACK31-NEXT:    pushl %esi
-; FALLBACK31-NEXT:    subl $188, %esp
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FALLBACK31-NEXT:    vmovups (%eax), %ymm0
-; FALLBACK31-NEXT:    vmovups 32(%eax), %xmm1
-; FALLBACK31-NEXT:    movl 48(%eax), %edx
-; FALLBACK31-NEXT:    movl 52(%eax), %esi
-; FALLBACK31-NEXT:    movl 56(%eax), %edi
-; FALLBACK31-NEXT:    movl 60(%eax), %eax
-; FALLBACK31-NEXT:    movl (%ecx), %ecx
-; FALLBACK31-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %edi, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %edx, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    sarl $31, %eax
-; FALLBACK31-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; FALLBACK31-NEXT:    movl %ecx, %ebp
-; FALLBACK31-NEXT:    andl $60, %ebp
-; FALLBACK31-NEXT:    movl 56(%esp,%ebp), %edx
-; FALLBACK31-NEXT:    movl 52(%esp,%ebp), %eax
-; FALLBACK31-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shll $3, %ecx
-; FALLBACK31-NEXT:    andl $24, %ecx
-; FALLBACK31-NEXT:    shrdl %cl, %edx, %eax
-; FALLBACK31-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 64(%esp,%ebp), %edi
-; FALLBACK31-NEXT:    movl 60(%esp,%ebp), %eax
-; FALLBACK31-NEXT:    movl %eax, %esi
-; FALLBACK31-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK31-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 72(%esp,%ebp), %esi
-; FALLBACK31-NEXT:    movl 68(%esp,%ebp), %eax
-; FALLBACK31-NEXT:    movl %eax, %edx
-; FALLBACK31-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK31-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 80(%esp,%ebp), %edi
-; FALLBACK31-NEXT:    movl 76(%esp,%ebp), %eax
-; FALLBACK31-NEXT:    movl %eax, %edx
-; FALLBACK31-NEXT:    shrdl %cl, %edi, %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %esi
-; FALLBACK31-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 88(%esp,%ebp), %ebx
-; FALLBACK31-NEXT:    movl 84(%esp,%ebp), %eax
-; FALLBACK31-NEXT:    movl %eax, %edx
-; FALLBACK31-NEXT:    shrdl %cl, %ebx, %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %edi
-; FALLBACK31-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    movl 96(%esp,%ebp), %esi
-; FALLBACK31-NEXT:    movl 92(%esp,%ebp), %eax
-; FALLBACK31-NEXT:    movl %eax, %edx
-; FALLBACK31-NEXT:    shrdl %cl, %esi, %edx
-; FALLBACK31-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %ebx
-; FALLBACK31-NEXT:    movl 104(%esp,%ebp), %eax
-; FALLBACK31-NEXT:    movl 100(%esp,%ebp), %edi
-; FALLBACK31-NEXT:    movl %edi, %edx
-; FALLBACK31-NEXT:    shrdl %cl, %eax, %edx
-; FALLBACK31-NEXT:    shrdl %cl, %edi, %esi
-; FALLBACK31-NEXT:    movl 48(%esp,%ebp), %edi
-; FALLBACK31-NEXT:    movl 108(%esp,%ebp), %ebp
-; FALLBACK31-NEXT:    movl %ebp, (%esp) # 4-byte Spill
-; FALLBACK31-NEXT:    shrdl %cl, %ebp, %eax
-; FALLBACK31-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; FALLBACK31-NEXT:    movl %eax, 56(%ebp)
-; FALLBACK31-NEXT:    movl %esi, 48(%ebp)
-; FALLBACK31-NEXT:    movl %edx, 52(%ebp)
-; FALLBACK31-NEXT:    movl %ebx, 40(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 44(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 32(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 36(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 24(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 28(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 16(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 20(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 8(%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; FALLBACK31-NEXT:    movl %eax, 12(%ebp)
-; FALLBACK31-NEXT:    sarxl %ecx, (%esp), %eax # 4-byte Folded Reload
-; FALLBACK31-NEXT:    # kill: def $cl killed $cl killed $ecx
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; FALLBACK31-NEXT:    shrdl %cl, %edx, %edi
-; FALLBACK31-NEXT:    movl %edi, (%ebp)
-; FALLBACK31-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; FALLBACK31-NEXT:    movl %ecx, 4(%ebp)
-; FALLBACK31-NEXT:    movl %eax, 60(%ebp)
-; FALLBACK31-NEXT:    addl $188, %esp
-; FALLBACK31-NEXT:    popl %esi
-; FALLBACK31-NEXT:    popl %edi
-; FALLBACK31-NEXT:    popl %ebx
-; FALLBACK31-NEXT:    popl %ebp
-; FALLBACK31-NEXT:    vzeroupper
-; FALLBACK31-NEXT:    retl
+; X64-NO-SHLD-NO-BMI2-SSE2-LABEL: ashr_64bytes:
+; X64-NO-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %r13
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %r12
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 32(%rdi), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 40(%rdi), %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 48(%rdi), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq 56(%rdi), %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl (%rsi), %edi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    sarq $63, %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (,%rdi,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    andl $56, %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    andl $56, %edi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -128(%rsp,%rdi), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -120(%rsp,%rdi), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -112(%rsp,%rdi), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leaq (%rbx,%rbx), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r11, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    addq %r8, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r10, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -104(%rsp,%rdi), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -96(%rsp,%rdi), %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leaq (%r14,%r14), %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r15, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    addq %r10, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %rbx, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -88(%rsp,%rdi), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rbx, %r12
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -80(%rsp,%rdi), %r13
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leaq (%r13,%r13), %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r12, %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    addq %rbx, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r14, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrq %cl, %r13
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq -72(%rsp,%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    leaq (%rdi,%rdi), %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    shlq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    orq %r13, %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    sarq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %rbx, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r15, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r11, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %r12
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %r13
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    popq %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE2-LABEL: ashr_64bytes:
+; X64-HAVE-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushq %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushq %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 24(%rdi), %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 32(%rdi), %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 40(%rdi), %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 48(%rdi), %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq 56(%rdi), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -112(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -128(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -120(%rsp,%rax), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r9, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %rdi, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -96(%rsp,%rax), %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -104(%rsp,%rax), %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r11, %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %r10, %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %r11, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -80(%rsp,%rax), %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -88(%rsp,%rax), %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r14, %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %r11, %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %r14, %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq -72(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %rax, %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdq %cl, %r9, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    sarq %cl, %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r11, 48(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rax, 56(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r10, 32(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r15, 40(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rdi, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rbx, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %rsi, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movq %r8, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popq %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popq %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popq %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE2-LABEL: ashr_64bytes:
+; X64-NO-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 32(%rdi), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 40(%rdi), %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 48(%rdi), %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 56(%rdi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%rsi), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    sarq $63, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (,%rax,8), %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $56, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $56, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -120(%rsp,%rax), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -112(%rsp,%rax), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rsi, %r8, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    notb %cl
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leaq (%r10,%r10), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r9, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rsi, -128(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addq %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r9, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -104(%rsp,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rsi, %r11, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -96(%rsp,%rax), %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leaq (%r14,%r14), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %rbx, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rsi, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addq %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %r10, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -88(%rsp,%rax), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rsi, %r10, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -80(%rsp,%rax), %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leaq (%r15,%r15), %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r12, %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %rbx, %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rsi, %r14, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addq %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %rbx, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxq %rsi, %r15, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -72(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leaq (%rax,%rax), %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxq %rcx, %r14, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orq %rbx, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    sarxq %rsi, %rax, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 56(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r10, 32(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r12, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addq $8, %rsp
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: ashr_64bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq (%rdi), %rcx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 8(%rdi), %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 16(%rdi), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 24(%rdi), %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 32(%rdi), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 40(%rdi), %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 48(%rdi), %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq 56(%rdi), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r14, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r11, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r10, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -112(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -128(%rsp,%rax), %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -120(%rsp,%rax), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r9, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %rdi, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -96(%rsp,%rax), %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -104(%rsp,%rax), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r11, %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %r10, %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %r11, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -80(%rsp,%rax), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -88(%rsp,%rax), %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r14, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %r11, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %r14, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq -72(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %rax, %r11
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    sarxq %rcx, %rax, %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdq %cl, %r9, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r11, 48(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r10, 32(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r15, 40(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rdi, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rbx, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rsi, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %r8, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movq %rax, 56(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-SSE4-LABEL: ashr_64bytes:
+; X64-NO-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %rbp
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r13
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushq %rax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 32(%rdi), %xmm2
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq 48(%rdi), %rax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq 56(%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl (%rsi), %edi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    sarq $63, %rcx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (,%rdi,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    andl $56, %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    andl $56, %edi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -128(%rsp,%rdi), %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -120(%rsp,%rdi), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leaq (%r9,%r9), %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r10, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -104(%rsp,%rdi), %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -96(%rsp,%rdi), %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leaq (%r12,%r12), %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %rbx, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -112(%rsp,%rdi), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rbx, %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    addq %r10, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r14, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -88(%rsp,%rdi), %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r14, %r13
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r13
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -80(%rsp,%rdi), %rbp
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leaq (%rbp,%rbp), %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r13, %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    addq %r14, %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r12, %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %rbp
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq -72(%rsp,%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    leaq (%rdi,%rdi), %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %rbp, %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    addq %rbx, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    orq %r9, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    sarq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %rbx, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r12, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r14, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r15, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r11, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    movq %r8, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    addq $8, %rsp
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %r13
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    popq %rbp
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-SSE4-LABEL: ashr_64bytes:
+; X64-HAVE-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushq %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 32(%rdi), %xmm2
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq 48(%rdi), %rcx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq 56(%rdi), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -96(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -104(%rsp,%rax), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %rdi, %rsi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -112(%rsp,%rax), %r10
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r10, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %r9, %r8
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -80(%rsp,%rax), %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -88(%rsp,%rax), %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r11, %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %r9, %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %r11, %rdi
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -72(%rsp,%rax), %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %r11, %r9
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -128(%rsp,%rax), %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq -120(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rax, %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %r10, %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdq %cl, %rax, %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    sarq %cl, %r11
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r15, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r9, 48(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r11, 56(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rdi, 32(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rbx, 40(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movq %r14, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popq %rbx
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popq %r14
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popq %r15
+; X64-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-SSE4-LABEL: ashr_64bytes:
+; X64-NO-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %r13
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 32(%rdi), %xmm2
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq 48(%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq 56(%rdi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl (%rsi), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    sarq $63, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (,%rax,8), %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $56, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $56, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, -128(%rsp,%rax), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    notb %cl
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -120(%rsp,%rax), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -112(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leaq (%r10,%r10), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r8, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -104(%rsp,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %r11, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -96(%rsp,%rax), %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leaq (%r14,%r14), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rbx, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %r9, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addq %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %rbx, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -88(%rsp,%rax), %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %rbx, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -80(%rsp,%rax), %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leaq (%r12,%r12), %r13
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r13, %r13
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r15, %r13
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %r14, %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addq %rbx, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %rbx, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r14, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %r12, %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -72(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leaq (%rax,%rax), %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r15, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r14, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxq %rsi, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addq %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxq %rcx, %r9, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orq %r10, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    sarxq %rsi, %rax, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, 56(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rcx, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r15, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rbx, 32(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r13, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r8, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %r12
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %r13
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: ashr_64bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%rdi), %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%rdi), %xmm1
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 32(%rdi), %xmm2
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq 48(%rdi), %rcx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq 56(%rdi), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -96(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -104(%rsp,%rax), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r9, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %rdi, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -112(%rsp,%rax), %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r10, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %r9, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -80(%rsp,%rax), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -88(%rsp,%rax), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r11, %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %r9, %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %r11, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -72(%rsp,%rax), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %r11, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -128(%rsp,%rax), %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq -120(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rax, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %r10, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    sarxq %rcx, %r11, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdq %cl, %rax, %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r15, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r9, 48(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rdi, 32(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rbx, 40(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r14, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movq %r10, 56(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    retq
+;
+; X64-NO-SHLD-NO-BMI2-AVX-LABEL: ashr_64bytes:
+; X64-NO-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    pushq %rbp
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    pushq %r15
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    pushq %r14
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    pushq %r13
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    pushq %r12
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    pushq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    pushq %rax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups 32(%rdi), %xmm1
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq 48(%rdi), %rax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq 56(%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl (%rsi), %edi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    sarq $63, %rcx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    leal (,%rdi,8), %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    andl $56, %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    andl $56, %edi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -128(%rsp,%rdi), %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -120(%rsp,%rdi), %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %esi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    notb %sil
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    leaq (%r9,%r9), %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r10, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -104(%rsp,%rdi), %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -96(%rsp,%rdi), %r12
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    leaq (%r12,%r12), %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %rbx, %r11
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -112(%rsp,%rdi), %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rbx, %r14
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    addq %r10, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r14, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -88(%rsp,%rdi), %r14
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r14, %r13
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r13
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -80(%rsp,%rdi), %rbp
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    leaq (%rbp,%rbp), %r15
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r15
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r13, %r15
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    addq %r14, %r14
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r14
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r12, %r14
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %rbp
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq -72(%rsp,%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    leaq (%rdi,%rdi), %r12
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %rbp, %r12
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shrq %cl, %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    addq %rbx, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    shlq %cl, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    orq %r9, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    sarq %cl, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %rbx, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r12, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r14, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r15, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r11, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    movq %r8, (%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    addq $8, %rsp
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    popq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    popq %r12
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    popq %r13
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    popq %r14
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    popq %r15
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    popq %rbp
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    vzeroupper
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT:    retq
+;
+; X64-HAVE-SHLD-NO-BMI2-AVX-LABEL: ashr_64bytes:
+; X64-HAVE-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushq %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushq %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups 32(%rdi), %xmm1
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq 48(%rdi), %rcx
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq 56(%rdi), %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -96(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -104(%rsp,%rax), %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r9, %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %rdi, %rsi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -112(%rsp,%rax), %r10
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r10, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %r9, %r8
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -80(%rsp,%rax), %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -88(%rsp,%rax), %r11
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r11, %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %r9, %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %r11, %rdi
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -72(%rsp,%rax), %r11
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %r11, %r9
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -128(%rsp,%rax), %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq -120(%rsp,%rax), %rax
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rax, %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %r10, %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdq %cl, %rax, %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    sarq %cl, %r11
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r15, 8(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r9, 48(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r11, 56(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rdi, 32(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rbx, 40(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movq %r14, (%rdx)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popq %rbx
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popq %r14
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popq %r15
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vzeroupper
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT:    retq
+;
+; X64-NO-SHLD-HAVE-BMI2-AVX-LABEL: ashr_64bytes:
+; X64-NO-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushq %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushq %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushq %r13
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushq %r12
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushq %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups 32(%rdi), %xmm1
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq 48(%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq 56(%rdi), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl (%rsi), %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    sarq $63, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (,%rax,8), %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    andl $56, %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    andl $56, %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rsi, -128(%rsp,%rax), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    notb %cl
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -120(%rsp,%rax), %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -112(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leaq (%r10,%r10), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %r8, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -104(%rsp,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rsi, %r11, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -96(%rsp,%rax), %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leaq (%r14,%r14), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %rbx, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rsi, %r9, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addq %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %rbx, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -88(%rsp,%rax), %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rsi, %rbx, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -80(%rsp,%rax), %r12
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leaq (%r12,%r12), %r13
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %r13, %r13
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %r15, %r13
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rsi, %r14, %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addq %rbx, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %rbx, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %r14, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rsi, %r12, %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq -72(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leaq (%rax,%rax), %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %r15, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %r14, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxq %rsi, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addq %r9, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxq %rcx, %r9, %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orq %r10, %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    sarxq %rsi, %rax, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, 56(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rcx, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r15, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rbx, 32(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r13, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r11, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r8, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popq %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popq %r12
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popq %r13
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popq %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popq %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vzeroupper
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT:    retq
+;
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-LABEL: ashr_64bytes:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%rdi), %ymm0
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups 32(%rdi), %xmm1
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq 48(%rdi), %rcx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq 56(%rdi), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl (%rsi), %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    sarq $63, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    leal (,%rax,8), %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    andl $56, %ecx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    andl $56, %eax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -96(%rsp,%rax), %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -104(%rsp,%rax), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r9, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %rdi, %rsi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -112(%rsp,%rax), %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r10, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %r9, %r8
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -80(%rsp,%rax), %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -88(%rsp,%rax), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r11, %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %r9, %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %r11, %rdi
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -72(%rsp,%rax), %r11
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %r11, %r9
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -128(%rsp,%rax), %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq -120(%rsp,%rax), %rax
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rax, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %r10, %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    sarxq %rcx, %r11, %r10
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdq %cl, %rax, %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r15, 8(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r9, 48(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rdi, 32(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rbx, 40(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r8, 16(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %rsi, 24(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r14, (%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movq %r10, 56(%rdx)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popq %rbx
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popq %r14
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popq %r15
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vzeroupper
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    retq
+;
+; X86-NO-SHLD-NO-BMI2-SSE2-LABEL: ashr_64bytes:
+; X86-NO-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    subl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl (%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 32(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 36(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 40(%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 44(%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 48(%ecx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 52(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 56(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 60(%ecx), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl (%ebp), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    sarl $31, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    andl $60, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 68(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll $3, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    andl $24, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 72(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    notb %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 64(%esp,%esi), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %bl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 76(%esp,%esi), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %bl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 80(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%edi,%edi), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %bl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %ebp, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %eax, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 84(%esp,%esi), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %bl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 88(%esp,%esi), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%esi,%esi), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %eax, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %bl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 92(%esp,%edx), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %bl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 96(%esp,%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%edi,%edi), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %bl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %ebp, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %esi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 100(%esp,%edx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %bl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 104(%esp,%edx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%esi,%esi), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %eax, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 108(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 112(%esp,%ebp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%ecx,%ecx), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %eax, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %bl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 116(%esp,%edx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 120(%esp,%edx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%edx,%edx), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %eax, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %bl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %bl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl 124(%esp,%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    leal (%ebx,%ebx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    orl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    sarl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, 60(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, 56(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, 48(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, 52(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, 40(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 44(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 32(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 36(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, (%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-SSE2-LABEL: ashr_64bytes:
+; X86-HAVE-SHLD-NO-BMI2-SSE2:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%eax), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 4(%eax), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 8(%eax), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 12(%eax), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 16(%eax), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 20(%eax), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 24(%eax), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 28(%eax), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 32(%eax), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 36(%eax), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 40(%eax), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 44(%eax), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 48(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 52(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 56(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 60(%eax), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%ecx), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%esp), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    sarl $31, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 56(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 64(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 72(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 68(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 80(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 76(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 88(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 84(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 96(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 92(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %esi, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 104(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 100(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 48(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl 108(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edx, 56(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    sarl %cl, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 60(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %esi, 48(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %edi, 52(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 40(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %ebx, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    movl %eax, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE2-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-SSE2-LABEL: ashr_64bytes:
+; X86-NO-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    subl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 28(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 32(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 36(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 40(%eax), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 44(%eax), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 48(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 52(%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 56(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 60(%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    sarl $31, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (,%eax,8), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $24, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $60, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 68(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 72(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    notb %dl
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%edi,%edi), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, 64(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 80(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%esi,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 76(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 88(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%eax,%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 84(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 96(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%esi,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 92(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 104(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%eax,%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 100(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 112(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%eax,%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 108(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 120(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%edi,%edi), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %ecx, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 116(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %ebp, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 124(%esp,%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    leal (%eax,%eax), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shlxl %edx, %ebp, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    shrxl %ebx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    orl %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    sarxl %ebx, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, 60(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, 56(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 48(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, 52(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 40(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 44(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 32(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 36(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 28(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, (%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    addl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: ashr_64bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%eax), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 4(%eax), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 8(%eax), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 12(%eax), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 16(%eax), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 20(%eax), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 24(%eax), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 28(%eax), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 32(%eax), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 36(%eax), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 40(%eax), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 44(%eax), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 48(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 52(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 56(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 60(%eax), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%ecx), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%esp), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    sarl $31, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 56(%esp,%ebp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 64(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 72(%esp,%ebp), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 68(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 80(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 76(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 88(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 84(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %ebx, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 96(%esp,%ebp), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 92(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 104(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 100(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 48(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl 108(%esp,%ebp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %ebp, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 56(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %esi, 48(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edx, 52(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ebx, 40(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    sarxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %edi, (%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %ecx, 4(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    movl %eax, 60(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE2-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-SSE4-LABEL: ashr_64bytes:
+; X86-NO-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    subl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movups 32(%ecx), %xmm2
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 48(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 52(%ecx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 56(%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 60(%ecx), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    sarl $31, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    andl $60, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 68(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll $3, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    andl $24, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 72(%esp,%esi), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%ecx,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %ch
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    notb %ch
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 64(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %edi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 76(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 80(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%edi,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 84(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 88(%esp,%esi), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 92(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 96(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%edi,%edi), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 100(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 104(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%edx,%edx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 108(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 112(%esp,%esi), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%ecx,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 116(%esp,%esi), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 120(%esp,%edx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%eax,%eax), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl 124(%esp,%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    leal (%ebx,%ebx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    orl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    sarl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, 60(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, 56(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, 48(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebp, 52(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, 40(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 44(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 32(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 36(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, (%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-SSE4-LABEL: ashr_64bytes:
+; X86-HAVE-SHLD-NO-BMI2-SSE4:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups (%eax), %xmm0
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 16(%eax), %xmm1
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movups 32(%eax), %xmm2
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 48(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 52(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 56(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 60(%eax), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl (%ecx), %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    sarl $31, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 56(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %edx, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 64(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 72(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 68(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 80(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 76(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 88(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 84(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 96(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 92(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %esi, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 104(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 100(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 48(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl 108(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edx, 56(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    shrdl %cl, %edx, %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    sarl %cl, %eax
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 60(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %esi, 48(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %edi, 52(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 40(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %ebx, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    movl %eax, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-SSE4-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-SSE4-LABEL: ashr_64bytes:
+; X86-NO-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    subl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%ecx), %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%ecx), %xmm1
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 32(%ecx), %xmm2
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 48(%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 52(%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 56(%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 60(%ecx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl (%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    sarl $31, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (,%eax,8), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $24, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $60, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 68(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 72(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    notb %dl
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%edi,%edi), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, 64(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 80(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%esi,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 76(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 88(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%eax,%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 84(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 96(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%esi,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 92(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 104(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%eax,%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 100(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 112(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%eax,%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 108(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 120(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%edi,%edi), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %ecx, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 116(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %ebp, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 124(%esp,%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    leal (%eax,%eax), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shlxl %edx, %ebp, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    shrxl %ebx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    orl %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    sarxl %ebx, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, 60(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, 56(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 48(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, 52(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 40(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 44(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 32(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 36(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 28(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, (%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    addl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: ashr_64bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups (%eax), %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 16(%eax), %xmm1
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movups 32(%eax), %xmm2
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 48(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 52(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 56(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 60(%eax), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl (%ecx), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    sarl $31, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 56(%esp,%ebp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %edx, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 64(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 72(%esp,%ebp), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 68(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 80(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 76(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 88(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 84(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %ebx, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 96(%esp,%ebp), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 92(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 104(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 100(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 48(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl 108(%esp,%ebp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebp, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %ebp, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 56(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %esi, 48(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edx, 52(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ebx, 40(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    sarxl %ecx, (%esp), %eax # 4-byte Folded Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %edi, (%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %ecx, 4(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    movl %eax, 60(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-SSE4-NEXT:    retl
+;
+; X86-NO-SHLD-NO-BMI2-AVX-LABEL: ashr_64bytes:
+; X86-NO-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    pushl %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    subl $204, %esp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%ecx), %ymm0
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups 32(%ecx), %xmm1
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 48(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 52(%ecx), %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 56(%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 60(%ecx), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    sarl $31, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    andl $60, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 68(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll $3, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    andl $24, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 72(%esp,%esi), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    leal (%ecx,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %al, %ch
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    notb %ch
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 64(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %edi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 76(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 80(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    leal (%edi,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %edx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 84(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 88(%esp,%esi), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 92(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 96(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    leal (%edi,%edi), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 100(%esp,%esi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 104(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    leal (%edx,%edx), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %ebx, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %edi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 108(%esp,%esi), %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 112(%esp,%esi), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    leal (%ecx,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 116(%esp,%esi), %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 120(%esp,%edx), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    leal (%eax,%eax), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl 124(%esp,%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    leal (%ebx,%ebx), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    orl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    sarl %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, 60(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, 56(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, 48(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ebp, 52(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, 40(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 44(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 32(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 36(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, (%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    popl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    vzeroupper
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT:    retl
+;
+; X86-HAVE-SHLD-NO-BMI2-AVX-LABEL: ashr_64bytes:
+; X86-HAVE-SHLD-NO-BMI2-AVX:       # %bb.0:
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    pushl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups (%eax), %ymm0
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups 32(%eax), %xmm1
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 48(%eax), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 52(%eax), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 56(%eax), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 60(%eax), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl (%ecx), %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    sarl $31, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 56(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %edx, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 64(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 72(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 68(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 80(%esp,%ebp), %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 76(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 88(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 84(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 96(%esp,%ebp), %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 92(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %esi, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 104(%esp,%ebp), %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 100(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 48(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl 108(%esp,%ebp), %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edx, 56(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    shrdl %cl, %edx, %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    sarl %cl, %eax
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 60(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %esi, 48(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %edi, 52(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 40(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %ebx, (%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    movl %eax, 4(%ebp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %esi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %edi
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %ebx
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    popl %ebp
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    vzeroupper
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT:    retl
+;
+; X86-NO-SHLD-HAVE-BMI2-AVX-LABEL: ashr_64bytes:
+; X86-NO-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    subl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%ecx), %ymm0
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups 32(%ecx), %xmm1
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 48(%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 52(%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 56(%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 60(%ecx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl (%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    sarl $31, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (,%eax,8), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    andl $24, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    andl $60, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 68(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 72(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ebx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    notb %dl
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (%edi,%edi), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %edx, %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ebx, 64(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 80(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (%esi,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 76(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %edx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 88(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (%eax,%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 84(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 96(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (%esi,%esi), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 92(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %edx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 104(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (%eax,%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 100(%esp,%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ebx, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ebx, %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %edx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 112(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (%eax,%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 108(%esp,%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ebx, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl %esi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %edx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 120(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (%edi,%edi), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %edx, %ecx, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 116(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ebx, %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %ebp, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %edx, %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %ebp, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl 124(%esp,%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    leal (%eax,%eax), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shlxl %edx, %ebp, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    shrxl %ebx, %edi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    orl %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    sarxl %ebx, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, 60(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, 56(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 48(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, 52(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 40(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 44(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 32(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 36(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 24(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 28(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 16(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 20(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 8(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 12(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, (%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 4(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    addl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    vzeroupper
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT:    retl
+;
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-LABEL: ashr_64bytes:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX:       # %bb.0:
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    pushl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    subl $188, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups (%eax), %ymm0
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups 32(%eax), %xmm1
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 48(%eax), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 52(%eax), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 56(%eax), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 60(%eax), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl (%ecx), %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    sarl $31, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    andl $60, %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 56(%esp,%ebp), %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 52(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shll $3, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    andl $24, %ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %edx, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 64(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 60(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 72(%esp,%ebp), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 68(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 80(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 76(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %edi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %eax, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 88(%esp,%ebp), %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 84(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %ebx, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %eax, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 96(%esp,%ebp), %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 92(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %esi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %eax, %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 104(%esp,%ebp), %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 100(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %eax, %edx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %edi, %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 48(%esp,%ebp), %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl 108(%esp,%ebp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebp, (%esp) # 4-byte Spill
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %ebp, %eax
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 56(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %esi, 48(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edx, 52(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ebx, 40(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 44(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 32(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 36(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 24(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 28(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 16(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 20(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 8(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 12(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    sarxl %ecx, (%esp), %eax # 4-byte Folded Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    shrdl %cl, %edx, %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %edi, (%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %ecx, 4(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    movl %eax, 60(%ebp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    addl $188, %esp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %esi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %edi
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebx
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    popl %ebp
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    vzeroupper
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT:    retl
   %src = load i512, ptr %src.ptr, align 1
   %byteOff = load i512, ptr %byteOff.ptr, align 1
   %bitOff = shl i512 %byteOff, 3
diff --git a/llvm/test/Transforms/InferAlignment/ptrtoint.ll b/llvm/test/Transforms/InferAlignment/ptrtoint.ll
new file mode 100644
index 0000000000000..3eb1f95d9c026
--- /dev/null
+++ b/llvm/test/Transforms/InferAlignment/ptrtoint.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=infer-alignment -S | FileCheck %s
+
+define i64 @base(ptr %ptr) {
+; CHECK-LABEL: define i64 @base(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[PTR]], align 4
+; CHECK-NEXT:    [[V3:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[V5:%.*]] = and i64 [[V3]], 2
+; CHECK-NEXT:    ret i64 0
+;
+  %v1 = load i32, ptr %ptr, align 4
+  %v3 = ptrtoint ptr %ptr to i64
+  %v5 = and i64 %v3, 2
+  ret i64 %v5
+}
+
+define i64 @best_alignment(ptr %ptr) {
+; CHECK-LABEL: define i64 @best_alignment(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[PTR]], align 8
+; CHECK-NEXT:    [[V2:%.*]] = load i32, ptr [[PTR]], align 16
+; CHECK-NEXT:    [[V3:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[V5:%.*]] = and i64 [[V3]], 15
+; CHECK-NEXT:    ret i64 0
+;
+  %v1 = load i32, ptr %ptr, align 8
+  %v2 = load i32, ptr %ptr, align 16
+  %v3 = ptrtoint ptr %ptr to i64
+  %v5 = and i64 %v3, 15
+  ret i64 %v5
+}
+
+declare void @func()
+
+define i64 @negative_test(ptr %ptr) {
+; CHECK-LABEL: define i64 @negative_test(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:    [[V3:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[V5:%.*]] = and i64 [[V3]], 2
+; CHECK-NEXT:    call void @func()
+; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[PTR]], align 4
+; CHECK-NEXT:    ret i64 [[V5]]
+;
+  %v3 = ptrtoint ptr %ptr to i64
+  %v5 = and i64 %v3, 2
+  call void @func()
+  %v1 = load i32, ptr %ptr, align 4
+  ret i64 %v5
+}
+
+define i64 @ptrtoaddr(ptr %ptr) {
+; CHECK-LABEL: define i64 @ptrtoaddr(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:    [[V3:%.*]] = ptrtoaddr ptr [[PTR]] to i64
+; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[PTR]], align 4
+; CHECK-NEXT:    [[V5:%.*]] = and i64 [[V3]], 2
+; CHECK-NEXT:    ret i64 0
+;
+  %v3 = ptrtoaddr ptr %ptr to i64
+  %v1 = load i32, ptr %ptr, align 4
+  %v5 = and i64 %v3, 2
+  ret i64 %v5
+}
diff --git a/mlir/include/mlir/Target/LLVMIR/LLVMImportInterface.h b/mlir/include/mlir/Target/LLVMIR/LLVMImportInterface.h
index 6a42627e17e60..0e50fac7e85dd 100644
--- a/mlir/include/mlir/Target/LLVMIR/LLVMImportInterface.h
+++ b/mlir/include/mlir/Target/LLVMIR/LLVMImportInterface.h
@@ -84,10 +84,14 @@ class LLVMImportDialectInterface
 
   /// Hook for derived dialect interfaces to publish the supported metadata
   /// kinds. As every metadata kind has a unique integer identifier, the
-  /// function returns the list of supported metadata identifiers. `ctx` can be
-  /// used to obtain IDs of metadata kinds that do not have a fixed static one.
-  virtual ArrayRef<unsigned>
-  getSupportedMetadata(llvm::LLVMContext &ctx) const {
+  /// function returns the list of supported metadata identifiers. The
+  /// `llvmContext` parameter is used to obtain identifiers for metadata kinds
+  /// that do not have a fixed static identifier. Since different LLVM contexts
+  /// can assign different identifiers to these non-static metadata kinds, the
+  /// function must recompute the list of supported metadata identifiers on each
+  /// call.
+  virtual SmallVector<unsigned>
+  getSupportedMetadata(llvm::LLVMContext &llvmContext) const {
     return {};
   }
 };
diff --git a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
index bc2f2f2a3fe93..d4b4c46d3c758 100644
--- a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
+++ b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
@@ -107,16 +107,16 @@ struct ConvertIndexCeilDivSPattern final : OpConversionPattern<CeilDivSOp> {
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     Value n = adaptor.getLhs();
-    Type n_type = n.getType();
+    Type nType = n.getType();
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = spirv::ConstantOp::create(rewriter, loc, n_type,
-                                           IntegerAttr::get(n_type, 0));
-    Value posOne = spirv::ConstantOp::create(rewriter, loc, n_type,
-                                             IntegerAttr::get(n_type, 1));
-    Value negOne = spirv::ConstantOp::create(rewriter, loc, n_type,
-                                             IntegerAttr::get(n_type, -1));
+    Value zero = spirv::ConstantOp::create(rewriter, loc, nType,
+                                           IntegerAttr::get(nType, 0));
+    Value posOne = spirv::ConstantOp::create(rewriter, loc, nType,
+                                             IntegerAttr::get(nType, 1));
+    Value negOne = spirv::ConstantOp::create(rewriter, loc, nType,
+                                             IntegerAttr::get(nType, -1));
 
     // Compute `x`.
     Value mPos = spirv::SGreaterThanOp::create(rewriter, loc, m, zero);
@@ -157,14 +157,14 @@ struct ConvertIndexCeilDivUPattern final : OpConversionPattern<CeilDivUOp> {
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     Value n = adaptor.getLhs();
-    Type n_type = n.getType();
+    Type nType = n.getType();
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = spirv::ConstantOp::create(rewriter, loc, n_type,
-                                           IntegerAttr::get(n_type, 0));
-    Value one = spirv::ConstantOp::create(rewriter, loc, n_type,
-                                          IntegerAttr::get(n_type, 1));
+    Value zero = spirv::ConstantOp::create(rewriter, loc, nType,
+                                           IntegerAttr::get(nType, 0));
+    Value one = spirv::ConstantOp::create(rewriter, loc, nType,
+                                          IntegerAttr::get(nType, 1));
 
     // Compute the non-zero result.
     Value minusOne = spirv::ISubOp::create(rewriter, loc, n, one);
@@ -193,16 +193,16 @@ struct ConvertIndexFloorDivSPattern final : OpConversionPattern<FloorDivSOp> {
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
     Value n = adaptor.getLhs();
-    Type n_type = n.getType();
+    Type nType = n.getType();
     Value m = adaptor.getRhs();
 
     // Define the constants
-    Value zero = spirv::ConstantOp::create(rewriter, loc, n_type,
-                                           IntegerAttr::get(n_type, 0));
-    Value posOne = spirv::ConstantOp::create(rewriter, loc, n_type,
-                                             IntegerAttr::get(n_type, 1));
-    Value negOne = spirv::ConstantOp::create(rewriter, loc, n_type,
-                                             IntegerAttr::get(n_type, -1));
+    Value zero = spirv::ConstantOp::create(rewriter, loc, nType,
+                                           IntegerAttr::get(nType, 0));
+    Value posOne = spirv::ConstantOp::create(rewriter, loc, nType,
+                                             IntegerAttr::get(nType, 1));
+    Value negOne = spirv::ConstantOp::create(rewriter, loc, nType,
+                                             IntegerAttr::get(nType, -1));
 
     // Compute `x`.
     Value mNeg = spirv::SLessThanOp::create(rewriter, loc, m, zero);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp
index 81c9da1d98c40..2d4a18cc4b145 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp
@@ -80,8 +80,9 @@ static LogicalResult convertIntrinsicImpl(OpBuilder &odsBuilder,
 
 /// Returns the list of LLVM IR metadata kinds that are convertible to MLIR LLVM
 /// dialect attributes.
-static ArrayRef<unsigned> getSupportedMetadataImpl(llvm::LLVMContext &context) {
-  static const SmallVector<unsigned> convertibleMetadata = {
+static SmallVector<unsigned>
+getSupportedMetadataImpl(llvm::LLVMContext &llvmContext) {
+  SmallVector<unsigned> convertibleMetadata = {
       llvm::LLVMContext::MD_prof,
       llvm::LLVMContext::MD_tbaa,
       llvm::LLVMContext::MD_access_group,
@@ -91,10 +92,10 @@ static ArrayRef<unsigned> getSupportedMetadataImpl(llvm::LLVMContext &context) {
       llvm::LLVMContext::MD_dereferenceable,
       llvm::LLVMContext::MD_dereferenceable_or_null,
       llvm::LLVMContext::MD_mmra,
-      context.getMDKindID(vecTypeHintMDName),
-      context.getMDKindID(workGroupSizeHintMDName),
-      context.getMDKindID(reqdWorkGroupSizeMDName),
-      context.getMDKindID(intelReqdSubGroupSizeMDName)};
+      llvmContext.getMDKindID(vecTypeHintMDName),
+      llvmContext.getMDKindID(workGroupSizeHintMDName),
+      llvmContext.getMDKindID(reqdWorkGroupSizeMDName),
+      llvmContext.getMDKindID(intelReqdSubGroupSizeMDName)};
   return convertibleMetadata;
 }
 
@@ -505,9 +506,9 @@ class LLVMDialectLLVMIRImportInterface : public LLVMImportDialectInterface {
 
   /// Returns the list of LLVM IR metadata kinds that are convertible to MLIR
   /// LLVM dialect attributes.
-  ArrayRef<unsigned>
-  getSupportedMetadata(llvm::LLVMContext &context) const final {
-    return getSupportedMetadataImpl(context);
+  SmallVector<unsigned>
+  getSupportedMetadata(llvm::LLVMContext &llvmContext) const final {
+    return getSupportedMetadataImpl(llvmContext);
   }
 };
 } // namespace
diff --git a/mlir/test/Examples/NVGPU/Ch0.py b/mlir/test/Examples/NVGPU/Ch0.py
index 8f60088178d11..e09720a0f3b75 100644
--- a/mlir/test/Examples/NVGPU/Ch0.py
+++ b/mlir/test/Examples/NVGPU/Ch0.py
@@ -1,5 +1,9 @@
 # RUN: env SUPPORT_LIB=%mlir_cuda_runtime \
-# RUN:   %PYTHON %s | FileCheck %s
+# RUN: sh -c 'if [[ "%mlir_run_cuda_sm90_tests" == "1" ]]; \
+# RUN: then %PYTHON %s | FileCheck %s; \
+# RUN: else export MLIR_NVDSL_PRINT_IR=1; \
+# RUN: %PYTHON %s | FileCheck %s --check-prefix=DUMPIR; fi'
+
 
 # ===----------------------------------------------------------------------===//
 #  Chapter 0 : Hello World
@@ -33,7 +37,7 @@ def kernel():
         # + operator generates arith.addi
         myValue = alpha + tidx
         # Print from a GPU thread
-        gpu.printf("GPU thread %llu has %llu\n", [tidx, myValue])
+        gpu.printf("GPU thread %llu has %llu\n", tidx, myValue)
 
     # 3. Call the GPU kernel
     kernel()
@@ -43,8 +47,24 @@ def kernel():
 # 4. The `mlir_func` decorator JIT compiles the IR and executes the MLIR function.
 main(alpha)
 
-
 # CHECK: GPU thread 0 has 100
 # CHECK: GPU thread 1 has 101
 # CHECK: GPU thread 2 has 102
 # CHECK: GPU thread 3 has 103
+
+# DUMPIR:   func.func @main(%arg0: index) attributes {llvm.emit_c_interface} {
+# DUMPIR:     %[[C0_I32:.*]] = arith.constant 0 : i32
+# DUMPIR:     %[[C1:.*]] = arith.constant 1 : index
+# DUMPIR:     %[[C1_0:.*]] = arith.constant 1 : index
+# DUMPIR:     %[[C1_1:.*]] = arith.constant 1 : index
+# DUMPIR:     %[[C4:.*]] = arith.constant 4 : index
+# DUMPIR:     %[[C1_2:.*]] = arith.constant 1 : index
+# DUMPIR:     %[[C1_3:.*]] = arith.constant 1 : index
+# DUMPIR:     gpu.launch blocks(%arg1, %arg2, %arg3) in (%arg7 = %[[C1]], %arg8 = %[[C1_0]], %arg9 = %[[C1_1]]) threads(%arg4, %arg5, %arg6) in (%arg10 = %[[C4]], %arg11 = %[[C1_2]], %arg12 = %[[C1_3]]) dynamic_shared_memory_size %[[C0_I32]] {
+# DUMPIR:       %[[TIDX:.*]] = gpu.thread_id  x
+# DUMPIR:       %[[MYVAL:.*]] = arith.addi %arg0, %[[TIDX]] : index
+# DUMPIR:       gpu.printf "GPU thread %llu has %llu\0A", %[[TIDX]], %[[MYVAL]] : index, index
+# DUMPIR:       gpu.terminator
+# DUMPIR:     }
+# DUMPIR:     return
+# DUMPIR:   }
diff --git a/mlir/test/Examples/NVGPU/Ch1.py b/mlir/test/Examples/NVGPU/Ch1.py
index cfb48d56f8d49..6e44e4d04fa06 100644
--- a/mlir/test/Examples/NVGPU/Ch1.py
+++ b/mlir/test/Examples/NVGPU/Ch1.py
@@ -1,5 +1,9 @@
 # RUN: env SUPPORT_LIB=%mlir_cuda_runtime \
-# RUN:   %PYTHON %s | FileCheck %s
+# RUN: sh -c 'if [[ "%mlir_run_cuda_sm90_tests" == "1" ]]; \
+# RUN: then %PYTHON %s | FileCheck %s; \
+# RUN: else export MLIR_NVDSL_PRINT_IR=1; \
+# RUN: %PYTHON %s | FileCheck %s --check-prefix=DUMPIR; fi'
+
 
 # ===----------------------------------------------------------------------===//
 #  Chapter 1 : 2D Saxpy
@@ -24,12 +28,12 @@
 def saxpy(x, y, alpha):
     # 1. Use MLIR GPU dialect to allocate and copy memory
     token_ty = gpu.AsyncTokenType.get()
-    t1 = gpu.wait(token_ty, [])
+    t1 = gpu.wait([])
     x_dev, t2 = gpu.alloc(x.type, token_ty, [t1], [], [])
     y_dev, t3 = gpu.alloc(y.type, token_ty, [t2], [], [])
     t4 = gpu.memcpy(token_ty, [t3], x_dev, x)
     t5 = gpu.memcpy(token_ty, [t4], y_dev, y)
-    t6 = gpu.wait(token_ty, [t5])
+    t6 = gpu.wait([t5])
 
     # 2. Compute 2D SAXPY kernel
     @NVDSL.mlir_gpu_launch(grid=(M, 1, 1), block=(N, 1, 1))
@@ -47,7 +51,7 @@ def saxpy_kernel():
     saxpy_kernel()
 
     t7 = gpu.memcpy(token_ty, [t6], y, y_dev)
-    gpu.wait(token_ty, [t7])
+    gpu.wait([t7])
 
 
 # 3. Pass numpy arrays to MLIR
@@ -56,11 +60,32 @@ def saxpy_kernel():
 alpha = 2.0
 x = np.random.randn(M, N).astype(np.float32)
 y = np.ones((M, N), np.float32)
+
 saxpy(x, y, alpha)
 
-#  4. Verify MLIR with reference computation
-ref = np.ones((M, N), np.float32)
-ref += x * alpha
-np.testing.assert_allclose(y, ref, rtol=5e-03, atol=1e-01)
-print("PASS")
+if os.getenv("MLIR_NVDSL_PRINT_IR") != "1":
+    # 4. Verify MLIR with reference computation
+    ref = np.ones((M, N), np.float32)
+    ref += x * alpha
+    np.testing.assert_allclose(y, ref, rtol=5e-03, atol=1e-01)
+    print("PASS")
 # CHECK-NOT: Mismatched elements
+# CHECK: PASS
+
+# DUMPIR:   func.func @saxpy(%[[ARG0:.*]]: memref<256x32xf32>, %[[ARG1:.*]]: memref<256x32xf32>, %[[ARG2:.*]]: f32) attributes {llvm.emit_c_interface} {
+# DUMPIR:     %[[WAIT0:.*]] = gpu.wait async
+# DUMPIR:     %[[MEMREF:.*]], %[[ASYNC0:.*]] = gpu.alloc async [%[[WAIT0]]] () : memref<256x32xf32>
+# DUMPIR:     %[[MEMREF0:.*]], %[[ASYNC1:.*]] = gpu.alloc async [%[[ASYNC0]]] () : memref<256x32xf32>
+# DUMPIR:     %[[MEMCPY1:.*]] = gpu.memcpy async [%[[ASYNC1]]] %[[MEMREF]], %[[ARG0]] : memref<256x32xf32>, memref<256x32xf32>
+# DUMPIR:     %[[MEMCPY2:.*]] = gpu.memcpy async [%[[MEMCPY1]]] %[[MEMREF0]], %[[ARG1]] : memref<256x32xf32>, memref<256x32xf32>
+# DUMPIR:     %[[WAIT1:.*]] = gpu.wait async [%[[MEMCPY2]]]
+# DUMPIR:       %[[LD0:.*]] = memref.load %[[MEMREF]][%{{.*}}, %{{.*}}] : memref<256x32xf32>
+# DUMPIR:       %[[LD1:.*]] = memref.load %[[MEMREF0]][%{{.*}}, %{{.*}}] : memref<256x32xf32>
+# DUMPIR:       %[[MUL:.*]] = arith.mulf %[[LD0]], %[[ARG2]] : f32
+# DUMPIR:       %[[ADD:.*]] = arith.addf %[[LD1]], %[[MUL]] : f32
+# DUMPIR:       memref.store %[[ADD]], %[[MEMREF0]][%{{.*}}, %{{.*}}] : memref<256x32xf32>
+# DUMPIR:       gpu.terminator
+# DUMPIR:     %[[MEMCPY3:.*]] = gpu.memcpy async [%[[WAIT1]]] %[[ARG1]], %[[MEMREF0]] : memref<256x32xf32>, memref<256x32xf32>
+# DUMPIR:     %[[WAIT2:.*]] = gpu.wait async [%[[MEMCPY3]]]
+# DUMPIR:     return
+# DUMPIR:   }
diff --git a/mlir/test/Examples/NVGPU/Ch2.py b/mlir/test/Examples/NVGPU/Ch2.py
index 729913c6d5c4f..aba610cee0b34 100644
--- a/mlir/test/Examples/NVGPU/Ch2.py
+++ b/mlir/test/Examples/NVGPU/Ch2.py
@@ -1,5 +1,9 @@
 # RUN: env SUPPORT_LIB=%mlir_cuda_runtime \
-# RUN:   %PYTHON %s | FileCheck %s
+# RUN: sh -c 'if [[ "%mlir_run_cuda_sm90_tests" == "1" ]]; \
+# RUN: then %PYTHON %s | FileCheck %s; \
+# RUN: else export MLIR_NVDSL_PRINT_IR=1; \
+# RUN: %PYTHON %s | FileCheck %s --check-prefix=DUMPIR; fi'
+
 
 # ===----------------------------------------------------------------------===//
 #  Chapter 2 : 2D Saxpy with TMA
@@ -28,12 +32,12 @@
 @NVDSL.mlir_func
 def saxpy(x, y, alpha):
     token_ty = gpu.AsyncTokenType.get()
-    t1 = gpu.wait(token_ty, [])
+    t1 = gpu.wait([])
     x_dev, t2 = gpu.alloc(x.type, token_ty, [t1], [], [])
     y_dev, t3 = gpu.alloc(y.type, token_ty, [t2], [], [])
     t4 = gpu.memcpy(token_ty, [t3], x_dev, x)
     t5 = gpu.memcpy(token_ty, [t4], y_dev, y)
-    t6 = gpu.wait(token_ty, [t5])
+    t6 = gpu.wait([t5])
 
     x_tma = TMA([1, N], x.type)
     y_tma = TMA([1, N], y.type)
@@ -74,7 +78,7 @@ def saxpy_tma_kernel():
     saxpy_tma_kernel()
 
     t7 = gpu.memcpy(token_ty, [t6], y, y_dev)
-    gpu.wait(token_ty, [t7])
+    gpu.wait([t7])
 
 
 # 3. Pass numpy arrays to MLIR
@@ -85,9 +89,46 @@ def saxpy_tma_kernel():
 y = np.ones((M, N), np.float32)
 saxpy(x, y, alpha)
 
-#  4. Verify MLIR with reference computation
-ref = np.ones((M, N), np.float32)
-ref += x * alpha
-np.testing.assert_allclose(y, ref, rtol=5e-03, atol=1e-01)
-print("PASS")
+if os.getenv("MLIR_NVDSL_PRINT_IR") != "1":
+    #  4. Verify MLIR with reference computation
+    ref = np.ones((M, N), np.float32)
+    ref += x * alpha
+    np.testing.assert_allclose(y, ref, rtol=5e-03, atol=1e-01)
+    print("PASS")
 # CHECK-NOT: Mismatched elements
+# CHECK: PASS
+
+# DUMPIR:   func.func @saxpy(%{{.*}}: memref<256x32xf32>, %[[ARG1:.*]]: memref<256x32xf32>, %[[ARG2:.*]]: f32) attributes {llvm.emit_c_interface} {
+# DUMPIR:     %[[WAIT0:.*]] = gpu.wait async
+# DUMPIR:     %[[MEMREF:.*]], %[[ASYNC0:.*]] = gpu.alloc async [%[[WAIT0]]] () : memref<256x32xf32>
+# DUMPIR:     %[[CAST:.*]] = memref.cast %[[MEMREF]] : memref<256x32xf32> to memref<*xf32>
+# DUMPIR:     %[[C1:.*]] = arith.constant 1 : index
+# DUMPIR:     %[[C32:.*]] = arith.constant 32 : index
+# DUMPIR:     %[[TMA0:.*]] = nvgpu.tma.create.descriptor %[[CAST]] box[%[[C1]], %[[C32]]] : memref<*xf32> -> <tensor = memref<1x32xf32, 3>, swizzle = none, l2promo = none, oob = zero, interleave = none>
+# DUMPIR:       %[[C0:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[EQ:.*]] = arith.cmpi eq, %{{.*}}, %[[C0]] : index
+# DUMPIR:       %[[MB:.*]] = nvgpu.mbarrier.create -> <memorySpace = #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C0_10:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[C1_11:.*]] = arith.constant 1 : index
+# DUMPIR:       nvgpu.mbarrier.init %[[MB]][%[[C0_10]]], %[[C1_11]], predicate = %[[EQ]] : <memorySpace = #gpu.address_space<workgroup>>
+# DUMPIR:       %[[DSM0:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C0_12:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[VIEW:.*]] = memref.view %[[DSM0]][%[[C0_12]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<1x32xf32, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[DSM1:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C128:.*]] = arith.constant 128 : index
+# DUMPIR:       %[[VIEW_13:.*]] = memref.view %[[DSM1]][%[[C128]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<1x32xf32, #gpu.address_space<workgroup>>
+# DUMPIR:       nvgpu.tma.async.load %[[TMA0]][%{{.*}}, %{{.*}}], %[[MB]][%{{.*}}] to %[[VIEW]], predicate = %[[EQ]] : <tensor = memref<1x32xf32, 3>, swizzle = none, l2promo = none, oob = zero, interleave = none>, <memorySpace = #gpu.address_space<workgroup>> -> memref<1x32xf32, #gpu.address_space<workgroup>>
+# DUMPIR:       nvgpu.mbarrier.arrive.expect_tx %[[MB]][%{{.*}}], %{{.*}}, predicate = %[[EQ]] : <memorySpace = #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C0_20:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[C10000000:.*]] = arith.constant 10000000 : index
+# DUMPIR:       %[[FALSE:.*]] = arith.constant false
+# DUMPIR:       nvgpu.mbarrier.try_wait.parity %[[MB]][%[[C0_20]]], %[[FALSE]], %[[C10000000]] : <memorySpace = #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C0_21:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[LD0:.*]] = memref.load %[[VIEW]][%[[C0_21]], %{{.*}}] : memref<1x32xf32, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C0_22:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[LD1:.*]] = memref.load %[[VIEW_13]][%[[C0_22]], %{{.*}}] : memref<1x32xf32, #gpu.address_space<workgroup>>
+# DUMPIR:       memref.store %{{.*}}, %{{.*}}[%{{.*}}, %{{.*}}] : memref<256x32xf32>
+# DUMPIR:     %[[MEMCPY3:.*]] = gpu.memcpy async [%{{.*}}] %[[ARG1]], %{{.*}} : memref<256x32xf32>, memref<256x32xf32>
+# DUMPIR:     %{{.*}} = gpu.wait async [%[[MEMCPY3]]]
+# DUMPIR:     return
+# DUMPIR:   }
diff --git a/mlir/test/Examples/NVGPU/Ch3.py b/mlir/test/Examples/NVGPU/Ch3.py
index eb96b11c63416..fe11575416866 100644
--- a/mlir/test/Examples/NVGPU/Ch3.py
+++ b/mlir/test/Examples/NVGPU/Ch3.py
@@ -1,5 +1,9 @@
 # RUN: env SUPPORT_LIB=%mlir_cuda_runtime \
-# RUN:   %PYTHON %s | FileCheck %s
+# RUN: sh -c 'if [[ "%mlir_run_cuda_sm90_tests" == "1" ]]; \
+# RUN: then %PYTHON %s | FileCheck %s; \
+# RUN: else export MLIR_NVDSL_PRINT_IR=1; \
+# RUN: %PYTHON %s | FileCheck %s --check-prefix=DUMPIR; fi'
+
 
 # ===----------------------------------------------------------------------===//
 #  Chapter 3 : GEMM 128x128x64 with Tensor Core
@@ -60,13 +64,13 @@ def tma_load(
 @NVDSL.mlir_func
 def gemm_128_128_64(a, b, d):
     token_ty = gpu.AsyncTokenType.get()
-    t1 = gpu.wait(token_ty, [])
+    t1 = gpu.wait([])
     a_dev, t2 = gpu.alloc(a.type, token_ty, [t1], [], [])
     b_dev, t3 = gpu.alloc(b.type, token_ty, [t2], [], [])
     d_dev, t4 = gpu.alloc(d.type, token_ty, [t3], [], [])
     t5 = gpu.memcpy(token_ty, [t4], a_dev, a)
     t6 = gpu.memcpy(token_ty, [t5], b_dev, b)
-    t7 = gpu.wait(token_ty, [t6])
+    t7 = gpu.wait([t6])
 
     sw = nvgpu.TensorMapSwizzleKind.SWIZZLE_128B
     a_tma = TMA([128, 64], a.type, swizzle=sw)
@@ -111,7 +115,7 @@ def gemm_tma_kernel():
     gemm_tma_kernel()
 
     t8 = gpu.memcpy(token_ty, [t7], d, d_dev)
-    gpu.wait(None, [t8])
+    gpu.wait([t8])
 
 
 # Python pass arguments to MLIR
@@ -123,7 +127,73 @@ def gemm_tma_kernel():
 d = np.zeros((M, N), np.float32)
 gemm_128_128_64(a, b, d)
 
-ref_d = a.astype(np.float16) @ b.astype(np.float16)
-np.testing.assert_allclose(d, ref_d, rtol=5e-03, atol=1e-01)
-print("PASS")
+if os.getenv("MLIR_NVDSL_PRINT_IR") != "1":
+    # Verify MLIR program with reference computation in python
+    ref_d = a.astype(np.float16) @ b.astype(np.float16)
+    np.testing.assert_allclose(d, ref_d, rtol=5e-03, atol=1e-01)
+    print("PASS")
 # CHECK-NOT: Mismatched elements
+# CHECK: PASS
+
+# DUMPIR:   func.func @gemm_128_128_64(%{{.*}}: memref<128x64xf16>, %{{.*}}: memref<64x128xf16>, %[[ARG2:.*]]: memref<128x128xf32>) attributes {llvm.emit_c_interface} {
+# DUMPIR:     %[[C128:.*]] = arith.constant 128 : index
+# DUMPIR:     %[[C64:.*]] = arith.constant 64 : index
+# DUMPIR:     %[[TMA0:.*]] = nvgpu.tma.create.descriptor %{{.*}} box[%[[C128]], %[[C64]]] : memref<*xf16> -> <tensor = memref<128x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>
+# DUMPIR:     %[[CAST1:.*]] = memref.cast %{{.*}} : memref<64x128xf16> to memref<*xf16>
+# DUMPIR:     %[[C64_5:.*]] = arith.constant 64 : index
+# DUMPIR:     %[[C64_6:.*]] = arith.constant 64 : index
+# DUMPIR:     %[[TMA1:.*]] = nvgpu.tma.create.descriptor %[[CAST1]] box[%[[C64_5]], %[[C64_6]]] : memref<*xf16> -> <tensor = memref<64x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>
+# DUMPIR:       %[[THREADID:.*]] = gpu.thread_id  x
+# DUMPIR:       %[[MB:.*]] = nvgpu.mbarrier.create -> <memorySpace = #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C0:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[EQ:.*]] = arith.cmpi eq, %[[THREADID]], %[[C0]] : index
+# DUMPIR:       %[[C0_12:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[C1_13:.*]] = arith.constant 1 : index
+# DUMPIR:       nvgpu.mbarrier.init %[[MB]][%[[C0_12]]], %[[C1_13]], predicate = %[[EQ]] : <memorySpace = #gpu.address_space<workgroup>>
+# DUMPIR:       nvgpu.tma.prefetch.descriptor %[[TMA0]], predicate = %[[EQ]] : <tensor = memref<128x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>
+# DUMPIR:       nvgpu.tma.prefetch.descriptor %[[TMA1]], predicate = %[[EQ]] : <tensor = memref<64x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>
+# DUMPIR:       %[[DSM0:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C0_14:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[VIEW:.*]] = memref.view %[[DSM0]][%[[C0_14]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<128x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[DSM1:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C16384:.*]] = arith.constant 16384 : index
+# DUMPIR:       %[[VIEW_15:.*]] = memref.view %[[DSM1]][%[[C16384]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<64x128xf16, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[DSM2:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C0_16:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[VIEW_17:.*]] = memref.view %[[DSM2]][%[[C0_16]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<128x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[DSM3:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C16384_18:.*]] = arith.constant 16384 : index
+# DUMPIR:       %[[VIEW_19:.*]] = memref.view %[[DSM3]][%[[C16384_18]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<64x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[DSM4:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C24576:.*]] = arith.constant 24576 : index
+# DUMPIR:       %[[VIEW_20:.*]] = memref.view %[[DSM4]][%[[C24576]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<64x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C0_21:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[C32768:.*]] = arith.constant 32768 : index
+# DUMPIR:       nvgpu.mbarrier.arrive.expect_tx %[[MB]][%[[C0_21]]], %[[C32768]], predicate = %[[EQ]] : <memorySpace = #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C0_22:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[C0_23:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[C0_24:.*]] = arith.constant 0 : index
+# DUMPIR:       nvgpu.tma.async.load %[[TMA0]][%[[C0_23]], %[[C0_24]]], %[[MB]][%[[C0_22]]] to %[[VIEW_17]], predicate = %[[EQ]] : <tensor = memref<128x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>, <memorySpace = #gpu.address_space<workgroup>> -> memref<128x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C0_25:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[C0_26:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[C0_27:.*]] = arith.constant 0 : index
+# DUMPIR:       nvgpu.tma.async.load %[[TMA1]][%[[C0_26]], %[[C0_27]]], %[[MB]][%[[C0_25]]] to %[[VIEW_19]], predicate = %[[EQ]] : <tensor = memref<64x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>, <memorySpace = #gpu.address_space<workgroup>> -> memref<64x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C0_28:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[C64_29:.*]] = arith.constant 64 : index
+# DUMPIR:       %[[C0_30:.*]] = arith.constant 0 : index
+# DUMPIR:       nvgpu.tma.async.load %[[TMA1]][%[[C64_29]], %[[C0_30]]], %[[MB]][%[[C0_28]]] to %[[VIEW_20]], predicate = %[[EQ]] : <tensor = memref<64x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>, <memorySpace = #gpu.address_space<workgroup>> -> memref<64x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C0_31:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[C10000000:.*]] = arith.constant 10000000 : index
+# DUMPIR:       %[[FALSE:.*]] = arith.constant false
+# DUMPIR:       nvgpu.mbarrier.try_wait.parity %[[MB]][%[[C0_31]]], %[[FALSE]], %[[C10000000]] : <memorySpace = #gpu.address_space<workgroup>>
+# DUMPIR:       %[[WG_ACC:.*]] = nvgpu.warpgroup.mma.init.accumulator -> <fragmented = vector<128x128xf32>>
+# DUMPIR:       %[[GEN0:.*]] = nvgpu.warpgroup.generate.descriptor %[[VIEW]], %[[TMA0]] : memref<128x64xf16, #gpu.address_space<workgroup>>, <tensor = memref<128x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none> -> <tensor = memref<128x64xf16, #gpu.address_space<workgroup>>>
+# DUMPIR:       %[[GEN1:.*]] = nvgpu.warpgroup.generate.descriptor %[[VIEW_15]], %[[TMA1]] : memref<64x128xf16, #gpu.address_space<workgroup>>, <tensor = memref<64x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none> -> <tensor = memref<64x128xf16, #gpu.address_space<workgroup>>>
+# DUMPIR:       %[[MMA:.*]] = nvgpu.warpgroup.mma %[[GEN0]], %[[GEN1]], %[[WG_ACC]] {transposeB} : <tensor = memref<128x64xf16, #gpu.address_space<workgroup>>>, <tensor = memref<64x128xf16, #gpu.address_space<workgroup>>>, <fragmented = vector<128x128xf32>> -> <fragmented = vector<128x128xf32>>
+# DUMPIR:       nvgpu.warpgroup.mma.store %[[MMA]], %{{.*}} : <fragmented = vector<128x128xf32>> to memref<128x128xf32>
+# DUMPIR:       gpu.terminator
+# DUMPIR:     }
+# DUMPIR:     %[[CPY3:.*]] = gpu.memcpy async [%{{.*}}] %[[ARG2]], %{{.*}} : memref<128x128xf32>, memref<128x128xf32>
+# DUMPIR:     gpu.wait async [%[[CPY3]]]
+# DUMPIR:     return
+# DUMPIR:   }
diff --git a/mlir/test/Examples/NVGPU/Ch4.py b/mlir/test/Examples/NVGPU/Ch4.py
index 0e3460ff8d63b..dffafda7f21c9 100644
--- a/mlir/test/Examples/NVGPU/Ch4.py
+++ b/mlir/test/Examples/NVGPU/Ch4.py
@@ -1,5 +1,9 @@
 # RUN: env SUPPORT_LIB=%mlir_cuda_runtime \
-# RUN:   %PYTHON %s | FileCheck %s
+# RUN: sh -c 'if [[ "%mlir_run_cuda_sm90_tests" == "1" ]]; \
+# RUN: then %PYTHON %s | FileCheck %s; \
+# RUN: else export MLIR_NVDSL_PRINT_IR=1; \
+# RUN: %PYTHON %s | FileCheck %s --check-prefix=DUMPIR; fi'
+
 
 # ===----------------------------------------------------------------------===//
 #  Chapter 4 : Multistage GEMM with Tensor Core
@@ -259,13 +263,13 @@ def epilogue(D: WGMMAMatrix, d_dev):
 @NVDSL.mlir_func
 def gemm_multistage(a, b, d, num_stages):
     token_ty = gpu.AsyncTokenType.get()
-    t1 = gpu.wait(token_ty, [])
+    t1 = gpu.wait([])
     a_dev, t2 = gpu.alloc(a.type, token_ty, [t1], [], [])
     b_dev, t3 = gpu.alloc(b.type, token_ty, [t2], [], [])
     d_dev, t4 = gpu.alloc(d.type, token_ty, [t3], [], [])
     t5 = gpu.memcpy(token_ty, [t4], a_dev, a)
     t6 = gpu.memcpy(token_ty, [t5], b_dev, b)
-    t7 = gpu.wait(token_ty, [t6])
+    t7 = gpu.wait([t6])
 
     sw = nvgpu.TensorMapSwizzleKind.SWIZZLE_128B
     a_tma = TMA([128, 64], a.type, swizzle=sw)
@@ -297,7 +301,7 @@ def gemm_multistage_kernel():
     gemm_multistage_kernel()
 
     t8 = gpu.memcpy(token_ty, [t7], d, d_dev)
-    gpu.wait(None, [t8])
+    gpu.wait([t8])
 
 
 # Python pass arguments to MLIR
@@ -313,11 +317,153 @@ def gemm_multistage_kernel():
 
 gemm_multistage(a, b, d, num_stages=7)
 
+if os.getenv("MLIR_NVDSL_PRINT_IR") != "1":
+    # Verify MLIR with reference computation
+    ref_d = a.astype(np.float16) @ b.astype(np.float16)
+    np.testing.assert_allclose(d, ref_d, rtol=5e-03, atol=1e-01)
 
-# Verify MLIR with reference computation
-ref_d = a.astype(np.float16) @ b.astype(np.float16)
-np.testing.assert_allclose(d, ref_d, rtol=5e-03, atol=1e-01)
-
-
-print("PASS")
+    print("PASS")
 # CHECK-NOT: Mismatched elements
+# CHECK: PASS
+
+# DUMPIR:   func.func @gemm_multistage(%{{.*}}: memref<512x1024xf16>, %{{.*}}: memref<1024x256xf16>, %{{.*}}: memref<512x256xf32>) attributes {llvm.emit_c_interface} {
+# DUMPIR:       scf.if %{{.*}} {
+# DUMPIR:         %[[C0_INIT:.*]] = arith.constant 0 : index
+# DUMPIR:         %[[C7:.*]] = arith.constant 7 : index
+# DUMPIR:         %[[C1_INIT:.*]] = arith.constant 1 : index
+# DUMPIR:         scf.for %arg15 = %[[C0_INIT]] to %[[C7]] step %[[C1_INIT]] {
+# DUMPIR:           %[[C1_MBAR:.*]] = arith.constant 1 : index
+# DUMPIR:           nvgpu.mbarrier.init %{{.*}}[%arg15], %[[C1_MBAR]] : <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7>
+# DUMPIR:         }
+# DUMPIR:         nvgpu.tma.prefetch.descriptor %{{.*}} : <tensor = memref<128x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>
+# DUMPIR:         nvgpu.tma.prefetch.descriptor %{{.*}} : <tensor = memref<64x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>
+# DUMPIR:       }
+# DUMPIR:       %[[C0_PROLOGUE:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[C6:.*]] = arith.constant 6 : index
+# DUMPIR:       %[[C1_PROLOGUE:.*]] = arith.constant 1 : index
+# DUMPIR:       scf.for %arg15 = %[[C0_PROLOGUE]] to %[[C6]] step %[[C1_PROLOGUE]] {
+# DUMPIR:         %[[BID_X_P:.*]] = gpu.block_id  x
+# DUMPIR:         %[[BID_Y_P:.*]] = gpu.block_id  y
+# DUMPIR:         %[[C128_P1:.*]] = arith.constant 128 : index
+# DUMPIR:         %[[DIMX_P:.*]] = arith.muli %[[BID_X_P]], %[[C128_P1]] : index
+# DUMPIR:         %[[C128_P2:.*]] = arith.constant 128 : index
+# DUMPIR:         %[[DIMY_P:.*]] = arith.muli %[[BID_Y_P]], %[[C128_P2]] : index
+# DUMPIR:         %{{.*}} = gpu.thread_id  x
+# DUMPIR:         %[[TID_X_P:.*]] = gpu.thread_id  x
+# DUMPIR:         %[[C0_P:.*]] = arith.constant 0 : index
+# DUMPIR:         %[[PRED_P:.*]] = arith.cmpi eq, %[[TID_X_P]], %[[C0_P]] : index
+# DUMPIR:         %[[C16384_P1:.*]] = arith.constant 16384 : index
+# DUMPIR:         %[[OFF_A_P:.*]] = arith.muli %arg15, %[[C16384_P1]] : index
+# DUMPIR:         %[[C16384_P2:.*]] = arith.constant 16384 : index
+# DUMPIR:         %[[OFF_B_BASE_P:.*]] = arith.muli %arg15, %[[C16384_P2]] : index
+# DUMPIR:         %[[C114688:.*]] = arith.constant 114688 : index
+# DUMPIR:         %[[OFF_B1_P:.*]] = arith.addi %[[OFF_B_BASE_P]], %[[C114688]] : index
+# DUMPIR:         %[[C8192:.*]] = arith.constant 8192 : index
+# DUMPIR:         %[[OFF_B2_P:.*]] = arith.addi %[[OFF_B1_P]], %[[C8192]] : index
+# DUMPIR:         %[[SMEM_A_P:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[VIEW_A_P:.*]] = memref.view %[[SMEM_A_P]][%[[OFF_A_P]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<128x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[SMEM_B1_P:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[VIEW_B1_P:.*]] = memref.view %[[SMEM_B1_P]][%[[OFF_B1_P]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<64x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[SMEM_B2_P:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[VIEW_B2_P:.*]] = memref.view %[[SMEM_B2_P]][%[[OFF_B2_P]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<64x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[C32768:.*]] = arith.constant 32768 : index
+# DUMPIR:         nvgpu.mbarrier.arrive.expect_tx %{{.*}}[%arg15], %[[C32768]], predicate = %[[PRED_P]] : <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7>
+# DUMPIR:         %[[C64_K_P:.*]] = arith.constant 64 : index
+# DUMPIR:         %[[K_COORD_P:.*]] = arith.muli %arg15, %[[C64_K_P]] : index
+# DUMPIR:         nvgpu.tma.async.load %{{.*}}[%[[K_COORD_P]], %[[DIMX_P]]], %{{.*}}[%arg15] to %[[VIEW_A_P]], predicate = %[[PRED_P]] : <tensor = memref<128x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>, <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7> -> memref<128x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:         nvgpu.tma.async.load %{{.*}}[%[[DIMY_P]], %[[K_COORD_P]]], %{{.*}}[%arg15] to %[[VIEW_B1_P]], predicate = %[[PRED_P]] : <tensor = memref<64x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>, <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7> -> memref<64x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[C64_OFF:.*]] = arith.constant 64 : index
+# DUMPIR:         %[[DIMY_P_OFF:.*]] = arith.addi %[[DIMY_P]], %[[C64_OFF]] : index
+# DUMPIR:         nvgpu.tma.async.load %{{.*}}[%[[DIMY_P_OFF]], %[[K_COORD_P]]], %{{.*}}[%arg15] to %[[VIEW_B2_P]], predicate = %[[PRED_P]] : <tensor = memref<64x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>, <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7> -> memref<64x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:       }
+# DUMPIR:       %[[TID_X_LOOP:.*]] = gpu.thread_id  x
+# DUMPIR:       %[[ACC_INIT:.*]] = nvgpu.warpgroup.mma.init.accumulator -> <fragmented = vector<128x128xf32>>
+# DUMPIR:       %[[FALSE_LOOP:.*]] = arith.constant false
+# DUMPIR:       %[[C0_LOOP:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[C16_LOOP:.*]] = arith.constant 16 : index
+# DUMPIR:       %[[C1_LOOP:.*]] = arith.constant 1 : index
+# DUMPIR:       %[[LOOP_RES:.*]]:2 = scf.for %arg15 = %[[C0_LOOP]] to %[[C16_LOOP]] step %[[C1_LOOP]] iter_args(%arg16 = %[[ACC_INIT]], %arg17 = %[[FALSE_LOOP]]) -> (!nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>, i1) {
+# DUMPIR:         %[[C7_L:.*]] = arith.constant 7 : index
+# DUMPIR:         %[[STAGE_L:.*]] = arith.remui %arg15, %[[C7_L]] : index
+# DUMPIR:         %[[C10M:.*]] = arith.constant 10000000 : index
+# DUMPIR:         nvgpu.mbarrier.try_wait.parity %{{.*}}[%[[STAGE_L]]], %arg17, %[[C10M]] : <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7>
+# DUMPIR:         %[[C16384_L:.*]] = arith.constant 16384 : index
+# DUMPIR:         %[[OFF_A_L:.*]] = arith.muli %[[STAGE_L]], %[[C16384_L]] : index
+# DUMPIR:         %[[C114688_L:.*]] = arith.constant 114688 : index
+# DUMPIR:         %[[OFF_B_L:.*]] = arith.addi %[[OFF_A_L]], %[[C114688_L]] : index
+# DUMPIR:         %[[SMEM_A_L:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[VIEW_A_L:.*]] = memref.view %[[SMEM_A_L]][%[[OFF_A_L]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<128x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[SMEM_B_L:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[VIEW_B_L:.*]] = memref.view %[[SMEM_B_L]][%[[OFF_B_L]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<64x128xf16, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[DESC_A_L:.*]] = nvgpu.warpgroup.generate.descriptor %[[VIEW_A_L]], %{{.*}} : memref<128x64xf16, #gpu.address_space<workgroup>>, <tensor = memref<128x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none> -> <tensor = memref<128x64xf16, #gpu.address_space<workgroup>>>
+# DUMPIR:         %[[DESC_B_L:.*]] = nvgpu.warpgroup.generate.descriptor %[[VIEW_B_L]], %{{.*}} : memref<64x128xf16, #gpu.address_space<workgroup>>, <tensor = memref<64x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none> -> <tensor = memref<64x128xf16, #gpu.address_space<workgroup>>>
+# DUMPIR:         %[[ACC_L:.*]] = nvgpu.warpgroup.mma %[[DESC_A_L]], %[[DESC_B_L]], %arg16 {transposeB} : <tensor = memref<128x64xf16, #gpu.address_space<workgroup>>>, <tensor = memref<64x128xf16, #gpu.address_space<workgroup>>>, <fragmented = vector<128x128xf32>> -> <fragmented = vector<128x128xf32>>
+# DUMPIR:         %[[C6_NEXT:.*]] = arith.constant 6 : index
+# DUMPIR:         %[[ITER_NEXT:.*]] = arith.addi %arg15, %[[C6_NEXT]] : index
+# DUMPIR:         %[[C16_CMP:.*]] = arith.constant 16 : index
+# DUMPIR:         %[[IN_RANGE:.*]] = arith.cmpi ult, %[[ITER_NEXT]], %[[C16_CMP]] : index
+# DUMPIR:         %[[C0_CMP:.*]] = arith.constant 0 : index
+# DUMPIR:         %[[IS_THREAD0_L:.*]] = arith.cmpi eq, %[[TID_X_LOOP]], %[[C0_CMP]] : index
+# DUMPIR:         %[[DO_LOAD:.*]] = arith.andi %[[IN_RANGE]], %[[IS_THREAD0_L]] : i1
+# DUMPIR:         %[[C6_STAGE:.*]] = arith.constant 6 : index
+# DUMPIR:         %[[STAGE_NEXT_L:.*]] = arith.addi %arg15, %[[C6_STAGE]] : index
+# DUMPIR:         %[[C7_MOD:.*]] = arith.constant 7 : index
+# DUMPIR:         %[[STAGE_LOAD:.*]] = arith.remui %[[STAGE_NEXT_L]], %[[C7_MOD]] : index
+# DUMPIR:         %[[BID_X_L:.*]] = gpu.block_id  x
+# DUMPIR:         %[[BID_Y_L:.*]] = gpu.block_id  y
+# DUMPIR:         %[[C128_L1:.*]] = arith.constant 128 : index
+# DUMPIR:         %[[DIMX_L:.*]] = arith.muli %[[BID_X_L]], %[[C128_L1]] : index
+# DUMPIR:         %[[C128_L2:.*]] = arith.constant 128 : index
+# DUMPIR:         %[[DIMY_L:.*]] = arith.muli %[[BID_Y_L]], %[[C128_L2]] : index
+# DUMPIR:         %[[TID_X_L1:.*]] = gpu.thread_id  x
+# DUMPIR:         %[[TID_X_L2:.*]] = gpu.thread_id  x
+# DUMPIR:         %[[C16384_LA1:.*]] = arith.constant 16384 : index
+# DUMPIR:         %[[OFF_A_LOAD:.*]] = arith.muli %[[STAGE_LOAD]], %[[C16384_LA1]] : index
+# DUMPIR:         %[[C16384_LA2:.*]] = arith.constant 16384 : index
+# DUMPIR:         %[[OFF_B_BASE_LOAD:.*]] = arith.muli %[[STAGE_LOAD]], %[[C16384_LA2]] : index
+# DUMPIR:         %[[C114688_LOAD:.*]] = arith.constant 114688 : index
+# DUMPIR:         %[[OFF_B1_LOAD:.*]] = arith.addi %[[OFF_B_BASE_LOAD]], %[[C114688_LOAD]] : index
+# DUMPIR:         %[[C8192_LOAD:.*]] = arith.constant 8192 : index
+# DUMPIR:         %[[OFF_B2_LOAD:.*]] = arith.addi %[[OFF_B1_LOAD]], %[[C8192_LOAD]] : index
+# DUMPIR:         %[[SMEM_A_LOAD:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[VIEW_A_LOAD:.*]] = memref.view %[[SMEM_A_LOAD]][%[[OFF_A_LOAD]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<128x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[SMEM_B1_LOAD:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[VIEW_B1_LOAD:.*]] = memref.view %[[SMEM_B1_LOAD]][%[[OFF_B1_LOAD]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<64x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[SMEM_B2_LOAD:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[VIEW_B2_LOAD:.*]] = memref.view %[[SMEM_B2_LOAD]][%[[OFF_B2_LOAD]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<64x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[C32768_LOAD:.*]] = arith.constant 32768 : index
+# DUMPIR:         nvgpu.mbarrier.arrive.expect_tx %{{.*}}[%[[STAGE_LOAD]]], %[[C32768_LOAD]], predicate = %[[DO_LOAD]] : <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7>
+# DUMPIR:         %[[C64_K_LOAD:.*]] = arith.constant 64 : index
+# DUMPIR:         %[[K_COORD_LOAD:.*]] = arith.muli %[[STAGE_NEXT_L]], %[[C64_K_LOAD]] : index
+# DUMPIR:         nvgpu.tma.async.load %{{.*}}[%[[K_COORD_LOAD]], %[[DIMX_L]]], %{{.*}}[%[[STAGE_LOAD]]] to %[[VIEW_A_LOAD]], predicate = %[[DO_LOAD]] : <tensor = memref<128x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>, <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7> -> memref<128x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:         nvgpu.tma.async.load %{{.*}}[%[[DIMY_L]], %[[K_COORD_LOAD]]], %{{.*}}[%[[STAGE_LOAD]]] to %[[VIEW_B1_LOAD]], predicate = %[[DO_LOAD]] : <tensor = memref<64x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>, <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7> -> memref<64x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[C64_OFF_LOAD:.*]] = arith.constant 64 : index
+# DUMPIR:         %[[DIMY_L_OFF:.*]] = arith.addi %[[DIMY_L]], %[[C64_OFF_LOAD]] : index
+# DUMPIR:         nvgpu.tma.async.load %{{.*}}[%[[DIMY_L_OFF]], %[[K_COORD_LOAD]]], %{{.*}}[%[[STAGE_LOAD]]] to %[[VIEW_B2_LOAD]], predicate = %[[DO_LOAD]] : <tensor = memref<64x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>, <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7> -> memref<64x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[C6_FLIP:.*]] = arith.constant 6 : index
+# DUMPIR:         %[[IS_STAGE6:.*]] = arith.cmpi eq, %[[STAGE_L]], %[[C6_FLIP]] : index
+# DUMPIR:         %[[TRUE:.*]] = arith.constant true
+# DUMPIR:         %[[PARITY_FLIP:.*]] = arith.xori %arg17, %[[TRUE]] : i1
+# DUMPIR:         %[[NEW_PARITY:.*]] = arith.select %[[IS_STAGE6]], %[[PARITY_FLIP]], %arg17 : i1
+# DUMPIR:         scf.yield %[[ACC_L]], %[[NEW_PARITY]] : !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>, i1
+# DUMPIR:       }
+# DUMPIR:       nvvm.wgmma.wait.group.sync.aligned 0
+# DUMPIR:       %[[TID_X_EPI:.*]] = gpu.thread_id  x
+# DUMPIR:       %[[BID_X_EPI:.*]] = gpu.block_id  x
+# DUMPIR:       %[[BID_Y_EPI:.*]] = gpu.block_id  y
+# DUMPIR:       %[[C128_EPI1:.*]] = arith.constant 128 : index
+# DUMPIR:       %[[DIMX_EPI:.*]] = arith.muli %[[BID_X_EPI]], %[[C128_EPI1]] : index
+# DUMPIR:       %[[C128_EPI2:.*]] = arith.constant 128 : index
+# DUMPIR:       %[[DIMY_EPI:.*]] = arith.muli %[[BID_Y_EPI]], %[[C128_EPI2]] : index
+# DUMPIR:       %[[SMEM_EPI:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[C0_VIEW:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[VIEW_EPI:.*]] = memref.view %[[SMEM_EPI]][%[[C0_VIEW]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<128x128xf32, #gpu.address_space<workgroup>>
+# DUMPIR:       %[[SUBVIEW_EPI:.*]] = memref.subview %{{.*}}[%[[DIMX_EPI]], %[[DIMY_EPI]]] [128, 128] [1, 1] : memref<512x256xf32> to memref<128x128xf32, strided<[256, 1], offset: ?>>
+# DUMPIR:       nvgpu.warpgroup.mma.store %[[LOOP_RES]]#0, %[[VIEW_EPI]] : <fragmented = vector<128x128xf32>> to memref<128x128xf32, #gpu.address_space<workgroup>>
+# DUMPIR:       gpu.barrier
+# DUMPIR:       %[[C0_STORE:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[C128_STORE:.*]] = arith.constant 128 : index
+# DUMPIR:       %[[C1_STORE:.*]] = arith.constant 1 : index
+# DUMPIR:       scf.for %arg15 = %[[C0_STORE]] to %[[C128_STORE]] step %[[C1_STORE]] {
+# DUMPIR:         %[[VAL_LOAD:.*]] = memref.load %[[VIEW_EPI]][%arg15, %[[TID_X_EPI]]] : memref<128x128xf32, #gpu.address_space<workgroup>>
+# DUMPIR:         memref.store %[[VAL_LOAD]], %[[SUBVIEW_EPI]][%arg15, %[[TID_X_EPI]]] : memref<128x128xf32, strided<[256, 1], offset: ?>>
diff --git a/mlir/test/Examples/NVGPU/Ch5.py b/mlir/test/Examples/NVGPU/Ch5.py
index f98cfd758a75f..b725e50d8f44b 100644
--- a/mlir/test/Examples/NVGPU/Ch5.py
+++ b/mlir/test/Examples/NVGPU/Ch5.py
@@ -1,5 +1,9 @@
 # RUN: env SUPPORT_LIB=%mlir_cuda_runtime \
-# RUN:   %PYTHON %s | FileCheck %s
+# RUN: sh -c 'if [[ "%mlir_run_cuda_sm90_tests" == "1" ]]; \
+# RUN: then %PYTHON %s | FileCheck %s; \
+# RUN: else export MLIR_NVDSL_PRINT_IR=1; \
+# RUN: %PYTHON %s | FileCheck %s --check-prefix=DUMPIR; fi'
+
 
 # ===----------------------------------------------------------------------===//
 #  Chapter 5 : Warp Specialized GEMM with Tensor Core
@@ -156,7 +160,7 @@ def producer_loop(
 ):
     phase = const(True, ty=T.bool())
 
-    for iv, phase in scf.for_(0, (K // TILE_K), 1, [phase]):
+    for iv, phase, _ in scf.for_(0, (K // TILE_K), 1, [phase]):
         stage = iv % num_stages
         # Wait MMA to be done
         mbar_mma[stage].try_wait(phase)
@@ -253,13 +257,13 @@ def epilogue(D: WGMMAMatrix, d_dev):
 @NVDSL.mlir_func
 def gemm_warp_specialized(a, b, d, num_stages):
     token_ty = gpu.AsyncTokenType.get()
-    t1 = gpu.wait(token_ty, [])
+    t1 = gpu.wait([])
     a_dev, t2 = gpu.alloc(a.type, token_ty, [t1], [], [])
     b_dev, t3 = gpu.alloc(b.type, token_ty, [t2], [], [])
     d_dev, t4 = gpu.alloc(d.type, token_ty, [t3], [], [])
     t5 = gpu.memcpy(token_ty, [t4], a_dev, a)
     t6 = gpu.memcpy(token_ty, [t5], b_dev, b)
-    t7 = gpu.wait(token_ty, [t6])
+    t7 = gpu.wait([t6])
 
     sw = nvgpu.TensorMapSwizzleKind.SWIZZLE_128B
     a_tma = TMA([128, 64], a.type, swizzle=sw)
@@ -295,7 +299,7 @@ def gemm_warp_specialized_kernel():
     gemm_warp_specialized_kernel()
 
     t8 = gpu.memcpy(token_ty, [t7], d, d_dev)
-    gpu.wait(None, [t8])
+    gpu.wait([t8])
 
 
 # Python pass arguments to MLIR
@@ -311,11 +315,166 @@ def gemm_warp_specialized_kernel():
 
 gemm_warp_specialized(a, b, d, num_stages=7)
 
+if os.getenv("MLIR_NVDSL_PRINT_IR") != "1":
+    # Verify MLIR with reference computation
+    ref_d = a.astype(np.float16) @ b.astype(np.float16)
+    np.testing.assert_allclose(d, ref_d, rtol=5e-03, atol=1e-01)
 
-# Verify MLIR with reference computation
-ref_d = a.astype(np.float16) @ b.astype(np.float16)
-np.testing.assert_allclose(d, ref_d, rtol=5e-03, atol=1e-01)
-
-
-print("PASS")
+    print("PASS")
 # CHECK-NOT: Mismatched elements
+# CHECK: PASS
+
+# DUMPIR:       %[[TID_X:.*]] = gpu.thread_id  x
+# DUMPIR:       %[[C128:.*]] = arith.constant 128 : index
+# DUMPIR:       %[[REM1:.*]] = arith.remui %[[TID_X]], %[[C128]] : index
+# DUMPIR:       %[[C0:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[IS_PRIMARY:.*]] = arith.cmpi eq, %[[REM1]], %[[C0]] : index
+# DUMPIR:       %[[C128_1:.*]] = arith.constant 128 : index
+# DUMPIR:       %[[DIV1:.*]] = arith.divui %[[TID_X]], %[[C128_1]] : index
+# DUMPIR:       %[[C1:.*]] = arith.constant 1 : index
+# DUMPIR:       %[[IS_PRODUCER:.*]] = arith.cmpi eq, %[[DIV1]], %[[C1]] : index
+# DUMPIR:       %[[TID_X_2:.*]] = gpu.thread_id  x
+# DUMPIR:       %[[C128_2:.*]] = arith.constant 128 : index
+# DUMPIR:       %[[REM2:.*]] = arith.remui %[[TID_X_2]], %[[C128_2]] : index
+# DUMPIR:       %[[C0_2:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[IS_PRIMARY_2:.*]] = arith.cmpi eq, %[[REM2]], %[[C0_2]] : index
+# DUMPIR:       %[[C128_3:.*]] = arith.constant 128 : index
+# DUMPIR:       %[[DIV2:.*]] = arith.divui %[[TID_X_2]], %[[C128_3]] : index
+# DUMPIR:       %[[C0_3:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[IS_CONSUMER:.*]] = arith.cmpi eq, %[[DIV2]], %[[C0_3]] : index
+# DUMPIR:       %[[TID_X_3:.*]] = gpu.thread_id  x
+# DUMPIR:       %[[MBAR_MMA:.*]] = nvgpu.mbarrier.create -> <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7>
+# DUMPIR:       %[[MBAR_TMA:.*]] = nvgpu.mbarrier.create -> <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7>
+# DUMPIR:       %[[C0_4:.*]] = arith.constant 0 : index
+# DUMPIR:       %[[IS_THREAD0:.*]] = arith.cmpi eq, %[[TID_X_3]], %[[C0_4]] : index
+# DUMPIR:       scf.if %[[IS_THREAD0]] {
+# DUMPIR:         %[[C0_INIT:.*]] = arith.constant 0 : index
+# DUMPIR:         %[[C7:.*]] = arith.constant 7 : index
+# DUMPIR:         %[[C1_INIT:.*]] = arith.constant 1 : index
+# DUMPIR:         scf.for %arg15 = %[[C0_INIT]] to %[[C7]] step %[[C1_INIT]] {
+# DUMPIR:           %[[C1_INIT_VAL:.*]] = arith.constant 1 : index
+# DUMPIR:           nvgpu.mbarrier.init %[[MBAR_MMA]][%arg15], %[[C1_INIT_VAL]] : <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7>
+# DUMPIR:           %[[C1_INIT_VAL_2:.*]] = arith.constant 1 : index
+# DUMPIR:           nvgpu.mbarrier.init %[[MBAR_TMA]][%arg15], %[[C1_INIT_VAL_2]] : <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7>
+# DUMPIR:         }
+# DUMPIR:         nvgpu.tma.prefetch.descriptor %{{.*}} : <tensor = memref<128x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>
+# DUMPIR:         nvgpu.tma.prefetch.descriptor %{{.*}} : <tensor = memref<64x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>
+# DUMPIR:       }
+# DUMPIR:       scf.if %[[IS_PRODUCER]] {
+# DUMPIR:         nvvm.setmaxregister  decrease 40
+# DUMPIR:         %[[TRUE:.*]] = arith.constant true
+# DUMPIR:         %[[C0_PROD:.*]] = arith.constant 0 : index
+# DUMPIR:         %[[C16:.*]] = arith.constant 16 : index
+# DUMPIR:         %[[C1_PROD:.*]] = arith.constant 1 : index
+# DUMPIR:         %[[PROD_LOOP:.*]] = scf.for %arg15 = %[[C0_PROD]] to %[[C16]] step %[[C1_PROD]] iter_args(%arg16 = %[[TRUE]]) -> (i1) {
+# DUMPIR:           %[[C7_PROD:.*]] = arith.constant 7 : index
+# DUMPIR:           %[[SLOT:.*]] = arith.remui %arg15, %[[C7_PROD]] : index
+# DUMPIR:           %[[TIMEOUT:.*]] = arith.constant 10000000 : index
+# DUMPIR:           nvgpu.mbarrier.try_wait.parity %[[MBAR_MMA]][%[[SLOT]]], %arg16, %[[TIMEOUT]] : <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7>
+# DUMPIR:           %[[C6:.*]] = arith.constant 6 : index
+# DUMPIR:           %[[IS_LAST:.*]] = arith.cmpi eq, %[[SLOT]], %[[C6]] : index
+# DUMPIR:           %[[TRUE_2:.*]] = arith.constant true
+# DUMPIR:           %[[FLIP:.*]] = arith.xori %arg16, %[[TRUE_2]] : i1
+# DUMPIR:           %[[PHASE:.*]] = arith.select %[[IS_LAST]], %[[FLIP]], %arg16 : i1
+# DUMPIR:           %[[BID_X:.*]] = gpu.block_id  x
+# DUMPIR:           %[[BID_Y:.*]] = gpu.block_id  y
+# DUMPIR:           %[[C128_TILE:.*]] = arith.constant 128 : index
+# DUMPIR:           %[[DIM_X:.*]] = arith.muli %[[BID_X]], %[[C128_TILE]] : index
+# DUMPIR:           %[[C128_TILE_2:.*]] = arith.constant 128 : index
+# DUMPIR:           %[[DIM_Y:.*]] = arith.muli %[[BID_Y]], %[[C128_TILE_2]] : index
+# DUMPIR:           %[[TID_PROD:.*]] = gpu.thread_id  x
+# DUMPIR:           %[[C16384:.*]] = arith.constant 16384 : index
+# DUMPIR:           %[[OFF_A:.*]] = arith.muli %[[SLOT]], %[[C16384]] : index
+# DUMPIR:           %[[C16384_2:.*]] = arith.constant 16384 : index
+# DUMPIR:           %[[OFF_B_BASE:.*]] = arith.muli %[[SLOT]], %[[C16384_2]] : index
+# DUMPIR:           %[[C114688:.*]] = arith.constant 114688 : index
+# DUMPIR:           %[[OFF_B1:.*]] = arith.addi %[[OFF_B_BASE]], %[[C114688]] : index
+# DUMPIR:           %[[C8192:.*]] = arith.constant 8192 : index
+# DUMPIR:           %[[OFF_B2:.*]] = arith.addi %[[OFF_B1]], %[[C8192]] : index
+# DUMPIR:           %[[SMEM:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:           %[[VIEW_A:.*]] = memref.view %[[SMEM]][%[[OFF_A]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<128x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:           %[[SMEM_2:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:           %[[VIEW_B1:.*]] = memref.view %[[SMEM_2]][%[[OFF_B1]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<64x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:           %[[SMEM_3:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:           %[[VIEW_B2:.*]] = memref.view %[[SMEM_3]][%[[OFF_B2]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<64x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:           %[[TX_COUNT:.*]] = arith.constant 32768 : index
+# DUMPIR:           nvgpu.mbarrier.arrive.expect_tx %[[MBAR_TMA]][%[[SLOT]]], %[[TX_COUNT]], predicate = %[[IS_PRIMARY]] : <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7>
+# DUMPIR:           %[[C128_WG:.*]] = arith.constant 128 : index
+# DUMPIR:           %[[TID_MOD:.*]] = arith.remui %[[TID_PROD]], %[[C128_WG]] : index
+# DUMPIR:           %[[C0_TMA:.*]] = arith.constant 0 : index
+# DUMPIR:           %[[IS_TMA_THREAD:.*]] = arith.cmpi eq, %[[TID_MOD]], %[[C0_TMA]] : index
+# DUMPIR:           %[[C64:.*]] = arith.constant 64 : index
+# DUMPIR:           %[[K_COORD:.*]] = arith.muli %arg15, %[[C64]] : index
+# DUMPIR:           nvgpu.tma.async.load %{{.*}}[%[[K_COORD]], %[[DIM_X]]], %[[MBAR_TMA]][%[[SLOT]]] to %[[VIEW_A]], predicate = %[[IS_TMA_THREAD]] : <tensor = memref<128x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>, <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7> -> memref<128x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:           nvgpu.tma.async.load %{{.*}}[%[[DIM_Y]], %[[K_COORD]]], %[[MBAR_TMA]][%[[SLOT]]] to %[[VIEW_B1]], predicate = %[[IS_TMA_THREAD]] : <tensor = memref<64x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>, <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7> -> memref<64x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:           %[[C64_OFF:.*]] = arith.constant 64 : index
+# DUMPIR:           %[[DIM_Y_OFF:.*]] = arith.addi %[[DIM_Y]], %[[C64_OFF]] : index
+# DUMPIR:           nvgpu.tma.async.load %{{.*}}[%[[DIM_Y_OFF]], %[[K_COORD]]], %[[MBAR_TMA]][%[[SLOT]]] to %[[VIEW_B2]], predicate = %[[IS_TMA_THREAD]] : <tensor = memref<64x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>, <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7> -> memref<64x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:           scf.yield %[[PHASE]] : i1
+# DUMPIR:         }
+# DUMPIR:       }
+# DUMPIR:       scf.if %[[IS_CONSUMER]] {
+# DUMPIR:         nvvm.setmaxregister  increase 232
+# DUMPIR:         %[[FALSE:.*]] = arith.constant false
+# DUMPIR:         %[[ACC_INIT:.*]] = nvgpu.warpgroup.mma.init.accumulator -> <fragmented = vector<128x128xf32>>
+# DUMPIR:         %[[C0_CONS:.*]] = arith.constant 0 : index
+# DUMPIR:         %[[C16_CONS:.*]] = arith.constant 16 : index
+# DUMPIR:         %[[C1_CONS:.*]] = arith.constant 1 : index
+# DUMPIR:         %[[CONS_LOOP:.*]]:2 = scf.for %arg15 = %[[C0_CONS]] to %[[C16_CONS]] step %[[C1_CONS]] iter_args(%arg16 = %[[ACC_INIT]], %arg17 = %[[FALSE]]) -> (!nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>, i1) {
+# DUMPIR:           %[[C7_CONS:.*]] = arith.constant 7 : index
+# DUMPIR:           %[[SLOT_CONS:.*]] = arith.remui %arg15, %[[C7_CONS]] : index
+# DUMPIR:           %[[TIMEOUT_CONS:.*]] = arith.constant 10000000 : index
+# DUMPIR:           nvgpu.mbarrier.try_wait.parity %[[MBAR_TMA]][%[[SLOT_CONS]]], %arg17, %[[TIMEOUT_CONS]] : <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7>
+# DUMPIR:           %[[C16384_CONS:.*]] = arith.constant 16384 : index
+# DUMPIR:           %[[OFF_A_CONS:.*]] = arith.muli %[[SLOT_CONS]], %[[C16384_CONS]] : index
+# DUMPIR:           %[[C114688_CONS:.*]] = arith.constant 114688 : index
+# DUMPIR:           %[[OFF_B_CONS:.*]] = arith.addi %[[OFF_A_CONS]], %[[C114688_CONS]] : index
+# DUMPIR:           %[[SMEM_CONS:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:           %[[VIEW_A_CONS:.*]] = memref.view %[[SMEM_CONS]][%[[OFF_A_CONS]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<128x64xf16, #gpu.address_space<workgroup>>
+# DUMPIR:           %[[SMEM_CONS_2:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:           %[[VIEW_B_CONS:.*]] = memref.view %[[SMEM_CONS_2]][%[[OFF_B_CONS]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<64x128xf16, #gpu.address_space<workgroup>>
+# DUMPIR:           %[[DESC_A:.*]] = nvgpu.warpgroup.generate.descriptor %[[VIEW_A_CONS]], %{{.*}} : memref<128x64xf16, #gpu.address_space<workgroup>>, <tensor = memref<128x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none> -> <tensor = memref<128x64xf16, #gpu.address_space<workgroup>>>
+# DUMPIR:           %[[DESC_B:.*]] = nvgpu.warpgroup.generate.descriptor %[[VIEW_B_CONS]], %{{.*}} : memref<64x128xf16, #gpu.address_space<workgroup>>, <tensor = memref<64x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none> -> <tensor = memref<64x128xf16, #gpu.address_space<workgroup>>>
+# DUMPIR:           %[[ACC:.*]] = nvgpu.warpgroup.mma %[[DESC_A]], %[[DESC_B]], %arg16 {transposeB} : <tensor = memref<128x64xf16, #gpu.address_space<workgroup>>>, <tensor = memref<64x128xf16, #gpu.address_space<workgroup>>>, <fragmented = vector<128x128xf32>> -> <fragmented = vector<128x128xf32>>
+# DUMPIR:           %[[C0_CMP:.*]] = arith.constant 0 : index
+# DUMPIR:           %[[IS_NOT_FIRST:.*]] = arith.cmpi ugt, %arg15, %[[C0_CMP]] : index
+# DUMPIR:           %[[ARRIVE_PRED:.*]] = arith.andi %[[IS_NOT_FIRST]], %[[IS_PRIMARY_2]] : i1
+# DUMPIR:           scf.if %[[ARRIVE_PRED]] {
+# DUMPIR:             %[[C0_ARR:.*]] = arith.constant 0 : index
+# DUMPIR:             %[[IS_ZERO:.*]] = arith.cmpi eq, %[[SLOT_CONS]], %[[C0_ARR]] : index
+# DUMPIR:             %[[C6_WRAP:.*]] = arith.constant 6 : index
+# DUMPIR:             %[[C1_SUB:.*]] = arith.constant 1 : index
+# DUMPIR:             %[[PREV_SLOT:.*]] = arith.subi %[[SLOT_CONS]], %[[C1_SUB]] : index
+# DUMPIR:             %[[BARR_ID:.*]] = arith.select %[[IS_ZERO]], %[[C6_WRAP]], %[[PREV_SLOT]] : index
+# DUMPIR:             %{{.*}} = nvgpu.mbarrier.arrive %[[MBAR_MMA]][%[[BARR_ID]]] : <memorySpace = #gpu.address_space<workgroup>, num_barriers = 7> -> !nvgpu.mbarrier.token
+# DUMPIR:           }
+# DUMPIR:           %[[C6_LAST:.*]] = arith.constant 6 : index
+# DUMPIR:           %[[IS_LAST_CONS:.*]] = arith.cmpi eq, %[[SLOT_CONS]], %[[C6_LAST]] : index
+# DUMPIR:           %[[TRUE_CONS:.*]] = arith.constant true
+# DUMPIR:           %[[FLIP_CONS:.*]] = arith.xori %arg17, %[[TRUE_CONS]] : i1
+# DUMPIR:           %[[PHASE_CONS:.*]] = arith.select %[[IS_LAST_CONS]], %[[FLIP_CONS]], %arg17 : i1
+# DUMPIR:           scf.yield %[[ACC]], %[[PHASE_CONS]] : !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>, i1
+# DUMPIR:         }
+# DUMPIR:         nvvm.wgmma.wait.group.sync.aligned 0
+# DUMPIR:         %[[TID_EPI:.*]] = gpu.thread_id  x
+# DUMPIR:         %[[BID_X_EPI:.*]] = gpu.block_id  x
+# DUMPIR:         %[[BID_Y_EPI:.*]] = gpu.block_id  y
+# DUMPIR:         %[[C128_EPI:.*]] = arith.constant 128 : index
+# DUMPIR:         %[[DIM_X_EPI:.*]] = arith.muli %[[BID_X_EPI]], %[[C128_EPI]] : index
+# DUMPIR:         %[[C128_EPI_2:.*]] = arith.constant 128 : index
+# DUMPIR:         %[[DIM_Y_EPI:.*]] = arith.muli %[[BID_Y_EPI]], %[[C128_EPI_2]] : index
+# DUMPIR:         %[[SMEM_EPI:.*]] = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[C0_EPI:.*]] = arith.constant 0 : index
+# DUMPIR:         %[[VIEW_EPI:.*]] = memref.view %[[SMEM_EPI]][%[[C0_EPI]]][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<128x128xf32, #gpu.address_space<workgroup>>
+# DUMPIR:         %[[SUBVIEW:.*]] = memref.subview %{{.*}}[%[[DIM_X_EPI]], %[[DIM_Y_EPI]]] [128, 128] [1, 1] : memref<512x256xf32> to memref<128x128xf32, strided<[256, 1], offset: ?>>
+# DUMPIR:         nvgpu.warpgroup.mma.store %[[CONS_LOOP]]#0, %[[VIEW_EPI]] : <fragmented = vector<128x128xf32>> to memref<128x128xf32, #gpu.address_space<workgroup>>
+# DUMPIR:         gpu.barrier
+# DUMPIR:         %[[C0_STORE:.*]] = arith.constant 0 : index
+# DUMPIR:         %[[C128_STORE:.*]] = arith.constant 128 : index
+# DUMPIR:         %[[C1_STORE:.*]] = arith.constant 1 : index
+# DUMPIR:         scf.for %arg15 = %[[C0_STORE]] to %[[C128_STORE]] step %[[C1_STORE]] {
+# DUMPIR:           %{{.*}} = memref.load %[[VIEW_EPI]][%arg15, %[[TID_EPI]]] : memref<128x128xf32, #gpu.address_space<workgroup>>
+# DUMPIR:           memref.store %{{.*}}, %[[SUBVIEW]][%arg15, %[[TID_EPI]]] : memref<128x128xf32, strided<[256, 1], offset: ?>>
+# DUMPIR:         }
+# DUMPIR:       }
+# DUMPIR:       gpu.terminator
diff --git a/mlir/test/Examples/NVGPU/lit.local.cfg b/mlir/test/Examples/NVGPU/lit.local.cfg
index 689cd252e7a25..cf975dad53db3 100644
--- a/mlir/test/Examples/NVGPU/lit.local.cfg
+++ b/mlir/test/Examples/NVGPU/lit.local.cfg
@@ -1,4 +1,4 @@
 config.unsupported = False
-if not config.enable_cuda_runner or not config.mlir_run_cuda_sm90_tests:
+if not config.enable_cuda_runner:
   config.unsupported = True
   
\ No newline at end of file
diff --git a/mlir/test/Examples/NVGPU/tools/nvdsl.py b/mlir/test/Examples/NVGPU/tools/nvdsl.py
index 90dbb2355e1c8..856107293470d 100644
--- a/mlir/test/Examples/NVGPU/tools/nvdsl.py
+++ b/mlir/test/Examples/NVGPU/tools/nvdsl.py
@@ -9,6 +9,7 @@
 from tools import nvgpucompiler
 
 MLIR_DYNAMIC = -9223372036854775808
+DUMP_ONLY = os.getenv("MLIR_NVDSL_PRINT_IR") == "1"
 
 
 def const(value: int, ty=None):
@@ -84,9 +85,7 @@ def arrive(self, txcount: int = 0, predicate=None):
                 self.mbar_group_op, txcount_op, self.id_op, predicate=predicate
             )
         else:
-            nvgpu.mbarrier_arrive(
-                ir.Type.parse("!nvgpu.mbarrier.token"), self.mbar_group_op, self.id_op
-            )
+            nvgpu.mbarrier_arrive(self.mbar_group_op, self.id_op)
 
     def try_wait(self, phase: bool = False, ticks: int = 10000000):
         ticks_op = const(ticks)
@@ -144,7 +143,9 @@ def create_descriptor(self, device_ptr):
             device_ptr,
         )
         self.tma_descriptor = nvgpu.TmaCreateDescriptorOp(
-            tma_descriptor_ty, device_unranked_memref, map(const, self.tma_box_shape)
+            tma_descriptor_ty,
+            device_unranked_memref,
+            list(map(const, self.tma_box_shape)),
         )
         return self.tma_descriptor.result
 
@@ -156,7 +157,7 @@ def load(self, dest, mbarrier: Mbarriers, coords=[0], predicate=None):
             dest,
             mbarrier.mbar_group_op,
             self.tma_descriptor,
-            coordinates=map(const, coords),
+            coordinates=list(map(const, coords)),
             mbarId=mbarrier.id_op,
             predicate=predicate,
         )
@@ -310,13 +311,10 @@ def decorator(func):
             @functools.wraps(func)
             def wrapper(*args, **kwargs):
                 launch_op = gpu.LaunchOp(
-                    None,
-                    [],
-                    *map(const, grid),
-                    *map(const, block),
-                    dynamicSharedMemorySize=arith.constant(T.i32(), smem),
+                    grid_size=grid,
+                    block_size=block,
+                    dynamic_shared_memory_size=arith.constant(T.i32(), smem),
                 )
-                launch_op.body.blocks.append(*([T.index()] * 12))
                 with ir.InsertionPoint(launch_op.body.blocks[0]):
                     result = func(*args, **kwargs)
                     gpu.terminator()
@@ -334,13 +332,11 @@ def wrapper(*args, **kwargs):
 
             def saveIR(module):
                 """Save generated IR"""
-                if True:  # self.saveIR:
-                    # print(mlir_nvgpu_module)
-                    original_stdout = sys.stdout
-                    with open("nvdsl.mlir", "w") as f:
-                        sys.stdout = f
-                        print(module)
-                        sys.stdout = original_stdout
+                original_stdout = sys.stdout
+                with open("nvdsl.mlir", "w") as f:
+                    sys.stdout = f
+                    print(module)
+                    sys.stdout = original_stdout
 
             def _binary_op(lhs, rhs, op: str, predAtt="") -> "ArithValue":
                 """Generate MLIR's Arith dialects binary operations."""
@@ -429,6 +425,9 @@ def __str__(self):
 
                 # Save IR in a file
                 # saveIR(module)
+                if DUMP_ONLY:
+                    print(module)
+                    return 0
 
                 # Verify the module
                 module.operation.verify()
diff --git a/mlir/test/Examples/NVGPU/tools/nvgpucompiler.py b/mlir/test/Examples/NVGPU/tools/nvgpucompiler.py
index 1c9cc74fcd169..4b661f8df6a9f 100644
--- a/mlir/test/Examples/NVGPU/tools/nvgpucompiler.py
+++ b/mlir/test/Examples/NVGPU/tools/nvgpucompiler.py
@@ -35,9 +35,11 @@ def compile(self, module: ir.Module):
 
     def jit(self, module: ir.Module) -> execution_engine.ExecutionEngine:
         """Wraps the module in a JIT execution engine."""
-        return execution_engine.ExecutionEngine(
+        ee = execution_engine.ExecutionEngine(
             module, opt_level=self.opt_level, shared_libs=self.shared_libs
         )
+        ee.initialize()
+        return ee
 
     def compile_and_jit(self, module: ir.Module) -> execution_engine.ExecutionEngine:
         """Compiles and jits the module."""
diff --git a/mlir/test/Integration/GPU/CUDA/TensorCore/sm80/wmma-matmul-f64.mlir b/mlir/test/Integration/GPU/CUDA/TensorCore/sm80/wmma-matmul-f64.mlir
index a016a60022699..c3dd35bf905db 100644
--- a/mlir/test/Integration/GPU/CUDA/TensorCore/sm80/wmma-matmul-f64.mlir
+++ b/mlir/test/Integration/GPU/CUDA/TensorCore/sm80/wmma-matmul-f64.mlir
@@ -44,6 +44,7 @@ func.func @main() {
   gpu.host_register %2 : memref<*xf64>
   gpu.host_register %20 : memref<*xf64>
   gpu.host_register %33 : memref<*xf64>
+  gpu.host_register %34 : memref<*xf64>
 
   gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %c1, %grid_y = %c1, %grid_z = %c1)
              threads(%tx, %ty, %tz) in (%block_x = %c32, %block_y = %c1, %block_z = %c1) {
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index e063de015506b..a3a0747a71003 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -1752,8 +1752,8 @@ libc_support_library(
 )
 
 libc_support_library(
-    name = "__support_time_linux_clock_conversion",
-    hdrs = ["src/__support/time/linux/clock_conversion.h"],
+    name = "__support_time_clock_conversion",
+    hdrs = ["src/__support/time/clock_conversion.h"],
     target_compatible_with = select({
         "@platforms//os:linux": [],
         "//conditions:default": ["@platforms//:incompatible"],
@@ -1789,8 +1789,8 @@ libc_support_library(
     }),
     deps = [
         ":__support_libc_assert",
+        ":__support_time_clock_conversion",
         ":__support_time_linux_abs_timeout",
-        ":__support_time_linux_clock_conversion",
         ":hdr_time_macros",
     ],
 )
@@ -3064,6 +3064,24 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "__support_math_expm1",
+    hdrs = ["src/__support/math/expm1.h"],
+    deps = [
+        ":__support_fputil_double_double",
+        ":__support_fputil_dyadic_float",
+        ":__support_fputil_except_value_utils",
+        ":__support_fputil_multiply_add",
+        ":__support_fputil_polyeval",
+        ":__support_fputil_rounding_mode",
+        ":__support_fputil_triple_double",
+        ":__support_integer_literals",
+        ":__support_macros_optimization",
+        ":__support_math_common_constants",
+        ":__support_math_exp_constants",
+    ],
+)
+
 libc_support_library(
     name = "__support_range_reduction_double",
     hdrs = [
@@ -3792,15 +3810,7 @@ libc_math_function(
 libc_math_function(
     name = "expm1",
     additional_deps = [
-        ":__support_fputil_double_double",
-        ":__support_fputil_dyadic_float",
-        ":__support_fputil_multiply_add",
-        ":__support_fputil_polyeval",
-        ":__support_fputil_rounding_mode",
-        ":__support_fputil_triple_double",
-        ":__support_integer_literals",
-        ":__support_macros_optimization",
-        ":__support_math_common_constants",
+        ":__support_math_expm1",
     ],
 )