Assertion `isa<X>(Val) && "cast<Ty>() argument of incompatible type!"' failed. when running opt -passes=loop-deletion

[mikaelholmen]

llvm commit: 0365c54
Reproduce with: opt -passes=loop-deletion -o /dev/null bbi-67899.ll

Result:

opt: ../include/llvm/Support/Casting.h:269: typename cast_retty<X, Y *>::ret_type llvm::cast(Y *) [X = llvm::Argument, Y = llvm::Value]: Assertion `isa<X>(Val) && "cast<Ty>() argument of incompatible type!"' failed.
PLEASE submit a bug report to https://github.com/llvm/llvm-project/issues/ and include the crash backtrace.
Stack dump:
0.	Program arguments: build-all/bin/opt -passes=loop-deletion -o /dev/null bbi-67899.ll
 #0 0x0000000002d0f923 llvm::sys::PrintStackTrace(llvm::raw_ostream&, int) (build-all/bin/opt+0x2d0f923)
 #1 0x0000000002d0d59e llvm::sys::RunSignalHandlers() (build-all/bin/opt+0x2d0d59e)
 #2 0x0000000002d0fca6 SignalHandler(int) Signals.cpp:0:0
 #3 0x00007f7638922630 __restore_rt sigaction.c:0:0
 #4 0x00007f7636055387 raise (/lib64/libc.so.6+0x36387)
 #5 0x00007f7636056a78 abort (/lib64/libc.so.6+0x37a78)
 #6 0x00007f763604e1a6 __assert_fail_base (/lib64/libc.so.6+0x2f1a6)
 #7 0x00007f763604e252 (/lib64/libc.so.6+0x2f252)
 #8 0x0000000001b99523 computePointerICmp(llvm::CmpInst::Predicate, llvm::Value*, llvm::Value*, llvm::SimplifyQuery const&) InstructionSimplify.cpp:0:0
 #9 0x0000000001b8165c SimplifyICmpInst(unsigned int, llvm::Value*, llvm::Value*, llvm::SimplifyQuery const&, unsigned int) InstructionSimplify.cpp:0:0
#10 0x0000000002a66e55 getValueOnFirstIteration(llvm::Value*, llvm::DenseMap<llvm::Value*, llvm::Value*, llvm::DenseMapInfo<llvm::Value*, void>, llvm::detail::DenseMapPair<llvm::Value*, llvm::Value*> >&, llvm::SimplifyQuery const&) LoopDeletion.cpp:0:0
#11 0x0000000002a65301 breakBackedgeIfNotTaken(llvm::Loop*, llvm::DominatorTree&, llvm::ScalarEvolution&, llvm::LoopInfo&, llvm::MemorySSA*, llvm::OptimizationRemarkEmitter&) LoopDeletion.cpp:0:0
#12 0x0000000002a63928 llvm::LoopDeletionPass::run(llvm::Loop&, llvm::AnalysisManager<llvm::Loop, llvm::LoopStandardAnalysisResults&>&, llvm::LoopStandardAnalysisResults&, llvm::LPMUpdater&) (build-all/bin/opt+0x2a63928)
#13 0x000000000304447d llvm::detail::PassModel<llvm::Loop, llvm::LoopDeletionPass, llvm::PreservedAnalyses, llvm::AnalysisManager<llvm::Loop, llvm::LoopStandardAnalysisResults&>, llvm::LoopStandardAnalysisResults&, llvm::LPMUpdater&>::run(llvm::Loop&, llvm::AnalysisManager<llvm::Loop, llvm::LoopStandardAnalysisResults&>&, llvm::LoopStandardAnalysisResults&, llvm::LPMUpdater&) crtstuff.c:0:0
#14 0x0000000003534f6f llvm::Optional<llvm::PreservedAnalyses> llvm::PassManager<llvm::Loop, llvm::AnalysisManager<llvm::Loop, llvm::LoopStandardAnalysisResults&>, llvm::LoopStandardAnalysisResults&, llvm::LPMUpdater&>::runSinglePass<llvm::Loop, std::unique_ptr<llvm::detail::PassConcept<llvm::Loop, llvm::AnalysisManager<llvm::Loop, llvm::LoopStandardAnalysisResults&>, llvm::LoopStandardAnalysisResults&, llvm::LPMUpdater&>, std::default_delete<llvm::detail::PassConcept<llvm::Loop, llvm::AnalysisManager<llvm::Loop, llvm::LoopStandardAnalysisResults&>, llvm::LoopStandardAnalysisResults&, llvm::LPMUpdater&> > > >(llvm::Loop&, std::unique_ptr<llvm::detail::PassConcept<llvm::Loop, llvm::AnalysisManager<llvm::Loop, llvm::LoopStandardAnalysisResults&>, llvm::LoopStandardAnalysisResults&, llvm::LPMUpdater&>, std::default_delete<llvm::detail::PassConcept<llvm::Loop, llvm::AnalysisManager<llvm::Loop, llvm::LoopStandardAnalysisResults&>, llvm::LoopStandardAnalysisResults&, llvm::LPMUpdater&> > >&, llvm::AnalysisManager<llvm::Loop, llvm::LoopStandardAnalysisResults&>&, llvm::LoopStandardAnalysisResults&, llvm::LPMUpdater&, llvm::PassInstrumentation&) (build-all/bin/opt+0x3534f6f)
#15 0x0000000003534b93 llvm::PassManager<llvm::Loop, llvm::AnalysisManager<llvm::Loop, llvm::LoopStandardAnalysisResults&>, llvm::LoopStandardAnalysisResults&, llvm::LPMUpdater&>::runWithoutLoopNestPasses(llvm::Loop&, llvm::AnalysisManager<llvm::Loop, llvm::LoopStandardAnalysisResults&>&, llvm::LoopStandardAnalysisResults&, llvm::LPMUpdater&) (build-all/bin/opt+0x3534b93)
#16 0x0000000003534258 llvm::PassManager<llvm::Loop, llvm::AnalysisManager<llvm::Loop, llvm::LoopStandardAnalysisResults&>, llvm::LoopStandardAnalysisResults&, llvm::LPMUpdater&>::run(llvm::Loop&, llvm::AnalysisManager<llvm::Loop, llvm::LoopStandardAnalysisResults&>&, llvm::LoopStandardAnalysisResults&, llvm::LPMUpdater&) (build-all/bin/opt+0x3534258)
#17 0x000000000302798d llvm::detail::PassModel<llvm::Loop, llvm::PassManager<llvm::Loop, llvm::AnalysisManager<llvm::Loop, llvm::LoopStandardAnalysisResults&>, llvm::LoopStandardAnalysisResults&, llvm::LPMUpdater&>, llvm::PreservedAnalyses, llvm::AnalysisManager<llvm::Loop, llvm::LoopStandardAnalysisResults&>, llvm::LoopStandardAnalysisResults&, llvm::LPMUpdater&>::run(llvm::Loop&, llvm::AnalysisManager<llvm::Loop, llvm::LoopStandardAnalysisResults&>&, llvm::LoopStandardAnalysisResults&, llvm::LPMUpdater&) crtstuff.c:0:0
#18 0x000000000353626b llvm::FunctionToLoopPassAdaptor::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (build-all/bin/opt+0x353626b)
#19 0x0000000003043cad llvm::detail::PassModel<llvm::Function, llvm::FunctionToLoopPassAdaptor, llvm::PreservedAnalyses, llvm::AnalysisManager<llvm::Function> >::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) crtstuff.c:0:0
#20 0x000000000244b485 llvm::PassManager<llvm::Function, llvm::AnalysisManager<llvm::Function> >::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (build-all/bin/opt+0x244b485)
#21 0x0000000000af5edd llvm::detail::PassModel<llvm::Function, llvm::PassManager<llvm::Function, llvm::AnalysisManager<llvm::Function> >, llvm::PreservedAnalyses, llvm::AnalysisManager<llvm::Function> >::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) crtstuff.c:0:0
#22 0x000000000244f8aa llvm::ModuleToFunctionPassAdaptor::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) (build-all/bin/opt+0x244f8aa)
#23 0x0000000000af5cbd llvm::detail::PassModel<llvm::Module, llvm::ModuleToFunctionPassAdaptor, llvm::PreservedAnalyses, llvm::AnalysisManager<llvm::Module> >::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) crtstuff.c:0:0
#24 0x000000000244a5d8 llvm::PassManager<llvm::Module, llvm::AnalysisManager<llvm::Module> >::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) (build-all/bin/opt+0x244a5d8)
#25 0x000000000077974f llvm::runPassPipeline(llvm::StringRef, llvm::Module&, llvm::TargetMachine*, llvm::TargetLibraryInfoImpl*, llvm::ToolOutputFile*, llvm::ToolOutputFile*, llvm::ToolOutputFile*, llvm::StringRef, llvm::ArrayRef<llvm::StringRef>, llvm::ArrayRef<llvm::PassPlugin>, llvm::opt_tool::OutputKind, llvm::opt_tool::VerifierKind, bool, bool, bool, bool, bool) (build-all/bin/opt+0x77974f)
#26 0x000000000078a05a main (build-all/bin/opt+0x78a05a)
#27 0x00007f7636041555 __libc_start_main (/lib64/libc.so.6+0x22555)
#28 0x0000000000774780 _start (build-all/bin/opt+0x774780)
Abort

bbi-67899.ll.gz

[mikaelholmen]

Starts crashing with commit ff2e4c0:
[instsimplify] Assume storage for byval args doesn't overlap allocas, globals, or other byval args

This allows us to discharge many pointer comparisons based on byval arguments.

Differential Revision: https://reviews.llvm.org/D120133

[nikic]

The problem here is this code assumes an Instruction or Argument, while it can also be a GlobalVariable:

llvm-project/llvm/lib/Analysis/InstructionSimplify.cpp

Lines 2695 to 2699 in c204cee

	auto *F = [](Value *V) {
	if (auto *I = dyn_cast<Instruction>(V))
	return I->getFunction();
	return cast<Argument>(V)->getParent();
	}(LHS);

And the reason why this doesn't happen more commonly is this code:

llvm-project/llvm/lib/Analysis/InstructionSimplify.cpp

Lines 3544 to 3551 in c204cee

	if (Constant *CLHS = dyn_cast<Constant>(LHS)) {
	if (Constant *CRHS = dyn_cast<Constant>(RHS))
	return ConstantFoldCompareInstOperands(Pred, CLHS, CRHS, Q.DL, Q.TLI);
	// If we have a constant, make sure it is on the RHS.
	std::swap(LHS, RHS);
	Pred = CmpInst::getSwappedPredicate(Pred);
	}

So we need the particular interaction where the original inputs to the icmp are both instructions, but one becomes a GlobalVariable after pointer cast stripping. Normally (e.g. when trying to reproduce via -instsimplify the bitcast will be converted into a constant expression first, gets canonicalized to the right and we don't hit this issue.)

[nikic]

Slightly cleaned up test case:

@g = external global i32

define void @test(i32* byval(i32) %arg) {
  br label %for.cond

for.cond:
  %cast = bitcast i32* @g to i32*
  %cmp = icmp eq i32* %cast, %arg
  br i1 %cmp, label %for.end, label %for.cond

for.end:
  ret void
}