[RFC] Initial reference pass-plugin in LLVM

clang/test/Misc/Inputs/pypass-plugin.c

@@ -0,0 +1,2 @@
+void f() {}
+void g() {}

clang/test/Misc/lit.local.cfg

@@ -0,0 +1,21 @@
+import os
+import platform
+import sysconfig
+
+# Run end-to-end tests if the reference pass-plugin exists in LLVM
+pypass_plugin = f"pypass-plugin{config.llvm_plugin_ext}"
+pypass_plugin_shlib = os.path.join(config.llvm_libs_dir, pypass_plugin)
+if os.path.exists(pypass_plugin_shlib):
+    config.available_features.add("pypass-plugin")
+
+# Disable ASAN's leak detection for Python tests
+config.environment["ASAN_OPTIONS"] = "detect_leaks=0"
+
+if platform.system() != "Windows":
+    libdir = sysconfig.get_config_var("LIBDIR")
+    dylib = sysconfig.get_config_var("LDLIBRARY")
+    config.substitutions.append(("%libpython", os.path.join(libdir, dylib)))
+
+# FIXME: %llvmshlibdir is broken in standalone builds
+config.substitutions.append(("%plugindir", config.llvm_libs_dir))
+config.suffixes.add(".py")

clang/test/Misc/pypass-plugin-entrypoints.py

@@ -0,0 +1,127 @@
+# REQUIRES: native, system-linux, plugins, pypass-plugin
+
+# Entry-points in default and -O0 pipeline
+#
+# RUN: env LLVM_PYPASS_SCRIPT=%s \
+# RUN: env LLVM_PYPASS_DYLIB=%libpython \
+# RUN:   %clang -fpass-plugin=%plugindir/pypass-plugin%pluginext \
+# RUN:          -o /dev/null -S -emit-llvm %S/Inputs/pypass-plugin.c | FileCheck --check-prefix=EP %s
+#
+# RUN: env LLVM_PYPASS_SCRIPT=%s \
+# RUN: env LLVM_PYPASS_DYLIB=%libpython \
+# RUN:   %clang -fpass-plugin=%plugindir/pypass-plugin%pluginext -flto=full -O0 \
+# RUN:          -o /dev/null -S -emit-llvm %S/Inputs/pypass-plugin.c | FileCheck --check-prefix=EP %s
+#
+# RUN: env LLVM_PYPASS_SCRIPT=%s \
+# RUN: env LLVM_PYPASS_DYLIB=%libpython \
+# RUN:   %clang -fpass-plugin=%plugindir/pypass-plugin%pluginext -flto=thin -O0 \
+# RUN:          -o /dev/null -S -emit-llvm %S/Inputs/pypass-plugin.c | FileCheck --check-prefix=EP %s
+#
+# EP-NOT: PeepholeEPCallback
+# EP-NOT: Optimizer{{.*}}EPCallback
+# EP-NOT: ScalarOptimizer{{.*}}EPCallback
+# EP-NOT: FullLinkTimeOptimization{{.*}}EPCallback
+#
+# EP: PipelineStartEPCallback
+# EP: PipelineEarlySimplificationEPCallback
+# EP: OptimizerEarlyEPCallback
+# EP: OptimizerLastEPCallback
+
+# Entry-points in optimizer pipeline
+#
+# RUN: env LLVM_PYPASS_SCRIPT=%s \
+# RUN: env LLVM_PYPASS_DYLIB=%libpython \
+# RUN:   %clang -fpass-plugin=%plugindir/pypass-plugin%pluginext -O2 \
+# RUN:          -o /dev/null -S -emit-llvm %S/Inputs/pypass-plugin.c | FileCheck --check-prefix=EP-OPT %s
+#
+# RUN: env LLVM_PYPASS_SCRIPT=%s \
+# RUN: env LLVM_PYPASS_DYLIB=%libpython \
+# RUN:   %clang -fpass-plugin=%plugindir/pypass-plugin%pluginext -O2 -flto=full \
+# RUN:          -o /dev/null -S -emit-llvm %S/Inputs/pypass-plugin.c | FileCheck --check-prefix=EP-OPT %s
+#
+# RUN: env LLVM_PYPASS_SCRIPT=%s \
+# RUN: env LLVM_PYPASS_DYLIB=%libpython \
+# RUN:   %clang -fpass-plugin=%plugindir/pypass-plugin%pluginext -O2 -ffat-lto-objects \
+# RUN:          -o /dev/null -S -emit-llvm %S/Inputs/pypass-plugin.c | FileCheck --check-prefix=EP-OPT %s
+#
+# EP-OPT:     PipelineStartEPCallback
+# EP-OPT:     PipelineEarlySimplificationEPCallback
+# EP-OPT:     PeepholeEPCallback
+# EP-OPT:     ScalarOptimizerLateEPCallback
+# EP-OPT:     PeepholeEPCallback
+# EP-OPT:     OptimizerEarlyEPCallback
+# EP-OPT:     VectorizerStartEPCallback
+# EP-OPT:     VectorizerEndEPCallback
+# EP-OPT:     OptimizerLastEPCallback
+
+# FIXME: Thin-LTO does not invoke vectorizer callbacks
+#
+# RUN: env LLVM_PYPASS_SCRIPT=%s \
+# RUN: env LLVM_PYPASS_DYLIB=%libpython \
+# RUN:   %clang -fpass-plugin=%plugindir/pypass-plugin%pluginext -O2 -flto=thin \
+# RUN:          -o /dev/null -S -emit-llvm %S/Inputs/pypass-plugin.c | FileCheck --check-prefix=EP-LTO-THIN %s
+#
+# EP-LTO-THIN:     PipelineStartEPCallback
+# EP-LTO-THIN:     PipelineEarlySimplificationEPCallback
+# EP-LTO-THIN:     PeepholeEPCallback
+# EP-LTO-THIN:     ScalarOptimizerLateEPCallback
+# EP-LTO-THIN:     OptimizerEarlyEPCallback
+# EP-LTO-THIN-NOT: Vectorizer{{.*}}EPCallback
+# EP-LTO-THIN:     OptimizerLastEPCallback
+
+
+def registerPipelineStartEPCallback():
+    """Module pass at the start of the pipeline"""
+    return True
+
+
+def registerPipelineEarlySimplificationEPCallback():
+    """Module pass after basic simplification of input IR"""
+    return True
+
+
+def registerOptimizerEarlyEPCallback():
+    """Module pass before the function optimization pipeline"""
+    return True
+
+
+def registerOptimizerLastEPCallback():
+    """Module pass after the function optimization pipeline"""
+    return True
+
+
+def registerPeepholeEPCallback():
+    """Function pass after each instance of the instruction combiner pass"""
+    return True
+
+
+def registerScalarOptimizerLateEPCallback():
+    """Function pass after most of the main optimizations, but before the last
+    cleanup-ish optimizations"""
+    return True
+
+
+def registerVectorizerStartEPCallback():
+    """Function pass before the vectorizer and other highly target specific
+    optimization passes are executed"""
+    return True
+
+
+def registerVectorizerEndEPCallback():
+    """Function pass after the vectorizer and other highly target specific
+    optimization passes are executed"""
+    return True
+
+
+def registerFullLinkTimeOptimizationEarlyEPCallback():
+    """Module pass at the start of the full LTO pipeline"""
+    return True
+
+
+def registerFullLinkTimeOptimizationLastEPCallback():
+    """Module pass at the end of the full LTO pipeline"""
+    return True
+
+
+def run(input, ctx, stage):
+    print(f"0x{input:016x} {stage}")

clang/test/Misc/pypass-plugin-params.py

@@ -0,0 +1,30 @@
+# REQUIRES: native, system-linux, llvm-dylib, plugins, pypass-plugin
+
+# XFAIL: *
+#
+# RUN: %clang -fpass-plugin=%plugindir/pypass-plugin%pluginext -S -emit-llvm \
+# RUN:        -mllvm -pypass-script=%s \
+# RUN:        -mllvm -pypass-dylib=%libpython \
+# RUN:        -Xclang -fdebug-pass-manager \
+# RUN:        -o /dev/null -S -emit-llvm %S/Inputs/pypass-plugin.c 2>&1 | FileCheck %s
+#
+# CHECK: Unknown command line argument
+
+# Plugin parameters only work with the extra `-Xclang -load -Xclang <path>`
+# RUN: %clang -fpass-plugin=%plugindir/pypass-plugin%pluginext -S -emit-llvm \
+# RUN:        -Xclang -load -Xclang %plugindir/pypass-plugin%pluginext \
+# RUN:        -mllvm -pypass-script=%s \
+# RUN:        -mllvm -pypass-dylib=%libpython \
+# RUN:        -Xclang -fdebug-pass-manager \
+# RUN:        -o /dev/null -S -emit-llvm %S/Inputs/pypass-plugin.c 2>&1 | FileCheck %s
+#
+# CHECK: Running pass: PyPass
+
+
+def registerPipelineEarlySimplificationEPCallback():
+    """Module pass after basic simplification of input IR"""
+    return True
+
+
+def run(input, ctx, stage):
+    print(f"0x{input:016x} {stage}")

clang/test/Misc/pypass-plugin-repl.py

@@ -0,0 +1,17 @@
+# REQUIRES: native, system-linux, llvm-dylib, plugins, pypass-plugin
+
+# RUN: echo "int a = 1;" | \
+# RUN:   env LLVM_PYPASS_SCRIPT=%s \
+# RUN:   env LLVM_PYPASS_DYLIB=%libpython \
+# RUN:     clang-repl -Xcc -fpass-plugin=%plugindir/pypass-plugin%pluginext | FileCheck %s
+#
+# CHECK: PipelineEarlySimplificationEPCallback
+
+
+def registerPipelineEarlySimplificationEPCallback():
+    """Module pass after basic simplification of input IR"""
+    return True
+
+
+def run(input, ctx, stage):
+    print(f"0x{input:016x} {stage}")

clang/test/lit.cfg.py

@@ -279,6 +279,9 @@ def have_host_clang_repl_cuda():
 if not (config.build_shared_libs or config.link_llvm_dylib or config.link_clang_dylib):
     config.available_features.add("static-libs")
 
+if config.link_llvm_dylib:
+    config.available_features.add("llvm-dylib")
+
 # Plugins (loadable modules)
 if config.has_plugins and config.llvm_plugin_ext:
     config.available_features.add("plugins")

lld/test/CMakeLists.txt

@@ -5,6 +5,7 @@ llvm_canonicalize_cmake_booleans(
   LLVM_ENABLE_ZSTD
   LLVM_ENABLE_LIBXML2
   LLD_DEFAULT_LD_LLD_IS_MINGW
+  LLVM_LINK_LLVM_DYLIB
   LLVM_BUILD_EXAMPLES
   LLVM_ENABLE_PLUGINS
   LLVM_BYE_LINK_INTO_TOOLS

lld/test/ELF/lto/Inputs/pypass-plugin.ll

@@ -0,0 +1,7 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @main() {
+entry:
+  ret i32 0
+}

lld/test/ELF/lto/lit.local.cfg

@@ -0,0 +1,23 @@
+import os
+import platform
+import sysconfig
+
+pypass_plugin = f"pypass-plugin{config.llvm_shlib_ext}"
+pypass_plugin_shlib = os.path.join(config.llvm_shlib_dir, pypass_plugin)
+
+# Run end-to-end tests if the reference pass-plugin exists in LLVM
+if os.path.exists(pypass_plugin_shlib):
+    config.available_features.add("pypass-plugin")
+
+# Disable ASAN's leak detection for Python tests
+config.environment["ASAN_OPTIONS"] = "detect_leaks=0"
+
+if platform.system() != "Windows":
+    libdir = sysconfig.get_config_var("LIBDIR")
+    dylib = sysconfig.get_config_var("LDLIBRARY")
+    config.substitutions.append(("%libpython", os.path.join(libdir, dylib)))
+
+# FIXME: %llvmshlibdir is broken in standalone builds
+config.substitutions.append(("%plugindir", config.llvm_libs_dir))
+config.substitutions.append(("%pluginext", config.llvm_shlib_ext))
+config.suffixes.add(".py")

lld/test/ELF/lto/pypass-plugin-entrypoints.py

@@ -0,0 +1,100 @@
+# REQUIRES: native, system-linux, llvm-dylib, plugins, pypass-plugin
+
+# Entry-points in pipeline for regular/monolithic LTO
+#
+# RUN: opt %S/Inputs/pypass-plugin.ll -o %t.o
+#
+# RUN: env LLVM_PYPASS_SCRIPT=%s \
+# RUN: env LLVM_PYPASS_DYLIB=%libpython \
+# RUN:   ld.lld --load-pass-plugin=%plugindir/pypass-plugin%pluginext %t.o \
+# RUN:           -shared -o /dev/null | FileCheck --check-prefix=REGULAR %s
+#
+# REGULAR-NOT: PipelineStartEPCallback
+# REGULAR-NOT: PipelineEarlySimplificationEPCallback
+# REGULAR-NOT: PeepholeEPCallback
+# REGULAR-NOT: ScalarOptimizerLateEPCallback
+# REGULAR-NOT: Vectorizer{{.*}}EPCallback
+# REGULAR-NOT: Optimizer{{.*}}EPCallback
+#
+# REGULAR: FullLinkTimeOptimizationEarlyEPCallback
+# REGULAR: FullLinkTimeOptimizationLastEPCallback
+
+# Entry-points in Thin-LTO pipeline
+#
+# RUN: opt --thinlto-bc %S/Inputs/pypass-plugin.ll -o %t_thin1.o
+# RUN: opt -module-summary %S/Inputs/pypass-plugin.ll -o %t_thin2.bc
+#
+# RUN: env LLVM_PYPASS_SCRIPT=%s \
+# RUN: env LLVM_PYPASS_DYLIB=%libpython \
+# RUN:   ld.lld --load-pass-plugin=%plugindir/pypass-plugin%pluginext %t_thin2.bc \
+# RUN:           -shared -o /dev/null | FileCheck --check-prefix=THIN %s
+#
+# THIN-NOT: FullLinkTimeOptimizationEarlyEPCallback
+# THIN-NOT: FullLinkTimeOptimizationLastEPCallback
+# THIN-NOT: PipelineStartEPCallback
+#
+# THIN: PipelineEarlySimplificationEPCallback
+# THIN: PeepholeEPCallback
+# THIN: ScalarOptimizerLateEPCallback
+# THIN: PeepholeEPCallback
+# THIN: OptimizerEarlyEPCallback
+# THIN: VectorizerStartEPCallback
+# THIN: VectorizerEndEPCallback
+# THIN: OptimizerLastEPCallback
+
+
+def registerPipelineStartEPCallback():
+    """Module pass at the start of the pipeline"""
+    return True
+
+
+def registerPipelineEarlySimplificationEPCallback():
+    """Module pass after basic simplification of input IR"""
+    return True
+
+
+def registerOptimizerEarlyEPCallback():
+    """Module pass before the function optimization pipeline"""
+    return True
+
+
+def registerOptimizerLastEPCallback():
+    """Module pass after the function optimization pipeline"""
+    return True
+
+
+def registerPeepholeEPCallback():
+    """Function pass after each instance of the instruction combiner pass"""
+    return True
+
+
+def registerScalarOptimizerLateEPCallback():
+    """Function pass after most of the main optimizations, but before the last
+    cleanup-ish optimizations"""
+    return True
+
+
+def registerVectorizerStartEPCallback():
+    """Function pass before the vectorizer and other highly target specific
+    optimization passes are executed"""
+    return True
+
+
+def registerVectorizerEndEPCallback():
+    """Function pass after the vectorizer and other highly target specific
+    optimization passes are executed"""
+    return True
+
+
+def registerFullLinkTimeOptimizationEarlyEPCallback():
+    """Module pass at the start of the full LTO pipeline"""
+    return True
+
+
+def registerFullLinkTimeOptimizationLastEPCallback():
+    """Module pass at the end of the full LTO pipeline"""
+    return True
+
+
+def run(input, ctx, stage):
+    print(f"0x{input:016x} {stage}")

lld/test/ELF/lto/pypass-plugin-pipeline-parsing.py

@@ -0,0 +1,18 @@
+# REQUIRES: native, system-linux, llvm-dylib, plugins, pypass-plugin
+#
+# RUN: opt %S/Inputs/pypass-plugin.ll -o %t.o
+#
+# RUN: env LLVM_PYPASS_SCRIPT=%s \
+# RUN: env LLVM_PYPASS_DYLIB=%libpython \
+# RUN:   ld.lld --load-pass-plugin=%plugindir/pypass-plugin%pluginext \
+# RUN:          --lto-newpm-passes=pypass %t.o -o /dev/null | FileCheck %s
+#
+# CHECK: 0x{{[0-9a-f]+}} Module
+
+
+def registerModulePipelineParsingCallback():
+    return True
+
+
+def run(input, ctx, stage):
+    print(f"0x{input:016x} {stage}")

lld/test/lit.cfg.py

@@ -143,6 +143,9 @@
 if config.has_plugins:
     config.available_features.add("plugins")
 
+if config.link_llvm_dylib:
+    config.available_features.add("llvm-dylib")
+
 if config.build_examples:
     config.available_features.add("examples")
 

lld/test/lit.site.cfg.py.in

@@ -26,6 +26,7 @@ config.ld_lld_default_mingw = @LLD_DEFAULT_LD_LLD_IS_MINGW@
 config.build_examples = @LLVM_BUILD_EXAMPLES@
 config.has_plugins = @LLVM_ENABLE_PLUGINS@
 config.linked_bye_extension = @LLVM_BYE_LINK_INTO_TOOLS@
+config.link_llvm_dylib = @LLVM_LINK_LLVM_DYLIB@
 config.enable_threads = @LLVM_ENABLE_THREADS@
 
 import lit.llvm