diff --git a/gradle-plugins/src/main/kotlin/io/opentelemetry/javaagent/muzzle/matcher/MuzzleGradlePluginUtil.kt b/gradle-plugins/src/main/kotlin/io/opentelemetry/javaagent/muzzle/matcher/MuzzleGradlePluginUtil.kt
index 7b4ab0b360b9..f7c3b8039227 100644
--- a/gradle-plugins/src/main/kotlin/io/opentelemetry/javaagent/muzzle/matcher/MuzzleGradlePluginUtil.kt
+++ b/gradle-plugins/src/main/kotlin/io/opentelemetry/javaagent/muzzle/matcher/MuzzleGradlePluginUtil.kt
@@ -57,9 +57,6 @@ class MuzzleGradlePluginUtil {
 
       val matcherClass = agentClassLoader.loadClass("io.opentelemetry.javaagent.tooling.muzzle.ClassLoaderMatcher")
 
-      // We cannot reference Mismatch class directly here, because we are loaded from a different
-      // class loader.
-
       // We cannot reference Mismatch class directly here, because we are loaded from a different
       // class loader.
       val allMismatches = matcherClass
diff --git a/javaagent-extension-api/src/main/java/io/opentelemetry/javaagent/bootstrap/internal/ClassLoaderMatcherCacheHolder.java b/javaagent-extension-api/src/main/java/io/opentelemetry/javaagent/bootstrap/internal/ClassLoaderMatcherCacheHolder.java
index 1c23650bbc7f..583c40d180e8 100644
--- a/javaagent-extension-api/src/main/java/io/opentelemetry/javaagent/bootstrap/internal/ClassLoaderMatcherCacheHolder.java
+++ b/javaagent-extension-api/src/main/java/io/opentelemetry/javaagent/bootstrap/internal/ClassLoaderMatcherCacheHolder.java
@@ -22,11 +22,11 @@
 public final class ClassLoaderMatcherCacheHolder {
 
   @GuardedBy("allCaches")
-  private static final List<Cache<ClassLoader, Boolean>> allCaches = new ArrayList<>();
+  private static final List<Cache<ClassLoader, ?>> allCaches = new ArrayList<>();
 
   private ClassLoaderMatcherCacheHolder() {}
 
-  public static void addCache(Cache<ClassLoader, Boolean> cache) {
+  public static void addCache(Cache<ClassLoader, ?> cache) {
     synchronized (allCaches) {
       allCaches.add(cache);
     }
@@ -34,7 +34,7 @@ public static void addCache(Cache<ClassLoader, Boolean> cache) {
 
   public static void invalidateAllCachesForClassLoader(ClassLoader loader) {
     synchronized (allCaches) {
-      for (Cache<ClassLoader, Boolean> cache : allCaches) {
+      for (Cache<ClassLoader, ?> cache : allCaches) {
         cache.remove(loader);
       }
     }
diff --git a/javaagent-extension-api/src/main/java/io/opentelemetry/javaagent/extension/matcher/ClassLoaderHasClassesNamedMatcher.java b/javaagent-extension-api/src/main/java/io/opentelemetry/javaagent/extension/matcher/ClassLoaderHasClassesNamedMatcher.java
index 368effcd3447..cd6523e2e4bd 100644
--- a/javaagent-extension-api/src/main/java/io/opentelemetry/javaagent/extension/matcher/ClassLoaderHasClassesNamedMatcher.java
+++ b/javaagent-extension-api/src/main/java/io/opentelemetry/javaagent/extension/matcher/ClassLoaderHasClassesNamedMatcher.java
@@ -8,20 +8,29 @@
 import io.opentelemetry.instrumentation.api.internal.cache.Cache;
 import io.opentelemetry.javaagent.bootstrap.internal.ClassLoaderMatcherCacheHolder;
 import io.opentelemetry.javaagent.bootstrap.internal.InClassLoaderMatcher;
+import java.util.BitSet;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicInteger;
 import net.bytebuddy.matcher.ElementMatcher;
 
 class ClassLoaderHasClassesNamedMatcher extends ElementMatcher.Junction.AbstractBase<ClassLoader> {
-
-  private final Cache<ClassLoader, Boolean> cache = Cache.weak();
+  // caching is disabled for build time muzzle checks
+  // this field is set via reflection from ClassLoaderMatcher
+  static boolean useCache = true;
+  private static final AtomicInteger counter = new AtomicInteger();
 
   private final String[] resources;
+  private final int index = counter.getAndIncrement();
 
   ClassLoaderHasClassesNamedMatcher(String... classNames) {
     resources = classNames;
     for (int i = 0; i < resources.length; i++) {
       resources[i] = resources[i].replace(".", "/") + ".class";
     }
-    ClassLoaderMatcherCacheHolder.addCache(cache);
+    if (useCache) {
+      Manager.INSTANCE.add(this);
+    }
   }
 
   @Override
@@ -30,10 +39,14 @@ public boolean matches(ClassLoader cl) {
       // Can't match the bootstrap class loader.
       return false;
     }
-    return cache.computeIfAbsent(cl, this::hasResources);
+    if (useCache) {
+      return Manager.INSTANCE.match(this, cl);
+    } else {
+      return hasResources(cl, resources);
+    }
   }
 
-  private boolean hasResources(ClassLoader cl) {
+  private static boolean hasResources(ClassLoader cl, String... resources) {
     boolean priorValue = InClassLoaderMatcher.getAndSet(true);
     try {
       for (String resource : resources) {
@@ -46,4 +59,41 @@ private boolean hasResources(ClassLoader cl) {
     }
     return true;
   }
+
+  private static class Manager {
+    private static final BitSet EMPTY = new BitSet(0);
+    static final Manager INSTANCE = new Manager();
+    private final List<ClassLoaderHasClassesNamedMatcher> matchers = new CopyOnWriteArrayList<>();
+    private final Cache<ClassLoader, BitSet> enabled = Cache.weak();
+    private volatile boolean matchCalled = false;
+
+    Manager() {
+      ClassLoaderMatcherCacheHolder.addCache(enabled);
+    }
+
+    void add(ClassLoaderHasClassesNamedMatcher matcher) {
+      if (matchCalled) {
+        throw new IllegalStateException("All matchers should be create before match is called");
+      }
+      matchers.add(matcher);
+    }
+
+    boolean match(ClassLoaderHasClassesNamedMatcher matcher, ClassLoader cl) {
+      matchCalled = true;
+      BitSet set = enabled.get(cl);
+      if (set == null) {
+        set = new BitSet(counter.get());
+        for (ClassLoaderHasClassesNamedMatcher m : matchers) {
+          if (hasResources(cl, m.resources)) {
+            set.set(m.index);
+          }
+        }
+        enabled.put(cl, set.isEmpty() ? EMPTY : set);
+        enabled.put(cl, set);
+      } else if (set.isEmpty()) {
+        return false;
+      }
+      return set.get(matcher.index);
+    }
+  }
 }
diff --git a/muzzle/src/main/java/io/opentelemetry/javaagent/tooling/muzzle/ClassLoaderMatcher.java b/muzzle/src/main/java/io/opentelemetry/javaagent/tooling/muzzle/ClassLoaderMatcher.java
index aa1a66ad4555..7b63fe4a96b7 100644
--- a/muzzle/src/main/java/io/opentelemetry/javaagent/tooling/muzzle/ClassLoaderMatcher.java
+++ b/muzzle/src/main/java/io/opentelemetry/javaagent/tooling/muzzle/ClassLoaderMatcher.java
@@ -7,6 +7,7 @@
 
 import io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule;
 import io.opentelemetry.javaagent.tooling.HelperInjector;
+import java.lang.reflect.Field;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -32,6 +33,8 @@ public class ClassLoaderMatcher {
    */
   public static Map<String, List<Mismatch>> matchesAll(
       ClassLoader classLoader, boolean injectHelpers, Set<String> excludedInstrumentationNames) {
+    disableMatcherCache();
+
     Map<String, List<Mismatch>> result = new HashMap<>();
     ServiceLoader.load(InstrumentationModule.class, ClassLoaderMatcher.class.getClassLoader())
         .forEach(
@@ -101,5 +104,19 @@ private static List<Mismatch> checkHelperInjection(
     return mismatches;
   }
 
+  private static void disableMatcherCache() {
+    try {
+      Class<?> matcherClass =
+          Class.forName(
+              "io.opentelemetry.javaagent.extension.matcher.ClassLoaderHasClassesNamedMatcher");
+      Field field = matcherClass.getDeclaredField("useCache");
+      field.setAccessible(true);
+      field.setBoolean(null, false);
+    } catch (Exception exception) {
+      throw new IllegalStateException(
+          "Failed to disable cache for ClassLoaderHasClassesNamedMatcher", exception);
+    }
+  }
+
   private ClassLoaderMatcher() {}
 }