Keep track of public classes in compact filter

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/outline/IsPublicFilter.java

@@ -0,0 +1,11 @@
+package datadog.trace.agent.tooling.bytebuddy.outline;
+
+import datadog.trace.agent.tooling.bytebuddy.ClassCodeFilter;
+import datadog.trace.api.InstrumenterConfig;
+
+/** Compact filter that records public types. */
+final class IsPublicFilter extends ClassCodeFilter {
+  IsPublicFilter() {
+    super(InstrumenterConfig.get().getResolverVisibilitySize());
+  }
+}

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/outline/TypeFactory.java

@@ -70,6 +70,9 @@ final class TypeFactory {
   private static final boolean OUTLINING_ENABLED =
       InstrumenterConfig.get().isResolverOutliningEnabled();
 
+  private static final boolean MEMOIZING_ENABLED =
+      InstrumenterConfig.get().isResolverMemoizingEnabled();
+
   private static final TypeParser outlineTypeParser = new OutlineTypeParser();
 
   private static final TypeParser fullTypeParser = new FullTypeParser();
@@ -83,6 +86,8 @@ final class TypeFactory {
   private static final TypeInfoCache<TypeDescription> fullTypes =
       new TypeInfoCache<>(InstrumenterConfig.get().getResolverTypePoolSize());
 
+  static final IsPublicFilter isPublicFilter = new IsPublicFilter();
+
   /** Small local cache to help deduplicate lookups when matching/transforming. */
   private final DDCache<String, LazyType> deferredTypes = DDCaches.newFixedSizeCache(16);
 
@@ -273,6 +278,12 @@ private TypeDescription lookupType(
 
     InstrumenterMetrics.buildTypeDescription(fromTick, isOutline);
 
+    if (MEMOIZING_ENABLED && null != type) {
+      if (type.isPublic()) {
+        isPublicFilter.add(name);
+      }
+    }
+
     // share result, whether we found it or not
     types.share(name, classLoader, classFile, type);
 
@@ -377,6 +388,11 @@ public TypeDescription getDeclaringType() {
       return outline().getDeclaringType();
     }
 
+    @Override
+    public boolean isPublic() {
+      return isPublicFilter.contains(name) || super.isPublic();
+    }
+
     private TypeDescription outline() {
       if (null != delegate) {
         return delegate; // will be at least an outline, no need to re-resolve

internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java

@@ -350,6 +350,10 @@ public int getResolverNoMatchesSize() {
     return resolverCacheConfig.noMatchesSize();
   }
 
+  public int getResolverVisibilitySize() {
+    return resolverCacheConfig.visibilitySize();
+  }
+
   public boolean isResolverMemoizingEnabled() {
     return resolverCacheConfig.memoPoolSize() > 0;
   }

internal-api/src/main/java/datadog/trace/api/ResolverCacheConfig.java

@@ -10,6 +10,11 @@ public int noMatchesSize() {
       return 65536;
     }
 
+    @Override
+    public int visibilitySize() {
+      return 4096;
+    }
+
     @Override
     public int memoPoolSize() {
       return 4096;
@@ -33,6 +38,11 @@ public int noMatchesSize() {
       return 16384;
     }
 
+    @Override
+    public int visibilitySize() {
+      return 1024;
+    }
+
     @Override
     public int memoPoolSize() {
       return 2048;
@@ -56,6 +66,11 @@ public int noMatchesSize() {
       return 0;
     }
 
+    @Override
+    public int visibilitySize() {
+      return 0;
+    }
+
     @Override
     public int memoPoolSize() {
       return 0;
@@ -79,6 +94,11 @@ public int noMatchesSize() {
       return 0;
     }
 
+    @Override
+    public int visibilitySize() {
+      return 0;
+    }
+
     @Override
     public int memoPoolSize() {
       return 0;
@@ -102,6 +122,11 @@ public int noMatchesSize() {
       return 0;
     }
 
+    @Override
+    public int visibilitySize() {
+      return 0;
+    }
+
     @Override
     public int memoPoolSize() {
       return 0;
@@ -120,6 +145,8 @@ public int typePoolSize() {
 
   public abstract int noMatchesSize();
 
+  public abstract int visibilitySize();
+
   public abstract int memoPoolSize();
 
   public abstract int outlinePoolSize();