WINDUP-1327: Support ignored files for sources as well as parts of zi…

…p files (#1295)

graph/api/src/main/java/org/jboss/windup/graph/service/GraphService.java

@@ -130,7 +130,7 @@ public Iterable<T> findAllByProperty(final String key, final Object value, final
     }
 
     @Override
-    public Iterable<T> findAllWithoutProperty(final String key, final Object value)
+    public List<T> findAllWithoutProperty(final String key, final Object value)
     {
         return ExecutionStatistics.performBenchmarked("GraphService.findAllWithoutProperty(" + key + ")", () -> {
             //return (List<T>)findAllQuery().traverse(g -> g.hasNot(key).or(g.has(key, P.neq(value)))).toList(type);
@@ -142,7 +142,7 @@ public Iterable<T> findAllWithoutProperty(final String key, final Object value)
     }
 
     @Override
-    public Iterable<T> findAllWithoutProperty(final String key)
+    public List<T> findAllWithoutProperty(final String key)
     {
         return ExecutionStatistics.performBenchmarked("GraphService.findAllWithoutProperty(" + key + ")", () -> {
             return (List<T>)findAllQuery().traverse(g -> g.hasNot(key)).toList(type);

rules-java/api/src/main/java/org/jboss/windup/rules/apps/java/decompiler/AbstractDecompilerOperation.java

@@ -1,10 +1,12 @@
 package org.jboss.windup.rules.apps.java.decompiler;
 
 import java.util.List;
+import java.util.stream.Collectors;
 
 import org.jboss.windup.config.operation.GraphOperation;
 import org.jboss.windup.graph.GraphContext;
 import org.jboss.windup.graph.model.resource.FileModel;
+import org.jboss.windup.graph.model.resource.IgnoredFileModel;
 import org.jboss.windup.graph.service.FileService;
 import org.jboss.windup.graph.service.GraphService;
 import org.jboss.windup.rules.apps.java.model.JavaClassFileModel;
@@ -42,7 +44,9 @@ public void setFilesToDecompile(Iterable<JavaClassFileModel> filesToDecompile)
     private Iterable<JavaClassFileModel> getDefaultFilesToDecompile(GraphContext context)
     {
         GraphService<JavaClassFileModel> classFileService = new GraphService<>(context, JavaClassFileModel.class);
-        return classFileService.findAllWithoutProperty(JavaClassFileModel.SKIP_DECOMPILATION, true);
+        return classFileService.findAllWithoutProperty(JavaClassFileModel.SKIP_DECOMPILATION, true).stream()
+                .filter(fileModel -> !(fileModel instanceof IgnoredFileModel))
+                .collect(Collectors.toList());
     }
 
     protected void setupClassToJavaConnections(GraphContext context, List<String> classFilesPaths, JavaSourceFileModel decompiledJavaFile)

rules-java/api/src/main/java/org/jboss/windup/rules/apps/java/scan/ast/AnalyzeJavaFilesRuleProvider.java

@@ -51,6 +51,7 @@
 import org.jboss.windup.graph.model.TechnologyReferenceModel;
 import org.jboss.windup.graph.model.WindupConfigurationModel;
 import org.jboss.windup.graph.model.resource.FileModel;
+import org.jboss.windup.graph.model.resource.IgnoredFileModel;
 import org.jboss.windup.graph.service.GraphService;
 import org.jboss.windup.graph.service.WindupConfigurationService;
 import org.jboss.windup.reporting.service.ClassificationService;
@@ -123,7 +124,10 @@ public void perform(final GraphRewrite event, EvaluationContext context)
                 final GraphContext graphContext = event.getGraphContext();
                 WindupJavaConfigurationService windupJavaConfigurationService = new WindupJavaConfigurationService(graphContext);
 
-                Iterable<JavaSourceFileModel> allJavaSourceModels = graphContext.service(JavaSourceFileModel.class).findAll();
+                List<JavaSourceFileModel> allJavaSourceModels = graphContext.service(JavaSourceFileModel.class).findAll();
+                allJavaSourceModels = allJavaSourceModels.stream()
+                        .filter(fileModel -> !(fileModel instanceof IgnoredFileModel))
+                        .collect(Collectors.toList());
 
 
                 final Map<ProjectModel, JavaAnalysisBatch> batchMap = new HashMap<>();

rules-java/api/src/main/java/org/jboss/windup/rules/apps/java/scan/operation/AddArchiveReferenceInformation.java

@@ -8,6 +8,7 @@
 import org.jboss.windup.graph.model.ArchiveModel;
 import org.jboss.windup.graph.model.resource.FileModel;
 import org.jboss.windup.graph.service.GraphService;
+import org.jboss.windup.rules.apps.java.service.WindupJavaConfigurationService;
 import org.ocpsoft.rewrite.context.EvaluationContext;
 
 public class AddArchiveReferenceInformation extends AbstractIterationOperation<FileModel>
@@ -36,6 +37,9 @@ public void perform(GraphRewrite event, EvaluationContext context, FileModel fil
         archiveModel.setArchiveName(file.getName());
 
         ApplicationArchiveModel appArchiveModel = GraphService.addTypeToModel(event.getGraphContext(), fileModel, ApplicationArchiveModel.class);
+
+        // This line will cause the file to be marked if it is to be ignored
+        new WindupJavaConfigurationService(event.getGraphContext()).checkIfIgnored(event, fileModel);
         ///appArchiveModel.setApplicationName(file.getName()); // Removed because not used.
     }
 

rules-java/api/src/main/java/org/jboss/windup/rules/apps/java/scan/operation/RecurseDirectoryAndAddFiles.java

@@ -5,6 +5,7 @@
 import org.jboss.windup.config.operation.iteration.AbstractIterationOperation;
 import org.jboss.windup.graph.model.resource.FileModel;
 import org.jboss.windup.graph.service.FileService;
+import org.jboss.windup.rules.apps.java.service.WindupJavaConfigurationService;
 import org.ocpsoft.rewrite.context.EvaluationContext;
 
     /**
@@ -40,15 +41,18 @@ public String toString()
     public void perform(GraphRewrite event, EvaluationContext context, FileModel resourceModel)
     {
         FileService fileModelService = new FileService(event.getGraphContext());
-        recurseAndAddFiles(event, context,  fileModelService, resourceModel);
+        WindupJavaConfigurationService javaConfigurationService = new WindupJavaConfigurationService(event.getGraphContext());
+        recurseAndAddFiles(event, fileModelService, javaConfigurationService, resourceModel);
     }
 
     /**
      * Recurses the given folder and creates the FileModels vertices for the child files to the graph.
      */
-    private void recurseAndAddFiles(GraphRewrite event, EvaluationContext context,
-                FileService fileService, FileModel file)
+    private void recurseAndAddFiles(GraphRewrite event, FileService fileService, WindupJavaConfigurationService javaConfigurationService, FileModel file)
     {
+        if (javaConfigurationService.checkIfIgnored(event, file))
+            return;
+
         String filePath = file.getFilePath();
         File fileReference = new File(filePath);
 
@@ -60,7 +64,7 @@ private void recurseAndAddFiles(GraphRewrite event, EvaluationContext context,
                 for (File reference : subFiles)
                 {
                     FileModel subFile = fileService.createByFilePath(file, reference.getAbsolutePath());
-                    recurseAndAddFiles(event, context, fileService, subFile);
+                    recurseAndAddFiles(event, fileService, javaConfigurationService, subFile);
                 }
             }
         }

rules-java/api/src/main/java/org/jboss/windup/rules/apps/java/scan/operation/UnzipArchiveToOutputFolder.java

@@ -57,6 +57,9 @@ public static UnzipArchiveToOutputFolder unzip()
     @Override
     public void perform(GraphRewrite event, EvaluationContext context, ArchiveModel payload)
     {
+        if (new WindupJavaConfigurationService(event.getGraphContext()).checkIfIgnored(event, payload))
+            return;
+
         LOG.info("Unzipping archive: " + payload.toPrettyString());
         File zipFile = payload.asFile();
 
@@ -189,7 +192,7 @@ private void recurseAndAddFiles(GraphRewrite event, EvaluationContext context,
             FileModel subFileModel = fileService.createByFilePath(parentFileModel, subFile.getAbsolutePath());
 
             // check if this file should be ignored
-            if (checkIfIgnored(event, subFileModel, windupJavaConfigurationService.getIgnoredFileRegexes()))
+            if (windupJavaConfigurationService.checkIfIgnored(event, subFileModel))
                 continue;
 
             numberAdded++;
@@ -237,32 +240,6 @@ private void recurseAndAddFiles(GraphRewrite event, EvaluationContext context,
         }
     }
 
-    /**
-     * Checks if the {@link FileModel#getFilePath()} + {@link FileModel#getFileName()} is ignored by any of the specified regular expressions.
-     */
-    private boolean checkIfIgnored(final GraphRewrite event, FileModel file, List<String> patterns)
-    {
-        boolean ignored = false;
-        if (patterns != null && !patterns.isEmpty())
-        {
-            for (String pattern : patterns)
-            {
-                if (file.getFilePath().matches(pattern))
-                {
-                    IgnoredFileModel ignoredFileModel = GraphService.addTypeToModel(event.getGraphContext(), file, IgnoredFileModel.class);
-                    ignoredFileModel.setIgnoredRegex(pattern);
-                    LOG.info("File/Directory placed in " + file.getFilePath() + " was ignored, because matched [" + pattern + "].");
-                    ignored = true;
-                    break;
-                }
-            }
-        }
-
-        return ignored;
-    }
-
-
-
     private static Path getNonexistentDirForAppArchive(Path tempFolder, String appArchiveName)
     {
         Path appArchiveFolder = Paths.get(tempFolder.toString(), appArchiveName);

rules-java/api/src/main/java/org/jboss/windup/rules/apps/java/scan/provider/DiscoverPropertiesFilesRuleProvider.java

@@ -12,6 +12,7 @@
 import org.jboss.windup.reporting.model.TechnologyTagLevel;
 import org.jboss.windup.reporting.service.TechnologyTagService;
 import org.jboss.windup.rules.apps.java.model.PropertiesModel;
+import org.jboss.windup.rules.apps.java.service.WindupJavaConfigurationService;
 import org.jboss.windup.util.ExecutionStatistics;
 import org.ocpsoft.rewrite.config.ConditionBuilder;
 import org.ocpsoft.rewrite.context.EvaluationContext;
@@ -37,6 +38,9 @@ public ConditionBuilder when()
 
     public void perform(GraphRewrite event, EvaluationContext context, FileModel payload)
     {
+        if (new WindupJavaConfigurationService(event.getGraphContext()).checkIfIgnored(event, payload))
+            return;
+
         ExecutionStatistics.get().begin("DiscoverPropertiesFilesRuleProvider.perform");
         GraphService<PropertiesModel> service = new GraphService<>(event.getGraphContext(), PropertiesModel.class);
         TechnologyTagService technologyTagService = new TechnologyTagService(event.getGraphContext());

rules-java/api/src/main/java/org/jboss/windup/rules/apps/java/service/WindupJavaConfigurationService.java

@@ -2,13 +2,18 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.logging.Logger;
 
 import org.apache.commons.io.FilenameUtils;
+import org.jboss.windup.config.GraphRewrite;
 import org.jboss.windup.graph.GraphContext;
 import org.jboss.windup.graph.model.report.IgnoredFileRegexModel;
+import org.jboss.windup.graph.model.resource.FileModel;
+import org.jboss.windup.graph.model.resource.IgnoredFileModel;
 import org.jboss.windup.graph.service.GraphService;
 import org.jboss.windup.rules.apps.java.model.PackageModel;
 import org.jboss.windup.rules.apps.java.model.WindupJavaConfigurationModel;
+import org.jboss.windup.util.Logging;
 
 /**
  * Provides methods for loading and working with {@link WindupJavaConfigurationModel} objects.
@@ -18,6 +23,7 @@
  */
 public class WindupJavaConfigurationService extends GraphService<WindupJavaConfigurationModel>
 {
+    private static final Logger LOG = Logging.get(WindupJavaConfigurationService.class);
 
     private List<String> ignoredRegexes;
 
@@ -38,6 +44,31 @@ public static synchronized WindupJavaConfigurationModel getJavaConfigurationMode
         return config;
     }
 
+    /**
+     * Checks if the {@link FileModel#getFilePath()} + {@link FileModel#getFileName()} is ignored by any of the specified regular expressions.
+     */
+    public boolean checkIfIgnored(final GraphRewrite event, FileModel file)
+    {
+        List<String> patterns = getIgnoredFileRegexes();
+        boolean ignored = false;
+        if (patterns != null && !patterns.isEmpty())
+        {
+            for (String pattern : patterns)
+            {
+                if (file.getFilePath().matches(pattern))
+                {
+                    IgnoredFileModel ignoredFileModel = GraphService.addTypeToModel(event.getGraphContext(), file, IgnoredFileModel.class);
+                    ignoredFileModel.setIgnoredRegex(pattern);
+                    LOG.info("File/Directory placed in " + file.getFilePath() + " was ignored, because matched [" + pattern + "].");
+                    ignored = true;
+                    break;
+                }
+            }
+        }
+
+        return ignored;
+    }
+
     public List<String> getIgnoredFileRegexes()
     {
         if (ignoredRegexes == null)

rules-java/tests/src/test/java/org/jboss/windup/rules/java/ignore/JavaIgnoreRegexesTest.java

@@ -38,6 +38,7 @@
 import org.jboss.windup.graph.GraphContext;
 import org.jboss.windup.graph.GraphContextFactory;
 import org.jboss.windup.graph.model.ProjectModel;
+import org.jboss.windup.graph.model.report.IgnoredFileRegexModel;
 import org.jboss.windup.graph.model.resource.FileModel;
 import org.jboss.windup.graph.service.GraphService;
 import org.jboss.windup.reporting.config.Hint;
@@ -52,6 +53,7 @@
 import org.jboss.windup.rules.apps.java.scan.ast.AnalyzeJavaFilesRuleProvider;
 import org.jboss.windup.rules.apps.java.scan.provider.FindUnboundJavaReferencesRuleProvider;
 import org.jboss.windup.rules.apps.java.scan.provider.IndexJavaSourceFilesRuleProvider;
+import org.jboss.windup.rules.apps.java.service.WindupJavaConfigurationService;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -86,7 +88,7 @@ public static AddonArchive getDeployment()
     private GraphContextFactory factory;
 
     @Test
-    public void testHintsAndClassificationOperation() throws Exception
+    public void testRegexIgnore() throws Exception
     {
         try (GraphContext context = factory.create(true))
         {
@@ -125,6 +127,11 @@ public void testHintsAndClassificationOperation() throws Exception
                                         new NotPredicate(new EnumeratedRuleProviderPredicate(FindUnboundJavaReferencesRuleProvider.class))
                             );
 
+                IgnoredFileRegexModel ignoredFileRegexModel = new GraphService<IgnoredFileRegexModel>(context, IgnoredFileRegexModel.class).create();
+                ignoredFileRegexModel.setRegex(".*JavaClassTestFile1.*");
+
+                WindupJavaConfigurationService.getJavaConfigurationModel(context).addIgnoredFileRegex(ignoredFileRegexModel);
+
                 WindupConfiguration configuration = new WindupConfiguration()
                             .setGraphContext(context)
                             .setRuleProviderFilter(predicate)
@@ -144,7 +151,7 @@ public void testHintsAndClassificationOperation() throws Exception
                 Iterable<JavaTypeReferenceModel> typeReferences = typeRefService.findAll();
                 Assert.assertTrue(typeReferences.iterator().hasNext());
 
-                Assert.assertEquals(4, provider.getTypeReferences().size());
+                Assert.assertEquals(2, provider.getTypeReferences().size());
                 List<InlineHintModel> hints = Iterators.asList(hintService.findAll());
 
                 boolean foundAddonDep1 = false;
@@ -179,16 +186,16 @@ else if (hint.getHint().contains("util.Iterators"))
                     }
                 }
                 Assert.assertTrue(foundAddonDep1);
-                Assert.assertTrue(foundAddonDep2);
-                Assert.assertTrue(foundIterators);
+                Assert.assertFalse(foundAddonDep2);
+                Assert.assertFalse(foundIterators);
                 Assert.assertTrue(foundCallables);
 
                 List<ClassificationModel> classifications = Iterators.asList(classificationService.findAll());
                 Assert.assertEquals(1, classifications.size());
                 Assert.assertTrue(classifications.get(0).getDescription().contains("JavaClassTestFile"));
 
                 Iterable<FileModel> fileModels = classifications.get(0).getFileModels();
-                Assert.assertEquals(2, Iterators.asList(fileModels).size());
+                Assert.assertEquals(1, Iterators.asList(fileModels).size());
             }
             finally
             {