From 210b82f602ce85b67c5778499aec1886ea47f98b Mon Sep 17 00:00:00 2001 From: Robert Merget Date: Wed, 21 May 2025 08:44:06 +0400 Subject: [PATCH 1/5] added uuid to scan job descriptions --- .../java/de/rub/nds/crawler/data/ScanJobDescription.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/de/rub/nds/crawler/data/ScanJobDescription.java b/src/main/java/de/rub/nds/crawler/data/ScanJobDescription.java index 841b410..3bd92a7 100644 --- a/src/main/java/de/rub/nds/crawler/data/ScanJobDescription.java +++ b/src/main/java/de/rub/nds/crawler/data/ScanJobDescription.java @@ -12,9 +12,12 @@ import java.io.IOException; import java.io.Serializable; import java.util.Optional; +import java.util.UUID; public class ScanJobDescription implements Serializable { + private final UUID id = UUID.randomUUID(); + private final ScanTarget scanTarget; // Metadata @@ -52,6 +55,10 @@ public ScanJobDescription(ScanTarget scanTarget, BulkScan bulkScan, JobStatus st status); } + public UUID getId() { + return id; + } + private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException { // handle deserialization, cf. https://stackoverflow.com/a/3960558 From c553b6f32c38a15561b2356e28926c65f9503af5 Mon Sep 17 00:00:00 2001 From: Robert Merget Date: Wed, 21 May 2025 09:25:12 +0400 Subject: [PATCH 2/5] Added retrieval functions --- .../persistence/IPersistenceProvider.java | 23 ++++++ .../persistence/MongoPersistenceProvider.java | 75 +++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/src/main/java/de/rub/nds/crawler/persistence/IPersistenceProvider.java b/src/main/java/de/rub/nds/crawler/persistence/IPersistenceProvider.java index 50e3626..2e6fb81 100644 --- a/src/main/java/de/rub/nds/crawler/persistence/IPersistenceProvider.java +++ b/src/main/java/de/rub/nds/crawler/persistence/IPersistenceProvider.java @@ -11,6 +11,7 @@ import de.rub.nds.crawler.data.BulkScan; import de.rub.nds.crawler.data.ScanJobDescription; import de.rub.nds.crawler.data.ScanResult; +import java.util.List; /** * Persistence provider interface. Exposes methods to write out the different stages of a task to a @@ -40,4 +41,26 @@ public interface IPersistenceProvider { * @param bulkScan The bulk scan to update. */ void updateBulkScan(BulkScan bulkScan); + + /** + * Retrieve scan results for a specific target hostname or IP. + * + * @param dbName The database name where the scan results are stored. + * @param collectionName The collection name where the scan results are stored. + * @param target The hostname or IP address to search for. + * @param limit The maximum number of results to retrieve. If null, all results are retrieved. + * @return A list of scan results matching the target. + */ + List getScanResultsByTarget( + String dbName, String collectionName, String target); + + /** + * Retrieve a specific scan result by its ID. + * + * @param dbName The database name where the scan result is stored. + * @param collectionName The collection name where the scan result is stored. + * @param id The ID of the scan result to retrieve. + * @return The scan result, or null if not found. + */ + ScanResult getScanResultById(String dbName, String collectionName, String id); } diff --git a/src/main/java/de/rub/nds/crawler/persistence/MongoPersistenceProvider.java b/src/main/java/de/rub/nds/crawler/persistence/MongoPersistenceProvider.java index 22c0999..a99daaa 100644 --- a/src/main/java/de/rub/nds/crawler/persistence/MongoPersistenceProvider.java +++ b/src/main/java/de/rub/nds/crawler/persistence/MongoPersistenceProvider.java @@ -35,7 +35,9 @@ import java.math.BigDecimal; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.tuple.Pair; @@ -270,4 +272,77 @@ public void insertScanResult(ScanResult scanResult, ScanJobDescription scanJobDe } } } + + @Override + public List getScanResultsByTarget( + String dbName, String collectionName, String target) { + LOGGER.info( + "Retrieving scan results for target {} from collection: {}.{}", + target, + dbName, + collectionName); + + try { + var collection = resultCollectionCache.getUnchecked(Pair.of(dbName, collectionName)); + + // Create a query that matches either hostname or IP + var query = new org.bson.Document(); + var orQuery = new ArrayList(); + orQuery.add(new org.bson.Document("scanTarget.hostname", target)); + orQuery.add(new org.bson.Document("scanTarget.ip", target)); + query.append("$or", orQuery); + + var iterable = collection.find(query); + + + + List results = new ArrayList<>(); + iterable.forEach(results::add); + + LOGGER.info( + "Retrieved {} scan results for target {} from collection: {}.{}", + results.size(), + target, + dbName, + collectionName); + + return results; + } catch (Exception e) { + LOGGER.error("Exception while retrieving scan results from MongoDB: ", e); + throw new RuntimeException("Failed to retrieve scan results for target: " + target, e); + } + } + + @Override + public ScanResult getScanResultById(String dbName, String collectionName, String id) { + LOGGER.info( + "Retrieving scan result with ID {} from collection: {}.{}", + id, + dbName, + collectionName); + + try { + var collection = resultCollectionCache.getUnchecked(Pair.of(dbName, collectionName)); + var result = collection.findOneById(id); + + if (result == null) { + LOGGER.warn( + "No scan result found with ID: {} in collection: {}.{}", + id, + dbName, + collectionName); + } else { + LOGGER.info( + "Retrieved scan result with ID: {} from collection: {}.{}", + id, + dbName, + collectionName); + } + + return result; + } catch (Exception e) { + LOGGER.error("Exception while retrieving scan result from MongoDB: ", e); + throw new RuntimeException("Failed to retrieve scan result with ID: " + id, e); + } + } } From ed36a5036bb31a4a1db64cf313ec887e16dcf560 Mon Sep 17 00:00:00 2001 From: Juan del Carmen Grados Vasquez Date: Wed, 22 Oct 2025 13:22:16 +0400 Subject: [PATCH 3/5] Moving Scanner-Core dependency from scanner-core 5.5.0 to 6.2.3 and fix deserialization issues Version: - Moving Scanner-Core dependency from scanner-core 5.5.0 to 6.2.3 Fixes: 1. Add Jackson deserialization support to ScanResult - Add @JsonCreator annotation to private constructor - Add @JsonProperty annotations for all constructor parameters - Map 'resultStatus' JSON field to 'jobStatus' Java field - Fixes MongoDB deserialization via MongoJack 2. Fix Javadoc compilation error in IPersistenceProvider - Remove @param limit documentation for non-existent parameter 3. Implement missing methods in DummyPersistenceProvider - Add getScanResultsByTarget() returning empty list - Add getScanResultById() returning null - Fixes test compilation after interface updates --- pom.xml | 2 +- .../java/de/rub/nds/crawler/data/ScanResult.java | 8 +++++++- .../crawler/persistence/IPersistenceProvider.java | 4 +--- .../crawler/dummy/DummyPersistenceProvider.java | 15 +++++++++++++++ 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 2ca6ca8..2f88a1a 100644 --- a/pom.xml +++ b/pom.xml @@ -125,7 +125,7 @@ de.rub.nds scanner-core - 5.5.0 + 6.2.3 org.apache.commons diff --git a/src/main/java/de/rub/nds/crawler/data/ScanResult.java b/src/main/java/de/rub/nds/crawler/data/ScanResult.java index ebd5de5..f011984 100644 --- a/src/main/java/de/rub/nds/crawler/data/ScanResult.java +++ b/src/main/java/de/rub/nds/crawler/data/ScanResult.java @@ -8,6 +8,7 @@ */ package de.rub.nds.crawler.data; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import de.rub.nds.crawler.constant.JobStatus; import java.io.Serializable; @@ -26,8 +27,13 @@ public class ScanResult implements Serializable { private final Document result; + @JsonCreator private ScanResult( - String bulkScan, ScanTarget scanTarget, JobStatus jobStatus, Document result) { + @JsonProperty("bulkScan") String bulkScan, + @JsonProperty("scanTarget") ScanTarget scanTarget, + @JsonProperty("resultStatus") + JobStatus jobStatus, // Note: field is 'jobStatus' but JSON is 'resultStatus' + @JsonProperty("result") Document result) { this.id = UUID.randomUUID().toString(); this.bulkScan = bulkScan; this.scanTarget = scanTarget; diff --git a/src/main/java/de/rub/nds/crawler/persistence/IPersistenceProvider.java b/src/main/java/de/rub/nds/crawler/persistence/IPersistenceProvider.java index 2e6fb81..734876c 100644 --- a/src/main/java/de/rub/nds/crawler/persistence/IPersistenceProvider.java +++ b/src/main/java/de/rub/nds/crawler/persistence/IPersistenceProvider.java @@ -48,11 +48,9 @@ public interface IPersistenceProvider { * @param dbName The database name where the scan results are stored. * @param collectionName The collection name where the scan results are stored. * @param target The hostname or IP address to search for. - * @param limit The maximum number of results to retrieve. If null, all results are retrieved. * @return A list of scan results matching the target. */ - List getScanResultsByTarget( - String dbName, String collectionName, String target); + List getScanResultsByTarget(String dbName, String collectionName, String target); /** * Retrieve a specific scan result by its ID. diff --git a/src/test/java/de/rub/nds/crawler/dummy/DummyPersistenceProvider.java b/src/test/java/de/rub/nds/crawler/dummy/DummyPersistenceProvider.java index 9c2bd00..dabd334 100644 --- a/src/test/java/de/rub/nds/crawler/dummy/DummyPersistenceProvider.java +++ b/src/test/java/de/rub/nds/crawler/dummy/DummyPersistenceProvider.java @@ -13,6 +13,7 @@ import de.rub.nds.crawler.data.ScanResult; import de.rub.nds.crawler.persistence.IPersistenceProvider; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; public class DummyPersistenceProvider implements IPersistenceProvider { @@ -31,4 +32,18 @@ public void insertBulkScan(BulkScan bulkScan) { @Override public void updateBulkScan(BulkScan bulkScan) {} + + @Override + public List getScanResultsByTarget( + String dbName, String collectionName, String target) { + return new LinkedList<>(); + } + + @Override + public ScanResult getScanResultById(String dbName, String collectionName, String id) { + return results.stream() + .filter(result -> result.getId().equals(id)) + .findFirst() + .orElse(null); + } } From 6d5328a1cd9551ba5d25e80db652b9f210ffcd94 Mon Sep 17 00:00:00 2001 From: Juan del Carmen Grados Vasquez Date: Wed, 22 Oct 2025 13:25:19 +0400 Subject: [PATCH 4/5] Fixing format issues --- src/main/java/de/rub/nds/crawler/data/ScanResult.java | 2 +- .../rub/nds/crawler/persistence/MongoPersistenceProvider.java | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/de/rub/nds/crawler/data/ScanResult.java b/src/main/java/de/rub/nds/crawler/data/ScanResult.java index f011984..d993422 100644 --- a/src/main/java/de/rub/nds/crawler/data/ScanResult.java +++ b/src/main/java/de/rub/nds/crawler/data/ScanResult.java @@ -32,7 +32,7 @@ private ScanResult( @JsonProperty("bulkScan") String bulkScan, @JsonProperty("scanTarget") ScanTarget scanTarget, @JsonProperty("resultStatus") - JobStatus jobStatus, // Note: field is 'jobStatus' but JSON is 'resultStatus' + JobStatus jobStatus, @JsonProperty("result") Document result) { this.id = UUID.randomUUID().toString(); this.bulkScan = bulkScan; diff --git a/src/main/java/de/rub/nds/crawler/persistence/MongoPersistenceProvider.java b/src/main/java/de/rub/nds/crawler/persistence/MongoPersistenceProvider.java index a99daaa..902b05b 100644 --- a/src/main/java/de/rub/nds/crawler/persistence/MongoPersistenceProvider.java +++ b/src/main/java/de/rub/nds/crawler/persistence/MongoPersistenceProvider.java @@ -294,8 +294,6 @@ public List getScanResultsByTarget( var iterable = collection.find(query); - - List results = new ArrayList<>(); iterable.forEach(results::add); From 921b332160109f99099ee09bbbb8d84f74333d79 Mon Sep 17 00:00:00 2001 From: Juan del Carmen Grados Vasquez Date: Wed, 22 Oct 2025 13:27:30 +0400 Subject: [PATCH 5/5] Fixing format --- src/main/java/de/rub/nds/crawler/data/ScanResult.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/de/rub/nds/crawler/data/ScanResult.java b/src/main/java/de/rub/nds/crawler/data/ScanResult.java index d993422..ffea2a6 100644 --- a/src/main/java/de/rub/nds/crawler/data/ScanResult.java +++ b/src/main/java/de/rub/nds/crawler/data/ScanResult.java @@ -31,8 +31,7 @@ public class ScanResult implements Serializable { private ScanResult( @JsonProperty("bulkScan") String bulkScan, @JsonProperty("scanTarget") ScanTarget scanTarget, - @JsonProperty("resultStatus") - JobStatus jobStatus, + @JsonProperty("resultStatus") JobStatus jobStatus, @JsonProperty("result") Document result) { this.id = UUID.randomUUID().toString(); this.bulkScan = bulkScan;