diff --git a/src/main/java/emissary/core/IBaseDataObjectHelper.java b/src/main/java/emissary/core/IBaseDataObjectHelper.java index 6310c72c37..96c9a9aefe 100644 --- a/src/main/java/emissary/core/IBaseDataObjectHelper.java +++ b/src/main/java/emissary/core/IBaseDataObjectHelper.java @@ -11,7 +11,6 @@ import java.io.IOException; import java.lang.reflect.Field; -import java.security.NoSuchAlgorithmException; import java.util.Date; import java.util.List; import java.util.Map; @@ -173,11 +172,7 @@ public static void addParentInformationToChild(final IBaseDataObject parentIBase KffDataObjectHandler.parentToChild(childIBaseDataObject); // Hash the new child data, overwrites parent hashes if any - try { - kffDataObjectHandler.hash(childIBaseDataObject, true); - } catch (NoSuchAlgorithmException | IOException e) { - // Do not add the hash parameters - } + kffDataObjectHandler.hash(childIBaseDataObject); } /** diff --git a/src/main/java/emissary/core/channels/SeekableByteChannelHelper.java b/src/main/java/emissary/core/channels/SeekableByteChannelHelper.java index 89dcd1062d..7bc83d1258 100644 --- a/src/main/java/emissary/core/channels/SeekableByteChannelHelper.java +++ b/src/main/java/emissary/core/channels/SeekableByteChannelHelper.java @@ -19,6 +19,9 @@ public final class SeekableByteChannelHelper { private static final Logger logger = LoggerFactory.getLogger(SeekableByteChannelHelper.class); + /** Channel factory backed by an empty byte array. Used for situations when a BDO should have its payload discarded. */ + public static final SeekableByteChannelFactory EMPTY_CHANNEL_FACTORY = memory(new byte[0]); + private SeekableByteChannelHelper() {} /** diff --git a/src/main/java/emissary/kff/KffDataObjectHandler.java b/src/main/java/emissary/kff/KffDataObjectHandler.java index b42cc9fb03..52bbe64bd1 100755 --- a/src/main/java/emissary/kff/KffDataObjectHandler.java +++ b/src/main/java/emissary/kff/KffDataObjectHandler.java @@ -2,6 +2,7 @@ import emissary.core.IBaseDataObject; import emissary.core.channels.SeekableByteChannelFactory; +import emissary.core.channels.SeekableByteChannelHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,51 +71,14 @@ public KffDataObjectHandler(boolean truncateKnownData, boolean setFormOnKnownDat * Compute the configure hashes and return as a map Also include entries indicating the know file or duplicate file * status if so configured * - * @param data the bytes to hash - * @param name th name of the data (for reporting) + * @param sbcf the data to hash + * @param name the name of the data (for reporting) * @return parameter entries suitable for a BaseDataObject + * @throws IOException if the data can't be read + * @throws NoSuchAlgorithmException if the checksum can't be computed */ - public Map hashData(byte[] data, String name) { - return hashData(data, name, ""); - } - - /** - * Compute the configure hashes and return as a map Also include entries indicating the know file or duplicate file - * status if so configured - * - * @param data the bytes to hash - * @param name th name of the data (for reporting) - * @param prefix prepended to hash name entries - * @return parameter entries suitable for a BaseDataObject - */ - public Map hashData(@Nullable byte[] data, String name, @Nullable String prefix) { - Map results = new HashMap(); - - if (prefix == null) { - prefix = ""; - } - - if (data != null && data.length > 0) { - try { - KffResult kffCheck = kff.check(name, data); - - // Store all computed results in data object params - for (String alg : kffCheck.getResultNames()) { - results.put(prefix + KFF_PARAM_BASE + alg, kffCheck.getResultString(alg)); - } - - // Set params if we have a hit - if (kffCheck.isKnown()) { - results.put(prefix + KFF_PARAM_KNOWN_FILTER_NAME, kffCheck.getFilterName()); - } - if (kffCheck.isDupe()) { - results.put(prefix + KFF_PARAM_DUPE_FILTER_NAME, kffCheck.getFilterName()); - } - } catch (Exception kffex) { - logger.warn("Unable to compute kff on " + name, kffex); - } - } - return results; + public Map hashData(final SeekableByteChannelFactory sbcf, final String name) throws NoSuchAlgorithmException, IOException { + return hashData(sbcf, name, ""); } /** @@ -122,7 +86,7 @@ public Map hashData(@Nullable byte[] data, String name, @Nullabl * status if so configured * * @param sbcf the data to hash - * @param name th name of the data (for reporting) + * @param name the name of the data (for reporting) * @param prefix prepended to hash name entries * @return parameter entries suitable for a BaseDataObject * @throws IOException if the data can't be read @@ -166,22 +130,6 @@ public Map hashData(final SeekableByteChannelFactory sbcf, final * @param d the data object */ public void hash(@Nullable final IBaseDataObject d) { - try { - hash(d, false); - } catch (NoSuchAlgorithmException | IOException e) { - // Do nothing - } - } - - /** - * Compute the hash of a data object's data - * - * @param d the data object - * @param useSbc use the {@link SeekableByteChannel} interface - * @throws IOException if the data can't be read - * @throws NoSuchAlgorithmException if the checksum can't be computed - */ - public void hash(@Nullable final IBaseDataObject d, final boolean useSbc) throws NoSuchAlgorithmException, IOException { if (d != null) { removeHash(d); } @@ -190,13 +138,15 @@ public void hash(@Nullable final IBaseDataObject d, final boolean useSbc) throws return; } - // Compute and add the hashes - if (useSbc && d.getChannelSize() > 0) { - d.putParameters(hashData(d.getChannelFactory(), d.shortName(), "")); - } else if (!useSbc && d.dataLength() > 0) { - d.putParameters(hashData(d.data(), d.shortName())); - } else { - return; + try { + // Compute and add the hashes + if (d.getChannelSize() > 0) { + d.putParameters(hashData(d.getChannelFactory(), d.shortName())); + } else { + return; + } + } catch (NoSuchAlgorithmException | IOException e) { + logger.error("Couldn't hash data {}", d.shortName()); } // Set params if we have a hit @@ -208,7 +158,7 @@ public void hash(@Nullable final IBaseDataObject d, final boolean useSbc) throws d.replaceCurrentForm(KFF_DUPE_CURRENT_FORM); } if (truncateKnownData) { - d.setData(null); + d.setChannelFactory(SeekableByteChannelHelper.EMPTY_CHANNEL_FACTORY); } } } diff --git a/src/main/java/emissary/place/KffHashPlace.java b/src/main/java/emissary/place/KffHashPlace.java index 74c2e6719c..a9ca084238 100644 --- a/src/main/java/emissary/place/KffHashPlace.java +++ b/src/main/java/emissary/place/KffHashPlace.java @@ -6,7 +6,6 @@ import java.io.IOException; import java.io.InputStream; -import java.security.NoSuchAlgorithmException; /** * Hashing place to hash payload unless hashes are set or skip flag is set. This place is intended to execute in the @@ -22,8 +21,6 @@ public class KffHashPlace extends ServiceProviderPlace { public static final String SKIP_KFF_HASH = "SKIP_KFF_HASH"; - private boolean useSbc = false; - public KffHashPlace(String thePlaceLocation) throws IOException { super(thePlaceLocation); } @@ -51,7 +48,6 @@ public KffHashPlace(InputStream configStream, String placeLocation) throws IOExc @Override protected void setupPlace(String theDir, String placeLocation) throws IOException { super.setupPlace(theDir, placeLocation); - useSbc = configG.findBooleanEntry("USE_SBC", useSbc); initKff(); } @@ -62,11 +58,7 @@ public void process(IBaseDataObject payload) throws ResourceException { return; } - try { - kff.hash(payload, useSbc); - } catch (final NoSuchAlgorithmException | IOException e) { - logger.error("KffHashPlace failed to hash data for {} - this shouldn't happen", payload.shortName(), e); - } + kff.hash(payload); } } diff --git a/src/main/resources/emissary/place/KffHashPlace.cfg b/src/main/resources/emissary/place/KffHashPlace.cfg index 16029da4d4..9b785fdeb3 100644 --- a/src/main/resources/emissary/place/KffHashPlace.cfg +++ b/src/main/resources/emissary/place/KffHashPlace.cfg @@ -6,6 +6,3 @@ SERVICE_COST = 10 SERVICE_QUALITY = 100 SERVICE_PROXY = "UNKNOWN" - -# Use SeekableByteChannel-related methods (true) vs byte array based (false) -USE_SBC = false \ No newline at end of file diff --git a/src/test/java/emissary/core/IBaseDataObjectHelperTest.java b/src/test/java/emissary/core/IBaseDataObjectHelperTest.java index d4cc8b05ec..a395f596a4 100644 --- a/src/test/java/emissary/core/IBaseDataObjectHelperTest.java +++ b/src/test/java/emissary/core/IBaseDataObjectHelperTest.java @@ -1,6 +1,7 @@ package emissary.core; import emissary.core.channels.InMemoryChannelFactory; +import emissary.core.channels.SeekableByteChannelFactory; import emissary.kff.KffDataObjectHandler; import emissary.parser.SessionParser; @@ -317,7 +318,8 @@ void testAddParentInformationToChild() throws Exception { final IBaseDataObject childIbdo1 = new BaseDataObject(); childIbdo1.setChannelFactory(InMemoryChannelFactory.create("0123456789".getBytes(StandardCharsets.US_ASCII))); - Mockito.doThrow(NoSuchAlgorithmException.class).when(mockKffDataObjectHandler1).hash(Mockito.any(BaseDataObject.class), Mockito.anyBoolean()); + Mockito.doThrow(NoSuchAlgorithmException.class).when(mockKffDataObjectHandler1).hashData(Mockito.any(SeekableByteChannelFactory.class), + Mockito.anyString()); IBaseDataObjectHelper.addParentInformationToChild(parentIbdo, childIbdo1, true, alwaysCopyMetadataKeys, placeKey, mockKffDataObjectHandler1); assertFalse(KffDataObjectHandler.hashPresent(childIbdo1)); diff --git a/src/test/java/emissary/kff/KffDataObjectHandlerTest.java b/src/test/java/emissary/kff/KffDataObjectHandlerTest.java index adcea8f362..04c9994af1 100644 --- a/src/test/java/emissary/kff/KffDataObjectHandlerTest.java +++ b/src/test/java/emissary/kff/KffDataObjectHandlerTest.java @@ -2,6 +2,9 @@ import emissary.core.DataObjectFactory; import emissary.core.IBaseDataObject; +import emissary.core.channels.AbstractSeekableByteChannel; +import emissary.core.channels.SeekableByteChannelFactory; +import emissary.core.channels.SeekableByteChannelHelper; import emissary.test.core.junit5.UnitTest; import emissary.util.io.ResourceReader; @@ -11,17 +14,24 @@ import java.io.IOException; import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; class KffDataObjectHandlerTest extends UnitTest { - static final byte[] DATA = "This is a test".getBytes(); + static final SeekableByteChannelFactory DATA = SeekableByteChannelHelper.memory("This is a test".getBytes()); // echo -n "This is a test" | openssl sha1 static final String DATA_SHA1 = "a54d88e06612d820bc3be72877c74f257b561b19"; @@ -72,19 +82,19 @@ public void tearDown() throws Exception { } @Test - void testMapWithEmptyPrefix() { + void testMapWithEmptyPrefix() throws NoSuchAlgorithmException, IOException { Map m = kff.hashData(DATA, "junk"); assertNotNull(m.get(KffDataObjectHandler.KFF_PARAM_MD5), "Empty prefix returns normal values"); } @Test - void testMapWithNullPrefix() { + void testMapWithNullPrefix() throws NoSuchAlgorithmException, IOException { Map m = kff.hashData(DATA, "junk", null); assertNotNull(m.get(KffDataObjectHandler.KFF_PARAM_MD5), "Null prefix returns normal values"); } @Test - void testMapWithPrefix() { + void testMapWithPrefix() throws NoSuchAlgorithmException, IOException { Map m = kff.hashData(DATA, "name", "foo"); assertNotNull(m.get("foo" + KffDataObjectHandler.KFF_PARAM_MD5), "Prefix prepends on normal key names but we got " + m.keySet()); } @@ -131,4 +141,107 @@ void testSetAndGetHash() { payload.deleteParameter(KffDataObjectHandler.KFF_PARAM_SHA512); assertEquals(DATA_SHA384, KffDataObjectHandler.getBestAvailableHash(payload)); } + + @Test + void testWithChannelFactory() { + kff = new KffDataObjectHandler(true, true, true); + payload.setParameter(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME, "test.filter"); + payload.setChannelFactory(DATA); + kff.hash(payload); + assertEquals("test.filter", payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "FILTERED_BY")); + assertTrue(KffDataObjectHandler.hashPresent(payload)); + assertEquals(DATA_MD5, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_MD5)); + assertEquals(DATA_CRC32, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "CRC32")); + assertEquals(DATA_SSDEEP, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SSDEEP)); + assertEquals(DATA_SHA1, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA1)); + assertEquals(DATA_SHA256, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA256)); + assertEquals(KffDataObjectHandler.KFF_DUPE_CURRENT_FORM, payload.getFileType()); + assertArrayEquals(new byte[0], payload.data()); + } + + @Test + void testWithEmptyChannelFactory() { + kff = new KffDataObjectHandler(true, true, true); + payload.setParameter(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME, "test.filter"); + payload.setChannelFactory(SeekableByteChannelHelper.EMPTY_CHANNEL_FACTORY); + kff.hash(payload); + assertEquals("test.filter", payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "FILTERED_BY")); + assertFalse(KffDataObjectHandler.hashPresent(payload)); + assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_MD5)); + assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "CRC32")); + assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SSDEEP)); + assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA1)); + assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA256)); + assertEquals("test", payload.getFileType()); + assertArrayEquals(new byte[0], payload.data()); + assertEquals(SeekableByteChannelHelper.EMPTY_CHANNEL_FACTORY, payload.getChannelFactory()); + } + + @Test + void testNullPayload() { + assertDoesNotThrow(() -> kff.hash(null)); + } + + @Test + void testRemovingHash() { + final SeekableByteChannelFactory exceptionSbcf = new SeekableByteChannelFactory() { + + @Override + public SeekableByteChannel create() { + return new AbstractSeekableByteChannel() { + + @Override + protected void closeImpl() throws IOException { + // Do nothing + } + + @Override + protected int readImpl(ByteBuffer byteBuffer) throws IOException { + throw new IOException("Test exception"); + } + + @Override + protected long sizeImpl() throws IOException { + throw new IOException("Test exception"); + } + + }; + } + + }; + + payload.setChannelFactory(exceptionSbcf); + kff.hash(payload); + assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "FILTERED_BY")); + assertFalse(KffDataObjectHandler.hashPresent(payload)); + assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_MD5)); + assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "CRC32")); + assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SSDEEP)); + assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA1)); + assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA256)); + assertNotEquals(KffDataObjectHandler.KFF_DUPE_CURRENT_FORM, payload.getFileType()); + + payload.setParameter(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME, "test.filter"); + payload.setChannelFactory(DATA); + kff.hash(payload); + assertEquals("test.filter", payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "FILTERED_BY")); + assertTrue(KffDataObjectHandler.hashPresent(payload)); + assertEquals(DATA_MD5, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_MD5)); + assertEquals(DATA_CRC32, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "CRC32")); + assertEquals(DATA_SSDEEP, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SSDEEP)); + assertEquals(DATA_SHA1, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA1)); + assertEquals(DATA_SHA256, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA256)); + assertEquals(KffDataObjectHandler.KFF_DUPE_CURRENT_FORM, payload.getFileType()); + + } + + @Test + void testNullHashData() throws NoSuchAlgorithmException, IOException { + assertEquals(new HashMap<>(), kff.hashData(null, null)); + } + + @Test + void testEmptySbcf() throws NoSuchAlgorithmException, IOException { + assertEquals(new HashMap<>(), kff.hashData(SeekableByteChannelHelper.EMPTY_CHANNEL_FACTORY, null)); + } }