Fix #346 by adding system to support the different types of external …

…lobs

Implemented the option to add filter modules in between import and export modules when running dbptk:
   DBPTK help now shows available filter module names;
   Using -help <filter name> displays the list of parameters received by that filter;
   Using --filter or -f with the name of one or more filters (separated by ',') injects these filters (in the same order they are written in) into the workflow;
   Passing parameters to the filters is done using --filter<n>-<param long name> or -f<n><param short name> (where 'n' is the position of the filter in the list [starting at 1]).

Implemented base system to support different types of external lobs:
   Added filter module to support handling of external lobs which can be added to the workflow using '--filter external-lobs', this filter requires at least two parameters:
      '--filter<n>-column-list [path to file]' which points to the file containing the list of columns in the database that reference external lobs (this file follows the same syntax as the one exported from the 'list-tables' module;
      '--filter<n>-reference-type [reference type]' which defines the strategy used to fetch the external lobs (ex.: file-system);
   Implemented abstract class ExternalLOBSCellHandler to ease the development of solutions for different types of external lobs by simply initializing a implementation of this class and passing it to the ExternalLOBSFilter.

Implemented solution for external lobs on local file system

dbptk-core/pom.xml

@@ -142,6 +142,11 @@
             <artifactId>dbptk-module-sql-server</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>com.databasepreservation</groupId>
+            <artifactId>dbptk-filter-external-lobs</artifactId>
+        </dependency>
+
         <!-- common -->
         <dependency>
             <groupId>commons-codec</groupId>

dbptk-core/src/main/java/com/databasepreservation/Main.java

@@ -62,7 +62,8 @@ private static Reporter getReporter() {
    *          the console arguments
    */
   public static void main(String[] args) {
-    CLI cli = new CLI(Arrays.asList(args), ReflectionUtils.collectDatabaseModuleFactories());
+    CLI cli = new CLI(Arrays.asList(args), ReflectionUtils.collectDatabaseModuleFactories(),
+      ReflectionUtils.collectDatabaseFilterFactory());
     System.exit(internalMain(cli));
   }
 
@@ -72,7 +73,8 @@ public static int internalMainUsedOnlyByTestClasses(String... args) {
     if (reporter == null) {
       reporter = new NoOpReporter();
     }
-    CLI cli = new CLI(Arrays.asList(args), ReflectionUtils.collectDatabaseModuleFactories());
+    CLI cli = new CLI(Arrays.asList(args), ReflectionUtils.collectDatabaseModuleFactories(),
+      ReflectionUtils.collectDatabaseFilterFactory());
     return internalMain(cli);
   }
 
@@ -124,7 +126,8 @@ private static int run(CLI cli) {
     try {
       databaseMigration = DatabaseMigration.newInstance().importModule(cli.getImportModuleFactory())
         .exportModule(cli.getExportModuleFactory()).importModuleParameters(cli.getImportModuleParameters())
-        .exportModuleParameters(cli.getExportModuleParameters()).reporter(getReporter());
+        .exportModuleParameters(cli.getExportModuleParameters()).filterFactories(cli.getFilterFactories())
+        .filterParameters(cli.getFilterParameters()).reporter(getReporter());
     } catch (ParseException e) {
       LOGGER.error(e.getMessage(), e);
       logProgramFinish(EXIT_CODE_COMMAND_PARSE_ERROR);
@@ -134,6 +137,7 @@ private static int run(CLI cli) {
     // adds a default filter, which for now just does progress logging
     databaseMigration.filter(new ObservableFilter(new ProgressLoggerObserver()));
 
+
     // converts the database using the specified modules, module parameters, and
     // filters
     int exitStatus = EXIT_CODE_GENERIC_ERROR;

dbptk-core/src/test/java/com/databasepreservation/testing/unit/cli/ModuleFactoryTestHelper.java

@@ -74,14 +74,14 @@ private Map<Parameter, String> getModuleArguments(boolean forImportModule, List<
 
       Method getModuleFactories = CLI.class.getDeclaredMethod("getModuleFactories", List.class);
       getModuleFactories.setAccessible(true);
-      CLI.DatabaseModuleFactoriesPair databaseModuleFactoriesPair = (CLI.DatabaseModuleFactoriesPair) getModuleFactories
+      CLI.DatabaseModuleFactories databaseModuleFactories = (CLI.DatabaseModuleFactories) getModuleFactories
         .invoke(cli, args);
 
       Method getModuleArguments = CLI.class.getDeclaredMethod("getModuleArguments",
-        CLI.DatabaseModuleFactoriesPair.class, List.class);
+        CLI.DatabaseModuleFactories.class, List.class);
       getModuleArguments.setAccessible(true);
       CLI.DatabaseModuleFactoriesArguments databaseModuleFactoriesArguments = (CLI.DatabaseModuleFactoriesArguments) getModuleArguments
-        .invoke(cli, databaseModuleFactoriesPair, args);
+        .invoke(cli, databaseModuleFactories, args);
 
       if (forImportModule) {
         return databaseModuleFactoriesArguments.getImportModuleArguments();
@@ -103,10 +103,10 @@ private Map<String, Parameter> getModuleParameters(List<String> args) {
 
       Method getModuleFactories = CLI.class.getDeclaredMethod("getModuleFactories", List.class);
       getModuleFactories.setAccessible(true);
-      CLI.DatabaseModuleFactoriesPair databaseModuleFactoriesPair = (CLI.DatabaseModuleFactoriesPair) getModuleFactories
+      CLI.DatabaseModuleFactories databaseModuleFactories = (CLI.DatabaseModuleFactories) getModuleFactories
         .invoke(cli, args);
 
-      return databaseModuleFactoriesPair.getImportModuleFactory().getAllParameters();
+      return databaseModuleFactories.getImportModuleFactory().getAllParameters();
     } catch (NoSuchMethodException e) {
       throw new RuntimeException(e);
     } catch (InvocationTargetException e) {

dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java

@@ -13,6 +13,7 @@
 import java.util.List;
 import java.util.Map;
 
+import com.databasepreservation.model.modules.filters.DatabaseFilterFactory;
 import org.apache.commons.lang3.tuple.Pair;
 
 import com.databasepreservation.model.Reporter;
@@ -36,6 +37,9 @@ public class DatabaseMigration {
   private DatabaseModuleFactory exportModuleFactory;
   private HashMap<String, String> exportModuleFactoryStringParameters = new HashMap<>();
 
+  private List<DatabaseFilterFactory> filterFactories;
+  private List<HashMap<String, String>> filterFactoriesStringParameters = new ArrayList<>();
+
   private List<DatabaseFilterModule> filterModules = new ArrayList<>();
 
   private DatabaseMigration() {
@@ -58,17 +62,30 @@ public void migrate() throws ModuleException {
     DatabaseImportModule importModule = importModuleFactory.buildImportModule(importParameters, reporter);
     DatabaseExportModule exportModule = exportModuleFactory.buildExportModule(exportParameters, reporter);
 
+    List<DatabaseFilterModule> userDefinedFilterModules = new ArrayList<>();
+    for(int i=0; i<filterFactories.size(); i++) {
+      Map<Parameter, String> filterParameters = buildParametersFromStringParameters(filterFactories.get(i), filterFactoriesStringParameters.get(i));
+      userDefinedFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter));
+    }
+
     // set reporters
     importModule.setOnceReporter(reporter);
+    for (DatabaseFilterModule userDefinedFilterModule : userDefinedFilterModules) {
+      userDefinedFilterModule.setOnceReporter(reporter);
+    }
     for (DatabaseFilterModule filterModule : filterModules) {
       filterModule.setOnceReporter(reporter);
     }
     exportModule.setOnceReporter(reporter);
 
     // create module chain with filters in the middle
     Collections.reverse(filterModules);
+    Collections.reverse(userDefinedFilterModules);
 
     DatabaseExportModule moduleChain = exportModule;
+    for (DatabaseFilterModule userDefinedFilterModule : userDefinedFilterModules) {
+      moduleChain = userDefinedFilterModule.migrateDatabaseTo(moduleChain);
+    }
     for (DatabaseFilterModule filterModule : filterModules) {
       moduleChain = filterModule.migrateDatabaseTo(moduleChain);
     }
@@ -159,6 +176,39 @@ public DatabaseMigration exportModuleParameters(Map<Parameter, String> parameter
     return this;
   }
 
+  /**
+   * Sets the filter factories that will be used to produce the user specified filters
+   */
+  public DatabaseMigration filterFactories(List<DatabaseFilterFactory> filterFactories) {
+    this.filterFactories = filterFactories;
+
+    return this;
+  }
+
+  /**
+   * Adds the specified parameter to be used in the specific filter during the
+   * migration
+   */
+  public DatabaseMigration filterParameter(String parameterLongName, String parameterValue, int filterIndex) {
+    filterFactoriesStringParameters.get(filterIndex).put(parameterLongName, parameterValue);
+    return this;
+  }
+
+  /**
+   * Adds the specified parameters to be used in the export module during the
+   * migration
+   */
+  public DatabaseMigration filterParameters(List<Map<Parameter, String>> parameters) {
+    for(int i=0;i<parameters.size();i++) {
+      filterFactoriesStringParameters.add(new HashMap<String, String>());
+
+      for (Map.Entry<Parameter, String> entry : parameters.get(i).entrySet()) {
+        filterParameter(entry.getKey().longName(), entry.getValue(), i);
+      }
+    }
+    return this;
+  }
+
   /**
    * Adds the specified filter to this database migration. Multiple filters can be
    * added, and the database information will go through them in the same order
@@ -208,4 +258,18 @@ private static Map<Parameter, String> buildParametersFromStringParameters(Databa
 
     return parameters;
   }
+
+  private static Map<Parameter, String> buildParametersFromStringParameters(DatabaseFilterFactory filterFactory,
+    HashMap<String, String> stringFilterFactoryParameters) {
+    Map<Parameter, String> parameters = new HashMap<>();
+
+    for (Map.Entry<String, String> entry : stringFilterFactoryParameters.entrySet()) {
+      Parameter key = filterFactory.getAllParameters().get(entry.getKey());
+      if (key != null) {
+        parameters.put(key, entry.getValue());
+      }
+    }
+
+    return parameters;
+  }
 }

dbptk-model/src/main/java/com/databasepreservation/model/modules/filters/DatabaseFilterFactory.java

@@ -0,0 +1,26 @@
+package com.databasepreservation.model.modules.filters;
+
+import java.util.Map;
+
+import com.databasepreservation.model.Reporter;
+import com.databasepreservation.model.exception.ModuleException;
+import com.databasepreservation.model.parameters.Parameter;
+import com.databasepreservation.model.parameters.Parameters;
+
+/**
+ * Defines a factory used to create Filter Modules. This factory should also be
+ * able to inform the parameters needed to create a new filter module.
+ *
+ * @author Tomás Ferreira <tferreira@keep.pt>
+ */
+public interface DatabaseFilterFactory {
+  String getFilterName();
+
+  boolean isEnabled();
+
+  Parameters getParameters();
+
+  Map<String, Parameter> getAllParameters();
+
+  DatabaseFilterModule buildFilterModule(Map<Parameter, String> parameters, Reporter reporter) throws ModuleException;
+}

dbptk-model/src/main/java/com/databasepreservation/model/structure/type/SimpleTypeBinary.java

@@ -23,6 +23,12 @@ public class SimpleTypeBinary extends Type {
 
   private Integer length;
 
+  /**
+   * True if this binary type was created from a reference to a LOB outside the
+   * database
+   */
+  private boolean isOutsideDatabase;
+
   /**
    * Binary type constructor, with no optional fields. Format registry name and
    * key will be null
@@ -145,4 +151,20 @@ public String toString() {
     return super.toString() + "-->SimpleTypeBinary{" + "formatRegistryName='" + formatRegistryName + '\''
       + ", formatRegistryKey='" + formatRegistryKey + '\'' + ", length=" + length + '}';
   }
+
+  /**
+   * @return True if this binary type was created from a reference to a LOB
+   *         outside the database
+   */
+  public boolean isOutsideDatabase() {
+    return isOutsideDatabase;
+  }
+
+  /**
+   * Set by external lob filter module to indicate that this LOB type was created
+   * from a reference to a LOB outside the database
+   */
+  public void setOutsideDatabase(boolean outsideDatabase) {
+    isOutsideDatabase = outsideDatabase;
+  }
 }

dbptk-model/src/main/java/com/databasepreservation/utils/ReflectionUtils.java

@@ -13,20 +13,24 @@
 import java.util.List;
 import java.util.Set;
 
+import com.databasepreservation.model.modules.filters.DatabaseFilterFactory;
 import org.reflections.Reflections;
 import org.reflections.scanners.SubTypesScanner;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.databasepreservation.model.modules.DatabaseModuleFactory;
 
+import javax.xml.crypto.Data;
+
 /**
  * @author Bruno Ferreira <bferreira@keep.pt>
  */
 public class ReflectionUtils {
   private static Logger LOGGER = LoggerFactory.getLogger(ReflectionUtils.class);
 
   private static List<Constructor<? extends DatabaseModuleFactory>> databaseModuleFactoryConstructors = new ArrayList<>();
+  private static List<Constructor<? extends DatabaseFilterFactory>> databaseFilterFactoryConstructors = new ArrayList<>();
 
   public static Set<DatabaseModuleFactory> collectDatabaseModuleFactories() {
     return collectDatabaseModuleFactories(false);
@@ -64,4 +68,41 @@ public static Set<DatabaseModuleFactory> collectDatabaseModuleFactories(boolean
     return databaseModuleFactories;
   }
 
+  public static List<DatabaseFilterFactory> collectDatabaseFilterFactory() {
+    return collectDatabaseFilterFactory(false);
+  }
+
+  public static List<DatabaseFilterFactory> collectDatabaseFilterFactory(boolean includeDisabled) {
+    List<DatabaseFilterFactory> databaseFilterFactories = new ArrayList<>();
+
+    if (databaseFilterFactoryConstructors.isEmpty()) {
+      Reflections reflections = new Reflections("com.databasepreservation.modules", new SubTypesScanner());
+
+      Set<Class<? extends DatabaseFilterFactory>> filterFactoryClasses = reflections
+              .getSubTypesOf(DatabaseFilterFactory.class);
+
+      for (Class<? extends DatabaseFilterFactory> filterFactoryClass : filterFactoryClasses) {
+        try {
+          Constructor<? extends DatabaseFilterFactory> constructor = filterFactoryClass.getConstructor();
+          databaseFilterFactoryConstructors.add(constructor);
+        } catch (NoSuchMethodException e) {
+          LOGGER.info("Filter factory {} could not be loaded", filterFactoryClass.getName(), e);
+        }
+      }
+    }
+
+    for (Constructor<? extends DatabaseFilterFactory> constructor : databaseFilterFactoryConstructors) {
+      try {
+        DatabaseFilterFactory instance = constructor.newInstance();
+        if (includeDisabled || instance.isEnabled()) {
+          databaseFilterFactories.add(instance);
+        }
+      } catch (Exception e) {
+        LOGGER.info("Filter factory {} could not be loaded", constructor.getDeclaringClass().getName(), e);
+      }
+    }
+
+    return databaseFilterFactories;
+  }
+
 }

dbptk-modules/dbptk-filter-external-lobs/pom.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>dbptk-modules</artifactId>
+        <groupId>com.databasepreservation</groupId>
+        <version>2.2.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <name>dbptk-filter-external-lobs</name>
+    <groupId>com.databasepreservation</groupId>
+    <artifactId>dbptk-filter-external-lobs</artifactId>
+    <version>2.2.0-SNAPSHOT</version>
+
+    <dependencies>
+        <!-- internal dependencies -->
+        <dependency>
+            <groupId>com.databasepreservation</groupId>
+            <artifactId>dbptk-model</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.databasepreservation</groupId>
+            <artifactId>dbptk-module-list-tables</artifactId>
+        </dependency>
+    </dependencies>
+</project>

dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java

@@ -0,0 +1,44 @@
+package com.databasepreservation.modules.externalLobs.CellHandlers;
+
+import com.databasepreservation.model.data.BinaryCell;
+import com.databasepreservation.model.data.Cell;
+import com.databasepreservation.model.exception.ModuleException;
+import com.databasepreservation.modules.externalLobs.ExternalLOBSCellHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+public class ExternalLOBSCellHandlerFileSystem implements ExternalLOBSCellHandler {
+  private static final Logger LOGGER = LoggerFactory.getLogger(ExternalLOBSCellHandlerFileSystem.class);
+  private Path basePath;
+
+  public ExternalLOBSCellHandlerFileSystem() {
+    basePath = null;
+  }
+
+  public ExternalLOBSCellHandlerFileSystem(Path basePath) {
+    this.basePath = basePath;
+  }
+
+  @Override
+  public BinaryCell handleCell(String cellId, String cellValue) throws ModuleException {
+    Path blobPath = basePath.resolve(cellValue);
+    BinaryCell newCell = null;
+
+    try (InputStream stream = Files.newInputStream(blobPath)) {
+      newCell = new BinaryCell(cellId, stream);
+    } catch (IOException e) {
+      LOGGER.debug("Could not open stream to file", e);
+    }
+    return newCell;
+  }
+
+  @Override
+  public String handleTypeDescription(String originalTypeDescription) {
+    return "Converted to LOB referenced by file system path (original description: '"+originalTypeDescription+"')";
+  }
+}