From c9041fceaf423a1c0b509824ea2d58d4b6d78093 Mon Sep 17 00:00:00 2001
From: kstamatis <kstamatis@ekt.gr>
Date: Fri, 4 Oct 2013 23:30:57 +0300
Subject: [PATCH] Support v0.9 of BTE

---
 dspace-api/pom.xml                            |  17 +-
 .../app/itemimport/DataLoaderService.java     |  25 +-
 .../org/dspace/app/itemimport/ItemImport.java | 102 +++---
 dspace/config/spring/api/bte.xml              | 336 +++++++++---------
 4 files changed, 257 insertions(+), 223 deletions(-)

diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml
index fe07777d263e..c83141474d33 100644
--- a/dspace-api/pom.xml
+++ b/dspace-api/pom.xml
@@ -434,19 +434,14 @@
             <version>1.0</version>
         </dependency>
         <dependency>
-            <groupId>gr.ekt</groupId>
-            <artifactId>biblio-transformation-engine</artifactId>
-            <version>0.82</version>
+            <groupId>gr.ekt.bte</groupId>
+            <artifactId>bte-core</artifactId>
+            <version>0.9</version>
         </dependency>
         <dependency>
-            <groupId>net.sf.opencsv</groupId>
-            <artifactId>opencsv</artifactId>
-            <version>2.3</version>
-        </dependency>
-        <dependency>
-            <groupId>org.jbibtex</groupId>
-            <artifactId>jbibtex</artifactId>
-            <version>1.0.0</version>
+            <groupId>gr.ekt.bte</groupId>
+            <artifactId>bte-io</artifactId>
+            <version>0.9</version>
         </dependency>
         <dependency>
             <groupId>org.apache.solr</groupId>
diff --git a/dspace-api/src/main/java/org/dspace/app/itemimport/DataLoaderService.java b/dspace-api/src/main/java/org/dspace/app/itemimport/DataLoaderService.java
index 99cb0f9f95e1..30356c07994e 100644
--- a/dspace-api/src/main/java/org/dspace/app/itemimport/DataLoaderService.java
+++ b/dspace-api/src/main/java/org/dspace/app/itemimport/DataLoaderService.java
@@ -7,11 +7,11 @@
  */
 package org.dspace.app.itemimport;
 
+import gr.ekt.bte.core.DataLoader;
+
 import java.util.HashMap;
 import java.util.Map;
 
-import gr.ekt.transformationengine.core.DataLoader;
-
 
 
 
@@ -22,6 +22,7 @@ public class DataLoaderService
 {
 
     Map<String, DataLoader> dataLoaders = new HashMap<String, DataLoader>();
+    Map<String, String> outputMap = new HashMap<String,String>();
     
     /**
      * Default constructor
@@ -48,5 +49,21 @@ public Map<String, DataLoader> getDataLoaders()
     {
         return dataLoaders;
     }
-    
-}
+
+    /**
+     * 
+     * @return the outputMapping
+     */
+	public Map<String, String> getOutputMap() {
+		return outputMap;
+	}
+
+	/**
+	 * Setter method for the outputMapping
+	 * @param outputMapping
+	 */
+	public void setOutputMap(Map<String, String> outputMap) {
+		this.outputMap = outputMap;
+	}
+  
+}
\ No newline at end of file
diff --git a/dspace-api/src/main/java/org/dspace/app/itemimport/ItemImport.java b/dspace-api/src/main/java/org/dspace/app/itemimport/ItemImport.java
index dce235d9497e..9f8bf4b75dfe 100644
--- a/dspace-api/src/main/java/org/dspace/app/itemimport/ItemImport.java
+++ b/dspace-api/src/main/java/org/dspace/app/itemimport/ItemImport.java
@@ -7,13 +7,12 @@
  */
 package org.dspace.app.itemimport;
 
-import gr.ekt.transformationengine.core.DataLoader;
-import gr.ekt.transformationengine.core.TransformationEngine;
-import gr.ekt.transformationengine.exceptions.UnimplementedAbstractMethod;
-import gr.ekt.transformationengine.exceptions.UnknownClassifierException;
-import gr.ekt.transformationengine.exceptions.UnknownInputFileType;
-import gr.ekt.transformationengine.exceptions.UnsupportedComparatorMode;
-import gr.ekt.transformationengine.exceptions.UnsupportedCriterion;
+import gr.ekt.bte.core.DataLoader;
+import gr.ekt.bte.core.TransformationEngine;
+import gr.ekt.bte.core.TransformationResult;
+import gr.ekt.bte.core.TransformationSpec;
+import gr.ekt.bte.dataloader.FileDataLoader;
+import gr.ekt.bteio.generators.DSpaceOutputGenerator;
 
 import java.io.*;
 import java.sql.SQLException;
@@ -54,6 +53,7 @@
 import org.dspace.eperson.Group;
 import org.dspace.handle.HandleManager;
 import org.dspace.search.DSIndexer;
+import org.dspace.utils.DSpace;
 import org.dspace.workflow.WorkflowManager;
 import org.dspace.xmlworkflow.XmlWorkflowManager;
 import org.w3c.dom.Document;
@@ -62,7 +62,6 @@
 import org.w3c.dom.NodeList;
 import org.xml.sax.SAXException;
 
-import org.dspace.utils.DSpace;
 
 /**
  * Import items into DSpace. The conventional use is upload files by copying
@@ -206,7 +205,7 @@ public static void main(String[] argv) throws Exception
             
             if (line.hasOption('i'))
             {
-                bteInputType = line.getOptionValue('i');;
+                bteInputType = line.getOptionValue('i');
             }
 
             if (line.hasOption('w'))
@@ -314,13 +313,7 @@ else if ("add".equals(command) || "replace".equals(command))
             }
             else if ("add-bte".equals(command))
             {
-            	if (sourcedir == null)
-                {
-                    System.out
-                            .println("Error - a source file containing items must be set");
-                    System.out.println(" (run with -h flag for details)");
-                    System.exit(1);
-                }
+            	//Source dir can be null, the user can specify the parameters for his loader in the Spring XML configuration file
             	
                 if (mapfile == null)
                 {
@@ -638,36 +631,51 @@ private void addBTEItems(Context c, Collection[] mycollections,
 
         DataLoaderService dls  = new DSpace().getSingletonService(DataLoaderService.class);
         DataLoader dataLoader = dls.getDataLoaders().get(inputType);
+        Map<String, String> outputMap = dls.getOutputMap();
 
+        if (dataLoader==null){
+        	System.out.println("ERROR: The key used in -i parameter must match a valid DataLoader in the BTE Spring XML configuration file!");
+        	return;
+        }
+        
+        if (outputMap==null){
+        	System.out.println("ERROR: The key used in -i parameter must match a valid outputMapping in the BTE Spring XML configuration file!");
+        	return;
+        }
+        
+        if (dataLoader instanceof FileDataLoader){
+        	FileDataLoader fdl = (FileDataLoader) dataLoader;
+        	if (!"".equals(sourceDir)){
+        		System.out.println("INFO: Dataloader will load data from the file specified in the command prompt (and not from the Spring XML configuration file)");
+        		fdl.setFilename(sourceDir);
+        	}
+        }
+        
         if (dataLoader!=null){
-            System.out.println("INFO: Dataloader " + dataLoader.toString()+" will be used for the import!");
-            
-            dataLoader.setFileName(sourceDir);
-            te.setDataLoader(dataLoader);
+        	System.out.println("INFO: Dataloader " + dataLoader.toString()+" will be used for the import!");
 
-            try {
-                te.transform();
-            } catch (UnknownClassifierException e) {
-                e.printStackTrace();
-            } catch (UnknownInputFileType e) {
-                e.printStackTrace();
-            } catch (UnimplementedAbstractMethod e) {
-                e.printStackTrace();
-            } catch (UnsupportedComparatorMode e) {
-                e.printStackTrace();
-            } catch (UnsupportedCriterion e) {
-                e.printStackTrace();
-            }
+        	te.setDataLoader(dataLoader);
+        	
+        	DSpaceOutputGenerator outputGenerator = new DSpaceOutputGenerator(outputMap);
+        	outputGenerator.setOutputDirectory("./.bte_output_dspace");
+        	
+        	te.setOutputGenerator(outputGenerator);
 
-            ItemImport myloader = new ItemImport();
-            myloader.addItems(c, mycollections, "./bte_output_dspace", mapFile, template);
 
-            //remove files from output generator
-            deleteDirectory(new File("./bte_output_dspace"));
-        }
-        else {
-            System.out.println("Error: The key used in -i parameter must match a valid DataLoader in the BTE Spring XML configuration file!");
-            return;
+        	try {
+        		TransformationResult res = te.transform(new TransformationSpec());
+        		List<String> output = res.getOutput();
+        		outputGenerator.writeOutput(output);
+        	} catch (Exception e) {
+        		System.err.println("Exception");
+        		e.printStackTrace();
+        	}
+
+        	ItemImport myloader = new ItemImport();
+        	myloader.addItems(c, mycollections, "./.bte_output_dspace", mapFile, template);
+
+        	//remove files from output generator
+        	deleteDirectory(new File("./.bte_output_dspace"));
         }
     }
 
@@ -1546,7 +1554,7 @@ private void registerBitstream(Context c, Item i, int assetstore,
         	// find the bundle
 	        Bundle[] bundles = i.getBundles(newBundleName);
 	        Bundle targetBundle = null;
-	            
+
 	        if( bundles.length < 1 )
 	        {
 	            // not found, create a new one
@@ -1557,19 +1565,19 @@ private void registerBitstream(Context c, Item i, int assetstore,
 	            // put bitstreams into first bundle
 	            targetBundle = bundles[0];
 	        }
-	
+
 	        // now add the bitstream
 	        bs = targetBundle.registerBitstream(assetstore, bitstreamPath);
-	
+
 	        // set the name to just the filename
 	        int iLastSlash = bitstreamPath.lastIndexOf('/');
 	        bs.setName(bitstreamPath.substring(iLastSlash + 1));
-	
+
 	        // Identify the format
 	        // FIXME - guessing format guesses license.txt incorrectly as a text file format!
 	        BitstreamFormat bf = FormatIdentifier.guessFormat(c, bs);
 	        bs.setFormat(bf);
-	
+
 	        bs.update();
         }
     }
@@ -1876,4 +1884,4 @@ private static boolean deleteDirectory(File path)
         boolean pathDeleted = path.delete();
         return (pathDeleted);
     }
-}
+}
\ No newline at end of file
diff --git a/dspace/config/spring/api/bte.xml b/dspace/config/spring/api/bte.xml
index dc714f83f510..b061df680c6f 100644
--- a/dspace/config/spring/api/bte.xml
+++ b/dspace/config/spring/api/bte.xml
@@ -3,175 +3,189 @@
 <!-- TEST-SPRING.XML -->
 
 <beans xmlns="http://www.springframework.org/schema/beans"
-	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-	xsi:schemaLocation="http://www.springframework.org/schema/beans
-	   http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+                           http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
 
-	<bean id="org.dspace.app.itemimport.DataLoaderService" class="org.dspace.app.itemimport.DataLoaderService">
-		<property name="dataLoaders">
-			<map>
-				<entry>
-					<key><value>bibtex</value></key>
-					<ref bean="gr.ekt.transformationengine.dataloaders.dspace.BibTexDataLoader"/>
-				</entry>
-				<entry>
-					<key><value>csv</value></key>
-					<ref bean="gr.ekt.transformationengine.dataloaders.dspace.CSVDataLoader"/>
-				</entry>
-				<entry>
-					<key><value>tsv</value></key>
-					<ref bean="gr.ekt.transformationengine.dataloaders.dspace.TSVDataLoader"/>
-				</entry>
-				<entry>
-					<key><value>endnote</value></key>
-					<ref bean="gr.ekt.transformationengine.dataloaders.dspace.EndnoteDataLoader"/>
-				</entry>
-				<entry>
-					<key><value>ris</value></key>
-					<ref bean="gr.ekt.transformationengine.dataloaders.dspace.RISDataLoader"/>
-				</entry>
-			</map>
-		</property>
-	</bean>
+  <bean id="gr.ekt.bte.core.TransformationEngine" class="gr.ekt.bte.core.TransformationEngine">
+    <property name="workflow" ref="linearWorkflow"/>
+  </bean>
 
-	<bean id="gr.ekt.transformationengine.core.TransformationEngine" class="gr.ekt.transformationengine.core.TransformationEngine">
-		<property name="workflow" ref="conjunctionTransformationWorkflow"/>
-		<property name="outputGenerator" ref="outputGenerator" />
-	</bean>
+  <bean id="org.dspace.app.itemimport.DataLoaderService" class="org.dspace.app.itemimport.DataLoaderService">
+    <property name="dataLoaders">
+      <map>
+        <entry>
+          <key><value>bibtex</value></key>
+          <ref bean="gr.ekt.bteio.loaders.BibTeXDataLoader"/>
+        </entry>
+        <entry>
+          <key><value>csv</value></key>
+          <ref bean="gr.ekt.bteio.loaders.CSVDataLoader"/>
+        </entry>
+        <entry>
+          <key><value>tsv</value></key>
+          <ref bean="gr.ekt.bteio.loaders.TSVDataLoader"/>
+        </entry>
+        <entry>
+          <key><value>ris</value></key>
+          <ref bean="gr.ekt.bteio.loaders.RISDataLoader"/>
+        </entry>
+        <!-- <entry>
+             <key><value>endnote</value></key>
+             <ref bean="gr.ekt.bteio.loaders.EndnoteDataLoader"/>
+             </entry> -->
+      </map>
+    </property>
+    
+    <!-- 
+    	The output generator needs a configuration on how to map internal
+    	records to the DSpace metadata fields. The followinf map specifies
+    	this relationship. The value needs to be the label that the record 
+    	internally associates with the specific data and it is specified in
+    	the data loader beans. The key is in the format of 
+    	<schema>.<element>[.<qualifier>] and specified the dspace metadata
+    	field that the value will map to.
+     -->
+    <property name="outputMap">
+      <map>
+        <entry value="Title" key="dc.title"/>
+        <entry value="Type" key="dc.type"/>
+        <entry value="Author" key="dc.contributor.author"/>
+        <entry value="Abstract" key="dc.description.abstract"/>
+        <entry value="Journal" key="dc.source"/>
+        <entry value="Year" key="dc.date"/>
+        <entry value="ISSN" key="dc.identifier"/>
+      </map>
+    </property>
+  </bean>
 
-	<bean id="gr.ekt.transformationengine.dataloaders.dspace.BibTexDataLoader"
-		class="gr.ekt.transformationengine.dataloaders.dspace.BibTexDataLoader"/>
+  <!-- DataLoader beans -->
+  <!--
+      Each dataloader needs a mapping that defines how the input maps to
+      records. Internally every record organizes the data as key-list
+      of values pairs, and in order to be able to recall the values
+      the keys should have distinct names. Each data format has a way
+      to address individual data, and this is the key of the map. The
+      value is the label that the record internally associates with
+      the specific data and is used in the output mapping as well.
+  -->
 
-	<bean id="gr.ekt.transformationengine.dataloaders.dspace.CSVDataLoader" 
-		class="gr.ekt.transformationengine.dataloaders.dspace.CSVDataLoader"/>
-	
-	<bean id="gr.ekt.transformationengine.dataloaders.dspace.EndnoteDataLoader"
-		class="gr.ekt.transformationengine.dataloaders.dspace.EndnoteDataLoader"/>
+  <!--
+      The key of the map is just the BibTeX label (e.g. author, date,
+      pages etc). The value the label that the record internally associates 
+      with the specific data.
+  -->
+  <bean id="gr.ekt.bteio.loaders.BibTeXDataLoader"
+        class="gr.ekt.bteio.loaders.BibTeXDataLoader">
+    <property name="fieldMap">
+      <map>
+        <entry key="title" value="Title"/>
+        <entry key="author" value="Author"/>
+        <entry key="journal" value="Journal"/>
+        <entry key="year" value="Year"/>
+        <entry key="ISSN" value="ISSN"/>
+      </map>
+    </property>
+  </bean>
 
-	<bean id="gr.ekt.transformationengine.dataloaders.dspace.RISDataLoader" 
-		class="gr.ekt.transformationengine.dataloaders.dspace.RISDataLoader"/>
+  <!--
+      Each entry in a CSV is a row, and each column represents the
+      same data in each entry. For example the first column might
+      record the item title, the second the authors etc. The key of
+      the field map is this number. Note that the first column is
+      number 0.
 
-	<bean id="gr.ekt.transformationengine.dataloaders.dspace.TSVDataLoader" 
-		class="gr.ekt.transformationengine.dataloaders.dspace.TSVDataLoader"/>
+      The CSV data loader has these extra parameters that configure
+      its behaviour:
 
+      - skipLines: a number that instructs the reader to ignore the
+      first lines in the input file. Default value: 0
+      - separator: a character that signifies how the values are
+      separated. Default value ','
+      -quoteChar: individual valued could include the separator
+      character. For example if the separator is ',' and there is an
+      abstract, there is a high probability that it will contain
+      commas. If the value is quoted using the quoteChar then
+      separator characters inside it will be ignored. Default value
+      '"'.
+      -valueSeparator: There are cases where a value is the
+      concatenation of multiple values. For instance a list of
+      authors. In this case the CSVDataLoader can split the CSV value
+      into its individual values, if they are separated with
+      valueSeparator. This can be a full java regular
+      expression. Default value: null (the csv value is not used by
+      default).
+  -->
+  <bean id="gr.ekt.bteio.loaders.CSVDataLoader"
+        class="gr.ekt.bteio.loaders.CSVDataLoader">
+    <property name="fieldMap">
+      <map>
+        <entry key="1" value="Title"/>
+        <entry key="0" value="Author"/>
+        <entry key="2" value="Year"/>
+        <entry key="3" value="Journal"/>
+        <entry key="14" value="Abstract"/>
+        <entry key="31" value="ISSN"/>
+        <entry key="38" value="Type"/>
+      </map>
+    </property>
+    <property name="skipLines" value="1"/>
+  </bean>
 
-	<bean id="conjunctionTransformationWorkflow"
-		class="gr.ekt.transformationengine.core.ConjunctionTransformationWorkflow">
-		<property name="steps">
-			<list>
-			</list>
-		</property>
-	</bean>
+  <!--
+      A TSV file is exactly the same as a CSV one, with the difference
+      that the separator is a tab instead of a comma. See the comments
+      in the previous section.
+  -->
+  <bean id="gr.ekt.bteio.loaders.TSVDataLoader"
+        class="gr.ekt.bteio.loaders.CSVDataLoader">
+    <property name="fieldMap">
+      <map>
+        <entry key="7" value="Title"/>
+        <entry key="1" value="Author"/>
+        <entry key="37" value="Year"/>
+        <entry key="8" value="Journal"/>
+        <entry key="19" value="Abstract"/>
+        <entry key="31" value="ISSN"/>
+        <entry key="0" value="Type"/>
+      </map>
+    </property>
+    <!-- This makes the CSV data loader able to load TSV data -->
+    <property name="separator" value="\u0009"/>
+    <property name="skipLines" value="1"/>
+  </bean>
 
-	<bean id="dcMap" class="java.util.HashMap">
-		<constructor-arg>
-			<map>
-				<entry>
-					<key>
-						<value>Title</value>
-					</key>
-					<ref bean="titleDSpaceMetadata" />
-				</entry>
-				<entry>
-					<key>
-						<value>Type</value>
-					</key>
-					<ref bean="typeDSpaceMetadata" />
-				</entry>
-				<entry>
-					<key>
-						<value>Author</value>
-					</key>
-					<ref bean="authorDSpaceMetadata" />
-				</entry>
-				<entry>
-					<key>
-						<value>Journal</value>
-					</key>
-					<ref bean="sourceDSpaceMetadata" />
-				</entry>
-				<entry>
-					<key>
-						<value>Year</value>
-					</key>
-					<ref bean="dateDSpaceMetadata" />
-				</entry>
-				<entry>
-					<key>
-						<value>ISSN</value>
-					</key>
-					<ref bean="issnDSpaceMetadata" />
-				</entry>
-			</map>
-		</constructor-arg>
-	</bean>
+  <!--
+      The map for the RIS data loader has to have as keys the RIS
+      tags that need to be loaded.
+  -->
+  <bean id="gr.ekt.bteio.loaders.RISDataLoader"
+        class="gr.ekt.bteio.loaders.RISDataLoader">
+    <property name="fieldMap">
+      <map>
+        <entry key="T1" value="Title"/>
+        <entry key="AU" value="Author"/>
+        <entry key="SO" value="Journal"/>
+        <entry key="PY" value="Year"/>
+        <entry key="SN" value="ISSN"/>
+        <entry key="PT" value="Type"/>
+        <entry key="AB" value="Abstract"/>
+      </map>
+    </property>
+  </bean>
 
-	<bean id="ektMap" class="java.util.HashMap">
-		<constructor-arg>
-			<map>
-				<entry>
-					<key>
-						<value>Title</value>
-					</key>
-					<ref bean="titleDSpaceMetadata" />
-				</entry>
-				<entry>
-					<key>
-						<value>Type</value>
-					</key>
-					<ref bean="typeDSpaceMetadata" />
-				</entry>
-			</map>
-		</constructor-arg>
-	</bean>
+  <!--<bean id="gr.ekt.bteio.loaders.EndnoteDataLoader"
+      class="gr.ekt.bteio.loaders.EndnoteDataLoader">
+      </bean> -->
 
-	<bean id="outputGenerator"
-		class="gr.ekt.transformationengine.outputGenerators.DSpaceOutputGenerator">
-		<property name="mappings">
-			<map>
-				<entry>
-					<key>
-						<value>dc</value>
-					</key>
-					<ref bean="dcMap" />
-				</entry>
-				<entry>
-					<key>
-						<value>ekt</value>
-					</key>
-					<ref bean="ektMap" />
-				</entry>
-			</map>
-		</property>
-	</bean>
-	<bean id="titleDSpaceMetadata" class="gr.ekt.transformationengine.dspace.DSpaceMetadata">
-		<property name="schema" value="dc" />
-		<property name="element" value="title" />
-	</bean>
-	<bean id="typeDSpaceMetadata" class="gr.ekt.transformationengine.dspace.DSpaceMetadata">
-		<property name="schema" value="dc" />
-		<property name="element" value="type" />
-	</bean>
-	<bean id="authorDSpaceMetadata" class="gr.ekt.transformationengine.dspace.DSpaceMetadata">
-		<property name="schema" value="dc" />
-		<property name="element" value="contributor" />
-		<property name="qualifier" value="author" />
-	</bean>
-	<bean id="abstractDSpaceMetadata" class="gr.ekt.transformationengine.dspace.DSpaceMetadata">
-		<property name="schema" value="dc" />
-		<property name="element" value="description" />
-		<property name="qualifier" value="abstract" />
-	</bean>
-	<bean id="sourceDSpaceMetadata" class="gr.ekt.transformationengine.dspace.DSpaceMetadata">
-		<property name="schema" value="dc" />
-		<property name="element" value="source" />
-	</bean>
-	<bean id="dateDSpaceMetadata" class="gr.ekt.transformationengine.dspace.DSpaceMetadata">
-		<property name="schema" value="dc" />
-		<property name="element" value="date" />
-	</bean>
-	<bean id="issnDSpaceMetadata" class="gr.ekt.transformationengine.dspace.DSpaceMetadata">
-		<property name="schema" value="dc" />
-		<property name="element" value="identifier" />
-	</bean>
-</beans> 
\ No newline at end of file
+
+  <!-- Specify here any filters or modifiers to run before the output -->
+  <bean id="linearWorkflow"
+        class="gr.ekt.bte.core.LinearWorkflow">
+    <property name="process">
+      <list>
+      </list>
+    </property>
+  </bean>
+
+</beans>
\ No newline at end of file