diff --git a/common/common-core/pom.xml b/common/common-core/pom.xml
index 55e9708862..229ae97f83 100644
--- a/common/common-core/pom.xml
+++ b/common/common-core/pom.xml
@@ -14,18 +14,95 @@
1.4
-
+
+
+
org.bitrepository.reference
bitrepository-core
${bitrepository.version}
+
org.bitrepository.reference
bitrepository-client
${bitrepository.version}
+
+
+
+
+
+
+
org.slf4j
slf4j-api
@@ -55,8 +132,14 @@
jaxen
jaxen
1.1
-
+
+
+ xerces
+ xercesImpl
+
+
+
commons-io
commons-io
@@ -134,6 +217,10 @@
xml-apis
xml-apis
+
+ xerces
+ xercesImpl
+
diff --git a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClient.java b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClient.java
index edb82552ea..7e0c538193 100644
--- a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClient.java
+++ b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClient.java
@@ -29,7 +29,6 @@
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import java.util.regex.Pattern;
@@ -50,11 +49,9 @@
import dk.netarkivet.common.exceptions.NotImplementedException;
import dk.netarkivet.common.exceptions.IllegalState;
import dk.netarkivet.common.exceptions.PermissionDenied;
-import dk.netarkivet.common.utils.ChecksumCalculator;
import dk.netarkivet.common.utils.FileUtils;
import dk.netarkivet.common.utils.Settings;
import dk.netarkivet.common.utils.batch.BatchLocalFiles;
-import dk.netarkivet.common.utils.batch.ChecksumJob;
import dk.netarkivet.common.utils.batch.FileBatchJob;
/**
@@ -71,7 +68,7 @@ public class BitmagArcRepositoryClient implements ArcRepositoryClient {
private static final Logger log = LoggerFactory.getLogger(BitmagArcRepositoryClient.class);
/** The default place in classpath where the settings file can be found. */
- private static String defaultSettingsClasspath = "dk/netarkivet/common/distribute/arcrepository/"
+ private static String defaultSettingsClasspath = "dk/netarkivet/common/distribute/arcrepository/bitrepository/"
+ "BitmagArcRepositoryClientSettings.xml";
/*
@@ -82,7 +79,7 @@ public class BitmagArcRepositoryClient implements ArcRepositoryClient {
Settings.addDefaultClasspathSettings(defaultSettingsClasspath);
}
- private static final String ARCREPOSITORY_TEMPDIR = "settings.common.arcrepositoryClient.bitrepository.tempdir";
+ private static final String BITREPOSITORY_TEMPDIR = "settings.common.arcrepositoryClient.bitrepository.tempdir";
private static final String BITREPOSITORY_SETTINGS_DIR = "settings.common.arcrepositoryClient.bitrepository.settingsDir";
@@ -90,12 +87,26 @@ public class BitmagArcRepositoryClient implements ArcRepositoryClient {
private static final String BITREPOSITORY_STORE_MAX_PILLAR_FAILURES = "settings.common.arcrepositoryClient.bitrepository.storeMaxPillarFailures"; //TODO necessary?
- private static final String BITREPOSITORY_COLLECTIONID = "settings.common.arcrepositoryClient.bitrepository.collectionid";
+ private static final String BITREPOSITORY_COLLECTIONID = "settings.common.arcrepositoryClient.bitrepository.collectionID";
+
+ private final String collectionId;
+
+ private File tempdir;
+
+ private int maxStoreFailures;
+
+ private Bitrepository bitrep;
/** Create a new BitmagArcRepositoryClient based on current settings. */
public BitmagArcRepositoryClient() {
-
+ File configDir = Settings.getFile(BITREPOSITORY_SETTINGS_DIR);
+ File keyfile = Settings.getFile(BITREPOSITORY_KEYFILE);
+ this.collectionId = Settings.get(BITREPOSITORY_COLLECTIONID);
+ this.tempdir = Settings.getFile(BITREPOSITORY_TEMPDIR);
+ this.maxStoreFailures = Settings.getInt(BITREPOSITORY_STORE_MAX_PILLAR_FAILURES);
+ // Initialize connection to the bitrepository
+ this.bitrep = new Bitrepository(configDir, keyfile, maxStoreFailures);
}
@Override
@@ -114,13 +125,19 @@ public void close() {
public void store(File file) throws IOFailure, ArgumentNotValid {
ArgumentNotValid.checkNotNull(file, "File file");
ArgumentNotValid.checkTrue(file.exists(), "File '" + file + "' does not exist");
-
// Check if file already exists
-
- // upload file
-
-
-
+ if (bitrep.existsInCollection(file.getName(), collectionId)) {
+ log.warn("The file '{}' is already in collection '{}'", file.getName(), collectionId);
+ return;
+ } else {
+ // upload file
+ boolean uploadSuccessful = bitrep.uploadFile(file, collectionId);
+ if (!uploadSuccessful) {
+ throw new IOFailure("Upload to collection '" + collectionId + "' of file '" + file.getName() + "' failed.");
+ } else {
+ log.info("Upload to collection '{}' of file '{}' was successfull", collectionId, file.getName());
+ }
+ }
}
/**
@@ -136,40 +153,36 @@ public void store(File file) throws IOFailure, ArgumentNotValid {
public BitarchiveRecord get(String arcfile, long index) throws ArgumentNotValid {
ArgumentNotValid.checkNotNullOrEmpty(arcfile, "String arcfile");
ArgumentNotValid.checkNotNegative(index, "long index");
- //FIXME
- // Initial implementation. fetch file, and retrieve the record on the local file.
- return null;
-/*
- File f = findFile(arcfile);
- if (f == null) {
- log.warn("File '{}' does not exist. Null BitarchiveRecord returned", arcfile);
- return null;
- }
- ArchiveReader reader = null;
- ArchiveRecord record = null;
- try {
- reader = ArchiveReaderFactory.get(f, index);
- record = reader.get();
- return new BitarchiveRecord(record, arcfile);
- } catch (IOException e) {
- throw new IOFailure("Error reading record from '" + arcfile + "' offset " + index, e);
- } finally {
- if (record != null) {
- try {
- record.close();
- } catch (IOException e) {
- log.warn("Error closing ARC record '{}'", record, e);
+ if (!bitrep.existsInCollection(arcfile, collectionId)) {
+ log.warn("The file '{}' is not in collection '{}'. Returning null BitarchiveRecord", arcfile, collectionId);
+ return null;
+ } else {
+ File f = bitrep.getFile(arcfile, collectionId , null);
+ ArchiveReader reader = null;
+ ArchiveRecord record = null;
+ try {
+ reader = ArchiveReaderFactory.get(f, index);
+ record = reader.get();
+ return new BitarchiveRecord(record, arcfile);
+ } catch (IOException e) {
+ throw new IOFailure("Error reading record from '" + arcfile + "' offset " + index, e);
+ } finally {
+ if (record != null) {
+ try {
+ record.close();
+ } catch (IOException e) {
+ log.warn("Error closing ARC record '{}'", record, e);
+ }
}
- }
- if (reader != null) {
- try {
- reader.close();
- } catch (IOException e) {
- log.warn("Error closing ARC reader '{}'", reader, e);
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (IOException e) {
+ log.warn("Error closing ARC reader '{}'", reader, e);
+ }
}
}
}
-*/
}
/**
@@ -186,18 +199,16 @@ record = reader.get();
public void getFile(String arcfilename, Replica replica, File toFile) {
ArgumentNotValid.checkNotNullOrEmpty(arcfilename, "String arcfilename");
ArgumentNotValid.checkNotNull(toFile, "File toFile");
- //FIXME
-/*
- File f = findFile(arcfilename);
- if (f != null) {
- FileUtils.copyFile(f, toFile);
+
+ if (!bitrep.existsInCollection(arcfilename, collectionId)) {
+ log.warn("The file '{}' is not in collection '{}'.", arcfilename, collectionId);
+ return;
} else {
- throw new IOFailure("File '" + arcfilename + "' does not exist");
+ File f = bitrep.getFile(arcfilename, collectionId , null);
+ FileUtils.copyFile(f, toFile);
}
-
-*/
}
-
+
/**
* Runs a batch job on each file in the ArcRepository.
*
diff --git a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/Bitrepository.java b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/Bitrepository.java
index ae95e57aeb..2f60515ab5 100644
--- a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/Bitrepository.java
+++ b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/Bitrepository.java
@@ -102,18 +102,20 @@ public class Bitrepository {
private MessageBus bitMagMessageBus;
/** The maximum number of failing pillars. Default is 0. */
- private int maxNumberOfFailingPillars = 0; //Settings.getInt...
+ private final int maxNumberOfFailingPillars;
/**
* Constructor for the BitRepository class.
* @param configDir A Bitrepository settingsdirectory
+ * @param maxStoreFailures Max number of acceptable store failures
* @param bitmagKeyFile (optional, we hope)
* @throws ArgumentNotValid if configFile is null
*/
- public Bitrepository(File configDir, File bitmagKeyfile) {
+ public Bitrepository(File configDir, File bitmagKeyfile, int maxStoreFailures) {
ArgumentNotValid.checkExistsDirectory(configDir, "File configDir");
//ArgumentNotValid.checkExistsNormalFile(bitmagKeyfile, "File bitmagKeyfile");
componentId = BitrepositoryUtils.generateComponentID();
+ maxNumberOfFailingPillars = maxStoreFailures;
this.settingsDir = configDir;
this.privateKeyFile = bitmagKeyfile;
initBitmagSettings();
@@ -418,7 +420,7 @@ private List getCollectionPillars(String collectionID) {
/**
* Helper method for computing the clientTimeout. The clientTimeout is the identificationTimeout
* plus the OperationTimeout.
- * @param bitmagSettings The bitmagsettingg
+ * @param bitmagSettings The bitmagsetting
* @return the clientTimeout
*/
private long getClientTimeout(Settings bitmagSettings) {
@@ -451,4 +453,28 @@ public List getKnownCollections() {
public ChecksumSpecTYPE getDefaultChecksum() {
return ChecksumUtils.getDefault(bitmagSettings);
}
+
+ //FIXME can this method be made to work?
+ public List getFileIds(String regex, String collectionID) {
+
+ OutputHandler output = new DefaultOutputHandler(Bitrepository.class);
+
+ output.debug("Instantiation GetFileID outputFormatter.");
+ // TODO: change to non pagingClient
+ GetFileIDsOutputFormatter outputFormatter = new GetFileIDsInfoFormatter(output);
+
+ long timeout = getClientTimeout(bitmagSettings);
+
+ output.debug("Instantiation GetFileID paging client.");
+ PagingGetFileIDsClient pagingClient = new PagingGetFileIDsClient(
+ bitMagGetFileIDsClient, timeout, outputFormatter, output);
+
+ Boolean success = pagingClient.getFileIDs(collectionID, "packageId",
+ getCollectionPillars(collectionID));
+ return null;
+ }
+
+
+
+
}
diff --git a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitrepositoryTest3.java b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitrepositoryTest3.java
new file mode 100644
index 0000000000..db2ca20bb3
--- /dev/null
+++ b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitrepositoryTest3.java
@@ -0,0 +1,26 @@
+package dk.netarkivet.common.distribute.arcrepository.bitrepository;
+
+import java.io.File;
+import dk.netarkivet.common.distribute.arcrepository.bitrepository.Bitrepository;
+
+public class BitrepositoryTest3 {
+
+ // FIXME kan kun køres hvis xmlParserAPIs-2.6.2.jar (fra 2003) ikke findes i classpath (skal bruge xercesImpl-2.9.1.jar istedet)
+ // Jf. http://stackoverflow.com/questions/20473689/org-xml-sax-saxnotrecognizedexception-feature-http-javax-xml-xmlconstants-fe
+
+ public static void main(String[] args) {
+
+ //File configDir = new File("/home/svc/bitmag-releasetest-conf");
+ //File bitmagKeyfile = new File(configDir, "client-certificate.pem");
+ //Bitrepository bitrep = new Bitrepository(configDir, bitmagKeyfile);
+ //bitrep.shutdown();
+
+ File configDir = new File("/home/svc/bitrepository-quickstart/commandline/conf");
+ //File bitmagKeyfile = new File(configDir, "client-certificate.pem");
+ Bitrepository bitrep = new Bitrepository(configDir, null, 1);
+ bitrep.shutdown();
+
+ BitmagArcRepositoryClient client = new BitmagArcRepositoryClient();
+
+ }
+}
\ No newline at end of file
diff --git a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository.java b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository.java
index 718803a519..c19b11fdab 100644
--- a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository.java
+++ b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository.java
@@ -17,9 +17,10 @@ public static void main(String[] args) {
File configDir = new File("/home/svc/bitrepository-quickstart/commandline/conf");
//File bitmagKeyfile = new File(configDir, "client-certificate.pem");
- Bitrepository bitrep = new Bitrepository(configDir, null);
+ Bitrepository bitrep = new Bitrepository(configDir, null, 1);
bitrep.shutdown();
+ BitmagArcRepositoryClient client = new BitmagArcRepositoryClient();
}
diff --git a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository2.java b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository2.java
index b37791e4be..5d5a6f0792 100644
--- a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository2.java
+++ b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository2.java
@@ -17,10 +17,14 @@ public static void main(String[] args) {
File configDir = new File("/home/svc/bitrepository-quickstart/commandline/conf");
//File bitmagKeyfile = new File(configDir, "client-certificate.pem");
- Bitrepository bitrep = new Bitrepository(configDir, null);
+ Bitrepository bitrep = new Bitrepository(configDir, null, 1);
bitrep.shutdown();
+
+ BitmagArcRepositoryClient client = new BitmagArcRepositoryClient();
+
+
}
}
diff --git a/common/common-core/src/main/resources/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClientSettings.xml b/common/common-core/src/main/resources/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClientSettings.xml
index 65f86c4dfb..e0ae7ded8c 100644
--- a/common/common-core/src/main/resources/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClientSettings.xml
+++ b/common/common-core/src/main/resources/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClientSettings.xml
@@ -40,7 +40,7 @@ National Library.
1
arcrepositoryTemp
-
+ dvds
diff --git a/common/common-test/src/main/java/TestBitrepository.java b/common/common-test/src/main/java/TestBitrepository.java
index 037f888c1b..09a26f860c 100644
--- a/common/common-test/src/main/java/TestBitrepository.java
+++ b/common/common-test/src/main/java/TestBitrepository.java
@@ -7,7 +7,7 @@ public class TestBitrepository {
public static void main(String[] args) {
File configDir = new File("/home/svc/bitmag-releasetest-conf");
File bitmagKeyfile = new File(configDir, "client-certificate.pem");
- Bitrepository bitrep = new Bitrepository(configDir, bitmagKeyfile);
+ Bitrepository bitrep = new Bitrepository(configDir, bitmagKeyfile, 1);
bitrep.shutdown();
}