diff --git a/common/common-core/pom.xml b/common/common-core/pom.xml index 55e9708862..229ae97f83 100644 --- a/common/common-core/pom.xml +++ b/common/common-core/pom.xml @@ -14,18 +14,95 @@ 1.4 - + + + org.bitrepository.reference bitrepository-core ${bitrepository.version} + org.bitrepository.reference bitrepository-client ${bitrepository.version} + + + + + + + org.slf4j slf4j-api @@ -55,8 +132,14 @@ jaxen jaxen 1.1 - + + + xerces + xercesImpl + + + commons-io commons-io @@ -134,6 +217,10 @@ xml-apis xml-apis + + xerces + xercesImpl + diff --git a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClient.java b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClient.java index edb82552ea..7e0c538193 100644 --- a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClient.java +++ b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClient.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.regex.Pattern; @@ -50,11 +49,9 @@ import dk.netarkivet.common.exceptions.NotImplementedException; import dk.netarkivet.common.exceptions.IllegalState; import dk.netarkivet.common.exceptions.PermissionDenied; -import dk.netarkivet.common.utils.ChecksumCalculator; import dk.netarkivet.common.utils.FileUtils; import dk.netarkivet.common.utils.Settings; import dk.netarkivet.common.utils.batch.BatchLocalFiles; -import dk.netarkivet.common.utils.batch.ChecksumJob; import dk.netarkivet.common.utils.batch.FileBatchJob; /** @@ -71,7 +68,7 @@ public class BitmagArcRepositoryClient implements ArcRepositoryClient { private static final Logger log = LoggerFactory.getLogger(BitmagArcRepositoryClient.class); /** The default place in classpath where the settings file can be found. */ - private static String defaultSettingsClasspath = "dk/netarkivet/common/distribute/arcrepository/" + private static String defaultSettingsClasspath = "dk/netarkivet/common/distribute/arcrepository/bitrepository/" + "BitmagArcRepositoryClientSettings.xml"; /* @@ -82,7 +79,7 @@ public class BitmagArcRepositoryClient implements ArcRepositoryClient { Settings.addDefaultClasspathSettings(defaultSettingsClasspath); } - private static final String ARCREPOSITORY_TEMPDIR = "settings.common.arcrepositoryClient.bitrepository.tempdir"; + private static final String BITREPOSITORY_TEMPDIR = "settings.common.arcrepositoryClient.bitrepository.tempdir"; private static final String BITREPOSITORY_SETTINGS_DIR = "settings.common.arcrepositoryClient.bitrepository.settingsDir"; @@ -90,12 +87,26 @@ public class BitmagArcRepositoryClient implements ArcRepositoryClient { private static final String BITREPOSITORY_STORE_MAX_PILLAR_FAILURES = "settings.common.arcrepositoryClient.bitrepository.storeMaxPillarFailures"; //TODO necessary? - private static final String BITREPOSITORY_COLLECTIONID = "settings.common.arcrepositoryClient.bitrepository.collectionid"; + private static final String BITREPOSITORY_COLLECTIONID = "settings.common.arcrepositoryClient.bitrepository.collectionID"; + + private final String collectionId; + + private File tempdir; + + private int maxStoreFailures; + + private Bitrepository bitrep; /** Create a new BitmagArcRepositoryClient based on current settings. */ public BitmagArcRepositoryClient() { - + File configDir = Settings.getFile(BITREPOSITORY_SETTINGS_DIR); + File keyfile = Settings.getFile(BITREPOSITORY_KEYFILE); + this.collectionId = Settings.get(BITREPOSITORY_COLLECTIONID); + this.tempdir = Settings.getFile(BITREPOSITORY_TEMPDIR); + this.maxStoreFailures = Settings.getInt(BITREPOSITORY_STORE_MAX_PILLAR_FAILURES); + // Initialize connection to the bitrepository + this.bitrep = new Bitrepository(configDir, keyfile, maxStoreFailures); } @Override @@ -114,13 +125,19 @@ public void close() { public void store(File file) throws IOFailure, ArgumentNotValid { ArgumentNotValid.checkNotNull(file, "File file"); ArgumentNotValid.checkTrue(file.exists(), "File '" + file + "' does not exist"); - // Check if file already exists - - // upload file - - - + if (bitrep.existsInCollection(file.getName(), collectionId)) { + log.warn("The file '{}' is already in collection '{}'", file.getName(), collectionId); + return; + } else { + // upload file + boolean uploadSuccessful = bitrep.uploadFile(file, collectionId); + if (!uploadSuccessful) { + throw new IOFailure("Upload to collection '" + collectionId + "' of file '" + file.getName() + "' failed."); + } else { + log.info("Upload to collection '{}' of file '{}' was successfull", collectionId, file.getName()); + } + } } /** @@ -136,40 +153,36 @@ public void store(File file) throws IOFailure, ArgumentNotValid { public BitarchiveRecord get(String arcfile, long index) throws ArgumentNotValid { ArgumentNotValid.checkNotNullOrEmpty(arcfile, "String arcfile"); ArgumentNotValid.checkNotNegative(index, "long index"); - //FIXME - // Initial implementation. fetch file, and retrieve the record on the local file. - return null; -/* - File f = findFile(arcfile); - if (f == null) { - log.warn("File '{}' does not exist. Null BitarchiveRecord returned", arcfile); - return null; - } - ArchiveReader reader = null; - ArchiveRecord record = null; - try { - reader = ArchiveReaderFactory.get(f, index); - record = reader.get(); - return new BitarchiveRecord(record, arcfile); - } catch (IOException e) { - throw new IOFailure("Error reading record from '" + arcfile + "' offset " + index, e); - } finally { - if (record != null) { - try { - record.close(); - } catch (IOException e) { - log.warn("Error closing ARC record '{}'", record, e); + if (!bitrep.existsInCollection(arcfile, collectionId)) { + log.warn("The file '{}' is not in collection '{}'. Returning null BitarchiveRecord", arcfile, collectionId); + return null; + } else { + File f = bitrep.getFile(arcfile, collectionId , null); + ArchiveReader reader = null; + ArchiveRecord record = null; + try { + reader = ArchiveReaderFactory.get(f, index); + record = reader.get(); + return new BitarchiveRecord(record, arcfile); + } catch (IOException e) { + throw new IOFailure("Error reading record from '" + arcfile + "' offset " + index, e); + } finally { + if (record != null) { + try { + record.close(); + } catch (IOException e) { + log.warn("Error closing ARC record '{}'", record, e); + } } - } - if (reader != null) { - try { - reader.close(); - } catch (IOException e) { - log.warn("Error closing ARC reader '{}'", reader, e); + if (reader != null) { + try { + reader.close(); + } catch (IOException e) { + log.warn("Error closing ARC reader '{}'", reader, e); + } } } } -*/ } /** @@ -186,18 +199,16 @@ record = reader.get(); public void getFile(String arcfilename, Replica replica, File toFile) { ArgumentNotValid.checkNotNullOrEmpty(arcfilename, "String arcfilename"); ArgumentNotValid.checkNotNull(toFile, "File toFile"); - //FIXME -/* - File f = findFile(arcfilename); - if (f != null) { - FileUtils.copyFile(f, toFile); + + if (!bitrep.existsInCollection(arcfilename, collectionId)) { + log.warn("The file '{}' is not in collection '{}'.", arcfilename, collectionId); + return; } else { - throw new IOFailure("File '" + arcfilename + "' does not exist"); + File f = bitrep.getFile(arcfilename, collectionId , null); + FileUtils.copyFile(f, toFile); } - -*/ } - + /** * Runs a batch job on each file in the ArcRepository. * diff --git a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/Bitrepository.java b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/Bitrepository.java index ae95e57aeb..2f60515ab5 100644 --- a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/Bitrepository.java +++ b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/Bitrepository.java @@ -102,18 +102,20 @@ public class Bitrepository { private MessageBus bitMagMessageBus; /** The maximum number of failing pillars. Default is 0. */ - private int maxNumberOfFailingPillars = 0; //Settings.getInt... + private final int maxNumberOfFailingPillars; /** * Constructor for the BitRepository class. * @param configDir A Bitrepository settingsdirectory + * @param maxStoreFailures Max number of acceptable store failures * @param bitmagKeyFile (optional, we hope) * @throws ArgumentNotValid if configFile is null */ - public Bitrepository(File configDir, File bitmagKeyfile) { + public Bitrepository(File configDir, File bitmagKeyfile, int maxStoreFailures) { ArgumentNotValid.checkExistsDirectory(configDir, "File configDir"); //ArgumentNotValid.checkExistsNormalFile(bitmagKeyfile, "File bitmagKeyfile"); componentId = BitrepositoryUtils.generateComponentID(); + maxNumberOfFailingPillars = maxStoreFailures; this.settingsDir = configDir; this.privateKeyFile = bitmagKeyfile; initBitmagSettings(); @@ -418,7 +420,7 @@ private List getCollectionPillars(String collectionID) { /** * Helper method for computing the clientTimeout. The clientTimeout is the identificationTimeout * plus the OperationTimeout. - * @param bitmagSettings The bitmagsettingg + * @param bitmagSettings The bitmagsetting * @return the clientTimeout */ private long getClientTimeout(Settings bitmagSettings) { @@ -451,4 +453,28 @@ public List getKnownCollections() { public ChecksumSpecTYPE getDefaultChecksum() { return ChecksumUtils.getDefault(bitmagSettings); } + + //FIXME can this method be made to work? + public List getFileIds(String regex, String collectionID) { + + OutputHandler output = new DefaultOutputHandler(Bitrepository.class); + + output.debug("Instantiation GetFileID outputFormatter."); + // TODO: change to non pagingClient + GetFileIDsOutputFormatter outputFormatter = new GetFileIDsInfoFormatter(output); + + long timeout = getClientTimeout(bitmagSettings); + + output.debug("Instantiation GetFileID paging client."); + PagingGetFileIDsClient pagingClient = new PagingGetFileIDsClient( + bitMagGetFileIDsClient, timeout, outputFormatter, output); + + Boolean success = pagingClient.getFileIDs(collectionID, "packageId", + getCollectionPillars(collectionID)); + return null; + } + + + + } diff --git a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitrepositoryTest3.java b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitrepositoryTest3.java new file mode 100644 index 0000000000..db2ca20bb3 --- /dev/null +++ b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitrepositoryTest3.java @@ -0,0 +1,26 @@ +package dk.netarkivet.common.distribute.arcrepository.bitrepository; + +import java.io.File; +import dk.netarkivet.common.distribute.arcrepository.bitrepository.Bitrepository; + +public class BitrepositoryTest3 { + + // FIXME kan kun køres hvis xmlParserAPIs-2.6.2.jar (fra 2003) ikke findes i classpath (skal bruge xercesImpl-2.9.1.jar istedet) + // Jf. http://stackoverflow.com/questions/20473689/org-xml-sax-saxnotrecognizedexception-feature-http-javax-xml-xmlconstants-fe + + public static void main(String[] args) { + + //File configDir = new File("/home/svc/bitmag-releasetest-conf"); + //File bitmagKeyfile = new File(configDir, "client-certificate.pem"); + //Bitrepository bitrep = new Bitrepository(configDir, bitmagKeyfile); + //bitrep.shutdown(); + + File configDir = new File("/home/svc/bitrepository-quickstart/commandline/conf"); + //File bitmagKeyfile = new File(configDir, "client-certificate.pem"); + Bitrepository bitrep = new Bitrepository(configDir, null, 1); + bitrep.shutdown(); + + BitmagArcRepositoryClient client = new BitmagArcRepositoryClient(); + + } +} \ No newline at end of file diff --git a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository.java b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository.java index 718803a519..c19b11fdab 100644 --- a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository.java +++ b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository.java @@ -17,9 +17,10 @@ public static void main(String[] args) { File configDir = new File("/home/svc/bitrepository-quickstart/commandline/conf"); //File bitmagKeyfile = new File(configDir, "client-certificate.pem"); - Bitrepository bitrep = new Bitrepository(configDir, null); + Bitrepository bitrep = new Bitrepository(configDir, null, 1); bitrep.shutdown(); + BitmagArcRepositoryClient client = new BitmagArcRepositoryClient(); } diff --git a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository2.java b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository2.java index b37791e4be..5d5a6f0792 100644 --- a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository2.java +++ b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository2.java @@ -17,10 +17,14 @@ public static void main(String[] args) { File configDir = new File("/home/svc/bitrepository-quickstart/commandline/conf"); //File bitmagKeyfile = new File(configDir, "client-certificate.pem"); - Bitrepository bitrep = new Bitrepository(configDir, null); + Bitrepository bitrep = new Bitrepository(configDir, null, 1); bitrep.shutdown(); + + BitmagArcRepositoryClient client = new BitmagArcRepositoryClient(); + + } } diff --git a/common/common-core/src/main/resources/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClientSettings.xml b/common/common-core/src/main/resources/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClientSettings.xml index 65f86c4dfb..e0ae7ded8c 100644 --- a/common/common-core/src/main/resources/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClientSettings.xml +++ b/common/common-core/src/main/resources/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClientSettings.xml @@ -40,7 +40,7 @@ National Library. 1 arcrepositoryTemp - + dvds diff --git a/common/common-test/src/main/java/TestBitrepository.java b/common/common-test/src/main/java/TestBitrepository.java index 037f888c1b..09a26f860c 100644 --- a/common/common-test/src/main/java/TestBitrepository.java +++ b/common/common-test/src/main/java/TestBitrepository.java @@ -7,7 +7,7 @@ public class TestBitrepository { public static void main(String[] args) { File configDir = new File("/home/svc/bitmag-releasetest-conf"); File bitmagKeyfile = new File(configDir, "client-certificate.pem"); - Bitrepository bitrep = new Bitrepository(configDir, bitmagKeyfile); + Bitrepository bitrep = new Bitrepository(configDir, bitmagKeyfile, 1); bitrep.shutdown(); }