Skip to content

Commit

Permalink
Work on bitmag integration
Browse files Browse the repository at this point in the history
  • Loading branch information
Søren Vejrup Carlsen committed Sep 8, 2015
1 parent e730c8a commit 618a86e
Show file tree
Hide file tree
Showing 8 changed files with 216 additions and 61 deletions.
91 changes: 89 additions & 2 deletions common/common-core/pom.xml
View file
Expand Up @@ -14,18 +14,95 @@
<properties>
<bitrepository.version>1.4</bitrepository.version>
</properties>

<dependencies>

<!-- bitrepository dependencies -->

<dependency>
<groupId>org.bitrepository.reference</groupId>
<artifactId>bitrepository-core</artifactId>
<version>${bitrepository.version}</version>
<!-- <exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
-->
</dependency>
<dependency>
<groupId>org.bitrepository.reference</groupId>
<artifactId>bitrepository-client</artifactId>
<version>${bitrepository.version}</version>
<!-- <exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
-->
</dependency>
<!--
<dependency>
<groupId>org.bitrepository</groupId>
<artifactId>message-xml-java</artifactId>
<version>28</version>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.bitrepository</groupId>
<artifactId>message-xml-xsd</artifactId>
<version>28</version>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.jvnet.jaxb2_commons</groupId>
<artifactId>jaxb2-basics-runtime</artifactId>
<version>0.6.0</version>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.bitrepository.repository-settings</groupId>
<artifactId>repository-settings-java</artifactId>
<version>11</version>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.bitrepository.repository-settings</groupId>
<artifactId>repository-settings-xsd</artifactId>
<version>11</version>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
-->

<!-- bitrepository settings complete -->


<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
Expand Down Expand Up @@ -55,8 +132,14 @@
<groupId>jaxen</groupId>
<artifactId>jaxen</artifactId>
<version>1.1</version>
</dependency>
<exclusions>
<exclusion>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
</exclusion>
</exclusions>

</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
Expand Down Expand Up @@ -134,6 +217,10 @@
<artifactId>xml-apis</artifactId>
<groupId>xml-apis</groupId>
</exclusion>
<exclusion>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
</exclusion>
</exclusions>
</dependency>

Expand Down
115 changes: 63 additions & 52 deletions ...k/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClient.java
View file
Expand Up @@ -29,7 +29,6 @@
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Pattern;

Expand All @@ -50,11 +49,9 @@
import dk.netarkivet.common.exceptions.NotImplementedException;
import dk.netarkivet.common.exceptions.IllegalState;
import dk.netarkivet.common.exceptions.PermissionDenied;
import dk.netarkivet.common.utils.ChecksumCalculator;
import dk.netarkivet.common.utils.FileUtils;
import dk.netarkivet.common.utils.Settings;
import dk.netarkivet.common.utils.batch.BatchLocalFiles;
import dk.netarkivet.common.utils.batch.ChecksumJob;
import dk.netarkivet.common.utils.batch.FileBatchJob;

/**
Expand All @@ -71,7 +68,7 @@ public class BitmagArcRepositoryClient implements ArcRepositoryClient {
private static final Logger log = LoggerFactory.getLogger(BitmagArcRepositoryClient.class);

/** The default place in classpath where the settings file can be found. */
private static String defaultSettingsClasspath = "dk/netarkivet/common/distribute/arcrepository/"
private static String defaultSettingsClasspath = "dk/netarkivet/common/distribute/arcrepository/bitrepository/"
+ "BitmagArcRepositoryClientSettings.xml";

/*
Expand All @@ -82,20 +79,34 @@ public class BitmagArcRepositoryClient implements ArcRepositoryClient {
Settings.addDefaultClasspathSettings(defaultSettingsClasspath);
}

private static final String ARCREPOSITORY_TEMPDIR = "settings.common.arcrepositoryClient.bitrepository.tempdir";
private static final String BITREPOSITORY_TEMPDIR = "settings.common.arcrepositoryClient.bitrepository.tempdir";

private static final String BITREPOSITORY_SETTINGS_DIR = "settings.common.arcrepositoryClient.bitrepository.settingsDir";

private static final String BITREPOSITORY_KEYFILE = "settings.common.arcrepositoryClient.bitrepository.keyfile"; //TODO necessary? optional so we don't force the user to use credentials.

private static final String BITREPOSITORY_STORE_MAX_PILLAR_FAILURES = "settings.common.arcrepositoryClient.bitrepository.storeMaxPillarFailures"; //TODO necessary?

private static final String BITREPOSITORY_COLLECTIONID = "settings.common.arcrepositoryClient.bitrepository.collectionid";
private static final String BITREPOSITORY_COLLECTIONID = "settings.common.arcrepositoryClient.bitrepository.collectionID";

private final String collectionId;

private File tempdir;

private int maxStoreFailures;

private Bitrepository bitrep;


/** Create a new BitmagArcRepositoryClient based on current settings. */
public BitmagArcRepositoryClient() {

File configDir = Settings.getFile(BITREPOSITORY_SETTINGS_DIR);
File keyfile = Settings.getFile(BITREPOSITORY_KEYFILE);
this.collectionId = Settings.get(BITREPOSITORY_COLLECTIONID);
this.tempdir = Settings.getFile(BITREPOSITORY_TEMPDIR);
this.maxStoreFailures = Settings.getInt(BITREPOSITORY_STORE_MAX_PILLAR_FAILURES);
// Initialize connection to the bitrepository
this.bitrep = new Bitrepository(configDir, keyfile, maxStoreFailures);
}

@Override
Expand All @@ -114,13 +125,19 @@ public void close() {
public void store(File file) throws IOFailure, ArgumentNotValid {
ArgumentNotValid.checkNotNull(file, "File file");
ArgumentNotValid.checkTrue(file.exists(), "File '" + file + "' does not exist");

// Check if file already exists

// upload file



if (bitrep.existsInCollection(file.getName(), collectionId)) {
log.warn("The file '{}' is already in collection '{}'", file.getName(), collectionId);
return;
} else {
// upload file
boolean uploadSuccessful = bitrep.uploadFile(file, collectionId);
if (!uploadSuccessful) {
throw new IOFailure("Upload to collection '" + collectionId + "' of file '" + file.getName() + "' failed.");
} else {
log.info("Upload to collection '{}' of file '{}' was successfull", collectionId, file.getName());
}
}
}

/**
Expand All @@ -136,40 +153,36 @@ public void store(File file) throws IOFailure, ArgumentNotValid {
public BitarchiveRecord get(String arcfile, long index) throws ArgumentNotValid {
ArgumentNotValid.checkNotNullOrEmpty(arcfile, "String arcfile");
ArgumentNotValid.checkNotNegative(index, "long index");
//FIXME
// Initial implementation. fetch file, and retrieve the record on the local file.
return null;
/*
File f = findFile(arcfile);
if (f == null) {
log.warn("File '{}' does not exist. Null BitarchiveRecord returned", arcfile);
return null;
}
ArchiveReader reader = null;
ArchiveRecord record = null;
try {
reader = ArchiveReaderFactory.get(f, index);
record = reader.get();
return new BitarchiveRecord(record, arcfile);
} catch (IOException e) {
throw new IOFailure("Error reading record from '" + arcfile + "' offset " + index, e);
} finally {
if (record != null) {
try {
record.close();
} catch (IOException e) {
log.warn("Error closing ARC record '{}'", record, e);
if (!bitrep.existsInCollection(arcfile, collectionId)) {
log.warn("The file '{}' is not in collection '{}'. Returning null BitarchiveRecord", arcfile, collectionId);
return null;
} else {
File f = bitrep.getFile(arcfile, collectionId , null);
ArchiveReader reader = null;
ArchiveRecord record = null;
try {
reader = ArchiveReaderFactory.get(f, index);
record = reader.get();
return new BitarchiveRecord(record, arcfile);
} catch (IOException e) {
throw new IOFailure("Error reading record from '" + arcfile + "' offset " + index, e);
} finally {
if (record != null) {
try {
record.close();
} catch (IOException e) {
log.warn("Error closing ARC record '{}'", record, e);
}
}
}
if (reader != null) {
try {
reader.close();
} catch (IOException e) {
log.warn("Error closing ARC reader '{}'", reader, e);
if (reader != null) {
try {
reader.close();
} catch (IOException e) {
log.warn("Error closing ARC reader '{}'", reader, e);
}
}
}
}
*/
}

/**
Expand All @@ -186,18 +199,16 @@ record = reader.get();
public void getFile(String arcfilename, Replica replica, File toFile) {
ArgumentNotValid.checkNotNullOrEmpty(arcfilename, "String arcfilename");
ArgumentNotValid.checkNotNull(toFile, "File toFile");
//FIXME
/*
File f = findFile(arcfilename);
if (f != null) {
FileUtils.copyFile(f, toFile);

if (!bitrep.existsInCollection(arcfilename, collectionId)) {
log.warn("The file '{}' is not in collection '{}'.", arcfilename, collectionId);
return;
} else {
throw new IOFailure("File '" + arcfilename + "' does not exist");
File f = bitrep.getFile(arcfilename, collectionId , null);
FileUtils.copyFile(f, toFile);
}
*/
}

/**
* Runs a batch job on each file in the ArcRepository.
*
Expand Down
32 changes: 29 additions & 3 deletions .../main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/Bitrepository.java
View file
Expand Up @@ -102,18 +102,20 @@ public class Bitrepository {
private MessageBus bitMagMessageBus;

/** The maximum number of failing pillars. Default is 0. */
private int maxNumberOfFailingPillars = 0; //Settings.getInt...
private final int maxNumberOfFailingPillars;

/**
* Constructor for the BitRepository class.
* @param configDir A Bitrepository settingsdirectory
* @param maxStoreFailures Max number of acceptable store failures
* @param bitmagKeyFile (optional, we hope)
* @throws ArgumentNotValid if configFile is null
*/
public Bitrepository(File configDir, File bitmagKeyfile) {
public Bitrepository(File configDir, File bitmagKeyfile, int maxStoreFailures) {
ArgumentNotValid.checkExistsDirectory(configDir, "File configDir");
//ArgumentNotValid.checkExistsNormalFile(bitmagKeyfile, "File bitmagKeyfile");
componentId = BitrepositoryUtils.generateComponentID();
maxNumberOfFailingPillars = maxStoreFailures;
this.settingsDir = configDir;
this.privateKeyFile = bitmagKeyfile;
initBitmagSettings();
Expand Down Expand Up @@ -418,7 +420,7 @@ private List<String> getCollectionPillars(String collectionID) {
/**
* Helper method for computing the clientTimeout. The clientTimeout is the identificationTimeout
* plus the OperationTimeout.
* @param bitmagSettings The bitmagsettingg
* @param bitmagSettings The bitmagsetting
* @return the clientTimeout
*/
private long getClientTimeout(Settings bitmagSettings) {
Expand Down Expand Up @@ -451,4 +453,28 @@ public List<String> getKnownCollections() {
public ChecksumSpecTYPE getDefaultChecksum() {
return ChecksumUtils.getDefault(bitmagSettings);
}

//FIXME can this method be made to work?
public List<String> getFileIds(String regex, String collectionID) {

OutputHandler output = new DefaultOutputHandler(Bitrepository.class);

output.debug("Instantiation GetFileID outputFormatter.");
// TODO: change to non pagingClient
GetFileIDsOutputFormatter outputFormatter = new GetFileIDsInfoFormatter(output);

long timeout = getClientTimeout(bitmagSettings);

output.debug("Instantiation GetFileID paging client.");
PagingGetFileIDsClient pagingClient = new PagingGetFileIDsClient(
bitMagGetFileIDsClient, timeout, outputFormatter, output);

Boolean success = pagingClient.getFileIDs(collectionID, "packageId",
getCollectionPillars(collectionID));
return null;
}




}
26 changes: 26 additions & 0 deletions .../java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitrepositoryTest3.java
View file
@@ -0,0 +1,26 @@
package dk.netarkivet.common.distribute.arcrepository.bitrepository;

import java.io.File;
import dk.netarkivet.common.distribute.arcrepository.bitrepository.Bitrepository;

public class BitrepositoryTest3 {

// FIXME kan kun køres hvis xmlParserAPIs-2.6.2.jar (fra 2003) ikke findes i classpath (skal bruge xercesImpl-2.9.1.jar istedet)
// Jf. http://stackoverflow.com/questions/20473689/org-xml-sax-saxnotrecognizedexception-feature-http-javax-xml-xmlconstants-fe

public static void main(String[] args) {

//File configDir = new File("/home/svc/bitmag-releasetest-conf");
//File bitmagKeyfile = new File(configDir, "client-certificate.pem");
//Bitrepository bitrep = new Bitrepository(configDir, bitmagKeyfile);
//bitrep.shutdown();

File configDir = new File("/home/svc/bitrepository-quickstart/commandline/conf");
//File bitmagKeyfile = new File(configDir, "client-certificate.pem");
Bitrepository bitrep = new Bitrepository(configDir, null, 1);
bitrep.shutdown();

BitmagArcRepositoryClient client = new BitmagArcRepositoryClient();

}
}

0 comments on commit 618a86e

Please sign in to comment.