Skip to content

Commit

Permalink
Fixing error with getting getFileIds from bitrepos that doen't use ce…
Browse files Browse the repository at this point in the history 
…rtificates. The trick was assign a non-existing filename as keyfile
  • Loading branch information
svcarlsen committed Sep 17, 2015
1 parent da847f9 commit ee14ebf
Show file tree
Hide file tree
Showing 10 changed files with 257 additions and 230 deletions.
6 changes: 5 additions & 1 deletion common/common-core/pom.xml
View file
Expand Up @@ -70,7 +70,11 @@
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.4.1</version>
</dependency>
<dependency>
<groupId>xalan</groupId>
<artifactId>xalan</artifactId>
Expand Down
109 changes: 53 additions & 56 deletions ...k/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClient.java
View file
Expand Up @@ -25,12 +25,10 @@

import java.io.File;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;

import org.archive.io.ArchiveReader;
import org.archive.io.ArchiveReaderFactory;
Expand Down Expand Up @@ -83,9 +81,10 @@ public class BitmagArcRepositoryClient implements ArcRepositoryClient {

private static final String BITREPOSITORY_SETTINGS_DIR = "settings.common.arcrepositoryClient.bitrepository.settingsDir";

private static final String BITREPOSITORY_KEYFILE = "settings.common.arcrepositoryClient.bitrepository.keyfile"; //TODO necessary? optional so we don't force the user to use credentials.
// optional so we don't force the user to use credentials.
private static final String BITREPOSITORY_KEYFILENAME = "settings.common.arcrepositoryClient.bitrepository.keyfilename";

private static final String BITREPOSITORY_STORE_MAX_PILLAR_FAILURES = "settings.common.arcrepositoryClient.bitrepository.storeMaxPillarFailures"; //TODO necessary?
private static final String BITREPOSITORY_STORE_MAX_PILLAR_FAILURES = "settings.common.arcrepositoryClient.bitrepository.storeMaxPillarFailures";

private static final String BITREPOSITORY_COLLECTIONID = "settings.common.arcrepositoryClient.bitrepository.collectionID";

Expand All @@ -104,13 +103,13 @@ public class BitmagArcRepositoryClient implements ArcRepositoryClient {
/** Create a new BitmagArcRepositoryClient based on current settings. */
public BitmagArcRepositoryClient() {
File configDir = Settings.getFile(BITREPOSITORY_SETTINGS_DIR);
File keyfile = Settings.getFile(BITREPOSITORY_KEYFILE);
String keyfilename = Settings.get(BITREPOSITORY_KEYFILENAME);
this.collectionId = Settings.get(BITREPOSITORY_COLLECTIONID);
this.tempdir = Settings.getFile(BITREPOSITORY_TEMPDIR);
this.maxStoreFailures = Settings.getInt(BITREPOSITORY_STORE_MAX_PILLAR_FAILURES);
this.usepillar = Settings.get(BITREPOSITORY_USEPILLAR);
// Initialize connection to the bitrepository
this.bitrep = new Bitrepository(configDir, keyfile, maxStoreFailures, usepillar);
this.bitrep = new Bitrepository(configDir, keyfilename, maxStoreFailures, usepillar);
}

@Override
Expand Down Expand Up @@ -220,66 +219,64 @@ public void getFile(String arcfilename, Replica replica, File toFile) {
* processing and the finish() method will be called afterwards. The process() method will be called with each File
* entry. An optional function postProcess() allows handling the combined results of the batchjob, e.g. summing the
* results, sorting, etc.
* @param replicaId The archive to execute the job on.
* @param replicaId The archive to execute the job on. Argument Ignored replaced by usepillar (or reuse replicaId for bitmaguse)
* @param args The arguments for the batchjob. This can be null.
* @return The status of the batch job after it ended.
* @throws ArgumentNotValid If the job is null or the replicaId is either null or the empty string.
* @throws IOFailure If a problem occurs during processing the batchjob.
*/
@Override
public BatchStatus batch(final FileBatchJob job, String replicaId, String... args) throws ArgumentNotValid,
IOFailure {
ArgumentNotValid.checkNotNull(job, "FileBatchJob job");
ArgumentNotValid.checkNotNullOrEmpty(replicaId, "String replicaId");
//FIXME
// Use this pattern to request the fileIds to match this pattern
// and then fetch the matching files to local storage
//Pattern filenamePattern = job.getFilenamePattern;
log.info("pattern: " + job.getFilenamePattern().pattern());
System.out.println("pattern: " + job.getFilenamePattern().pattern());

OutputStream os = null;
File resultFile;
try {
resultFile = File.createTempFile("batch", replicaId, FileUtils.getTempDir());
os = new FileOutputStream(resultFile);
IOFailure {
ArgumentNotValid.checkNotNull(job, "FileBatchJob job");
ArgumentNotValid.checkNotNullOrEmpty(replicaId, "String replicaId");

// Deduce the remote file to run the batchjob on from the job.getFilenamePattern()
// e.g. "22-metadata-[0-9]+.(w)?arc" => 22-metadata-1.warc
log.info("Trying to deducing requested file to run batch on from pattern {}", job.getFilenamePattern().pattern());

String patternAsString = job.getFilenamePattern().pattern();
if (!patternAsString.contains("metadata-")) {
log.warn("deducing requested file to run batch on from pattern {} failed. Is not a metadata file", job.getFilenamePattern().pattern());
return null;
} else {
// With 22-metadata-[0-9]+.(w)?arc
// nameparts will be ["22", "metadata", "[0", "9]+.(w)?arc"]
String nameParts[] = patternAsString.split("-");
String nameToFetch = nameParts[0] + "-metadata-1.warc";
List<File> files = new ArrayList<File>();
final FilenameFilter filenameFilter = new FilenameFilter() {
public boolean accept(File dir, String name) {
Pattern filenamePattern = job.getFilenamePattern();
return new File(dir, name).isFile()
&& (filenamePattern == null || filenamePattern.matcher(name).matches());
}
};
/*
for (File dir : storageDirs) {
File[] filesInDir = dir.listFiles(filenameFilter);
if (filesInDir != null) {
files.addAll(Arrays.asList(filesInDir));
}
}
*/

BatchLocalFiles batcher = new BatchLocalFiles(files.toArray(new File[files.size()]));
batcher.run(job, os);
} catch (IOException e) {
throw new IOFailure("Cannot perform batch '" + job + "'", e);
} finally {
if (os != null) {
try {
os.close();
} catch (IOException e) {
log.warn("Error closing batch output stream '{}'", os, e);
}
}
}
return new BatchStatus(replicaId, job.getFilesFailed(), job.getNoOfFilesProcessed(), new FileRemoteFile(
resultFile), job.getExceptions());

if (!bitrep.existsInCollection(nameToFetch, collectionId)) {
log.warn("The file '{}' is not in collection '{}'.", nameToFetch, collectionId);
} else {
File workFile = bitrep.getFile(nameToFetch, this.collectionId, null);
files.add(workFile);
}

OutputStream os = null;
File resultFile;
try {
resultFile = File.createTempFile("batch", replicaId, FileUtils.getTempDir());
os = new FileOutputStream(resultFile);

BatchLocalFiles batcher = new BatchLocalFiles(files.toArray(new File[files.size()]));
batcher.run(job, os);
} catch (IOException e) {
throw new IOFailure("Cannot perform batch '" + job + "'", e);
} finally {
if (os != null) {
try {
os.close();
} catch (IOException e) {
log.warn("Error closing batch output stream '{}'", os, e);
}
}
}
return new BatchStatus(replicaId, job.getFilesFailed(), job.getNoOfFilesProcessed(), new FileRemoteFile(
resultFile), job.getExceptions());
}
}


/////////////////// The rest of the API is not implemented for the bitrepository system ///////////////////////////

/**
Expand Down
63 changes: 32 additions & 31 deletions .../main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/Bitrepository.java
View file
Expand Up @@ -11,6 +11,7 @@
import java.util.Map;

import javax.jms.JMSException;

import dk.netarkivet.common.exceptions.*;

import org.bitrepository.access.AccessComponentFactory;
Expand All @@ -32,7 +33,7 @@
import org.bitrepository.commandline.eventhandler.GetFileEventHandler;
import org.bitrepository.commandline.output.DefaultOutputHandler;
import org.bitrepository.commandline.output.OutputHandler;
import org.bitrepository.commandline.outputformatter.GetFileIDsInfoFormatter;
import org.bitrepository.commandline.outputformatter.GetFileIDsOutputFormatter;
import org.bitrepository.common.exceptions.OperationFailedException;
import org.bitrepository.common.settings.Settings;
import org.bitrepository.common.settings.SettingsProvider;
Expand Down Expand Up @@ -113,22 +114,26 @@ public class Bitrepository {
* @param configDir A Bitrepository settingsdirectory
* @param maxStoreFailures Max number of acceptable store failures
* @param usepillar The pillar to use
* @param bitmagKeyFile Optional certificate filename relative to configDir
* @param bitmagKeyFilename Optional certificate filename relative to configDir
* @throws ArgumentNotValid if configFile is null
*/
public Bitrepository(File configDir, File bitmagKeyfile, int maxStoreFailures, String usepillar) {
public Bitrepository(File configDir, String bitmagKeyFilename, int maxStoreFailures, String usepillar) {
ArgumentNotValid.checkExistsDirectory(configDir, "File configDir");
componentId = BitrepositoryUtils.generateComponentID();
maxNumberOfFailingPillars = maxStoreFailures;
this.usepillar = usepillar;
usepillarListOnly = new ArrayList<String>();
usepillarListOnly.add(usepillar);
this.settingsDir = configDir;
this.privateKeyFile = bitmagKeyfile;
initBitmagSettings();
if (bitmagKeyfile != null){
initBitmagSecurityManager(); // Is this mandatory?
if (bitmagKeyFilename == null){
this.privateKeyFile = new File(configDir, "dummy-certificate.pem"); // This file should never exist
} else {
this.privateKeyFile = new File(configDir, bitmagKeyFilename);
}

initBitmagSettings();
initBitmagSecurityManager(); // Mandatory,even if we point to a nonexisting file dummy-certificate.pem

bitMagMessageBus = ProtocolComponentFactory.getInstance().getMessageBus(
bitmagSettings, bitMagSecurityManager); // Is bitMagSecurityManager mandatory?
initBitMagClients();
Expand Down Expand Up @@ -242,12 +247,12 @@ public File getFile(final String fileId, final String collectionId, final FilePa
// Note that this eventHandler is blocking
CompleteEventAwaiter eventHandler = new GetFileEventHandler(this.bitmagSettings, output);
output.debug("Initiating the GetFile conversation.");
String auditTrailInformation = "Retrieving package '" + fileId + "' from collection '" + collectionId + "'";
bitMagGetClient.getFileFromFastestPillar(collectionId, fileId, filePart, fileUrl, eventHandler,
auditTrailInformation);
String auditTrailInformation = "Retrieving package '" + fileId + "' from collection '" + collectionId + "' using pillar '" + usepillar + "'";
logger.info(auditTrailInformation);
//bitMagGetClient.getFileFromFastestPillar(collectionId, fileId, filePart, fileUrl, eventHandler,
// auditTrailInformation);

//TODO get from specific pillar instead of fastest pillar?
//bitMagGetClient.getFileFromSpecificPillar(collectionID, fileId, filePart, uploadUrl, pillarId, eventHandler, auditTrailInformation);
bitMagGetClient.getFileFromSpecificPillar(collectionId, fileId, filePart, fileUrl, usepillar, eventHandler, auditTrailInformation);

OperationEvent finalEvent = eventHandler.getFinish();
if(finalEvent.getEventType() == OperationEventType.COMPLETE) {
Expand Down Expand Up @@ -307,13 +312,12 @@ public boolean existsInCollection(String packageId, String collectionID) {

OutputHandler output = new DefaultOutputHandler(Bitrepository.class);

output.debug("Instantiation GetFileID outputFormatter.");
// TODO: change to non pagingClient
GetFileIDsInfoFormatter outputFormatter = new GetFileIDsInfoFormatter(output);

//output.debug("Instantiation GetFileID outputFormatter.");
//GetFileIDsListFormatter outputFormatter = new GetFileIDsListFormatter(output);
GetFileIDsOutputFormatter outputFormatter = new GetFileIDsNoFormatter(output);
long timeout = getClientTimeout(bitmagSettings);

output.debug("Instantiation GetFileID paging client.");
//output.debug("Instantiation GetFileID paging client.");
PagingGetFileIDsClient pagingClient = new PagingGetFileIDsClient(
bitMagGetFileIDsClient, timeout, outputFormatter, output);

Expand Down Expand Up @@ -373,16 +377,15 @@ public Map<String, ChecksumsCompletePillarEvent> getChecksums(String packageID,
* Initialize the BITMAG security manager.
*/
private void initBitmagSecurityManager() {
PermissionStore permissionStore = new PermissionStore();
MessageAuthenticator authenticator = new BasicMessageAuthenticator(permissionStore);
MessageSigner signer = new BasicMessageSigner();
OperationAuthorizor authorizer = new BasicOperationAuthorizor(permissionStore);
if (getPrivateKeyFile() != null) {
bitMagSecurityManager = new BasicSecurityManager(bitmagSettings.getRepositorySettings(),
getPrivateKeyFile().getAbsolutePath(),
authenticator, signer, authorizer, permissionStore,
bitmagSettings.getComponentID());
}
PermissionStore permissionStore = new PermissionStore();
MessageAuthenticator authenticator = new BasicMessageAuthenticator(permissionStore);
MessageSigner signer = new BasicMessageSigner();
OperationAuthorizor authorizer = new BasicOperationAuthorizor(permissionStore);

bitMagSecurityManager = new BasicSecurityManager(bitmagSettings.getRepositorySettings(),
getPrivateKeyFile().getAbsolutePath(),
authenticator, signer, authorizer, permissionStore,
bitmagSettings.getComponentID());
}

private File getPrivateKeyFile() {
Expand Down Expand Up @@ -466,15 +469,13 @@ public ChecksumSpecTYPE getDefaultChecksum() {
public List<String> getFileIds(String collectionID) {

OutputHandler output = new DefaultOutputHandler(Bitrepository.class);

output.debug("Instantiation GetFileID outputFormatter.");
// TODO: change to non pagingClient
//output.debug("Instantiation GetFileID outputFormatter.");
GetFileIDsListFormatter outputFormatter = new GetFileIDsListFormatter(output);

long timeout = getClientTimeout(bitmagSettings);
List<String> usepillarListOnly = new ArrayList<String>();
usepillarListOnly.add(usepillar);
output.debug("Instantiation GetFileID paging client.");
//output.debug("Instantiation GetFileID paging client.");
PagingGetFileIDsClient pagingClient = new PagingGetFileIDsClient(
bitMagGetFileIDsClient, timeout, outputFormatter, output);
Boolean success = pagingClient.getFileIDs(collectionID, null,
Expand Down
50 changes: 0 additions & 50 deletions .../java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitrepositoryTest3.java
View file

This file was deleted.

6 changes: 1 addition & 5 deletions .../dk/netarkivet/common/distribute/arcrepository/bitrepository/GetFileIDsListFormatter.java
View file
Expand Up @@ -12,10 +12,8 @@
public class GetFileIDsListFormatter implements GetFileIDsOutputFormatter {

List<String> result = new ArrayList<String>();
//private OutputHandler outputHandler;

public GetFileIDsListFormatter(OutputHandler outputHandler) {
//this.outputHandler = outputHandler;
}

@Override
Expand All @@ -28,9 +26,7 @@ public void formatResult(Collection<FileIDsResult> results) {
result.add(a.getID());
}
}

public List<String> getFoundIds() {
return result;
}

}
}
20 changes: 20 additions & 0 deletions ...va/dk/netarkivet/common/distribute/arcrepository/bitrepository/GetFileIDsNoFormatter.java
View file
@@ -0,0 +1,20 @@
package dk.netarkivet.common.distribute.arcrepository.bitrepository;

import java.util.Collection;
import org.bitrepository.commandline.output.OutputHandler;
import org.bitrepository.commandline.outputformatter.GetFileIDsOutputFormatter;
import org.bitrepository.commandline.resultmodel.FileIDsResult;

public class GetFileIDsNoFormatter implements GetFileIDsOutputFormatter {

public GetFileIDsNoFormatter(OutputHandler outputHandler) {
}

@Override
public void formatHeader() {
}

@Override
public void formatResult(Collection<FileIDsResult> results) {
}
}

0 comments on commit ee14ebf

Please sign in to comment.