From ee14ebfb8cd247303241b1e03a509c1e77af6d2f Mon Sep 17 00:00:00 2001 From: svcarlsen Date: Thu, 17 Sep 2015 18:31:30 +0200 Subject: [PATCH] Fixing error with getting getFileIds from bitrepos that doen't use certificates. The trick was assign a non-existing filename as keyfile --- common/common-core/pom.xml | 6 +- .../BitmagArcRepositoryClient.java | 109 +++++------ .../bitrepository/Bitrepository.java | 63 +++--- .../bitrepository/BitrepositoryTest3.java | 50 ----- .../GetFileIDsListFormatter.java | 6 +- .../bitrepository/GetFileIDsNoFormatter.java | 20 ++ .../bitrepository/TestBitrepository.java | 182 ++++++++++++++---- .../bitrepository/TestBitrepository2.java | 30 --- .../BitmagArcRepositoryClientSettings.xml | 6 +- .../src/main/java/TestBitrepository.java | 15 -- 10 files changed, 257 insertions(+), 230 deletions(-) delete mode 100644 common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitrepositoryTest3.java create mode 100644 common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/GetFileIDsNoFormatter.java delete mode 100644 common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository2.java delete mode 100644 common/common-test/src/main/java/TestBitrepository.java diff --git a/common/common-core/pom.xml b/common/common-core/pom.xml index ab1e15bf49..107f2f1bcb 100644 --- a/common/common-core/pom.xml +++ b/common/common-core/pom.xml @@ -70,7 +70,11 @@ - + + org.apache.httpcomponents + httpcore + 4.4.1 + xalan xalan diff --git a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClient.java b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClient.java index c4369932bc..3e6f1dcf0a 100644 --- a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClient.java +++ b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClient.java @@ -25,12 +25,10 @@ import java.io.File; import java.io.FileOutputStream; -import java.io.FilenameFilter; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; import java.util.List; -import java.util.regex.Pattern; import org.archive.io.ArchiveReader; import org.archive.io.ArchiveReaderFactory; @@ -83,9 +81,10 @@ public class BitmagArcRepositoryClient implements ArcRepositoryClient { private static final String BITREPOSITORY_SETTINGS_DIR = "settings.common.arcrepositoryClient.bitrepository.settingsDir"; - private static final String BITREPOSITORY_KEYFILE = "settings.common.arcrepositoryClient.bitrepository.keyfile"; //TODO necessary? optional so we don't force the user to use credentials. + // optional so we don't force the user to use credentials. + private static final String BITREPOSITORY_KEYFILENAME = "settings.common.arcrepositoryClient.bitrepository.keyfilename"; - private static final String BITREPOSITORY_STORE_MAX_PILLAR_FAILURES = "settings.common.arcrepositoryClient.bitrepository.storeMaxPillarFailures"; //TODO necessary? + private static final String BITREPOSITORY_STORE_MAX_PILLAR_FAILURES = "settings.common.arcrepositoryClient.bitrepository.storeMaxPillarFailures"; private static final String BITREPOSITORY_COLLECTIONID = "settings.common.arcrepositoryClient.bitrepository.collectionID"; @@ -104,13 +103,13 @@ public class BitmagArcRepositoryClient implements ArcRepositoryClient { /** Create a new BitmagArcRepositoryClient based on current settings. */ public BitmagArcRepositoryClient() { File configDir = Settings.getFile(BITREPOSITORY_SETTINGS_DIR); - File keyfile = Settings.getFile(BITREPOSITORY_KEYFILE); + String keyfilename = Settings.get(BITREPOSITORY_KEYFILENAME); this.collectionId = Settings.get(BITREPOSITORY_COLLECTIONID); this.tempdir = Settings.getFile(BITREPOSITORY_TEMPDIR); this.maxStoreFailures = Settings.getInt(BITREPOSITORY_STORE_MAX_PILLAR_FAILURES); this.usepillar = Settings.get(BITREPOSITORY_USEPILLAR); // Initialize connection to the bitrepository - this.bitrep = new Bitrepository(configDir, keyfile, maxStoreFailures, usepillar); + this.bitrep = new Bitrepository(configDir, keyfilename, maxStoreFailures, usepillar); } @Override @@ -220,7 +219,7 @@ public void getFile(String arcfilename, Replica replica, File toFile) { * processing and the finish() method will be called afterwards. The process() method will be called with each File * entry. An optional function postProcess() allows handling the combined results of the batchjob, e.g. summing the * results, sorting, etc. - * @param replicaId The archive to execute the job on. + * @param replicaId The archive to execute the job on. Argument Ignored replaced by usepillar (or reuse replicaId for bitmaguse) * @param args The arguments for the batchjob. This can be null. * @return The status of the batch job after it ended. * @throws ArgumentNotValid If the job is null or the replicaId is either null or the empty string. @@ -228,58 +227,56 @@ public void getFile(String arcfilename, Replica replica, File toFile) { */ @Override public BatchStatus batch(final FileBatchJob job, String replicaId, String... args) throws ArgumentNotValid, - IOFailure { - ArgumentNotValid.checkNotNull(job, "FileBatchJob job"); - ArgumentNotValid.checkNotNullOrEmpty(replicaId, "String replicaId"); - //FIXME - // Use this pattern to request the fileIds to match this pattern - // and then fetch the matching files to local storage - //Pattern filenamePattern = job.getFilenamePattern; - - log.info("pattern: " + job.getFilenamePattern().pattern()); - System.out.println("pattern: " + job.getFilenamePattern().pattern()); - - - OutputStream os = null; - File resultFile; - try { - resultFile = File.createTempFile("batch", replicaId, FileUtils.getTempDir()); - os = new FileOutputStream(resultFile); + IOFailure { + ArgumentNotValid.checkNotNull(job, "FileBatchJob job"); + ArgumentNotValid.checkNotNullOrEmpty(replicaId, "String replicaId"); + + // Deduce the remote file to run the batchjob on from the job.getFilenamePattern() + // e.g. "22-metadata-[0-9]+.(w)?arc" => 22-metadata-1.warc + log.info("Trying to deducing requested file to run batch on from pattern {}", job.getFilenamePattern().pattern()); + + String patternAsString = job.getFilenamePattern().pattern(); + if (!patternAsString.contains("metadata-")) { + log.warn("deducing requested file to run batch on from pattern {} failed. Is not a metadata file", job.getFilenamePattern().pattern()); + return null; + } else { + // With 22-metadata-[0-9]+.(w)?arc + // nameparts will be ["22", "metadata", "[0", "9]+.(w)?arc"] + String nameParts[] = patternAsString.split("-"); + String nameToFetch = nameParts[0] + "-metadata-1.warc"; List files = new ArrayList(); - final FilenameFilter filenameFilter = new FilenameFilter() { - public boolean accept(File dir, String name) { - Pattern filenamePattern = job.getFilenamePattern(); - return new File(dir, name).isFile() - && (filenamePattern == null || filenamePattern.matcher(name).matches()); - } - }; -/* - for (File dir : storageDirs) { - File[] filesInDir = dir.listFiles(filenameFilter); - if (filesInDir != null) { - files.addAll(Arrays.asList(filesInDir)); - } - } -*/ - - BatchLocalFiles batcher = new BatchLocalFiles(files.toArray(new File[files.size()])); - batcher.run(job, os); - } catch (IOException e) { - throw new IOFailure("Cannot perform batch '" + job + "'", e); - } finally { - if (os != null) { - try { - os.close(); - } catch (IOException e) { - log.warn("Error closing batch output stream '{}'", os, e); - } - } - } - return new BatchStatus(replicaId, job.getFilesFailed(), job.getNoOfFilesProcessed(), new FileRemoteFile( - resultFile), job.getExceptions()); + + if (!bitrep.existsInCollection(nameToFetch, collectionId)) { + log.warn("The file '{}' is not in collection '{}'.", nameToFetch, collectionId); + } else { + File workFile = bitrep.getFile(nameToFetch, this.collectionId, null); + files.add(workFile); + } + + OutputStream os = null; + File resultFile; + try { + resultFile = File.createTempFile("batch", replicaId, FileUtils.getTempDir()); + os = new FileOutputStream(resultFile); + + BatchLocalFiles batcher = new BatchLocalFiles(files.toArray(new File[files.size()])); + batcher.run(job, os); + } catch (IOException e) { + throw new IOFailure("Cannot perform batch '" + job + "'", e); + } finally { + if (os != null) { + try { + os.close(); + } catch (IOException e) { + log.warn("Error closing batch output stream '{}'", os, e); + } + } + } + return new BatchStatus(replicaId, job.getFilesFailed(), job.getNoOfFilesProcessed(), new FileRemoteFile( + resultFile), job.getExceptions()); + } } - /////////////////// The rest of the API is not implemented for the bitrepository system /////////////////////////// /** diff --git a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/Bitrepository.java b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/Bitrepository.java index e9319dff9e..bf929690b8 100644 --- a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/Bitrepository.java +++ b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/Bitrepository.java @@ -11,6 +11,7 @@ import java.util.Map; import javax.jms.JMSException; + import dk.netarkivet.common.exceptions.*; import org.bitrepository.access.AccessComponentFactory; @@ -32,7 +33,7 @@ import org.bitrepository.commandline.eventhandler.GetFileEventHandler; import org.bitrepository.commandline.output.DefaultOutputHandler; import org.bitrepository.commandline.output.OutputHandler; -import org.bitrepository.commandline.outputformatter.GetFileIDsInfoFormatter; +import org.bitrepository.commandline.outputformatter.GetFileIDsOutputFormatter; import org.bitrepository.common.exceptions.OperationFailedException; import org.bitrepository.common.settings.Settings; import org.bitrepository.common.settings.SettingsProvider; @@ -113,10 +114,10 @@ public class Bitrepository { * @param configDir A Bitrepository settingsdirectory * @param maxStoreFailures Max number of acceptable store failures * @param usepillar The pillar to use - * @param bitmagKeyFile Optional certificate filename relative to configDir + * @param bitmagKeyFilename Optional certificate filename relative to configDir * @throws ArgumentNotValid if configFile is null */ - public Bitrepository(File configDir, File bitmagKeyfile, int maxStoreFailures, String usepillar) { + public Bitrepository(File configDir, String bitmagKeyFilename, int maxStoreFailures, String usepillar) { ArgumentNotValid.checkExistsDirectory(configDir, "File configDir"); componentId = BitrepositoryUtils.generateComponentID(); maxNumberOfFailingPillars = maxStoreFailures; @@ -124,11 +125,15 @@ public Bitrepository(File configDir, File bitmagKeyfile, int maxStoreFailures, S usepillarListOnly = new ArrayList(); usepillarListOnly.add(usepillar); this.settingsDir = configDir; - this.privateKeyFile = bitmagKeyfile; - initBitmagSettings(); - if (bitmagKeyfile != null){ - initBitmagSecurityManager(); // Is this mandatory? + if (bitmagKeyFilename == null){ + this.privateKeyFile = new File(configDir, "dummy-certificate.pem"); // This file should never exist + } else { + this.privateKeyFile = new File(configDir, bitmagKeyFilename); } + + initBitmagSettings(); + initBitmagSecurityManager(); // Mandatory,even if we point to a nonexisting file dummy-certificate.pem + bitMagMessageBus = ProtocolComponentFactory.getInstance().getMessageBus( bitmagSettings, bitMagSecurityManager); // Is bitMagSecurityManager mandatory? initBitMagClients(); @@ -242,12 +247,12 @@ public File getFile(final String fileId, final String collectionId, final FilePa // Note that this eventHandler is blocking CompleteEventAwaiter eventHandler = new GetFileEventHandler(this.bitmagSettings, output); output.debug("Initiating the GetFile conversation."); - String auditTrailInformation = "Retrieving package '" + fileId + "' from collection '" + collectionId + "'"; - bitMagGetClient.getFileFromFastestPillar(collectionId, fileId, filePart, fileUrl, eventHandler, - auditTrailInformation); + String auditTrailInformation = "Retrieving package '" + fileId + "' from collection '" + collectionId + "' using pillar '" + usepillar + "'"; + logger.info(auditTrailInformation); + //bitMagGetClient.getFileFromFastestPillar(collectionId, fileId, filePart, fileUrl, eventHandler, + // auditTrailInformation); - //TODO get from specific pillar instead of fastest pillar? - //bitMagGetClient.getFileFromSpecificPillar(collectionID, fileId, filePart, uploadUrl, pillarId, eventHandler, auditTrailInformation); + bitMagGetClient.getFileFromSpecificPillar(collectionId, fileId, filePart, fileUrl, usepillar, eventHandler, auditTrailInformation); OperationEvent finalEvent = eventHandler.getFinish(); if(finalEvent.getEventType() == OperationEventType.COMPLETE) { @@ -307,13 +312,12 @@ public boolean existsInCollection(String packageId, String collectionID) { OutputHandler output = new DefaultOutputHandler(Bitrepository.class); - output.debug("Instantiation GetFileID outputFormatter."); - // TODO: change to non pagingClient - GetFileIDsInfoFormatter outputFormatter = new GetFileIDsInfoFormatter(output); - + //output.debug("Instantiation GetFileID outputFormatter."); + //GetFileIDsListFormatter outputFormatter = new GetFileIDsListFormatter(output); + GetFileIDsOutputFormatter outputFormatter = new GetFileIDsNoFormatter(output); long timeout = getClientTimeout(bitmagSettings); - output.debug("Instantiation GetFileID paging client."); + //output.debug("Instantiation GetFileID paging client."); PagingGetFileIDsClient pagingClient = new PagingGetFileIDsClient( bitMagGetFileIDsClient, timeout, outputFormatter, output); @@ -373,16 +377,15 @@ public Map getChecksums(String packageID, * Initialize the BITMAG security manager. */ private void initBitmagSecurityManager() { - PermissionStore permissionStore = new PermissionStore(); - MessageAuthenticator authenticator = new BasicMessageAuthenticator(permissionStore); - MessageSigner signer = new BasicMessageSigner(); - OperationAuthorizor authorizer = new BasicOperationAuthorizor(permissionStore); - if (getPrivateKeyFile() != null) { - bitMagSecurityManager = new BasicSecurityManager(bitmagSettings.getRepositorySettings(), - getPrivateKeyFile().getAbsolutePath(), - authenticator, signer, authorizer, permissionStore, - bitmagSettings.getComponentID()); - } + PermissionStore permissionStore = new PermissionStore(); + MessageAuthenticator authenticator = new BasicMessageAuthenticator(permissionStore); + MessageSigner signer = new BasicMessageSigner(); + OperationAuthorizor authorizer = new BasicOperationAuthorizor(permissionStore); + + bitMagSecurityManager = new BasicSecurityManager(bitmagSettings.getRepositorySettings(), + getPrivateKeyFile().getAbsolutePath(), + authenticator, signer, authorizer, permissionStore, + bitmagSettings.getComponentID()); } private File getPrivateKeyFile() { @@ -466,15 +469,13 @@ public ChecksumSpecTYPE getDefaultChecksum() { public List getFileIds(String collectionID) { OutputHandler output = new DefaultOutputHandler(Bitrepository.class); - - output.debug("Instantiation GetFileID outputFormatter."); - // TODO: change to non pagingClient + //output.debug("Instantiation GetFileID outputFormatter."); GetFileIDsListFormatter outputFormatter = new GetFileIDsListFormatter(output); long timeout = getClientTimeout(bitmagSettings); List usepillarListOnly = new ArrayList(); usepillarListOnly.add(usepillar); - output.debug("Instantiation GetFileID paging client."); + //output.debug("Instantiation GetFileID paging client."); PagingGetFileIDsClient pagingClient = new PagingGetFileIDsClient( bitMagGetFileIDsClient, timeout, outputFormatter, output); Boolean success = pagingClient.getFileIDs(collectionID, null, diff --git a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitrepositoryTest3.java b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitrepositoryTest3.java deleted file mode 100644 index 5daf5b79af..0000000000 --- a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitrepositoryTest3.java +++ /dev/null @@ -1,50 +0,0 @@ -package dk.netarkivet.common.distribute.arcrepository.bitrepository; - -import java.io.File; -import java.util.List; - -import dk.netarkivet.common.distribute.arcrepository.bitrepository.Bitrepository; - -public class BitrepositoryTest3 { - - public static void main(String[] args) { - - //File configDir = new File("/home/svc/bitmag-releasetest-conf"); - //File bitmagKeyfile = new File(configDir, "client-certificate.pem"); - //Bitrepository bitrep = new Bitrepository(configDir, bitmagKeyfile); - //bitrep.shutdown(); - - //File configDir = new File("/home/svc/bitrepository-quickstart/commandline/conf"); - //File bitmagKeyfile = new File(configDir, "client-certificate.pem"); - //Bitrepository bitrep = new Bitrepository(configDir, null, 1); - //bitrep.shutdown(); - - BitmagArcRepositoryClient client = new BitmagArcRepositoryClient(); - Bitrepository bitrep = client.getBitrepository(); - boolean succes = bitrep.existsInCollection("packageId", "dvds"); - System.out.println(succes); - List pillarList = bitrep.getCollectionPillars("dvds"); - for (String id: pillarList) { - System.out.println(id); - } - - - - - - List fileList = bitrep.getFileIds("dvds"); - - if (fileList != null) { - System.out.println("Found '" + fileList.size() + "' ids"); - for (String id: fileList) { - System.out.println(id); - } - } else { - System.out.println("No fileList returned"); - } - - - bitrep.shutdown(); - - } -} \ No newline at end of file diff --git a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/GetFileIDsListFormatter.java b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/GetFileIDsListFormatter.java index 2ae9769f4d..e50f97abb2 100644 --- a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/GetFileIDsListFormatter.java +++ b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/GetFileIDsListFormatter.java @@ -12,10 +12,8 @@ public class GetFileIDsListFormatter implements GetFileIDsOutputFormatter { List result = new ArrayList(); - //private OutputHandler outputHandler; public GetFileIDsListFormatter(OutputHandler outputHandler) { - //this.outputHandler = outputHandler; } @Override @@ -28,9 +26,7 @@ public void formatResult(Collection results) { result.add(a.getID()); } } - public List getFoundIds() { return result; - } - + } } diff --git a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/GetFileIDsNoFormatter.java b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/GetFileIDsNoFormatter.java new file mode 100644 index 0000000000..d50910a79b --- /dev/null +++ b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/GetFileIDsNoFormatter.java @@ -0,0 +1,20 @@ +package dk.netarkivet.common.distribute.arcrepository.bitrepository; + +import java.util.Collection; +import org.bitrepository.commandline.output.OutputHandler; +import org.bitrepository.commandline.outputformatter.GetFileIDsOutputFormatter; +import org.bitrepository.commandline.resultmodel.FileIDsResult; + +public class GetFileIDsNoFormatter implements GetFileIDsOutputFormatter { + + public GetFileIDsNoFormatter(OutputHandler outputHandler) { + } + + @Override + public void formatHeader() { + } + + @Override + public void formatResult(Collection results) { + } +} diff --git a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository.java b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository.java index e96e2d69dc..90f6313cb6 100644 --- a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository.java +++ b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository.java @@ -1,56 +1,156 @@ package dk.netarkivet.common.distribute.arcrepository.bitrepository; import java.io.File; +import java.util.List; + import dk.netarkivet.common.distribute.arcrepository.bitrepository.Bitrepository; /** * - * netarkiv-pillars: - checksum2 - sbdisk1 + * * */ - - public class TestBitrepository { public static void main(String[] args) { + //runnerOne(); + runnerTwo(); + } + public static void runnerTwo(){ + File configDir = new File("/home/svc/devel/bitrepository-quickstart-netarchivesuite/commandline/conf"); + String bitmagKeyfilename = "dummy-certificate.pem"; + //Bitrepository bitrep = new Bitrepository(configDir, null, 1, "sbdisk1"); + Bitrepository bitrep = new Bitrepository(configDir, bitmagKeyfilename, 1, "file1-pillar"); + int i=0; + for (String col: bitrep.getKnownCollections()) { + i++; + System.out.println("col " + i + ": " + col); + } + i=0; + for ( String pillar: bitrep.getCollectionPillars("books")) { + i++; + System.out.println("pillar " + i + ": " + pillar); + } + + if (bitrep.existsInCollection("47-metadata-1.warc", "books")){ + System.out.println("47-metadata-1.warc found in collection books"); + } else { + System.out.println("47-metadata-1.warc NOT found in collection books"); + } + System.out.println("books-ids:"); + for (String id: bitrep.getFileIds("books")) { + System.out.println(id); + } + + + + bitrep.shutdown(); + } + + /** + * TEST That integrates with our bitrepository integration system. + * netarkiv-pillars: + checksum2 + sbdisk1 + */ + public static void runnerOne(){ + File configDir = new File("/home/svc/bitmag-releasetest-conf"); + String bitmagKeyfilename = "client-certificate.pem"; + Bitrepository bitrep = new Bitrepository(configDir, bitmagKeyfilename, 1, "sbdisk1"); + for (String col: bitrep.getKnownCollections()) { + System.out.println(col); + } + + for ( String pillar: bitrep.getCollectionPillars("netarkiv")) { + System.out.println(pillar); + } + System.out.println(); + System.out.println("netarkiv-pillars:"); + for (String col: bitrep.getCollectionPillars("netarkiv")) { + System.out.println(col); + } + + System.out.println("netarkiv-ids:"); + for (String id: bitrep.getFileIds("netarkiv")) { + System.out.println(id); + } + + + //bitrep.shutdown(); + + /* + File configDir = new File("/home/svc/bitrepository-quickstart/commandline/conf"); + //File bitmagKeyfile = new File(configDir, "client-certificate.pem"); + Bitrepository bitrep = new Bitrepository(configDir, null, 1, "kbpillar-test-linux"); + bitrep.shutdown(); + */ + //BitmagArcRepositoryClient client = new BitmagArcRepositoryClient(); + + + bitrep.shutdown(); + + } + + public static void runnerThree() { + File configDir = new File("/home/svc/bitmag-releasetest-conf"); + String bitmagKeyfilename = "client-certificate.pem"; + Bitrepository bitrep = new Bitrepository(configDir, bitmagKeyfilename, 1, "kbpillar-test-linux"); + bitrep.shutdown(); + } + public static void runnerFour() { + + //File configDir = new File("/home/svc/bitmag-releasetest-conf"); + //File bitmagKeyfile = new File(configDir, "client-certificate.pem"); + //Bitrepository bitrep = new Bitrepository(configDir, bitmagKeyfile); + //bitrep.shutdown(); + + //File configDir = new File("/home/svc/bitrepository-quickstart/commandline/conf"); + //File bitmagKeyfile = new File(configDir, "client-certificate.pem"); + //Bitrepository bitrep = new Bitrepository(configDir, null, 1); + //bitrep.shutdown(); + + BitmagArcRepositoryClient client = new BitmagArcRepositoryClient(); + Bitrepository bitrep = client.getBitrepository(); + boolean succes = bitrep.existsInCollection("packageId", "dvds"); + System.out.println(succes); + List pillarList = bitrep.getCollectionPillars("dvds"); + for (String id: pillarList) { + System.out.println(id); + } - File configDir = new File("/home/svc/bitmag-releasetest-conf"); - File bitmagKeyfile = new File(configDir, "client-certificate.pem"); - Bitrepository bitrep = new Bitrepository(configDir, bitmagKeyfile, 1, "sbdisk1"); - for (String col: bitrep.getKnownCollections()) { - System.out.println(col); - } - - for ( String pillar: bitrep.getCollectionPillars("netarkiv")) { - System.out.println(pillar); - } - System.out.println(); - System.out.println("netarkiv-pillars:"); - for (String col: bitrep.getCollectionPillars("netarkiv")) { - System.out.println(col); - } - - System.out.println("netarkiv-ids:"); - for (String id: bitrep.getFileIds("netarkiv")) { - System.out.println(id); - } - - - //bitrep.shutdown(); - - /* - File configDir = new File("/home/svc/bitrepository-quickstart/commandline/conf"); - //File bitmagKeyfile = new File(configDir, "client-certificate.pem"); - Bitrepository bitrep = new Bitrepository(configDir, null, 1, "kbpillar-test-linux"); - bitrep.shutdown(); - */ - //BitmagArcRepositoryClient client = new BitmagArcRepositoryClient(); - - - bitrep.shutdown(); - - } + List fileList = bitrep.getFileIds("dvds"); + + if (fileList != null) { + System.out.println("Found '" + fileList.size() + "' ids"); + for (String id: fileList) { + System.out.println(id); + } + } else { + System.out.println("No fileList returned"); + } + bitrep.shutdown(); + + } + + public static void runnerFive() { + //File configDir = new File("/home/svc/bitmag-releasetest-conf"); + //File bitmagKeyfile = new File(configDir, "client-certificate.pem"); + //Bitrepository bitrep = new Bitrepository(configDir, bitmagKeyfile); + //bitrep.shutdown(); + + File configDir = new File("/home/svc/bitrepository-quickstart/commandline/conf"); + //File bitmagKeyfile = new File(configDir, "client-certificate.pem"); + Bitrepository bitrep = new Bitrepository(configDir, null, 1, "kbpillar-test-linux"); + bitrep.shutdown(); + + + + BitmagArcRepositoryClient client = new BitmagArcRepositoryClient(); + + } + +} + + diff --git a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository2.java b/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository2.java deleted file mode 100644 index 72bf9ae6c3..0000000000 --- a/common/common-core/src/main/java/dk/netarkivet/common/distribute/arcrepository/bitrepository/TestBitrepository2.java +++ /dev/null @@ -1,30 +0,0 @@ -package dk.netarkivet.common.distribute.arcrepository.bitrepository; - -import java.io.File; -import dk.netarkivet.common.distribute.arcrepository.bitrepository.Bitrepository; - -public class TestBitrepository2 { - - // FIXME kan kun køres hvis xmlParserAPIs-2.6.2.jar (fra 2003) ikke findes i classpath (skal bruge xercesImpl-2.9.1.jar istedet) - // Jf. http://stackoverflow.com/questions/20473689/org-xml-sax-saxnotrecognizedexception-feature-http-javax-xml-xmlconstants-fe - - public static void main(String[] args) { - - //File configDir = new File("/home/svc/bitmag-releasetest-conf"); - //File bitmagKeyfile = new File(configDir, "client-certificate.pem"); - //Bitrepository bitrep = new Bitrepository(configDir, bitmagKeyfile); - //bitrep.shutdown(); - - File configDir = new File("/home/svc/bitrepository-quickstart/commandline/conf"); - //File bitmagKeyfile = new File(configDir, "client-certificate.pem"); - Bitrepository bitrep = new Bitrepository(configDir, null, 1, "kbpillar-test-linux"); - bitrep.shutdown(); - - - - BitmagArcRepositoryClient client = new BitmagArcRepositoryClient(); - - - } - - } diff --git a/common/common-core/src/main/resources/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClientSettings.xml b/common/common-core/src/main/resources/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClientSettings.xml index e0ae7ded8c..363089b85d 100644 --- a/common/common-core/src/main/resources/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClientSettings.xml +++ b/common/common-core/src/main/resources/dk/netarkivet/common/distribute/arcrepository/bitrepository/BitmagArcRepositoryClientSettings.xml @@ -36,11 +36,15 @@ National Library. - + + 1 arcrepositoryTemp dvds + diff --git a/common/common-test/src/main/java/TestBitrepository.java b/common/common-test/src/main/java/TestBitrepository.java deleted file mode 100644 index 05928e6f82..0000000000 --- a/common/common-test/src/main/java/TestBitrepository.java +++ /dev/null @@ -1,15 +0,0 @@ -import java.io.File; - -import dk.netarkivet.common.distribute.arcrepository.bitrepository.Bitrepository; - -public class TestBitrepository { - - public static void main(String[] args) { - File configDir = new File("/home/svc/bitmag-releasetest-conf"); - File bitmagKeyfile = new File(configDir, "client-certificate.pem"); - Bitrepository bitrep = new Bitrepository(configDir, bitmagKeyfile, 1, "kbpillar-test-linux"); - bitrep.shutdown(); - - } - -}