/
Bitrepository.java
480 lines (421 loc) · 20.9 KB
/
Bitrepository.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
package dk.netarkivet.common.distribute.arcrepository.bitrepository;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.jms.JMSException;
import dk.netarkivet.common.exceptions.*;
import org.bitrepository.access.AccessComponentFactory;
import org.bitrepository.access.getchecksums.BlockingGetChecksumsClient;
import org.bitrepository.access.getchecksums.GetChecksumsClient;
import org.bitrepository.access.getchecksums.conversation.ChecksumsCompletePillarEvent;
import org.bitrepository.access.getfile.GetFileClient;
import org.bitrepository.access.getfileids.GetFileIDsClient;
import org.bitrepository.bitrepositoryelements.ChecksumDataForFileTYPE;
import org.bitrepository.bitrepositoryelements.ChecksumSpecTYPE;
import org.bitrepository.bitrepositoryelements.FilePart;
import org.bitrepository.client.eventhandler.BlockingEventHandler;
import org.bitrepository.client.eventhandler.ContributorEvent;
import org.bitrepository.client.eventhandler.OperationEvent;
import org.bitrepository.client.eventhandler.OperationEvent.OperationEventType;
import org.bitrepository.client.exceptions.NegativeResponseException;
import org.bitrepository.commandline.clients.PagingGetFileIDsClient;
import org.bitrepository.commandline.eventhandler.CompleteEventAwaiter;
import org.bitrepository.commandline.eventhandler.GetFileEventHandler;
import org.bitrepository.commandline.output.DefaultOutputHandler;
import org.bitrepository.commandline.output.OutputHandler;
import org.bitrepository.commandline.outputformatter.GetFileIDsInfoFormatter;
import org.bitrepository.commandline.outputformatter.GetFileIDsOutputFormatter;
import org.bitrepository.common.exceptions.OperationFailedException;
import org.bitrepository.common.settings.Settings;
import org.bitrepository.common.settings.SettingsProvider;
import org.bitrepository.common.settings.XMLFileSettingsLoader;
import org.bitrepository.common.utils.ChecksumUtils;
import org.bitrepository.common.utils.SettingsUtils;
import org.bitrepository.modify.ModifyComponentFactory;
import org.bitrepository.modify.putfile.BlockingPutFileClient;
import org.bitrepository.modify.putfile.PutFileClient;
import org.bitrepository.protocol.FileExchange;
import org.bitrepository.protocol.ProtocolComponentFactory;
import org.bitrepository.protocol.messagebus.MessageBus;
import org.bitrepository.protocol.security.BasicMessageAuthenticator;
import org.bitrepository.protocol.security.BasicMessageSigner;
import org.bitrepository.protocol.security.BasicOperationAuthorizor;
import org.bitrepository.protocol.security.BasicSecurityManager;
import org.bitrepository.protocol.security.MessageAuthenticator;
import org.bitrepository.protocol.security.MessageSigner;
import org.bitrepository.protocol.security.OperationAuthorizor;
import org.bitrepository.protocol.security.PermissionStore;
import org.bitrepository.protocol.security.SecurityManager;
import org.bitrepository.settings.repositorysettings.ClientSettings;
import org.bitrepository.settings.repositorysettings.Collection;
import dk.netarkivet.common.exceptions.ArgumentNotValid;
/**
* The class for interacting with the BitRepository, e.g. put files, get files, etc.
*/
public class Bitrepository {
/** Logging mechanism. */
private static final Logger logger = Logger.getLogger(Bitrepository.class.getName());
/** The archive settings directory needed to upload to
* a bitmag style repository */
private File settingsDir = null;
/** National bitrepository settings. */
private Settings bitmagSettings = null;
/** The bitrepository component id. */
private final String componentId;
/** The bitmag security manager.*/
private SecurityManager bitMagSecurityManager;
/** The client for performing the PutFile operation.*/
private PutFileClient bitMagPutClient;
/** The client for performing the GetFile operation.*/
private GetFileClient bitMagGetClient;
/** The client for performing the GetFileID operation.*/
private GetFileIDsClient bitMagGetFileIDsClient;
/** The client for performing the GetChecksums operation.*/
private GetChecksumsClient bitMagGetChecksumsClient;
/** The authentication key used by the putfileClient. */
private File privateKeyFile;
/** The message bus used by the putfileClient. */
private MessageBus bitMagMessageBus;
/** The maximum number of failing pillars. Default is 0. */
private final int maxNumberOfFailingPillars;
/**
* Constructor for the BitRepository class.
* @param configDir A Bitrepository settingsdirectory
* @param maxStoreFailures Max number of acceptable store failures
* @param bitmagKeyFile (optional, we hope)
* @throws ArgumentNotValid if configFile is null
*/
public Bitrepository(File configDir, File bitmagKeyfile, int maxStoreFailures) {
ArgumentNotValid.checkExistsDirectory(configDir, "File configDir");
//ArgumentNotValid.checkExistsNormalFile(bitmagKeyfile, "File bitmagKeyfile");
componentId = BitrepositoryUtils.generateComponentID();
maxNumberOfFailingPillars = maxStoreFailures;
this.settingsDir = configDir;
this.privateKeyFile = bitmagKeyfile;
initBitmagSettings();
if (bitmagKeyfile != null){
initBitmagSecurityManager(); // Is this mandatory?
}
bitMagMessageBus = ProtocolComponentFactory.getInstance().getMessageBus(
bitmagSettings, bitMagSecurityManager); // Is bitMagSecurityManager mandatory?
initBitMagClients();
}
/**
* Initialization of the various bitmag client.
*/
private void initBitMagClients() {
bitMagPutClient = ModifyComponentFactory.getInstance().retrievePutClient(
bitmagSettings, bitMagSecurityManager, componentId);
AccessComponentFactory acf = AccessComponentFactory.getInstance();
bitMagGetClient = acf.createGetFileClient(bitmagSettings, bitMagSecurityManager, componentId);
bitMagGetFileIDsClient = acf.createGetFileIDsClient(bitmagSettings, bitMagSecurityManager, componentId);
bitMagGetChecksumsClient = acf.createGetChecksumsClient(bitmagSettings, bitMagSecurityManager, componentId);
}
/**
* Attempts to upload a given file.
*
* @param file The file to upload. Should exist. The packageId is the name of the file
* @param collectionId The Id of the collection to upload to
* @return true if the upload succeeded, false otherwise.
*/
public boolean uploadFile(final File file, final String collectionId) {
//ArgumentCheck.checkExistsNormalFile(file, "File file");
// Does collection exists? If not return false
if (getCollectionPillars(collectionId).isEmpty()) {
logger.warning("The given collection Id does not exist");
return false;
}
boolean success = false;
try {
OperationEventType finalEvent = putTheFile(bitMagPutClient, file, collectionId);
if(finalEvent == OperationEventType.COMPLETE) {
success = true;
logger.info("File '" + file.getAbsolutePath() + "' uploaded successfully. ");
} else {
logger.warning("Upload of file '" + file.getAbsolutePath()
+ "' failed with event-type '" + finalEvent + "'.");
}
} catch (Exception e) {
logger.warning("Unexpected error while storing file '"
+ file.getAbsolutePath() + "': " + e);
success = false;
}
return success;
}
/**
* Upload the file to the uploadserver, initiate the PutFile request, and wait for the
* request to finish.
* @param client the PutFileClient responsible for the put operation.
* @param packageFile The package to upload
* @param collectionID The ID of the collection to upload to.
* @return OperationEventType.FAILED if operation failed; otherwise returns OperationEventType.COMPLETE
* @throws IOException If unable to upload the packageFile to the uploadserver
*/
private OperationEventType putTheFile(PutFileClient client, File packageFile, String collectionID)
throws IOException, URISyntaxException {
FileExchange fileexchange = ProtocolComponentFactory.getInstance().getFileExchange(this.bitmagSettings);
BlockingPutFileClient bpfc = new BlockingPutFileClient(client);
URL url = fileexchange.uploadToServer(packageFile);
String fileId = packageFile.getName();
ChecksumSpecTYPE csSpec = ChecksumUtils.getDefault(this.bitmagSettings);
ChecksumDataForFileTYPE validationChecksum = BitrepositoryUtils.getValidationChecksum(
packageFile,csSpec);
ChecksumSpecTYPE requestChecksum = null;
String putFileMessage = "Putting the file '" + packageFile + "' with the file id '"
+ fileId + "' from Netarchivesuite";
NetarchivesuiteBlockingEventHandler putFileEventHandler = new NetarchivesuiteBlockingEventHandler(collectionID,
maxNumberOfFailingPillars);
try {
bpfc.putFile(collectionID, url, fileId, packageFile.length(), validationChecksum, requestChecksum,
putFileEventHandler, putFileMessage);
} catch (OperationFailedException e) {
logger.log(Level.WARNING, "The putFile Operation was not a complete success (" + putFileMessage + ")."
+ " Checksum whether we accept anyway.", e);
if(putFileEventHandler.hasFailed()) {
return OperationEventType.FAILED;
} else {
return OperationEventType.COMPLETE;
}
} finally {
// delete the uploaded file from server
fileexchange.deleteFromServer(url);
}
logger.info("The putFile Operation succeeded (" + putFileMessage + ")");
return OperationEventType.COMPLETE;
}
/**
* Get a file with a given fileId from a given collection.
* @param fileId A fileId of a package known to exist in the repository
* @param collectionId A given collection in the repository
* @param filePart The part of the file to 'get'. Set to null, if retrieving the whole file.
* @return the file if found. Otherwise an exception is thrown
* @throws IOFailure If not found or an error occurred during the fetch process.
*/
public File getFile(final String fileId, final String collectionId, final FilePart filePart)
throws IOFailure{
//ArgumentCheck.checkNotNullOrEmpty(fileId, "String fileId");
//ArgumentCheck.checkNotNullOrEmpty(collectionId, "String collectionId");
// Does collection exists? If not throw exception
if (getCollectionPillars(collectionId).isEmpty()) {
throw new IOFailure("The given collection Id does not exist");
}
OutputHandler output = new DefaultOutputHandler(Bitrepository.class);
URL fileUrl = getDeliveryUrl(fileId);
// Note that this eventHandler is blocking
CompleteEventAwaiter eventHandler = new GetFileEventHandler(this.bitmagSettings, output);
output.debug("Initiating the GetFile conversation.");
String auditTrailInformation = "Retrieving package '" + fileId + "' from collection '" + collectionId + "'";
bitMagGetClient.getFileFromFastestPillar(collectionId, fileId, filePart, fileUrl, eventHandler,
auditTrailInformation);
OperationEvent finalEvent = eventHandler.getFinish();
if(finalEvent.getEventType() == OperationEventType.COMPLETE) {
File result = null;
try {
result = downloadFile(fileUrl);
} catch (IOException e) {
throw new IOFailure(
"Download was successful, but we failed to create result File: ", e);
}
return result;
} else {
throw new IOFailure("Download of package w/ id '" + fileId + "' failed. Reason: "
+ finalEvent.getInfo());
}
}
/**
* Downloads the file from the URL defined in the conversation.
* @throws IOException
*/
private File downloadFile(URL fileUrl) throws IOException {
File outputFile = File.createTempFile("Extracted", null);
FileExchange fileexchange = getFileExchange(bitmagSettings);
fileexchange.downloadFromServer(outputFile, fileUrl.toExternalForm());
return outputFile;
}
/**
* Generates the URL for where the file should be delivered from the GetFile operation.
* @param fileId The id of the file.
* @return The URL where the file should be located.
*/
private URL getDeliveryUrl(String fileId) {
try {
return getFileExchange(bitmagSettings).getURL(fileId);
} catch (MalformedURLException e) {
throw new IllegalStateException("Could not make an URL for the file '"
+ fileId + "'.", e);
}
}
/**
* Check if a package with the following id exists within a specific collection.
* @param packageId A given packageId
* @param collectionID A given collection ID
* @return true, if a package with the given ID exists within the given collection. Otherwise returns false
*/
public boolean existsInCollection(String packageId, String collectionID) {
//ArgumentCheck.checkNotNullOrEmpty(packageId, "String packageId");
//ArgumentCheck.checkNotNullOrEmpty(collectionID, "String collectionId");
// Does collection exists? If not return false
if (getCollectionPillars(collectionID).isEmpty()) {
logger.warning("The given collection Id does not exist");
return false;
}
OutputHandler output = new DefaultOutputHandler(Bitrepository.class);
output.debug("Instantiation GetFileID outputFormatter.");
// TODO: change to non pagingClient
GetFileIDsOutputFormatter outputFormatter = new GetFileIDsInfoFormatter(output);
long timeout = getClientTimeout(bitmagSettings);
output.debug("Instantiation GetFileID paging client.");
PagingGetFileIDsClient pagingClient = new PagingGetFileIDsClient(
bitMagGetFileIDsClient, timeout, outputFormatter, output);
Boolean success = pagingClient.getFileIDs(collectionID, packageId,
getCollectionPillars(collectionID));
return success;
}
/**
* Check the checksums for a whole collection, or only a single packageId in a collection.
* @param packageID A given package ID (if null, checksums for the whole collection is requested)
* @param collectionID A given collection ID
* @return a map with the results from the pillars
* @throws YggdrasilException If it fails to retrieve the checksums.
*/
public Map<String, ChecksumsCompletePillarEvent> getChecksums(String packageID, String collectionID)
throws IOFailure {
//ArgumentCheck.checkNotNullOrEmpty(collectionID, "String collectionId");
//If packageID = null, checksum is requested for all files in the collection.
if (packageID != null) {
logger.info("Collecting checksums for package '" + packageID + "' in collection '" + collectionID + "'");
} else {
logger.info("Collecting checksums for all packages in collection '" + collectionID + "'");
}
BlockingGetChecksumsClient client = new BlockingGetChecksumsClient(bitMagGetChecksumsClient);
ChecksumSpecTYPE checksumSpec = ChecksumUtils.getDefault(bitmagSettings);
BlockingEventHandler eventhandler = new BlockingEventHandler();
try {
client.getChecksums(collectionID, null, packageID, checksumSpec, null, eventhandler, null);
} catch (NegativeResponseException e) {
throw new IOFailure("Got bad feedback from the bitrepository " + e);
}
int failures = eventhandler.getFailures().size();
int results = eventhandler.getResults().size();
if (failures > 0) {
logger.warning("Got back " + eventhandler.getFailures().size() + " failures");
}
if (results > 0) {
logger.info("Got back " + eventhandler.getResults().size() + " successful responses");
}
Map<String, ChecksumsCompletePillarEvent> resultsMap = new HashMap<String,
ChecksumsCompletePillarEvent>();
for (ContributorEvent e : eventhandler.getResults()) {
ChecksumsCompletePillarEvent event = (ChecksumsCompletePillarEvent) e;
resultsMap.put(event.getContributorID(), event);
}
return resultsMap;
}
/**
* Initialize the BITMAG security manager.
*/
private void initBitmagSecurityManager() {
PermissionStore permissionStore = new PermissionStore();
MessageAuthenticator authenticator = new BasicMessageAuthenticator(permissionStore);
MessageSigner signer = new BasicMessageSigner();
OperationAuthorizor authorizer = new BasicOperationAuthorizor(permissionStore);
if (getPrivateKeyFile() != null) {
bitMagSecurityManager = new BasicSecurityManager(bitmagSettings.getRepositorySettings(),
getPrivateKeyFile().getAbsolutePath(),
authenticator, signer, authorizer, permissionStore,
bitmagSettings.getComponentID());
}
}
private File getPrivateKeyFile() {
return this.privateKeyFile;
}
/**
* Load BitMag settings, if not already done.
*/
private void initBitmagSettings() {
if (bitmagSettings == null) {
SettingsProvider settingsLoader =
new SettingsProvider(
new XMLFileSettingsLoader(
settingsDir.getAbsolutePath()),
componentId);
bitmagSettings = settingsLoader.getSettings();
SettingsUtils.initialize(bitmagSettings);
}
}
/**
* Shutdown the messagebus.
*/
public void shutdown() {
if (bitMagMessageBus != null) {
try {
bitMagMessageBus.close();
} catch (JMSException e) {
logger.warning("JMSException caught during shutdown of messagebus " + e);
}
}
}
/**
* Helper method for reading the list of pillars preserving the given collection.
* @param collectionID The ID of a specific collection.
* @return the list of pillars preserving the collection with the given ID.
*/
private List<String> getCollectionPillars(String collectionID) {
return SettingsUtils.getPillarIDsForCollection(collectionID);
}
/**
* Helper method for computing the clientTimeout. The clientTimeout is the identificationTimeout
* plus the OperationTimeout.
* @param bitmagSettings The bitmagsetting
* @return the clientTimeout
*/
private long getClientTimeout(Settings bitmagSettings) {
ClientSettings clSettings = bitmagSettings.getRepositorySettings().getClientSettings();
return clSettings.getIdentificationTimeout().longValue()
+ clSettings.getOperationTimeout().longValue();
}
private FileExchange getFileExchange(Settings bitmagSettings) {
return ProtocolComponentFactory.getInstance().getFileExchange(
bitmagSettings);
}
/**
* @return a set of known CollectionIDs
*/
public List<String> getKnownCollections() {
List<Collection> knownCollections = bitmagSettings.getRepositorySettings()
.getCollections().getCollection();
List<String> collectionIDs = new ArrayList<String>();
for (Collection c: knownCollections) {
collectionIDs.add(c.getID());
}
return collectionIDs;
}
/**
* @return The default checksum spec from the settings.
*/
public ChecksumSpecTYPE getDefaultChecksum() {
return ChecksumUtils.getDefault(bitmagSettings);
}
//FIXME can this method be made to work?
public List<String> getFileIds(String regex, String collectionID) {
OutputHandler output = new DefaultOutputHandler(Bitrepository.class);
output.debug("Instantiation GetFileID outputFormatter.");
// TODO: change to non pagingClient
GetFileIDsOutputFormatter outputFormatter = new GetFileIDsInfoFormatter(output);
long timeout = getClientTimeout(bitmagSettings);
output.debug("Instantiation GetFileID paging client.");
PagingGetFileIDsClient pagingClient = new PagingGetFileIDsClient(
bitMagGetFileIDsClient, timeout, outputFormatter, output);
Boolean success = pagingClient.getFileIDs(collectionID, "packageId",
getCollectionPillars(collectionID));
return null;
}
}