diff --git a/fcrepo-connector-file/pom.xml b/fcrepo-connector-file/pom.xml new file mode 100644 index 0000000000..4d6a78638e --- /dev/null +++ b/fcrepo-connector-file/pom.xml @@ -0,0 +1,70 @@ + + 4.0.0 + + org.fcrepo + fcrepo + 4.0.0-alpha-4-SNAPSHOT + + fcrepo-connector-file + Fedora Repository FileSystem Connector Module + The Fedora Commons repository filesystem connector module: Provides repository projection over hierarchical files/directories on the filesystem. + jar + + + + + org.fcrepo + fcrepo-kernel + ${project.version} + + + javax.jcr + jcr + + + + org.fcrepo + fcrepo-http-api + ${project.version} + test + + + org.fcrepo + fcrepo-http-commons + ${project.version} + test + + + org.fcrepo + fcrepo-http-commons + ${project.version} + test-jar + test + + + javax.inject + javax.inject + test + + + javax + javaee-api + test + + + org.springframework + spring-test + + + org.springframework + spring-beans + + + ch.qos.logback + logback-classic + test + + + + + diff --git a/fcrepo-connector-file/src/main/java/org/fcrepo/connector/file/FedoraFileSystemConnector.java b/fcrepo-connector-file/src/main/java/org/fcrepo/connector/file/FedoraFileSystemConnector.java new file mode 100644 index 0000000000..5b1e72328e --- /dev/null +++ b/fcrepo-connector-file/src/main/java/org/fcrepo/connector/file/FedoraFileSystemConnector.java @@ -0,0 +1,144 @@ +/** + * Copyright 2013 DuraSpace, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.fcrepo.connector.file; + +import static org.fcrepo.jcr.FedoraJcrTypes.CONTENT_DIGEST; +import static org.fcrepo.jcr.FedoraJcrTypes.CONTENT_SIZE; +import static org.fcrepo.jcr.FedoraJcrTypes.FEDORA_BINARY; +import static org.fcrepo.jcr.FedoraJcrTypes.FEDORA_DATASTREAM; +import static org.fcrepo.jcr.FedoraJcrTypes.FEDORA_RESOURCE; +import static org.fcrepo.jcr.FedoraJcrTypes.JCR_CREATED; +import static org.fcrepo.jcr.FedoraJcrTypes.JCR_LASTMODIFIED; +import static org.fcrepo.kernel.utils.ContentDigest.asURI; +import static org.modeshape.jcr.api.JcrConstants.JCR_DATA; +import static org.modeshape.jcr.api.JcrConstants.JCR_PRIMARY_TYPE; +import static org.modeshape.jcr.api.JcrConstants.NT_FILE; +import static org.modeshape.jcr.api.JcrConstants.NT_RESOURCE; + +import java.util.Map; + +import org.infinispan.schematic.document.Document; +import org.modeshape.connector.filesystem.FileSystemConnector; +import org.modeshape.jcr.federation.spi.DocumentReader; +import org.modeshape.jcr.federation.spi.DocumentWriter; +import org.modeshape.jcr.value.BinaryValue; +import org.modeshape.jcr.value.Name; +import org.modeshape.jcr.value.Property; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class extends the {@link FileSystemConnector} to enable the autocreation of Fedora-specific datastream and + * content properties. + * + * @author Andrew Woods + * Date: 1/30/14 + */ +public class FedoraFileSystemConnector extends FileSystemConnector { + + private static final Logger LOGGER = LoggerFactory.getLogger(FedoraFileSystemConnector.class); + + + /** + * This method returns the object/document for the node with the federated arg 'id'. + *

+ * Additionally, this method adds Fedora datastream and content properties to the result of the parent class + * implementation. + */ + @Override + public Document getDocumentById(final String id) { + LOGGER.debug("Getting Federated document: {}", id); + if (null == id || id.isEmpty()) { + LOGGER.warn("Can not get document with null id"); + return null; + } + + final Document doc = super.getDocumentById(id); + + final DocumentReader docReader = readDocument(doc); + final DocumentWriter docWriter = writeDocument(doc); + + final String primaryType = docReader.getPrimaryTypeName(); + + if (!docReader.getMixinTypeNames().contains(FEDORA_RESOURCE)) { + LOGGER.trace("Adding mixin: {}, to {}", FEDORA_RESOURCE, id); + docWriter.addMixinType(FEDORA_RESOURCE); + } + + // Is Fedora Datastream? + if (primaryType.equals(NT_FILE)) { + decorateDatastreamNode(docReader, docWriter); + + // Is Fedora Content? + } else if (primaryType.equals(NT_RESOURCE)) { + decorateContentNode(docReader, docWriter); + } + + // Persist new properties + if (!isReadonly()) { + saveProperties(docReader); + } + + return docWriter.document(); + } + + private static void decorateDatastreamNode(final DocumentReader docReader, final DocumentWriter docWriter) { + if (!docReader.getMixinTypeNames().contains(FEDORA_DATASTREAM)) { + LOGGER.trace("Adding mixin: {}, to {}", FEDORA_DATASTREAM, docReader.getDocumentId()); + docWriter.addMixinType(FEDORA_DATASTREAM); + } + } + + private static void decorateContentNode(final DocumentReader docReader, final DocumentWriter docWriter) { + if (!docReader.getMixinTypeNames().contains(FEDORA_BINARY)) { + LOGGER.trace("Adding mixin: {}, to {}", FEDORA_BINARY, docReader.getDocumentId()); + docWriter.addMixinType(FEDORA_BINARY); + } + + if (null == docReader.getProperty(CONTENT_DIGEST)) { + final BinaryValue binaryValue = getBinaryValue(docReader); + final String dsChecksum = binaryValue.getHexHash(); + final String dsURI = asURI("SHA-1", dsChecksum).toString(); + + LOGGER.trace("Adding {} property of {} to {}", CONTENT_DIGEST, dsURI, docReader.getDocumentId()); + docWriter.addProperty(CONTENT_DIGEST, dsURI); + } + + if (null == docReader.getProperty(CONTENT_SIZE)) { + final BinaryValue binaryValue = getBinaryValue(docReader); + final long binarySize = binaryValue.getSize(); + + LOGGER.trace("Adding {} property of {} to {}", CONTENT_SIZE, binarySize, docReader.getDocumentId()); + docWriter.addProperty(CONTENT_SIZE, binarySize); + } + + LOGGER.debug("Decorated data property at path: {}", docReader.getDocumentId()); + } + + private static BinaryValue getBinaryValue(final DocumentReader docReader) { + final Property binaryProperty = docReader.getProperty(JCR_DATA); + return (BinaryValue) binaryProperty.getFirstValue(); + } + + private void saveProperties(final DocumentReader docReader) { + LOGGER.trace("Persisting properties for {}", docReader.getDocumentId()); + final Map properties = docReader.getProperties(); + final ExtraProperties extraProperties = extraPropertiesFor(docReader.getDocumentId(), true); + extraProperties.addAll(properties).except(JCR_PRIMARY_TYPE, JCR_CREATED, JCR_LASTMODIFIED, JCR_DATA); + extraProperties.save(); + } + +} diff --git a/fcrepo-connector-file/src/test/java/org/fcrepo/connector/file/FedoraFileSystemConnectorTest.java b/fcrepo-connector-file/src/test/java/org/fcrepo/connector/file/FedoraFileSystemConnectorTest.java new file mode 100644 index 0000000000..49c04e2272 --- /dev/null +++ b/fcrepo-connector-file/src/test/java/org/fcrepo/connector/file/FedoraFileSystemConnectorTest.java @@ -0,0 +1,168 @@ +/** + * Copyright 2013 DuraSpace, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.fcrepo.connector.file; + +import org.junit.AfterClass; +import org.infinispan.schematic.document.Document; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mock; +import org.modeshape.jcr.ExecutionContext; +import org.modeshape.jcr.api.nodetype.NodeTypeManager; +import org.modeshape.jcr.cache.document.DocumentTranslator; +import org.modeshape.jcr.federation.spi.ExtraPropertiesStore; +import org.modeshape.jcr.value.BinaryValue; +import org.modeshape.jcr.value.NameFactory; +import org.modeshape.jcr.value.Property; +import org.modeshape.jcr.value.ValueFactories; +import org.modeshape.jcr.value.basic.BasicName; +import org.slf4j.Logger; + +import javax.jcr.NamespaceRegistry; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Path; + +import static java.nio.file.Files.createTempDirectory; +import static java.nio.file.Files.createTempFile; +import static org.fcrepo.http.commons.test.util.TestHelpers.setField; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.eq; +import static org.mockito.MockitoAnnotations.initMocks; +import static org.modeshape.jcr.api.JcrConstants.JCR_DATA; +import static org.modeshape.jcr.api.JcrConstants.NT_FILE; +import static org.modeshape.jcr.api.JcrConstants.NT_RESOURCE; +import static org.slf4j.LoggerFactory.getLogger; + +/** + * @author Andrew Woods + * Date: 2/3/14 + */ +public class FedoraFileSystemConnectorTest { + + private FedoraFileSystemConnector connector; + + private static Path directoryPath; + + private static File tmpFile; + + @Mock + private NamespaceRegistry mockRegistry; + + @Mock + private NodeTypeManager mockNodeTypeManager; + + @Mock + private DocumentTranslator mockTranslator; + + @Mock + private NameFactory mockNameFactory; + + @Mock + private ValueFactories mockValueFactories; + + @Mock + private ExtraPropertiesStore mockExtraPropertiesStore; + + @Mock + private Property binaryProperty; + + @Mock + private BinaryValue binaryValue; + + private ExecutionContext mockContext = new ExecutionContext(); + + private static final Logger logger = + getLogger(FedoraFileSystemConnectorTest.class); + + @BeforeClass + public static void beforeClass() throws IOException { + directoryPath = createTempDirectory("fedora-filesystemtest"); + tmpFile = + createTempFile(directoryPath, "fedora-filesystemtestfile", + "txt").toFile(); + try (FileOutputStream outputStream = new FileOutputStream(tmpFile)) { + outputStream.write("hello".getBytes()); + } catch (final IOException e) { + logger.error("Error creating: {} - {}", tmpFile.getAbsolutePath(), + e.getMessage()); + } + } + + @AfterClass + public static void afterClass() { + try { + tmpFile.delete(); + } catch (final Exception e) { + logger.error("Error deleting: " + tmpFile.getAbsolutePath() + + " - " + e.getMessage()); + } + } + + @Before + public void setUp() throws Exception { + initMocks(this); + + connector = new FedoraFileSystemConnector(); + setField(connector, "directoryPath", directoryPath.toString()); + setField(connector, "translator", mockTranslator); + setField(connector, "context", mockContext); + setField(connector, "extraPropertiesStore", mockExtraPropertiesStore); + setField(mockTranslator, "names", mockNameFactory); + connector.initialize(mockRegistry, mockNodeTypeManager); + } + + @Test + public void testGetDocumentByIdNull() throws Exception { + final Document doc = connector.getDocumentById(null); + assertNull(doc); + } + + @Test + public void testGetDocumentByIdDatastream() throws Exception { + when(mockTranslator.getPrimaryTypeName(any(Document.class))) + .thenReturn(NT_FILE); + when(mockNameFactory.create(anyString())).thenReturn( + new BasicName("", tmpFile.getName())); + + final Document doc = connector.getDocumentById("/" + tmpFile.getName()); + assertNotNull(doc); + } + + @Test + public void testGetDocumentByIdContent() throws Exception { + when(mockTranslator.getPrimaryTypeName(any(Document.class))) + .thenReturn(NT_RESOURCE); + when(mockNameFactory.create(anyString())).thenReturn( + new BasicName("", tmpFile.getName())); + + when(binaryProperty.getFirstValue()).thenReturn(binaryValue); + when(mockTranslator.getProperty(any(Document.class), eq(JCR_DATA))) + .thenReturn(binaryProperty); + + final Document doc = connector.getDocumentById("/" + tmpFile.getName()); + assertNotNull(doc); + } + +} diff --git a/fcrepo-connector-file/src/test/java/org/fcrepo/integration/connector/file/FedoraFileSystemConnectorIT.java b/fcrepo-connector-file/src/test/java/org/fcrepo/integration/connector/file/FedoraFileSystemConnectorIT.java new file mode 100644 index 0000000000..cddb861515 --- /dev/null +++ b/fcrepo-connector-file/src/test/java/org/fcrepo/integration/connector/file/FedoraFileSystemConnectorIT.java @@ -0,0 +1,258 @@ +/** + * Copyright 2013 DuraSpace, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.fcrepo.integration.connector.file; + +import static java.lang.System.clearProperty; +import static java.lang.System.getProperty; +import static java.lang.System.setProperty; +import static org.fcrepo.jcr.FedoraJcrTypes.CONTENT_SIZE; +import static org.fcrepo.jcr.FedoraJcrTypes.FEDORA_BINARY; +import static org.fcrepo.jcr.FedoraJcrTypes.FEDORA_DATASTREAM; +import static org.fcrepo.kernel.utils.ContentDigest.asURI; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.modeshape.common.util.SecureHash.getHash; +import static org.modeshape.common.util.SecureHash.Algorithm.SHA_1; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.security.NoSuchAlgorithmException; +import java.util.Collection; +import java.util.Iterator; + +import javax.inject.Inject; +import javax.jcr.Node; +import javax.jcr.PathNotFoundException; +import javax.jcr.Property; +import javax.jcr.Repository; +import javax.jcr.RepositoryException; +import javax.jcr.Session; +import javax.jcr.nodetype.NodeType; + +import org.fcrepo.kernel.FedoraObject; +import org.fcrepo.kernel.FedoraResource; +import org.fcrepo.kernel.rdf.GraphSubjects; +import org.fcrepo.kernel.rdf.impl.DefaultGraphSubjects; +import org.fcrepo.kernel.services.DatastreamService; +import org.fcrepo.kernel.services.NodeService; +import org.fcrepo.kernel.services.ObjectService; +import org.fcrepo.kernel.utils.FixityResult; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + + + + +/** + * @author Andrew Woods + * Date: 2/3/14 + */ +@ContextConfiguration({"/spring-test/repo.xml"}) +@RunWith(SpringJUnit4ClassRunner.class) +public class FedoraFileSystemConnectorIT { + + @Inject + private Repository repo; + + @Inject + private NodeService nodeService; + + @Inject + private ObjectService objectService; + + @Inject + private DatastreamService datastreamService; + + private final String testFile = "/federated/config/testing/repository.json"; + + private final static String PROP_TEST_DIR = "fcrepo.test.dir"; + + @BeforeClass + public static void beforeClass() { + // Note: This property is used in the repository.json + setProperty(PROP_TEST_DIR, new File("target/test-classes").getAbsolutePath()); + } + + @AfterClass + public static void afterClass() { + clearProperty(PROP_TEST_DIR); + } + + @Test + public void testGetFederatedObject() throws RepositoryException { + final Session session = repo.login(); + + final FedoraObject object = objectService.getObject(session, testFile); + assertNotNull(object); + + final Node node = object.getNode(); + final NodeType[] mixins = node.getMixinNodeTypes(); + assertEquals(2, mixins.length); + + boolean found = false; + for (final NodeType nodeType : mixins) { + if (nodeType.getName().equals(FEDORA_DATASTREAM)) { + found = true; + } + } + assertTrue("Mixin not found: " + FEDORA_DATASTREAM, found); + + session.save(); + session.logout(); + } + + @Test + public void testGetFederatedContent() throws RepositoryException { + final Session session = repo.login(); + + final Node node = datastreamService.getDatastreamNode(session, testFile + "/jcr:content"); + assertNotNull(node); + + final NodeType[] mixins = node.getMixinNodeTypes(); + assertEquals(2, mixins.length); + + boolean found = false; + for (final NodeType nodeType : mixins) { + if (nodeType.getName().equals(FEDORA_BINARY)) { + found = true; + } + } + assertTrue("Mixin not found: " + FEDORA_BINARY, found); + + final Property size = node.getProperty(CONTENT_SIZE); + + final File file = new File(testFile.replace("/federated", "target/test-classes")); + assertTrue(file.getAbsolutePath(), file.exists()); + assertEquals(file.length(), size.getLong()); + + session.save(); + session.logout(); + } + + @Test + public void testWriteProperty() throws RepositoryException { + final Session session = repo.login(); + + final FedoraResource object = nodeService.getObject(session, testFile); + assertNotNull(object); + + final String sparql = "PREFIX fedora: " + + "INSERT DATA { " + + " " + + "fedora:name " + + "'some-test-name' }"; + + // Write the properties + object.updatePropertiesDataset(new DefaultGraphSubjects(session), + sparql); + + // Verify + final Property property = object.getNode().getProperty("fedora:name"); + assertNotNull(property); + assertEquals("some-test-name", property.getValues()[0].toString()); + + session.save(); + session.logout(); + } + + @Test + public void testRemoveProperty() throws RepositoryException { + Session session = repo.login(); + + final FedoraResource object = nodeService.getObject(session, testFile); + assertNotNull(object); + + final String sparql = "PREFIX fedora: " + + "INSERT DATA { " + + " " + + "fedora:remove " + + "'some-property-to-remove' }"; + + // Write the properties + final GraphSubjects graphSubjects = new DefaultGraphSubjects(session); + object.updatePropertiesDataset(graphSubjects, sparql); + + // Verify property exists + final Property property = object.getNode().getProperty("fedora:remove"); + assertNotNull(property); + assertEquals("some-property-to-remove", property.getValues()[0].getString()); + + final String sparqlRemove = "PREFIX fedora: " + + "DELETE {" + + " fedora:remove ?s " + + "} WHERE { " + + " fedora:remove ?s" + + "}"; + + // Remove the properties + final GraphSubjects graphSubjectsRemove = new DefaultGraphSubjects(session); + object.updatePropertiesDataset(graphSubjectsRemove, sparqlRemove); + + // Persist the object (although the propery will be removed from memory without this.) + session.save(); + + // Verify + boolean thrown = false; + try{ + object.getNode().getProperty("fedora:remove"); + } catch (PathNotFoundException e){ + thrown = true; + } + assertTrue("Exception expected - property should be missing", thrown); + + session.logout(); + } + + @Test + public void testFixity() throws RepositoryException, IOException, NoSuchAlgorithmException { + final Session session = repo.login(); + + final Node node = datastreamService.getDatastreamNode(session, testFile + "/jcr:content"); + assertNotNull(node); + + final File file = + new File(getProperty(PROP_TEST_DIR), testFile.replace("federated", + "")); + final byte[] hash = getHash(SHA_1, file); + + final URI calculatedChecksum = asURI(SHA_1.toString(), hash); + + final Collection results = + datastreamService + .getFixity(node, calculatedChecksum, file.length()); + assertNotNull(results); + + assertFalse("Found no results!", results.isEmpty()); + + final Iterator resultIterator = results.iterator(); + while (resultIterator.hasNext()) { + final FixityResult result = resultIterator.next(); + assertTrue(result.isSuccess()); + } + + session.save(); + session.logout(); + } + +} diff --git a/fcrepo-connector-file/src/test/resources/config/testing/repository.json b/fcrepo-connector-file/src/test/resources/config/testing/repository.json new file mode 100644 index 0000000000..c9d60ed838 --- /dev/null +++ b/fcrepo-connector-file/src/test/resources/config/testing/repository.json @@ -0,0 +1,32 @@ +{ + "name" : "repo", + "jndiName" : "", + "workspaces" : { + "predefined" : [], + "default" : "default", + "allowCreation" : true + }, + "externalSources" : { + "federated-directory" : { + "classname" : "org.fcrepo.connector.file.FedoraFileSystemConnector", + "directoryPath" : "${fcrepo.test.dir:must-be-provided}", + "projections" : [ "default:/federated => /" ], + "contentBasedSha1" : "false", + "readonly" : false, + "extraPropertiesStorage" : "json" + } + }, + "storage" : { + "binaryStorage" : { + "type" : "transient", + "minimumBinarySizeInBytes" : 40 + } + }, + "security" : { + "anonymous" : { + "roles" : ["readonly","readwrite","admin"], + "useOnFailedLogin" : true + } + }, + "node-types" : ["fedora-node-types.cnd"] +} \ No newline at end of file diff --git a/fcrepo-connector-file/src/test/resources/logback-test.xml b/fcrepo-connector-file/src/test/resources/logback-test.xml new file mode 100644 index 0000000000..5fdd0336e7 --- /dev/null +++ b/fcrepo-connector-file/src/test/resources/logback-test.xml @@ -0,0 +1,16 @@ + + + + + + %p %d{HH:mm:ss.SSS} \(%c{0}\) %m%n + + + + + + + + + + diff --git a/fcrepo-connector-file/src/test/resources/spring-test/repo.xml b/fcrepo-connector-file/src/test/resources/spring-test/repo.xml new file mode 100644 index 0000000000..6c3a3f5316 --- /dev/null +++ b/fcrepo-connector-file/src/test/resources/spring-test/repo.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + diff --git a/fcrepo-webapp/pom.xml b/fcrepo-webapp/pom.xml index 3125a268a9..66bf278496 100644 --- a/fcrepo-webapp/pom.xml +++ b/fcrepo-webapp/pom.xml @@ -92,6 +92,12 @@ fcrepo-storage-policy ${project.version} + + org.fcrepo + fcrepo-connector-file + ${project.version} + runtime + org.eclipse.jetty jetty-webapp diff --git a/pom.xml b/pom.xml index 7147dff47b..8e457f0f56 100644 --- a/pom.xml +++ b/pom.xml @@ -74,6 +74,7 @@ fcrepo-auth-roles-common fcrepo-auth-roles-basic fcrepo-kernel-api + fcrepo-connector-file