[performance] move storage binary resources from dbx files to filesys…

…tem. Yields into performance increase for (concurrent) access of binary files (=queries). Internally binary resources are dealt as Streams Patch created by Alex Milowski svn path=/trunk/eXist/; revision=8222
eXist-db · Oct 7, 2008 · ac5472a628a153cd2430e72c243588e0e8b4a920 · ac5472a
1 parent e6f3637
commit ac5472a628a153cd2430e72c243588e0e8b4a920
diff --git a/extensions/fluent/src/org/exist/fluent/Document.java b/extensions/fluent/src/org/exist/fluent/Document.java
@@ -322,9 +322,15 @@ private DocumentImpl moveOrCopy(Folder destination, Name name, boolean copy) {
 	public String contentsAsString() {
 		DBBroker broker = db.acquireBroker();
 		try {
-  
-			return new String(broker.getBinaryResource((BinaryDocument) doc), db.defaultCharacterEncoding);
+  
+			InputStream is = broker.getBinaryResource((BinaryDocument) doc);
+  
+			byte [] data = new byte[(int)broker.getBinaryResourceSize((BinaryDocument) doc)];
+  
+			is.read(data);
+  
+			is.close();
+  
+			return new String(data, db.defaultCharacterEncoding);
 		} catch (UnsupportedEncodingException e) {
 			throw new DatabaseException(e);
+  
+		} catch (IOException e) {
+  
+			throw new DatabaseException(e);
 		} finally {
 			db.releaseBroker(broker);
 		}
diff --git a/extensions/fluent/test/src/org/exist/fluent/DocumentTest.java b/extensions/fluent/test/src/org/exist/fluent/DocumentTest.java
@@ -36,6 +36,16 @@
 		assertEquals("helloworld", original.contentsAsString());
 		assertEquals("helloworld", copy.contentsAsString());
 	}
+  
+
+  
+	@Test public void copy2() {
+  
+		Folder c1 = db.createFolder("/c1"), c2 = db.createFolder("/c2");
+  
+		Document original = c1.documents().load(Name.create("original.xml"), Source.xml("<original/>"));
+  
+		Document copy = original.copy(c2, Name.keepCreate());
+  
+		assertEquals(1, c1.documents().size());
+  
+		assertEquals(1, c2.documents().size());
+  
+		assertEquals("<original/>", original.contentsAsString());
+  
+		assertEquals("<original/>", copy.contentsAsString());
+  
+	}
 	@Test public void move1() {
 		Folder c1 = db.createFolder("/c1"), c2 = db.createFolder("/c2");
@@ -46,5 +56,15 @@
 		assertEquals("/c2/original", doc.path());
 		assertEquals("helloworld", doc.contentsAsString());
 	}
+  
+
+  
+	@Test public void move2() {
+  
+		Folder c1 = db.createFolder("/c1"), c2 = db.createFolder("/c2");
+  
+		Document doc = c1.documents().load(Name.create("original.xml"), Source.xml("<original/>"));
+  
+		doc.move(c2, Name.keepCreate());
+  
+		assertEquals(0, c1.documents().size());
+  
+		assertEquals(1, c2.documents().size());
+  
+		assertEquals("/c2/original.xml", doc.path());
+  
+		assertEquals("<original/>", doc.contentsAsString());
+  
+	}
 }
diff --git a/extensions/modules/src/org/exist/xquery/modules/compression/TarFunction.java b/extensions/modules/src/org/exist/xquery/modules/compression/TarFunction.java
@@ -21,6 +21,7 @@
 */
 package org.exist.xquery.modules.compression;
+  
+import java.io.InputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.Iterator;
@@ -210,9 +211,13 @@ private void tarResource(TarOutputStream tos, DocumentImpl doc,
 			tos.write(strDoc.getBytes());
 		} else if (doc.getResourceType() == DocumentImpl.BINARY_FILE) {
 			// binary file
-  
-			byte[] data = context.getBroker().getBinaryResource(
-  
-					(BinaryDocument) doc);
-  
-			tos.write(data);
+  
+                        InputStream is = context.getBroker().getBinaryResource((BinaryDocument)doc);
+  
+			byte[] data = new byte[16384];
+  
+                        int len = 0;
+  
+                        while ((len=is.read(data,0,data.length))>0) {
+  
+			   tos.write(data,0,len);
+  
+                        }
+  
+                        is.close();
 		}
 		// close the entry in the Tar
@@ -261,4 +266,4 @@ private void tarCollection(TarOutputStream tos, Collection col,
 			tarCollection(tos, childCol, useHierarchy, stripOffset);
 		}
 	}
-  
-}
+  
+}
diff --git a/extensions/modules/src/org/exist/xquery/modules/compression/ZipFunction.java b/extensions/modules/src/org/exist/xquery/modules/compression/ZipFunction.java
@@ -22,6 +22,7 @@
 package org.exist.xquery.modules.compression;
 import java.io.ByteArrayOutputStream;
+  
+import java.io.InputStream;
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.zip.ZipEntry;
@@ -210,9 +211,14 @@ private void zipResource(ZipOutputStream zos, DocumentImpl doc,
 			zos.write(strDoc.getBytes());
 		} else if (doc.getResourceType() == DocumentImpl.BINARY_FILE) {
 			// binary file
-  
-			byte[] data = context.getBroker().getBinaryResource(
+  
+			InputStream is = context.getBroker().getBinaryResource(
 					(BinaryDocument) doc);
-  
-			zos.write(data);
+  
+                        byte [] data = new byte[16384];
+  
+                        int len;
+  
+                        while ((len=is.read(data))>0) {
+  
+			   zos.write(data,0,len);
+  
+                        }
+  
+                        is.close();
 		}
 		// close the entry in the Zip
diff --git a/extensions/modules/src/org/exist/xquery/modules/image/GetThumbnailsFunction.java b/extensions/modules/src/org/exist/xquery/modules/image/GetThumbnailsFunction.java
@@ -2,6 +2,7 @@
 import java.awt.Image;
 import java.awt.image.BufferedImage;
+  
+import java.io.InputStream;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -213,11 +214,10 @@ public boolean accept(File dir, String name) {
 						binImage = (BinaryDocument) docImage;
 						// get a byte array representing the image
-  
-						imgData = dbbroker.getBinaryResource(binImage);
 						try {
-  
-							image = ImageIO.read(new ByteArrayInputStream(
-  
-									imgData));
+  
+                                                   InputStream is = dbbroker.getBinaryResource(binImage);
+  
+							image = ImageIO.read(is);
 						} catch (IOException ioe) {
 							throw new XPathException(getASTNode(),ioe.getMessage());
 						}
diff --git a/src/org/exist/cluster/ClusterCollection.java b/src/org/exist/cluster/ClusterCollection.java
@@ -165,7 +165,11 @@ public void store(Txn transaction, DBBroker broker, IndexInfo info, InputSource
    public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
 			XmldbURI name, byte[] data, String mimeType) throws EXistException,
            PermissionDeniedException, LockException, TriggerException {
+  
+       try {
        return collection.addBinaryResource(transaction, broker, name, data, mimeType);
+  
+       } catch (IOException ex) {
+  
+          throw new EXistException("Cannot add binary due to I/O error.",ex);
+  
+       }
    }
    public Lock getLock() {
diff --git a/src/org/exist/collections/Collection.java b/src/org/exist/collections/Collection.java
@@ -865,7 +865,11 @@ public void removeBinaryResource(Txn transaction, DBBroker broker, DocumentImpl
            if (trigger != null)
                trigger.prepare(Trigger.REMOVE_DOCUMENT_EVENT, broker, transaction, doc.getURI(), doc);
-  
-            broker.removeBinaryResource(transaction, (BinaryDocument) doc);
+  
+            try {
+  
+               broker.removeBinaryResource(transaction, (BinaryDocument) doc);
+  
+            } catch (IOException ex) {
+  
+               throw new PermissionDeniedException("Cannot delete file.");
+  
+            }
            documents.remove(doc.getFileURI().getRawCollectionPath());
            if (trigger != null) {
@@ -1361,29 +1365,29 @@ private DocumentTrigger setupTriggers(DBBroker broker, XmldbURI docUri, boolean
    // Blob
    public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
                XmldbURI docUri, byte[] data, String mimeType)
-  
-            throws EXistException, PermissionDeniedException, LockException, TriggerException {
+  
+            throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException {
        return addBinaryResource(transaction, broker, docUri, data, mimeType, null, null);
    }
    // Blob
    public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
    		XmldbURI docUri, byte[] data, String mimeType, Date created, Date modified)
-  
-            throws EXistException, PermissionDeniedException, LockException, TriggerException {
+  
+            throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException {
        return addBinaryResource(transaction, broker, docUri, 
                                new ByteArrayInputStream(data), mimeType, data.length, created, modified);
    }
    // Streaming
    public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
    		XmldbURI docUri, InputStream is, String mimeType, int size)
-  
-            throws EXistException, PermissionDeniedException, LockException, TriggerException {
+  
+            throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException {
        return addBinaryResource(transaction, broker, docUri, is, mimeType, size, null, null);
    }
    // Streaming
    public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
    		XmldbURI docUri, InputStream is, String mimeType, int size, Date created, Date modified)
-  
-            throws EXistException, PermissionDeniedException, LockException, TriggerException {
+  
+            throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException {
        if (broker.isReadOnly())
            throw new PermissionDeniedException("Database is read-only");
@@ -1400,6 +1404,7 @@ public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
 	        checkPermissions(transaction, broker, oldDoc);
 	        DocumentTrigger trigger = null;
 	        int event = 0;
+  
+/*
 	        if (triggersEnabled) {
 	            CollectionConfiguration config = getConfiguration(broker);
 	            if (config != null) {
@@ -1414,6 +1419,7 @@ public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
 	                }
 	            }
 	        }
+  
+*/
 	        manageDocumentInformation(broker, oldDoc, blob );
 	        DocumentMetadata metadata = blob.getMetadata();
@@ -1438,8 +1444,24 @@ public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
 	        addDocument(transaction, broker, blob);
 	        broker.storeXMLResource(transaction, blob);
+  
+	        if (triggersEnabled) {
+  
+	            CollectionConfiguration config = getConfiguration(broker);
+  
+	            if (config != null) {
+  
+	                event = oldDoc != null ? Trigger.UPDATE_DOCUMENT_EVENT : Trigger.STORE_DOCUMENT_EVENT;
+  
+                    try {
+  
+                        trigger = (DocumentTrigger) config.newTrigger(event, broker, this);
+  
+                    } catch (CollectionConfigurationException e) {
+  
+                        LOG.debug("An error occurred while initializing a trigger for collection " + getURI() + ": " + e.getMessage(), e);
+  
+                    }
+  
+                    if (trigger != null) {
+  
+	                    trigger.prepare(event, broker, transaction, blob.getURI(), blob);
+  
+	                }
+  
+	            }
+  
+	        }
+  
+
-  
-	        broker.closeDocument();
+  
+                // This is no longer needed as the dom.dbx isn't used
+  
+	        //broker.closeDocument();
 	        if (trigger != null) {
 	            trigger.finish(event, broker, transaction, blob.getURI(), blob);
diff --git a/src/org/exist/collections/triggers/XQueryTrigger.java b/src/org/exist/collections/triggers/XQueryTrigger.java
@@ -1,6 +1,7 @@
 package org.exist.collections.triggers;
 import java.io.IOException;
+  
+import java.io.InputStream;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Properties;
@@ -219,7 +220,10 @@ else if (existingDocument instanceof BinaryDocument)
        	{
        		//binary document
        		BinaryDocument bin = (BinaryDocument)existingDocument;
-  
-                byte[] data = context.getBroker().getBinaryResource(bin);
+  
+                        InputStream is = broker.getBinaryResource(bin);
+  
+                        byte [] data = new byte[(int)broker.getBinaryResourceSize(bin)];
+  
+                        is.read(data);
+  
+                        is.close();
                context.declareVariable(bindingPrefix + "document", new Base64Binary(data));
        	}
@@ -302,7 +306,10 @@ else if (document instanceof BinaryDocument)
        	{
        		//Binary document
        		BinaryDocument bin = (BinaryDocument)document;
-  
-                byte[] data = context.getBroker().getBinaryResource(bin);
+  
+                        InputStream is = broker.getBinaryResource(bin);
+  
+                        byte [] data = new byte[(int)broker.getBinaryResourceSize(bin)];
+  
+                        is.read(data);
+  
+                        is.close();
                context.declareVariable(bindingPrefix + "document", new Base64Binary(data));
        	}	
diff --git a/src/org/exist/http/SOAPServer.java b/src/org/exist/http/SOAPServer.java
@@ -1309,9 +1309,18 @@ private BinaryDocument getXQWS(DBBroker broker, String path) throws PermissionDe
         */
        private byte[] getXQWSData(DBBroker broker, BinaryDocument docXQWS)
        {
-  
-        	byte[] data = broker.getBinaryResource(docXQWS);
-  
-
-  
-        	return data;
+  
+           try {
+  
+               InputStream is = broker.getBinaryResource(docXQWS);
+  
+               byte [] data = new byte[(int)broker.getBinaryResourceSize(docXQWS)];
+  
+               is.read(data);
+  
+               is.close();
+  
+
+  
+                   return data;
+  
+           } catch (IOException ex) {
+  
+              // TODO: where should this go?
+  
+              ex.printStackTrace();
+  
+           }
+  
+           return null;
        }
        /**
diff --git a/src/org/exist/soap/AdminSoapBindingImpl.java b/src/org/exist/soap/AdminSoapBindingImpl.java
@@ -1,4 +1,12 @@
 package org.exist.soap;
+  
+import java.io.ByteArrayInputStream;
+  
+import java.io.IOException;
+  
+import java.io.InputStream;
+  
+import java.io.StringReader;
+  
+import java.net.URISyntaxException;
+  
+import java.rmi.RemoteException;
+  
+import java.util.Iterator;
+  
+import java.util.Vector;
 import org.apache.log4j.Logger;
 import org.exist.EXistException;
@@ -449,7 +457,11 @@ public void storeBinary(java.lang.String sessionId, byte[] data, XmldbURI path,
                        + " is not a binary resource");
            if(!doc.getPermissions().validate(session.getUser(), Permission.READ))
                throw new PermissionDeniedException("Insufficient privileges to read resource");
-  
-            return broker.getBinaryResource( (BinaryDocument) doc );
+  
+            InputStream is = broker.getBinaryResource((BinaryDocument) doc);
+  
+            byte [] data = new byte[(int)broker.getBinaryResourceSize((BinaryDocument) doc)];
+  
+            is.read(data);
+  
+            is.close();
+  
+            return data;
        } catch (Exception ex) {
            throw new RemoteException(ex.getMessage());
        } finally {
diff --git a/src/org/exist/source/DBSource.java b/src/org/exist/source/DBSource.java
@@ -21,6 +21,7 @@
 */
 package org.exist.source;
+  
+import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -108,18 +109,22 @@ public int isValid(Source other) {
     * @see org.exist.source.Source#getReader()
     */
    public Reader getReader() throws IOException {
-  
-        byte[] data = broker.getBinaryResource(doc);
-  
-        ByteArrayInputStream is = new ByteArrayInputStream(data);
-  
-        checkEncoding(is);
-  
-        is.reset();
-  
-        return new InputStreamReader(is, encoding);
+  
+        InputStream is = broker.getBinaryResource(doc);
+  
+        BufferedInputStream bis = new BufferedInputStream(is);
+  
+        bis.mark(64);
+  
+        checkEncoding(bis);
+  
+        bis.reset();
+  
+        return new InputStreamReader(bis, encoding);
    }
    /* (non-Javadoc)
     * @see org.exist.source.Source#getContent()
     */
    public String getContent() throws IOException {
-  
-        byte[] data = broker.getBinaryResource(doc);
+  
+        InputStream raw = broker.getBinaryResource(doc);
+  
+        byte [] data = new byte[(int)broker.getBinaryResourceSize(doc)];
+  
+        raw.read(data);
+  
+        raw.close();
        ByteArrayInputStream is = new ByteArrayInputStream(data);
        checkEncoding(is);
        return new String(data, encoding);