[performance] move storage binary resources from dbx files to filesys…

…tem.

Yields into performance increase for (concurrent) access of binary files (=queries).
Internally binary resources are dealt as Streams 
Patch created by Alex Milowski

svn path=/trunk/eXist/; revision=8222

extensions/fluent/src/org/exist/fluent/Document.java

@@ -322,9 +322,15 @@ private DocumentImpl moveOrCopy(Folder destination, Name name, boolean copy) {
 	public String contentsAsString() {
 		DBBroker broker = db.acquireBroker();
 		try {
-			return new String(broker.getBinaryResource((BinaryDocument) doc), db.defaultCharacterEncoding);
+			InputStream is = broker.getBinaryResource((BinaryDocument) doc);
+			byte [] data = new byte[(int)broker.getBinaryResourceSize((BinaryDocument) doc)];
+			is.read(data);
+			is.close();
+			return new String(data, db.defaultCharacterEncoding);
 		} catch (UnsupportedEncodingException e) {
 			throw new DatabaseException(e);
+		} catch (IOException e) {
+			throw new DatabaseException(e);
 		} finally {
 			db.releaseBroker(broker);
 		}

extensions/fluent/test/src/org/exist/fluent/DocumentTest.java

@@ -36,6 +36,16 @@ public class DocumentTest extends DatabaseTestCase {
 		assertEquals("helloworld", original.contentsAsString());
 		assertEquals("helloworld", copy.contentsAsString());
 	}
+
+	@Test public void copy2() {
+		Folder c1 = db.createFolder("/c1"), c2 = db.createFolder("/c2");
+		Document original = c1.documents().load(Name.create("original.xml"), Source.xml("<original/>"));
+		Document copy = original.copy(c2, Name.keepCreate());
+		assertEquals(1, c1.documents().size());
+		assertEquals(1, c2.documents().size());
+		assertEquals("<original/>", original.contentsAsString());
+		assertEquals("<original/>", copy.contentsAsString());
+	}
 
 	@Test public void move1() {
 		Folder c1 = db.createFolder("/c1"), c2 = db.createFolder("/c2");
@@ -46,5 +56,15 @@ public class DocumentTest extends DatabaseTestCase {
 		assertEquals("/c2/original", doc.path());
 		assertEquals("helloworld", doc.contentsAsString());
 	}
+
+	@Test public void move2() {
+		Folder c1 = db.createFolder("/c1"), c2 = db.createFolder("/c2");
+		Document doc = c1.documents().load(Name.create("original.xml"), Source.xml("<original/>"));
+		doc.move(c2, Name.keepCreate());
+		assertEquals(0, c1.documents().size());
+		assertEquals(1, c2.documents().size());
+		assertEquals("/c2/original.xml", doc.path());
+		assertEquals("<original/>", doc.contentsAsString());
+	}
 
 }

extensions/modules/src/org/exist/xquery/modules/compression/TarFunction.java

@@ -21,6 +21,7 @@
  */
 package org.exist.xquery.modules.compression;
 
+import java.io.InputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.Iterator;
@@ -210,9 +211,13 @@ private void tarResource(TarOutputStream tos, DocumentImpl doc,
 			tos.write(strDoc.getBytes());
 		} else if (doc.getResourceType() == DocumentImpl.BINARY_FILE) {
 			// binary file
-			byte[] data = context.getBroker().getBinaryResource(
-					(BinaryDocument) doc);
-			tos.write(data);
+                        InputStream is = context.getBroker().getBinaryResource((BinaryDocument)doc);
+			byte[] data = new byte[16384];
+                        int len = 0;
+                        while ((len=is.read(data,0,data.length))>0) {
+			   tos.write(data,0,len);
+                        }
+                        is.close();
 		}
 
 		// close the entry in the Tar
@@ -261,4 +266,4 @@ private void tarCollection(TarOutputStream tos, Collection col,
 			tarCollection(tos, childCol, useHierarchy, stripOffset);
 		}
 	}
-}
+}

extensions/modules/src/org/exist/xquery/modules/compression/ZipFunction.java

@@ -22,6 +22,7 @@
 package org.exist.xquery.modules.compression;
 
 import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.zip.ZipEntry;
@@ -210,9 +211,14 @@ private void zipResource(ZipOutputStream zos, DocumentImpl doc,
 			zos.write(strDoc.getBytes());
 		} else if (doc.getResourceType() == DocumentImpl.BINARY_FILE) {
 			// binary file
-			byte[] data = context.getBroker().getBinaryResource(
+			InputStream is = context.getBroker().getBinaryResource(
 					(BinaryDocument) doc);
-			zos.write(data);
+                        byte [] data = new byte[16384];
+                        int len;
+                        while ((len=is.read(data))>0) {
+			   zos.write(data,0,len);
+                        }
+                        is.close();
 		}
 
 		// close the entry in the Zip

extensions/modules/src/org/exist/xquery/modules/image/GetThumbnailsFunction.java

@@ -2,6 +2,7 @@
 
 import java.awt.Image;
 import java.awt.image.BufferedImage;
+import java.io.InputStream;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -213,11 +214,10 @@ public boolean accept(File dir, String name) {
 						binImage = (BinaryDocument) docImage;
 
 						// get a byte array representing the image
-						imgData = dbbroker.getBinaryResource(binImage);
 
 						try {
-							image = ImageIO.read(new ByteArrayInputStream(
-									imgData));
+                                                   InputStream is = dbbroker.getBinaryResource(binImage);
+							image = ImageIO.read(is);
 						} catch (IOException ioe) {
 							throw new XPathException(getASTNode(),ioe.getMessage());
 						}

src/org/exist/cluster/ClusterCollection.java

@@ -165,7 +165,11 @@ public void store(Txn transaction, DBBroker broker, IndexInfo info, InputSource
     public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
 			XmldbURI name, byte[] data, String mimeType) throws EXistException,
             PermissionDeniedException, LockException, TriggerException {
+       try {
         return collection.addBinaryResource(transaction, broker, name, data, mimeType);
+       } catch (IOException ex) {
+          throw new EXistException("Cannot add binary due to I/O error.",ex);
+       }
     }
 
     public Lock getLock() {

src/org/exist/collections/Collection.java

@@ -865,7 +865,11 @@ public void removeBinaryResource(Txn transaction, DBBroker broker, DocumentImpl
             if (trigger != null)
                 trigger.prepare(Trigger.REMOVE_DOCUMENT_EVENT, broker, transaction, doc.getURI(), doc);
 
-            broker.removeBinaryResource(transaction, (BinaryDocument) doc);
+            try {
+               broker.removeBinaryResource(transaction, (BinaryDocument) doc);
+            } catch (IOException ex) {
+               throw new PermissionDeniedException("Cannot delete file.");
+            }
             documents.remove(doc.getFileURI().getRawCollectionPath());
 
             if (trigger != null) {
@@ -1361,29 +1365,29 @@ private DocumentTrigger setupTriggers(DBBroker broker, XmldbURI docUri, boolean
     // Blob
     public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
                 XmldbURI docUri, byte[] data, String mimeType)
-            throws EXistException, PermissionDeniedException, LockException, TriggerException {
+            throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException {
         return addBinaryResource(transaction, broker, docUri, data, mimeType, null, null);
     }
 
     // Blob
     public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
     		XmldbURI docUri, byte[] data, String mimeType, Date created, Date modified)
-            throws EXistException, PermissionDeniedException, LockException, TriggerException {
+            throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException {
         return addBinaryResource(transaction, broker, docUri, 
                                 new ByteArrayInputStream(data), mimeType, data.length, created, modified);
     }
 
     // Streaming
     public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
     		XmldbURI docUri, InputStream is, String mimeType, int size)
-            throws EXistException, PermissionDeniedException, LockException, TriggerException {
+            throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException {
         return addBinaryResource(transaction, broker, docUri, is, mimeType, size, null, null);
     }
 
     // Streaming
     public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
     		XmldbURI docUri, InputStream is, String mimeType, int size, Date created, Date modified)
-            throws EXistException, PermissionDeniedException, LockException, TriggerException {
+            throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException {
 
         if (broker.isReadOnly())
             throw new PermissionDeniedException("Database is read-only");
@@ -1400,6 +1404,7 @@ public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
 	        checkPermissions(transaction, broker, oldDoc);
 	        DocumentTrigger trigger = null;
 	        int event = 0;
+/*
 	        if (triggersEnabled) {
 	            CollectionConfiguration config = getConfiguration(broker);
 	            if (config != null) {
@@ -1414,6 +1419,7 @@ public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
 	                }
 	            }
 	        }
+*/
 
 	        manageDocumentInformation(broker, oldDoc, blob );
 	        DocumentMetadata metadata = blob.getMetadata();
@@ -1438,8 +1444,24 @@ public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
 	        addDocument(transaction, broker, blob);
 
 	        broker.storeXMLResource(transaction, blob);
+	        if (triggersEnabled) {
+	            CollectionConfiguration config = getConfiguration(broker);
+	            if (config != null) {
+	                event = oldDoc != null ? Trigger.UPDATE_DOCUMENT_EVENT : Trigger.STORE_DOCUMENT_EVENT;
+                    try {
+                        trigger = (DocumentTrigger) config.newTrigger(event, broker, this);
+                    } catch (CollectionConfigurationException e) {
+                        LOG.debug("An error occurred while initializing a trigger for collection " + getURI() + ": " + e.getMessage(), e);
+                    }
+                    if (trigger != null) {
+	                    trigger.prepare(event, broker, transaction, blob.getURI(), blob);
+	                }
+	            }
+	        }
+
 
-	        broker.closeDocument();
+                // This is no longer needed as the dom.dbx isn't used
+	        //broker.closeDocument();
 
 	        if (trigger != null) {
 	            trigger.finish(event, broker, transaction, blob.getURI(), blob);

src/org/exist/collections/triggers/XQueryTrigger.java

@@ -1,6 +1,7 @@
 package org.exist.collections.triggers;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Properties;
@@ -219,7 +220,10 @@ else if (existingDocument instanceof BinaryDocument)
         	{
         		//binary document
         		BinaryDocument bin = (BinaryDocument)existingDocument;
-                byte[] data = context.getBroker().getBinaryResource(bin);
+                        InputStream is = broker.getBinaryResource(bin);
+                        byte [] data = new byte[(int)broker.getBinaryResourceSize(bin)];
+                        is.read(data);
+                        is.close();
 
                 context.declareVariable(bindingPrefix + "document", new Base64Binary(data));
         	}
@@ -302,7 +306,10 @@ else if (document instanceof BinaryDocument)
         	{
         		//Binary document
         		BinaryDocument bin = (BinaryDocument)document;
-                byte[] data = context.getBroker().getBinaryResource(bin);
+                        InputStream is = broker.getBinaryResource(bin);
+                        byte [] data = new byte[(int)broker.getBinaryResourceSize(bin)];
+                        is.read(data);
+                        is.close();
 
                 context.declareVariable(bindingPrefix + "document", new Base64Binary(data));
         	}	

src/org/exist/http/SOAPServer.java

@@ -1309,9 +1309,18 @@ private BinaryDocument getXQWS(DBBroker broker, String path) throws PermissionDe
          */
         private byte[] getXQWSData(DBBroker broker, BinaryDocument docXQWS)
         {
-        	byte[] data = broker.getBinaryResource(docXQWS);
-
-        	return data;
+           try {
+               InputStream is = broker.getBinaryResource(docXQWS);
+               byte [] data = new byte[(int)broker.getBinaryResourceSize(docXQWS)];
+               is.read(data);
+               is.close();
+
+                   return data;
+           } catch (IOException ex) {
+              // TODO: where should this go?
+              ex.printStackTrace();
+           }
+           return null;
         }
 
         /**

src/org/exist/soap/AdminSoapBindingImpl.java

@@ -1,4 +1,12 @@
 package org.exist.soap;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringReader;
+import java.net.URISyntaxException;
+import java.rmi.RemoteException;
+import java.util.Iterator;
+import java.util.Vector;
 
 import org.apache.log4j.Logger;
 import org.exist.EXistException;
@@ -449,7 +457,11 @@ public byte[] getBinaryResource(java.lang.String sessionId, XmldbURI name) throw
                         + " is not a binary resource");
             if(!doc.getPermissions().validate(session.getUser(), Permission.READ))
                 throw new PermissionDeniedException("Insufficient privileges to read resource");
-            return broker.getBinaryResource( (BinaryDocument) doc );
+            InputStream is = broker.getBinaryResource((BinaryDocument) doc);
+            byte [] data = new byte[(int)broker.getBinaryResourceSize((BinaryDocument) doc)];
+            is.read(data);
+            is.close();
+            return data;
         } catch (Exception ex) {
             throw new RemoteException(ex.getMessage());
         } finally {

src/org/exist/source/DBSource.java

@@ -21,6 +21,7 @@
  */
 package org.exist.source;
 
+import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -108,18 +109,22 @@ public int isValid(Source other) {
      * @see org.exist.source.Source#getReader()
      */
     public Reader getReader() throws IOException {
-        byte[] data = broker.getBinaryResource(doc);
-        ByteArrayInputStream is = new ByteArrayInputStream(data);
-        checkEncoding(is);
-        is.reset();
-        return new InputStreamReader(is, encoding);
+        InputStream is = broker.getBinaryResource(doc);
+        BufferedInputStream bis = new BufferedInputStream(is);
+        bis.mark(64);
+        checkEncoding(bis);
+        bis.reset();
+        return new InputStreamReader(bis, encoding);
     }
 
     /* (non-Javadoc)
      * @see org.exist.source.Source#getContent()
      */
     public String getContent() throws IOException {
-        byte[] data = broker.getBinaryResource(doc);
+        InputStream raw = broker.getBinaryResource(doc);
+        byte [] data = new byte[(int)broker.getBinaryResourceSize(doc)];
+        raw.read(data);
+        raw.close();
         ByteArrayInputStream is = new ByteArrayInputStream(data);
         checkEncoding(is);
         return new String(data, encoding);