Permalink
Browse files

[performance] move storage binary resources from dbx files to filesys…

…tem.

Yields into performance increase for (concurrent) access of binary files (=queries).
Internally binary resources are dealt as Streams 
Patch created by Alex Milowski

svn path=/trunk/eXist/; revision=8222
  • Loading branch information...
dizzzz committed Oct 7, 2008
1 parent e6f3637 commit ac5472a628a153cd2430e72c243588e0e8b4a920
Showing with 879 additions and 79 deletions.
  1. +7 −1 extensions/fluent/src/org/exist/fluent/Document.java
  2. +20 −0 extensions/fluent/test/src/org/exist/fluent/DocumentTest.java
  3. +9 −4 extensions/modules/src/org/exist/xquery/modules/compression/TarFunction.java
  4. +8 −2 extensions/modules/src/org/exist/xquery/modules/compression/ZipFunction.java
  5. +3 −3 extensions/modules/src/org/exist/xquery/modules/image/GetThumbnailsFunction.java
  6. +4 −0 src/org/exist/cluster/ClusterCollection.java
  7. +28 −6 src/org/exist/collections/Collection.java
  8. +9 −2 src/org/exist/collections/triggers/XQueryTrigger.java
  9. +12 −3 src/org/exist/http/SOAPServer.java
  10. +13 −1 src/org/exist/soap/AdminSoapBindingImpl.java
  11. +11 −6 src/org/exist/source/DBSource.java
  12. +89 −0 src/org/exist/storage/CreateBinaryLoggable.java
  13. +18 −6 src/org/exist/storage/DBBroker.java
  14. +282 −33 src/org/exist/storage/NativeBroker.java
  15. +98 −0 src/org/exist/storage/RenameBinaryLoggable.java
  16. +108 −0 src/org/exist/storage/UpdateBinaryLoggable.java
  17. +1 −1 src/org/exist/storage/dom/DOMFile.java
  18. +18 −0 src/org/exist/storage/journal/Journal.java
  19. +1 −0 src/org/exist/storage/recovery/RecoveryManager.java
  20. +80 −0 src/org/exist/util/FileUtils.java
  21. +9 −1 src/org/exist/xmldb/LocalBinaryResource.java
  22. +19 −4 src/org/exist/xmlrpc/RpcConnection.java
  23. +8 −1 src/org/exist/xquery/functions/util/BinaryDoc.java
  24. +6 −1 test/src/org/exist/storage/RecoverBinaryTest.java
  25. +9 −2 test/src/org/exist/storage/RecoveryTest.java
  26. +9 −2 test/src/org/exist/storage/ResourceTest.java
@@ -322,9 +322,15 @@ private DocumentImpl moveOrCopy(Folder destination, Name name, boolean copy) {
public String contentsAsString() {
DBBroker broker = db.acquireBroker();
try {
return new String(broker.getBinaryResource((BinaryDocument) doc), db.defaultCharacterEncoding);
InputStream is = broker.getBinaryResource((BinaryDocument) doc);
byte [] data = new byte[(int)broker.getBinaryResourceSize((BinaryDocument) doc)];
is.read(data);
is.close();
return new String(data, db.defaultCharacterEncoding);
} catch (UnsupportedEncodingException e) {
throw new DatabaseException(e);
} catch (IOException e) {
throw new DatabaseException(e);
} finally {
db.releaseBroker(broker);
}
@@ -36,6 +36,16 @@
assertEquals("helloworld", original.contentsAsString());
assertEquals("helloworld", copy.contentsAsString());
}

@Test public void copy2() {
Folder c1 = db.createFolder("/c1"), c2 = db.createFolder("/c2");
Document original = c1.documents().load(Name.create("original.xml"), Source.xml("<original/>"));
Document copy = original.copy(c2, Name.keepCreate());
assertEquals(1, c1.documents().size());
assertEquals(1, c2.documents().size());
assertEquals("<original/>", original.contentsAsString());
assertEquals("<original/>", copy.contentsAsString());
}

@Test public void move1() {
Folder c1 = db.createFolder("/c1"), c2 = db.createFolder("/c2");
@@ -46,5 +56,15 @@
assertEquals("/c2/original", doc.path());
assertEquals("helloworld", doc.contentsAsString());
}

@Test public void move2() {
Folder c1 = db.createFolder("/c1"), c2 = db.createFolder("/c2");
Document doc = c1.documents().load(Name.create("original.xml"), Source.xml("<original/>"));
doc.move(c2, Name.keepCreate());
assertEquals(0, c1.documents().size());
assertEquals(1, c2.documents().size());
assertEquals("/c2/original.xml", doc.path());
assertEquals("<original/>", doc.contentsAsString());
}

}
@@ -21,6 +21,7 @@
*/
package org.exist.xquery.modules.compression;

import java.io.InputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Iterator;
@@ -210,9 +211,13 @@ private void tarResource(TarOutputStream tos, DocumentImpl doc,
tos.write(strDoc.getBytes());
} else if (doc.getResourceType() == DocumentImpl.BINARY_FILE) {
// binary file
byte[] data = context.getBroker().getBinaryResource(
(BinaryDocument) doc);
tos.write(data);
InputStream is = context.getBroker().getBinaryResource((BinaryDocument)doc);
byte[] data = new byte[16384];
int len = 0;
while ((len=is.read(data,0,data.length))>0) {
tos.write(data,0,len);
}
is.close();
}

// close the entry in the Tar
@@ -261,4 +266,4 @@ private void tarCollection(TarOutputStream tos, Collection col,
tarCollection(tos, childCol, useHierarchy, stripOffset);
}
}
}
}
@@ -22,6 +22,7 @@
package org.exist.xquery.modules.compression;

import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.IOException;
import java.util.Iterator;
import java.util.zip.ZipEntry;
@@ -210,9 +211,14 @@ private void zipResource(ZipOutputStream zos, DocumentImpl doc,
zos.write(strDoc.getBytes());
} else if (doc.getResourceType() == DocumentImpl.BINARY_FILE) {
// binary file
byte[] data = context.getBroker().getBinaryResource(
InputStream is = context.getBroker().getBinaryResource(
(BinaryDocument) doc);
zos.write(data);
byte [] data = new byte[16384];
int len;
while ((len=is.read(data))>0) {
zos.write(data,0,len);
}
is.close();
}

// close the entry in the Zip
@@ -2,6 +2,7 @@

import java.awt.Image;
import java.awt.image.BufferedImage;
import java.io.InputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -213,11 +214,10 @@ public boolean accept(File dir, String name) {
binImage = (BinaryDocument) docImage;

// get a byte array representing the image
imgData = dbbroker.getBinaryResource(binImage);

try {
image = ImageIO.read(new ByteArrayInputStream(
imgData));
InputStream is = dbbroker.getBinaryResource(binImage);
image = ImageIO.read(is);
} catch (IOException ioe) {
throw new XPathException(getASTNode(),ioe.getMessage());
}
@@ -165,7 +165,11 @@ public void store(Txn transaction, DBBroker broker, IndexInfo info, InputSource
public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
XmldbURI name, byte[] data, String mimeType) throws EXistException,
PermissionDeniedException, LockException, TriggerException {
try {
return collection.addBinaryResource(transaction, broker, name, data, mimeType);
} catch (IOException ex) {
throw new EXistException("Cannot add binary due to I/O error.",ex);
}
}

public Lock getLock() {
@@ -865,7 +865,11 @@ public void removeBinaryResource(Txn transaction, DBBroker broker, DocumentImpl
if (trigger != null)
trigger.prepare(Trigger.REMOVE_DOCUMENT_EVENT, broker, transaction, doc.getURI(), doc);

broker.removeBinaryResource(transaction, (BinaryDocument) doc);
try {
broker.removeBinaryResource(transaction, (BinaryDocument) doc);
} catch (IOException ex) {
throw new PermissionDeniedException("Cannot delete file.");
}
documents.remove(doc.getFileURI().getRawCollectionPath());

if (trigger != null) {
@@ -1361,29 +1365,29 @@ private DocumentTrigger setupTriggers(DBBroker broker, XmldbURI docUri, boolean
// Blob
public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
XmldbURI docUri, byte[] data, String mimeType)
throws EXistException, PermissionDeniedException, LockException, TriggerException {
throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException {
return addBinaryResource(transaction, broker, docUri, data, mimeType, null, null);
}

// Blob
public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
XmldbURI docUri, byte[] data, String mimeType, Date created, Date modified)
throws EXistException, PermissionDeniedException, LockException, TriggerException {
throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException {
return addBinaryResource(transaction, broker, docUri,
new ByteArrayInputStream(data), mimeType, data.length, created, modified);
}

// Streaming
public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
XmldbURI docUri, InputStream is, String mimeType, int size)
throws EXistException, PermissionDeniedException, LockException, TriggerException {
throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException {
return addBinaryResource(transaction, broker, docUri, is, mimeType, size, null, null);
}

// Streaming
public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
XmldbURI docUri, InputStream is, String mimeType, int size, Date created, Date modified)
throws EXistException, PermissionDeniedException, LockException, TriggerException {
throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException {

if (broker.isReadOnly())
throw new PermissionDeniedException("Database is read-only");
@@ -1400,6 +1404,7 @@ public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
checkPermissions(transaction, broker, oldDoc);
DocumentTrigger trigger = null;
int event = 0;
/*
if (triggersEnabled) {
CollectionConfiguration config = getConfiguration(broker);
if (config != null) {
@@ -1414,6 +1419,7 @@ public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
}
}
}
*/

manageDocumentInformation(broker, oldDoc, blob );
DocumentMetadata metadata = blob.getMetadata();
@@ -1438,8 +1444,24 @@ public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
addDocument(transaction, broker, blob);

broker.storeXMLResource(transaction, blob);
if (triggersEnabled) {
CollectionConfiguration config = getConfiguration(broker);
if (config != null) {
event = oldDoc != null ? Trigger.UPDATE_DOCUMENT_EVENT : Trigger.STORE_DOCUMENT_EVENT;
try {
trigger = (DocumentTrigger) config.newTrigger(event, broker, this);
} catch (CollectionConfigurationException e) {
LOG.debug("An error occurred while initializing a trigger for collection " + getURI() + ": " + e.getMessage(), e);
}
if (trigger != null) {
trigger.prepare(event, broker, transaction, blob.getURI(), blob);
}
}
}


broker.closeDocument();
// This is no longer needed as the dom.dbx isn't used
//broker.closeDocument();

if (trigger != null) {
trigger.finish(event, broker, transaction, blob.getURI(), blob);
@@ -1,6 +1,7 @@
package org.exist.collections.triggers;

import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
@@ -219,7 +220,10 @@ else if (existingDocument instanceof BinaryDocument)
{
//binary document
BinaryDocument bin = (BinaryDocument)existingDocument;
byte[] data = context.getBroker().getBinaryResource(bin);
InputStream is = broker.getBinaryResource(bin);
byte [] data = new byte[(int)broker.getBinaryResourceSize(bin)];
is.read(data);
is.close();

context.declareVariable(bindingPrefix + "document", new Base64Binary(data));
}
@@ -302,7 +306,10 @@ else if (document instanceof BinaryDocument)
{
//Binary document
BinaryDocument bin = (BinaryDocument)document;
byte[] data = context.getBroker().getBinaryResource(bin);
InputStream is = broker.getBinaryResource(bin);
byte [] data = new byte[(int)broker.getBinaryResourceSize(bin)];
is.read(data);
is.close();

context.declareVariable(bindingPrefix + "document", new Base64Binary(data));
}
@@ -1309,9 +1309,18 @@ private BinaryDocument getXQWS(DBBroker broker, String path) throws PermissionDe
*/
private byte[] getXQWSData(DBBroker broker, BinaryDocument docXQWS)
{
byte[] data = broker.getBinaryResource(docXQWS);

return data;
try {
InputStream is = broker.getBinaryResource(docXQWS);
byte [] data = new byte[(int)broker.getBinaryResourceSize(docXQWS)];
is.read(data);
is.close();

return data;
} catch (IOException ex) {
// TODO: where should this go?
ex.printStackTrace();
}
return null;
}

/**
@@ -1,4 +1,12 @@
package org.exist.soap;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
import java.net.URISyntaxException;
import java.rmi.RemoteException;
import java.util.Iterator;
import java.util.Vector;

import org.apache.log4j.Logger;
import org.exist.EXistException;
@@ -449,7 +457,11 @@ public void storeBinary(java.lang.String sessionId, byte[] data, XmldbURI path,
+ " is not a binary resource");
if(!doc.getPermissions().validate(session.getUser(), Permission.READ))
throw new PermissionDeniedException("Insufficient privileges to read resource");
return broker.getBinaryResource( (BinaryDocument) doc );
InputStream is = broker.getBinaryResource((BinaryDocument) doc);
byte [] data = new byte[(int)broker.getBinaryResourceSize((BinaryDocument) doc)];
is.read(data);
is.close();
return data;
} catch (Exception ex) {
throw new RemoteException(ex.getMessage());
} finally {
@@ -21,6 +21,7 @@
*/
package org.exist.source;

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -108,18 +109,22 @@ public int isValid(Source other) {
* @see org.exist.source.Source#getReader()
*/
public Reader getReader() throws IOException {
byte[] data = broker.getBinaryResource(doc);
ByteArrayInputStream is = new ByteArrayInputStream(data);
checkEncoding(is);
is.reset();
return new InputStreamReader(is, encoding);
InputStream is = broker.getBinaryResource(doc);
BufferedInputStream bis = new BufferedInputStream(is);
bis.mark(64);
checkEncoding(bis);
bis.reset();
return new InputStreamReader(bis, encoding);
}

/* (non-Javadoc)
* @see org.exist.source.Source#getContent()
*/
public String getContent() throws IOException {
byte[] data = broker.getBinaryResource(doc);
InputStream raw = broker.getBinaryResource(doc);
byte [] data = new byte[(int)broker.getBinaryResourceSize(doc)];
raw.read(data);
raw.close();
ByteArrayInputStream is = new ByteArrayInputStream(data);
checkEncoding(is);
return new String(data, encoding);
Oops, something went wrong.

0 comments on commit ac5472a

Please sign in to comment.