From ac5472a628a153cd2430e72c243588e0e8b4a920 Mon Sep 17 00:00:00 2001 From: Dannes Wessels Date: Tue, 7 Oct 2008 20:02:15 +0000 Subject: [PATCH] [performance] move storage binary resources from dbx files to filesystem. Yields into performance increase for (concurrent) access of binary files (=queries). Internally binary resources are dealt as Streams Patch created by Alex Milowski svn path=/trunk/eXist/; revision=8222 --- .../fluent/src/org/exist/fluent/Document.java | 8 +- .../src/org/exist/fluent/DocumentTest.java | 20 ++ .../modules/compression/TarFunction.java | 13 +- .../modules/compression/ZipFunction.java | 10 +- .../modules/image/GetThumbnailsFunction.java | 6 +- src/org/exist/cluster/ClusterCollection.java | 4 + src/org/exist/collections/Collection.java | 34 +- .../collections/triggers/XQueryTrigger.java | 11 +- src/org/exist/http/SOAPServer.java | 15 +- src/org/exist/soap/AdminSoapBindingImpl.java | 14 +- src/org/exist/source/DBSource.java | 17 +- .../exist/storage/CreateBinaryLoggable.java | 89 +++++ src/org/exist/storage/DBBroker.java | 24 +- src/org/exist/storage/NativeBroker.java | 315 ++++++++++++++++-- .../exist/storage/RenameBinaryLoggable.java | 98 ++++++ .../exist/storage/UpdateBinaryLoggable.java | 108 ++++++ src/org/exist/storage/dom/DOMFile.java | 2 +- src/org/exist/storage/journal/Journal.java | 18 + .../storage/recovery/RecoveryManager.java | 1 + src/org/exist/util/FileUtils.java | 80 +++++ src/org/exist/xmldb/LocalBinaryResource.java | 10 +- src/org/exist/xmlrpc/RpcConnection.java | 23 +- .../xquery/functions/util/BinaryDoc.java | 9 +- .../org/exist/storage/RecoverBinaryTest.java | 7 +- test/src/org/exist/storage/RecoveryTest.java | 11 +- test/src/org/exist/storage/ResourceTest.java | 11 +- 26 files changed, 879 insertions(+), 79 deletions(-) create mode 100644 src/org/exist/storage/CreateBinaryLoggable.java create mode 100644 src/org/exist/storage/RenameBinaryLoggable.java create mode 100644 src/org/exist/storage/UpdateBinaryLoggable.java create mode 100644 src/org/exist/util/FileUtils.java diff --git a/extensions/fluent/src/org/exist/fluent/Document.java b/extensions/fluent/src/org/exist/fluent/Document.java index 03efdbb556c..4b1f48f126d 100644 --- a/extensions/fluent/src/org/exist/fluent/Document.java +++ b/extensions/fluent/src/org/exist/fluent/Document.java @@ -322,9 +322,15 @@ private DocumentImpl moveOrCopy(Folder destination, Name name, boolean copy) { public String contentsAsString() { DBBroker broker = db.acquireBroker(); try { - return new String(broker.getBinaryResource((BinaryDocument) doc), db.defaultCharacterEncoding); + InputStream is = broker.getBinaryResource((BinaryDocument) doc); + byte [] data = new byte[(int)broker.getBinaryResourceSize((BinaryDocument) doc)]; + is.read(data); + is.close(); + return new String(data, db.defaultCharacterEncoding); } catch (UnsupportedEncodingException e) { throw new DatabaseException(e); + } catch (IOException e) { + throw new DatabaseException(e); } finally { db.releaseBroker(broker); } diff --git a/extensions/fluent/test/src/org/exist/fluent/DocumentTest.java b/extensions/fluent/test/src/org/exist/fluent/DocumentTest.java index cc9f52bb365..9e592dd2746 100644 --- a/extensions/fluent/test/src/org/exist/fluent/DocumentTest.java +++ b/extensions/fluent/test/src/org/exist/fluent/DocumentTest.java @@ -36,6 +36,16 @@ public class DocumentTest extends DatabaseTestCase { assertEquals("helloworld", original.contentsAsString()); assertEquals("helloworld", copy.contentsAsString()); } + + @Test public void copy2() { + Folder c1 = db.createFolder("/c1"), c2 = db.createFolder("/c2"); + Document original = c1.documents().load(Name.create("original.xml"), Source.xml("")); + Document copy = original.copy(c2, Name.keepCreate()); + assertEquals(1, c1.documents().size()); + assertEquals(1, c2.documents().size()); + assertEquals("", original.contentsAsString()); + assertEquals("", copy.contentsAsString()); + } @Test public void move1() { Folder c1 = db.createFolder("/c1"), c2 = db.createFolder("/c2"); @@ -46,5 +56,15 @@ public class DocumentTest extends DatabaseTestCase { assertEquals("/c2/original", doc.path()); assertEquals("helloworld", doc.contentsAsString()); } + + @Test public void move2() { + Folder c1 = db.createFolder("/c1"), c2 = db.createFolder("/c2"); + Document doc = c1.documents().load(Name.create("original.xml"), Source.xml("")); + doc.move(c2, Name.keepCreate()); + assertEquals(0, c1.documents().size()); + assertEquals(1, c2.documents().size()); + assertEquals("/c2/original.xml", doc.path()); + assertEquals("", doc.contentsAsString()); + } } diff --git a/extensions/modules/src/org/exist/xquery/modules/compression/TarFunction.java b/extensions/modules/src/org/exist/xquery/modules/compression/TarFunction.java index 4b15ab1bbc4..6b790209ac0 100644 --- a/extensions/modules/src/org/exist/xquery/modules/compression/TarFunction.java +++ b/extensions/modules/src/org/exist/xquery/modules/compression/TarFunction.java @@ -21,6 +21,7 @@ */ package org.exist.xquery.modules.compression; +import java.io.InputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Iterator; @@ -210,9 +211,13 @@ private void tarResource(TarOutputStream tos, DocumentImpl doc, tos.write(strDoc.getBytes()); } else if (doc.getResourceType() == DocumentImpl.BINARY_FILE) { // binary file - byte[] data = context.getBroker().getBinaryResource( - (BinaryDocument) doc); - tos.write(data); + InputStream is = context.getBroker().getBinaryResource((BinaryDocument)doc); + byte[] data = new byte[16384]; + int len = 0; + while ((len=is.read(data,0,data.length))>0) { + tos.write(data,0,len); + } + is.close(); } // close the entry in the Tar @@ -261,4 +266,4 @@ private void tarCollection(TarOutputStream tos, Collection col, tarCollection(tos, childCol, useHierarchy, stripOffset); } } -} \ No newline at end of file +} diff --git a/extensions/modules/src/org/exist/xquery/modules/compression/ZipFunction.java b/extensions/modules/src/org/exist/xquery/modules/compression/ZipFunction.java index e544a04a5b0..dd620c5bff3 100644 --- a/extensions/modules/src/org/exist/xquery/modules/compression/ZipFunction.java +++ b/extensions/modules/src/org/exist/xquery/modules/compression/ZipFunction.java @@ -22,6 +22,7 @@ package org.exist.xquery.modules.compression; import java.io.ByteArrayOutputStream; +import java.io.InputStream; import java.io.IOException; import java.util.Iterator; import java.util.zip.ZipEntry; @@ -210,9 +211,14 @@ private void zipResource(ZipOutputStream zos, DocumentImpl doc, zos.write(strDoc.getBytes()); } else if (doc.getResourceType() == DocumentImpl.BINARY_FILE) { // binary file - byte[] data = context.getBroker().getBinaryResource( + InputStream is = context.getBroker().getBinaryResource( (BinaryDocument) doc); - zos.write(data); + byte [] data = new byte[16384]; + int len; + while ((len=is.read(data))>0) { + zos.write(data,0,len); + } + is.close(); } // close the entry in the Zip diff --git a/extensions/modules/src/org/exist/xquery/modules/image/GetThumbnailsFunction.java b/extensions/modules/src/org/exist/xquery/modules/image/GetThumbnailsFunction.java index 2539e66ed89..4bf941470b0 100644 --- a/extensions/modules/src/org/exist/xquery/modules/image/GetThumbnailsFunction.java +++ b/extensions/modules/src/org/exist/xquery/modules/image/GetThumbnailsFunction.java @@ -2,6 +2,7 @@ import java.awt.Image; import java.awt.image.BufferedImage; +import java.io.InputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; @@ -213,11 +214,10 @@ public boolean accept(File dir, String name) { binImage = (BinaryDocument) docImage; // get a byte array representing the image - imgData = dbbroker.getBinaryResource(binImage); try { - image = ImageIO.read(new ByteArrayInputStream( - imgData)); + InputStream is = dbbroker.getBinaryResource(binImage); + image = ImageIO.read(is); } catch (IOException ioe) { throw new XPathException(getASTNode(),ioe.getMessage()); } diff --git a/src/org/exist/cluster/ClusterCollection.java b/src/org/exist/cluster/ClusterCollection.java index 5aa23b7ab25..1fef00dd449 100644 --- a/src/org/exist/cluster/ClusterCollection.java +++ b/src/org/exist/cluster/ClusterCollection.java @@ -165,7 +165,11 @@ public void store(Txn transaction, DBBroker broker, IndexInfo info, InputSource public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker, XmldbURI name, byte[] data, String mimeType) throws EXistException, PermissionDeniedException, LockException, TriggerException { + try { return collection.addBinaryResource(transaction, broker, name, data, mimeType); + } catch (IOException ex) { + throw new EXistException("Cannot add binary due to I/O error.",ex); + } } public Lock getLock() { diff --git a/src/org/exist/collections/Collection.java b/src/org/exist/collections/Collection.java index 55b11524bc8..531b2c18dd4 100644 --- a/src/org/exist/collections/Collection.java +++ b/src/org/exist/collections/Collection.java @@ -865,7 +865,11 @@ public void removeBinaryResource(Txn transaction, DBBroker broker, DocumentImpl if (trigger != null) trigger.prepare(Trigger.REMOVE_DOCUMENT_EVENT, broker, transaction, doc.getURI(), doc); - broker.removeBinaryResource(transaction, (BinaryDocument) doc); + try { + broker.removeBinaryResource(transaction, (BinaryDocument) doc); + } catch (IOException ex) { + throw new PermissionDeniedException("Cannot delete file."); + } documents.remove(doc.getFileURI().getRawCollectionPath()); if (trigger != null) { @@ -1361,14 +1365,14 @@ private DocumentTrigger setupTriggers(DBBroker broker, XmldbURI docUri, boolean // Blob public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker, XmldbURI docUri, byte[] data, String mimeType) - throws EXistException, PermissionDeniedException, LockException, TriggerException { + throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException { return addBinaryResource(transaction, broker, docUri, data, mimeType, null, null); } // Blob public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker, XmldbURI docUri, byte[] data, String mimeType, Date created, Date modified) - throws EXistException, PermissionDeniedException, LockException, TriggerException { + throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException { return addBinaryResource(transaction, broker, docUri, new ByteArrayInputStream(data), mimeType, data.length, created, modified); } @@ -1376,14 +1380,14 @@ public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker, // Streaming public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker, XmldbURI docUri, InputStream is, String mimeType, int size) - throws EXistException, PermissionDeniedException, LockException, TriggerException { + throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException { return addBinaryResource(transaction, broker, docUri, is, mimeType, size, null, null); } // Streaming public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker, XmldbURI docUri, InputStream is, String mimeType, int size, Date created, Date modified) - throws EXistException, PermissionDeniedException, LockException, TriggerException { + throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException { if (broker.isReadOnly()) throw new PermissionDeniedException("Database is read-only"); @@ -1400,6 +1404,7 @@ public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker, checkPermissions(transaction, broker, oldDoc); DocumentTrigger trigger = null; int event = 0; +/* if (triggersEnabled) { CollectionConfiguration config = getConfiguration(broker); if (config != null) { @@ -1414,6 +1419,7 @@ public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker, } } } +*/ manageDocumentInformation(broker, oldDoc, blob ); DocumentMetadata metadata = blob.getMetadata(); @@ -1438,8 +1444,24 @@ public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker, addDocument(transaction, broker, blob); broker.storeXMLResource(transaction, blob); + if (triggersEnabled) { + CollectionConfiguration config = getConfiguration(broker); + if (config != null) { + event = oldDoc != null ? Trigger.UPDATE_DOCUMENT_EVENT : Trigger.STORE_DOCUMENT_EVENT; + try { + trigger = (DocumentTrigger) config.newTrigger(event, broker, this); + } catch (CollectionConfigurationException e) { + LOG.debug("An error occurred while initializing a trigger for collection " + getURI() + ": " + e.getMessage(), e); + } + if (trigger != null) { + trigger.prepare(event, broker, transaction, blob.getURI(), blob); + } + } + } + - broker.closeDocument(); + // This is no longer needed as the dom.dbx isn't used + //broker.closeDocument(); if (trigger != null) { trigger.finish(event, broker, transaction, blob.getURI(), blob); diff --git a/src/org/exist/collections/triggers/XQueryTrigger.java b/src/org/exist/collections/triggers/XQueryTrigger.java index 341f7ccc7ff..c923b162d80 100644 --- a/src/org/exist/collections/triggers/XQueryTrigger.java +++ b/src/org/exist/collections/triggers/XQueryTrigger.java @@ -1,6 +1,7 @@ package org.exist.collections.triggers; import java.io.IOException; +import java.io.InputStream; import java.util.Iterator; import java.util.Map; import java.util.Properties; @@ -219,7 +220,10 @@ else if (existingDocument instanceof BinaryDocument) { //binary document BinaryDocument bin = (BinaryDocument)existingDocument; - byte[] data = context.getBroker().getBinaryResource(bin); + InputStream is = broker.getBinaryResource(bin); + byte [] data = new byte[(int)broker.getBinaryResourceSize(bin)]; + is.read(data); + is.close(); context.declareVariable(bindingPrefix + "document", new Base64Binary(data)); } @@ -302,7 +306,10 @@ else if (document instanceof BinaryDocument) { //Binary document BinaryDocument bin = (BinaryDocument)document; - byte[] data = context.getBroker().getBinaryResource(bin); + InputStream is = broker.getBinaryResource(bin); + byte [] data = new byte[(int)broker.getBinaryResourceSize(bin)]; + is.read(data); + is.close(); context.declareVariable(bindingPrefix + "document", new Base64Binary(data)); } diff --git a/src/org/exist/http/SOAPServer.java b/src/org/exist/http/SOAPServer.java index 279def92344..969ac0cfb62 100644 --- a/src/org/exist/http/SOAPServer.java +++ b/src/org/exist/http/SOAPServer.java @@ -1309,9 +1309,18 @@ private BinaryDocument getXQWS(DBBroker broker, String path) throws PermissionDe */ private byte[] getXQWSData(DBBroker broker, BinaryDocument docXQWS) { - byte[] data = broker.getBinaryResource(docXQWS); - - return data; + try { + InputStream is = broker.getBinaryResource(docXQWS); + byte [] data = new byte[(int)broker.getBinaryResourceSize(docXQWS)]; + is.read(data); + is.close(); + + return data; + } catch (IOException ex) { + // TODO: where should this go? + ex.printStackTrace(); + } + return null; } /** diff --git a/src/org/exist/soap/AdminSoapBindingImpl.java b/src/org/exist/soap/AdminSoapBindingImpl.java index 4b64b4bd980..f0a9adbb967 100644 --- a/src/org/exist/soap/AdminSoapBindingImpl.java +++ b/src/org/exist/soap/AdminSoapBindingImpl.java @@ -1,4 +1,12 @@ package org.exist.soap; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringReader; +import java.net.URISyntaxException; +import java.rmi.RemoteException; +import java.util.Iterator; +import java.util.Vector; import org.apache.log4j.Logger; import org.exist.EXistException; @@ -449,7 +457,11 @@ public byte[] getBinaryResource(java.lang.String sessionId, XmldbURI name) throw + " is not a binary resource"); if(!doc.getPermissions().validate(session.getUser(), Permission.READ)) throw new PermissionDeniedException("Insufficient privileges to read resource"); - return broker.getBinaryResource( (BinaryDocument) doc ); + InputStream is = broker.getBinaryResource((BinaryDocument) doc); + byte [] data = new byte[(int)broker.getBinaryResourceSize((BinaryDocument) doc)]; + is.read(data); + is.close(); + return data; } catch (Exception ex) { throw new RemoteException(ex.getMessage()); } finally { diff --git a/src/org/exist/source/DBSource.java b/src/org/exist/source/DBSource.java index 7a27c795b1c..d96dfbf42b7 100644 --- a/src/org/exist/source/DBSource.java +++ b/src/org/exist/source/DBSource.java @@ -21,6 +21,7 @@ */ package org.exist.source; +import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -108,18 +109,22 @@ public int isValid(Source other) { * @see org.exist.source.Source#getReader() */ public Reader getReader() throws IOException { - byte[] data = broker.getBinaryResource(doc); - ByteArrayInputStream is = new ByteArrayInputStream(data); - checkEncoding(is); - is.reset(); - return new InputStreamReader(is, encoding); + InputStream is = broker.getBinaryResource(doc); + BufferedInputStream bis = new BufferedInputStream(is); + bis.mark(64); + checkEncoding(bis); + bis.reset(); + return new InputStreamReader(bis, encoding); } /* (non-Javadoc) * @see org.exist.source.Source#getContent() */ public String getContent() throws IOException { - byte[] data = broker.getBinaryResource(doc); + InputStream raw = broker.getBinaryResource(doc); + byte [] data = new byte[(int)broker.getBinaryResourceSize(doc)]; + raw.read(data); + raw.close(); ByteArrayInputStream is = new ByteArrayInputStream(data); checkEncoding(is); return new String(data, encoding); diff --git a/src/org/exist/storage/CreateBinaryLoggable.java b/src/org/exist/storage/CreateBinaryLoggable.java new file mode 100644 index 00000000000..87a82006969 --- /dev/null +++ b/src/org/exist/storage/CreateBinaryLoggable.java @@ -0,0 +1,89 @@ +/* + * RenameBinaryLoggable.java + * + * Created on December 9, 2007, 1:57 PM + * + * To change this template, choose Tools | Template Manager + * and open the template in the editor. + */ + +package org.exist.storage; + +import java.io.File; +import java.nio.ByteBuffer; +import org.apache.log4j.Logger; +import org.exist.storage.journal.AbstractLoggable; +import org.exist.storage.journal.LogException; +import org.exist.storage.txn.Txn; + +/** + * + * @author alex + */ +public class CreateBinaryLoggable extends AbstractLoggable { + + protected final static Logger LOG = Logger.getLogger(RenameBinaryLoggable.class); + + DBBroker broker; + File original; + /** + * Creates a new instance of RenameBinaryLoggable + */ + public CreateBinaryLoggable(DBBroker broker,Txn txn,File original) + { + super(NativeBroker.LOG_CREATE_BINARY,txn.getId()); + System.out.println("CreateBinaryLoggable created"); + System.out.flush(); + this.broker = broker; + this.original = original; + } + + public CreateBinaryLoggable(DBBroker broker,long transactionId) { + super(NativeBroker.LOG_CREATE_BINARY,transactionId); + System.out.println("CreateBinaryLoggable created"); + System.out.flush(); + this.broker = broker; + } + + /* (non-Javadoc) + * @see org.exist.storage.log.Loggable#write(java.nio.ByteBuffer) + */ + public void write(ByteBuffer out) { + String originalPath = original.getAbsolutePath(); + byte [] data = originalPath.getBytes(); + out.putInt(data.length); + out.put(data); + } + + /* (non-Javadoc) + * @see org.exist.storage.log.Loggable#read(java.nio.ByteBuffer) + */ + public void read(ByteBuffer in) { + int size = in.getInt(); + byte [] data = new byte[size]; + in.get(data); + original = new File(new String(data)); + } + + /* (non-Javadoc) + * @see org.exist.storage.log.Loggable#getLogSize() + */ + public int getLogSize() { + return 4 + original.getAbsolutePath().getBytes().length; + } + + public void redo() throws LogException { + // TODO: do we need to redo? The file was stored... + } + + public void undo() throws LogException { + if (!original.delete()) { + throw new LogException("Cannot delete binary resource "+original); + } + } + + public String dump() { + return super.dump() + " - create binary "+original; + } + +} diff --git a/src/org/exist/storage/DBBroker.java b/src/org/exist/storage/DBBroker.java index cee951a53d7..a6921f57c51 100644 --- a/src/org/exist/storage/DBBroker.java +++ b/src/org/exist/storage/DBBroker.java @@ -523,7 +523,8 @@ public void endElement(final StoredNode node, NodePath currentPath, String conte * the document binary data */ public abstract void storeBinaryResource(Txn transaction, - BinaryDocument blob, byte[] data); + BinaryDocument blob, byte[] data) + throws IOException; /** * Stores the given data under the given binary resource descriptor @@ -535,7 +536,8 @@ public abstract void storeBinaryResource(Txn transaction, * the document binary data as input stream */ public abstract void storeBinaryResource(Txn transaction, - BinaryDocument blob, InputStream is); + BinaryDocument blob, InputStream is) + throws IOException; public abstract void getCollectionResources(Collection collection); @@ -547,11 +549,21 @@ public abstract void storeBinaryResource(Txn transaction, * the binary document descriptor * @return the document binary data */ - public abstract byte[] getBinaryResource(BinaryDocument blob); + /* + public abstract byte[] getBinaryResource(BinaryDocument blob) + throws IOException; + */ public abstract void readBinaryResource(final BinaryDocument blob, - final OutputStream os); - + final OutputStream os) + throws IOException; + + public abstract InputStream getBinaryResource(final BinaryDocument blob) + throws IOException; + + public abstract long getBinaryResourceSize(final BinaryDocument blob) + throws IOException; + public abstract void getResourceMetadata(DocumentImpl doc); /** @@ -563,7 +575,7 @@ public abstract void readBinaryResource(final BinaryDocument blob, * if you don't have the right to do this */ public abstract void removeBinaryResource(Txn transaction, - BinaryDocument blob) throws PermissionDeniedException; + BinaryDocument blob) throws PermissionDeniedException,IOException; /** * Move a collection and all its subcollections to another collection and diff --git a/src/org/exist/storage/NativeBroker.java b/src/org/exist/storage/NativeBroker.java index 8ee62a6f89e..a4c34459716 100644 --- a/src/org/exist/storage/NativeBroker.java +++ b/src/org/exist/storage/NativeBroker.java @@ -92,12 +92,17 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.FileInputStream; import java.io.OutputStream; +import java.io.FileOutputStream; import java.io.UnsupportedEncodingException; import java.text.NumberFormat; import java.util.Iterator; import java.util.Observer; import java.util.Stack; +import org.exist.storage.journal.Journal; +import org.exist.storage.journal.LogEntryTypes; +import org.exist.storage.journal.Loggable; import java.util.StringTokenizer; /** @@ -121,6 +126,16 @@ */ public class NativeBroker extends DBBroker { + public final static byte LOG_RENAME_BINARY = 0x40; + public final static byte LOG_CREATE_BINARY = 0x41; + public final static byte LOG_UPDATE_BINARY = 0x42; + + static { + LogEntryTypes.addEntryType(LOG_RENAME_BINARY, RenameBinaryLoggable.class); + LogEntryTypes.addEntryType(LOG_CREATE_BINARY, CreateBinaryLoggable.class); + LogEntryTypes.addEntryType(LOG_UPDATE_BINARY, UpdateBinaryLoggable.class); + } + public static final byte PREPEND_DB_ALWAYS = 0; public static final byte PREPEND_DB_NEVER = 1; public static final byte PREPEND_DB_AS_NEEDED = 2; @@ -178,6 +193,9 @@ public class NativeBroker extends DBBroker { protected int nodesCount = 0; protected String dataDir; + protected File fsDir; + protected File fsBackupDir; + protected int pageSize; protected byte prepend; @@ -186,10 +204,13 @@ public class NativeBroker extends DBBroker { private NodeProcessor nodeProcessor = new NodeProcessor(); private EmbeddedXMLStreamReader streamReader = null; + + protected Journal logManager; /** initialize database; read configuration, etc. */ public NativeBroker(BrokerPool pool, Configuration config) throws EXistException { super(pool, config); + this.logManager = pool.getTransactionManager().getJournal(); LOG.debug("Initializing broker " + hashCode()); String prependDB = (String) config.getProperty("db-connection.prepend-db"); @@ -204,6 +225,20 @@ public NativeBroker(BrokerPool pool, Configuration config) throws EXistException dataDir = (String) config.getProperty(BrokerPool.PROPERTY_DATA_DIR); if (dataDir == null) dataDir = DEFAULT_DATA_DIR; + + fsDir = new File(new File(dataDir),"fs"); + if (!fsDir.exists()) { + if (!fsDir.mkdir()) { + throw new EXistException("Cannot make collection filesystem directory: "+fsDir); + } + } + fsBackupDir = new File(new File(dataDir),"fs.journal"); + if (!fsBackupDir.exists()) { + if (!fsBackupDir.mkdir()) { + throw new EXistException("Cannot make collection filesystem directory: "+fsBackupDir); + } + } + defaultIndexDepth = config.getInteger(PROPERTY_INDEX_DEPTH); if (defaultIndexDepth < 0) @@ -773,8 +808,13 @@ public void copyCollection(Txn transaction, Collection collection, Collection de BinaryDocument newDoc = new BinaryDocument(this, destCollection, child.getFileURI()); newDoc.copyOf(child); newDoc.setDocId(getNextResourceId(transaction, destination)); + /* byte[] data = getBinaryResource((BinaryDocument) child); storeBinaryResource(transaction, newDoc, data); + */ + InputStream is = getBinaryResource((BinaryDocument)child); + storeBinaryResource(transaction,newDoc,is); + is.close(); storeXMLResource(transaction, newDoc); destCollection.addDocument(transaction, this, newDoc); } @@ -822,6 +862,9 @@ public void moveCollection(Txn transaction, Collection collection, Collection de if(!destination.getPermissions().validate(user, Permission.WRITE)) throw new PermissionDeniedException("Insufficient privileges on target collection " + destination.getURI()); + + File sourceDir = getCollectionFile(fsDir,collection.getURI(),false); + File targetDir = getCollectionFile(fsDir,destination.getURI(),false); // check if another collection with the same name exists at the destination Collection old = openCollection(destination.getURI().append(newName), Lock.WRITE_LOCK); if(old != null) { @@ -881,6 +924,18 @@ public void moveCollection(Txn transaction, Collection collection, Collection de } } } + if (sourceDir.exists()) { + if (sourceDir.renameTo(targetDir)) { + Loggable loggable = new RenameBinaryLoggable(this,transaction,sourceDir,targetDir); + try { + logManager.writeToLog(loggable); + } catch (TransactionException e) { + LOG.warn(e.getMessage(), e); + } + } else { + LOG.fatal("Cannot move "+sourceDir+" to "+targetDir); + } + } } private void canRemoveCollection(Collection collection) throws PermissionDeniedException { @@ -921,6 +976,8 @@ public boolean removeCollection(final Txn transaction, Collection collection) th long start = System.currentTimeMillis(); final CollectionCache collectionsCache = pool.getCollectionsCache(); + File sourceDir = getCollectionFile(fsDir,collection.getURI(),false); + File targetDir = getCollectionFile(fsBackupDir,transaction,collection.getURI(),true); synchronized(collectionsCache) { @@ -1102,6 +1159,20 @@ public Object start() freeResourceId(transaction, doc.getDocId()); } + if (sourceDir.exists()) { + targetDir.getParentFile().mkdirs(); + if (sourceDir.renameTo(targetDir)) { + Loggable loggable = new RenameBinaryLoggable(this,transaction,sourceDir,targetDir); + try { + logManager.writeToLog(loggable); + } catch (TransactionException e) { + LOG.warn(e.getMessage(), e); + } + + } else { + LOG.fatal("Cannot rename "+sourceDir+" to "+targetDir); + } + } if(LOG.isDebugEnabled()) LOG.debug("Removing collection '" + collName + "' took " + (System.currentTimeMillis() - start)); @@ -1524,34 +1595,113 @@ public void storeXMLResource(final Txn transaction, final DocumentImpl doc) { } } - public void storeBinaryResource(final Txn transaction, final BinaryDocument blob, final byte[] data) { - if (data.length == 0) { - blob.setPage(Page.NO_PAGE); - return; - } - new DOMTransaction(this, domDb, Lock.WRITE_LOCK) { - public Object start() throws ReadOnlyException { - LOG.debug("Storing binary resource " + blob.getFileURI()); - blob.setPage(domDb.addBinary(transaction, blob, data)); - return null; - } - } - .run(); + private File getCollectionFile(File dir,XmldbURI uri,boolean create) + throws IOException + { + return getCollectionFile(dir,null,uri,create); + } + + private File getCollectionFile(File dir,Txn transaction,XmldbURI uri,boolean create) + throws IOException + { + if (transaction!=null) { + dir = new File(dir,"txn."+transaction.getId()); + if (create && !dir.exists()) { + if (!dir.mkdir()) { + throw new IOException("Cannot make transaction filesystem directory: "+dir); + } + } + } + XmldbURI [] segments = uri.getPathSegments(); + File binFile = dir; + int last = segments.length-1; + for (int i=0; i=0) { + if (len>0) { + os.write(buffer,0,len); + } + } + os.close(); + + if (exists) { + Loggable loggable = new UpdateBinaryLoggable(this,transaction,binFile,backupFile); + try { + logManager.writeToLog(loggable); + } catch (TransactionException e) { + LOG.warn(e.getMessage(), e); + } + } else { + Loggable loggable = new CreateBinaryLoggable(this,transaction,binFile); + try { + logManager.writeToLog(loggable); + } catch (TransactionException e) { + LOG.warn(e.getMessage(), e); + } + } } @@ -1584,6 +1734,14 @@ public Document getXMLResource(XmldbURI fileName) throws PermissionDeniedExcepti return null; } + if (doc.getResourceType() == DocumentImpl.BINARY_FILE) { + BinaryDocument bin = (BinaryDocument)doc; + try { + bin.setContentLength((int)getBinaryResourceSize(bin)); + } catch (IOException ex) { + LOG.fatal("Cannot get content size for "+bin.getURI(),ex); + } + } // if (!doc.getPermissions().validate(user, Permission.READ)) // throw new PermissionDeniedException("not allowed to read document"); @@ -1619,6 +1777,14 @@ public DocumentImpl getXMLResource(XmldbURI fileName, int lockMode) throws Permi // if (!doc.getPermissions().validate(user, Permission.READ)) // throw new PermissionDeniedException("not allowed to read document"); + if (doc.getResourceType() == DocumentImpl.BINARY_FILE) { + BinaryDocument bin = (BinaryDocument)doc; + try { + bin.setContentLength((int)getBinaryResourceSize(bin)); + } catch (IOException ex) { + LOG.fatal("Cannot get content size for "+bin.getURI(),ex); + } + } return doc; } catch (LockException e) { LOG.warn("Could not acquire lock on document " + fileName, e); @@ -1631,7 +1797,10 @@ public DocumentImpl getXMLResource(XmldbURI fileName, int lockMode) throws Permi return null; } - public byte[] getBinaryResource(final BinaryDocument blob) { + /* + public byte[] getBinaryResource(final BinaryDocument blob) + throws IOException + { if (blob.getPage() == Page.NO_PAGE) return new byte[0]; byte[] data = (byte[]) new DOMTransaction(this, domDb, Lock.WRITE_LOCK) { @@ -1641,9 +1810,16 @@ public Object start() throws ReadOnlyException { } .run(); return data; + ByteArrayOutputStream os = new ByteArrayOutputStream(); + readBinaryResource(blob,os); + return os.toByteArray(); } + */ - public void readBinaryResource(final BinaryDocument blob, final OutputStream os) { + public void readBinaryResource(final BinaryDocument blob, final OutputStream os) + throws IOException + { + /* if (blob.getPage() == Page.NO_PAGE) return; new DOMTransaction(this, domDb, Lock.WRITE_LOCK) { @@ -1652,6 +1828,37 @@ public Object start() throws ReadOnlyException { return null; } }.run(); + */ + InputStream is = getBinaryResource(blob); + byte [] buffer = new byte[4096]; + int len; + while ((len=is.read(buffer))>=0) { + os.write(buffer,0,len); + } + is.close(); + } + + public long getBinaryResourceSize(final BinaryDocument blob) + throws IOException + { + File binFile = getCollectionFile(fsDir,blob.getURI(),false); + return binFile.length(); + } + public InputStream getBinaryResource(final BinaryDocument blob) + throws IOException + { + /* + if (blob.getPage() == Page.NO_PAGE) + return; + new DOMTransaction(this, domDb, Lock.WRITE_LOCK) { + public Object start() throws ReadOnlyException { + domDb.readBinary(blob.getPage(), os); + return null; + } + }.run(); + */ + File binFile = getCollectionFile(fsDir,blob.getURI(),false); + return new FileInputStream(binFile); } //TODO : consider a better cooperation with Collection -pb @@ -1821,8 +2028,8 @@ public void copyXMLResource(Txn transaction, DocumentImpl doc, Collection destin destination.getURI()); } if (doc.getResourceType() == DocumentImpl.BINARY_FILE) { - byte[] data = getBinaryResource((BinaryDocument) doc); - destination.addBinaryResource(transaction, this, newName, data, doc.getMetadata().getMimeType()); + InputStream is = getBinaryResource((BinaryDocument) doc); + destination.addBinaryResource(transaction, this, newName, is, doc.getMetadata().getMimeType(),-1); } else { //TODO : put a lock on newDoc ? DocumentImpl newDoc = new DocumentImpl(this, destination, newName); @@ -1836,6 +2043,8 @@ public void copyXMLResource(Txn transaction, DocumentImpl doc, Collection destin // saveCollection(destination); } catch (EXistException e) { LOG.warn("An error occurred while copying resource", e); + } catch (IOException e) { + LOG.warn("An error occurred while copying resource", e); } catch (TriggerException e) { throw new PermissionDeniedException(e.getMessage()); } finally { @@ -1862,6 +2071,10 @@ private void copyXMLResource(Txn transaction, DocumentImpl oldDoc, DocumentImpl /** move Resource to another collection, with possible rename */ public void moveXMLResource(Txn transaction, DocumentImpl doc, Collection destination, XmldbURI newName) throws PermissionDeniedException, LockException, IOException { + + /* Copy reference to original document */ + File originalDocument = getCollectionFile(fsDir,doc.getURI(),true); + if (readOnly) throw new PermissionDeniedException(DATABASE_IS_READ_ONLY); @@ -1907,7 +2120,8 @@ public void moveXMLResource(Txn transaction, DocumentImpl doc, Collection destin if (!destination.getPermissions().validate(user, Permission.WRITE)) throw new PermissionDeniedException("Insufficient privileges on target collection " + destination.getURI()); - + + boolean renameOnly = collection.getId() == destination.getId(); collection.unlinkDocument(doc); removeResourceMetadata(transaction, doc); @@ -1927,7 +2141,26 @@ public void moveXMLResource(Txn transaction, DocumentImpl doc, Collection destin } } else { // binary resource - destination.addDocument(transaction, this, doc); + destination.addDocument(transaction, this, doc); + + File colDir = getCollectionFile(fsDir,destination.getURI(),true); + File binFile = new File(colDir,newName.lastSegment().toString()); + File sourceFile = getCollectionFile(fsDir,doc.getURI(),false); + + /* Create required directories */ + binFile.getParentFile().mkdirs(); + + /* Rename original file to new location */ + if (originalDocument.renameTo(binFile)) { + Loggable loggable = new RenameBinaryLoggable(this,transaction,sourceFile,binFile); + try { + logManager.writeToLog(loggable); + } catch (TransactionException e) { + LOG.warn(e.getMessage(), e); + } + } else { + LOG.fatal("Cannot rename "+sourceFile+" to "+binFile+" for journaling of binary resource move."); + } } storeXMLResource(transaction, doc); saveCollection(transaction, destination); @@ -2004,10 +2237,25 @@ private void dropIndex(Txn transaction, DocumentImpl document) throws ReadOnlyEx } public void removeBinaryResource(final Txn transaction, final BinaryDocument blob) - throws PermissionDeniedException { + throws PermissionDeniedException,IOException + { if (readOnly) throw new PermissionDeniedException(DATABASE_IS_READ_ONLY); LOG.info("removing binary resource " + blob.getDocId() + "..."); + File binFile = getCollectionFile(fsDir,blob.getURI(),false); + if (binFile.exists()) { + File binBackupFile = getCollectionFile(fsBackupDir,transaction,blob.getURI(),true); + Loggable loggable = new RenameBinaryLoggable(this,transaction,binFile,binBackupFile); + if (!binFile.renameTo(binBackupFile)) { + throw new IOException("Cannot move file "+binFile+" for delete journal to "+binBackupFile); + } + try { + logManager.writeToLog(loggable); + } catch (TransactionException e) { + LOG.warn(e.getMessage(), e); + } + } + /* if (blob.getPage() != Page.NO_PAGE) { new DOMTransaction(this, domDb, Lock.WRITE_LOCK) { public Object start() throws ReadOnlyException { @@ -2017,6 +2265,7 @@ public Object start() throws ReadOnlyException { } .run(); } + */ removeResourceMetadata(transaction, blob); } diff --git a/src/org/exist/storage/RenameBinaryLoggable.java b/src/org/exist/storage/RenameBinaryLoggable.java new file mode 100644 index 00000000000..c8eb6128be0 --- /dev/null +++ b/src/org/exist/storage/RenameBinaryLoggable.java @@ -0,0 +1,98 @@ +/* + * RenameBinaryLoggable.java + * + * Created on December 9, 2007, 1:57 PM + * + * To change this template, choose Tools | Template Manager + * and open the template in the editor. + */ + +package org.exist.storage; + +import java.io.File; +import java.nio.ByteBuffer; +import org.apache.log4j.Logger; +import org.exist.storage.journal.AbstractLoggable; +import org.exist.storage.journal.LogException; +import org.exist.storage.txn.Txn; + +/** + * + * @author alex + */ +public class RenameBinaryLoggable extends AbstractLoggable { + + protected final static Logger LOG = Logger.getLogger(RenameBinaryLoggable.class); + + DBBroker broker; + File original; + File backup; + /** + * Creates a new instance of RenameBinaryLoggable + */ + public RenameBinaryLoggable(DBBroker broker,Txn txn,File original,File backup) + { + super(NativeBroker.LOG_RENAME_BINARY,txn.getId()); + this.broker = broker; + this.original = original; + this.backup = backup; + System.out.println("Rename binary created "+original+" -> "+backup); + } + + public RenameBinaryLoggable(DBBroker broker,long transactionId) { + super(NativeBroker.LOG_RENAME_BINARY,transactionId); + this.broker = broker; + System.out.println("Rename binary created ..."); + } + + /* (non-Javadoc) + * @see org.exist.storage.log.Loggable#write(java.nio.ByteBuffer) + */ + public void write(ByteBuffer out) { + String originalPath = original.getAbsolutePath(); + byte [] data = originalPath.getBytes(); + out.putInt(data.length); + out.put(data); + String backupPath = backup.getAbsolutePath(); + data = backupPath.getBytes(); + out.putInt(data.length); + out.put(data); + } + + /* (non-Javadoc) + * @see org.exist.storage.log.Loggable#read(java.nio.ByteBuffer) + */ + public void read(ByteBuffer in) { + int size = in.getInt(); + byte [] data = new byte[size]; + in.get(data); + original = new File(new String(data)); + size = in.getInt(); + data = new byte[size]; + in.get(data); + backup = new File(new String(data)); + System.out.println("Rename binary read: "+original+" -> "+backup); + } + + /* (non-Javadoc) + * @see org.exist.storage.log.Loggable#getLogSize() + */ + public int getLogSize() { + return 8 + original.getAbsolutePath().getBytes().length + backup.getAbsolutePath().getBytes().length; + } + + public void redo() throws LogException { + } + + public void undo() throws LogException { + System.out.println("Undo rename: "+original); + if (!backup.renameTo(original)) { + throw new LogException("Cannot move original "+original+" to backup file "+backup); + } + } + + public String dump() { + return super.dump() + " - rename "+original+" to "+backup; + } + +} diff --git a/src/org/exist/storage/UpdateBinaryLoggable.java b/src/org/exist/storage/UpdateBinaryLoggable.java new file mode 100644 index 00000000000..99fc46986b4 --- /dev/null +++ b/src/org/exist/storage/UpdateBinaryLoggable.java @@ -0,0 +1,108 @@ +/* + * RenameBinaryLoggable.java + * + * Created on December 9, 2007, 1:57 PM + * + * To change this template, choose Tools | Template Manager + * and open the template in the editor. + */ + +package org.exist.storage; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.log4j.Logger; +import org.exist.storage.journal.AbstractLoggable; +import org.exist.storage.journal.LogException; +import org.exist.storage.txn.Txn; + +/** + * + * @author alex + */ +public class UpdateBinaryLoggable extends AbstractLoggable { + + protected final static Logger LOG = Logger.getLogger(RenameBinaryLoggable.class); + + DBBroker broker; + File original; + File backup; + /** + * Creates a new instance of RenameBinaryLoggable + */ + public UpdateBinaryLoggable(DBBroker broker,Txn txn,File original,File backup) + { + super(NativeBroker.LOG_UPDATE_BINARY,txn.getId()); + this.broker = broker; + this.original = original; + this.backup = backup; + } + + public UpdateBinaryLoggable(DBBroker broker,long transactionId) { + super(NativeBroker.LOG_UPDATE_BINARY,transactionId); + this.broker = broker; + } + + /* (non-Javadoc) + * @see org.exist.storage.log.Loggable#write(java.nio.ByteBuffer) + */ + public void write(ByteBuffer out) { + String originalPath = original.getAbsolutePath(); + byte [] data = originalPath.getBytes(); + out.putInt(data.length); + out.put(data); + String backupPath = backup.getAbsolutePath(); + data = backupPath.getBytes(); + out.putInt(data.length); + out.put(data); + } + + /* (non-Javadoc) + * @see org.exist.storage.log.Loggable#read(java.nio.ByteBuffer) + */ + public void read(ByteBuffer in) { + int size = in.getInt(); + byte [] data = new byte[size]; + in.get(data); + original = new File(new String(data)); + size = in.getInt(); + data = new byte[size]; + in.get(data); + backup = new File(new String(data)); + } + + /* (non-Javadoc) + * @see org.exist.storage.log.Loggable#getLogSize() + */ + public int getLogSize() { + return 8 + original.getAbsolutePath().getBytes().length + backup.getAbsolutePath().getBytes().length; + } + + public void redo() throws LogException { + // TODO: is there something to do? The file has been written + } + + public void undo() throws LogException { + try { + FileInputStream is = new FileInputStream(backup); + FileOutputStream os = new FileOutputStream(original); + byte [] buffer = new byte[4096]; + int len; + while ((len=is.read(buffer))>=0) { + os.write(buffer,0,len); + } + os.close(); + is.close(); + } catch (IOException ex) { + + } + } + + public String dump() { + return super.dump() + " - update "+original+" to "+backup; + } + +} diff --git a/src/org/exist/storage/dom/DOMFile.java b/src/org/exist/storage/dom/DOMFile.java index 9a988712978..3ede22a00c3 100644 --- a/src/org/exist/storage/dom/DOMFile.java +++ b/src/org/exist/storage/dom/DOMFile.java @@ -157,7 +157,7 @@ public class DOMFile extends BTree implements Lockable { LogEntryTypes.addEntryType(LOG_UPDATE_LINK, UpdateLinkLoggable.class); } - public final static short FILE_FORMAT_VERSION_ID = 7; + public final static short FILE_FORMAT_VERSION_ID = 8; // page types public final static byte LOB = 21; diff --git a/src/org/exist/storage/journal/Journal.java b/src/org/exist/storage/journal/Journal.java index 3a63fc50efd..ea24571e412 100644 --- a/src/org/exist/storage/journal/Journal.java +++ b/src/org/exist/storage/journal/Journal.java @@ -22,6 +22,7 @@ package org.exist.storage.journal; import java.io.File; +import java.io.FileFilter; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileFilter; @@ -37,6 +38,7 @@ import org.exist.storage.lock.FileLock; import org.exist.storage.txn.Checkpoint; import org.exist.storage.txn.TransactionException; +import org.exist.util.FileUtils; import org.exist.util.ReadOnlyException; import org.exist.util.sanity.SanityCheck; @@ -143,9 +145,12 @@ public class Journal { /** if set to true, a sync will be triggered on the log file after every commit */ private boolean syncOnCommit = true; + private File fsJournalDir; + public Journal(BrokerPool pool, File directory) throws EXistException { this.dir = directory; this.pool = pool; + this.fsJournalDir = new File(directory,"fs.journal"); // we use a 1 megabyte buffer: currentBuffer = ByteBuffer.allocateDirect(1024 * 1024); @@ -329,6 +334,7 @@ public void checkpoint(long txnId, boolean switchLogFiles) throws TransactionExc } rt.start(); } + clearBackupFiles(); } catch (IOException e) { LOG.warn("IOException while writing checkpoint", e); } @@ -343,6 +349,18 @@ public void setCurrentFileNum(int fileNum) { currentFile = fileNum; } + public void clearBackupFiles() { + fsJournalDir.listFiles(new FileFilter() { + public boolean accept(File file) { + LOG.info("Checkpoint deleting "+file); + if (!FileUtils.delete(file)) { + LOG.fatal("Cannot delete file "+file+" from backup journal."); + } + return false; + } + }); + } + /** * Create a new journal with a larger file number * than the previous file. diff --git a/src/org/exist/storage/recovery/RecoveryManager.java b/src/org/exist/storage/recovery/RecoveryManager.java index e73931e7992..c6d2a436fa9 100644 --- a/src/org/exist/storage/recovery/RecoveryManager.java +++ b/src/org/exist/storage/recovery/RecoveryManager.java @@ -156,6 +156,7 @@ public boolean recover() throws LogException { } logManager.setCurrentFileNum(lastNum); logManager.switchFiles(); + logManager.clearBackupFiles(); return recoveryRun; } diff --git a/src/org/exist/util/FileUtils.java b/src/org/exist/util/FileUtils.java new file mode 100644 index 00000000000..10f24c89be7 --- /dev/null +++ b/src/org/exist/util/FileUtils.java @@ -0,0 +1,80 @@ +/* + * FileUtils.java + * + * Created on December 10, 2007, 1:11 PM + * + * To change this template, choose Tools | Template Manager + * and open the template in the editor. + */ + +package org.exist.util; + +import java.io.File; +import java.io.FileFilter; + +/** + * + * @author alex + */ +public class FileUtils +{ + + // Why is this here? Because we can't use generics because we're + // still in the dark ages of Java 1.4 + + static class FileRef { + File file; + FileRef next; + FileRef(FileRef next,File file) { + this.next = next; + this.file = file; + } + FileRef(File file) { + this.next = null; + this.file = file; + } + } + static class DeleteDir { + FileRef current; + boolean ok; + DeleteDir(File dir) { + current = new FileRef(dir); + ok = true; + } + public boolean delete() { + while (ok && current!=null) { + FileRef work = current; + current.file.listFiles(new FileFilter() { + public boolean accept(File file) { + if (file.isDirectory()) { + current = new FileRef(current,file); + } else { + ok = file.delete(); + } + return false; + } + }); + if (current==work) { + ok = current.file.delete(); + current = current.next; + } + } + return ok; + } + } + /** Creates a new instance of FileUtils */ + private FileUtils() + { + } + + public static boolean delete(File dir) + { + if (!dir.isDirectory()) { + return dir.delete(); + } + DeleteDir doDelete = new DeleteDir(dir); + return doDelete.delete(); + + } + +} diff --git a/src/org/exist/xmldb/LocalBinaryResource.java b/src/org/exist/xmldb/LocalBinaryResource.java index 4a2a6d58ccd..ebc9b69f3ac 100644 --- a/src/org/exist/xmldb/LocalBinaryResource.java +++ b/src/org/exist/xmldb/LocalBinaryResource.java @@ -29,6 +29,7 @@ import java.io.FileNotFoundException; import java.io.InputStream; import java.io.IOException; +import java.io.InputStream; import java.util.Date; import org.exist.EXistException; @@ -105,10 +106,17 @@ public Object getContent() throws XMLDBException { if(!blob.getPermissions().validate(user, Permission.READ)) throw new XMLDBException(ErrorCodes.PERMISSION_DENIED, "Permission denied to read resource"); - rawData = broker.getBinaryResource(blob); + InputStream is = broker.getBinaryResource(blob); + rawData = new byte[(int)broker.getBinaryResourceSize(blob)]; + is.read(rawData); + is.close(); + } catch(EXistException e) { throw new XMLDBException(ErrorCodes.VENDOR_ERROR, "error while loading binary resource " + getId(), e); + } catch(IOException e) { + throw new XMLDBException(ErrorCodes.VENDOR_ERROR, + "error while loading binary resource " + getId(), e); } finally { parent.getCollection().releaseDocument(blob, Lock.READ_LOCK); pool.release(broker); diff --git a/src/org/exist/xmlrpc/RpcConnection.java b/src/org/exist/xmlrpc/RpcConnection.java index 067116d20c3..ef81c16aa9e 100644 --- a/src/org/exist/xmlrpc/RpcConnection.java +++ b/src/org/exist/xmlrpc/RpcConnection.java @@ -802,9 +802,16 @@ public Hashtable getDocumentData(User user, String docName, Hashtable parameters result.put("offset", new Integer(firstChunk.length)); } else { - byte[] data = broker.getBinaryResource((BinaryDocument)doc); - result.put("data", data); - result.put("offset", new Integer(0)); + try { + InputStream is = broker.getBinaryResource((BinaryDocument)doc); + byte[] data = new byte[(int)broker.getBinaryResourceSize((BinaryDocument)doc)]; + is.read(data); + is.close(); + result.put("data", data); + result.put("offset", new Integer(0)); + } catch (IOException ex) { + throw new EXistException("I/O error while reading resource.",ex); + } } return result; } @@ -904,7 +911,15 @@ public byte[] getBinaryResource(User user, XmldbURI name) + " is not a binary resource"); if(!doc.getPermissions().validate(user, Permission.READ)) throw new PermissionDeniedException("Insufficient privileges to read resource"); - return broker.getBinaryResource((BinaryDocument) doc); + try { + InputStream is = broker.getBinaryResource((BinaryDocument)doc); + byte[] data = new byte[(int)broker.getBinaryResourceSize((BinaryDocument)doc)]; + is.read(data); + is.close(); + return data; + } catch (IOException ex) { + throw new EXistException("I/O error while reading resource.",ex); + } } finally { if(doc != null) doc.getUpdateLock().release(Lock.READ_LOCK); diff --git a/src/org/exist/xquery/functions/util/BinaryDoc.java b/src/org/exist/xquery/functions/util/BinaryDoc.java index 1d3b25e9ac9..488f50d7254 100644 --- a/src/org/exist/xquery/functions/util/BinaryDoc.java +++ b/src/org/exist/xquery/functions/util/BinaryDoc.java @@ -21,6 +21,8 @@ */ package org.exist.xquery.functions.util; +import java.io.IOException; +import java.io.InputStream; import java.net.URISyntaxException; import org.exist.dom.BinaryDocument; @@ -82,7 +84,10 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) return defaultReturn; if (isCalledAs("binary-doc")) { BinaryDocument bin = (BinaryDocument) doc; - byte[] data = context.getBroker().getBinaryResource(bin); + InputStream is = context.getBroker().getBinaryResource(bin); + byte[] data = new byte[(int)context.getBroker().getBinaryResourceSize(bin)]; + is.read(data); + is.close(); return new Base64Binary(data); } else return BooleanValue.TRUE; @@ -90,6 +95,8 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) throw new XPathException(getASTNode(), "Invalid resource uri",e); } catch (PermissionDeniedException e) { throw new XPathException(getASTNode(), path + ": permission denied to read resource"); + } catch (IOException e) { + throw new XPathException(getASTNode(), path + ": I/O error while reading resource",e); } finally { if (doc != null) doc.getUpdateLock().release(Lock.READ_LOCK); diff --git a/test/src/org/exist/storage/RecoverBinaryTest.java b/test/src/org/exist/storage/RecoverBinaryTest.java index 244a2406a23..1f8c0bd4d5a 100644 --- a/test/src/org/exist/storage/RecoverBinaryTest.java +++ b/test/src/org/exist/storage/RecoverBinaryTest.java @@ -23,6 +23,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.InputStream; import java.io.FileInputStream; import junit.framework.TestCase; @@ -106,7 +107,11 @@ public void testLoad() { assertNotNull(broker); BinaryDocument binDoc = (BinaryDocument) broker.getXMLResource(TestConstants.TEST_COLLECTION_URI.append(TestConstants.TEST_BINARY_URI), Lock.READ_LOCK); assertNotNull("Binary document is null", binDoc); - String data = new String(broker.getBinaryResource(binDoc)); + InputStream is = broker.getBinaryResource(binDoc); + byte [] bdata = new byte[(int)broker.getBinaryResourceSize(binDoc)]; + is.read(bdata); + is.close(); + String data = new String(bdata); assertNotNull(data); System.out.println(data); } catch (Exception e) { diff --git a/test/src/org/exist/storage/RecoveryTest.java b/test/src/org/exist/storage/RecoveryTest.java index 73048895dc4..47b8d98d724 100644 --- a/test/src/org/exist/storage/RecoveryTest.java +++ b/test/src/org/exist/storage/RecoveryTest.java @@ -21,6 +21,7 @@ */ package org.exist.storage; +import java.io.InputStream; import java.io.File; import java.io.StringWriter; import java.io.Writer; @@ -218,7 +219,11 @@ public void testRead() { BinaryDocument binDoc = (BinaryDocument) broker.getXMLResource(TestConstants.TEST_COLLECTION_URI2.append(TestConstants.TEST_BINARY_URI), Lock.READ_LOCK); assertNotNull("Binary document is null", binDoc); - data = new String(broker.getBinaryResource(binDoc)); + InputStream is = broker.getBinaryResource(binDoc); + byte [] bdata = new byte[(int)broker.getBinaryResourceSize(binDoc)]; + is.read(bdata); + is.close(); + data = new String(bdata); assertNotNull(data); System.out.println(data); @@ -241,8 +246,10 @@ public void testRead() { transact.commit(transaction); System.out.println("Transaction commited ..."); - } catch (Exception e) { + } catch (Exception e) { + if (transact!=null) { transact.abort(transaction); + } fail(e.getMessage()); e.printStackTrace(); } finally { diff --git a/test/src/org/exist/storage/ResourceTest.java b/test/src/org/exist/storage/ResourceTest.java index cc3edb9bc3a..0407a86e074 100644 --- a/test/src/org/exist/storage/ResourceTest.java +++ b/test/src/org/exist/storage/ResourceTest.java @@ -22,6 +22,7 @@ package org.exist.storage; +import java.io.InputStream; import junit.framework.TestCase; import junit.textui.TestRunner; @@ -127,7 +128,10 @@ public void testRead() { if(binDoc == null){ fail("Binary document '" + docPath + " does not exist."); } else { - data = broker.getBinaryResource(binDoc); + InputStream is = broker.getBinaryResource(binDoc); + data = new byte[(int)broker.getBinaryResourceSize(binDoc)]; + is.read(data); + is.close(); binDoc.getUpdateLock().release(Lock.READ_LOCK); } @@ -181,7 +185,10 @@ public void testRemoveCollection() { if(binDoc == null){ fail("Binary document '" + docPath + " does not exist."); } else { - data = broker.getBinaryResource(binDoc); + InputStream is = broker.getBinaryResource(binDoc); + data = new byte[(int)broker.getBinaryResourceSize(binDoc)]; + is.read(data); + is.close(); binDoc.getUpdateLock().release(Lock.READ_LOCK); }