Skip to content

Commit ac5472a

Browse files
dizzzzdizzzz
committed
[performance] move storage binary resources from dbx files to filesystem.
Yields into performance increase for (concurrent) access of binary files (=queries). Internally binary resources are dealt as Streams Patch created by Alex Milowski svn path=/trunk/eXist/; revision=8222
1 parent e6f3637 commit ac5472a

File tree

26 files changed

+879
-79
lines changed

26 files changed

+879
-79
lines changed

‎extensions/fluent/src/org/exist/fluent/Document.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -322,9 +322,15 @@ private DocumentImpl moveOrCopy(Folder destination, Name name, boolean copy) {
322322
public String contentsAsString() {
323323
DBBroker broker = db.acquireBroker();
324324
try {
325-
return new String(broker.getBinaryResource((BinaryDocument) doc), db.defaultCharacterEncoding);
325+
InputStream is = broker.getBinaryResource((BinaryDocument) doc);
326+
byte [] data = new byte[(int)broker.getBinaryResourceSize((BinaryDocument) doc)];
327+
is.read(data);
328+
is.close();
329+
return new String(data, db.defaultCharacterEncoding);
326330
} catch (UnsupportedEncodingException e) {
327331
throw new DatabaseException(e);
332+
} catch (IOException e) {
333+
throw new DatabaseException(e);
328334
} finally {
329335
db.releaseBroker(broker);
330336
}

‎extensions/fluent/test/src/org/exist/fluent/DocumentTest.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,16 @@ public class DocumentTest extends DatabaseTestCase {
3636
assertEquals("helloworld", original.contentsAsString());
3737
assertEquals("helloworld", copy.contentsAsString());
3838
}
39+
40+
@Test public void copy2() {
41+
Folder c1 = db.createFolder("/c1"), c2 = db.createFolder("/c2");
42+
Document original = c1.documents().load(Name.create("original.xml"), Source.xml("<original/>"));
43+
Document copy = original.copy(c2, Name.keepCreate());
44+
assertEquals(1, c1.documents().size());
45+
assertEquals(1, c2.documents().size());
46+
assertEquals("<original/>", original.contentsAsString());
47+
assertEquals("<original/>", copy.contentsAsString());
48+
}
3949

4050
@Test public void move1() {
4151
Folder c1 = db.createFolder("/c1"), c2 = db.createFolder("/c2");
@@ -46,5 +56,15 @@ public class DocumentTest extends DatabaseTestCase {
4656
assertEquals("/c2/original", doc.path());
4757
assertEquals("helloworld", doc.contentsAsString());
4858
}
59+
60+
@Test public void move2() {
61+
Folder c1 = db.createFolder("/c1"), c2 = db.createFolder("/c2");
62+
Document doc = c1.documents().load(Name.create("original.xml"), Source.xml("<original/>"));
63+
doc.move(c2, Name.keepCreate());
64+
assertEquals(0, c1.documents().size());
65+
assertEquals(1, c2.documents().size());
66+
assertEquals("/c2/original.xml", doc.path());
67+
assertEquals("<original/>", doc.contentsAsString());
68+
}
4969

5070
}

‎extensions/modules/src/org/exist/xquery/modules/compression/TarFunction.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
*/
2222
package org.exist.xquery.modules.compression;
2323

24+
import java.io.InputStream;
2425
import java.io.ByteArrayOutputStream;
2526
import java.io.IOException;
2627
import java.util.Iterator;
@@ -210,9 +211,13 @@ private void tarResource(TarOutputStream tos, DocumentImpl doc,
210211
tos.write(strDoc.getBytes());
211212
} else if (doc.getResourceType() == DocumentImpl.BINARY_FILE) {
212213
// binary file
213-
byte[] data = context.getBroker().getBinaryResource(
214-
(BinaryDocument) doc);
215-
tos.write(data);
214+
InputStream is = context.getBroker().getBinaryResource((BinaryDocument)doc);
215+
byte[] data = new byte[16384];
216+
int len = 0;
217+
while ((len=is.read(data,0,data.length))>0) {
218+
tos.write(data,0,len);
219+
}
220+
is.close();
216221
}
217222

218223
// close the entry in the Tar
@@ -261,4 +266,4 @@ private void tarCollection(TarOutputStream tos, Collection col,
261266
tarCollection(tos, childCol, useHierarchy, stripOffset);
262267
}
263268
}
264-
}
269+
}

‎extensions/modules/src/org/exist/xquery/modules/compression/ZipFunction.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
package org.exist.xquery.modules.compression;
2323

2424
import java.io.ByteArrayOutputStream;
25+
import java.io.InputStream;
2526
import java.io.IOException;
2627
import java.util.Iterator;
2728
import java.util.zip.ZipEntry;
@@ -210,9 +211,14 @@ private void zipResource(ZipOutputStream zos, DocumentImpl doc,
210211
zos.write(strDoc.getBytes());
211212
} else if (doc.getResourceType() == DocumentImpl.BINARY_FILE) {
212213
// binary file
213-
byte[] data = context.getBroker().getBinaryResource(
214+
InputStream is = context.getBroker().getBinaryResource(
214215
(BinaryDocument) doc);
215-
zos.write(data);
216+
byte [] data = new byte[16384];
217+
int len;
218+
while ((len=is.read(data))>0) {
219+
zos.write(data,0,len);
220+
}
221+
is.close();
216222
}
217223

218224
// close the entry in the Zip

‎extensions/modules/src/org/exist/xquery/modules/image/GetThumbnailsFunction.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import java.awt.Image;
44
import java.awt.image.BufferedImage;
5+
import java.io.InputStream;
56
import java.io.ByteArrayInputStream;
67
import java.io.ByteArrayOutputStream;
78
import java.io.File;
@@ -213,11 +214,10 @@ public boolean accept(File dir, String name) {
213214
binImage = (BinaryDocument) docImage;
214215

215216
// get a byte array representing the image
216-
imgData = dbbroker.getBinaryResource(binImage);
217217

218218
try {
219-
image = ImageIO.read(new ByteArrayInputStream(
220-
imgData));
219+
InputStream is = dbbroker.getBinaryResource(binImage);
220+
image = ImageIO.read(is);
221221
} catch (IOException ioe) {
222222
throw new XPathException(getASTNode(),ioe.getMessage());
223223
}

‎src/org/exist/cluster/ClusterCollection.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,11 @@ public void store(Txn transaction, DBBroker broker, IndexInfo info, InputSource
165165
public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
166166
XmldbURI name, byte[] data, String mimeType) throws EXistException,
167167
PermissionDeniedException, LockException, TriggerException {
168+
try {
168169
return collection.addBinaryResource(transaction, broker, name, data, mimeType);
170+
} catch (IOException ex) {
171+
throw new EXistException("Cannot add binary due to I/O error.",ex);
172+
}
169173
}
170174

171175
public Lock getLock() {

‎src/org/exist/collections/Collection.java‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -865,7 +865,11 @@ public void removeBinaryResource(Txn transaction, DBBroker broker, DocumentImpl
865865
if (trigger != null)
866866
trigger.prepare(Trigger.REMOVE_DOCUMENT_EVENT, broker, transaction, doc.getURI(), doc);
867867

868-
broker.removeBinaryResource(transaction, (BinaryDocument) doc);
868+
try {
869+
broker.removeBinaryResource(transaction, (BinaryDocument) doc);
870+
} catch (IOException ex) {
871+
throw new PermissionDeniedException("Cannot delete file.");
872+
}
869873
documents.remove(doc.getFileURI().getRawCollectionPath());
870874

871875
if (trigger != null) {
@@ -1361,29 +1365,29 @@ private DocumentTrigger setupTriggers(DBBroker broker, XmldbURI docUri, boolean
13611365
// Blob
13621366
public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
13631367
XmldbURI docUri, byte[] data, String mimeType)
1364-
throws EXistException, PermissionDeniedException, LockException, TriggerException {
1368+
throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException {
13651369
return addBinaryResource(transaction, broker, docUri, data, mimeType, null, null);
13661370
}
13671371

13681372
// Blob
13691373
public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
13701374
XmldbURI docUri, byte[] data, String mimeType, Date created, Date modified)
1371-
throws EXistException, PermissionDeniedException, LockException, TriggerException {
1375+
throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException {
13721376
return addBinaryResource(transaction, broker, docUri,
13731377
new ByteArrayInputStream(data), mimeType, data.length, created, modified);
13741378
}
13751379

13761380
// Streaming
13771381
public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
13781382
XmldbURI docUri, InputStream is, String mimeType, int size)
1379-
throws EXistException, PermissionDeniedException, LockException, TriggerException {
1383+
throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException {
13801384
return addBinaryResource(transaction, broker, docUri, is, mimeType, size, null, null);
13811385
}
13821386

13831387
// Streaming
13841388
public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
13851389
XmldbURI docUri, InputStream is, String mimeType, int size, Date created, Date modified)
1386-
throws EXistException, PermissionDeniedException, LockException, TriggerException {
1390+
throws EXistException, PermissionDeniedException, LockException, TriggerException,IOException {
13871391

13881392
if (broker.isReadOnly())
13891393
throw new PermissionDeniedException("Database is read-only");
@@ -1400,6 +1404,7 @@ public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
14001404
checkPermissions(transaction, broker, oldDoc);
14011405
DocumentTrigger trigger = null;
14021406
int event = 0;
1407+
/*
14031408
if (triggersEnabled) {
14041409
CollectionConfiguration config = getConfiguration(broker);
14051410
if (config != null) {
@@ -1414,6 +1419,7 @@ public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
14141419
}
14151420
}
14161421
}
1422+
*/
14171423

14181424
manageDocumentInformation(broker, oldDoc, blob );
14191425
DocumentMetadata metadata = blob.getMetadata();
@@ -1438,8 +1444,24 @@ public BinaryDocument addBinaryResource(Txn transaction, DBBroker broker,
14381444
addDocument(transaction, broker, blob);
14391445

14401446
broker.storeXMLResource(transaction, blob);
1447+
if (triggersEnabled) {
1448+
CollectionConfiguration config = getConfiguration(broker);
1449+
if (config != null) {
1450+
event = oldDoc != null ? Trigger.UPDATE_DOCUMENT_EVENT : Trigger.STORE_DOCUMENT_EVENT;
1451+
try {
1452+
trigger = (DocumentTrigger) config.newTrigger(event, broker, this);
1453+
} catch (CollectionConfigurationException e) {
1454+
LOG.debug("An error occurred while initializing a trigger for collection " + getURI() + ": " + e.getMessage(), e);
1455+
}
1456+
if (trigger != null) {
1457+
trigger.prepare(event, broker, transaction, blob.getURI(), blob);
1458+
}
1459+
}
1460+
}
1461+
14411462

1442-
broker.closeDocument();
1463+
// This is no longer needed as the dom.dbx isn't used
1464+
//broker.closeDocument();
14431465

14441466
if (trigger != null) {
14451467
trigger.finish(event, broker, transaction, blob.getURI(), blob);

‎src/org/exist/collections/triggers/XQueryTrigger.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.exist.collections.triggers;
22

33
import java.io.IOException;
4+
import java.io.InputStream;
45
import java.util.Iterator;
56
import java.util.Map;
67
import java.util.Properties;
@@ -219,7 +220,10 @@ else if (existingDocument instanceof BinaryDocument)
219220
{
220221
//binary document
221222
BinaryDocument bin = (BinaryDocument)existingDocument;
222-
byte[] data = context.getBroker().getBinaryResource(bin);
223+
InputStream is = broker.getBinaryResource(bin);
224+
byte [] data = new byte[(int)broker.getBinaryResourceSize(bin)];
225+
is.read(data);
226+
is.close();
223227

224228
context.declareVariable(bindingPrefix + "document", new Base64Binary(data));
225229
}
@@ -302,7 +306,10 @@ else if (document instanceof BinaryDocument)
302306
{
303307
//Binary document
304308
BinaryDocument bin = (BinaryDocument)document;
305-
byte[] data = context.getBroker().getBinaryResource(bin);
309+
InputStream is = broker.getBinaryResource(bin);
310+
byte [] data = new byte[(int)broker.getBinaryResourceSize(bin)];
311+
is.read(data);
312+
is.close();
306313

307314
context.declareVariable(bindingPrefix + "document", new Base64Binary(data));
308315
}

‎src/org/exist/http/SOAPServer.java‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1309,9 +1309,18 @@ private BinaryDocument getXQWS(DBBroker broker, String path) throws PermissionDe
13091309
*/
13101310
private byte[] getXQWSData(DBBroker broker, BinaryDocument docXQWS)
13111311
{
1312-
byte[] data = broker.getBinaryResource(docXQWS);
1313-
1314-
return data;
1312+
try {
1313+
InputStream is = broker.getBinaryResource(docXQWS);
1314+
byte [] data = new byte[(int)broker.getBinaryResourceSize(docXQWS)];
1315+
is.read(data);
1316+
is.close();
1317+
1318+
return data;
1319+
} catch (IOException ex) {
1320+
// TODO: where should this go?
1321+
ex.printStackTrace();
1322+
}
1323+
return null;
13151324
}
13161325

13171326
/**

‎src/org/exist/soap/AdminSoapBindingImpl.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,12 @@
11
package org.exist.soap;
2+
import java.io.ByteArrayInputStream;
3+
import java.io.IOException;
4+
import java.io.InputStream;
5+
import java.io.StringReader;
6+
import java.net.URISyntaxException;
7+
import java.rmi.RemoteException;
8+
import java.util.Iterator;
9+
import java.util.Vector;
210

311
import org.apache.log4j.Logger;
412
import org.exist.EXistException;
@@ -449,7 +457,11 @@ public byte[] getBinaryResource(java.lang.String sessionId, XmldbURI name) throw
449457
+ " is not a binary resource");
450458
if(!doc.getPermissions().validate(session.getUser(), Permission.READ))
451459
throw new PermissionDeniedException("Insufficient privileges to read resource");
452-
return broker.getBinaryResource( (BinaryDocument) doc );
460+
InputStream is = broker.getBinaryResource((BinaryDocument) doc);
461+
byte [] data = new byte[(int)broker.getBinaryResourceSize((BinaryDocument) doc)];
462+
is.read(data);
463+
is.close();
464+
return data;
453465
} catch (Exception ex) {
454466
throw new RemoteException(ex.getMessage());
455467
} finally {

0 commit comments

Comments
 (0)