Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Speedup #30

Merged
merged 22 commits into from
Mar 28, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
d23bbd9
Modified access to resource files so that it works on Windows and
altaite Feb 28, 2017
9c4052b
Replaced Jackson by MessagePack parsing code from Jmol. Jackson remains
altaite Mar 1, 2017
36d075f
Moving code from getNumBonds() to constructor results in 4-fold speedup
altaite Mar 1, 2017
5ce9fbc
FLOAT64 is now parsed as double, which corresponds to what Jackson did.
altaite Mar 1, 2017
7c10d24
Added a test comparing if results of Jackson and the new MessagePack
altaite Mar 5, 2017
c99723c
Moved MessagePack test into mmtf-codec.
altaite Mar 5, 2017
1570040
Version set to 1.05.
altaite Mar 6, 2017
3822c89
Renaming of mp package to quickmessagepackdeserialization.
altaite Mar 9, 2017
86c7a03
Zipped resource pdb_entry_type.txt. Replaced few other
altaite Mar 17, 2017
06455cb
Replaced zip with gz. Also corrected the loading of file using URL,
altaite Mar 18, 2017
5fc6b9a
Nicer code for reading text files as resources in gz. URL are used
altaite Mar 21, 2017
5f8057f
Minor formatting changes and removal of dead code in comments.
altaite Mar 22, 2017
acf8e1c
Message Pack parsing code from Bob Hanson's Jmol simplified and moved…
altaite Mar 23, 2017
032d454
PDB codes for testing now does not contain missing records (no codes …
altaite Mar 24, 2017
2999e37
Correction of versions to 1.0.4-SNAPSHOT.
altaite Mar 25, 2017
4f69622
Better API for Lines utility.
altaite Mar 26, 2017
ef9034f
Merge branch 'master' into speedup
altaite Mar 27, 2017
2d388ef
Better API for Lines utility. Also number of tested structures
altaite Mar 27, 2017
cd39c88
Merge origin/speedup into speedup
altaite Mar 27, 2017
457bf2c
Replaced Exception for ParseException in MessagePack decoding.
altaite Mar 28, 2017
07425c1
More appropriate name for test.
altaite Mar 28, 2017
72e468e
More descriptive method names in ObjectTree.
altaite Mar 28, 2017
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file modified mmtf-api/pom.xml
Whitespace-only changes.
Empty file modified mmtf-codec/pom.xml
Whitespace-only changes.
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ public GenericDecoder(MmtfStructure inputData) {
numGroups = inputData.getNumGroups();
numChains = inputData.getNumChains();
numModels = inputData.getNumModels();
numIntergroupBonds = interGroupBondOrders.length;
for(int groupIndex : groupList) {
numIntergroupBonds += groupMap[groupIndex].getBondOrderList().length;
}
ncsOperMatrixList = inputData.getNcsOperatorList();
}

Expand Down Expand Up @@ -183,6 +187,8 @@ public GenericDecoder(MmtfStructure inputData) {

private int numGroups;

private int numIntergroupBonds;

private double[][] ncsOperMatrixList;


Expand Down Expand Up @@ -452,10 +458,6 @@ public String getDepositionDate() {

@Override
public int getNumBonds() {
int numIntergroupBonds = interGroupBondOrders.length;
for(int groupIndex : groupList) {
numIntergroupBonds+=groupMap[groupIndex].getBondOrderList().length;
}
return numIntergroupBonds;
}

Expand Down
94 changes: 57 additions & 37 deletions mmtf-codec/src/main/java/org/rcsb/mmtf/decoder/ReaderUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.ParseException;
import java.util.zip.GZIPInputStream;

import org.rcsb.mmtf.dataholders.MmtfStructure;
Expand All @@ -15,110 +16,129 @@

/**
* A class of static utility methods for reading data.
*
* @author Anthony Bradley
*
*/
public class ReaderUtils {

/** The size of a chunk for a byte buffer. */
/**
* The size of a chunk for a byte buffer.
*/
private static final int BYTE_BUFFER_CHUNK_SIZE = 4096;

/**
* Find the message pack byte array from the web using the input code and a base url.
* Caches the file if possible.
* Find the message pack byte array from the web using the input code and a base url. Caches the
* file if possible.
*
* @param pdbCode the pdb code for the desired structure.
* @return the MMTFBean of the deserialized data
* @throws java.text.ParseException if MessagePack cannot be parsed
* @throws IOException if the data cannot be read from the URL
*/
public static MmtfStructure getDataFromUrl(String pdbCode) throws IOException {
public static MmtfStructure getDataFromUrl(String pdbCode)
throws ParseException, IOException {
// Get these as an inputstream
byte[] byteArr = getByteArrayFromUrl(pdbCode);
byte[] bytes = getByteArrayFromUrl(pdbCode);
// Now return the gzip deflated and deserialized byte array
MessagePackSerialization mmtfBeanSeDeMessagePackImpl = new MessagePackSerialization();
return mmtfBeanSeDeMessagePackImpl.deserialize(new ByteArrayInputStream(deflateGzip(byteArr)));
MessagePackSerialization mmtfBeanSeDeMessagePackImpl
= new MessagePackSerialization();
return mmtfBeanSeDeMessagePackImpl.deserialize(new ByteArrayInputStream(
deflateGzip(bytes)));
}

/**
* Get the GZIP compressed and messagepack serialized data from the MMTF servers
*
* @param pdbCode the PDB code for the data required
* @return the byte array (GZIP compressed) of the data from the URL
* @throws IOException an error reading the URL
*/
public static byte[] getByteArrayFromUrl(String pdbCode) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
InputStream inputStream = null;
public static byte[] getByteArrayFromUrl(String pdbCode)
throws IOException {
URL url = new URL(getUrl(pdbCode));
try {
inputStream = url.openStream();
byte[] byteChunk = new byte[BYTE_BUFFER_CHUNK_SIZE]; // Or whatever size you want to read in at a time.
try (InputStream inputStream = url.openStream();
ByteArrayOutputStream baos = new ByteArrayOutputStream();) {
byte[] byteChunk = new byte[BYTE_BUFFER_CHUNK_SIZE];
// Or whatever size you want to read in at a time.
int n;
while ((n = inputStream.read(byteChunk)) > 0) {
baos.write(byteChunk, 0, n);
}
} finally {
if (inputStream != null) { inputStream.close(); }
}
return baos.toByteArray();
}
}

/**
* Deflate a gzip byte array.
*
* @param inputBytes a gzip compressed byte array
* @return a deflated byte array
* @throws IOException error in gzip input stream
*/
public static byte[] deflateGzip(byte[] inputBytes) throws IOException {
// Start the byte input stream
ByteArrayInputStream byteInputStream = new ByteArrayInputStream(inputBytes);
GZIPInputStream gzipInputStream;
ByteArrayOutputStream byteArrayOutputStream = null;
try {
gzipInputStream = new GZIPInputStream(byteInputStream);

byteArrayOutputStream = new ByteArrayOutputStream();
// Make a buffer
ByteArrayOutputStream byteArrayOutputStream
= new ByteArrayOutputStream();
try (GZIPInputStream gzipInputStream = new GZIPInputStream(
new ByteArrayInputStream(inputBytes))) {
byte[] buffer = new byte[BYTE_BUFFER_CHUNK_SIZE];

while (gzipInputStream.available() == 1) {
int size = gzipInputStream.read(buffer);
if (size == -1) {
break;
}
byteArrayOutputStream.write(buffer, 0, size);
}

} finally {
if (byteArrayOutputStream != null) {
byteArrayOutputStream.close();
}
}
return byteArrayOutputStream.toByteArray();
}
}

/**
* A function to get MMTF data from a file path.
*
* @param filePath the full path of the file to be read
* @return the deserialized {@link MmtfStructure}
* @throws IOException an error reading the file
* @throws java.text.ParseException if MessagePack cannot be parsed
*/
public static MmtfStructure getDataFromFile(Path filePath) throws IOException {
public static MmtfStructure getDataFromFile(Path filePath)
throws IOException, ParseException {
// Now return the gzip deflated and deserialized byte array
try (InputStream is = new ByteArrayInputStream(readFile(filePath))) {
return getDataFromInputStream(is);
}
}

return getDataFromInputStream(Files.newInputStream(filePath));
/**
* Read a byte array from a file
*
* @param path the input file path
* @return the returned byte array
* @throws IOException an error reading the file
*/
private static byte[] readFile(Path path) throws IOException {
byte[] data = Files.readAllBytes(path);
return data;
}

/**
* Read an input stream to an {@link MmtfStructure} object.
*
* @param inStream the {@link InputStream} to read.
* @return the {@link MmtfStructure} to be returned
* @throws java.text.ParseException if MessagePack cannot be parsed
* @throws java.io.IOException if the inStream cannot be read
*/
public static MmtfStructure getDataFromInputStream(InputStream inStream) {
MessagePackSerialization mmtfBeanSeDeMessagePackImpl = new MessagePackSerialization();
public static MmtfStructure getDataFromInputStream(InputStream inStream)
throws ParseException, IOException {
MessagePackSerialization mmtfBeanSeDeMessagePackImpl
= new MessagePackSerialization();
return mmtfBeanSeDeMessagePackImpl.deserialize(inStream);

}

/**
* Get the URL to return a given PDB id
*
* @param pdbId the input PDB id
* @return the URL {@link String} to get data from
*/
Expand Down
11 changes: 5 additions & 6 deletions mmtf-codec/src/test/java/org/rcsb/mmtf/codec/TestRoundTrip.java
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.ParseException;

import org.junit.Test;
import org.rcsb.mmtf.api.StructureDataInterface;
Expand All @@ -24,7 +24,7 @@ public class TestRoundTrip {
* @throws IOException error reading the file from the resource
*/
@Test
public void testGenericGeneric() throws IOException {
public void testGenericGeneric() throws IOException, ParseException {
Utils.compare(getDefaultFullData());
}

Expand Down Expand Up @@ -58,9 +58,8 @@ public void testArrayRoundTrip() {
* @return a {@link StructureDataInterface} for the full data.
* @throws IOException
*/
private StructureDataInterface getDefaultFullData() throws IOException {
ClassLoader classLoader = getClass().getClassLoader();
Path inFile = Paths.get(classLoader.getResource("mmtf/4cup.mmtf").getFile());
return new GenericDecoder(ReaderUtils.getDataFromFile(inFile));
private StructureDataInterface getDefaultFullData() throws IOException, ParseException {
Path p = Utils.getResource("/mmtf/4cup.mmtf");
return new GenericDecoder(ReaderUtils.getDataFromFile(p));
}
}
13 changes: 13 additions & 0 deletions mmtf-codec/src/test/java/org/rcsb/mmtf/codec/Utils.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,15 @@
import java.beans.IntrospectionException;
import java.beans.Introspector;
import java.beans.PropertyDescriptor;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.nio.file.Path;
import java.nio.file.Paths;

import org.rcsb.mmtf.api.StructureDataInterface;
import org.rcsb.mmtf.decoder.GenericDecoder;
import org.rcsb.mmtf.decoder.ReaderUtils;
import org.rcsb.mmtf.encoder.GenericEncoder;

/**
Expand Down Expand Up @@ -72,4 +77,12 @@ public static void compareStructDataInfs(StructureDataInterface interfaceOne, St
assertArrayEquals(interfaceOne.getInterGroupBondIndices(), interfaceTwo.getInterGroupBondIndices());
assertArrayEquals(interfaceOne.getInterGroupBondOrders(), interfaceTwo.getInterGroupBondOrders());
}

/**
* path = "/mmtf/4cup.mmtf"
*/
public static Path getResource(String p) throws IOException {
File f = new File(Utils.class.getClass().getResource(p).getFile());
return Paths.get(f.getAbsolutePath());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@
import java.beans.PropertyDescriptor;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.nio.file.Paths;
import java.text.ParseException;

import org.junit.Test;
import org.rcsb.mmtf.codec.Utils;
import org.rcsb.mmtf.dataholders.MmtfStructure;
import org.unitils.reflectionassert.ReflectionAssert;

Expand All @@ -29,11 +30,13 @@ public class TestGenericDecoder {
* @throws IllegalAccessException an error with introspection
*/
@Test
public void testDecodeAllFields() throws IOException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, IntrospectionException {
public void testDecodeAllFields() throws IOException, ParseException, IllegalAccessException,
IllegalArgumentException, InvocationTargetException, IntrospectionException {
ClassLoader classLoader = getClass().getClassLoader();
MmtfStructure mmtfBean = ReaderUtils.getDataFromFile(Paths.get(classLoader.getResource("mmtf/4cup.mmtf").getFile()));
MmtfStructure mmtfBean = ReaderUtils.getDataFromFile(Utils.getResource("/mmtf/4cup.mmtf"));
GenericDecoder genericDecoder = new GenericDecoder(mmtfBean);
ReflectionAssert.assertPropertiesNotNull("Some properties null after decoding", genericDecoder);
ReflectionAssert.assertPropertiesNotNull("Some properties null after decoding",
genericDecoder);
for(PropertyDescriptor propertyDescriptor :
Introspector.getBeanInfo(MmtfStructure.class).getPropertyDescriptors()){
assertNotNull(propertyDescriptor.getReadMethod().invoke(mmtfBean));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@


import java.io.IOException;
import java.nio.file.Paths;
import java.text.ParseException;
import org.rcsb.mmtf.codec.Utils;

/**
* Test the reader utils class functions work.
Expand Down Expand Up @@ -81,9 +82,9 @@ public void testGzipDecompressText() throws IOException {
* @throws IOException error accesing the file
*/
@Test
public void testReadFromFile() throws IOException {
public void testReadFromFile() throws IOException, ParseException {
ClassLoader classLoader = getClass().getClassLoader();
MmtfStructure mmtfBean = ReaderUtils.getDataFromFile(Paths.get(classLoader.getResource("mmtf/4cup.mmtf").getFile()));
MmtfStructure mmtfBean = ReaderUtils.getDataFromFile(Utils.getResource("/mmtf/4cup.mmtf"));
assertNotEquals(mmtfBean, null);
assertEquals(mmtfBean.getDepositionDate(), "2014-03-21");
}
Expand Down
Loading