Skip to content

Commit

Permalink
calculate checksum with less consumption of memory
Browse files Browse the repository at this point in the history 
Signed-off-by: Markus Barchfeld <markus.barchfeld@gmx.de>
  • Loading branch information
Markus Barchfeld committed Jul 20, 2016
1 parent 307a6a3 commit 93fbeff
Show file tree
Hide file tree
Showing 4 changed files with 262 additions and 11 deletions.
77 changes: 77 additions & 0 deletions ...del/src/org/eclipse/emf/emfstore/internal/common/model/util/ChecksumCalculatorWriter.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/*******************************************************************************
* Copyright (c) 2011-2016 EclipseSource Muenchen GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* meegenm - initial API and implementation
******************************************************************************/
package org.eclipse.emf.emfstore.internal.common.model.util;

import java.io.IOException;
import java.io.Writer;

/**
* calculates the checksum of all bytes streamed without wasting space
*
* @author Marco van Meegen
*
*/
public class ChecksumCalculatorWriter extends Writer {
private long checksum = 1125899906842597L; // prime
private long trimmedStringChecksum = -1;
private boolean trimmingLeading = true;

/**
* update checksum with bytes written
*
* @see java.io.Writer#write(char[], int, int)
*/
@Override
public void write(char[] cbuf, int off, int len) throws IOException {
for (int idx = 0; idx < len; idx++) {
final char c = cbuf[idx + off];
// trim leading whitespace <= ' '
if (!trimmingLeading || c > ' ') {
checksum = 31 * checksum + c;
if (c > ' ') {
// trimmdStringChecksum will always be the last checksum calculated where a non-whitespace was found
trimmedStringChecksum = checksum;
}
trimmingLeading = false;
}
}
}

/**
* {@inheritDoc}
*
* @see java.io.Writer#flush()
*/
@Override
public void flush() throws IOException {
// ignore

}

/**
* {@inheritDoc}
*
* @see java.io.Writer#close()
*/
@Override
public void close() throws IOException {
// ignore
}

/**
* @return the checksum calculated for all characters written to the writer
*/
public long getChecksum() {
return trimmedStringChecksum != -1 ? trimmedStringChecksum : checksum;
}

}
77 changes: 66 additions & 11 deletions ...store.common.model/src/org/eclipse/emf/emfstore/internal/common/model/util/ModelUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,9 +287,7 @@ private static long computeChecksum(String eObjectString) {

for (int i = 0; i < len; i++) {
final char c = eObjectString.charAt(i);

h = 31 * h + c;

}

return h;
Expand All @@ -305,10 +303,56 @@ private static long computeChecksum(String eObjectString) {
* @throws SerializationException
* in case any errors occur during computation of the checksum
*/
public static long computeChecksum(EObject eObject) throws SerializationException {
public static long computeChecksumOld(EObject eObject) throws SerializationException {
return computeChecksum(eObjectToString(eObject, getChecksumSaveOptions()));
}

/**
* Converts the given {@link EObject} to a string.
*
* @param copy The copied {@link EObject}.
* @param resource The resource for the {@link EObject}.
* @param saveOptions define the format of the returned serialization.
* @return The checksum of the serialized {@link EObject}.
* @throws SerializationException If a serialization problem occurs.
*/
private static long computeChecksumForCopiedEObject(EObject copy, XMIResource resource, Map<?, ?> saveOptions)
throws SerializationException {
resource.getContents().add(copy);

final ChecksumCalculatorWriter checksumCalculatorWriter = new ChecksumCalculatorWriter();
final URIConverter.WriteableOutputStream uws = new URIConverter.WriteableOutputStream(checksumCalculatorWriter,
CommonUtil.getEncoding());

try {
resource.save(uws, saveOptions);
} catch (final IOException e) {
throw new SerializationException(e);
} finally {
try {
uws.close();
} catch (final IOException exception) {
logException(exception);
}
}

return checksumCalculatorWriter.getChecksum();
}

/**
* Computes the checksum for a given {@link EObject}.
*
* @param eObject
* the EObject for which to compute a checksum
* @return the computed checksum
*
* @throws SerializationException
* in case any errors occur during computation of the checksum
*/
public static long computeChecksum(EObject eObject) throws SerializationException {
return computeChecksumInternal(eObject, false);
}

/**
* Computes the checksum for a given {@link IdEObjectCollection}.
* The checksum for a collection is independent of the order of the
Expand All @@ -322,23 +366,34 @@ public static long computeChecksum(EObject eObject) throws SerializationExceptio
* in case any errors occur during computation of the checksum
*/
public static long computeChecksum(IdEObjectCollection collection) throws SerializationException {
return computeChecksumInternal(collection, true);
}

private static long computeChecksumInternal(EObject eObject, boolean sort) throws SerializationException {
final ResourceSetImpl resourceSetImpl = new ResourceSetImpl();
// TODO: do we need to instantiate the factory registry each time?
resourceSetImpl.setResourceFactoryRegistry(new ResourceFactoryRegistry());
final XMIResource res = (XMIResource) resourceSetImpl.createResource(VIRTUAL_URI);
((ResourceImpl) res).setIntrinsicIDToEObjectMap(new HashMap<String, EObject>());
final IdEObjectCollection copy = copyIdEObjectCollection(collection, res);

ECollections.sort(copy.getModelElements(), new Comparator<EObject>() {
public int compare(EObject o1, EObject o2) {
return copy.getModelElementId(o1).getId().compareTo(copy.getModelElementId(o2).getId());
EObject copy;
if (eObject instanceof IdEObjectCollection) {
copy = copyIdEObjectCollection((IdEObjectCollection) eObject, res);
final IdEObjectCollection castedCopy = (IdEObjectCollection) copy;
if (sort) {
ECollections.sort(castedCopy.getModelElements(), new Comparator<EObject>() {
public int compare(EObject o1, EObject o2) {
return castedCopy.getModelElementId(o1).getId()
.compareTo(castedCopy.getModelElementId(o2).getId());
}
});
}
});
} else {
copy = copyEObject(ModelUtil.getProject(eObject), eObject, res);
}

final String serialized = copiedEObjectToString(copy, res);
// before: final String serialized = copiedEObjectToString(copy, res);

return computeChecksum(serialized);
return computeChecksumForCopiedEObject(copy, res, getChecksumSaveOptions());
}

/**
Expand Down
44 changes: 44 additions & 0 deletions tests/org.eclipse.emf.emfstore.server.test/ChecksumStreamingTest.launch
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<launchConfiguration type="org.eclipse.pde.ui.JunitLaunchConfig">
<booleanAttribute key="append.args" value="true"/>
<stringAttribute key="application" value="org.eclipse.pde.junit.runtime.coretestapplication"/>
<booleanAttribute key="askclear" value="false"/>
<booleanAttribute key="automaticAdd" value="true"/>
<booleanAttribute key="automaticValidate" value="false"/>
<stringAttribute key="bootstrap" value=""/>
<stringAttribute key="checked" value="[NONE]"/>
<booleanAttribute key="clearConfig" value="true"/>
<booleanAttribute key="clearws" value="true"/>
<booleanAttribute key="clearwslog" value="false"/>
<stringAttribute key="configLocation" value="${workspace_loc}/.metadata/.plugins/org.eclipse.pde.core/pde-junit"/>
<booleanAttribute key="default" value="false"/>
<stringAttribute key="deselected_workspace_plugins" value="org.eclipse.emf.emfstore.client.transaction,org.eclipse.emf.emfstore.client.ui.transaction,org.eclipse.emf.emfstore.mongodb,org.eclipse.emf.emfstore.mongodb.client,org.eclipse.emf.emfstore.mongodb.server"/>
<booleanAttribute key="includeOptional" value="true"/>
<stringAttribute key="location" value="${workspace_loc}/../junit-workspace"/>
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
<listEntry value="/org.eclipse.emf.emfstore.server.test/src/org/eclipse/emf/emfstore/server/test/ChecksumStreamingTest.java"/>
</listAttribute>
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
<listEntry value="1"/>
</listAttribute>
<stringAttribute key="org.eclipse.jdt.junit.CONTAINER" value=""/>
<booleanAttribute key="org.eclipse.jdt.junit.KEEPRUNNING_ATTR" value="false"/>
<stringAttribute key="org.eclipse.jdt.junit.TESTNAME" value=""/>
<stringAttribute key="org.eclipse.jdt.junit.TEST_KIND" value="org.eclipse.jdt.junit.loader.junit4"/>
<stringAttribute key="org.eclipse.jdt.launching.JRE_CONTAINER" value="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.eclipse.emf.emfstore.server.test.ChecksumStreamingTest"/>
<stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="-os ${target.os} -ws ${target.ws} -arch ${target.arch} -nl ${target.nl} -consoleLog"/>
<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="org.eclipse.emf.emfstore.server.test"/>
<stringAttribute key="org.eclipse.jdt.launching.SOURCE_PATH_PROVIDER" value="org.eclipse.pde.ui.workbenchClasspathProvider"/>
<stringAttribute key="pde.version" value="3.3"/>
<stringAttribute key="product" value="org.eclipse.emf.cdo.server.product.tcp_h2"/>
<booleanAttribute key="run_in_ui_thread" value="true"/>
<stringAttribute key="selected_target_plugins" value="ch.qos.logback.classic@default:default,ch.qos.logback.core@default:default,ch.qos.logback.slf4j@default:false,com.google.guava@default:default,com.ibm.icu@default:default,javax.annotation*1.2.0.v201401042248@default:default,javax.inject@default:default,javax.servlet@default:default,javax.xml@default:default,org.apache.batik.css@default:default,org.apache.batik.util.gui@default:default,org.apache.batik.util@default:default,org.apache.commons.codec*1.6.0.v201305230611@default:default,org.eclipse.ant.core@default:default,org.eclipse.compare.core@default:default,org.eclipse.compare@default:default,org.eclipse.core.commands@default:default,org.eclipse.core.contenttype@default:default,org.eclipse.core.databinding.beans@default:default,org.eclipse.core.databinding.observable@default:default,org.eclipse.core.databinding.property@default:default,org.eclipse.core.databinding@default:default,org.eclipse.core.expressions@default:default,org.eclipse.core.filebuffers@default:default,org.eclipse.core.filesystem.win32.x86_64@default:false,org.eclipse.core.filesystem@default:default,org.eclipse.core.jobs@default:default,org.eclipse.core.resources@default:default,org.eclipse.core.runtime.compatibility.registry@default:false,org.eclipse.core.runtime@default:true,org.eclipse.core.variables@default:default,org.eclipse.e4.core.commands@default:default,org.eclipse.e4.core.contexts@default:default,org.eclipse.e4.core.di.extensions@default:default,org.eclipse.e4.core.di@default:default,org.eclipse.e4.core.services@default:default,org.eclipse.e4.ui.bindings@default:default,org.eclipse.e4.ui.css.core@default:default,org.eclipse.e4.ui.css.swt.theme@default:default,org.eclipse.e4.ui.css.swt@default:default,org.eclipse.e4.ui.di@default:default,org.eclipse.e4.ui.model.workbench@default:default,org.eclipse.e4.ui.services@default:default,org.eclipse.e4.ui.widgets@default:default,org.eclipse.e4.ui.workbench.addons.swt@default:default,org.eclipse.e4.ui.workbench.renderers.swt@default:default,org.eclipse.e4.ui.workbench.swt@default:default,org.eclipse.e4.ui.workbench3@default:default,org.eclipse.e4.ui.workbench@default:default,org.eclipse.emf.cdo.ecore.retrofit@default:false,org.eclipse.emf.common.ui@default:default,org.eclipse.emf.common@default:default,org.eclipse.emf.compare.edit*4.0.0.201501201328@default:default,org.eclipse.emf.compare.ide.ui@default:default,org.eclipse.emf.compare.ide@default:default,org.eclipse.emf.compare.rcp.ui@default:default,org.eclipse.emf.compare.rcp@default:default,org.eclipse.emf.compare@default:default,org.eclipse.emf.ecore.change@default:default,org.eclipse.emf.ecore.edit@default:default,org.eclipse.emf.ecore.xmi@default:default,org.eclipse.emf.ecore@default:default,org.eclipse.emf.edit.ui@default:default,org.eclipse.emf.edit@default:default,org.eclipse.emf.transaction@default:default,org.eclipse.emf.validation@default:default,org.eclipse.equinox.app@default:default,org.eclipse.equinox.common@2:true,org.eclipse.equinox.ds@1:true,org.eclipse.equinox.event@default:default,org.eclipse.equinox.p2.core@default:default,org.eclipse.equinox.p2.engine@default:default,org.eclipse.equinox.p2.metadata.repository@default:default,org.eclipse.equinox.p2.metadata@default:default,org.eclipse.equinox.p2.repository@default:default,org.eclipse.equinox.preferences@default:default,org.eclipse.equinox.registry@default:default,org.eclipse.equinox.security.win32.x86_64@default:false,org.eclipse.equinox.security@default:default,org.eclipse.equinox.servletbridge.extensionbundle@default:false,org.eclipse.equinox.transforms.hook@default:false,org.eclipse.equinox.util@default:default,org.eclipse.equinox.weaving.hook@default:false,org.eclipse.help@default:default,org.eclipse.jdt.junit.runtime*3.4.500.v20140527-1138@default:default,org.eclipse.jface.databinding@default:default,org.eclipse.jface.text@default:default,org.eclipse.jface@default:default,org.eclipse.osgi*3.8.2.v20130124-134944@-1:true,org.eclipse.osgi.services@default:default,org.eclipse.swt.win32.win32.x86_64@default:false,org.eclipse.swt@default:default,org.eclipse.swtbot.eclipse.core@default:default,org.eclipse.swtbot.eclipse.finder@default:default,org.eclipse.swtbot.forms.finder@default:default,org.eclipse.swtbot.junit4_x@default:default,org.eclipse.swtbot.swt.finder@default:default,org.eclipse.team.core@default:default,org.eclipse.team.ui@default:default,org.eclipse.text@default:default,org.eclipse.ui.editors@default:default,org.eclipse.ui.forms@default:default,org.eclipse.ui.ide@default:default,org.eclipse.ui.navigator@default:default,org.eclipse.ui.views@default:default,org.eclipse.ui.win32@default:false,org.eclipse.ui.workbench.texteditor@default:default,org.eclipse.ui.workbench@default:default,org.eclipse.ui@default:default,org.hamcrest.core*1.3.0.v201303031735@default:default,org.hamcrest.library@default:default,org.junit*4.11.0.v201303080030@default:default,org.slf4j.api@default:default,org.slf4j.jcl@default:default,org.slf4j.log4j@default:default,org.w3c.css.sac@default:default,org.w3c.dom.smil@default:default,org.w3c.dom.svg@default:default"/>
<stringAttribute key="selected_workspace_plugins" value="de.metus.model@default:default,org.eclipse.emf.emfstore.branding@default:default,org.eclipse.emf.emfstore.client.api.test@default:false,org.eclipse.emf.emfstore.client.changetracking.test@default:false,org.eclipse.emf.emfstore.client.conflictdetection.test@default:false,org.eclipse.emf.emfstore.client.example.test@default:default,org.eclipse.emf.emfstore.client.model.edit@default:default,org.eclipse.emf.emfstore.client.recording.test@default:false,org.eclipse.emf.emfstore.client.test.alltests@default:default,org.eclipse.emf.emfstore.client.test.ui@default:default,org.eclipse.emf.emfstore.client.test@default:default,org.eclipse.emf.emfstore.client.ui.historybrowsercomparator@default:default,org.eclipse.emf.emfstore.client.ui.rap@default:false,org.eclipse.emf.emfstore.client.ui.rcp@default:false,org.eclipse.emf.emfstore.client.ui.test@default:false,org.eclipse.emf.emfstore.client.ui@default:default,org.eclipse.emf.emfstore.client@default:default,org.eclipse.emf.emfstore.common.model.edit@default:default,org.eclipse.emf.emfstore.common.model@default:default,org.eclipse.emf.emfstore.common@default:default,org.eclipse.emf.emfstore.ecore@default:default,org.eclipse.emf.emfstore.example.helloworld@default:default,org.eclipse.emf.emfstore.example.installer@default:default,org.eclipse.emf.emfstore.example.merging@default:default,org.eclipse.emf.emfstore.example.sessionprovider@default:default,org.eclipse.emf.emfstore.examplemodel.edit@default:default,org.eclipse.emf.emfstore.examplemodel@default:default,org.eclipse.emf.emfstore.fuzzy.emf.diff.test@default:default,org.eclipse.emf.emfstore.fuzzy.emf.edit@default:default,org.eclipse.emf.emfstore.fuzzy.emf.editor@default:default,org.eclipse.emf.emfstore.fuzzy.emf.example@default:default,org.eclipse.emf.emfstore.fuzzy.emf.test@default:default,org.eclipse.emf.emfstore.fuzzy.emf@default:default,org.eclipse.emf.emfstore.migration@default:default,org.eclipse.emf.emfstore.modelmutator.test@default:default,org.eclipse.emf.emfstore.modelmutator@default:default,org.eclipse.emf.emfstore.performance.test@default:default,org.eclipse.emf.emfstore.server.model.edit@default:default,org.eclipse.emf.emfstore.server.model@default:default,org.eclipse.emf.emfstore.server.test@default:false,org.eclipse.emf.emfstore.server@default:default,org.eclipse.emf.emfstore.test.common@default:default,org.eclipse.emf.emfstore.test.model.edit@default:default,org.eclipse.emf.emfstore.test.model@default:default"/>
<booleanAttribute key="show_selected_only" value="false"/>
<booleanAttribute key="tracing" value="false"/>
<booleanAttribute key="useCustomFeatures" value="false"/>
<booleanAttribute key="useDefaultConfig" value="true"/>
<booleanAttribute key="useDefaultConfigArea" value="false"/>
<booleanAttribute key="useProduct" value="false"/>
</launchConfiguration>
75 changes: 75 additions & 0 deletions ....emfstore.server.test/src/org/eclipse/emf/emfstore/server/test/ChecksumStreamingTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
/*******************************************************************************
* Copyright (c) 2012-2013 EclipseSource Muenchen GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
******************************************************************************/
package org.eclipse.emf.emfstore.server.test;

import static org.junit.Assert.assertEquals;

import org.eclipse.emf.emfstore.client.ESLocalProject;
import org.eclipse.emf.emfstore.client.test.common.cases.ESTestWithLoggedInUserMock;
import org.eclipse.emf.emfstore.internal.client.model.ESWorkspaceProviderImpl;
import org.eclipse.emf.emfstore.internal.client.model.ProjectSpace;
import org.eclipse.emf.emfstore.internal.client.model.impl.api.ESLocalProjectImpl;
import org.eclipse.emf.emfstore.internal.common.model.util.ModelUtil;
import org.eclipse.emf.emfstore.internal.common.model.util.SerializationException;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;

public class ChecksumStreamingTest extends ESTestWithLoggedInUserMock {
@Override
@Before
public void before() {
org.junit.Assume.assumeTrue(transactionalEditingDomainNotInUse());
super.before();
}

public boolean transactionalEditingDomainNotInUse() {
return !ESWorkspaceProviderImpl.getInstance().getEditingDomain().getClass().getName().contains("Transactional"); //$NON-NLS-1$
}

@Override
@After
public void after() {
super.after();
}

@BeforeClass
public static void beforeClass() {
startEMFStore();
}

@AfterClass
public static void afterClass() {
stopEMFStore();
}

@Test
public void testChecksumOptimization() throws SerializationException {
final long checksumStreaming = computeChecksumStreamingOrNonStreaming(getLocalProject(), true);
System.out.println("Checksum streaming: " + checksumStreaming); //$NON-NLS-1$
final long checksum = computeChecksumStreamingOrNonStreaming(getLocalProject(), false);
System.out.println("Checksum old: " + checksum); //$NON-NLS-1$
assertEquals(checksum, checksumStreaming);

}

private static long computeChecksumStreamingOrNonStreaming(ESLocalProject localProject,
boolean useStreamingCalculation)
throws SerializationException {
final ProjectSpace projectSpace = ((ESLocalProjectImpl) localProject).toInternalAPI();
final long checksum = useStreamingCalculation ? ModelUtil.computeChecksum(projectSpace.getProject())
: ModelUtil.computeChecksumOld(projectSpace.getProject());
System.out.println(ModelUtil.eObjectToString(projectSpace.getProject()));
return checksum;
}
}

0 comments on commit 93fbeff

Please sign in to comment.