From b312e36a08562eb2d14aec2e361e7891be9b4b97 Mon Sep 17 00:00:00 2001 From: Robert Virkus Date: Mon, 5 May 2014 16:17:23 +0200 Subject: [PATCH] now being able to remove elements from a ChunkedStorageCollection that reside in the already persisted chunked area --- .../polish/io/ChunkedStorageCollection.java | 171 ++++++++++++++++-- .../io/ChunkedStorageCollectionTest.java | 59 +++++- 2 files changed, 210 insertions(+), 20 deletions(-) diff --git a/enough-polish-j2me/source/src/de/enough/polish/io/ChunkedStorageCollection.java b/enough-polish-j2me/source/src/de/enough/polish/io/ChunkedStorageCollection.java index 0f70547..8546a5e 100644 --- a/enough-polish-j2me/source/src/de/enough/polish/io/ChunkedStorageCollection.java +++ b/enough-polish-j2me/source/src/de/enough/polish/io/ChunkedStorageCollection.java @@ -32,6 +32,7 @@ import java.io.IOException; import de.enough.polish.util.ArrayList; +import de.enough.polish.util.IntList; /** * Allows to store lots of data that is retrieved lazily @@ -45,12 +46,14 @@ public abstract class ChunkedStorageCollection public final static int STORAGE_STRATEGY_CHUNKED = 2; //public final static int STORAGE_STRATEGY_MANUAL = 3; - private final static int PERSISTENCE_VERSION = 1; + private final static int PERSISTENCE_VERSION = 2; private int chunkSize; private String identifier; private int completeSize; + private int numberOfDeletedEntries; + private IntList deletedIndecesList; private int tailCollectionStartIndex; private ArrayList tailCollection; private boolean tailCollectionIsDirty; @@ -79,6 +82,13 @@ public synchronized void write(DataOutputStream out) throws IOException { out.writeInt(this.chunkSize); out.writeInt(this.completeSize); out.writeInt(this.tailCollectionStartIndex); + // persistence version 2+: + out.writeInt(this.numberOfDeletedEntries); + if (this.numberOfDeletedEntries > 0) + { + this.deletedIndecesList.write(out); + } + // persistence version 1+: int tailSize = getTailCollection().size(); out.writeInt(tailSize); Object[] internalObjects = getTailCollection().getInternalArray(); @@ -101,6 +111,17 @@ public synchronized void read(DataInputStream in) throws IOException { this.chunkSize = in.readInt(); this.completeSize = in.readInt(); this.tailCollectionStartIndex = in.readInt(); + // persistence version 2+: + if (version >= 2) + { + this.numberOfDeletedEntries = in.readInt(); + if (this.numberOfDeletedEntries > 0) + { + this.deletedIndecesList = new IntList(); + this.deletedIndecesList.read(in); + } + } + // persistence version 1+: int size = in.readInt(); if ( this.tailCollection == null ) { this.tailCollection = new ArrayList(chunkSize*2); @@ -119,7 +140,7 @@ public synchronized int size() { loadTailCollection(); } - return this.completeSize; + return this.completeSize - this.numberOfDeletedEntries; } public synchronized int sizeTail() @@ -139,18 +160,66 @@ private synchronized void fillCollection(ArrayList collection, int size, Mutable externalizable = createCollectionObject(); externalizable.read(in); collection.add(externalizable); - } + } } protected abstract Mutable createCollectionObject(); + protected int getExternalIndex(int index) + { + if (this.numberOfDeletedEntries == 0) + { + return index; + } + int[] internal = this.deletedIndecesList.getInternalArray(); + int internalSize = this.deletedIndecesList.size(); + for (int i = 0; i < internalSize; i++) + { + int deletedIndex = internal[i]; + if (deletedIndex <= index) + { + index--; + } + else + { + break; + } + } + return index; + } + + protected int getInternalIndex(int index) + { + if (this.numberOfDeletedEntries == 0) + { + return index; + } + int[] internal = this.deletedIndecesList.getInternalArray(); + int internalSize = this.deletedIndecesList.size(); + for (int i = 0; i < internalSize; i++) + { + int deletedIndex = internal[i]; + if (deletedIndex <= index) + { + index++; + } + else + { + break; + } + } + return index; + } + + public synchronized Object get(int index) { if (!this.tailCollectionIsLoaded) { loadTailCollection(); } + index = getInternalIndex(index); if (index < 0 || index >= this.completeSize) { throw new ArrayIndexOutOfBoundsException("for index " + index + ", completeSize=" + this.completeSize); @@ -185,14 +254,14 @@ public synchronized int indexOf(Mutable element) int index = getTailCollection().indexOf(element); if (index != -1) { - return this.tailCollectionStartIndex + index; + return getExternalIndex(this.tailCollectionStartIndex + index); } if (this.currentCollection != null) { index = this.currentCollection.indexOf(element); if (index != -1) { - return index + (this.chunkSize * this.currentCollectionIndex); + return getExternalIndex(index + (this.chunkSize * this.currentCollectionIndex)); } } return -1; @@ -304,15 +373,62 @@ private synchronized void saveChunk() public synchronized Object remove(int index) { - if (!isRemovable(index)) +// if (!isRemovable(index)) +// { +// throw new IllegalArgumentException("cannot remove already chunked element " + index); +// } + if (index < 0) + { + throw new IndexOutOfBoundsException("for " + index); + } + if (!this.tailCollectionIsLoaded) + { + loadTailCollection(); + } + int internalIndex = getInternalIndex(index); + if (internalIndex >= this.completeSize) + { + throw new IndexOutOfBoundsException("for " + index); + } + if (internalIndex >= this.tailCollectionStartIndex) + { + internalIndex -= this.tailCollectionStartIndex; + Object removed = getTailCollection().remove(internalIndex); + this.tailCollectionIsDirty = true; + this.completeSize--; + return removed; + } + else { - throw new IllegalArgumentException("cannot remove already chunked element " + index); + Object removed = set(index, null); + if (this.deletedIndecesList == null) + { + this.deletedIndecesList = new IntList(); + this.deletedIndecesList.add(internalIndex); + } + else + { + int[] internal = this.deletedIndecesList.getInternalArray(); + int internalSize = this.deletedIndecesList.size(); + boolean isInserted = false; + for (int i=0; i< internalSize; i++) + { + int deletedIndex = internal[i]; + if (deletedIndex > internalIndex) + { + this.deletedIndecesList.add(i, internalIndex); + isInserted = true; + break; + } + } + if (!isInserted) + { + this.deletedIndecesList.add(internalIndex); + } + } + this.numberOfDeletedEntries++; + return removed; } - index -= this.tailCollectionStartIndex; - Object removed = getTailCollection().remove(index); - this.tailCollectionIsDirty = true; - this.completeSize--; - return removed; } public synchronized boolean isRemovable(int index) @@ -321,7 +437,7 @@ public synchronized boolean isRemovable(int index) { loadTailCollection(); } - return (index >= this.tailCollectionStartIndex); + return true; // (index >= this.tailCollectionStartIndex); } @@ -331,13 +447,17 @@ public synchronized Object set(int index, Mutable element) { loadTailCollection(); } - if (index >= this.tailCollectionStartIndex) + int internalIndex = getInternalIndex(index); + if (internalIndex >= this.tailCollectionStartIndex) { - index -= this.tailCollectionStartIndex; + internalIndex -= this.tailCollectionStartIndex; this.tailCollectionIsDirty = true; - return getTailCollection().set(index, element); + return getTailCollection().set(internalIndex, element); } - throw new IllegalArgumentException("cannot set/replace already chunked element " + index); + Object previous = get(index); + int indexWithinChunk = internalIndex % this.chunkSize; + this.currentCollection.set(indexWithinChunk, element); + return previous; } private synchronized void loadTailCollection() { @@ -389,6 +509,21 @@ private synchronized void loadChunk(int chunkIndex) this.currentCollectionIndex = chunkIndex; fillCollection(this.currentCollection, this.chunkSize, in); in.close(); + // remove deleted elements to save memory: + if (this.numberOfDeletedEntries > 0) + { + int chunkStartIndex = chunkIndex * this.chunkSize; + int[] internal = this.deletedIndecesList.getInternalArray(); + int internalSize = this.deletedIndecesList.size(); + for (int i=0; i= chunkStartIndex && index < chunkStartIndex + this.chunkSize) + { + this.currentCollection.set(index - chunkStartIndex, null); + } + } + } } private synchronized void saveCurrentCollection() { @@ -516,7 +651,7 @@ private synchronized boolean containsDirtyElement(ArrayList collection) for (int i=0; i < size; i++) { Mutable mutable = (Mutable) objects[i]; - if (mutable.isDirty()) + if ((mutable != null) && mutable.isDirty()) { return true; } diff --git a/enough-polish-j2me/source/test/de/enough/polish/io/ChunkedStorageCollectionTest.java b/enough-polish-j2me/source/test/de/enough/polish/io/ChunkedStorageCollectionTest.java index 000b032..73d89d5 100644 --- a/enough-polish-j2me/source/test/de/enough/polish/io/ChunkedStorageCollectionTest.java +++ b/enough-polish-j2me/source/test/de/enough/polish/io/ChunkedStorageCollectionTest.java @@ -55,6 +55,51 @@ public void testChunking() assertEquals( (testSize + toAdd - 1) % 20 + 20, collection.sizeTail() ); } + public void testRemove() + { + TestChunkedStorageCollection collection = new TestChunkedStorageCollection(); + int testSize = 1003; + for (int i=0; i " + collection.get(9)); + for (int i=0; i<30; i++) + { + System.out.print( collection.getInternalIndex(i) + ": " + collection.get(i) + ", "); + } + System.out.println(); + int[] removeIndeces = new int[]{ 10, 0, 0, 999, 0, 26, 1, 767, 400, 0, 14, 7 }; + for (int i = 0; i < removeIndeces.length; i++) + { + int removeIndex = removeIndeces[i]; + System.out.print("removing " + removeIndex + ", internal=" + collection.getInternalIndex(removeIndex) + " -> " ); + Object previous = collection.remove(removeIndex); + System.out.println(previous); + System.out.println("now: internal of 2: " + collection.getInternalIndex(2) + " -> " + collection.get(2)); + System.out.println("now: internal of 3: " + collection.getInternalIndex(3) + " -> " + collection.get(3)); + System.out.println("now: internal of 4: " + collection.getInternalIndex(4) + " -> " + collection.get(4)); + assertNotNull(previous); + testSize--; + assertEquals( testSize, collection.size() ); + } + System.out.println("------ after removal ------"); + for (int i=0; i<30; i++) + { + System.out.print( i + "(" + collection.getInternalIndex(i) + "): " + collection.get(i) + ", "); + } + System.out.println(); + for (int i=0; i" + collection.getInternalIndex(i)); + } + } + } + static class TestData implements Mutable { private String data; @@ -76,7 +121,7 @@ public void read(DataInputStream in) throws IOException { public void setData(String data) { - this.data = data; + this.data = data; this.isDirty = true; } @@ -84,6 +129,11 @@ public boolean isDirty() { return this.isDirty; } + public String toString() + { + return this.data; + } + } static class TestChunkedStorageCollection extends ChunkedStorageCollection @@ -100,7 +150,12 @@ public Mutable createCollectionObject() { static class ChunkedStorageMemorySystem implements ChunkedStorageSystem { - private static Hashtable listsPerIdentifier = new Hashtable(); + private static Hashtable listsPerIdentifier; + + public ChunkedStorageMemorySystem() + { + listsPerIdentifier = new Hashtable(); + } public byte[] loadTailData(String identifier) throws IOException { return loadData(-1, identifier);