Skip to content

Commit

Permalink
now being able to remove elements from a ChunkedStorageCollection tha…
Browse files Browse the repository at this point in the history 
…t reside in the already persisted chunked area
  • Loading branch information
Robert Virkus committed May 5, 2014
1 parent 1a15a84 commit b312e36
Show file tree
Hide file tree
Showing 2 changed files with 210 additions and 20 deletions.
171 changes: 153 additions & 18 deletions enough-polish-j2me/source/src/de/enough/polish/io/ChunkedStorageCollection.java
View file
Expand Up @@ -32,6 +32,7 @@
import java.io.IOException;

import de.enough.polish.util.ArrayList;
import de.enough.polish.util.IntList;

/**
* Allows to store lots of data that is retrieved lazily
Expand All @@ -45,12 +46,14 @@ public abstract class ChunkedStorageCollection
public final static int STORAGE_STRATEGY_CHUNKED = 2;
//public final static int STORAGE_STRATEGY_MANUAL = 3;

private final static int PERSISTENCE_VERSION = 1;
private final static int PERSISTENCE_VERSION = 2;

private int chunkSize;
private String identifier;

private int completeSize;
private int numberOfDeletedEntries;
private IntList deletedIndecesList;
private int tailCollectionStartIndex;
private ArrayList tailCollection;
private boolean tailCollectionIsDirty;
Expand Down Expand Up @@ -79,6 +82,13 @@ public synchronized void write(DataOutputStream out) throws IOException {
out.writeInt(this.chunkSize);
out.writeInt(this.completeSize);
out.writeInt(this.tailCollectionStartIndex);
// persistence version 2+:
out.writeInt(this.numberOfDeletedEntries);
if (this.numberOfDeletedEntries > 0)
{
this.deletedIndecesList.write(out);
}
// persistence version 1+:
int tailSize = getTailCollection().size();
out.writeInt(tailSize);
Object[] internalObjects = getTailCollection().getInternalArray();
Expand All @@ -101,6 +111,17 @@ public synchronized void read(DataInputStream in) throws IOException {
this.chunkSize = in.readInt();
this.completeSize = in.readInt();
this.tailCollectionStartIndex = in.readInt();
// persistence version 2+:
if (version >= 2)
{
this.numberOfDeletedEntries = in.readInt();
if (this.numberOfDeletedEntries > 0)
{
this.deletedIndecesList = new IntList();
this.deletedIndecesList.read(in);
}
}
// persistence version 1+:
int size = in.readInt();
if ( this.tailCollection == null ) {
this.tailCollection = new ArrayList(chunkSize*2);
Expand All @@ -119,7 +140,7 @@ public synchronized int size()
{
loadTailCollection();
}
return this.completeSize;
return this.completeSize - this.numberOfDeletedEntries;
}

public synchronized int sizeTail()
Expand All @@ -139,18 +160,66 @@ private synchronized void fillCollection(ArrayList collection, int size,
Mutable externalizable = createCollectionObject();
externalizable.read(in);
collection.add(externalizable);
}
}
}


protected abstract Mutable createCollectionObject();

protected int getExternalIndex(int index)
{
if (this.numberOfDeletedEntries == 0)
{
return index;
}
int[] internal = this.deletedIndecesList.getInternalArray();
int internalSize = this.deletedIndecesList.size();
for (int i = 0; i < internalSize; i++)
{
int deletedIndex = internal[i];
if (deletedIndex <= index)
{
index--;
}
else
{
break;
}
}
return index;
}

protected int getInternalIndex(int index)
{
if (this.numberOfDeletedEntries == 0)
{
return index;
}
int[] internal = this.deletedIndecesList.getInternalArray();
int internalSize = this.deletedIndecesList.size();
for (int i = 0; i < internalSize; i++)
{
int deletedIndex = internal[i];
if (deletedIndex <= index)
{
index++;
}
else
{
break;
}
}
return index;
}


public synchronized Object get(int index)
{
if (!this.tailCollectionIsLoaded)
{
loadTailCollection();
}
index = getInternalIndex(index);
if (index < 0 || index >= this.completeSize)
{
throw new ArrayIndexOutOfBoundsException("for index " + index + ", completeSize=" + this.completeSize);
Expand Down Expand Up @@ -185,14 +254,14 @@ public synchronized int indexOf(Mutable element)
int index = getTailCollection().indexOf(element);
if (index != -1)
{
return this.tailCollectionStartIndex + index;
return getExternalIndex(this.tailCollectionStartIndex + index);
}
if (this.currentCollection != null)
{
index = this.currentCollection.indexOf(element);
if (index != -1)
{
return index + (this.chunkSize * this.currentCollectionIndex);
return getExternalIndex(index + (this.chunkSize * this.currentCollectionIndex));
}
}
return -1;
Expand Down Expand Up @@ -304,15 +373,62 @@ private synchronized void saveChunk()

public synchronized Object remove(int index)
{
if (!isRemovable(index))
// if (!isRemovable(index))
// {
// throw new IllegalArgumentException("cannot remove already chunked element " + index);
// }
if (index < 0)
{
throw new IndexOutOfBoundsException("for " + index);
}
if (!this.tailCollectionIsLoaded)
{
loadTailCollection();
}
int internalIndex = getInternalIndex(index);
if (internalIndex >= this.completeSize)
{
throw new IndexOutOfBoundsException("for " + index);
}
if (internalIndex >= this.tailCollectionStartIndex)
{
internalIndex -= this.tailCollectionStartIndex;
Object removed = getTailCollection().remove(internalIndex);
this.tailCollectionIsDirty = true;
this.completeSize--;
return removed;
}
else
{
throw new IllegalArgumentException("cannot remove already chunked element " + index);
Object removed = set(index, null);
if (this.deletedIndecesList == null)
{
this.deletedIndecesList = new IntList();
this.deletedIndecesList.add(internalIndex);
}
else
{
int[] internal = this.deletedIndecesList.getInternalArray();
int internalSize = this.deletedIndecesList.size();
boolean isInserted = false;
for (int i=0; i< internalSize; i++)
{
int deletedIndex = internal[i];
if (deletedIndex > internalIndex)
{
this.deletedIndecesList.add(i, internalIndex);
isInserted = true;
break;
}
}
if (!isInserted)
{
this.deletedIndecesList.add(internalIndex);
}
}
this.numberOfDeletedEntries++;
return removed;
}
index -= this.tailCollectionStartIndex;
Object removed = getTailCollection().remove(index);
this.tailCollectionIsDirty = true;
this.completeSize--;
return removed;
}

public synchronized boolean isRemovable(int index)
Expand All @@ -321,7 +437,7 @@ public synchronized boolean isRemovable(int index)
{
loadTailCollection();
}
return (index >= this.tailCollectionStartIndex);
return true; // (index >= this.tailCollectionStartIndex);
}


Expand All @@ -331,13 +447,17 @@ public synchronized Object set(int index, Mutable element)
{
loadTailCollection();
}
if (index >= this.tailCollectionStartIndex)
int internalIndex = getInternalIndex(index);
if (internalIndex >= this.tailCollectionStartIndex)
{
index -= this.tailCollectionStartIndex;
internalIndex -= this.tailCollectionStartIndex;
this.tailCollectionIsDirty = true;
return getTailCollection().set(index, element);
return getTailCollection().set(internalIndex, element);
}
throw new IllegalArgumentException("cannot set/replace already chunked element " + index);
Object previous = get(index);
int indexWithinChunk = internalIndex % this.chunkSize;
this.currentCollection.set(indexWithinChunk, element);
return previous;
}

private synchronized void loadTailCollection() {
Expand Down Expand Up @@ -389,6 +509,21 @@ private synchronized void loadChunk(int chunkIndex)
this.currentCollectionIndex = chunkIndex;
fillCollection(this.currentCollection, this.chunkSize, in);
in.close();
// remove deleted elements to save memory:
if (this.numberOfDeletedEntries > 0)
{
int chunkStartIndex = chunkIndex * this.chunkSize;
int[] internal = this.deletedIndecesList.getInternalArray();
int internalSize = this.deletedIndecesList.size();
for (int i=0; i<internalSize; i++)
{
int index = internal[i];
if (index >= chunkStartIndex && index < chunkStartIndex + this.chunkSize)
{
this.currentCollection.set(index - chunkStartIndex, null);
}
}
}
}

private synchronized void saveCurrentCollection() {
Expand Down Expand Up @@ -516,7 +651,7 @@ private synchronized boolean containsDirtyElement(ArrayList collection)
for (int i=0; i < size; i++)
{
Mutable mutable = (Mutable) objects[i];
if (mutable.isDirty())
if ((mutable != null) && mutable.isDirty())
{
return true;
}
Expand Down
59 changes: 57 additions & 2 deletions enough-polish-j2me/source/test/de/enough/polish/io/ChunkedStorageCollectionTest.java
View file
Expand Up @@ -55,6 +55,51 @@ public void testChunking()
assertEquals( (testSize + toAdd - 1) % 20 + 20, collection.sizeTail() );
}

public void testRemove()
{
TestChunkedStorageCollection collection = new TestChunkedStorageCollection();
int testSize = 1003;
for (int i=0; i<testSize; i++)
{
collection.add( new TestData(Integer.toString(i)));
}
assertEquals(testSize, collection.size());
System.out.println("------ before removal ------");
//System.out.println("now: internal of 9: " + collection.getInternalIndex(9) + " -> " + collection.get(9));
for (int i=0; i<30; i++)
{
System.out.print( collection.getInternalIndex(i) + ": " + collection.get(i) + ", ");
}
System.out.println();
int[] removeIndeces = new int[]{ 10, 0, 0, 999, 0, 26, 1, 767, 400, 0, 14, 7 };
for (int i = 0; i < removeIndeces.length; i++)
{
int removeIndex = removeIndeces[i];
System.out.print("removing " + removeIndex + ", internal=" + collection.getInternalIndex(removeIndex) + " -> " );
Object previous = collection.remove(removeIndex);
System.out.println(previous);
System.out.println("now: internal of 2: " + collection.getInternalIndex(2) + " -> " + collection.get(2));
System.out.println("now: internal of 3: " + collection.getInternalIndex(3) + " -> " + collection.get(3));
System.out.println("now: internal of 4: " + collection.getInternalIndex(4) + " -> " + collection.get(4));
assertNotNull(previous);
testSize--;
assertEquals( testSize, collection.size() );
}
System.out.println("------ after removal ------");
for (int i=0; i<30; i++)
{
System.out.print( i + "(" + collection.getInternalIndex(i) + "): " + collection.get(i) + ", ");
}
System.out.println();
for (int i=0; i<collection.size(); i++)
{
if (collection.get(i) == null)
{
fail("got null for index " + i + "->" + collection.getInternalIndex(i));
}
}
}

static class TestData implements Mutable
{
private String data;
Expand All @@ -76,14 +121,19 @@ public void read(DataInputStream in) throws IOException {

public void setData(String data)
{
this.data = data;
this.data = data;
this.isDirty = true;
}

public boolean isDirty() {
return this.isDirty;
}

public String toString()
{
return this.data;
}

}

static class TestChunkedStorageCollection extends ChunkedStorageCollection
Expand All @@ -100,7 +150,12 @@ public Mutable createCollectionObject() {
static class ChunkedStorageMemorySystem implements ChunkedStorageSystem
{

private static Hashtable listsPerIdentifier = new Hashtable();
private static Hashtable listsPerIdentifier;

public ChunkedStorageMemorySystem()
{
listsPerIdentifier = new Hashtable();
}

public byte[] loadTailData(String identifier) throws IOException {
return loadData(-1, identifier);
Expand Down

0 comments on commit b312e36

Please sign in to comment.