Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

add paging of the alldocs query

  • Loading branch information...
commit c9056c175b2fda064c5f7453a8b1dd0769b7e9cc 1 parent d803309
@tjake authored
Showing with 38 additions and 24 deletions.
  1. +38 −24 src/lucandra/LucandraAllTermDocs.java
View
62 src/lucandra/LucandraAllTermDocs.java
@@ -138,37 +138,51 @@ private void fillDocBuffer() throws IOException
ByteBuffer key = CassandraUtils.hashKeyBytes(indexName.getBytes("UTF-8"), CassandraUtils.delimeterBytes, "ids".getBytes("UTF-8"));
- ReadCommand cmd = new SliceFromReadCommand(CassandraUtils.keySpace, key,
- new ColumnParent(CassandraUtils.schemaInfoColumnFamily), ByteBufferUtil.EMPTY_BYTE_BUFFER,
- ByteBufferUtil.EMPTY_BYTE_BUFFER, false, Integer.MAX_VALUE);
+ ByteBuffer startCol = ByteBufferUtil.EMPTY_BYTE_BUFFER;
+ int pageSize = 4096;
+
+ while(true)
+ {
+
+ ReadCommand cmd = new SliceFromReadCommand(CassandraUtils.keySpace, key,
+ new ColumnParent(CassandraUtils.schemaInfoColumnFamily), startCol,
+ ByteBufferUtil.EMPTY_BYTE_BUFFER, false, pageSize);
- List<Row> rows = CassandraUtils.robustRead(CassandraUtils.consistency, cmd);
+ List<Row> rows = CassandraUtils.robustRead(CassandraUtils.consistency, cmd);
- if(rows.isEmpty())
- return;
+ if(rows.isEmpty())
+ return;
- Row row = rows.get(0);
+ Row row = rows.get(0);
- if(row == null || row.cf == null)
- return;
+ if(row == null || row.cf == null)
+ return;
- for(IColumn sc : row.cf.getSortedColumns()){
+ int actualPageSize = 0;
+ for(IColumn sc : row.cf.getSortedColumns()){
- Integer id = Integer.valueOf(ByteBufferUtil.string(sc.name()));
-
- if(!sc.isLive())
- continue;
+ actualPageSize++;
+
+ if(sc.name().equals(startCol))
+ continue;
- for(IColumn c : sc.getSubColumns())
- {
- //valid id
- if( !(c instanceof ExpiringColumn)){
- docBuffer.set(id);
- fillSize++;
- }
- }
- }
+ startCol = sc.name();
+
+ if(!sc.isLive())
+ continue;
+
+ Integer id = Integer.valueOf(ByteBufferUtil.string(sc.name()));
+
+ docBuffer.set(id);
+ fillSize++;
+ }
+
+ if(actualPageSize < pageSize)
+ break;
+
+ if(logger.isDebugEnabled())
+ logger.debug("Read "+fillSize+ " "+ ByteBufferUtil.string(startCol));
+ }
}
-
}
Please sign in to comment.
Something went wrong with that request. Please try again.