Skip to content

[Issue #265]: fix reading row group number. #266

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bianhq merged 1 commit into from
May 16, 2022
Merged

[Issue #265]: fix reading row group number. #266

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 16 additions & 20 deletions pixels-core/src/main/java/io/pixelsdb/pixels/core/reader/PixelsRecordReaderImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,10 +84,10 @@ public class PixelsRecordReaderImpl implements PixelsRecordReader
*/
private int[] targetRGs;
/**
* Target columns to read after matching reader option,
* each element represents a column id (column's index in the schema).
* Different from resultColumns, the ith column id in targetColumn
* corresponds to the ith true value in includedColumns.
* The target columns to read after matching reader option.
* Each element represents a column id (column's index in the schema).
* Different from resultColumns, the ith column id in targetColumns
* corresponds to the ith true value in this.includedColumns.
*/
private int[] targetColumns;
/**
Expand Down Expand Up @@ -154,24 +154,21 @@ private void checkBeforeRead() throws IOException
if (fileColTypes == null || fileColTypes.isEmpty())
{
checkValid = false;
//throw new IOException("ISSUE-103: type list is empty.");
return;
throw new IOException("type list is empty.");
}
fileSchema = TypeDescription.createSchema(fileColTypes);
if (fileSchema.getChildren() == null || fileSchema.getChildren().isEmpty())
{
checkValid = false;
//throw new IOException("ISSUE-103: file schema is empty.");
return;
throw new IOException("file schema is empty.");
}

// check RGStart and RGLen are within the range of actual number of row groups
int rgNum = footer.getRowGroupInfosCount();
if (RGStart >= rgNum)
{
checkValid = false;
//throw new IOException("ISSUE-103: row group start is out of bound.");
return;
throw new IOException("row group start (" + RGStart + ") is out of bound (" + rgNum + ").");
}
if (RGLen == -1)
{
Expand Down Expand Up @@ -217,9 +214,8 @@ private void checkBeforeRead() throws IOException
if (includedColumnNum != optionIncludedCols.length && !option.isTolerantSchemaEvolution())
{
checkValid = false;
//throw new IOException("ISSUE-103: includedColumnsNum is " + includedColumnsNum +
// " while optionIncludedCols.length is " + optionIncludedCols.length);
return;
throw new IOException("includedColumnsNum is " + includedColumnNum +
" whereas optionIncludedCols.length is " + optionIncludedCols.length);
}

// create result columns storing result column ids in user specified order
Expand Down Expand Up @@ -306,8 +302,8 @@ private boolean prepareRead() throws IOException
for (int i = 0; i < RGLen; i++)
{
includedRGs[i] = true;
includedRowNum += footer.getRowGroupInfos(RGStart + i).getNumberOfRows();
}
includedRowNum = postScript.getNumberOfRows();
}
else if (predicate.matchesNone())
{
Expand Down Expand Up @@ -362,7 +358,7 @@ else if (predicate.matchesNone())
{
// Issue #103: columnStatsMap should be cleared for each row group.
columnStatsMap.clear();
PixelsProto.RowGroupStatistic rowGroupStatistic = rowGroupStatistics.get(i + RGStart);
PixelsProto.RowGroupStatistic rowGroupStatistic = rowGroupStatistics.get(RGStart + i);
List<PixelsProto.ColumnStatistic> rgColumnStatistics =
rowGroupStatistic.getColumnChunkStatsList();
for (int id : targetColumns)
Expand All @@ -373,7 +369,7 @@ else if (predicate.matchesNone())
includedRGs[i] = predicate.matches(footer.getRowGroupInfos(i).getNumberOfRows(), columnStatsMap);
if (includedRGs[i] == true)
{
includedRowNum += footer.getRowGroupInfos(i).getNumberOfRows();
includedRowNum += footer.getRowGroupInfos(RGStart + i).getNumberOfRows();
}
}
}
Expand All @@ -384,8 +380,8 @@ else if (predicate.matchesNone())
for (int i = 0; i < RGLen; i++)
{
includedRGs[i] = true;
includedRowNum += footer.getRowGroupInfos(RGStart + i).getNumberOfRows();
}
includedRowNum = postScript.getNumberOfRows();
}

/**
Expand Down Expand Up @@ -790,7 +786,7 @@ public int prepareBatch(int batchSize) throws IOException

if (!everPrepared)
{
if (prepareRead() == false)
if (!prepareRead())
{
throw new IOException("Failed to prepare for read.");
}
Expand All @@ -803,7 +799,7 @@ public int prepareBatch(int batchSize) throws IOException
*/
if (includedColumnNum == 0)
{
if (endOfFile == false)
if (!endOfFile)
{
throw new IOException("EOF should be set in case of none projection columns");
}
Expand Down Expand Up @@ -880,7 +876,7 @@ public VectorizedRowBatch readBatch(int batchSize, boolean reuse)
* It should be EOF. And the batch size should have been set in prepareRead() and
* checked in read().
*/
if (endOfFile == false)
if (!endOfFile)
{
throw new IOException("EOF should be set in case of none projection columns");
}
Expand Down