Skip to content

Commit

Permalink
Rename rows-per-shard to max-shard-rows
Browse files Browse the repository at this point in the history
  • Loading branch information
@nileema
nileema committed Mar 12, 2015
1 parent 472f218 commit e3369dc
Show file tree
Hide file tree
Showing 4 changed files with 24 additions and 24 deletions.
20 changes: 10 additions & 10 deletions presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java
View file
Expand Up @@ -72,7 +72,7 @@ public class OrcStorageManager
private final DataSize orcMaxMergeDistance; private final DataSize orcMaxMergeDistance;
private final ShardRecoveryManager recoveryManager; private final ShardRecoveryManager recoveryManager;
private final Duration recoveryTimeout; private final Duration recoveryTimeout;
private final long rowsPerShard; private final long maxShardRows;
private final DataSize maxShardSize; private final DataSize maxShardSize;
private final DataSize maxBufferSize; private final DataSize maxBufferSize;


Expand All @@ -88,7 +88,7 @@ public OrcStorageManager(
config.getOrcMaxMergeDistance(), config.getOrcMaxMergeDistance(),
recoveryManager, recoveryManager,
config.getShardRecoveryTimeout(), config.getShardRecoveryTimeout(),
config.getRowsPerShard(), config.getMaxShardRows(),
config.getMaxShardSize(), config.getMaxShardSize(),
config.getMaxBufferSize()); config.getMaxBufferSize());
} }
Expand All @@ -99,7 +99,7 @@ public OrcStorageManager(
DataSize orcMaxMergeDistance, DataSize orcMaxMergeDistance,
ShardRecoveryManager recoveryManager, ShardRecoveryManager recoveryManager,
Duration shardRecoveryTimeout, Duration shardRecoveryTimeout,
long rowsPerShard, long maxShardRows,
DataSize maxShardSize, DataSize maxShardSize,
DataSize maxBufferSize) DataSize maxBufferSize)
{ {
Expand All @@ -109,8 +109,8 @@ public OrcStorageManager(
this.recoveryManager = checkNotNull(recoveryManager, "recoveryManager is null"); this.recoveryManager = checkNotNull(recoveryManager, "recoveryManager is null");
this.recoveryTimeout = checkNotNull(shardRecoveryTimeout, "shardRecoveryTimeout is null"); this.recoveryTimeout = checkNotNull(shardRecoveryTimeout, "shardRecoveryTimeout is null");


checkArgument(rowsPerShard > 0, "rowsPerShard must be > 0"); checkArgument(maxShardRows > 0, "maxShardRows must be > 0");
this.rowsPerShard = rowsPerShard; this.maxShardRows = maxShardRows;
this.maxShardSize = checkNotNull(maxShardSize, "maxShardSize is null"); this.maxShardSize = checkNotNull(maxShardSize, "maxShardSize is null");
this.maxBufferSize = checkNotNull(maxBufferSize, "maxBufferSize is null"); this.maxBufferSize = checkNotNull(maxBufferSize, "maxBufferSize is null");
} }
Expand Down Expand Up @@ -157,7 +157,7 @@ public ConnectorPageSource getPageSource(UUID shardUuid, List<Long> columnIds, L
@Override @Override
public StoragePageSink createStoragePageSink(List<Long> columnIds, List<Type> columnTypes) public StoragePageSink createStoragePageSink(List<Long> columnIds, List<Type> columnTypes)
{ {
return new OrcStoragePageSink(columnIds, columnTypes, rowsPerShard, maxShardSize); return new OrcStoragePageSink(columnIds, columnTypes, maxShardRows, maxShardSize);
} }


private void writeShard(UUID shardUuid) private void writeShard(UUID shardUuid)
Expand Down Expand Up @@ -272,16 +272,16 @@ private class OrcStoragePageSink
private final List<Type> columnTypes; private final List<Type> columnTypes;


private final List<ShardInfo> shards = new ArrayList<>(); private final List<ShardInfo> shards = new ArrayList<>();
private final long rowsPerShard; private final long maxShardRows;
private final DataSize maxShardSize; private final DataSize maxShardSize;


private boolean committed; private boolean committed;
private OrcFileWriter writer; private OrcFileWriter writer;
private UUID shardUuid; private UUID shardUuid;


public OrcStoragePageSink(List<Long> columnIds, List<Type> columnTypes, long rowsPerShard, DataSize maxShardSize) public OrcStoragePageSink(List<Long> columnIds, List<Type> columnTypes, long maxShardRows, DataSize maxShardSize)
{ {
this.rowsPerShard = rowsPerShard; this.maxShardRows = maxShardRows;
this.maxShardSize = maxShardSize; this.maxShardSize = maxShardSize;
this.columnIds = ImmutableList.copyOf(checkNotNull(columnIds, "columnIds is null")); this.columnIds = ImmutableList.copyOf(checkNotNull(columnIds, "columnIds is null"));
this.columnTypes = ImmutableList.copyOf(checkNotNull(columnTypes, "columnTypes is null")); this.columnTypes = ImmutableList.copyOf(checkNotNull(columnTypes, "columnTypes is null"));
Expand All @@ -307,7 +307,7 @@ public boolean isFull()
if (writer == null) { if (writer == null) {
return false; return false;
} }
return (writer.getRowCount() >= rowsPerShard) || (writer.getUncompressedSize() >= maxShardSize.toBytes()); return (writer.getRowCount() >= maxShardRows) || (writer.getUncompressedSize() >= maxShardSize.toBytes());
} }


@Override @Override
Expand Down
12 changes: 6 additions & 6 deletions presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java
View file
Expand Up @@ -39,7 +39,7 @@ public class StorageManagerConfig
private DataSize orcMaxMergeDistance = new DataSize(1, MEGABYTE); private DataSize orcMaxMergeDistance = new DataSize(1, MEGABYTE);
private int recoveryThreads = 10; private int recoveryThreads = 10;


private long rowsPerShard = 1_000_000; private long maxShardRows = 1_000_000;
private DataSize maxShardSize = new DataSize(256, MEGABYTE); private DataSize maxShardSize = new DataSize(256, MEGABYTE);
private DataSize maxBufferSize = new DataSize(256, MEGABYTE); private DataSize maxBufferSize = new DataSize(256, MEGABYTE);


Expand Down Expand Up @@ -126,16 +126,16 @@ public StorageManagerConfig setRecoveryThreads(int recoveryThreads)


@Min(1) @Min(1)
@Max(1_000_000_000) @Max(1_000_000_000)
public long getRowsPerShard() public long getMaxShardRows()
{ {
return rowsPerShard; return maxShardRows;
} }


@Config("storage.rows-per-shard") @Config("storage.max-shard-rows")
@ConfigDescription("Approximate maximum number of rows per shard") @ConfigDescription("Approximate maximum number of rows per shard")
public StorageManagerConfig setRowsPerShard(long rowsPerShard) public StorageManagerConfig setMaxShardRows(long maxShardRows)
{ {
this.rowsPerShard = rowsPerShard; this.maxShardRows = maxShardRows;
return this; return this;
} }


Expand Down
10 changes: 5 additions & 5 deletions presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java
View file
Expand Up @@ -93,7 +93,7 @@ public class TestOrcStorageManager
private static final DataSize ORC_MAX_MERGE_DISTANCE = new DataSize(1, MEGABYTE); private static final DataSize ORC_MAX_MERGE_DISTANCE = new DataSize(1, MEGABYTE);
private static final Duration SHARD_RECOVERY_TIMEOUT = new Duration(30, TimeUnit.SECONDS); private static final Duration SHARD_RECOVERY_TIMEOUT = new Duration(30, TimeUnit.SECONDS);
private static final DataSize MAX_BUFFER_SIZE = new DataSize(256, MEGABYTE); private static final DataSize MAX_BUFFER_SIZE = new DataSize(256, MEGABYTE);
private static final int ROWS_PER_SHARD = 100; private static final int MAX_SHARD_ROWS = 100;
private static final DataSize MAX_FILE_SIZE = new DataSize(1, MEGABYTE); private static final DataSize MAX_FILE_SIZE = new DataSize(1, MEGABYTE);


private final NodeManager nodeManager = new InMemoryNodeManager(); private final NodeManager nodeManager = new InMemoryNodeManager();
Expand Down Expand Up @@ -385,7 +385,7 @@ public void testShardStatsDateTimestamp()
} }


@Test @Test
public void testRowsPerShard() public void testMaxShardRows()
throws Exception throws Exception
{ {
OrcStorageManager manager = createOrcStorageManager(storageService, recoveryManager, 2, new DataSize(2, MEGABYTE)); OrcStorageManager manager = createOrcStorageManager(storageService, recoveryManager, 2, new DataSize(2, MEGABYTE));
Expand Down Expand Up @@ -426,12 +426,12 @@ public void testMaxFileSize()


public static OrcStorageManager createOrcStorageManager(StorageService storageService, ShardRecoveryManager recoveryManager) public static OrcStorageManager createOrcStorageManager(StorageService storageService, ShardRecoveryManager recoveryManager)
{ {
return createOrcStorageManager(storageService, recoveryManager, ROWS_PER_SHARD, MAX_FILE_SIZE); return createOrcStorageManager(storageService, recoveryManager, MAX_SHARD_ROWS, MAX_FILE_SIZE);
} }


public static OrcStorageManager createOrcStorageManager(StorageService storageService, ShardRecoveryManager recoveryManager, int rowsPerShard, DataSize maxFileSize) public static OrcStorageManager createOrcStorageManager(StorageService storageService, ShardRecoveryManager recoveryManager, int maxShardRows, DataSize maxFileSize)
{ {
return new OrcStorageManager(CURRENT_NODE, storageService, ORC_MAX_MERGE_DISTANCE, recoveryManager, SHARD_RECOVERY_TIMEOUT, rowsPerShard, maxFileSize, MAX_BUFFER_SIZE); return new OrcStorageManager(CURRENT_NODE, storageService, ORC_MAX_MERGE_DISTANCE, recoveryManager, SHARD_RECOVERY_TIMEOUT, maxShardRows, maxFileSize, MAX_BUFFER_SIZE);
} }


private static void assertColumnStats(List<ColumnStats> list, long columnId, Object min, Object max) private static void assertColumnStats(List<ColumnStats> list, long columnId, Object min, Object max)
Expand Down
6 changes: 3 additions & 3 deletions presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestStorageManagerConfig.java
View file
Expand Up @@ -44,7 +44,7 @@ public void testDefaults()
.setShardRecoveryTimeout(new Duration(30, SECONDS)) .setShardRecoveryTimeout(new Duration(30, SECONDS))
.setMissingShardDiscoveryInterval(new Duration(5, MINUTES)) .setMissingShardDiscoveryInterval(new Duration(5, MINUTES))
.setRecoveryThreads(10) .setRecoveryThreads(10)
.setRowsPerShard(1_000_000) .setMaxShardRows(1_000_000)
.setMaxShardSize(new DataSize(256, MEGABYTE)) .setMaxShardSize(new DataSize(256, MEGABYTE))
.setMaxBufferSize(new DataSize(256, MEGABYTE))); .setMaxBufferSize(new DataSize(256, MEGABYTE)));


Expand All @@ -60,7 +60,7 @@ public void testExplicitPropertyMappings()
.put("storage.shard-recovery-timeout", "1m") .put("storage.shard-recovery-timeout", "1m")
.put("storage.missing-shard-discovery-interval", "4m") .put("storage.missing-shard-discovery-interval", "4m")
.put("storage.max-recovery-threads", "12") .put("storage.max-recovery-threads", "12")
.put("storage.rows-per-shard", "10000") .put("storage.max-shard-rows", "10000")
.put("storage.max-shard-size", "10MB") .put("storage.max-shard-size", "10MB")
.put("storage.max-buffer-size", "512MB") .put("storage.max-buffer-size", "512MB")
.build(); .build();
Expand All @@ -72,7 +72,7 @@ public void testExplicitPropertyMappings()
.setShardRecoveryTimeout(new Duration(1, MINUTES)) .setShardRecoveryTimeout(new Duration(1, MINUTES))
.setMissingShardDiscoveryInterval(new Duration(4, MINUTES)) .setMissingShardDiscoveryInterval(new Duration(4, MINUTES))
.setRecoveryThreads(12) .setRecoveryThreads(12)
.setRowsPerShard(10_000) .setMaxShardRows(10_000)
.setMaxShardSize(new DataSize(10, MEGABYTE)) .setMaxShardSize(new DataSize(10, MEGABYTE))
.setMaxBufferSize(new DataSize(512, MEGABYTE)); .setMaxBufferSize(new DataSize(512, MEGABYTE));


Expand Down

0 comments on commit e3369dc

Please sign in to comment.