Skip to content

Commit ac76316

Browse files
Diksha628Diksha628
authored
HIVE-27058: Backport of HIVE-24316: ORC upgrade to 1.5.8 and HIVE-24391: TestORCFile fix (#4192)
Signed-off-by: Sankar Hariappan <sankarh@apache.org> Closes (#4192)
1 parent 7b2b35a commit ac76316

File tree

3 files changed

+36
-87
lines changed

3 files changed

+36
-87
lines changed

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,7 @@
184184
<log4j2.version>2.17.1</log4j2.version>
185185
<postgres.version>42.5.1</postgres.version>
186186
<opencsv.version>2.3</opencsv.version>
187-
<orc.version>1.5.6</orc.version>
187+
<orc.version>1.5.8</orc.version>
188188
<mockito-all.version>1.10.19</mockito-all.version>
189189
<powermock.version>1.7.4</powermock.version>
190190
<mina.version>2.0.0-M5</mina.version>

‎ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java‎

Lines changed: 28 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1928,68 +1928,37 @@ private BigRow createRandomRow(long[] intValues, double[] doubleValues,
19281928
new MiddleStruct(inner, inner2), list(), map(inner,inner2));
19291929
}
19301930

1931-
private static class MyMemoryManager implements MemoryManager {
1932-
final long totalSpace;
1933-
double rate;
1934-
Path path = null;
1935-
long lastAllocation = 0;
1936-
int rows = 0;
1937-
MemoryManager.Callback callback;
1938-
1939-
MyMemoryManager(Configuration conf, long totalSpace, double rate) {
1940-
this.totalSpace = totalSpace;
1941-
this.rate = rate;
1942-
}
1943-
1944-
@Override
1945-
public void addWriter(Path path, long requestedAllocation,
1946-
MemoryManager.Callback callback) {
1947-
this.path = path;
1948-
this.lastAllocation = requestedAllocation;
1949-
this.callback = callback;
1950-
}
1951-
1952-
@Override
1953-
public synchronized void removeWriter(Path path) {
1954-
this.path = null;
1955-
this.lastAllocation = 0;
1956-
}
1957-
1958-
@Override
1959-
public void addedRow(int count) throws IOException {
1960-
rows += count;
1961-
if (rows >= 100) {
1962-
callback.checkMemory(rate);
1963-
rows = 0;
1964-
}
1965-
}
1966-
}
1967-
19681931
@Test
19691932
public void testMemoryManagementV11() throws Exception {
1933+
OrcConf.ROWS_BETWEEN_CHECKS.setLong(conf, 100);
1934+
final long poolSize = 50_000;
19701935
ObjectInspector inspector;
19711936
synchronized (TestOrcFile.class) {
19721937
inspector = ObjectInspectorFactory.getReflectionObjectInspector
19731938
(InnerStruct.class,
19741939
ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
19751940
}
1976-
MyMemoryManager memory = new MyMemoryManager(conf, 10000, 0.1);
1941+
MemoryManager memoryManager = new MemoryManagerImpl(poolSize);
1942+
// set up 10 files that all request the full size.
1943+
MemoryManager.Callback ignore = newScale -> false;
1944+
for(int f=0; f < 9; ++f) {
1945+
memoryManager.addWriter(new Path("file-" + f), poolSize, ignore);
1946+
}
19771947
Writer writer = OrcFile.createWriter(testFilePath,
19781948
OrcFile.writerOptions(conf)
19791949
.inspector(inspector)
19801950
.compress(CompressionKind.NONE)
19811951
.stripeSize(50000)
19821952
.bufferSize(100)
19831953
.rowIndexStride(0)
1984-
.memory(memory)
1954+
.memory(memoryManager)
19851955
.batchSize(100)
19861956
.version(OrcFile.Version.V_0_11));
1987-
assertEquals(testFilePath, memory.path);
1957+
assertEquals(0.1, ((MemoryManagerImpl) memoryManager).getAllocationScale());
19881958
for(int i=0; i < 2500; ++i) {
19891959
writer.addRow(new InnerStruct(i*300, Integer.toHexString(10*i)));
19901960
}
19911961
writer.close();
1992-
assertEquals(null, memory.path);
19931962
Reader reader = OrcFile.createReader(testFilePath,
19941963
OrcFile.readerOptions(conf).filesystem(fs));
19951964
int i = 0;
@@ -2004,29 +1973,35 @@ public void testMemoryManagementV11() throws Exception {
20041973

20051974
@Test
20061975
public void testMemoryManagementV12() throws Exception {
1976+
OrcConf.ROWS_BETWEEN_CHECKS.setLong(conf, 100);
1977+
final long poolSize = 50_000;
20071978
ObjectInspector inspector;
20081979
synchronized (TestOrcFile.class) {
20091980
inspector = ObjectInspectorFactory.getReflectionObjectInspector
20101981
(InnerStruct.class,
20111982
ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
20121983
}
2013-
MyMemoryManager memory = new MyMemoryManager(conf, 10000, 0.1);
1984+
MemoryManager memoryManager = new MemoryManagerImpl(poolSize);
1985+
// set up 10 files that all request the full size.
1986+
MemoryManager.Callback ignore = newScale -> false;
1987+
for(int f=0; f < 9; ++f) {
1988+
memoryManager.addWriter(new Path("file-" + f), poolSize, ignore);
1989+
}
20141990
Writer writer = OrcFile.createWriter(testFilePath,
2015-
OrcFile.writerOptions(conf)
2016-
.inspector(inspector)
2017-
.compress(CompressionKind.NONE)
2018-
.stripeSize(50000)
2019-
.bufferSize(100)
2020-
.rowIndexStride(0)
2021-
.memory(memory)
2022-
.batchSize(100)
2023-
.version(OrcFile.Version.V_0_12));
2024-
assertEquals(testFilePath, memory.path);
1991+
OrcFile.writerOptions(conf)
1992+
.inspector(inspector)
1993+
.compress(CompressionKind.NONE)
1994+
.stripeSize(50000)
1995+
.bufferSize(100)
1996+
.rowIndexStride(0)
1997+
.memory(memoryManager)
1998+
.batchSize(100)
1999+
.version(OrcFile.Version.V_0_12));
2000+
assertEquals(0.1, ((MemoryManagerImpl) memoryManager).getAllocationScale());
20252001
for(int i=0; i < 2500; ++i) {
20262002
writer.addRow(new InnerStruct(i*300, Integer.toHexString(10*i)));
20272003
}
20282004
writer.close();
2029-
assertEquals(null, memory.path);
20302005
Reader reader = OrcFile.createReader(testFilePath,
20312006
OrcFile.readerOptions(conf).filesystem(fs));
20322007
int i = 0;

‎ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java‎

Lines changed: 7 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,8 @@
2323
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
2424
import org.apache.hadoop.hive.ql.io.BucketCodec;
2525
import org.apache.orc.CompressionKind;
26-
import org.apache.orc.MemoryManager;
26+
import org.apache.orc.OrcConf;
2727
import org.apache.orc.StripeInformation;
28-
import org.apache.orc.impl.MemoryManagerImpl;
2928
import org.apache.orc.impl.OrcAcidUtils;
3029
import org.junit.Assert;
3130
import org.junit.Rule;
@@ -1064,6 +1063,7 @@ static String getColumnTypesProperty() {
10641063
public void testRecordReaderOldBaseAndDelta() throws Exception {
10651064
final int BUCKET = 10;
10661065
Configuration conf = new Configuration();
1066+
OrcConf.ROWS_BETWEEN_CHECKS.setLong(conf, 2);
10671067
OrcOutputFormat of = new OrcOutputFormat();
10681068
FileSystem fs = FileSystem.getLocal(conf);
10691069
Path root = new Path(tmpDir, "testOldBaseAndDelta").makeQualified(fs);
@@ -1073,25 +1073,11 @@ public void testRecordReaderOldBaseAndDelta() throws Exception {
10731073
inspector = ObjectInspectorFactory.getReflectionObjectInspector
10741074
(BigRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
10751075
}
1076-
1077-
// write the base
1078-
MemoryManager mgr = new MemoryManagerImpl(conf){
1079-
int rowsAddedSinceCheck = 0;
1080-
1081-
@Override
1082-
public synchronized void addedRow(int rows) throws IOException {
1083-
rowsAddedSinceCheck += rows;
1084-
if (rowsAddedSinceCheck >= 2) {
1085-
notifyWriters();
1086-
rowsAddedSinceCheck = 0;
1087-
}
1088-
}
1089-
};
10901076
// make 5 stripes with 2 rows each
10911077
Writer writer = OrcFile.createWriter(new Path(root, "0000010_0"),
10921078
OrcFile.writerOptions(conf).inspector(inspector).fileSystem(fs)
10931079
.blockPadding(false).bufferSize(10000).compress(CompressionKind.NONE)
1094-
.stripeSize(1).memory(mgr).batchSize(2).version(OrcFile.Version.V_0_11));
1080+
.stripeSize(1).batchSize(2).version(OrcFile.Version.V_0_11));
10951081
String[] values= new String[]{"ignore.1", "0.1", "ignore.2", "ignore.3",
10961082
"2.0", "2.1", "3.0", "ignore.4", "ignore.5", "ignore.6"};
10971083
for(int i=0; i < values.length; ++i) {
@@ -1195,6 +1181,7 @@ public synchronized void addedRow(int rows) throws IOException {
11951181
public void testRecordReaderNewBaseAndDelta() throws Exception {
11961182
final int BUCKET = 11;
11971183
Configuration conf = new Configuration();
1184+
OrcConf.ROWS_BETWEEN_CHECKS.setLong(conf, 2);
11981185
OrcOutputFormat of = new OrcOutputFormat();
11991186
FileSystem fs = FileSystem.getLocal(conf);
12001187
Path root = new Path(tmpDir, "testRecordReaderNewBaseAndDelta").makeQualified(fs);
@@ -1205,20 +1192,6 @@ public void testRecordReaderNewBaseAndDelta() throws Exception {
12051192
(BigRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
12061193
}
12071194

1208-
// write the base
1209-
MemoryManager mgr = new MemoryManagerImpl(conf){
1210-
int rowsAddedSinceCheck = 0;
1211-
1212-
@Override
1213-
public synchronized void addedRow(int rows) throws IOException {
1214-
rowsAddedSinceCheck += rows;
1215-
if (rowsAddedSinceCheck >= 2) {
1216-
notifyWriters();
1217-
rowsAddedSinceCheck = 0;
1218-
}
1219-
}
1220-
};
1221-
12221195
// make 5 stripes with 2 rows each
12231196
OrcRecordUpdater.OrcOptions options = (OrcRecordUpdater.OrcOptions)
12241197
new OrcRecordUpdater.OrcOptions(conf)
@@ -1228,8 +1201,9 @@ public synchronized void addedRow(int rows) throws IOException {
12281201
final int BUCKET_PROPERTY = BucketCodec.V1.encode(options);
12291202

12301203
options.orcOptions(OrcFile.writerOptions(conf)
1231-
.stripeSize(1).blockPadding(false).compress(CompressionKind.NONE)
1232-
.memory(mgr).batchSize(2));
1204+
.stripeSize(1).blockPadding(false)
1205+
.compress(CompressionKind.NONE)
1206+
.batchSize(2));
12331207
options.finalDestination(root);
12341208
RecordUpdater ru = of.getRecordUpdater(root, options);
12351209
String[] values= new String[]{"ignore.1", "0.1", "ignore.2", "ignore.3",

0 commit comments

Comments
 (0)