From bb53585f82e09fa326826e79a673cc8e75ffa569 Mon Sep 17 00:00:00 2001 From: cartershanklin Date: Wed, 11 Dec 2013 18:38:18 -0800 Subject: [PATCH] Splitting flat and partition load settings. --- settings/load-flat.sql | 13 +++++++++++++ settings/load-partitioned.sql | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 settings/load-flat.sql create mode 100644 settings/load-partitioned.sql diff --git a/settings/load-flat.sql b/settings/load-flat.sql new file mode 100644 index 00000000..f53f40fb --- /dev/null +++ b/settings/load-flat.sql @@ -0,0 +1,13 @@ +set hive.enforce.bucketing=true; +set hive.enforce.sorting=true; +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.exec.max.dynamic.partitions.pernode=1000000; +set hive.exec.max.dynamic.partitions=1000000; +set hive.exec.max.created.files=1000000; + +set mapred.min.split.size=240000000; +set mapred.max.split.size=240000000; +set mapred.min.split.size.per.node=240000000; +set mapred.min.split.size.per.rack=240000000; +set hive.exec.parallel=true; +set hive.stats.autogather=false; diff --git a/settings/load-partitioned.sql b/settings/load-partitioned.sql new file mode 100644 index 00000000..60462d06 --- /dev/null +++ b/settings/load-partitioned.sql @@ -0,0 +1,19 @@ +set hive.enforce.bucketing=true; +set hive.enforce.sorting=true; +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.exec.max.dynamic.partitions.pernode=1000000; +set hive.exec.max.dynamic.partitions=1000000; +set hive.exec.max.created.files=1000000; + +set mapred.min.split.size=240000000; +set mapred.max.split.size=240000000; +set mapred.min.split.size.per.node=240000000; +set mapred.min.split.size.per.rack=240000000; +set hive.exec.parallel=true; +set hive.stats.autogather=false; + +set mapred.map.child.java.opts=-server -Xmx2248m -Djava.net.preferIPv4Stack=true; +set mapred.reduce.child.java.opts=-server -Xms1024m -Xmx9000m -Djava.net.preferIPv4Stack=true; +set mapreduce.map.memory.mb=3072; +set mapreduce.reduce.memory.mb=12288; +set io.sort.mb=800;