From 032e5fd3d5235d975031b5bda4bd18b06b196c96 Mon Sep 17 00:00:00 2001 From: buom <024025@gmail.com> Date: Fri, 25 Nov 2016 00:12:46 +0700 Subject: [PATCH 1/3] Update FileOutputCommitterContainer.java Wrong the location of table directory. See: https://github.com/apache/hive/blob/master/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java#L120 https://github.com/apache/hive/blob/master/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java#L163 --- .../hive/hcatalog/mapreduce/FileOutputCommitterContainer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java index 847c3bc5204e..618f0904a906 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java @@ -364,7 +364,7 @@ private Partition constructPartition( // Now, we need to de-scratchify this location - i.e., get rid of any // _SCRATCH[\d].?[\d]+ from the location. String jobLocation = jobInfo.getLocation(); - String finalLocn = jobLocation.replaceAll(Path.SEPARATOR + SCRATCH_DIR_NAME + "\\d\\.?\\d+",""); + String finalLocn = jobLocation.replaceAll(Path.SEPARATOR + SCRATCH_DIR_NAME + "\\d\\.?\\d+.*",""); partPath = new Path(finalLocn); } else { partPath = new Path(partLocnRoot); From f6e365dd0a565f0ae1a35194eedbe33ca63b0988 Mon Sep 17 00:00:00 2001 From: buom <024025@gmail.com> Date: Fri, 25 Nov 2016 18:10:55 +0700 Subject: [PATCH 2/3] Wrong the location of table directory --- .../hive/hcatalog/mapreduce/FileOutputCommitterContainer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java index 618f0904a906..9dbe85a771ca 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java @@ -364,7 +364,7 @@ private Partition constructPartition( // Now, we need to de-scratchify this location - i.e., get rid of any // _SCRATCH[\d].?[\d]+ from the location. String jobLocation = jobInfo.getLocation(); - String finalLocn = jobLocation.replaceAll(Path.SEPARATOR + SCRATCH_DIR_NAME + "\\d\\.?\\d+.*",""); + String finalLocn = jobLocation.replaceAll(Path.SEPARATOR + SCRATCH_DIR_NAME + "\\d\\.?[\\dE-]+",""); partPath = new Path(finalLocn); } else { partPath = new Path(partLocnRoot); From 0e8e634d8558a7e122b4093cde97a3640d1c8117 Mon Sep 17 00:00:00 2001 From: buom <024025@gmail.com> Date: Fri, 25 Nov 2016 18:55:57 +0700 Subject: [PATCH 3/3] Can not create a Path from an empty string Caused by: org.apache.hive.hcatalog.common.HCatException : 2006 : Error adding partition to metastore. Cause : java.lang.IllegalArgumentException: Can not create a Path from an empty string at org.apache.hive.hcatalog.mapreduce.FileOutputCommitterContainer.registerPartitions(FileOutputCommitterContainer.java:975) at org.apache.hive.hcatalog.mapreduce.FileOutputCommitterContainer.commitJob(FileOutputCommitterContainer.java:250) ... 20 more --- .../hive/hcatalog/mapreduce/FileOutputCommitterContainer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java index 9dbe85a771ca..62dc0fcbb968 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java @@ -891,7 +891,9 @@ private void registerPartitions(JobContext context) throws IOException{ if (src.toString().matches(".*" + Path.SEPARATOR + SCRATCH_DIR_NAME + "\\d\\.?\\d+.*")){ // src is scratch directory, need to trim the part key value pairs from path String diff = StringUtils.difference(src.toString(), dest.toString()); - fs.delete(new Path(StringUtils.substringBefore(src.toString(), diff)), true); + if (StringUtils.isNotBlank(diff)) { + fs.delete(new Path(StringUtils.substringBefore(src.toString(), diff)), true); + } } else { fs.delete(src, true); }