From 8e403b209ae743f8ec3d78494cce9925dfe4d585 Mon Sep 17 00:00:00 2001 From: Abhinay Nagpal Date: Fri, 28 Sep 2012 11:47:43 -0700 Subject: [PATCH] Added copyright and fixed code formatting --- .../readonly/disk/HadoopStoreWriter.java | 16 ++ .../disk/HadoopStoreWriterPerBucket.java | 16 ++ .../store/readonly/disk/KeyValueWriter.java | 16 ++ .../store/readonly/fetcher/HdfsFetcher.java | 9 +- .../readonly/mr/AvroStoreBuilderMapper.java | 18 +- .../readonly/mr/AvroStoreBuilderReducer.java | 20 ++ .../mr/AvroStoreBuilderReducerPerBucket.java | 20 ++ .../readonly/mr/HadoopStoreBuilderUtils.java | 16 ++ .../readonly/mr/HadoopStoreJobRunner.java | 16 ++ .../store/readonly/mr/HdfsDataFileChunk.java | 16 ++ .../store/readonly/mr/IdentityJsonMapper.java | 16 ++ .../readonly/mr/IdentityJsonReducer.java | 16 ++ .../voldemort/store/readonly/mr/JobState.java | 19 ++ .../mr/VoldemortStoreBuilderMapper.java | 16 ++ .../mr/azkaban/AbstractHadoopJob.java | 16 ++ .../AbstractVoldemortBatchCopyJob.java | 16 ++ .../store/readonly/mr/azkaban/Job.java | 16 ++ .../azkaban/StoreBuilderTransformation.java | 25 +- .../azkaban/UndefinedPropertyException.java | 16 ++ .../mr/azkaban/VoldemortBatchIndexJob.java | 16 ++ .../mr/azkaban/VoldemortBuildAndPushJob.java | 16 ++ .../VoldemortMultiStoreBuildAndPushJob.java | 16 ++ .../mr/azkaban/VoldemortRollbackJob.java | 16 ++ .../mr/azkaban/VoldemortStoreBuilderJob.java | 16 ++ .../readonly/mr/azkaban/VoldemortSwapJob.java | 16 ++ .../mr/azkaban/VoldemortSwapperUtils.java | 20 +- .../mr/serialization/JsonConfigurable.java | 16 ++ .../JsonDeserializerComparator.java | 189 +++++++-------- .../readonly/mr/serialization/JsonMapper.java | 97 +++++--- .../mr/serialization/JsonOutputCollector.java | 16 ++ .../mr/serialization/JsonReducer.java | 130 +++++----- .../JsonSequenceFileInputFormat.java | 16 ++ .../JsonSequenceFileOutputFormat.java | 152 ++++++------ .../store/readonly/mr/utils/AvroUtils.java | 16 ++ .../store/readonly/mr/utils/EmailMessage.java | 97 +++++--- .../store/readonly/mr/utils/HadoopUtils.java | 16 ++ .../store/readonly/mr/utils/JsonSchema.java | 16 ++ .../mr/utils/KeyValuePartitioner.java | 16 ++ .../mr/utils/MapperKeyValueWriter.java | 16 ++ .../readonly/mr/utils/VoldemortUtils.java | 229 +++++++++--------- 40 files changed, 1026 insertions(+), 431 deletions(-) diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriter.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriter.java index 66dc02b416..87bebe74d8 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriter.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriter.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.disk; import java.io.ByteArrayOutputStream; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriterPerBucket.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriterPerBucket.java index 3df92b85ba..6fdf34f910 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriterPerBucket.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriterPerBucket.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.disk; import java.io.ByteArrayOutputStream; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/KeyValueWriter.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/KeyValueWriter.java index c4a2763e0b..eda4341b35 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/KeyValueWriter.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/KeyValueWriter.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.disk; import java.io.IOException; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/fetcher/HdfsFetcher.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/fetcher/HdfsFetcher.java index 1b83fd52ad..a5cbb371d9 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/fetcher/HdfsFetcher.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/fetcher/HdfsFetcher.java @@ -327,16 +327,19 @@ private void copyFileWithCheckSum(FileSystem fs, } catch(IOException ioe) { success = false; logger.error("Error during copying file ", ioe); - if(attempt < NUM_RETRIES - 1) + ioe.printStackTrace(); + if(attempt < NUM_RETRIES - 1) { logger.info("retrying copying"); - else + } else { throw ioe; + } } finally { IOUtils.closeQuietly(output); IOUtils.closeQuietly(input); - if(success) + if(success) { break; + } } diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/AvroStoreBuilderMapper.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/AvroStoreBuilderMapper.java index 6dc468af91..726dc8a29f 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/AvroStoreBuilderMapper.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/AvroStoreBuilderMapper.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr; import java.io.IOException; @@ -32,7 +48,7 @@ import voldemort.xml.StoreDefinitionsMapper; import azkaban.common.utils.Props; -/* +/** * Avro container files are not sequence input format files they contain records * instead of k/v pairs to consume these files we use the AvroMapper */ diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/AvroStoreBuilderReducer.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/AvroStoreBuilderReducer.java index 94433e7b51..43173a78d8 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/AvroStoreBuilderReducer.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/AvroStoreBuilderReducer.java @@ -1,5 +1,21 @@ package voldemort.store.readonly.mr; +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; @@ -20,6 +36,10 @@ import voldemort.store.readonly.disk.KeyValueWriter; import azkaban.common.utils.Utils; +/** + * Take key md5s and value bytes and build a Avro read-only store from these + * values + */ public class AvroStoreBuilderReducer implements Reducer, AvroValue, Text, Text>, JobConfigurable, Closeable { diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/AvroStoreBuilderReducerPerBucket.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/AvroStoreBuilderReducerPerBucket.java index d1f53d678b..1d41419e6d 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/AvroStoreBuilderReducerPerBucket.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/AvroStoreBuilderReducerPerBucket.java @@ -1,5 +1,21 @@ package voldemort.store.readonly.mr; +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -21,6 +37,10 @@ import voldemort.store.readonly.disk.KeyValueWriter; import azkaban.common.utils.Utils; +/** + * Take key md5s and value bytes and build a Avro read-only store from these + * values + */ public class AvroStoreBuilderReducerPerBucket implements Reducer, AvroValue, Text, Text>, JobConfigurable, Closeable { diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/HadoopStoreBuilderUtils.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/HadoopStoreBuilderUtils.java index eaf3d438ff..79fb9aac6c 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/HadoopStoreBuilderUtils.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/HadoopStoreBuilderUtils.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr; import java.io.ByteArrayOutputStream; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/HadoopStoreJobRunner.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/HadoopStoreJobRunner.java index 69b0d70250..366f30f5f7 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/HadoopStoreJobRunner.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/HadoopStoreJobRunner.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr; import java.io.BufferedReader; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/HdfsDataFileChunk.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/HdfsDataFileChunk.java index ae4c6466a6..34b790f074 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/HdfsDataFileChunk.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/HdfsDataFileChunk.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr; import java.io.IOException; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/IdentityJsonMapper.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/IdentityJsonMapper.java index 3ea7783ca7..ce623d9b6f 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/IdentityJsonMapper.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/IdentityJsonMapper.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr; import java.io.IOException; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/IdentityJsonReducer.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/IdentityJsonReducer.java index e4c7f3a21e..b6dc13d0d1 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/IdentityJsonReducer.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/IdentityJsonReducer.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr; import java.io.IOException; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/JobState.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/JobState.java index cbb5d6af77..fd01c87135 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/JobState.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/JobState.java @@ -1,5 +1,24 @@ package voldemort.store.readonly.mr; +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * Captures the job state + */ import java.io.IOException; import java.io.StringReader; import java.util.List; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/VoldemortStoreBuilderMapper.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/VoldemortStoreBuilderMapper.java index d0473d4bfa..d074ca41e6 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/VoldemortStoreBuilderMapper.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/VoldemortStoreBuilderMapper.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr; import java.util.Map; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/AbstractHadoopJob.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/AbstractHadoopJob.java index afd8bc34e3..bcd0170e9f 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/AbstractHadoopJob.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/AbstractHadoopJob.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.azkaban; import java.io.IOException; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/AbstractVoldemortBatchCopyJob.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/AbstractVoldemortBatchCopyJob.java index 3e6860ed04..3bf5af72d4 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/AbstractVoldemortBatchCopyJob.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/AbstractVoldemortBatchCopyJob.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.azkaban; import java.io.IOException; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/Job.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/Job.java index 91ba74370e..57dc73d006 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/Job.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/Job.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.azkaban; import java.util.Properties; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/StoreBuilderTransformation.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/StoreBuilderTransformation.java index 6703f6879a..f496ab4cd6 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/StoreBuilderTransformation.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/StoreBuilderTransformation.java @@ -1,14 +1,29 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.azkaban; /** * An interface to use for processing rows in the voldemort store builder * * @author jkreps - * + * */ -public interface StoreBuilderTransformation -{ - - public Object transform(Object obj); +public interface StoreBuilderTransformation { + + public Object transform(Object obj); } diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/UndefinedPropertyException.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/UndefinedPropertyException.java index cbfe09504b..a3b6e7cc23 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/UndefinedPropertyException.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/UndefinedPropertyException.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.azkaban; public class UndefinedPropertyException extends RuntimeException { diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortBatchIndexJob.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortBatchIndexJob.java index cbe1d28f78..245887651f 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortBatchIndexJob.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortBatchIndexJob.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.azkaban; import java.io.ByteArrayInputStream; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortBuildAndPushJob.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortBuildAndPushJob.java index f56e66039b..a147a518b5 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortBuildAndPushJob.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortBuildAndPushJob.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.azkaban; import java.io.BufferedReader; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortMultiStoreBuildAndPushJob.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortMultiStoreBuildAndPushJob.java index b4f573881d..80280a5d53 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortMultiStoreBuildAndPushJob.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortMultiStoreBuildAndPushJob.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.azkaban; import java.io.File; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortRollbackJob.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortRollbackJob.java index 5e378fbb3b..bb22c1b1fd 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortRollbackJob.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortRollbackJob.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.azkaban; import java.io.IOException; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortStoreBuilderJob.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortStoreBuilderJob.java index 7de42de723..e35e264456 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortStoreBuilderJob.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortStoreBuilderJob.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.azkaban; import java.io.FileNotFoundException; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortSwapJob.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortSwapJob.java index 93f4a8bd60..f4e51b5dd1 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortSwapJob.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortSwapJob.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.azkaban; import java.io.IOException; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortSwapperUtils.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortSwapperUtils.java index a0685aba53..6f87b5c406 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortSwapperUtils.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortSwapperUtils.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.azkaban; import java.io.BufferedReader; @@ -37,8 +53,8 @@ public static void doSwap(String storeName, Node node, String destinationDir) connection.setRequestMethod("POST"); connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); - connection.setRequestProperty("Content-Length", "" - + Integer.toString(data.getBytes().length)); + connection.setRequestProperty("Content-Length", + "" + Integer.toString(data.getBytes().length)); connection.setRequestProperty("Content-Language", "en-US"); connection.setUseCaches(false); diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonConfigurable.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonConfigurable.java index e59df9d056..becba118cd 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonConfigurable.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonConfigurable.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.serialization; import java.io.Closeable; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonDeserializerComparator.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonDeserializerComparator.java index aa67a9dfea..50cb0243c1 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonDeserializerComparator.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonDeserializerComparator.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.serialization; import java.io.DataInputStream; @@ -13,106 +29,87 @@ import voldemort.serialization.json.JsonTypeSerializer; /** - * A hadoop RawComparator that deserializes first. Usefull for sorting JSON objects + * A hadoop RawComparator that deserializes first. Usefull for sorting JSON + * objects * * @author jay * */ -public class JsonDeserializerComparator implements RawComparator, - Configurable -{ - - /** - * Should be same as BytesWritable.Length - */ - private int LENGTH_BYTES = 4; - - private Configuration config; - private InputBuffer buffer = new InputBuffer(); - private DataInputStream dataInput = new DataInputStream(buffer); - private JsonTypeSerializer serializer; - - public Configuration getConf() - { - return this.config; - } - - public void setConf(Configuration config) - { - if (config.get("json.schema") == null) - throw new IllegalArgumentException("No schema has been set!"); - this.serializer = new JsonTypeSerializer(config.get("json.schema")); - } - - public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) - { - return compareBytes(b1, - s1 + LENGTH_BYTES, - l1 - LENGTH_BYTES, - b2, - s2 + LENGTH_BYTES, - l2 - LENGTH_BYTES); - } - - public int compareBytes(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) - { - if (serializer == null) - throw new SerializationException("No serializer has been set!"); - try - { - buffer.reset(b1, s1, l1); - Object key1 = serializer.toObject(dataInput); - - buffer.reset(b2, s2, l2); - Object key2 = serializer.toObject(dataInput); - - if (key1 instanceof Comparable) - { - return this.compareSerializedObjects(key1, key2); - } - else - { - return customCompare(key1, key2, serializer); - } +public class JsonDeserializerComparator implements RawComparator, Configurable { + + /** + * Should be same as BytesWritable.Length + */ + private int LENGTH_BYTES = 4; + + private Configuration config; + private InputBuffer buffer = new InputBuffer(); + private DataInputStream dataInput = new DataInputStream(buffer); + private JsonTypeSerializer serializer; + + public Configuration getConf() { + return this.config; + } + + public void setConf(Configuration config) { + if(config.get("json.schema") == null) + throw new IllegalArgumentException("No schema has been set!"); + this.serializer = new JsonTypeSerializer(config.get("json.schema")); + } + + public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { + return compareBytes(b1, + s1 + LENGTH_BYTES, + l1 - LENGTH_BYTES, + b2, + s2 + LENGTH_BYTES, + l2 - LENGTH_BYTES); } - catch (IOException e) - { - throw new SerializationException(e); + + public int compareBytes(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { + if(serializer == null) + throw new SerializationException("No serializer has been set!"); + try { + buffer.reset(b1, s1, l1); + Object key1 = serializer.toObject(dataInput); + + buffer.reset(b2, s2, l2); + Object key2 = serializer.toObject(dataInput); + + if(key1 instanceof Comparable) { + return this.compareSerializedObjects(key1, key2); + } else { + return customCompare(key1, key2, serializer); + } + } catch(IOException e) { + throw new SerializationException(e); + } + } + + public int customCompare(Object key1, Object key2, JsonTypeSerializer serializer) { + byte[] b1 = serializer.toBytes(key1); + byte[] b2 = serializer.toBytes(key2); + + return BytesWritable.Comparator.compareBytes(b1, 0, b1.length, b2, 0, b2.length); + } + + public int compare(BytesWritable o1, BytesWritable o2) { + return this.compareBytes(o1.getBytes(), 0, o1.getLength(), o2.getBytes(), 0, o2.getLength()); + } + + public int compareSerializedObjects(Object o1, Object o2) { + if(o1 == o2) + return 0; + else if(o1 == null) + return -1; + else if(o2 == null) + return 1; + else if(o1.getClass() != o2.getClass()) + throw new IllegalArgumentException("Attempt to compare two items of different classes: " + + o1.getClass() + " and " + o2.getClass()); + else if(o1 instanceof Comparable) + return ((Comparable) o1).compareTo(o2); + + throw new IllegalArgumentException("Incomparable object type!"); } - } - - public int customCompare(Object key1, Object key2, JsonTypeSerializer serializer) - { - byte[] b1 = serializer.toBytes(key1); - byte[] b2 = serializer.toBytes(key2); - - return BytesWritable.Comparator.compareBytes(b1, 0, b1.length, b2, 0, b2.length); - } - - public int compare(BytesWritable o1, BytesWritable o2) - { - return this.compareBytes(o1.getBytes(), - 0, - o1.getLength(), - o2.getBytes(), - 0, - o2.getLength()); - } - - public int compareSerializedObjects(Object o1, Object o2) - { - if (o1 == o2) - return 0; - else if (o1 == null) - return -1; - else if (o2 == null) - return 1; - else if (o1.getClass() != o2.getClass()) - throw new IllegalArgumentException("Attempt to compare two items of different classes: " - + o1.getClass() + " and " + o2.getClass()); - else if (o1 instanceof Comparable) - return ((Comparable) o1).compareTo(o2); - - throw new IllegalArgumentException("Incomparable object type!"); - } } diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonMapper.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonMapper.java index 104d1d8bdf..038519c70b 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonMapper.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonMapper.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.serialization; import java.io.IOException; @@ -14,45 +30,46 @@ * @author jkreps * */ -public abstract class JsonMapper extends JsonConfigurable implements Mapper -{ - - public abstract void mapObjects(Object key, Object value, OutputCollector output, Reporter reporter) throws IOException; - - public void configure(JobConf conf) - { - setInputKeySerializer(getSchemaFromJob(conf, "mapper.input.key.schema")); - setInputValueSerializer(getSchemaFromJob(conf, "mapper.input.value.schema")); - setOutputKeySerializer(getSchemaFromJob(conf, "mapper.output.key.schema")); - setOutputValueSerializer(getSchemaFromJob(conf, "mapper.output.value.schema")); - - // set comparator for input Key Schema - if (conf.getBoolean("use.json.comparator", false)) - { - conf.setOutputKeyComparatorClass(JsonDeserializerComparator.class); - conf.set("json.schema", conf.get("mapper.output.key.schema")); +public abstract class JsonMapper extends JsonConfigurable implements + Mapper { + + public abstract void mapObjects(Object key, + Object value, + OutputCollector output, + Reporter reporter) throws IOException; + + public void configure(JobConf conf) { + setInputKeySerializer(getSchemaFromJob(conf, "mapper.input.key.schema")); + setInputValueSerializer(getSchemaFromJob(conf, "mapper.input.value.schema")); + setOutputKeySerializer(getSchemaFromJob(conf, "mapper.output.key.schema")); + setOutputValueSerializer(getSchemaFromJob(conf, "mapper.output.value.schema")); + + // set comparator for input Key Schema + if(conf.getBoolean("use.json.comparator", false)) { + conf.setOutputKeyComparatorClass(JsonDeserializerComparator.class); + conf.set("json.schema", conf.get("mapper.output.key.schema")); + } + setConfigured(true); + } + + @SuppressWarnings("unchecked") + public void map(BytesWritable key, + BytesWritable value, + OutputCollector output, + Reporter reporter) throws IOException { + if(!isConfigured()) + throw new IllegalStateException("JsonMapper's configure method wasn't called. Please make sure that super.configure() is called."); + + mapObjects(getInputKeySerializer().toObject(key.get()), + getInputValueSerializer().toObject(value.get()), + getOutputCollector(output), + reporter); } - setConfigured(true); - } - - @SuppressWarnings("unchecked") - public void map(BytesWritable key, - BytesWritable value, - OutputCollector output, - Reporter reporter) throws IOException - { - if(!isConfigured()) - throw new IllegalStateException("JsonMapper's configure method wasn't called. Please make sure that super.configure() is called."); - - mapObjects(getInputKeySerializer().toObject(key.get()), - getInputValueSerializer().toObject(value.get()), - getOutputCollector(output), - reporter); - } - - protected OutputCollector getOutputCollector(OutputCollector output) - { - return new JsonOutputCollector(output, getOutputKeySerializer(), getOutputValueSerializer()); - } - + + protected OutputCollector getOutputCollector(OutputCollector output) { + return new JsonOutputCollector(output, + getOutputKeySerializer(), + getOutputValueSerializer()); + } + } diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonOutputCollector.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonOutputCollector.java index d2100ae06e..5309aeaec0 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonOutputCollector.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonOutputCollector.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.serialization; import java.io.IOException; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonReducer.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonReducer.java index 466cec503b..fdde87b834 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonReducer.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonReducer.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.serialization; import java.io.IOException; @@ -15,75 +31,67 @@ * Base reducer class that uses JSON serialization * * @author jkreps - * + * */ public abstract class JsonReducer extends JsonConfigurable implements - Reducer -{ - - public void configure(JobConf conf) - { - setInputKeySerializer(getSchemaFromJob(conf, "mapper.output.key.schema")); - setInputValueSerializer(getSchemaFromJob(conf, "mapper.output.value.schema")); - setOutputKeySerializer(getSchemaFromJob(conf, "reducer.output.key.schema")); - setOutputValueSerializer(getSchemaFromJob(conf, "reducer.output.value.schema")); - - // set comparator for input Key Schema - if (conf.getBoolean("use.json.comparator", false)) - { - conf.setOutputKeyComparatorClass(JsonDeserializerComparator.class); - conf.set("json.schema", conf.get("mapper.output.key.schema")); - } - setConfigured(true); - } - - public abstract void reduceObjects(Object key, - Iterator values, - OutputCollector collector, - Reporter reporter) throws IOException; - - public void reduce(BytesWritable key, - Iterator values, - OutputCollector output, - Reporter reporter) throws IOException - { - reduceObjects(getInputKeySerializer().toObject(key.get()), - new TranslatingIterator(getInputValueSerializer(), values), - getOutputCollector(output), - reporter); - } - - protected OutputCollector getOutputCollector(OutputCollector output) - { - return new JsonOutputCollector(output, getOutputKeySerializer(), getOutputValueSerializer()); - } - - private static class TranslatingIterator implements Iterator - { - - private final Serializer serializer; - private final Iterator inner; - - public TranslatingIterator(Serializer serializer, Iterator inner) - { - this.serializer = serializer; - this.inner = inner; + Reducer { + + public void configure(JobConf conf) { + setInputKeySerializer(getSchemaFromJob(conf, "mapper.output.key.schema")); + setInputValueSerializer(getSchemaFromJob(conf, "mapper.output.value.schema")); + setOutputKeySerializer(getSchemaFromJob(conf, "reducer.output.key.schema")); + setOutputValueSerializer(getSchemaFromJob(conf, "reducer.output.value.schema")); + + // set comparator for input Key Schema + if(conf.getBoolean("use.json.comparator", false)) { + conf.setOutputKeyComparatorClass(JsonDeserializerComparator.class); + conf.set("json.schema", conf.get("mapper.output.key.schema")); + } + setConfigured(true); } - public boolean hasNext() - { - return inner.hasNext(); + public abstract void reduceObjects(Object key, + Iterator values, + OutputCollector collector, + Reporter reporter) throws IOException; + + public void reduce(BytesWritable key, + Iterator values, + OutputCollector output, + Reporter reporter) throws IOException { + reduceObjects(getInputKeySerializer().toObject(key.get()), + new TranslatingIterator(getInputValueSerializer(), values), + getOutputCollector(output), + reporter); } - public Object next() - { - return serializer.toObject(inner.next().get()); + protected OutputCollector getOutputCollector(OutputCollector output) { + return new JsonOutputCollector(output, + getOutputKeySerializer(), + getOutputValueSerializer()); } - public void remove() - { - inner.remove(); + private static class TranslatingIterator implements Iterator { + + private final Serializer serializer; + private final Iterator inner; + + public TranslatingIterator(Serializer serializer, Iterator inner) { + this.serializer = serializer; + this.inner = inner; + } + + public boolean hasNext() { + return inner.hasNext(); + } + + public Object next() { + return serializer.toObject(inner.next().get()); + } + + public void remove() { + inner.remove(); + } } - } } diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonSequenceFileInputFormat.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonSequenceFileInputFormat.java index f0e26c1956..a1d0ccfd50 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonSequenceFileInputFormat.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonSequenceFileInputFormat.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.serialization; import java.io.IOException; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonSequenceFileOutputFormat.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonSequenceFileOutputFormat.java index 2bd0ca1fd2..b8b1f13dc9 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonSequenceFileOutputFormat.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/serialization/JsonSequenceFileOutputFormat.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.serialization; import java.io.IOException; @@ -6,8 +22,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.Text; import org.apache.hadoop.io.SequenceFile.CompressionType; +import org.apache.hadoop.io.Text; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.DefaultCodec; import org.apache.hadoop.mapred.FileOutputFormat; @@ -21,81 +37,73 @@ import voldemort.serialization.json.JsonTypeDefinition; /** - * Copy of hadoop's SequenceFileOutputFormat modified to set the schema as metadata on - * output files + * Copy of hadoop's SequenceFileOutputFormat modified to set the schema as + * metadata on output files * * @author jkreps * */ public class JsonSequenceFileOutputFormat extends - SequenceFileOutputFormat -{ - - public RecordWriter getRecordWriter(FileSystem ignored, - JobConf job, - String name, - Progressable progress) - throws IOException - { - - // Shamelessly copy in hadoop code to allow us to set the metadata with our schema - - // get the path of the temporary output file - Path file = FileOutputFormat.getTaskOutputPath(job, name); - - FileSystem fs = file.getFileSystem(job); - CompressionType compressionType = CompressionType.BLOCK; - // find the right codec - Class codecClass = getOutputCompressorClass(job, DefaultCodec.class); - CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, job); - - // set the schema metadata - /* begin jays code */ - SequenceFile.Metadata meta = new SequenceFile.Metadata(); - meta.set(new Text("key.schema"), - new Text(getSchema("reducer.output.key.schema", job))); - meta.set(new Text("value.schema"), new Text(getSchema("reducer.output.value.schema", - job))); - - final SequenceFile.Writer out = SequenceFile.createWriter(fs, - job, - file, - job.getOutputKeyClass(), - job.getOutputValueClass(), - compressionType, - codec, - progress, - meta); - /* end jays code */ - - return new RecordWriter() - { - - public void write(BytesWritable key, BytesWritable value) - throws IOException - { - - out.append(key, value); - } - - public void close(Reporter reporter) - throws IOException - { - out.close(); - } - }; - } - - private String getSchema(String prop, JobConf conf) - { - String schema = conf.get(prop); - if (schema == null) - throw new IllegalArgumentException("The required property '" + prop - + "' is not defined in the JobConf for this Hadoop job."); - // check that it is a valid schema definition - JsonTypeDefinition.fromJson(schema); - - return schema; - } + SequenceFileOutputFormat { + + public RecordWriter getRecordWriter(FileSystem ignored, + JobConf job, + String name, + Progressable progress) + throws IOException { + + // Shamelessly copy in hadoop code to allow us to set the metadata with + // our schema + + // get the path of the temporary output file + Path file = FileOutputFormat.getTaskOutputPath(job, name); + + FileSystem fs = file.getFileSystem(job); + CompressionType compressionType = CompressionType.BLOCK; + // find the right codec + Class codecClass = getOutputCompressorClass(job, DefaultCodec.class); + CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, job); + + // set the schema metadata + /* begin jays code */ + SequenceFile.Metadata meta = new SequenceFile.Metadata(); + meta.set(new Text("key.schema"), new Text(getSchema("reducer.output.key.schema", job))); + meta.set(new Text("value.schema"), new Text(getSchema("reducer.output.value.schema", job))); + + final SequenceFile.Writer out = SequenceFile.createWriter(fs, + job, + file, + job.getOutputKeyClass(), + job.getOutputValueClass(), + compressionType, + codec, + progress, + meta); + /* end jays code */ + + return new RecordWriter() { + + public void write(BytesWritable key, BytesWritable value) throws IOException { + + out.append(key, value); + } + + public void close(Reporter reporter) throws IOException { + out.close(); + } + }; + } + + private String getSchema(String prop, JobConf conf) { + String schema = conf.get(prop); + if(schema == null) + throw new IllegalArgumentException("The required property '" + + prop + + "' is not defined in the JobConf for this Hadoop job."); + // check that it is a valid schema definition + JsonTypeDefinition.fromJson(schema); + + return schema; + } } diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/AvroUtils.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/AvroUtils.java index 6e61dce04c..c6aeaf5a72 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/AvroUtils.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/AvroUtils.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.utils; import java.io.BufferedInputStream; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/EmailMessage.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/EmailMessage.java index 458f291b0c..f8902cb9cf 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/EmailMessage.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/EmailMessage.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.utils; import java.io.File; @@ -22,6 +38,7 @@ import javax.mail.internet.MimeMultipart; public class EmailMessage { + private List _toAddress = new ArrayList(); private String _mailHost; private String _mailUser; @@ -30,58 +47,58 @@ public class EmailMessage { private String _fromAddress; private String _mimeType = "text/plain"; private StringBuffer _body = new StringBuffer(); - + private ArrayList _attachments = new ArrayList(); - + public EmailMessage() { this("localhost", "", ""); } - + public EmailMessage(String host, String user, String password) { _mailUser = user; _mailHost = host; _mailPassword = password; } - + public EmailMessage setMailHost(String host) { _mailHost = host; return this; } - + public EmailMessage setMailUser(String user) { _mailUser = user; return this; } - + public EmailMessage setMailPassword(String password) { _mailPassword = password; return this; } - public EmailMessage addAllToAddress(Collection< ? extends String > addresses) { + public EmailMessage addAllToAddress(Collection addresses) { _toAddress.addAll(addresses); return this; } - + public EmailMessage addToAddress(String address) { _toAddress.add(address); return this; } - + public EmailMessage setSubject(String subject) { _subject = subject; return this; } - + public EmailMessage setFromAddress(String fromAddress) { _fromAddress = fromAddress; return this; } - + public EmailMessage addAttachment(File file) throws MessagingException { return addAttachment(file.getName(), file); } - + public EmailMessage addAttachment(String attachmentName, File file) throws MessagingException { BodyPart attachmentPart = new MimeBodyPart(); DataSource fileDataSource = new FileDataSource(file); @@ -90,48 +107,49 @@ public EmailMessage addAttachment(String attachmentName, File file) throws Messa _attachments.add(attachmentPart); return this; } - - public EmailMessage addAttachment(String attachmentName, InputStream stream) throws MessagingException { + + public EmailMessage addAttachment(String attachmentName, InputStream stream) + throws MessagingException { BodyPart attachmentPart = new MimeBodyPart(stream); attachmentPart.setFileName(attachmentName); _attachments.add(attachmentPart); return this; } - + private void checkSettings() { - if (_mailHost == null) { + if(_mailHost == null) { throw new RuntimeException("Mail host not set."); } - - if (_mailUser == null) { + + if(_mailUser == null) { throw new RuntimeException("Mail user not set."); } - - if (_mailPassword == null) { + + if(_mailPassword == null) { throw new RuntimeException("Mail password not set."); } - - if (_fromAddress == null || _fromAddress.length() == 0) { + + if(_fromAddress == null || _fromAddress.length() == 0) { throw new RuntimeException("From address not set."); } - - if (_subject == null ) { + + if(_subject == null) { throw new RuntimeException("Subject cannot be null"); } - - if (_toAddress.size() == 0) { + + if(_toAddress.size() == 0) { throw new RuntimeException("T"); } } - - public void sendEmail() throws MessagingException{ + + public void sendEmail() throws MessagingException { checkSettings(); Properties props = new Properties(); props.setProperty("mail.transport.protocol", "smtp"); props.put("mail.host", _mailHost); props.put("mail.user", _mailUser); props.put("mail.password", _mailPassword); - + Session session = Session.getDefaultInstance(props); Message message = new MimeMessage(session); InternetAddress from = new InternetAddress(_fromAddress, false); @@ -140,21 +158,20 @@ public void sendEmail() throws MessagingException{ message.addRecipient(Message.RecipientType.TO, new InternetAddress(toAddr, false)); message.setSubject(_subject); message.setSentDate(new Date()); - - if ( _attachments.size() > 0 ) { + + if(_attachments.size() > 0) { MimeMultipart multipart = new MimeMultipart("related"); // Add attachments for(BodyPart part: _attachments) { multipart.addBodyPart(part); } - + BodyPart messageBodyPart = new MimeBodyPart(); messageBodyPart.setContent(_body.toString(), _mimeType); multipart.addBodyPart(messageBodyPart); - - message.setContent(multipart); - } - else { + + message.setContent(multipart); + } else { message.setContent(_body.toString(), _mimeType); } @@ -163,21 +180,21 @@ public void sendEmail() throws MessagingException{ transport.sendMessage(message, message.getRecipients(Message.RecipientType.TO)); transport.close(); } - + public void setBody(String body) { setBody(body, _mimeType); } - + public void setBody(String body, String mimeType) { _body = new StringBuffer(body); _mimeType = mimeType; } - + public EmailMessage setMimeType(String mimeType) { _mimeType = mimeType; return this; } - + public EmailMessage println(Object str) { _body.append(str); diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/HadoopUtils.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/HadoopUtils.java index adccf3021a..08c0b0da7f 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/HadoopUtils.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/HadoopUtils.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.utils; import java.io.ByteArrayInputStream; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/JsonSchema.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/JsonSchema.java index aced7fcf52..f3619b303b 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/JsonSchema.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/JsonSchema.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.utils; import java.util.Arrays; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/KeyValuePartitioner.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/KeyValuePartitioner.java index 900fe844ed..4853c63534 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/KeyValuePartitioner.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/KeyValuePartitioner.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.utils; import voldemort.store.StoreDefinition; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/MapperKeyValueWriter.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/MapperKeyValueWriter.java index 574eead8aa..61f801d51a 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/MapperKeyValueWriter.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/MapperKeyValueWriter.java @@ -1,3 +1,19 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.utils; import java.io.IOException; diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/VoldemortUtils.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/VoldemortUtils.java index db45a503ff..37586b4b74 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/VoldemortUtils.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/utils/VoldemortUtils.java @@ -1,131 +1,138 @@ +/* + * Copyright 2008-2009 LinkedIn, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package voldemort.store.readonly.mr.utils; import java.io.StringReader; import java.util.List; -import com.google.common.collect.Lists; - import voldemort.store.StoreDefinition; import voldemort.utils.Utils; import voldemort.xml.StoreDefinitionsMapper; -public class VoldemortUtils -{ +import com.google.common.collect.Lists; + +public class VoldemortUtils { - /** - * Given the comma separated list of properties as a string, splits it multiple strings - * - * @param paramValue - * Concatenated string - * @param type - * Type of parameter ( to throw exception ) - * @return List of string properties - */ - public static List getCommaSeparatedStringValues(String paramValue, String type) - { - List commaSeparatedProps = Lists.newArrayList(); - for (String url : Utils.COMMA_SEP.split(paramValue.trim())) - if (url.trim().length() > 0) - commaSeparatedProps.add(url); + /** + * Given the comma separated list of properties as a string, splits it + * multiple strings + * + * @param paramValue Concatenated string + * @param type Type of parameter ( to throw exception ) + * @return List of string properties + */ + public static List getCommaSeparatedStringValues(String paramValue, String type) { + List commaSeparatedProps = Lists.newArrayList(); + for(String url: Utils.COMMA_SEP.split(paramValue.trim())) + if(url.trim().length() > 0) + commaSeparatedProps.add(url); - if (commaSeparatedProps.size() == 0) - { - throw new RuntimeException("Number of " + type + " should be greater than zero"); + if(commaSeparatedProps.size() == 0) { + throw new RuntimeException("Number of " + type + " should be greater than zero"); + } + return commaSeparatedProps; } - return commaSeparatedProps; - } - public static String getStoreDefXml(String storeName, - int replicationFactor, - int requiredReads, - int requiredWrites, - Integer preferredReads, - Integer preferredWrites, - String keySerializer, - String valSerializer, - String description, - String owners) - { - StringBuffer storeXml = new StringBuffer(); + public static String getStoreDefXml(String storeName, + int replicationFactor, + int requiredReads, + int requiredWrites, + Integer preferredReads, + Integer preferredWrites, + String keySerializer, + String valSerializer, + String description, + String owners) { + StringBuffer storeXml = new StringBuffer(); - storeXml.append("\n\t"); - storeXml.append(storeName); - storeXml.append("\n\tread-only\n\t"); - if (description.length() != 0) - { - storeXml.append(""); - storeXml.append(description); - storeXml.append("\n\t"); - } - if (owners.length() != 0) - { - storeXml.append(""); - storeXml.append(owners); - storeXml.append("\n\t"); - } - storeXml.append("client\n\t"); - storeXml.append(replicationFactor); - storeXml.append("\n\t"); - storeXml.append(requiredReads); - storeXml.append("\n\t"); - storeXml.append(requiredWrites); - storeXml.append("\n\t"); - if (preferredReads != null) - storeXml.append("") - .append(preferredReads) - .append("\n\t"); - if (preferredWrites != null) - storeXml.append("") - .append(preferredWrites) - .append("\n\t"); - storeXml.append(""); - storeXml.append(keySerializer); - storeXml.append("\n\t"); - storeXml.append(valSerializer); - storeXml.append("\n"); + storeXml.append("\n\t"); + storeXml.append(storeName); + storeXml.append("\n\tread-only\n\t"); + if(description.length() != 0) { + storeXml.append(""); + storeXml.append(description); + storeXml.append("\n\t"); + } + if(owners.length() != 0) { + storeXml.append(""); + storeXml.append(owners); + storeXml.append("\n\t"); + } + storeXml.append("client\n\t"); + storeXml.append(replicationFactor); + storeXml.append("\n\t"); + storeXml.append(requiredReads); + storeXml.append("\n\t"); + storeXml.append(requiredWrites); + storeXml.append("\n\t"); + if(preferredReads != null) + storeXml.append("") + .append(preferredReads) + .append("\n\t"); + if(preferredWrites != null) + storeXml.append("") + .append(preferredWrites) + .append("\n\t"); + storeXml.append(""); + storeXml.append(keySerializer); + storeXml.append("\n\t"); + storeXml.append(valSerializer); + storeXml.append("\n"); - return storeXml.toString(); - } + return storeXml.toString(); + } - public static String getStoreDefXml(String storeName, - int replicationFactor, - int requiredReads, - int requiredWrites, - Integer preferredReads, - Integer preferredWrites, - String keySerializer, - String valSerializer) - { - StringBuffer storeXml = new StringBuffer(); + public static String getStoreDefXml(String storeName, + int replicationFactor, + int requiredReads, + int requiredWrites, + Integer preferredReads, + Integer preferredWrites, + String keySerializer, + String valSerializer) { + StringBuffer storeXml = new StringBuffer(); - storeXml.append("\n\t"); - storeXml.append(storeName); - storeXml.append("\n\tread-only\n\tclient\n\t"); - storeXml.append(replicationFactor); - storeXml.append("\n\t"); - storeXml.append(requiredReads); - storeXml.append("\n\t"); - storeXml.append(requiredWrites); - storeXml.append("\n\t"); - if (preferredReads != null) - storeXml.append("") - .append(preferredReads) - .append("\n\t"); - if (preferredWrites != null) - storeXml.append("") - .append(preferredWrites) - .append("\n\t"); - storeXml.append(""); - storeXml.append(keySerializer); - storeXml.append("\n\t"); - storeXml.append(valSerializer); - storeXml.append("\n"); + storeXml.append("\n\t"); + storeXml.append(storeName); + storeXml.append("\n\tread-only\n\tclient\n\t"); + storeXml.append(replicationFactor); + storeXml.append("\n\t"); + storeXml.append(requiredReads); + storeXml.append("\n\t"); + storeXml.append(requiredWrites); + storeXml.append("\n\t"); + if(preferredReads != null) + storeXml.append("") + .append(preferredReads) + .append("\n\t"); + if(preferredWrites != null) + storeXml.append("") + .append(preferredWrites) + .append("\n\t"); + storeXml.append(""); + storeXml.append(keySerializer); + storeXml.append("\n\t"); + storeXml.append(valSerializer); + storeXml.append("\n"); - return storeXml.toString(); - } + return storeXml.toString(); + } - public static StoreDefinition getStoreDef(String xml) - { - return new StoreDefinitionsMapper().readStore(new StringReader(xml)); - } + public static StoreDefinition getStoreDef(String xml) { + return new StoreDefinitionsMapper().readStore(new StringReader(xml)); + } }