This repository has been archived by the owner on Jan 29, 2022. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 603
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Initial basics for Hive support, as well as InputFormat for BSON files.
Need to migrate to "old" APIs for the BSON Input it turns out.
- Loading branch information
Brendan W. McAdams
committed
Jun 17, 2012
1 parent
c7627b9
commit af2dabf
Showing
5 changed files
with
177 additions
and
17 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
155 changes: 155 additions & 0 deletions
155
core/src/main/java/com/mongodb/hadoop/input/BSONFileRecordReader.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,155 @@ | ||
| package com.mongodb.hadoop.input; | ||
|
|
||
| import com.mongodb.hadoop.io.BSONWritable; | ||
| import org.apache.commons.logging.Log; | ||
| import org.apache.commons.logging.LogFactory; | ||
| import org.apache.hadoop.conf.Configuration; | ||
| import org.apache.hadoop.fs.FSDataInputStream; | ||
| import org.apache.hadoop.fs.FileSystem; | ||
| import org.apache.hadoop.fs.Path; | ||
| import org.apache.hadoop.io.NullWritable; | ||
| import org.apache.hadoop.mapreduce.InputSplit; | ||
| import org.apache.hadoop.mapreduce.RecordReader; | ||
| import org.apache.hadoop.mapreduce.TaskAttemptContext; | ||
| import org.apache.hadoop.mapreduce.lib.input.FileSplit; | ||
| import org.bson.*; | ||
|
|
||
| import java.io.DataInputStream; | ||
| import java.io.IOException; | ||
| import java.io.InputStream; | ||
| import java.util.Iterator; | ||
| import java.util.NoSuchElementException; | ||
| import java.util.concurrent.atomic.AtomicBoolean; | ||
|
|
||
| /** | ||
| * Copyright (c) 2008 - 2012 10gen, Inc. <http://10gen.com> | ||
| * <p/> | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * <p/> | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * <p/> | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| public class BSONFileRecordReader extends RecordReader<NullWritable, BSONWritable> { | ||
| private FileSplit fileSplit; | ||
| private Configuration conf; | ||
| private BSONReader rdr; | ||
| private static final Log log = LogFactory.getLog(BSONFileRecordReader.class); | ||
| private Object key; | ||
| private BSONWritable value; | ||
|
|
||
| @Override | ||
| public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException, InterruptedException { | ||
| this.fileSplit = (FileSplit) inputSplit; | ||
| this.conf = context.getConfiguration(); | ||
| Path file = fileSplit.getPath(); | ||
| FileSystem fs = file.getFileSystem(conf); | ||
| FSDataInputStream in = null; | ||
| in = fs.open(file); | ||
| rdr = new BSONReader(in); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean nextKeyValue() throws IOException, InterruptedException { | ||
| if (rdr.hasNext()) { | ||
| value = new BSONWritable( rdr.next() ); | ||
| return true; | ||
| } else { | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public NullWritable getCurrentKey() throws IOException, InterruptedException { | ||
| return NullWritable.get(); | ||
| } | ||
|
|
||
| @Override | ||
| public BSONWritable getCurrentValue() throws IOException, InterruptedException { | ||
| return value; | ||
| } | ||
|
|
||
| @Override | ||
| public float getProgress() throws IOException, InterruptedException { | ||
| return rdr.hasNext() ? 1.0f : 0.0f; | ||
| } | ||
|
|
||
|
|
||
| @Override | ||
| public void close() throws IOException { | ||
| // do nothing | ||
| } | ||
|
|
||
| private class BSONReader implements Iterable<BSONObject>, Iterator<BSONObject> { | ||
|
|
||
| public BSONReader(final InputStream input) { | ||
| _input = new DataInputStream( input ); | ||
| } | ||
|
|
||
| public Iterator<BSONObject> iterator(){ | ||
| return this; | ||
| } | ||
|
|
||
| public boolean hasNext(){ | ||
| checkHeader(); | ||
| return hasMore.get(); | ||
| } | ||
|
|
||
| private synchronized void checkHeader(){ | ||
| // Read the BSON length from the start of the record | ||
| byte[] l = new byte[4]; | ||
| try { | ||
| _input.readFully( l ); | ||
| nextLen = org.bson.io.Bits.readInt( l ); | ||
| nextHdr = l; | ||
| hasMore.set( true ); | ||
| } catch (Exception e) { | ||
| log.debug( "Failed to get next header: " + e, e ); | ||
| hasMore.set( false ); | ||
| try { | ||
| _input.close(); | ||
| } | ||
| catch ( IOException e1 ) { } | ||
| } | ||
| } | ||
|
|
||
| public BSONObject next(){ | ||
| try { | ||
| byte[] data = new byte[nextLen + 4]; | ||
| System.arraycopy( nextHdr, 0, data, 0, 4 ); | ||
| _input.readFully( data, 4, nextLen - 4 ); | ||
| decoder.decode( data, callback ); | ||
| return (BSONObject) callback.get(); | ||
| } | ||
| catch ( IOException e ) { | ||
| /* If we can't read another length it's not an error, just return quietly. */ | ||
| log.info( "No Length Header available." + e ); | ||
| hasMore.set( false ); | ||
| try { | ||
| _input.close(); | ||
| } | ||
| catch ( IOException e1 ) { } | ||
| throw new NoSuchElementException("Iteration completed."); | ||
| } | ||
| } | ||
|
|
||
| public void remove(){ | ||
| throw new UnsupportedOperationException(); | ||
| } | ||
|
|
||
| private final BSONDecoder decoder = new BasicBSONDecoder(); | ||
| private final BSONCallback callback = new BasicBSONCallback(); | ||
|
|
||
| private volatile byte[] nextHdr; | ||
| private volatile int nextLen; | ||
| private AtomicBoolean hasMore = new AtomicBoolean( true ); | ||
| private final DataInputStream _input; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,4 @@ | ||
| addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.7.3") | ||
| addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.8.3") | ||
|
|
||
| resolvers += Resolver.url("sbt-plugin-releases", | ||
| new URL("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases/"))(Resolver.ivyStylePatterns) | ||
|
|
||
| addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.5.2") | ||
| resolvers += Resolver.url("artifactory", url("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns) | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters