Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
CDK-188: Add a Kite format backed by FileInputFormat.
This format relies on descriptor properties to configure the InputFormat class and whether the key, value, or both should be used as the record type. MR works like the other formats, but delegates to the underlying InputFormat to instantiate readers, which are then wrapped.
- Loading branch information
Showing
16 changed files
with
1,136 additions
and
40 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
201 changes: 201 additions & 0 deletions
201
...-data/kite-data-core/src/main/java/org/kitesdk/data/spi/filesystem/InputFormatReader.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,201 @@ | ||
| /* | ||
| * Copyright 2013 Cloudera Inc. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.kitesdk.data.spi.filesystem; | ||
|
|
||
| import com.google.common.base.Preconditions; | ||
| import java.io.IOException; | ||
| import java.util.Iterator; | ||
| import java.util.NoSuchElementException; | ||
| import org.apache.hadoop.conf.Configuration; | ||
| import org.apache.hadoop.fs.FileStatus; | ||
| import org.apache.hadoop.fs.FileSystem; | ||
| import org.apache.hadoop.fs.Path; | ||
| import org.apache.hadoop.mapreduce.InputSplit; | ||
| import org.apache.hadoop.mapreduce.Job; | ||
| import org.apache.hadoop.mapreduce.RecordReader; | ||
| import org.apache.hadoop.mapreduce.TaskAttemptContext; | ||
| import org.apache.hadoop.mapreduce.TaskAttemptID; | ||
| import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; | ||
| import org.kitesdk.compat.Hadoop; | ||
| import org.kitesdk.data.DatasetDescriptor; | ||
| import org.kitesdk.data.DatasetIOException; | ||
| import org.kitesdk.data.DatasetReaderException; | ||
| import org.kitesdk.data.spi.AbstractDatasetReader; | ||
| import org.kitesdk.data.spi.ReaderWriterState; | ||
|
|
||
| public class InputFormatReader<E> extends AbstractDatasetReader<E> { | ||
| private static final TaskAttemptID FAKE_ID = | ||
| new TaskAttemptID("", 0, false, 0, 0); | ||
|
|
||
| private final FileSystem fs; | ||
| private final Path path; | ||
| private final Configuration conf; | ||
| private final DatasetDescriptor descriptor; | ||
| private final TaskAttemptContext attemptContext; | ||
|
|
||
| // reader state | ||
| private ReaderWriterState state = ReaderWriterState.NEW; | ||
| private Iterator<InputSplit> splits; | ||
| private RecordReader<E, Void> currentReader = null; | ||
| private boolean hasNext = false; | ||
| private boolean shouldAdvance = false; | ||
|
|
||
| public InputFormatReader(FileSystem fs, Path path, DatasetDescriptor descriptor) { | ||
| this.fs = fs; | ||
| this.path = path; | ||
| this.descriptor = descriptor; | ||
| this.state = ReaderWriterState.NEW; | ||
|
|
||
| // set up the configuration from the descriptor properties | ||
| this.conf = new Configuration(fs.getConf()); | ||
| for (String prop : descriptor.listProperties()) { | ||
| conf.set(prop, descriptor.getProperty(prop)); | ||
| } | ||
|
|
||
| this.attemptContext = Hadoop.TaskAttemptContext.ctor.newInstance(conf, FAKE_ID); | ||
| } | ||
|
|
||
| @Override | ||
| public void initialize() { | ||
| Preconditions.checkState(ReaderWriterState.NEW.equals(state), | ||
| "A reader may not be opened more than once - current state:%s", state); | ||
|
|
||
| try { | ||
| FileInputFormat format = InputFormatUtil.newInputFormatInstance(descriptor); | ||
| Job job = Hadoop.Job.newInstance.invoke(conf); | ||
|
|
||
| FileInputFormat.addInputPath(job, path); | ||
| // attempt to minimize the number of InputSplits | ||
| FileStatus stat = fs.getFileStatus(path); | ||
| FileInputFormat.setMaxInputSplitSize(job, stat.getLen()); | ||
|
|
||
| this.splits = format.getSplits(job).iterator(); | ||
| this.shouldAdvance = true; | ||
| this.state = ReaderWriterState.OPEN; | ||
|
|
||
| } catch (RuntimeException e) { | ||
| this.state = ReaderWriterState.ERROR; | ||
| throw new DatasetReaderException("Cannot calculate splits", e); | ||
| } catch (IOException e) { | ||
| this.state = ReaderWriterState.ERROR; | ||
| throw new DatasetIOException("Cannot calculate splits", e); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public boolean hasNext() { | ||
| Preconditions.checkState(ReaderWriterState.OPEN.equals(state), | ||
| "Attempt to read from a file in state:%s", state); | ||
|
|
||
| // the Iterator contract requires that calls to hasNext() not change the | ||
| // iterator state. calling next() should advance the iterator. however, | ||
| // this wraps a RecordReader that reuses objects, so advancing in next | ||
| // after retrieving the key/value pair mutates the pair. this hack is a way | ||
| // to advance once per call to next(), but do it as late as possible. | ||
| if (shouldAdvance) { | ||
| this.hasNext = advance(); | ||
| this.shouldAdvance = false; | ||
| } | ||
| return hasNext; | ||
| } | ||
|
|
||
| @Override | ||
| public E next() { | ||
| Preconditions.checkState(ReaderWriterState.OPEN.equals(state), | ||
| "Attempt to read from a file in state:%s", state); | ||
|
|
||
| if (!hasNext()) { | ||
| throw new NoSuchElementException(); | ||
| } | ||
|
|
||
| try { | ||
| E record = currentReader.getCurrentKey(); | ||
|
|
||
| this.shouldAdvance = true; | ||
|
|
||
| return record; | ||
| } catch (RuntimeException e) { | ||
| this.state = ReaderWriterState.ERROR; | ||
| throw new DatasetReaderException("Cannot get record", e); | ||
| } catch (IOException e) { | ||
| this.state = ReaderWriterState.ERROR; | ||
| throw new DatasetIOException("Cannot get record", e); | ||
| } catch (InterruptedException e) { | ||
| // don't swallow the interrupt | ||
| Thread.currentThread().interrupt(); | ||
| // error: it is unclear whether the underlying reader is valid | ||
| this.state = ReaderWriterState.ERROR; | ||
| throw new DatasetReaderException("Interrupted", e); | ||
| } | ||
| } | ||
|
|
||
| private boolean advance() { | ||
| try { | ||
| if (currentReader != null && currentReader.nextKeyValue()) { | ||
| return true; | ||
| } else { | ||
| if (currentReader == null) { | ||
| this.currentReader = InputFormatUtil.newRecordReader(descriptor); | ||
| } | ||
| while (splits.hasNext()) { | ||
| // advance the reader and see if it has records | ||
| InputSplit nextSplit = splits.next(); | ||
| currentReader.initialize(nextSplit, attemptContext); | ||
| if (currentReader.nextKeyValue()) { | ||
| return true; | ||
| } | ||
| } | ||
| // either no next split or all readers were empty | ||
| return false; | ||
| } | ||
| } catch (RuntimeException e) { | ||
| this.state = ReaderWriterState.ERROR; | ||
| throw new DatasetReaderException("Cannot advance reader", e); | ||
| } catch (IOException e) { | ||
| this.state = ReaderWriterState.ERROR; | ||
| throw new DatasetIOException("Cannot advance reader", e); | ||
| } catch (InterruptedException e) { | ||
| // error: it is unclear whether the underlying reader is valid | ||
| this.state = ReaderWriterState.ERROR; | ||
| throw new DatasetReaderException("Interrupted", e); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void close() { | ||
| if (!state.equals(ReaderWriterState.OPEN)) { | ||
| return; | ||
| } | ||
|
|
||
| this.state = ReaderWriterState.CLOSED; | ||
|
|
||
| try { | ||
| if (currentReader != null) { | ||
| currentReader.close(); | ||
| } | ||
| } catch (IOException e) { | ||
| throw new DatasetReaderException("Unable to close reader path:" + path, e); | ||
| } | ||
|
|
||
| this.hasNext = false; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isOpen() { | ||
| return (ReaderWriterState.OPEN == state); | ||
| } | ||
| } |
Oops, something went wrong.