Skip to content

Commit

Permalink
DRILL-7261: Simplify Easy framework config for new scan
Browse files Browse the repository at this point in the history
Most format plugins are created using the Easy format plugin. A recent
change added support for the "row set" scan framework. After converting
the text and log reader plugins, it became clear that the setup code
could be made simpler.

* Add the user name to the "file scan" framework.
* Pass the file system, split and user name to the batch reader via
  the "schema negotiator" rather than via the constructor.
* Create the traditional "scan batch" scan or the new row-set scan via
  functions instead of classes.
* Add Easy config option and method to choose the kind of scan
  framework.
* Add Easy config options for some newer options such as whether the
  plugin supports statistics.

Simplified reader creation

* The batch reader can be created just by overriding a method.
* A default error context is provided if the plugin does not provide
  one.

Tested by running all unit tests for the CSV reader which is based on
the new framework, and by testing the converted log reader (that reader
is not part of this commit.)

closes #1796
  • Loading branch information
paul-rogers authored and arina-ielchiieva committed Jun 7, 2019
1 parent 8a7007f commit 1bf7f15
Show file tree
Hide file tree
Showing 7 changed files with 282 additions and 299 deletions.
Expand Up @@ -62,7 +62,8 @@


public class FileScanFramework extends ManagedScanFramework { public class FileScanFramework extends ManagedScanFramework {


private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileScanFramework.class); private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(FileScanFramework.class);


/** /**
* The file schema negotiator adds no behavior at present, but is * The file schema negotiator adds no behavior at present, but is
Expand All @@ -80,7 +81,6 @@ public interface FileSchemaNegotiator extends SchemaNegotiator {
/** /**
* Gives the Drill file system for this operator. * Gives the Drill file system for this operator.
*/ */

DrillFileSystem fileSystem(); DrillFileSystem fileSystem();


/** /**
Expand Down Expand Up @@ -186,6 +186,10 @@ public ManagedReader<? extends SchemaNegotiator> next() {
return newReader(); return newReader();
} }


public CustomErrorContext errorContext() {
return fileFramework == null ? null : fileFramework.errorContext();
}

public abstract ManagedReader<? extends FileSchemaNegotiator> newReader(); public abstract ManagedReader<? extends FileSchemaNegotiator> newReader();
} }


Expand Down
Expand Up @@ -135,10 +135,17 @@ public interface ReaderFactory {


public static class ScanFrameworkBuilder extends ScanOrchestratorBuilder { public static class ScanFrameworkBuilder extends ScanOrchestratorBuilder {
protected ReaderFactory readerFactory; protected ReaderFactory readerFactory;
protected String userName;


public void setReaderFactory(ReaderFactory readerFactory) { public void setReaderFactory(ReaderFactory readerFactory) {
this.readerFactory = readerFactory; this.readerFactory = readerFactory;
} }

public ReaderFactory readerFactory() { return readerFactory; }

public void setUserName(String userName) {
this.userName = userName;
}
} }


// Inputs // Inputs
Expand Down
Expand Up @@ -64,8 +64,15 @@ public interface SchemaNegotiator {
* Specify an advanced error context which allows the reader to * Specify an advanced error context which allows the reader to
* fill in custom context values. * fill in custom context values.
*/ */

void setErrorContext(CustomErrorContext context); void setErrorContext(CustomErrorContext context);


/*
* The name of the user running the query.
*/

String userName();

/** /**
* Specify the table schema if this is an early-schema reader. Need * Specify the table schema if this is an early-schema reader. Need
* not be called for a late-schema readers. The schema provided here, * not be called for a late-schema readers. The schema provided here,
Expand Down
Expand Up @@ -100,6 +100,11 @@ public void setBatchSize(int maxRecordsPerBatch) {
batchSize = maxRecordsPerBatch; batchSize = maxRecordsPerBatch;
} }


@Override
public String userName() {
return framework.builder.userName;
}

/** /**
* Callback from the schema negotiator to build the schema from information from * Callback from the schema negotiator to build the schema from information from
* both the table and scan operator. Returns the result set loader to be used * both the table and scan operator. Returns the result set loader to be used
Expand Down

0 comments on commit 1bf7f15

Please sign in to comment.