Skip to content

Commit

Permalink
[ASTERIXDB-2713][EXT] Change the name of the new adapter parameter fo…
Browse files Browse the repository at this point in the history 
…r escaping

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
Change the name of the new parameter "quote-escape" to just "escape".

Change-Id: I3b51fa317bf327fbad18b32f0fd45e47e853ca50
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6023
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>
  • Loading branch information
@AliSolaiman
AliSolaiman committed Apr 27, 2020
1 parent b1d8d52 commit 39c5df0
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 22 deletions.
16 changes: 7 additions & 9 deletions ...n/java/org/apache/asterix/external/input/record/reader/stream/QuotedLineRecordReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
public class QuotedLineRecordReader extends LineRecordReader {

private char quote;
private char quoteEscape;
private char escape;
private boolean prevCharEscape;
private int readLength;
private boolean inQuote;
Expand All @@ -54,7 +54,7 @@ public void configure(IHyracksTaskContext ctx, AsterixInputStream inputStream, M
String quoteString = config.get(ExternalDataConstants.KEY_QUOTE);
ExternalDataUtils.validateQuote(quoteString);
this.quote = quoteString.charAt(0);
this.quoteEscape = ExternalDataUtils.validateGetQuoteEscape(config);
this.escape = ExternalDataUtils.validateGetEscape(config);
}

@Override
Expand Down Expand Up @@ -117,7 +117,7 @@ public boolean hasNext() throws IOException {
}
boolean maybeInQuote = false;
for (; bufferPosn < bufferLength; ++bufferPosn) {
if (inputBuffer[bufferPosn] == quote && quoteEscape == quote) {
if (inputBuffer[bufferPosn] == quote && escape == quote) {
inQuote |= maybeInQuote;
prevCharEscape |= maybeInQuote;
}
Expand All @@ -135,18 +135,16 @@ public boolean hasNext() throws IOException {
prevCharCR = (inputBuffer[bufferPosn] == ExternalDataConstants.CR);
// if this is an opening quote, mark it
inQuote = inputBuffer[bufferPosn] == quote && !prevCharEscape;
// the quoteEscape != quote is for making an opening quote not an escape
prevCharEscape =
inputBuffer[bufferPosn] == quoteEscape && !prevCharEscape && quoteEscape != quote;
// the escape != quote is for making an opening quote not an escape
prevCharEscape = inputBuffer[bufferPosn] == escape && !prevCharEscape && escape != quote;
} else {
// if quote == quoteEscape and current char is quote, then it could be closing or escaping
// if quote == escape and current char is quote, then it could be closing or escaping
if (inputBuffer[bufferPosn] == quote && !prevCharEscape) {
// this is most likely a closing quote. the outcome depends on the next char
inQuote = false;
maybeInQuote = true;
}
prevCharEscape =
inputBuffer[bufferPosn] == quoteEscape && !prevCharEscape && quoteEscape != quote;
prevCharEscape = inputBuffer[bufferPosn] == escape && !prevCharEscape && escape != quote;
}
}
readLength = bufferPosn - startPosn;
Expand Down
2 changes: 1 addition & 1 deletion ...x-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ private ExternalDataConstants() {
public static final String KEY_LOCAL_SOCKET_PATH = "local-socket-path";
public static final String KEY_FORMAT = "format";
public static final String KEY_QUOTE = "quote";
public static final String KEY_QUOTE_ESCAPE = "quote-escape";
public static final String KEY_ESCAPE = "escape";
public static final String KEY_PARSER = "parser";
public static final String KEY_DATASET_RECORD = "dataset-record";
public static final String KEY_HIVE_SERDE = "hive-serde";
Expand Down
23 changes: 11 additions & 12 deletions ...terix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,13 +82,13 @@ public static char validateGetQuote(Map<String, String> configuration, char deli
return quote;
}

public static char validateGetQuoteEscape(Map<String, String> configuration) throws HyracksDataException {
String quoteEscapeValue = configuration.get(ExternalDataConstants.KEY_QUOTE_ESCAPE);
if (quoteEscapeValue == null) {
public static char validateGetEscape(Map<String, String> configuration) throws HyracksDataException {
String escapeValue = configuration.get(ExternalDataConstants.KEY_ESCAPE);
if (escapeValue == null) {
return ExternalDataConstants.ESCAPE;
}
validateQuoteEscape(quoteEscapeValue);
return quoteEscapeValue.charAt(0);
validateEscape(escapeValue);
return escapeValue.charAt(0);
}

public static void validateDataParserParameters(Map<String, String> configuration) throws AsterixException {
Expand Down Expand Up @@ -331,11 +331,11 @@ public static void defaultConfiguration(Map<String, String> configuration) {
if (format.equals(ExternalDataConstants.FORMAT_CSV)) {
configuration.putIfAbsent(ExternalDataConstants.KEY_DELIMITER, ExternalDataConstants.DEFAULT_DELIMITER);
configuration.putIfAbsent(ExternalDataConstants.KEY_QUOTE, ExternalDataConstants.DEFAULT_QUOTE);
configuration.putIfAbsent(ExternalDataConstants.KEY_QUOTE_ESCAPE, ExternalDataConstants.DEFAULT_QUOTE);
configuration.putIfAbsent(ExternalDataConstants.KEY_ESCAPE, ExternalDataConstants.DEFAULT_QUOTE);
} else if (format.equals(ExternalDataConstants.FORMAT_TSV)) {
configuration.putIfAbsent(ExternalDataConstants.KEY_DELIMITER, ExternalDataConstants.TAB_STR);
configuration.putIfAbsent(ExternalDataConstants.KEY_QUOTE, ExternalDataConstants.NULL_STR);
configuration.putIfAbsent(ExternalDataConstants.KEY_QUOTE_ESCAPE, ExternalDataConstants.NULL_STR);
configuration.putIfAbsent(ExternalDataConstants.KEY_ESCAPE, ExternalDataConstants.NULL_STR);
}
}
}
Expand Down Expand Up @@ -396,7 +396,7 @@ public static void validate(Map<String, String> configuration) throws HyracksDat
}
char delimiter = validateGetDelimiter(configuration);
validateGetQuote(configuration, delimiter);
validateGetQuoteEscape(configuration);
validateGetEscape(configuration);
String value = configuration.get(KEY_REDACT_WARNINGS);
if (value != null && !isBoolean(value)) {
throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL, KEY_REDACT_WARNINGS, value);
Expand Down Expand Up @@ -425,10 +425,9 @@ public static void validateQuote(String quote) throws RuntimeDataException {
}
}

private static void validateQuoteEscape(String quoteEsc) throws RuntimeDataException {
if (quoteEsc.length() != 1) {
throw new RuntimeDataException(ErrorCode.PARSER_INVALID_CHAR_LENGTH, quoteEsc,
ExternalDataConstants.KEY_QUOTE_ESCAPE);
private static void validateEscape(String esc) throws RuntimeDataException {
if (esc.length() != 1) {
throw new RuntimeDataException(ErrorCode.PARSER_INVALID_CHAR_LENGTH, esc, ExternalDataConstants.KEY_ESCAPE);
}
}

Expand Down

0 comments on commit 39c5df0

Please sign in to comment.