diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b9c8c3a0b..8ac4c55eb 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -55,14 +55,6 @@ updates: labels: - "dependabot" - "dependencies" - - directory: /spark/sql-13/ - open-pull-requests-limit: 1 - package-ecosystem: gradle - schedule: - interval: weekly - labels: - - "dependabot" - - "dependencies" - directory: /spark/sql-20/ open-pull-requests-limit: 1 package-ecosystem: gradle diff --git a/.github/workflows/build-hive.yml b/.github/workflows/build_hive.yml similarity index 93% rename from .github/workflows/build-hive.yml rename to .github/workflows/build_hive.yml index b4b1f4044..46f0360a3 100644 --- a/.github/workflows/build-hive.yml +++ b/.github/workflows/build_hive.yml @@ -1,4 +1,4 @@ -name: Build +name: Build and test hive on: [push, pull_request] diff --git a/.github/workflows/build-mr.yml b/.github/workflows/build_mr.yml similarity index 93% rename from .github/workflows/build-mr.yml rename to .github/workflows/build_mr.yml index 84f6ad1b1..4b0958013 100644 --- a/.github/workflows/build-mr.yml +++ b/.github/workflows/build_mr.yml @@ -1,4 +1,4 @@ -name: Build +name: Build and test MR on: [push, pull_request] diff --git a/.github/workflows/build-spark.yml b/.github/workflows/build_spark.yml similarity index 93% rename from .github/workflows/build-spark.yml rename to .github/workflows/build_spark.yml index 7fdb1299b..1f8ffa69f 100644 --- a/.github/workflows/build-spark.yml +++ b/.github/workflows/build_spark.yml @@ -1,4 +1,4 @@ -name: Build +name: Build and test spark on: [push, pull_request] diff --git a/.github/workflows/build-spark-20.yml b/.github/workflows/build_spark_20.yml similarity index 93% rename from .github/workflows/build-spark-20.yml rename to .github/workflows/build_spark_20.yml index eca74c17f..cddc422f5 100644 --- a/.github/workflows/build-spark-20.yml +++ b/.github/workflows/build_spark_20.yml @@ -1,4 +1,4 @@ -name: Build +name: Build and test spark 20 on: [push, pull_request] diff --git a/.github/workflows/build-spark-30.yml b/.github/workflows/build_spark_30.yml similarity index 93% rename from .github/workflows/build-spark-30.yml rename to .github/workflows/build_spark_30.yml index bef6815d8..aabf88099 100644 --- a/.github/workflows/build-spark-30.yml +++ b/.github/workflows/build_spark_30.yml @@ -1,4 +1,4 @@ -name: Build +name: Build and test spark 30 on: [push, pull_request] diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index 2cf7c5613..e916ccc2e 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -1,5 +1,5 @@ name: Gradle Precommit -on: [pull_request] +on: [push,pull_request] jobs: precommit: diff --git a/CHANGELOG.md b/CHANGELOG.md index a7a65ed5a..d6b7fe26d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Removed ### Fixed - Restored skipped push down tests ([125](https://github.com/opensearch-project/opensearch-hadoop/pull/125)) +- Fixed spark failured due to deserialization failed logic ([219](https://github.com/opensearch-project/opensearch-hadoop/pull/219)) + ### Security ### Dependencies - Bumps `com.google.guava:guava` from 16.0.1 to 23.0 diff --git a/mr/src/main/java/org/opensearch/hadoop/serialization/dto/mapping/Field.java b/mr/src/main/java/org/opensearch/hadoop/serialization/dto/mapping/Field.java index 5c59be7aa..9f553513e 100644 --- a/mr/src/main/java/org/opensearch/hadoop/serialization/dto/mapping/Field.java +++ b/mr/src/main/java/org/opensearch/hadoop/serialization/dto/mapping/Field.java @@ -31,6 +31,7 @@ import java.io.Serializable; import java.util.Arrays; import java.util.Collection; +import java.util.Objects; import org.opensearch.hadoop.serialization.FieldType; @@ -61,20 +62,35 @@ public Field(String name, FieldType type, Collection properties) { this.properties = properties; } + @JsonProperty("properties") public Field[] properties() { return properties; } + @JsonProperty("type") public FieldType type() { return type; } + @JsonProperty("name") public String name() { return name; } @Override public String toString() { - return String.format("%s=%s", name, ((type == FieldType.OBJECT || type == FieldType.NESTED) ? Arrays.toString(properties) : type)); + return String.format("%s=%s", name, + ((type == FieldType.OBJECT || type == FieldType.NESTED) ? Arrays.toString(properties) : type)); + } + + @Override + public boolean equals(Object o) { + if (o instanceof Field == false) { + return false; + } + Field other = (Field) o; + return Objects.equals(this.name, other.name) && + Objects.equals(this.type, other.type) && + Objects.deepEquals(this.properties, other.properties); } } \ No newline at end of file diff --git a/mr/src/main/java/org/opensearch/hadoop/serialization/dto/mapping/Mapping.java b/mr/src/main/java/org/opensearch/hadoop/serialization/dto/mapping/Mapping.java index b459867ee..d41747422 100644 --- a/mr/src/main/java/org/opensearch/hadoop/serialization/dto/mapping/Mapping.java +++ b/mr/src/main/java/org/opensearch/hadoop/serialization/dto/mapping/Mapping.java @@ -44,6 +44,8 @@ import com.amazonaws.thirdparty.jackson.annotation.JsonCreator; import com.amazonaws.thirdparty.jackson.annotation.JsonProperty; +import java.util.Objects; + /** * A mapping has a name and a collection of fields. */ @@ -170,4 +172,15 @@ public String toString() { return String.format("%s=%s", index, Arrays.toString(fields)); } } + + @Override + public boolean equals(Object o) { + if (o instanceof Mapping == false) { + return false; + } + Mapping other = (Mapping) o; + return Objects.equals(this.index, other.index) && + Objects.equals(this.type, other.type) && + Objects.deepEquals(this.fields, other.fields); + } } \ No newline at end of file diff --git a/mr/src/main/java/org/opensearch/hadoop/util/IOUtils.java b/mr/src/main/java/org/opensearch/hadoop/util/IOUtils.java index 6835740b8..29d44cb94 100644 --- a/mr/src/main/java/org/opensearch/hadoop/util/IOUtils.java +++ b/mr/src/main/java/org/opensearch/hadoop/util/IOUtils.java @@ -33,6 +33,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.ObjectOutputStream; import java.io.Serializable; import java.io.StringReader; import java.io.StringWriter; @@ -43,25 +44,26 @@ import java.net.URL; import java.util.Properties; +import javax.xml.bind.DatatypeConverter; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.opensearch.hadoop.OpenSearchHadoopIllegalArgumentException; import org.opensearch.hadoop.serialization.OpenSearchHadoopSerializationException; import com.amazonaws.thirdparty.jackson.core.JsonProcessingException; -import com.amazonaws.thirdparty.jackson.databind.DeserializationFeature; import com.amazonaws.thirdparty.jackson.databind.ObjectMapper; import com.amazonaws.thirdparty.jackson.databind.SerializationFeature; + /** * Utility class used internally for the Pig support. */ public abstract class IOUtils { private final static Field BYTE_ARRAY_BUFFER; - static final ObjectMapper mapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, - false).configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + static final ObjectMapper mapper = new ObjectMapper().configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); private static final Log log = LogFactory.getLog(IOUtils.class); private final boolean trace = log.isTraceEnabled(); @@ -89,14 +91,14 @@ public static T deserializeFromBase64(String data, Class clazz){ if (!StringUtils.hasLength(data)) { return null; } - Object object = null; + final T object; try { object = mapper.readValue(data, clazz); } catch (JsonProcessingException e) { - throw new OpenSearchHadoopSerializationException("Cannot deserialize object " + object, e); + throw new OpenSearchHadoopSerializationException("Cannot deserialize string " + data, e); } - return (T) object; + return object; } public static String propsToString(Properties props) { diff --git a/spark/sql-30/build.gradle b/spark/sql-30/build.gradle index 38c53d1b8..165e0637a 100644 --- a/spark/sql-30/build.gradle +++ b/spark/sql-30/build.gradle @@ -189,6 +189,4 @@ sparkVariants { } } } -} -sourceCompatibility = JavaVersion.VERSION_17 -targetCompatibility = JavaVersion.VERSION_17 \ No newline at end of file +} \ No newline at end of file