Skip to content
Permalink
Browse files
[MINOR] Fix some code style issues based on check-style plugin (#4532)
Co-authored-by: yuezhang <yuezhang@freewheel.tv>
  • Loading branch information
zhangyue19921010 and yuezhang committed Jan 9, 2022
1 parent 3679070 commit cf362fb2d5540d69b30604a227037bdff600735e
Show file tree
Hide file tree
Showing 11 changed files with 12 additions and 11 deletions.
@@ -40,7 +40,8 @@ public class HoodieBucketLayout extends HoodieStorageLayout {
add(WriteOperationType.DELETE);
add(WriteOperationType.COMPACT);
add(WriteOperationType.DELETE_PARTITION);
}};
}
};

public HoodieBucketLayout(HoodieWriteConfig config) {
super(config);
@@ -25,6 +25,7 @@
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.avro.model.HoodieFileStatus;

import static org.apache.hudi.common.model.HoodieFileFormat.ORC;
import static org.apache.hudi.common.model.HoodieFileFormat.PARQUET;

@@ -63,7 +63,8 @@ public class DataTypeUtils {
// String types
put(StringType$.class,
newHashSet(VarcharType$.class, StringType$.class));
}};
}
};

/**
* Validates whether one {@link StructType} is compatible w/ the other one.
@@ -52,6 +52,7 @@
import org.apache.orc.storage.serde2.io.DateWritable;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.orc.TypeDescription;

import static org.apache.avro.JsonProperties.NULL_VALUE;

/**
@@ -20,6 +20,7 @@

import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

@@ -31,13 +31,10 @@
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.Dataset;



public class HoodieSparkBootstrapExample {

private static String tableType = HoodieTableType.MERGE_ON_READ.name();


public static void main(String[] args) throws Exception {
if (args.length < 5) {
System.err.println("Usage: HoodieSparkBootstrapExample <recordKey> <tableName> <partitionPath> <preCombineField> <basePath>");
@@ -68,7 +65,7 @@ public static void main(String[] args) throws Exception {
.option(HoodieTableConfig.BASE_FILE_FORMAT.key(), HoodieFileFormat.ORC.name())
.option(HoodieBootstrapConfig.BASE_PATH.key(), basePath)
.option(HoodieBootstrapConfig.KEYGEN_CLASS_NAME.key(), NonpartitionedKeyGenerator.class.getCanonicalName())
.mode(SaveMode.Overwrite).save("/hudi/"+tableName);
.mode(SaveMode.Overwrite).save("/hudi/" + tableName);

df.count();
}
@@ -106,6 +106,7 @@ public static WorkflowDag convertYamlToDag(String yaml) throws IOException {
Entry<String, JsonNode> dagContentNode = contentItr.next();
allNodes.put(dagContentNode.getKey(), convertJsonToDagNode(allNodes, dagContentNode.getKey(), dagContentNode.getValue()));
}
break;
default:
break;
}
@@ -161,7 +161,6 @@ private Dataset<Row> getInputDf(ExecutionContext context, SparkSession session,
.filter("_hoodie_is_deleted != true");
}


private ExpressionEncoder getEncoder(StructType schema) {
List<Attribute> attributes = JavaConversions.asJavaCollection(schema.toAttributes()).stream()
.map(Attribute::toAttribute).collect(Collectors.toList());
@@ -38,7 +38,6 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;

/**
@@ -74,10 +73,10 @@ public Pair<Option<String>, String> getNextFilePathsAndMaxModificationTime(
// Say input data is as follow input/1, input/2, input/5 since 3,4 was rolled back and 5 is new generated data
// checkpoint from the latest commit metadata will be 2 since 3,4 has been rolled back. We need to set the
// next batch id correctly as 5 instead of 3
Optional<String> correctBatchIdDueToRollback = Arrays.stream(fileStatuses)
Option<String> correctBatchIdDueToRollback = Option.fromJavaOptional(Arrays.stream(fileStatuses)
.map(f -> f.getPath().toString().split("/")[f.getPath().toString().split("/").length - 1])
.filter(bid1 -> Integer.parseInt(bid1) > lastBatchId)
.min((bid1, bid2) -> Integer.min(Integer.parseInt(bid1), Integer.parseInt(bid2)));
.min((bid1, bid2) -> Integer.min(Integer.parseInt(bid1), Integer.parseInt(bid2))));
if (correctBatchIdDueToRollback.isPresent() && Integer.parseInt(correctBatchIdDueToRollback.get()) > nextBatchId) {
nextBatchId = Integer.parseInt(correctBatchIdDueToRollback.get());
}
@@ -35,7 +35,6 @@
import org.apache.hudi.integ.testsuite.schema.TestSuiteFileBasedSchemaProvider;
import org.apache.hudi.integ.testsuite.writer.DeltaOutputMode;
import org.apache.hudi.keygen.TimestampBasedKeyGenerator;
import org.apache.hudi.utilities.schema.FilebasedSchemaProvider;
import org.apache.hudi.utilities.sources.AvroDFSSource;
import org.apache.hudi.utilities.testutils.UtilitiesTestBase;

@@ -21,6 +21,7 @@
import java.util.ArrayList;
import java.util.List;
import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertEquals;

public class TestMultiPartKeysValueExtractor {

0 comments on commit cf362fb

Please sign in to comment.