Skip to content

Commit bf20a3e

Browse files
liunaijiecorgy-w
andauthored
[improve] typesense options (#9398)
Co-authored-by: corgy-w <73771213+corgy-w@users.noreply.github.com>
1 parent 145b687 commit bf20a3e

File tree

15 files changed

+76
-72
lines changed

15 files changed

+76
-72
lines changed

seatunnel-ci-tools/src/test/java/org/apache/seatunnel/api/ConnectorOptionCheckTest.java

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -190,8 +190,6 @@ public void checkConnectorOptionExist() {
190190
private Set<String> buildWhiteList() {
191191
Set<String> whiteList = new HashSet<>();
192192
whiteList.add("JdbcSinkOptions");
193-
whiteList.add("TypesenseSourceOptions");
194-
whiteList.add("TypesenseSinkOptions");
195193
whiteList.add("MongodbSinkOptions");
196194
whiteList.add("SelectDBSinkOptions");
197195
whiteList.add("PostgresIncrementalSourceOptions");

seatunnel-connectors-v2/connector-typesense/src/main/java/org/apache/seatunnel/connectors/seatunnel/typesense/catalog/TypesenseCatalogFactory.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.seatunnel.api.table.catalog.Catalog;
2323
import org.apache.seatunnel.api.table.factory.CatalogFactory;
2424
import org.apache.seatunnel.api.table.factory.Factory;
25+
import org.apache.seatunnel.connectors.seatunnel.typesense.config.TypesenseBaseOptions;
2526

2627
import com.google.auto.service.AutoService;
2728

@@ -35,11 +36,15 @@ public Catalog createCatalog(String catalogName, ReadonlyConfig options) {
3536

3637
@Override
3738
public String factoryIdentifier() {
38-
return "Typesense";
39+
return TypesenseBaseOptions.CONNECTOR_IDENTITY;
3940
}
4041

4142
@Override
4243
public OptionRule optionRule() {
43-
return OptionRule.builder().build();
44+
return OptionRule.builder()
45+
.required(TypesenseBaseOptions.HOSTS)
46+
.required(TypesenseBaseOptions.PROTOCOL)
47+
.required(TypesenseBaseOptions.APIKEY)
48+
.build();
4449
}
4550
}

seatunnel-connectors-v2/connector-typesense/src/main/java/org/apache/seatunnel/connectors/seatunnel/typesense/catalog/TypesenseTypeConverter.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
2727
import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
2828
import org.apache.seatunnel.connectors.seatunnel.typesense.client.TypesenseType;
29+
import org.apache.seatunnel.connectors.seatunnel.typesense.config.TypesenseBaseOptions;
2930

3031
import com.google.auto.service.AutoService;
3132

@@ -40,7 +41,7 @@ public class TypesenseTypeConverter implements BasicTypeConverter<BasicTypeDefin
4041

4142
@Override
4243
public String identifier() {
43-
return "Typesense";
44+
return TypesenseBaseOptions.CONNECTOR_IDENTITY;
4445
}
4546

4647
@Override

seatunnel-connectors-v2/connector-typesense/src/main/java/org/apache/seatunnel/connectors/seatunnel/typesense/client/TypesenseClient.java

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@
1919

2020
import org.apache.seatunnel.api.configuration.ReadonlyConfig;
2121
import org.apache.seatunnel.api.table.converter.BasicTypeDefine;
22-
import org.apache.seatunnel.connectors.seatunnel.typesense.config.TypesenseConnectionConfig;
22+
import org.apache.seatunnel.connectors.seatunnel.typesense.config.TypesenseBaseOptions;
23+
import org.apache.seatunnel.connectors.seatunnel.typesense.config.TypesenseSourceOptions;
2324
import org.apache.seatunnel.connectors.seatunnel.typesense.exception.TypesenseConnectorException;
2425
import org.apache.seatunnel.connectors.seatunnel.typesense.util.URLParamsConverter;
2526

@@ -47,7 +48,6 @@
4748
import java.util.List;
4849
import java.util.Map;
4950

50-
import static org.apache.seatunnel.connectors.seatunnel.typesense.config.SourceConfig.QUERY_BATCH_SIZE;
5151
import static org.apache.seatunnel.connectors.seatunnel.typesense.exception.TypesenseConnectorErrorCode.CREATE_COLLECTION_ERROR;
5252
import static org.apache.seatunnel.connectors.seatunnel.typesense.exception.TypesenseConnectorErrorCode.DELETE_COLLECTION_ERROR;
5353
import static org.apache.seatunnel.connectors.seatunnel.typesense.exception.TypesenseConnectorErrorCode.DROP_COLLECTION_ERROR;
@@ -69,9 +69,9 @@ public class TypesenseClient {
6969
}
7070

7171
public static TypesenseClient createInstance(ReadonlyConfig config) {
72-
List<String> hosts = config.get(TypesenseConnectionConfig.HOSTS);
73-
String protocol = config.get(TypesenseConnectionConfig.PROTOCOL);
74-
String apiKey = config.get(TypesenseConnectionConfig.APIKEY);
72+
List<String> hosts = config.get(TypesenseBaseOptions.HOSTS);
73+
String protocol = config.get(TypesenseBaseOptions.PROTOCOL);
74+
String apiKey = config.get(TypesenseBaseOptions.APIKEY);
7575
return createInstance(hosts, apiKey, protocol);
7676
}
7777

@@ -114,7 +114,8 @@ public void insert(String collection, List<String> documentList) {
114114
}
115115

116116
public SearchResult search(String collection, String query, int offset) throws Exception {
117-
return search(collection, query, offset, QUERY_BATCH_SIZE.defaultValue());
117+
return search(
118+
collection, query, offset, TypesenseSourceOptions.QUERY_BATCH_SIZE.defaultValue());
118119
}
119120

120121
public SearchResult search(String collection, String query, int offset, int pageSize)
Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@
2222

2323
import java.util.List;
2424

25-
public class TypesenseConnectionConfig {
25+
public class TypesenseBaseOptions {
26+
27+
public static final String CONNECTOR_IDENTITY = "Typesense";
2628

2729
public static final Option<List<String>> HOSTS =
2830
Options.key("hosts")
@@ -42,4 +44,10 @@ public class TypesenseConnectionConfig {
4244
.stringType()
4345
.defaultValue("http")
4446
.withDescription("Default is http , for Typesense Cloud use https");
47+
48+
public static final Option<String> COLLECTION =
49+
Options.key("collection")
50+
.stringType()
51+
.noDefaultValue()
52+
.withDescription("Typesense collection name");
4553
}
Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,7 @@
2929
import static org.apache.seatunnel.api.sink.DataSaveMode.DROP_DATA;
3030
import static org.apache.seatunnel.api.sink.DataSaveMode.ERROR_WHEN_DATA_EXISTS;
3131

32-
public class SinkConfig {
33-
34-
public static final Option<String> COLLECTION =
35-
Options.key("collection")
36-
.stringType()
37-
.noDefaultValue()
38-
.withDescription("Typesense collection name");
32+
public class TypesenseSinkOptions extends TypesenseBaseOptions {
3933

4034
public static final Option<List<String>> PRIMARY_KEYS =
4135
Options.key("primary_keys")
Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,7 @@
2020
import org.apache.seatunnel.api.configuration.Option;
2121
import org.apache.seatunnel.api.configuration.Options;
2222

23-
public class SourceConfig {
24-
25-
public static final Option<String> COLLECTION =
26-
Options.key("collection")
27-
.stringType()
28-
.noDefaultValue()
29-
.withDescription("Typesense collection name");
23+
public class TypesenseSourceOptions extends TypesenseBaseOptions {
3024

3125
public static final Option<String> QUERY =
3226
Options.key("query")

seatunnel-connectors-v2/connector-typesense/src/main/java/org/apache/seatunnel/connectors/seatunnel/typesense/dto/CollectionInfo.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
package org.apache.seatunnel.connectors.seatunnel.typesense.dto;
1919

2020
import org.apache.seatunnel.api.configuration.ReadonlyConfig;
21-
import org.apache.seatunnel.connectors.seatunnel.typesense.config.SinkConfig;
21+
import org.apache.seatunnel.connectors.seatunnel.typesense.config.TypesenseSinkOptions;
2222

2323
import lombok.Data;
2424

@@ -32,9 +32,9 @@ public class CollectionInfo {
3232

3333
public CollectionInfo(String collection, ReadonlyConfig config) {
3434
this.collection = collection;
35-
if (config.getOptional(SinkConfig.PRIMARY_KEYS).isPresent()) {
36-
primaryKeys = config.get(SinkConfig.PRIMARY_KEYS).toArray(new String[0]);
35+
if (config.getOptional(TypesenseSinkOptions.PRIMARY_KEYS).isPresent()) {
36+
primaryKeys = config.get(TypesenseSinkOptions.PRIMARY_KEYS).toArray(new String[0]);
3737
}
38-
keyDelimiter = config.get(SinkConfig.KEY_DELIMITER);
38+
keyDelimiter = config.get(TypesenseSinkOptions.KEY_DELIMITER);
3939
}
4040
}

seatunnel-connectors-v2/connector-typesense/src/main/java/org/apache/seatunnel/connectors/seatunnel/typesense/sink/TypesenseSink.java

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -32,16 +32,13 @@
3232
import org.apache.seatunnel.api.table.factory.CatalogFactory;
3333
import org.apache.seatunnel.api.table.factory.FactoryUtil;
3434
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
35-
import org.apache.seatunnel.connectors.seatunnel.typesense.config.SinkConfig;
35+
import org.apache.seatunnel.connectors.seatunnel.typesense.config.TypesenseSinkOptions;
3636
import org.apache.seatunnel.connectors.seatunnel.typesense.state.TypesenseAggregatedCommitInfo;
3737
import org.apache.seatunnel.connectors.seatunnel.typesense.state.TypesenseCommitInfo;
3838
import org.apache.seatunnel.connectors.seatunnel.typesense.state.TypesenseSinkState;
3939

4040
import java.util.Optional;
4141

42-
import static org.apache.seatunnel.connectors.seatunnel.typesense.config.SinkConfig.MAX_BATCH_SIZE;
43-
import static org.apache.seatunnel.connectors.seatunnel.typesense.config.SinkConfig.MAX_RETRY_COUNT;
44-
4542
public class TypesenseSink
4643
implements SeaTunnelSink<
4744
SeaTunnelRow,
@@ -59,13 +56,13 @@ public class TypesenseSink
5956
public TypesenseSink(ReadonlyConfig config, CatalogTable catalogTable) {
6057
this.config = config;
6158
this.catalogTable = catalogTable;
62-
maxBatchSize = config.get(MAX_BATCH_SIZE);
63-
maxRetryCount = config.get(MAX_RETRY_COUNT);
59+
maxBatchSize = config.get(TypesenseSinkOptions.MAX_BATCH_SIZE);
60+
maxRetryCount = config.get(TypesenseSinkOptions.MAX_RETRY_COUNT);
6461
}
6562

6663
@Override
6764
public String getPluginName() {
68-
return "Typesense";
65+
return TypesenseSinkOptions.CONNECTOR_IDENTITY;
6966
}
7067

7168
@Override
@@ -84,8 +81,8 @@ public Optional<SaveModeHandler> getSaveModeHandler() {
8481
return Optional.empty();
8582
}
8683
Catalog catalog = catalogFactory.createCatalog(catalogFactory.factoryIdentifier(), config);
87-
SchemaSaveMode schemaSaveMode = config.get(SinkConfig.SCHEMA_SAVE_MODE);
88-
DataSaveMode dataSaveMode = config.get(SinkConfig.DATA_SAVE_MODE);
84+
SchemaSaveMode schemaSaveMode = config.get(TypesenseSinkOptions.SCHEMA_SAVE_MODE);
85+
DataSaveMode dataSaveMode = config.get(TypesenseSinkOptions.DATA_SAVE_MODE);
8986

9087
TablePath tablePath = TablePath.of("", catalogTable.getTableId().getTableName());
9188
catalog.open();

seatunnel-connectors-v2/connector-typesense/src/main/java/org/apache/seatunnel/connectors/seatunnel/typesense/sink/TypesenseSinkFactory.java

Lines changed: 12 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -25,41 +25,38 @@
2525
import org.apache.seatunnel.api.table.factory.Factory;
2626
import org.apache.seatunnel.api.table.factory.TableSinkFactory;
2727
import org.apache.seatunnel.api.table.factory.TableSinkFactoryContext;
28-
import org.apache.seatunnel.connectors.seatunnel.typesense.config.SinkConfig;
28+
import org.apache.seatunnel.connectors.seatunnel.typesense.config.TypesenseSinkOptions;
2929

3030
import com.google.auto.service.AutoService;
3131

32-
import static org.apache.seatunnel.connectors.seatunnel.typesense.config.SinkConfig.COLLECTION;
33-
import static org.apache.seatunnel.connectors.seatunnel.typesense.config.SinkConfig.KEY_DELIMITER;
34-
import static org.apache.seatunnel.connectors.seatunnel.typesense.config.SinkConfig.PRIMARY_KEYS;
35-
import static org.apache.seatunnel.connectors.seatunnel.typesense.config.TypesenseConnectionConfig.APIKEY;
36-
import static org.apache.seatunnel.connectors.seatunnel.typesense.config.TypesenseConnectionConfig.HOSTS;
37-
3832
@AutoService(Factory.class)
3933
public class TypesenseSinkFactory implements TableSinkFactory {
4034

4135
@Override
4236
public String factoryIdentifier() {
43-
return "Typesense";
37+
return TypesenseSinkOptions.CONNECTOR_IDENTITY;
4438
}
4539

4640
@Override
4741
public OptionRule optionRule() {
4842
return OptionRule.builder()
4943
.required(
50-
HOSTS,
51-
COLLECTION,
52-
APIKEY,
53-
SinkConfig.SCHEMA_SAVE_MODE,
54-
SinkConfig.DATA_SAVE_MODE)
55-
.optional(PRIMARY_KEYS, KEY_DELIMITER)
44+
TypesenseSinkOptions.HOSTS,
45+
TypesenseSinkOptions.COLLECTION,
46+
TypesenseSinkOptions.APIKEY,
47+
TypesenseSinkOptions.SCHEMA_SAVE_MODE,
48+
TypesenseSinkOptions.DATA_SAVE_MODE)
49+
.optional(TypesenseSinkOptions.PRIMARY_KEYS)
50+
.optional(TypesenseSinkOptions.KEY_DELIMITER)
51+
.optional(TypesenseSinkOptions.MAX_BATCH_SIZE)
52+
.optional(TypesenseSinkOptions.MAX_RETRY_COUNT)
5653
.build();
5754
}
5855

5956
@Override
6057
public TableSink createSink(TableSinkFactoryContext context) {
6158
ReadonlyConfig readonlyConfig = context.getOptions();
62-
String original = readonlyConfig.get(COLLECTION);
59+
String original = readonlyConfig.get(TypesenseSinkOptions.COLLECTION);
6360
CatalogTable newTable =
6461
CatalogTable.of(
6562
TableIdentifier.of(

0 commit comments

Comments
 (0)