Merge branch 'dev' into rabbitmq_connector

docs/en/connector-v2/Error-Quick-Reference-Manual.md

@@ -66,4 +66,12 @@ This document records some common error codes and corresponding solutions of Sea
 | RABBITMQ-06 | messages could not be acknowledged with basicReject           | When users encounter this error code, it means that job has some problems, please check it whether is work well |
 | RABBITMQ-07 | parse uri failed                                              | When users encounter this error code, it means that rabbitmq connect uri incorrect, please check it             |
 | RABBITMQ-08 | initialize ssl context failed                                 | When users encounter this error code, it means that rabbitmq has some problems, please check it whether is work |
-| RABBITMQ-09 | setup ssl factory failed                                      | When users encounter this error code, it means that rabbitmq has some problems, please check it whether is work |
+| RABBITMQ-09 | setup ssl factory failed                                      | When users encounter this error code, it means that rabbitmq has some problems, please check it whether is work |
+
+## Socket Connector Error Codes
+
+| code      | description                                              | solution                                                                                                                       |
+|-----------|----------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------|
+| SOCKET-01 | Cannot connect to socket server                          | When the user encounters this error code, it means that the connection address may not match, please check                     |
+| SOCKET-02 | Failed to send message to socket server                  | When the user encounters this error code, it means that there is a problem sending data and retry is not enabled, please check |
+| SOCKET-03 | Unable to write; interrupted while doing another attempt | When the user encounters this error code, it means that the data writing is interrupted abnormally, please check               |

docs/en/connector-v2/source/Jira.md

@@ -0,0 +1,160 @@
+# Jira
+
+> Jira source connector
+
+## Description
+
+Used to read data from Jira.
+
+## Key features
+
+- [x] [batch](../../concept/connector-v2-features.md)
+- [ ] [stream](../../concept/connector-v2-features.md)
+- [ ] [exactly-once](../../concept/connector-v2-features.md)
+- [x] [schema projection](../../concept/connector-v2-features.md)
+- [ ] [parallelism](../../concept/connector-v2-features.md)
+- [ ] [support user-defined split](../../concept/connector-v2-features.md)
+
+##  Options
+
+| name                        | type   | required | default value |
+| --------------------------- | ------ | -------- | ------------- |
+| url                         | String | Yes      | -             |
+| email                       | String | Yes      | -             |
+| api_token                   | String | Yes      | -             |
+| method                      | String | No       | get           |
+| schema.fields               | Config | No       | -             |
+| format                      | String | No       | json          |
+| params                      | Map    | No       | -             |
+| body                        | String | No       | -             |
+| poll_interval_ms            | int    | No       | -             |
+| retry                       | int    | No       | -             |
+| retry_backoff_multiplier_ms | int    | No       | 100           |
+| retry_backoff_max_ms        | int    | No       | 10000         |
+| common-options              | config | No       | -             |
+
+### url [String]
+
+http request url
+
+### email [String]
+
+Jira Email
+
+### api_token [String]
+
+Jira API Token
+
+https://id.atlassian.com/manage-profile/security/api-tokens
+
+### method [String]
+
+http request method, only supports GET, POST method
+
+### params [Map]
+
+http params
+
+### body [String]
+
+http body
+
+### poll_interval_ms [int]
+
+request http api interval(millis) in stream mode
+
+### retry [int]
+
+The max retry times if request http return to `IOException`
+
+### retry_backoff_multiplier_ms [int]
+
+The retry-backoff times(millis) multiplier if request http failed
+
+### retry_backoff_max_ms [int]
+
+The maximum retry-backoff times(millis) if request http failed
+
+### format [String]
+
+the format of upstream data, now only support `json` `text`, default `json`.
+
+when you assign format is `json`, you should also assign schema option, for example:
+
+upstream data is the following:
+
+```json
+
+{"code":  200, "data":  "get success", "success":  true}
+
+```
+
+you should assign schema as the following:
+
+```hocon
+
+schema {
+    fields {
+        code = int
+        data = string
+        success = boolean
+    }
+}
+
+```
+
+connector will generate data as the following:
+
+| code | data        | success |
+|------|-------------|---------|
+| 200  | get success | true    |
+
+when you assign format is `text`, connector will do nothing for upstream data, for example:
+
+upstream data is the following:
+
+```json
+
+{"code":  200, "data":  "get success", "success":  true}
+
+```
+
+connector will generate data as the following:
+
+| content |
+|---------|
+| {"code":  200, "data":  "get success", "success":  true}        |
+
+### schema [Config]
+
+#### fields [Config]
+
+the schema fields of upstream data
+
+### common options 
+
+Source plugin common parameters, please refer to [Source Common Options](common-options.md) for details
+
+## Example
+
+```hocon
+Jira {
+    url = "https://liugddx.atlassian.net/rest/api/3/search"
+    email = "test@test.com"
+    api_token = "xxx" 
+    schema {
+       fields {
+         expand = string
+         startAt = bigint
+         maxResults = int
+         total = int
+       }
+    }
+}
+```
+
+## Changelog
+
+### next version
+
+- Add Jira Source Connector

docs/en/faq.md

@@ -345,7 +345,7 @@ Just configure hdfs-site.xml properly, refer to: https://www.cnblogs.com/suanec/
 
 ## I want to learn the source code of SeaTunnel, where should I start?
 
-SeaTunnel has a completely abstract and structured code implementation, and many people have chosen  SeaTunnel As a way to learn Spark, you can learn the source code from the main program entry: [Seatunnel.java](https://github.com/apache/incubator-seatunnel/blob/72b57b22688f17376fe7e5cf522b4bdd3f62cce0/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/Seatunnel.java)
+SeaTunnel has a completely abstract and structured code implementation, and many people have chosen  SeaTunnel As a way to learn Spark, you can learn the source code from the main program entry: Seatunnel.java
 
 ## When SeaTunnel developers develop their own plugins, do they need to understand the SeaTunnel code? Should these code integrated into the SeaTunnel project?
 

plugin-mapping.properties

@@ -152,6 +152,7 @@ seatunnel.source.Lemlist = connector-http-lemlist
 seatunnel.source.Klaviyo = connector-http-klaviyo
 seatunnel.sink.Slack = connector-slack
 seatunnel.source.OneSignal = connector-http-onesignal
+seatunnel.source.Jira = connector-http-jira
 seatunnel.source.Gitlab = connector-http-gitlab
 seatunnel.sink.RabbitMQ = connector-rabbitmq
 seatunnel.source.RabbitMQ = connector-rabbitmq

pom.xml

@@ -113,7 +113,7 @@
 
     <properties>
         <!--todo The classification is too confusing, reclassify by type-->
-        <revision>2.1.3-SNAPSHOT</revision>
+        <revision>2.3.1-SNAPSHOT</revision>
         <seatunnel.config.shade.version>2.1.1</seatunnel.config.shade.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <java.version>1.8</java.version>

seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigValidator.java

@@ -22,10 +22,9 @@
 import org.apache.seatunnel.api.configuration.Option;
 import org.apache.seatunnel.api.configuration.ReadonlyConfig;
 
-import java.util.HashSet;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
-import java.util.Set;
 
 public class ConfigValidator {
     private final ReadonlyConfig config;
@@ -65,8 +64,8 @@ void validate(RequiredOption requiredOption) {
         throw new UnsupportedOperationException(String.format("This type option(%s) of validation is not supported", requiredOption.getClass()));
     }
 
-    private Set<Option<?>> getAbsentOptions(Set<Option<?>> requiredOption) {
-        Set<Option<?>> absent = new HashSet<>();
+    private List<Option<?>> getAbsentOptions(List<Option<?>> requiredOption) {
+        List<Option<?>> absent = new ArrayList<>();
         for (Option<?> option : requiredOption) {
             if (!hasOption(option)) {
                 absent.add(option);
@@ -76,7 +75,7 @@ private Set<Option<?>> getAbsentOptions(Set<Option<?>> requiredOption) {
     }
 
     void validate(RequiredOption.AbsolutelyRequiredOptions requiredOption) {
-        Set<Option<?>> absentOptions = getAbsentOptions(requiredOption.getRequiredOption());
+        List<Option<?>> absentOptions = getAbsentOptions(requiredOption.getRequiredOption());
         if (absentOptions.size() == 0) {
             return;
         }
@@ -88,9 +87,9 @@ boolean hasOption(Option<?> option) {
     }
 
     boolean validate(RequiredOption.BundledRequiredOptions bundledRequiredOptions) {
-        Set<Option<?>> bundledOptions = bundledRequiredOptions.getRequiredOption();
-        Set<Option<?>> present = new HashSet<>();
-        Set<Option<?>> absent = new HashSet<>();
+        List<Option<?>> bundledOptions = bundledRequiredOptions.getRequiredOption();
+        List<Option<?>> present = new ArrayList<>();
+        List<Option<?>> absent = new ArrayList<>();
         for (Option<?> option : bundledOptions) {
             if (hasOption(option)) {
                 present.add(option);
@@ -109,8 +108,8 @@ boolean validate(RequiredOption.BundledRequiredOptions bundledRequiredOptions) {
     }
 
     void validate(RequiredOption.ExclusiveRequiredOptions exclusiveRequiredOptions) {
-        Set<RequiredOption.BundledRequiredOptions> presentBundledRequiredOptions = new HashSet<>();
-        Set<Option<?>> presentOptions = new HashSet<>();
+        List<RequiredOption.BundledRequiredOptions> presentBundledRequiredOptions = new ArrayList<>();
+        List<Option<?>> presentOptions = new ArrayList<>();
         for (RequiredOption.BundledRequiredOptions bundledOptions : exclusiveRequiredOptions.getExclusiveBundledOptions()) {
             if (validate(bundledOptions)) {
                 presentBundledRequiredOptions.add(bundledOptions);
@@ -142,7 +141,7 @@ void validate(RequiredOption.ConditionalRequiredOptions conditionalRequiredOptio
         if (!match) {
             return;
         }
-        Set<Option<?>> absentOptions = getAbsentOptions(conditionalRequiredOptions.getRequiredOption());
+        List<Option<?>> absentOptions = getAbsentOptions(conditionalRequiredOptions.getRequiredOption());
         if (absentOptions.size() == 0) {
             return;
         }

seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/OptionRule.java

@@ -21,7 +21,6 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
 
@@ -185,7 +184,7 @@ public Builder conditional(Expression expression, Option<?>... requiredOptions)
                 verifyRequiredOptionDefaultValue(o);
             }
             this.requiredOptions.add(RequiredOption.ConditionalRequiredOptions.of(expression,
-                new HashSet<>(Arrays.asList(requiredOptions))));
+                new ArrayList<>(Arrays.asList(requiredOptions))));
             return this;
         }
 

seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/OptionUtil.java

@@ -26,16 +26,14 @@
 import java.lang.reflect.Type;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 
 public class OptionUtil {
 
     private OptionUtil() {
     }
 
-    public static String getOptionKeys(Set<Option<?>> options) {
+    public static String getOptionKeys(List<Option<?>> options) {
         StringBuilder builder = new StringBuilder();
         boolean flag = false;
         for (Option<?> option : options) {
@@ -50,17 +48,18 @@ public static String getOptionKeys(Set<Option<?>> options) {
         return builder.toString();
     }
 
-    public static String getOptionKeys(Set<Option<?>> options, Set<RequiredOption.BundledRequiredOptions> bundledOptions) {
-        Set<Set<Option<?>>> optionSets = new HashSet<>();
+    public static String getOptionKeys(List<Option<?>> options,
+                                       List<RequiredOption.BundledRequiredOptions> bundledOptions) {
+        List<List<Option<?>>> optionList = new ArrayList<>();
         for (Option<?> option : options) {
-            optionSets.add(Collections.singleton(option));
+            optionList.add(Collections.singletonList(option));
         }
         for (RequiredOption.BundledRequiredOptions bundledOption : bundledOptions) {
-            optionSets.add(bundledOption.getRequiredOption());
+            optionList.add(bundledOption.getRequiredOption());
         }
         boolean flag = false;
         StringBuilder builder = new StringBuilder();
-        for (Set<Option<?>> optionSet : optionSets) {
+        for (List<Option<?>> optionSet : optionList) {
             if (flag) {
                 builder.append(", ");
             }
@@ -80,7 +79,8 @@ public static List<Option<?>> getOptions(Class<?> clazz) throws InstantiationExc
             field.setAccessible(true);
             OptionMark option = field.getAnnotation(OptionMark.class);
             if (option != null) {
-                options.add(new Option<>(!StringUtils.isNotBlank(option.name()) ? formatUnderScoreCase(field.getName()) : option.name(),
+                options.add(new Option<>(
+                    !StringUtils.isNotBlank(option.name()) ? formatUnderScoreCase(field.getName()) : option.name(),
                     new TypeReference<Object>() {
                         @Override
                         public Type getType() {