Skip to content

Commit 2723516

Browse files
fhussonnoisfhussonnois
committed
refactor(build): add mvn profiles for different storages
1 parent e9462c8 commit 2723516

File tree

5 files changed

+122
-42
lines changed

5 files changed

+122
-42
lines changed

‎Makefile

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
66

77
REPOSITORY = streamthoughts
88
IMAGE = kafka-connect-file-pulse
9+
PROFILES = gcs aws azure local
910

1011
.SILENT:
1112

@@ -26,17 +27,42 @@ clean-build:
2627
rm -rf ./docker-build;
2728

2829
build-images:
29-
echo "Building image \n========================================== ";
30-
echo "CONNECT_VERSION="$(CONNECT_VERSION)
31-
echo "GIT_COMMIT="$(GIT_COMMIT)
32-
echo "GIT_BRANCH="$(GIT_BRANCH)
33-
echo "==========================================\n "
34-
mvn clean package -q && \
30+
echo "Building Docker images \n========================================== ";
31+
echo "CONNECT_VERSION="$(CONNECT_VERSION);
32+
echo "GIT_COMMIT="$(GIT_COMMIT);
33+
echo "GIT_BRANCH="$(GIT_BRANCH);
34+
echo "==========================================\n ";
35+
./mvnw clean package -B -Dmaven.test.skip=true -P one-for-all && \
3536
docker build \
3637
--build-arg connectFilePulseVersion=${CONNECT_VERSION} \
3738
--build-arg connectFilePulseCommit=${GIT_COMMIT} \
3839
--build-arg connectFilePulseBranch=${GIT_BRANCH} \
3940
-t ${REPOSITORY}/${IMAGE}:latest . || exit 1 ;
4041
docker tag ${REPOSITORY}/${IMAGE}:latest ${REPOSITORY}/${IMAGE}:${CONNECT_VERSION} || exit 1 ;
4142

43+
for PROFILE in $(PROFILES); do\
44+
echo "Building Docker images \n========================================== ";\
45+
echo "PROFILE=$$PROFILE";\
46+
echo "==========================================\n ";\
47+
./mvnw clean package -B -Dmaven.test.skip=true -P $$PROFILE && \
48+
docker build \
49+
--build-arg connectFilePulseVersion=${CONNECT_VERSION} \
50+
--build-arg connectFilePulseCommit=${GIT_COMMIT} \
51+
--build-arg connectFilePulseBranch=${GIT_BRANCH} \
52+
-t ${REPOSITORY}/${IMAGE}:latest-$$PROFILE . || exit 1 ; \
53+
docker tag ${REPOSITORY}/${IMAGE}:latest-$$PROFILE ${REPOSITORY}/${IMAGE}:${CONNECT_VERSION}-$$PROFILE || exit 1 ;\
54+
done
55+
56+
push-images:
57+
echo "Pushing Docker images \n========================================== ";
58+
echo "CONNECT_VERSION="$(CONNECT_VERSION);
59+
echo "GIT_COMMIT="$(GIT_COMMIT);
60+
echo "GIT_BRANCH="$(GIT_BRANCH);
61+
echo "==========================================\n ";
62+
docker push ${REPOSITORY}/${IMAGE}:latest ${REPOSITORY}/${IMAGE}:${CONNECT_VERSION} || exit 1 ;
63+
docker push ${REPOSITORY}/${IMAGE}:latest ${REPOSITORY}/${IMAGE}:latest || exit 1 ;
64+
for PROFILE in $(PROFILES); do\
65+
docker push ${REPOSITORY}/${IMAGE}:${CONNECT_VERSION}-$$PROFILE || exit 1 ;\
66+
done
67+
4268
clean: clean-containers clean-images clean-build

‎connect-file-pulse-plugin/pom.xml

Lines changed: 88 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,16 @@
179179
</plugins>
180180
</build>
181181

182+
<dependencyManagement>
183+
<dependencies>
184+
<dependency>
185+
<groupId>org.apache.httpcomponents</groupId>
186+
<artifactId>httpclient</artifactId>
187+
<version>4.5.13</version>
188+
</dependency>
189+
</dependencies>
190+
</dependencyManagement>
191+
182192
<profiles>
183193
<profile>
184194
<id>standalone</id>
@@ -195,65 +205,105 @@
195205
</plugins>
196206
</build>
197207
</profile>
208+
<profile>
209+
<id>one-for-all</id>
210+
<activation>
211+
<activeByDefault>true</activeByDefault>
212+
</activation>
213+
<dependencies>
214+
<dependency>
215+
<groupId>io.streamthoughts</groupId>
216+
<artifactId>kafka-connect-filepulse-amazons3-fs</artifactId>
217+
<version>${project.version}</version>
218+
</dependency>
219+
<dependency>
220+
<groupId>io.streamthoughts</groupId>
221+
<artifactId>kafka-connect-filepulse-azure-storage-fs</artifactId>
222+
<version>${project.version}</version>
223+
</dependency>
224+
<dependency>
225+
<groupId>io.streamthoughts</groupId>
226+
<artifactId>kafka-connect-filepulse-google-cloud-storage-fs</artifactId>
227+
<version>${project.version}</version>
228+
</dependency>
229+
<dependency>
230+
<groupId>io.streamthoughts</groupId>
231+
<artifactId>kafka-connect-filepulse-local-fs</artifactId>
232+
<version>${project.version}</version>
233+
</dependency>
234+
</dependencies>
235+
</profile>
236+
<profile>
237+
<id>aws</id>
238+
<dependencies>
239+
<dependency>
240+
<groupId>io.streamthoughts</groupId>
241+
<artifactId>kafka-connect-filepulse-amazons3-fs</artifactId>
242+
<version>${project.version}</version>
243+
</dependency>
244+
</dependencies>
245+
</profile>
246+
<profile>
247+
<id>gcs</id>
248+
<dependencies>
249+
<dependency>
250+
<groupId>io.streamthoughts</groupId>
251+
<artifactId>kafka-connect-filepulse-google-cloud-storage-fs</artifactId>
252+
<version>${project.version}</version>
253+
</dependency>
254+
</dependencies>
255+
</profile>
256+
<profile>
257+
<id>azure</id>
258+
<dependencies>
259+
<dependency>
260+
<groupId>io.streamthoughts</groupId>
261+
<artifactId>kafka-connect-filepulse-azure-storage-fs</artifactId>
262+
<version>${project.version}</version>
263+
</dependency>
264+
</dependencies>
265+
</profile>
266+
<profile>
267+
<id>local</id>
268+
<dependencies>
269+
<dependency>
270+
<groupId>io.streamthoughts</groupId>
271+
<artifactId>kafka-connect-filepulse-local-fs</artifactId>
272+
<version>${project.version}</version>
273+
</dependency>
274+
</dependencies>
275+
</profile>
198276
</profiles>
199-
200-
<dependencyManagement>
201-
<dependencies>
202-
<dependency>
203-
<groupId>org.apache.httpcomponents</groupId>
204-
<artifactId>httpclient</artifactId>
205-
<version>4.5.13</version>
206-
</dependency>
207-
</dependencies>
208-
</dependencyManagement>
209-
210277
<dependencies>
211278
<dependency>
212279
<groupId>io.streamthoughts</groupId>
213280
<artifactId>kafka-connect-file-pulse-api</artifactId>
214281
<version>${project.version}</version>
215282
</dependency>
283+
216284
<dependency>
217285
<groupId>io.streamthoughts</groupId>
218286
<artifactId>kafka-connect-file-pulse-filters</artifactId>
219287
<version>${project.version}</version>
220288
</dependency>
221-
<dependency>
222-
<groupId>io.streamthoughts</groupId>
223-
<artifactId>kafka-connect-filepulse-local-fs</artifactId>
224-
<version>${project.version}</version>
225-
</dependency>
226-
<dependency>
227-
<groupId>io.streamthoughts</groupId>
228-
<artifactId>kafka-connect-filepulse-amazons3-fs</artifactId>
229-
<version>${project.version}</version>
230-
</dependency>
231-
<dependency>
232-
<groupId>io.streamthoughts</groupId>
233-
<artifactId>kafka-connect-filepulse-azure-storage-fs</artifactId>
234-
<version>${project.version}</version>
235-
</dependency>
236-
<dependency>
237-
<groupId>io.streamthoughts</groupId>
238-
<artifactId>kafka-connect-filepulse-google-cloud-storage-fs</artifactId>
239-
<version>${project.version}</version>
240-
</dependency>
289+
241290
<dependency>
242291
<groupId>org.apache.kafka</groupId>
243292
<artifactId>kafka-clients</artifactId>
244293
<scope>provided</scope>
245294
</dependency>
295+
246296
<dependency>
247297
<groupId>org.apache.kafka</groupId>
248298
<artifactId>connect-api</artifactId>
249299
<scope>provided</scope>
250300
</dependency>
301+
251302
<dependency>
252303
<groupId>commons-io</groupId>
253304
<artifactId>commons-io</artifactId>
254305
</dependency>
255306

256-
257307
<!-- START test dependencies-->
258308
<dependency>
259309
<groupId>junit</groupId>
@@ -283,6 +333,12 @@
283333
<groupId>io.rest-assured</groupId>
284334
<artifactId>json-path</artifactId>
285335
</dependency>
336+
<dependency>
337+
<groupId>io.streamthoughts</groupId>
338+
<artifactId>kafka-connect-filepulse-local-fs</artifactId>
339+
<version>${project.version}</version>
340+
<scope>test</scope>
341+
</dependency>
286342
<!-- END test dependencies-->
287343
</dependencies>
288344

‎connect-file-pulse-plugin/src/main/java/io/streamthoughts/kafka/connect/filepulse/config/CommonConfig.java

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
*/
1919
package io.streamthoughts.kafka.connect.filepulse.config;
2020

21-
import io.streamthoughts.kafka.connect.filepulse.fs.reader.LocalRowFileInputReader;
2221
import io.streamthoughts.kafka.connect.filepulse.offset.DefaultSourceOffsetPolicy;
2322
import io.streamthoughts.kafka.connect.filepulse.source.SourceOffsetPolicy;
2423
import io.streamthoughts.kafka.connect.filepulse.state.FileObjectStateBackingStore;
@@ -66,7 +65,6 @@ static ConfigDef getConf() {
6665
.define(
6766
TASKS_FILE_READER_CLASS_CONFIG,
6867
ConfigDef.Type.CLASS,
69-
LocalRowFileInputReader.class,
7068
ConfigDef.Importance.HIGH,
7169
TASKS_FILE_READER_CLASS_DOC,
7270
GROUP,

‎connect-file-pulse-plugin/src/main/java/io/streamthoughts/kafka/connect/filepulse/config/ConnectorConfig.java

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import io.streamthoughts.kafka.connect.filepulse.clean.FileCleanupPolicy;
2222
import io.streamthoughts.kafka.connect.filepulse.fs.FileSystemListing;
2323
import io.streamthoughts.kafka.connect.filepulse.fs.FileListFilter;
24-
import io.streamthoughts.kafka.connect.filepulse.fs.LocalFSDirectoryListing;
2524
import org.apache.kafka.common.config.ConfigDef;
2625

2726
import java.util.Collections;
@@ -66,7 +65,6 @@ public static ConfigDef getConf() {
6665
.define(
6766
FS_LISTING_CLASS_CONFIG,
6867
ConfigDef.Type.CLASS,
69-
LocalFSDirectoryListing.class,
7068
ConfigDef.Importance.HIGH,
7169
FS_LISTING_CLASS_DOC
7270
)

‎connect-file-pulse-plugin/src/test/java/io/streamthoughts/kafka/connect/filepulse/config/TaskConfigTest.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
import io.streamthoughts.kafka.connect.filepulse.filter.AppendFilter;
2222
import io.streamthoughts.kafka.connect.filepulse.filter.RecordFilter;
23+
import io.streamthoughts.kafka.connect.filepulse.fs.reader.LocalRowFileInputReader;
2324
import org.junit.Assert;
2425
import org.junit.Test;
2526

@@ -32,6 +33,7 @@ public class TaskConfigTest {
3233
public void should_configure_filter_given_on_failure() {
3334
final TaskConfig config = new TaskConfig(new HashMap<>() {{
3435
put(CommonConfig.OUTPUT_TOPIC_CONFIG, "Test");
36+
put(CommonConfig.TASKS_FILE_READER_CLASS_CONFIG, LocalRowFileInputReader.class.getName());
3537
put(TaskConfig.FILE_OBJECT_URIS_CONFIG, "/tmp");
3638
put(CommonConfig.FILTER_CONFIG, "Test");
3739
put(CommonConfig.FILTER_CONFIG + ".Test.type", AppendFilter.class.getName());

0 commit comments

Comments
 (0)