Skip to content

Commit

Permalink
Merge pull request Acxiom#366 from dafreels/metalus_2_0_0
Browse files Browse the repository at this point in the history 
Work on getting gcp project building
  • Loading branch information
@dafreels
dafreels committed Mar 15, 2023
2 parents fdad751 + 7093d14 commit fab6416
Show file tree
Hide file tree
Showing 24 changed files with 867 additions and 747 deletions.
3 changes: 2 additions & 1 deletion metalus-aws/src/main/resources/metadata/steps/aws-mappings.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
"alternativeStepCommands": [
{
"stepId": "8bcea87d-9574-4187-b539-b9bc2639b6bb8bcea87d-9574-4187-b539-b9bc2639b6bb",
"command": "LambdaSteps.executeLambda"
"command": "LambdaSteps.executeLambda",
"pkg": "com.acxiom.metalus.aws.steps"
}
]
}
2 changes: 1 addition & 1 deletion metalus-aws/src/main/scala/com/acxiom/metalus/aws/steps/LambdaSteps.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ object LambdaSteps {
-1 indicates a failure and stdErr should be reviewed for more information. This step will automatically
retry calls if non-code related issues occur. To adjust the retry behavior, include a RetryPolicy in
a global named awsRetryPolicy.""",
"Pipeline", "Utilities", List[String]("batch"))
"Pipeline", "Utilities", List[String]("batch", "aws"))
@StepParameters(Map("command" -> StepParameter(None, Some(true), None, None, None, None, Some("The Command object containing the execution information"))))
@StepResults(primaryType = "Int", secondaryTypes = Some(Map("stdOut" -> "String", "stdErr" -> "String")))
def executeLambda(command: Command, pipelineContext: PipelineContext): PipelineStepResponse = {
Expand Down
100 changes: 82 additions & 18 deletions metalus-gcp/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,17 @@
<description>Metalus GCP Step Library</description>
<inceptionYear>2018</inceptionYear>

<!-- <dependencyManagement>-->
<!-- <dependencies>-->
<!-- <dependency>-->
<!-- <groupId>com.google.cloud</groupId>-->
<!-- <artifactId>libraries-bom</artifactId>-->
<!-- <version>5.7.0</version>-->
<!-- <type>pom</type>-->
<!-- <scope>import</scope>-->
<!-- </dependency>-->
<!-- </dependencies>-->
<!-- </dependencyManagement>-->
<!-- <dependencyManagement>-->
<!-- <dependencies>-->
<!-- <dependency>-->
<!-- <groupId>com.google.cloud</groupId>-->
<!-- <artifactId>libraries-bom</artifactId>-->
<!-- <version>5.7.0</version>-->
<!-- <type>pom</type>-->
<!-- <scope>import</scope>-->
<!-- </dependency>-->
<!-- </dependencies>-->
<!-- </dependencyManagement>-->

<dependencies>
<dependency>
Expand All @@ -27,12 +27,62 @@
<version>${version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>2.0.6</version>
</dependency>
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-collection-compat_${scala.compat.version}</artifactId>
<version>2.9.0</version>
</dependency>
<!-- Google Libraries -->
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-storage</artifactId>
<version>2.20.1</version>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-secretmanager</artifactId>
<version>2.12.0</version>
<exclusions>
<exclusion>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty-shaded</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-pubsub</artifactId>
<version>1.123.5</version>
<exclusions>
<exclusion>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty-shaded</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-functions</artifactId>
<version>2.14.0</version>
</dependency>
<!-- Test -->
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.compat.version}</artifactId>
<version>3.2.9</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-nio</artifactId>
<version>0.126.8</version>
<scope>test</scope>
</dependency>
<!-- <dependency>-->
<!-- <groupId>io.grpc</groupId>-->
<!-- <artifactId>grpc-netty-shaded</artifactId>-->
Expand Down Expand Up @@ -157,8 +207,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>7</source>
<target>7</target>
<source>8</source>
<target>8</target>
</configuration>
</plugin>
<plugin>
Expand All @@ -176,6 +226,8 @@
<exclude>com.fasterxml.jackson.core:*:*:</exclude>
<exclude>org.apache.maven:lib:tests</exclude>
<exclude>org.apache.kafka:*:jar:</exclude>
<exclude>org.apache.commons:*:jar:</exclude>
<exclude>org.apache.httpcomponents:*:jar:</exclude>
<exclude>org.joda:*:jar:</exclude>
<exclude>joda-time:*:jar:</exclude>
<exclude>log4j:log4j:jar:</exclude>
Expand All @@ -188,6 +240,8 @@
<exclude>com.acxiom.pipeline:*:jar:</exclude>
<exclude>org.json4s:*:jar:</exclude>
<exclude>com.jcraft:*:jar:</exclude>
<exclude>commons-codec:*:jar:</exclude>
<exclude>javax.annotation:*:jar:</exclude>
</excludes>
</artifactSet>
<filters>
Expand All @@ -197,6 +251,8 @@
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>com/google/api/client/testing/**</exclude>
<exclude>com/google/api/client/googleapis/testing/**</exclude>
</excludes>
</filter>
</filters>
Expand All @@ -212,11 +268,19 @@
<relocation>
<pattern>com.google</pattern>
<shadedPattern>metalus.com.google</shadedPattern>
<excludes>
<exclude>com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem</exclude>
<exclude>com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS</exclude>
<exclude>com.google.cloud.spark.bigquery.DefaultSource</exclude>
</excludes>
<!-- <excludes>-->
<!-- <exclude>com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem</exclude>-->
<!-- <exclude>com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS</exclude>-->
<!-- <exclude>com.google.cloud.spark.bigquery.DefaultSource</exclude>-->
<!-- </excludes>-->
</relocation>
<relocation>
<pattern>io.opencensus</pattern>
<shadedPattern>metalus.io.opencensus</shadedPattern>
</relocation>
<relocation>
<pattern>io.grpc</pattern>
<shadedPattern>metalus.io.grpc</shadedPattern>
</relocation>
</relocations>
</configuration>
Expand Down
Loading

0 comments on commit fab6416

Please sign in to comment.