Skip to content

Commit

Permalink
Merge a8b23ed into 7e8a455
Browse files Browse the repository at this point in the history
  • Loading branch information
istreeter committed Mar 22, 2023
2 parents 7e8a455 + a8b23ed commit f667114
Show file tree
Hide file tree
Showing 30 changed files with 688 additions and 427 deletions.
31 changes: 15 additions & 16 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -75,56 +75,55 @@ lazy val loader = project
.settings(addCompilerPlugin(Dependencies.betterMonadicFor))
.settings(libraryDependencies ++= Dependencies.loaderDependencies)
.dependsOn(aws % "compile->compile;test->test;runtime->runtime", gcp % "compile->compile;test->test")
.enablePlugins(JavaAppPackaging, DockerPlugin, BuildInfoPlugin)

lazy val redshiftLoader = project
.in(file("modules/redshift-loader"))
.settings(BuildSettings.redshiftBuildSettings)
.settings(addCompilerPlugin(Dependencies.betterMonadicFor))
.settings(libraryDependencies ++= Dependencies.redshiftDependencies)
.dependsOn(loader % "compile->compile;test->test;runtime->runtime")
.enablePlugins(JavaAppPackaging, DockerPlugin, BuildInfoPlugin)
.enablePlugins(JavaAppPackaging, SnowplowDockerPlugin, BuildInfoPlugin)

lazy val redshiftLoaderDistroless = project
.in(file("modules/distroless/redshift-loader"))
.settings(sourceDirectory := (redshiftLoader / sourceDirectory).value)
.settings(BuildSettings.redshiftDistrolessBuildSettings)
.settings(BuildSettings.redshiftBuildSettings)
.settings(addCompilerPlugin(Dependencies.betterMonadicFor))
.settings(libraryDependencies ++= Dependencies.redshiftDependencies)
.dependsOn(loader % "compile->compile;test->test;runtime->runtime")
.enablePlugins(JavaAppPackaging, DockerPlugin, BuildInfoPlugin, LauncherJarPlugin)
.enablePlugins(JavaAppPackaging, SnowplowDistrolessDockerPlugin, BuildInfoPlugin)

lazy val snowflakeLoader = project
.in(file("modules/snowflake-loader"))
.settings(BuildSettings.snowflakeBuildSettings)
.settings(addCompilerPlugin(Dependencies.betterMonadicFor))
.settings(libraryDependencies ++= Dependencies.snowflakeDependencies)
.dependsOn(common % "compile->compile;test->test",loader % "compile->compile;test->test;runtime->runtime")
.enablePlugins(JavaAppPackaging, DockerPlugin, BuildInfoPlugin)
.enablePlugins(JavaAppPackaging, SnowplowDockerPlugin, BuildInfoPlugin)

lazy val snowflakeLoaderDistroless = project
.in(file("modules/distroless/snowflake-loader"))
.settings(sourceDirectory := (snowflakeLoader / sourceDirectory).value)
.settings(BuildSettings.snowflakeDistrolessBuildSettings)
.settings(BuildSettings.snowflakeBuildSettings)
.settings(addCompilerPlugin(Dependencies.betterMonadicFor))
.settings(libraryDependencies ++= Dependencies.snowflakeDependencies)
.dependsOn(loader % "compile->compile;test->test;runtime->runtime")
.enablePlugins(JavaAppPackaging, DockerPlugin, BuildInfoPlugin, LauncherJarPlugin)
.enablePlugins(JavaAppPackaging, SnowplowDistrolessDockerPlugin, BuildInfoPlugin)

lazy val databricksLoader = project
.in(file("modules/databricks-loader"))
.settings(BuildSettings.databricksBuildSettings)
.settings(addCompilerPlugin(Dependencies.betterMonadicFor))
.dependsOn(loader % "compile->compile;test->test;runtime->runtime")
.enablePlugins(JavaAppPackaging, DockerPlugin, BuildInfoPlugin)
.enablePlugins(JavaAppPackaging, SnowplowDockerPlugin, BuildInfoPlugin)

lazy val databricksLoaderDistroless = project
.in(file("modules/distroless/databricks-loader"))
.settings(sourceDirectory := (databricksLoader / sourceDirectory).value)
.settings(BuildSettings.databricksDistrolessBuildSettings)
.settings(BuildSettings.databricksBuildSettings)
.settings(addCompilerPlugin(Dependencies.betterMonadicFor))
.dependsOn(loader % "compile->compile;test->test;runtime->runtime")
.enablePlugins(JavaAppPackaging, DockerPlugin, BuildInfoPlugin, LauncherJarPlugin)
.enablePlugins(JavaAppPackaging, SnowplowDistrolessDockerPlugin, BuildInfoPlugin)

lazy val transformerBatch = project
.in(file("modules/transformer-batch"))
Expand All @@ -140,17 +139,17 @@ lazy val transformerKinesis = project
.settings(libraryDependencies ++= Dependencies.transformerKinesisDependencies)
.settings(excludeDependencies ++= Dependencies.commonStreamTransformerExclusions)
.dependsOn(commonTransformerStream % "compile->compile;test->test;runtime->runtime", aws % "compile->compile;test->test;runtime->runtime")
.enablePlugins(JavaAppPackaging, DockerPlugin, BuildInfoPlugin)
.enablePlugins(JavaAppPackaging, SnowplowDockerPlugin, BuildInfoPlugin)

lazy val transformerKinesisDistroless = project
.in(file("modules/distroless/transformer-kinesis"))
.settings(sourceDirectory := (transformerKinesis / sourceDirectory).value)
.settings(BuildSettings.transformerKinesisDistrolessBuildSettings)
.settings(BuildSettings.transformerKinesisBuildSettings)
.settings(addCompilerPlugin(Dependencies.betterMonadicFor))
.settings(libraryDependencies ++= Dependencies.transformerKinesisDependencies)
.settings(excludeDependencies ++= Dependencies.commonStreamTransformerExclusions)
.dependsOn(commonTransformerStream % "compile->compile;test->test;runtime->runtime", aws % "compile->compile;test->test;runtime->runtime")
.enablePlugins(JavaAppPackaging, DockerPlugin, BuildInfoPlugin, LauncherJarPlugin)
.enablePlugins(JavaAppPackaging, SnowplowDistrolessDockerPlugin, BuildInfoPlugin)

lazy val transformerPubsub = project
.in(file("modules/transformer-pubsub"))
Expand All @@ -159,14 +158,14 @@ lazy val transformerPubsub = project
.settings(libraryDependencies ++= Dependencies.transformerPubsubDependencies)
.settings(excludeDependencies ++= Dependencies.commonStreamTransformerExclusions)
.dependsOn(commonTransformerStream % "compile->compile;test->test;runtime->runtime", gcp % "compile->compile;test->test")
.enablePlugins(JavaAppPackaging, DockerPlugin, BuildInfoPlugin)
.enablePlugins(JavaAppPackaging, SnowplowDockerPlugin, BuildInfoPlugin)

lazy val transformerPubsubDistroless = project
.in(file("modules/distroless/transformer-pubsub"))
.settings(sourceDirectory := (transformerPubsub / sourceDirectory).value)
.settings(BuildSettings.transformerPubsubDistrolessBuildSettings)
.settings(BuildSettings.transformerPubsubBuildSettings)
.settings(addCompilerPlugin(Dependencies.betterMonadicFor))
.settings(libraryDependencies ++= Dependencies.transformerPubsubDependencies)
.settings(excludeDependencies ++= Dependencies.commonStreamTransformerExclusions)
.dependsOn(commonTransformerStream % "compile->compile;test->test;runtime->runtime", gcp % "compile->compile;test->test")
.enablePlugins(JavaAppPackaging, DockerPlugin, BuildInfoPlugin, LauncherJarPlugin)
.enablePlugins(JavaAppPackaging, SnowplowDistrolessDockerPlugin, BuildInfoPlugin)
6 changes: 4 additions & 2 deletions modules/loader/src/main/resources/application.conf
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@
},
"readyCheck": {
"backoff": "15 seconds",
"strategy": "CONSTANT"
"strategy": "CONSTANT",
"cumulativeBound": "10 minutes"
},
"initRetries": {
"backoff": "30 seconds",
Expand All @@ -33,7 +34,8 @@
"loading": "45 minutes",
"nonLoading": "10 minutes",
"sqsVisibility": "5 minutes",
"rollbackCommit": "20 minutes"
"rollbackCommit": "20 minutes",
"rollbackConnectionValidation": "1 minute"
},
"featureFlags": {
"addLoadTstampColumn": true
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/*
* Copyright (c) 2012-2023 Snowplow Analytics Ltd. All rights reserved.
*
* This program is licensed to you under the Apache License Version 2.0,
* and you may not use this file except in compliance with the Apache License Version 2.0.
* You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the Apache License Version 2.0 is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
*/
package com.snowplowanalytics.snowplow.rdbloader

import cats.Show
import cats.implicits._

/**
* An exception that is compatible with how use the monitoring webhook.
*
* If the app catches this type of exception, then it must either:
* - Send an alert, and then handle the exception
* - Re-throw the exception, so it can be handled higher up the stack
*
* This is the only exception for which it is acceptable to send a monitoring webhook. Other caught
* exceptions should go to sentry only, as they do provide helpful detail for the pipeline operator.
*/
class AlertableFatalException(prefix: AlertableFatalException.Explanation, cause: Throwable) extends Exception(prefix.show, cause) {

/**
* The message that appears in the alert sent to the webhook.
*
* The message comprises:
* - A controlled statement saying where the error occurred.
* - An un-controlled message taken from the underlying exception.
*/
def alertMessage: String = {
val suffix = cause match {
case ae: AlertableFatalException => ae.alertMessage
case other =>
Option(other.getMessage).getOrElse("")
}
show"$prefix: $suffix"
}
}

object AlertableFatalException {
sealed trait Explanation

/**
* A controlled set of terms for which the pipeline operator should receive an error alert.
*
* Items should appear on this list if it requies action on the warehouse to fix the problem. For
* example, by fixing permissions or credentials.
*/
object Explanation {
case object ManifestInit extends Explanation
case object InitialConnection extends Explanation
case object EventsTableInit extends Explanation
case object TargetQueryInit extends Explanation

implicit val show: Show[Explanation] =
Show.show {
case ManifestInit => "Could not initialize manifest table"
case EventsTableInit => "Could not initialize events table"
case InitialConnection => "Could not get connection to warehouse during startup"
case TargetQueryInit => "Could not acquire information necessary for startup"
}
}
}
Loading

0 comments on commit f667114

Please sign in to comment.