Skip to content

Commit

Permalink
Merge pull request #655 from atlanhq/DVX-385
Browse files Browse the repository at this point in the history
Adds secure credential-based object store support for S3, GCS, ADLS
  • Loading branch information
cmgrote committed May 24, 2024
2 parents c6aa89d + c62148f commit 97a8095
Show file tree
Hide file tree
Showing 25 changed files with 958 additions and 587 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ generated-packages/**
/typedef-toolkit/model/src/main/resources/META-INF/org/pkl/config/java/mapper/classes/
/package-toolkit/config/src/main/kotlin/com/atlan/pkg/Config.kt
/package-toolkit/config/src/main/kotlin/com/atlan/pkg/Connectors.kt
/package-toolkit/config/src/main/kotlin/com/atlan/pkg/Credential.kt
/package-toolkit/config/src/main/kotlin/com/atlan/pkg/Renderers.kt
/package-toolkit/config/src/main/resources/META-INF/org/pkl/config/java/mapper/classes/
/package-toolkit/config/src/main/resources/BuildInfo.pkl
Expand Down
2 changes: 2 additions & 0 deletions gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ jakarta-mail = "2.1.3"
angus-mail = "2.0.3"
pkl = "0.25.3"
adls = "12.19.0"
azure = "1.12.1"

[libraries]
jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson" }
Expand All @@ -44,6 +45,7 @@ gcs-bom = { module = "com.google.cloud:libraries-bom", version.ref = "gcs" }
gcs = { module = "com.google.cloud:google-cloud-storage" }
gcs-control = { module = "com.google.cloud:google-cloud-storage-control" }
adls = { module = "com.azure:azure-storage-file-datalake", version.ref = "adls" }
azure-identity = { module = "com.azure:azure-identity", version.ref = "azure" }
system-stubs = { module = "uk.org.webcompere:system-stubs-testng", version.ref = "system-stubs" }
fastcsv = { module = "de.siegmar:fastcsv", version.ref = "fastcsv" }
apache-poi = { module = "org.apache.poi:poi", version.ref = "poi" }
Expand Down
1 change: 1 addition & 0 deletions package-toolkit/config/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ pkl {
outputDir.set(layout.projectDirectory.dir("src/main"))
sourceModules.add(file("src/main/resources/Config.pkl"))
sourceModules.add(file("src/main/resources/Connectors.pkl"))
sourceModules.add(file("src/main/resources/Credential.pkl"))
sourceModules.add(file("src/main/resources/Renderers.pkl"))
}
}
Expand Down
134 changes: 1 addition & 133 deletions package-toolkit/config/src/main/resources/Config.pkl
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
/// | allowSchedule | | (Optional) Whether to allow the package to be scheduled (true) or only run immediately (false). | `true` |
/// | certified | | (Optional) Whether the package should be listed as certified (true) or not (false). | `true` |
/// | preview | | (Optional) Whether the package should be labeled as an early preview in the UI (true) or not (false). | `false` |
/// | credentialConfig | | (Optional) If the package will use a net-new type of credential, specify its configuration here. | |
/// | connectorType | | (Optional) If the package needs to configure a connector, specify its type here. | |
/// | category | | Name of the pill under which the package should be categorized in the marketplace in the UI. | `custom` |
@ModuleInfo { minPklVersion = "0.25.1" }
Expand Down Expand Up @@ -58,9 +57,6 @@ docsUrl: String
/// | **[rules][UIRule]** | | Listing of rules to control which inputs appear based on values selected in other inputs. |
uiConfig: UIConfig

/// (Optional) Configuration for any net-new connector credentials this custom package will use.
credentialConfig: CredentialConfig?

/// Coding language the package is implemented in.
/// This will control what (if any) strongly-typed configuration hand-over classes are generated by the toolkit.
/// (Note: if using 'Other', no strongly-typed configuration hand-over classes will be generated.)
Expand Down Expand Up @@ -237,131 +233,6 @@ class UIRule {
required: Listing<String>
}

/// Configuration for a new set of connector-specific credentials for the custom package.
///
/// | Variable | | Usage |
/// |---|---|---|
/// | **name** | | Name of this connector-specific credential configuration. |
/// | **source** | | Connector for which this credential configuration is applicable. |
/// | **icon** | | Icon to use for the credential configuration. |
/// | **helpdesk** | | Link to documentation for the credential configuration. |
/// | **logo** | | Logo to use for the credential configuration. |
/// | **[inputs][UIElement]** | | Mapping of common properties that exist across connection options, keyed by a unique variable name in lower_snake_case. |
/// | **[rules][UIRule]** | | Listing of rules to control which [inputs][UIElement] appear based on values selected in other inputs. |
class CredentialConfig {

/// Name of this connector-specific credential configuration.
hidden name: String

/// Connector for which this credential configuration is applicable.
hidden source: Connectors.Type

/// Icon to use for the credential configuration.
hidden icon: String

/// Link to documentation for the credential configuration.
hidden helpdesk: String

/// Logo to use for the credential configuration.
hidden logo: String

/// Default connector type to use for the credential configuration.
hidden connectorType: String = "jdbc"

/// TBC
hidden jdbcCredential: String = "{}"

/// TBC
hidden restCredential: String = "{}"

/// TBC
hidden odbcCredential: String = "{}"

/// TBC
hidden grpcCredential: String = "{}"

/// TBC
hidden restMetadata: String = ""

/// TBC
hidden restTransformer: String = ""

/// TBC
hidden sage: String?

/// TBC
hidden soda: String?

/// Mapping of common [inputs][UIElement] that exist across all or many of the different connectivity options,
/// keyed by a unique (variable) name for the input.
/// Note: the name of the variable (key of the map) should be in lower_snake_case.
///
/// Remember in Pkl to define a mapping, put the key in square brackets and the value in curly braces:
/// ```
/// inputs {
/// ["export_scope"] = new Radio {
/// ...
/// }
/// ["qn_prefix"] = new TextInput {
/// ...
/// }
/// ...
/// }
/// ```
hidden commonInputs: Mapping<String(lower_snake_case), UIElement>?

/// Label to use above the radio button of options
hidden optionsTitle: String = "Authentication"

/// Options for configuring the connector.
hidden options: Mapping<String, NestedInput>

/// (Generated) Details of all properties.
fixed properties: Map<String, UIElement> = new Mapping<String, UIElement> {
when (commonInputs != null) { ...commonInputs }
["auth-type"] = new Radio {
title = optionsTitle
possibleValues = new Mapping {
for (k, v in options) {
[k] = v.title
}
}
default = options.keys.first
required = true
}
...options
}.toMap()

/// Listing of rules to control which [inputs][UIElement] appear based on values selected in other inputs.
///
/// Remember in Pkl to define a listing, use curly braces and create new elements within:
/// ```
/// rules {
/// new UIRule {
/// whenInputs { ["export_scope"] = "ENRICHED_ONLY" }
/// required = { "qn_prefix" }
/// }
/// new UIRule {
/// ...
/// }
/// ...
/// }
/// ```
hidden rules: Listing<UIRule> = new Listing {}

/// (Generated) Details of all UI rules to use to control the UI.
fixed anyOf: List<UIRule>? =
new Listing {
for (k, _ in options) {
new UIRule {
whenInputs { ["auth-type"] = k }
required { k }
}
}
...rules
}.toList()
}

/// Class defining any outputs the package's logic will produce.
class WorkflowOutputs {
/// Files the package will produce in the local filesystem.
Expand Down Expand Up @@ -434,9 +305,6 @@ const function getOutputs(m): Mapping<String, FileOutput> = new Mapping {
["version.txt"] = Renderers.getVersionPy(m)
["Dockerfile"] = Renderers.getDockerfilePy(getPythonPkgName(m))
}
when (m.credentialConfig != null) {
["build/package/connectors/configmaps/\(m.credentialConfig.name).yaml"] = Renderers.getCredentialConfigMap(m)
}
}

/// Translate the model content into a set of files for both type definitions (JSON)
Expand Down Expand Up @@ -1108,7 +976,7 @@ class CredentialInput extends UIElement {
/// | **`inputs`** | | map of the sub-elements that should be nested within this input | |
/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` |
/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` |
class NestedInput extends UIElement {
open class NestedInput extends UIElement {
fixed type = "object"

/// Whether the widget will be shown in the UI (false) or not (true).
Expand Down
165 changes: 165 additions & 0 deletions package-toolkit/config/src/main/resources/Credential.pkl
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
/* SPDX-License-Identifier: Apache-2.0
Copyright 2024 Atlan Pte. Ltd. */

/// Template for defining configuration for a custom connector credential in Atlan.
/// Details provided through such credential configs are securely stored in an encrypted vault.
///
/// | Variable | | Usage | Default |
/// |---|---|---|---|
/// | **name** | | Name of this connector-specific credential configuration. |
/// | **source** | | Connector for which this credential configuration is applicable. |
/// | **icon** | | Icon to use for the credential configuration. |
/// | **helpdesk** | | Link to documentation for the credential configuration. |
/// | **logo** | | Logo to use for the credential configuration. |
/// | **[inputs][UIElement]** | | Mapping of common properties that exist across connection options, keyed by a unique variable name in lower_snake_case. |
/// | **[rules][UIRule]** | | Listing of rules to control which [inputs][UIElement] appear based on values selected in other inputs. |
@ModuleInfo { minPklVersion = "0.25.1" }
open module com.atlan.pkg.Credential

import "Config.pkl"
import "Renderers.pkl"

/// Name of this connector-specific credential configuration.
hidden name: String

/// Name of a connector for which this credential configuration is applicable.
hidden source: String

/// Icon to use for the credential configuration.
hidden icon: String

/// Link to documentation for the credential configuration.
hidden helpdesk: String

/// Logo to use for the credential configuration.
hidden logo: String

/// Default connector type to use for the credential configuration.
hidden connectorType: String = "rest"

/// TBC
hidden jdbcCredential: String = "{}"

/// TBC
hidden restCredential: String = "{}"

/// TBC
hidden odbcCredential: String = "{}"

/// TBC
hidden grpcCredential: String = "{}"

/// TBC
hidden restMetadata: String = ""

/// TBC
hidden restTransformer: String = ""

/// TBC
hidden sage: String?

/// TBC
hidden soda: String?

/// Mapping of fixed [inputs][UIElement] that are common across all or many of the different connectivity options.
/// Typically this is used to capture hidden properties once that can then be shared across different connectivity
/// options.
///
/// Remember in Pkl to define a mapping, put the key in square brackets and the value in curly braces:
/// ```
/// commonInputs {
/// ["host"] = new TextInput {
/// ...
/// }
/// ["port"] = new TextInput {
/// ...
/// }
/// ...
/// }
/// ```
hidden commonInputs: Mapping<CredentialAttribute, Config.UIElement>?

/// Label to use above the radio button of options
hidden optionsTitle: String = "Authentication"

/// Options for configuring the connector.
hidden options: Mapping<String, NestedCredentialInput>

/// (Generated) Details of all properties.
fixed properties: Map<String, Config.UIElement> = new Mapping<String, Config.UIElement> {
when (commonInputs != null) { ...commonInputs }
["auth-type"] = new Config.Radio {
title = optionsTitle
possibleValues = new Mapping {
for (k, v in options) {
[k] = v.title
}
}
default = options.keys.first
required = true
}
...options
}.toMap()

/// Listing of rules to control which [inputs][UIElement] appear based on values selected in other inputs.
///
/// Remember in Pkl to define a listing, use curly braces and create new elements within:
/// ```
/// rules {
/// new UIRule {
/// whenInputs { ["option1"] = "setting_a" }
/// required = { "field_name_z" }
/// }
/// new UIRule {
/// ...
/// }
/// ...
/// }
/// ```
hidden rules: Listing<Config.UIRule> = new Listing {}

/// (Generated) Details of all UI rules to use to control the UI.
fixed anyOf: List<Config.UIRule>? =
new Listing {
for (k, _ in options) {
new Config.UIRule {
whenInputs { ["auth-type"] = k }
required { k }
}
}
...rules
}.toList()

/// Valid attributes to map to in the credential.
/// Note: You can ONLY use these attributes to map credential information. If you require
/// any additional attributes, they should be a NestedInput inside the ["extra"].
typealias CredentialAttribute = "host"|"port"|"connection"|"username"|"password"|"extra"|"name"|"connector"|"connectorType"

/// Set up multiple outputs for the module, one for each configuration file.
/// - `m` the package config to generate outputs for
const function getOutputs(m): Mapping<String, FileOutput> = new Mapping {
["connectors/configmaps/\(m.name).yaml"] = Renderers.getCredentialConfigMap(m)
}

/// Translate the credential config content into a file for both type definitions (JSON)
/// and UI configuration (TypeScript).
const function getModuleOutput(m): ModuleOutput = new ModuleOutput {
files = getOutputs(m)
}

/// Set the output of the module to be a dedicated file for the credential config defined herein.
output = getModuleOutput(this)

/// Widget that allows you to configure multiple sub-elements all grouped together under a single parent
/// variable.
///
/// | Field | | Description | Default |
/// |---|---|---|---|
/// | **`title`** | | name to show in the UI for the widget | |
/// | **`inputs`** | | map of the sub-elements that should be nested within this input | |
/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` |
/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` |
class NestedCredentialInput extends Config.NestedInput {
/// Map of the sub-elements that should be nested within this input, keyed by unique lower_snake_case variable name.
hidden inputs: Mapping<CredentialAttribute, Config.UIElement>
}

0 comments on commit 97a8095

Please sign in to comment.