From f1b07ae571165ab6b76453a70ec2e73190a3a6fe Mon Sep 17 00:00:00 2001 From: Christopher Grote Date: Fri, 24 May 2024 00:58:27 +0100 Subject: [PATCH 1/3] Package toolkit refactoring, further validations Signed-off-by: Christopher Grote --- .gitignore | 1 + package-toolkit/config/build.gradle.kts | 1 + .../config/src/main/resources/Config.pkl | 134 +------- .../config/src/main/resources/Credential.pkl | 165 ++++++++++ .../config/src/main/resources/Renderers.pkl | 13 +- .../config/src/test/kotlin/CredentialTest.kt | 47 +++ .../src/test/kotlin/CustomPackageTest.kt | 19 -- .../src/test/resources/CredentialTest.pkl | 292 ++++++++++++++++++ .../src/test/resources/CustomPackageTest.pkl | 289 ----------------- package-toolkit/runtime/build.gradle.kts | 21 ++ .../src/main/kotlin/com/atlan/pkg/Utils.kt | 6 +- .../kotlin/com/atlan/pkg/model/Credential.kt | 56 ++++ .../com/atlan/pkg/objectstore/GCSSync.kt | 2 +- .../resources/csa-connectors-objectstore.pkl | 169 ++++++++++ .../src/main/resources/package.pkl | 122 -------- samples/typedefs/build.gradle.kts | 8 +- .../resources/MultiDimensionalDataset.pkl | 74 +++++ .../src/test/kotlin/CanonicalExampleTest.kt | 2 +- .../model/src/test/kotlin/ModelUnitTest.kt | 2 +- 19 files changed, 848 insertions(+), 575 deletions(-) create mode 100644 package-toolkit/config/src/main/resources/Credential.pkl create mode 100644 package-toolkit/config/src/test/kotlin/CredentialTest.kt create mode 100644 package-toolkit/config/src/test/resources/CredentialTest.pkl create mode 100644 package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/model/Credential.kt create mode 100644 package-toolkit/runtime/src/main/resources/csa-connectors-objectstore.pkl diff --git a/.gitignore b/.gitignore index 80a278f0f..f232d9849 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ generated-packages/** /typedef-toolkit/model/src/main/resources/META-INF/org/pkl/config/java/mapper/classes/ /package-toolkit/config/src/main/kotlin/com/atlan/pkg/Config.kt /package-toolkit/config/src/main/kotlin/com/atlan/pkg/Connectors.kt +/package-toolkit/config/src/main/kotlin/com/atlan/pkg/Credential.kt /package-toolkit/config/src/main/kotlin/com/atlan/pkg/Renderers.kt /package-toolkit/config/src/main/resources/META-INF/org/pkl/config/java/mapper/classes/ /package-toolkit/config/src/main/resources/BuildInfo.pkl diff --git a/package-toolkit/config/build.gradle.kts b/package-toolkit/config/build.gradle.kts index 9821a8c83..9e37f27ed 100644 --- a/package-toolkit/config/build.gradle.kts +++ b/package-toolkit/config/build.gradle.kts @@ -62,6 +62,7 @@ pkl { outputDir.set(layout.projectDirectory.dir("src/main")) sourceModules.add(file("src/main/resources/Config.pkl")) sourceModules.add(file("src/main/resources/Connectors.pkl")) + sourceModules.add(file("src/main/resources/Credential.pkl")) sourceModules.add(file("src/main/resources/Renderers.pkl")) } } diff --git a/package-toolkit/config/src/main/resources/Config.pkl b/package-toolkit/config/src/main/resources/Config.pkl index ebbbbce61..db4962655 100644 --- a/package-toolkit/config/src/main/resources/Config.pkl +++ b/package-toolkit/config/src/main/resources/Config.pkl @@ -21,7 +21,6 @@ /// | allowSchedule | | (Optional) Whether to allow the package to be scheduled (true) or only run immediately (false). | `true` | /// | certified | | (Optional) Whether the package should be listed as certified (true) or not (false). | `true` | /// | preview | | (Optional) Whether the package should be labeled as an early preview in the UI (true) or not (false). | `false` | -/// | credentialConfig | | (Optional) If the package will use a net-new type of credential, specify its configuration here. | | /// | connectorType | | (Optional) If the package needs to configure a connector, specify its type here. | | /// | category | | Name of the pill under which the package should be categorized in the marketplace in the UI. | `custom` | @ModuleInfo { minPklVersion = "0.25.1" } @@ -58,9 +57,6 @@ docsUrl: String /// | **[rules][UIRule]** | | Listing of rules to control which inputs appear based on values selected in other inputs. | uiConfig: UIConfig -/// (Optional) Configuration for any net-new connector credentials this custom package will use. -credentialConfig: CredentialConfig? - /// Coding language the package is implemented in. /// This will control what (if any) strongly-typed configuration hand-over classes are generated by the toolkit. /// (Note: if using 'Other', no strongly-typed configuration hand-over classes will be generated.) @@ -237,131 +233,6 @@ class UIRule { required: Listing } -/// Configuration for a new set of connector-specific credentials for the custom package. -/// -/// | Variable | | Usage | -/// |---|---|---| -/// | **name** | | Name of this connector-specific credential configuration. | -/// | **source** | | Connector for which this credential configuration is applicable. | -/// | **icon** | | Icon to use for the credential configuration. | -/// | **helpdesk** | | Link to documentation for the credential configuration. | -/// | **logo** | | Logo to use for the credential configuration. | -/// | **[inputs][UIElement]** | | Mapping of common properties that exist across connection options, keyed by a unique variable name in lower_snake_case. | -/// | **[rules][UIRule]** | | Listing of rules to control which [inputs][UIElement] appear based on values selected in other inputs. | -class CredentialConfig { - - /// Name of this connector-specific credential configuration. - hidden name: String - - /// Connector for which this credential configuration is applicable. - hidden source: Connectors.Type - - /// Icon to use for the credential configuration. - hidden icon: String - - /// Link to documentation for the credential configuration. - hidden helpdesk: String - - /// Logo to use for the credential configuration. - hidden logo: String - - /// Default connector type to use for the credential configuration. - hidden connectorType: String = "jdbc" - - /// TBC - hidden jdbcCredential: String = "{}" - - /// TBC - hidden restCredential: String = "{}" - - /// TBC - hidden odbcCredential: String = "{}" - - /// TBC - hidden grpcCredential: String = "{}" - - /// TBC - hidden restMetadata: String = "" - - /// TBC - hidden restTransformer: String = "" - - /// TBC - hidden sage: String? - - /// TBC - hidden soda: String? - - /// Mapping of common [inputs][UIElement] that exist across all or many of the different connectivity options, - /// keyed by a unique (variable) name for the input. - /// Note: the name of the variable (key of the map) should be in lower_snake_case. - /// - /// Remember in Pkl to define a mapping, put the key in square brackets and the value in curly braces: - /// ``` - /// inputs { - /// ["export_scope"] = new Radio { - /// ... - /// } - /// ["qn_prefix"] = new TextInput { - /// ... - /// } - /// ... - /// } - /// ``` - hidden commonInputs: Mapping? - - /// Label to use above the radio button of options - hidden optionsTitle: String = "Authentication" - - /// Options for configuring the connector. - hidden options: Mapping - - /// (Generated) Details of all properties. - fixed properties: Map = new Mapping { - when (commonInputs != null) { ...commonInputs } - ["auth-type"] = new Radio { - title = optionsTitle - possibleValues = new Mapping { - for (k, v in options) { - [k] = v.title - } - } - default = options.keys.first - required = true - } - ...options - }.toMap() - - /// Listing of rules to control which [inputs][UIElement] appear based on values selected in other inputs. - /// - /// Remember in Pkl to define a listing, use curly braces and create new elements within: - /// ``` - /// rules { - /// new UIRule { - /// whenInputs { ["export_scope"] = "ENRICHED_ONLY" } - /// required = { "qn_prefix" } - /// } - /// new UIRule { - /// ... - /// } - /// ... - /// } - /// ``` - hidden rules: Listing = new Listing {} - - /// (Generated) Details of all UI rules to use to control the UI. - fixed anyOf: List? = - new Listing { - for (k, _ in options) { - new UIRule { - whenInputs { ["auth-type"] = k } - required { k } - } - } - ...rules - }.toList() -} - /// Class defining any outputs the package's logic will produce. class WorkflowOutputs { /// Files the package will produce in the local filesystem. @@ -434,9 +305,6 @@ const function getOutputs(m): Mapping = new Mapping { ["version.txt"] = Renderers.getVersionPy(m) ["Dockerfile"] = Renderers.getDockerfilePy(getPythonPkgName(m)) } - when (m.credentialConfig != null) { - ["build/package/connectors/configmaps/\(m.credentialConfig.name).yaml"] = Renderers.getCredentialConfigMap(m) - } } /// Translate the model content into a set of files for both type definitions (JSON) @@ -1108,7 +976,7 @@ class CredentialInput extends UIElement { /// | **`inputs`** | | map of the sub-elements that should be nested within this input | | /// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | /// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` | -class NestedInput extends UIElement { +open class NestedInput extends UIElement { fixed type = "object" /// Whether the widget will be shown in the UI (false) or not (true). diff --git a/package-toolkit/config/src/main/resources/Credential.pkl b/package-toolkit/config/src/main/resources/Credential.pkl new file mode 100644 index 000000000..5183c91c0 --- /dev/null +++ b/package-toolkit/config/src/main/resources/Credential.pkl @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: Apache-2.0 + Copyright 2024 Atlan Pte. Ltd. */ + +/// Template for defining configuration for a custom connector credential in Atlan. +/// Details provided through such credential configs are securely stored in an encrypted vault. +/// +/// | Variable | | Usage | Default | +/// |---|---|---|---| +/// | **name** | | Name of this connector-specific credential configuration. | +/// | **source** | | Connector for which this credential configuration is applicable. | +/// | **icon** | | Icon to use for the credential configuration. | +/// | **helpdesk** | | Link to documentation for the credential configuration. | +/// | **logo** | | Logo to use for the credential configuration. | +/// | **[inputs][UIElement]** | | Mapping of common properties that exist across connection options, keyed by a unique variable name in lower_snake_case. | +/// | **[rules][UIRule]** | | Listing of rules to control which [inputs][UIElement] appear based on values selected in other inputs. | +@ModuleInfo { minPklVersion = "0.25.1" } +open module com.atlan.pkg.Credential + +import "Config.pkl" +import "Renderers.pkl" + +/// Name of this connector-specific credential configuration. +hidden name: String + +/// Name of a connector for which this credential configuration is applicable. +hidden source: String + +/// Icon to use for the credential configuration. +hidden icon: String + +/// Link to documentation for the credential configuration. +hidden helpdesk: String + +/// Logo to use for the credential configuration. +hidden logo: String + +/// Default connector type to use for the credential configuration. +hidden connectorType: String = "rest" + +/// TBC +hidden jdbcCredential: String = "{}" + +/// TBC +hidden restCredential: String = "{}" + +/// TBC +hidden odbcCredential: String = "{}" + +/// TBC +hidden grpcCredential: String = "{}" + +/// TBC +hidden restMetadata: String = "" + +/// TBC +hidden restTransformer: String = "" + +/// TBC +hidden sage: String? + +/// TBC +hidden soda: String? + +/// Mapping of fixed [inputs][UIElement] that are common across all or many of the different connectivity options. +/// Typically this is used to capture hidden properties once that can then be shared across different connectivity +/// options. +/// +/// Remember in Pkl to define a mapping, put the key in square brackets and the value in curly braces: +/// ``` +/// commonInputs { +/// ["host"] = new TextInput { +/// ... +/// } +/// ["port"] = new TextInput { +/// ... +/// } +/// ... +/// } +/// ``` +hidden commonInputs: Mapping? + +/// Label to use above the radio button of options +hidden optionsTitle: String = "Authentication" + +/// Options for configuring the connector. +hidden options: Mapping + +/// (Generated) Details of all properties. +fixed properties: Map = new Mapping { + when (commonInputs != null) { ...commonInputs } + ["auth-type"] = new Config.Radio { + title = optionsTitle + possibleValues = new Mapping { + for (k, v in options) { + [k] = v.title + } + } + default = options.keys.first + required = true + } + ...options +}.toMap() + +/// Listing of rules to control which [inputs][UIElement] appear based on values selected in other inputs. +/// +/// Remember in Pkl to define a listing, use curly braces and create new elements within: +/// ``` +/// rules { +/// new UIRule { +/// whenInputs { ["option1"] = "setting_a" } +/// required = { "field_name_z" } +/// } +/// new UIRule { +/// ... +/// } +/// ... +/// } +/// ``` +hidden rules: Listing = new Listing {} + +/// (Generated) Details of all UI rules to use to control the UI. +fixed anyOf: List? = + new Listing { + for (k, _ in options) { + new Config.UIRule { + whenInputs { ["auth-type"] = k } + required { k } + } + } + ...rules + }.toList() + +/// Valid attributes to map to in the credential. +/// Note: You can ONLY use these attributes to map credential information. If you require +/// any additional attributes, they should be a NestedInput inside the ["extra"]. +typealias CredentialAttribute = "host"|"port"|"connection"|"username"|"password"|"extra"|"name"|"connector"|"connectorType" + +/// Set up multiple outputs for the module, one for each configuration file. +/// - `m` the package config to generate outputs for +const function getOutputs(m): Mapping = new Mapping { + ["connectors/configmaps/\(m.name).yaml"] = Renderers.getCredentialConfigMap(m) +} + +/// Translate the credential config content into a file for both type definitions (JSON) +/// and UI configuration (TypeScript). +const function getModuleOutput(m): ModuleOutput = new ModuleOutput { + files = getOutputs(m) +} + +/// Set the output of the module to be a dedicated file for the credential config defined herein. +output = getModuleOutput(this) + +/// Widget that allows you to configure multiple sub-elements all grouped together under a single parent +/// variable. +/// +/// | Field | | Description | Default | +/// |---|---|---|---| +/// | **`title`** | | name to show in the UI for the widget | | +/// | **`inputs`** | | map of the sub-elements that should be nested within this input | | +/// | `helpText` | | informational text to place in a hover-over to describe the use of the input | `""` | +/// | `width` | | sizing of the input on the UI (8 is full-width, 4 is half-width) | `8` | +class NestedCredentialInput extends Config.NestedInput { + /// Map of the sub-elements that should be nested within this input, keyed by unique lower_snake_case variable name. + hidden inputs: Mapping +} diff --git a/package-toolkit/config/src/main/resources/Renderers.pkl b/package-toolkit/config/src/main/resources/Renderers.pkl index e12a5fa08..da0021a02 100644 --- a/package-toolkit/config/src/main/resources/Renderers.pkl +++ b/package-toolkit/config/src/main/resources/Renderers.pkl @@ -7,6 +7,7 @@ module com.atlan.pkg.Renderers import "Config.pkl" import "Connectors.pkl" +import "Credential.pkl" /// Render the configmap YAML file. const function getConfigMap(m: Config): FileOutput = new FileOutput { @@ -18,10 +19,10 @@ const function getConfigMap(m: Config): FileOutput = new FileOutput { } /// Render the connector-specific credential configmap YAML file. -const function getCredentialConfigMap(m: Config): FileOutput = new FileOutput { +const function getCredentialConfigMap(m: Credential): FileOutput = new FileOutput { value = new ConnectorConfigMap { name = m.name - config = m.credentialConfig!! + config = m } renderer = new YamlRenderer {} } @@ -956,18 +957,18 @@ local class WorkflowContainer { /// Used to render the configmap for a new connector type. local class ConnectorConfigMap extends ConfigMap { fixed metadata: Mapping = new Mapping { - ["name"] = config.name + ["name"] = name ["labels"] = new Mapping { ["workflows.argoproj.io/configmap-type"] = "Parameter" ["orchestration.atlan.com/version"] = "1" - ["orchestration.atlan.com/source"] = config.source.value + ["orchestration.atlan.com/source"] = config.source } } fixed data: Mapping = new Mapping { ["icon"] = config.icon ["helpdeskLink"] = config.helpdesk ["logo"] = config.logo - ["connector"] = config.source.value + ["connector"] = config.source ["defaultConnectorType"] = config.connectorType ["jdbcCredentialTemplate"] = config.jdbcCredential ["restCredentialTemplate"] = config.restCredential @@ -980,7 +981,7 @@ local class ConnectorConfigMap extends ConfigMap { ["config"] = new JsonRenderer {}.renderValue(config) } hidden name: String - hidden config: Config.CredentialConfig + hidden config: Credential } const function resolvePropertyToVar(key: String, property: Config.UIElement): NameValuePair = new NameValuePair { diff --git a/package-toolkit/config/src/test/kotlin/CredentialTest.kt b/package-toolkit/config/src/test/kotlin/CredentialTest.kt new file mode 100644 index 000000000..cbf469c0c --- /dev/null +++ b/package-toolkit/config/src/test/kotlin/CredentialTest.kt @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: Apache-2.0 + Copyright 2024 Atlan Pte. Ltd. */ +import com.atlan.pkg.Config +import com.atlan.pkg.Credential +import org.pkl.config.java.ConfigEvaluator +import org.pkl.config.kotlin.forKotlin +import org.pkl.config.kotlin.to +import org.pkl.core.ModuleSource +import org.testng.annotations.BeforeClass +import org.testng.annotations.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +object CredentialTest { + + lateinit var credential: Credential + + @BeforeClass + fun modelEval() { + val source = ModuleSource.path("src/test/resources/CredentialTest.pkl") + credential = ConfigEvaluator.preconfigured().forKotlin().use { evaluator -> + evaluator.evaluate(source).to() + } + } + + @Test + fun testConnectorCredential() { + assertNotNull(credential) + assertEquals(9, credential.properties.size) + val basic = credential.properties["basic"] + assertTrue(basic is Config.NestedInput) + assertEquals("Basic", basic.ui.label) + assertEquals("nested", basic.ui.widget) + assertEquals(3, basic.properties.size) + val extra = basic.properties["extra"] + assertTrue(extra is Config.NestedInput) + assertEquals("Role and Warehouse", extra.ui.label) + assertEquals(2, extra.properties.size) + val role = extra.properties["role"] + assertTrue(role is Config.SQLExecutor) + assertEquals("show grants", role.ui.query) + assertEquals(3, credential.anyOf?.size) + } + + // TODO: Test generated configmap YAML +} diff --git a/package-toolkit/config/src/test/kotlin/CustomPackageTest.kt b/package-toolkit/config/src/test/kotlin/CustomPackageTest.kt index 783051479..5b6900674 100644 --- a/package-toolkit/config/src/test/kotlin/CustomPackageTest.kt +++ b/package-toolkit/config/src/test/kotlin/CustomPackageTest.kt @@ -157,25 +157,6 @@ object CustomPackageTest { assertEquals("csa-connectors-gcs", widget.credentialType) } - @Test - fun testConnectorCredential() { - assertNotNull(config.credentialConfig) - assertEquals(9, config.credentialConfig?.properties?.size) - val basic = config.credentialConfig?.properties?.get("basic") - assertTrue(basic is Config.NestedInput) - assertEquals("Basic", basic.ui.label) - assertEquals("nested", basic.ui.widget) - assertEquals(3, basic.properties.size) - val extra = basic.properties["extra"] - assertTrue(extra is Config.NestedInput) - assertEquals("Role and Warehouse", extra.ui.label) - assertEquals(2, extra.properties.size) - val role = extra.properties["role"] - assertTrue(role is Config.SQLExecutor) - assertEquals("show grants", role.ui.query) - assertEquals(3, config.credentialConfig?.anyOf?.size) - } - // TODO: Test generated workflow template contents // TODO: Test generated package JSON contents } diff --git a/package-toolkit/config/src/test/resources/CredentialTest.pkl b/package-toolkit/config/src/test/resources/CredentialTest.pkl new file mode 100644 index 000000000..1775e69d4 --- /dev/null +++ b/package-toolkit/config/src/test/resources/CredentialTest.pkl @@ -0,0 +1,292 @@ +/* SPDX-License-Identifier: Apache-2.0 + Copyright 2024 Atlan Pte. Ltd. */ +amends "modulepath:/Credential.pkl" +import "modulepath:/Config.pkl" +import "modulepath:/Connectors.pkl" + +name = "atlan-connectors-snowflake" +source = Connectors.SNOWFLAKE.value +icon = "https://docs.snowflake.com/en/_images/logo-snowflake-sans-text.png" +helpdesk = "https://ask.atlan.com/hc/en-us/articles/4417168972689-How-to-set-up-your-Snowflake-connection-with-Atlan" +logo = "https://1amiydhcmj36tz3733v94f15-wpengine.netdna-ssl.com/wp-content/themes/snowflake/assets/img/logo-blue.svg" +jdbcCredential = """ + { + "className": "net.snowflake.client.jdbc.SnowflakeDriver", + "jarLink": "https://atlan-public.s3-eu-west-1.amazonaws.com/atlan/jdbc/snowflake.tar.gz", + "url": "jdbc:snowflake://{{ host }}?loginTimeout=5&networkTimeout=5&CLIENT_SESSION_KEEP_ALIVE=true&application=atlan{% if authType == "keypair" %}&private_key_file={{ "{{__jdbc_private_key}}" }}{% endif %}&CLIENT_RESULT_CHUNK_SIZE=100&CLIENT_MEMORY_LIMIT=1000", + "driverProperties": {% if authType == "keypair" %} + { "username" : "{{ username }}", "user": "{{ username }}", "__jdbc_private_key_pass": {{ extra.private_key_password | tojson }}, "__jdbc_private_key": {{ password | tojson }} } + {% elif authType == "oauth" %} + {"authenticator": "{{ authType }}", "role": "{{ extra.role }}", "warehouse": "{{ extra.warehouse }}", "token": "{{ extra.accessTokenSecret }}"} + {% elif authType == "okta" %} + {"user":"{{ username }}", "password": {{ password | tojson }}, "authenticator": "{{ extra.authenticator }}"} + {% else %} + {"username": "{{ username }}", "user": "{{ username }}", "password": {{ password | tojson }} } + {% endif %} + } + """ +sage = """ + { + "schemasCheck": { + "curls": [ + { + "name": "schemas", + "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' --data-raw '{\"query\": \"show atlan schemas\"}'", + "addCredential": true, + "credentialConnectorType": "jdbc" + } + ], + "responseTemplate": "{{- $includeFilter := dict}} {{- if eq `string` (printf `%T` (index .formData `include-filter`)) }} {{- $includeFilter = index .formData `include-filter` | fromJson}} {{- else }} {{- $includeFilter = index .formData `include-filter` }} {{- end }} {{- $allowedDatabases := list}} {{- $allowedSchemas := list}} {{- $missingObjectName := ``}} {{- $checkSuccess := true }} {{- range $schemaList := .schemas.results }} {{- $allowedDatabases = append $allowedDatabases $schemaList.TABLE_CATALOG }} {{- $allowedSchemas = append $allowedSchemas (print $schemaList.TABLE_CATALOG `.` $schemaList.TABLE_SCHEM )}} {{- end }} {{- range $filteredDb, $filteredSchemas := $includeFilter }} {{- $_db := upper $filteredDb | trimPrefix `^` | trimSuffix `$` }} {{- $checkSuccess = and $checkSuccess (has $_db $allowedDatabases) }} {{- if not (has $_db $allowedDatabases)}} {{- $missingObjectName = (print $_db ` ` `database`)}} {{- end }} {{- range $schmea := $filteredSchemas }} {{- $_schema := upper $schmea | trimPrefix `^` | trimSuffix `$` }} {{- $checkSuccess = and $checkSuccess (has (print $_db `.` (upper $_schema)) $allowedSchemas)}} {{- if not (has (print $_db `.` (upper $_schema)) $allowedSchemas)}} {{- $missingObjectName = (print $_db `.` (upper $_schema) ` ` `schema`)}} {{- end }} {{- end }} {{- end }} {{- $response := dict `successMessage` `` `failureMessage` `` `data` dict `response` dict }} {{- if $checkSuccess }} {{- $_ := set $response `successMessage` `Check successful` }} {{- else }} {{- $_ := set $response `failureMessage` (print `Check failed for ` $missingObjectName) }} {{- end }} {{- $response | toJson }}" + }, + "warehouseAccessCheck": { + "curls": [ + { + "name": "databases", + "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' --data-raw '{\"query\": \"show atlan databases\"}'", + "addCredential": true, + "credentialConnectorType": "jdbc" + }, + { + "name": "queries", + "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' --data-raw '{\"query\": \"SELECT count(*) from {{with .databases.results}}{{(index . 0).TABLE_CATALOG}}{{else}}public{{end}}.information_schema.tables\"}'", + "addCredential": true, + "credentialConnectorType": "jdbc" + } + ], + "responseTemplate": "{{- $errors := .queries.errors }} {{- $response := dict `successMessage` `Check successful` `failureMessage` `` `data` dict `response` dict }} {{- $_ := set $response `response` .queries | toJson }} {{- if $errors }} {{- if (gt (len $errors) 0) }} {{- $_ := set $response `failureMessage` `Can't query any of the database.` }} {{- end}} {{- end}} {{- $response | toJson -}}" + }, + "schemasCheckAU": { + "curls": [ + { + "name": "schemas", + "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' {{- $auDb := index . `account-usage-database-name`}} {{- $auSchema := index . `account-usage-schema-name`}} --data-raw '{\"query\": \"select CATALOG_NAME, SCHEMA_NAME from {{$auDb}}.{{$auSchema}}.SCHEMATA WHERE DELETED IS NULL;\"}'", + "addCredential": true, + "credentialConnectorType": "jdbc" + } + ], + "responseTemplate": "{{- $includeFilter := dict}} {{- if eq `string` (printf `%T` (index .formData `include-filter`)) }} {{- $includeFilter = index .formData `include-filter` | fromJson}} {{- else }} {{- $includeFilter = index .formData `include-filter` }} {{- end }} {{- $allowedDatabases := list}} {{- $allowedSchemas := list}} {{- $missingObjectName := ``}} {{- $checkSuccess := true }} {{- range $schemaList := .schemas.results }} {{- $allowedDatabases = append $allowedDatabases $schemaList.TABLE_CATALOG }} {{- $allowedSchemas = append $allowedSchemas (print $schemaList.TABLE_CATALOG `.` $schemaList.TABLE_SCHEM )}} {{- end }} {{- range $filteredDb, $filteredSchemas := $includeFilter }} {{- $_db := upper $filteredDb | trimPrefix `^` | trimSuffix `$` }} {{- $checkSuccess = and $checkSuccess (has $_db $allowedDatabases) }} {{- if not (has $_db $allowedDatabases)}} {{- $missingObjectName = (print $_db ` ` `database`)}} {{- end }} {{- range $schmea := $filteredSchemas }} {{- $_schema := upper $schmea | trimPrefix `^` | trimSuffix `$` }} {{- $checkSuccess = and $checkSuccess (has (print $_db `.` (upper $_schema)) $allowedSchemas)}} {{- if not (has (print $_db `.` (upper $_schema)) $allowedSchemas)}} {{- $missingObjectName = (print $_db `.` (upper $_schema) ` ` `schema`)}} {{- end }} {{- end }} {{- end }} {{- $response := dict `successMessage` `` `failureMessage` `` `data` dict `response` dict }} {{- if $checkSuccess }} {{- $_ := set $response `successMessage` `Check successful` }} {{- else }} {{- $_ := set $response `failureMessage` (print `Check failed for ` $missingObjectName) }} {{- end }} {{- $response | toJson }}" + }, + "minersQueryHistoryCheck": { + "curls": [ + { + "name": "history", + "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' --data-raw {{- $auDb := `SNOWFLAKE` }} {{- $auSc := `ACCOUNT_USAGE` }} {{- if eq (printf `%s` (index . `snowflake-database`)) `cloned` }} {{- $auDb = index . `database-name` }} {{- $auSc = index . `schema-name` }} {{- end }} '{\"query\": \"select * from {{$auDb}}.{{$auSc}}.QUERY_HISTORY LIMIT 1;\"}'", + "addCredential": true, + "credentialConnectorType": "jdbc" + } + ], + "responseTemplate": "{{- $errors := .history.errors }} {{- $response := dict `successMessage` `Check successful` `failureMessage` `` `data` dict `response` dict }} {{- if $errors }} {{- if (gt (len $errors) 0) }} {{- $_ := set $response `failureMessage` `Can't access the query history view. Please run the command in your snowflake instance: GRANT IMPORTED PRIVILEGES ON DATABASE snowflake TO ROLE atlan_user_role;` }} {{- end}} {{- end}} {{- $response | toJson -}}" + }, + "minersAccessHistoryCheck": { + "curls": [ + { + "name": "history", + "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' --data-raw {{- $auDb := `SNOWFLAKE` }} {{- $auSc := `ACCOUNT_USAGE` }} {{- if eq (printf `%s` (index . `snowflake-database`)) `cloned` }} {{- $auDb = index . `database-name` }} {{- $auSc = index . `schema-name` }} {{- end }} '{\"query\": \"select * from {{$auDb}}.{{$auSc}}.ACCESS_HISTORY LIMIT 1;\"}'", + "addCredential": true, + "credentialConnectorType": "jdbc" + } + ], + "responseTemplate": "{{- $errors := .history.errors }} {{- $response := dict `successMessage` `Check successful` `failureMessage` `` `data` dict `response` dict }} {{- if $errors }} {{- if (gt (len $errors) 0) }} {{- $_ := set $response `failureMessage` `Can't access the access history view. Please run the command in your snowflake instance: GRANT IMPORTED PRIVILEGES ON DATABASE snowflake TO ROLE atlan_user_role;` }} {{- end}} {{- end}} {{- $response | toJson -}}" + }, + "minersSessionsCheck": { + "curls": [ + { + "name": "history", + "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' --data-raw {{- $auDb := `SNOWFLAKE` }} {{- $auSc := `ACCOUNT_USAGE` }} {{- if eq (printf `%s` (index . `snowflake-database`)) `cloned` }} {{- $auDb = index . `database-name` }} {{- $auSc = index . `schema-name` }} {{- end }} '{\"query\": \"select * from {{$auDb}}.{{$auSc}}.SESSIONS LIMIT 1;\"}'", + "addCredential": true, + "credentialConnectorType": "jdbc" + } + ], + "responseTemplate": "{{- $errors := .history.errors }} {{- $response := dict `successMessage` `Check successful` `failureMessage` `` `data` dict `response` dict }} {{- if $errors }} {{- if (gt (len $errors) 0) }} {{- $_ := set $response `failureMessage` `Can't access the sessions view. Please run the command in your snowflake instance: GRANT IMPORTED PRIVILEGES ON DATABASE snowflake TO ROLE atlan_user_role;` }} {{- end}} {{- end}} {{- $response | toJson -}}" + }, + "crawlerCheck": { + "curls": [ + { + "name": "testConnector", + "curl": "curl --location --request POST 'http://atlas-service-atlas.atlas.svc.cluster.local/api/meta/search/indexsearch' --header 'Accept: application/json' --header 'Authorization: Bearer {{ index . \"atlan-token\"}}' --header 'Content-Type: application/json' --data-raw '{\"dsl\":{\"from\":0,\"size\":1,\"query\":{\"bool\":{\"filter\":{\"bool\":{\"must\":[{\"term\":{\"qualifiedName\":\"{{index . \"connection-qualified-name\"}}\"}},{\"term\":{\"__state\":\"ACTIVE\"}},{\"terms\":{\"__typeName.keyword\":[\"Connection\"]}}]}}}}}}'", + "addCredential": false, + "credentialConnectorType": "" + }, + { + "name": "testCurrentState", + "curl": "", + "addCredential": true, + "credentialConnectorType": "s3", + "extras": "{ \"s3_prefix\" : \"argo-artifacts/default/snowflake/{{(index . \"connection-qualified-name\") | trimPrefix \"default/snowflake/\"}}/current-state\", \"s3_bucket_name\": \"{{index . \"artifact-bucket-name\"}}\", \"ignore_empty\": true }" + } + ], + "responseTemplate": "{{ $response := dict `successMessage` `` `failureMessage` `` `data` dict `response` dict }} {{ if not .testConnector.entities }} {{ $_ := set $response `failureMessage` `Check failed. Connection does not exist.` }} {{ else }} {{ $failOnCurrentState := true}} {{ range $file := .testCurrentState }} {{ if contains `current-state` $file }} {{ $failOnCurrentState = false }} {{ end }} {{ end }} {{ if $failOnCurrentState }} {{ $_ := set $response `failureMessage` `Check failed. Workflow artifacts are missing. Please run the crawler workflow again.` }} {{ else }} {{ $_ := set $response `successMessage` `Check successful` }} {{ end }} {{ end }} {{- $response | toJson -}}" + }, + "minerS3Check": { + "curls": [ + { + "name": "s3", + "curl": "", + "addCredential": true, + "credentialConnectorType": "s3", + "extras": "{ \"s3_prefix\" : \"{{- index . `extraction-s3-prefix` }}\", \"s3_bucket_name\": \"{{- (index . `extraction-s3-bucket`) | trimPrefix `s3://` }}\" , \"s3_region\": \"\" }" + } + ], + "responseTemplate": "{{- $response := dict `successMessage` `Check successful` `failureMessage` `` `data` dict `response` dict }} {{- $_ := set $response `data` .s3 }} {{- $response | toJson -}}" + } + } + """ +soda = """ + { + "type": "snowflake", + "username": {{ username | tojson }}, + "account": {{ host | replace('/', '')| replace('.snowflakecomputing.com', '') | tojson }}, + "warehouse": {{ extra.warehouse | tojson }}, + "database": "", + "schema": "", + "role": {{ extra.role | tojson }}, + "passcode_in_password": false, + "private_key_path": null, + "authenticator": "snowflake", + "QUERY_TAG": null, + "QUOTED_IDENTIFIERS_IGNORE_CASE": null, + {% if authType == "basic" %} + "private_key": null, + "private_key_passphrase": null, + "password": {{ password | tojson }} + {% else %} + "private_key": {{ password | tojson }}, + "private_key_passphrase": {{ extra.private_key_password | tojson }}, + "password": null + {% endif %} + } + """ +commonInputs { + ["name"] = new Config.TextInput { + title = "Name" + required = false + hide = true + placeholderText = "Host Name" + } + ["connector"] = new Config.TextInput { + title = "Connector" + required = false + hide = true + placeholderText = "Connector" + } + ["connectorType"] = new Config.TextInput { + title = "connectorType" + required = false + hide = true + placeholderText = "connectorType" + } + ["host"] = new Config.TextInput { + title = "Account Identifiers (Host)" + placeholderText = "..aws.snowflakecomputing.com" + prepend = "https://" + width = 6 + } + ["port"] = new Config.NumericInput { + title = "Port" + default = 443 + enabled = false + width = 2 + } +} +options { + ["basic"] { + title = "Basic" + hide = true + inputs { + ["username"] = new Config.TextInput { + title = "Username" + required = true + defaultValue = "atlanadmin" + width = 4 + } + ["password"] = new Config.PasswordInput { + title = "Password" + required = true + width = 4 + } + ["extra"] = new Config.NestedInput { + title = "Role and Warehouse" + inputs { + ["role"] = new Config.SQLExecutor { + title = "Role" + sqlQuery = "show grants" + width = 4 + } + ["warehouse"] = new Config.SQLExecutor { + title = "Warehouse" + sqlQuery = "show warehouses" + width = 4 + } + } + } + } + } + ["keypair"] { + title = "Keypair" + inputs { + ["username"] = new Config.TextInput { + title = "Username" + placeholderText = "Username" + width = 4 + } + ["password"] = new Config.TextBoxInput { + title = "Encrypted Private Key" + placeholderText = "-----BEGIN ENCRYPTED PRIVATE KEY-----MIIE6TAbBgkqhkiG9w0BBQMwDgQILYPyCppzOwECAggABIIEyLiGSpeeGSe3xHP1wHLjfCYycUPennlX2bd8yX8xOxGSGfvB+99+PmSlex0FmY9ov1J8H1H9Y3lMWXbL...-----END ENCRYPTED PRIVATE KEY-----" + width = 4 + } + ["extra"] = new Config.NestedInput { + title = "Private Key Password" + inputs { + ["private_key_password"] = new Config.PasswordInput { + title = "Private Key Password" + width = 5 + } + ["role"] = new Config.SQLExecutor { + title = "Role" + sqlQuery = "show grants" + width = 4 + } + ["warehouse"] = new Config.SQLExecutor { + title = "Warehouse" + sqlQuery = "show warehouses" + width = 4 + } + } + } + } + } + ["okta"] { + title = "OKTA SSO" + inputs { + ["username"] = new Config.TextInput { + title = "Username" + placeholderText = "Username" + width = 4 + } + ["password"] = new Config.PasswordInput { + title = "Password" + width = 4 + } + ["extra"] = new Config.NestedInput { + title = "Private Key Password" + inputs { + ["authenticator"] = new Config.TextInput { + title = "Authenticator" + placeholderText = "Enter your authenticator code" + width = 4 + } + ["role"] = new Config.SQLExecutor { + title = "Role" + sqlQuery = "show grants" + width = 4 + } + ["warehouse"] = new Config.SQLExecutor { + title = "Warehouse" + sqlQuery = "show warehouses" + width = 4 + } + } + } + } + } +} diff --git a/package-toolkit/config/src/test/resources/CustomPackageTest.pkl b/package-toolkit/config/src/test/resources/CustomPackageTest.pkl index 2b4ff6c39..1c5935039 100644 --- a/package-toolkit/config/src/test/resources/CustomPackageTest.pkl +++ b/package-toolkit/config/src/test/resources/CustomPackageTest.pkl @@ -88,292 +88,3 @@ uiConfig { } } } - -credentialConfig { - name = "atlan-connectors-snowflake" - source = Connectors.SNOWFLAKE - icon = "https://docs.snowflake.com/en/_images/logo-snowflake-sans-text.png" - helpdesk = "https://ask.atlan.com/hc/en-us/articles/4417168972689-How-to-set-up-your-Snowflake-connection-with-Atlan" - logo = "https://1amiydhcmj36tz3733v94f15-wpengine.netdna-ssl.com/wp-content/themes/snowflake/assets/img/logo-blue.svg" - jdbcCredential = """ - { - "className": "net.snowflake.client.jdbc.SnowflakeDriver", - "jarLink": "https://atlan-public.s3-eu-west-1.amazonaws.com/atlan/jdbc/snowflake.tar.gz", - "url": "jdbc:snowflake://{{ host }}?loginTimeout=5&networkTimeout=5&CLIENT_SESSION_KEEP_ALIVE=true&application=atlan{% if authType == "keypair" %}&private_key_file={{ "{{__jdbc_private_key}}" }}{% endif %}&CLIENT_RESULT_CHUNK_SIZE=100&CLIENT_MEMORY_LIMIT=1000", - "driverProperties": {% if authType == "keypair" %} - { "username" : "{{ username }}", "user": "{{ username }}", "__jdbc_private_key_pass": {{ extra.private_key_password | tojson }}, "__jdbc_private_key": {{ password | tojson }} } - {% elif authType == "oauth" %} - {"authenticator": "{{ authType }}", "role": "{{ extra.role }}", "warehouse": "{{ extra.warehouse }}", "token": "{{ extra.accessTokenSecret }}"} - {% elif authType == "okta" %} - {"user":"{{ username }}", "password": {{ password | tojson }}, "authenticator": "{{ extra.authenticator }}"} - {% else %} - {"username": "{{ username }}", "user": "{{ username }}", "password": {{ password | tojson }} } - {% endif %} - } - """ - sage = """ - { - "schemasCheck": { - "curls": [ - { - "name": "schemas", - "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' --data-raw '{\"query\": \"show atlan schemas\"}'", - "addCredential": true, - "credentialConnectorType": "jdbc" - } - ], - "responseTemplate": "{{- $includeFilter := dict}} {{- if eq `string` (printf `%T` (index .formData `include-filter`)) }} {{- $includeFilter = index .formData `include-filter` | fromJson}} {{- else }} {{- $includeFilter = index .formData `include-filter` }} {{- end }} {{- $allowedDatabases := list}} {{- $allowedSchemas := list}} {{- $missingObjectName := ``}} {{- $checkSuccess := true }} {{- range $schemaList := .schemas.results }} {{- $allowedDatabases = append $allowedDatabases $schemaList.TABLE_CATALOG }} {{- $allowedSchemas = append $allowedSchemas (print $schemaList.TABLE_CATALOG `.` $schemaList.TABLE_SCHEM )}} {{- end }} {{- range $filteredDb, $filteredSchemas := $includeFilter }} {{- $_db := upper $filteredDb | trimPrefix `^` | trimSuffix `$` }} {{- $checkSuccess = and $checkSuccess (has $_db $allowedDatabases) }} {{- if not (has $_db $allowedDatabases)}} {{- $missingObjectName = (print $_db ` ` `database`)}} {{- end }} {{- range $schmea := $filteredSchemas }} {{- $_schema := upper $schmea | trimPrefix `^` | trimSuffix `$` }} {{- $checkSuccess = and $checkSuccess (has (print $_db `.` (upper $_schema)) $allowedSchemas)}} {{- if not (has (print $_db `.` (upper $_schema)) $allowedSchemas)}} {{- $missingObjectName = (print $_db `.` (upper $_schema) ` ` `schema`)}} {{- end }} {{- end }} {{- end }} {{- $response := dict `successMessage` `` `failureMessage` `` `data` dict `response` dict }} {{- if $checkSuccess }} {{- $_ := set $response `successMessage` `Check successful` }} {{- else }} {{- $_ := set $response `failureMessage` (print `Check failed for ` $missingObjectName) }} {{- end }} {{- $response | toJson }}" - }, - "warehouseAccessCheck": { - "curls": [ - { - "name": "databases", - "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' --data-raw '{\"query\": \"show atlan databases\"}'", - "addCredential": true, - "credentialConnectorType": "jdbc" - }, - { - "name": "queries", - "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' --data-raw '{\"query\": \"SELECT count(*) from {{with .databases.results}}{{(index . 0).TABLE_CATALOG}}{{else}}public{{end}}.information_schema.tables\"}'", - "addCredential": true, - "credentialConnectorType": "jdbc" - } - ], - "responseTemplate": "{{- $errors := .queries.errors }} {{- $response := dict `successMessage` `Check successful` `failureMessage` `` `data` dict `response` dict }} {{- $_ := set $response `response` .queries | toJson }} {{- if $errors }} {{- if (gt (len $errors) 0) }} {{- $_ := set $response `failureMessage` `Can't query any of the database.` }} {{- end}} {{- end}} {{- $response | toJson -}}" - }, - "schemasCheckAU": { - "curls": [ - { - "name": "schemas", - "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' {{- $auDb := index . `account-usage-database-name`}} {{- $auSchema := index . `account-usage-schema-name`}} --data-raw '{\"query\": \"select CATALOG_NAME, SCHEMA_NAME from {{$auDb}}.{{$auSchema}}.SCHEMATA WHERE DELETED IS NULL;\"}'", - "addCredential": true, - "credentialConnectorType": "jdbc" - } - ], - "responseTemplate": "{{- $includeFilter := dict}} {{- if eq `string` (printf `%T` (index .formData `include-filter`)) }} {{- $includeFilter = index .formData `include-filter` | fromJson}} {{- else }} {{- $includeFilter = index .formData `include-filter` }} {{- end }} {{- $allowedDatabases := list}} {{- $allowedSchemas := list}} {{- $missingObjectName := ``}} {{- $checkSuccess := true }} {{- range $schemaList := .schemas.results }} {{- $allowedDatabases = append $allowedDatabases $schemaList.TABLE_CATALOG }} {{- $allowedSchemas = append $allowedSchemas (print $schemaList.TABLE_CATALOG `.` $schemaList.TABLE_SCHEM )}} {{- end }} {{- range $filteredDb, $filteredSchemas := $includeFilter }} {{- $_db := upper $filteredDb | trimPrefix `^` | trimSuffix `$` }} {{- $checkSuccess = and $checkSuccess (has $_db $allowedDatabases) }} {{- if not (has $_db $allowedDatabases)}} {{- $missingObjectName = (print $_db ` ` `database`)}} {{- end }} {{- range $schmea := $filteredSchemas }} {{- $_schema := upper $schmea | trimPrefix `^` | trimSuffix `$` }} {{- $checkSuccess = and $checkSuccess (has (print $_db `.` (upper $_schema)) $allowedSchemas)}} {{- if not (has (print $_db `.` (upper $_schema)) $allowedSchemas)}} {{- $missingObjectName = (print $_db `.` (upper $_schema) ` ` `schema`)}} {{- end }} {{- end }} {{- end }} {{- $response := dict `successMessage` `` `failureMessage` `` `data` dict `response` dict }} {{- if $checkSuccess }} {{- $_ := set $response `successMessage` `Check successful` }} {{- else }} {{- $_ := set $response `failureMessage` (print `Check failed for ` $missingObjectName) }} {{- end }} {{- $response | toJson }}" - }, - "minersQueryHistoryCheck": { - "curls": [ - { - "name": "history", - "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' --data-raw {{- $auDb := `SNOWFLAKE` }} {{- $auSc := `ACCOUNT_USAGE` }} {{- if eq (printf `%s` (index . `snowflake-database`)) `cloned` }} {{- $auDb = index . `database-name` }} {{- $auSc = index . `schema-name` }} {{- end }} '{\"query\": \"select * from {{$auDb}}.{{$auSc}}.QUERY_HISTORY LIMIT 1;\"}'", - "addCredential": true, - "credentialConnectorType": "jdbc" - } - ], - "responseTemplate": "{{- $errors := .history.errors }} {{- $response := dict `successMessage` `Check successful` `failureMessage` `` `data` dict `response` dict }} {{- if $errors }} {{- if (gt (len $errors) 0) }} {{- $_ := set $response `failureMessage` `Can't access the query history view. Please run the command in your snowflake instance: GRANT IMPORTED PRIVILEGES ON DATABASE snowflake TO ROLE atlan_user_role;` }} {{- end}} {{- end}} {{- $response | toJson -}}" - }, - "minersAccessHistoryCheck": { - "curls": [ - { - "name": "history", - "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' --data-raw {{- $auDb := `SNOWFLAKE` }} {{- $auSc := `ACCOUNT_USAGE` }} {{- if eq (printf `%s` (index . `snowflake-database`)) `cloned` }} {{- $auDb = index . `database-name` }} {{- $auSc = index . `schema-name` }} {{- end }} '{\"query\": \"select * from {{$auDb}}.{{$auSc}}.ACCESS_HISTORY LIMIT 1;\"}'", - "addCredential": true, - "credentialConnectorType": "jdbc" - } - ], - "responseTemplate": "{{- $errors := .history.errors }} {{- $response := dict `successMessage` `Check successful` `failureMessage` `` `data` dict `response` dict }} {{- if $errors }} {{- if (gt (len $errors) 0) }} {{- $_ := set $response `failureMessage` `Can't access the access history view. Please run the command in your snowflake instance: GRANT IMPORTED PRIVILEGES ON DATABASE snowflake TO ROLE atlan_user_role;` }} {{- end}} {{- end}} {{- $response | toJson -}}" - }, - "minersSessionsCheck": { - "curls": [ - { - "name": "history", - "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' --data-raw {{- $auDb := `SNOWFLAKE` }} {{- $auSc := `ACCOUNT_USAGE` }} {{- if eq (printf `%s` (index . `snowflake-database`)) `cloned` }} {{- $auDb = index . `database-name` }} {{- $auSc = index . `schema-name` }} {{- end }} '{\"query\": \"select * from {{$auDb}}.{{$auSc}}.SESSIONS LIMIT 1;\"}'", - "addCredential": true, - "credentialConnectorType": "jdbc" - } - ], - "responseTemplate": "{{- $errors := .history.errors }} {{- $response := dict `successMessage` `Check successful` `failureMessage` `` `data` dict `response` dict }} {{- if $errors }} {{- if (gt (len $errors) 0) }} {{- $_ := set $response `failureMessage` `Can't access the sessions view. Please run the command in your snowflake instance: GRANT IMPORTED PRIVILEGES ON DATABASE snowflake TO ROLE atlan_user_role;` }} {{- end}} {{- end}} {{- $response | toJson -}}" - }, - "crawlerCheck": { - "curls": [ - { - "name": "testConnector", - "curl": "curl --location --request POST 'http://atlas-service-atlas.atlas.svc.cluster.local/api/meta/search/indexsearch' --header 'Accept: application/json' --header 'Authorization: Bearer {{ index . \"atlan-token\"}}' --header 'Content-Type: application/json' --data-raw '{\"dsl\":{\"from\":0,\"size\":1,\"query\":{\"bool\":{\"filter\":{\"bool\":{\"must\":[{\"term\":{\"qualifiedName\":\"{{index . \"connection-qualified-name\"}}\"}},{\"term\":{\"__state\":\"ACTIVE\"}},{\"terms\":{\"__typeName.keyword\":[\"Connection\"]}}]}}}}}}'", - "addCredential": false, - "credentialConnectorType": "" - }, - { - "name": "testCurrentState", - "curl": "", - "addCredential": true, - "credentialConnectorType": "s3", - "extras": "{ \"s3_prefix\" : \"argo-artifacts/default/snowflake/{{(index . \"connection-qualified-name\") | trimPrefix \"default/snowflake/\"}}/current-state\", \"s3_bucket_name\": \"{{index . \"artifact-bucket-name\"}}\", \"ignore_empty\": true }" - } - ], - "responseTemplate": "{{ $response := dict `successMessage` `` `failureMessage` `` `data` dict `response` dict }} {{ if not .testConnector.entities }} {{ $_ := set $response `failureMessage` `Check failed. Connection does not exist.` }} {{ else }} {{ $failOnCurrentState := true}} {{ range $file := .testCurrentState }} {{ if contains `current-state` $file }} {{ $failOnCurrentState = false }} {{ end }} {{ end }} {{ if $failOnCurrentState }} {{ $_ := set $response `failureMessage` `Check failed. Workflow artifacts are missing. Please run the crawler workflow again.` }} {{ else }} {{ $_ := set $response `successMessage` `Check successful` }} {{ end }} {{ end }} {{- $response | toJson -}}" - }, - "minerS3Check": { - "curls": [ - { - "name": "s3", - "curl": "", - "addCredential": true, - "credentialConnectorType": "s3", - "extras": "{ \"s3_prefix\" : \"{{- index . `extraction-s3-prefix` }}\", \"s3_bucket_name\": \"{{- (index . `extraction-s3-bucket`) | trimPrefix `s3://` }}\" , \"s3_region\": \"\" }" - } - ], - "responseTemplate": "{{- $response := dict `successMessage` `Check successful` `failureMessage` `` `data` dict `response` dict }} {{- $_ := set $response `data` .s3 }} {{- $response | toJson -}}" - } - } - """ - soda = """ - { - "type": "snowflake", - "username": {{ username | tojson }}, - "account": {{ host | replace('/', '')| replace('.snowflakecomputing.com', '') | tojson }}, - "warehouse": {{ extra.warehouse | tojson }}, - "database": "", - "schema": "", - "role": {{ extra.role | tojson }}, - "passcode_in_password": false, - "private_key_path": null, - "authenticator": "snowflake", - "QUERY_TAG": null, - "QUOTED_IDENTIFIERS_IGNORE_CASE": null, - {% if authType == "basic" %} - "private_key": null, - "private_key_passphrase": null, - "password": {{ password | tojson }} - {% else %} - "private_key": {{ password | tojson }}, - "private_key_passphrase": {{ extra.private_key_password | tojson }}, - "password": null - {% endif %} - } - """ - commonInputs { - ["name"] = new TextInput { - title = "Name" - required = false - hide = true - placeholderText = "Host Name" - } - ["connector"] = new TextInput { - title = "Connector" - required = false - hide = true - placeholderText = "Connector" - } - ["connector_type"] = new TextInput { - title = "connectorType" - required = false - hide = true - placeholderText = "connectorType" - } - ["host"] = new TextInput { - title = "Account Identifiers (Host)" - placeholderText = "..aws.snowflakecomputing.com" - prepend = "https://" - width = 6 - } - ["port"] = new NumericInput { - title = "Port" - default = 443 - enabled = false - width = 2 - } - } - options { - ["basic"] { - title = "Basic" - hide = true - inputs { - ["username"] = new TextInput { - title = "Username" - required = true - defaultValue = "atlanadmin" - width = 4 - } - ["password"] = new PasswordInput { - title = "Password" - required = true - width = 4 - } - ["extra"] = new NestedInput { - title = "Role and Warehouse" - inputs { - ["role"] = new SQLExecutor { - title = "Role" - sqlQuery = "show grants" - width = 4 - } - ["warehouse"] = new SQLExecutor { - title = "Warehouse" - sqlQuery = "show warehouses" - width = 4 - } - } - } - } - } - ["keypair"] { - title = "Keypair" - inputs { - ["username"] = new TextInput { - title = "Username" - placeholderText = "Username" - width = 4 - } - ["password"] = new TextBoxInput { - title = "Encrypted Private Key" - placeholderText = "-----BEGIN ENCRYPTED PRIVATE KEY-----MIIE6TAbBgkqhkiG9w0BBQMwDgQILYPyCppzOwECAggABIIEyLiGSpeeGSe3xHP1wHLjfCYycUPennlX2bd8yX8xOxGSGfvB+99+PmSlex0FmY9ov1J8H1H9Y3lMWXbL...-----END ENCRYPTED PRIVATE KEY-----" - width = 4 - } - ["extra"] = new NestedInput { - title = "Private Key Password" - inputs { - ["private_key_password"] = new PasswordInput { - title = "Private Key Password" - width = 5 - } - ["role"] = new SQLExecutor { - title = "Role" - sqlQuery = "show grants" - width = 4 - } - ["warehouse"] = new SQLExecutor { - title = "Warehouse" - sqlQuery = "show warehouses" - width = 4 - } - } - } - } - } - ["okta"] { - title = "OKTA SSO" - inputs { - ["username"] = new TextInput { - title = "Username" - placeholderText = "Username" - width = 4 - } - ["password"] = new PasswordInput { - title = "Password" - width = 4 - } - ["extra"] = new NestedInput { - title = "Private Key Password" - inputs { - ["authenticator"] = new TextInput { - title = "Authenticator" - placeholderText = "Enter your authenticator code" - width = 4 - } - ["role"] = new SQLExecutor { - title = "Role" - sqlQuery = "show grants" - width = 4 - } - ["warehouse"] = new SQLExecutor { - title = "Warehouse" - sqlQuery = "show warehouses" - width = 4 - } - } - } - } - } - } -} diff --git a/package-toolkit/runtime/build.gradle.kts b/package-toolkit/runtime/build.gradle.kts index aeb830057..2f1c3e001 100644 --- a/package-toolkit/runtime/build.gradle.kts +++ b/package-toolkit/runtime/build.gradle.kts @@ -4,6 +4,7 @@ val jarName = "package-toolkit-runtime" plugins { id("com.atlan.kotlin") + id("org.pkl-lang") alias(libs.plugins.shadow) `maven-publish` signing @@ -18,6 +19,7 @@ dependencies { because("version 1.0.0 pulled from elasticsearch-java has CWE-20 (CVE-2023-4043)") } } + api(libs.pkl.config) api(libs.jackson.kotlin) api(libs.fastcsv) api(libs.bundles.poi) @@ -193,6 +195,25 @@ tasks { attributes(Pair("Multi-Release", "true")) } } + + assemble { + dependsOn("genPklConnectors") + } +} + +pkl { + evaluators { + register("genPklConnectors") { + sourceModules.add("src/main/resources/csa-connectors-objectstore.pkl") + modulePath.from(file("../config/src/main/resources")) + outputFormat.set("yaml") + multipleFileOutputDir.set(layout.projectDirectory.dir("build")) + } + } +} + +tasks.getByName("genPklConnectors") { + dependsOn(":package-toolkit:config:generateBuildInfo") } java { diff --git a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/Utils.kt b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/Utils.kt index 27b239125..6b2099d3b 100644 --- a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/Utils.kt +++ b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/Utils.kt @@ -7,12 +7,13 @@ import com.atlan.exception.AtlanException import com.atlan.exception.NotFoundException import com.atlan.model.assets.Connection import com.atlan.model.enums.AssetCreationHandling -import com.atlan.pkg.Utils.getInputFile +import com.atlan.pkg.model.Credential import com.atlan.pkg.objectstore.ADLSSync import com.atlan.pkg.objectstore.GCSSync import com.atlan.pkg.objectstore.ObjectStorageSyncer import com.atlan.pkg.objectstore.S3Sync import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue import jakarta.activation.FileDataSource import jakarta.mail.Message import mu.KLogger @@ -24,7 +25,6 @@ import java.nio.file.Paths import java.util.concurrent.ThreadLocalRandom import java.util.concurrent.atomic.AtomicLong import kotlin.io.path.isDirectory -import kotlin.io.path.name import kotlin.io.path.readText import kotlin.math.round import kotlin.system.exitProcess @@ -499,6 +499,8 @@ object Utils { logger.info { "Cloud details: $cloudDetails" } val contents = Paths.get("tmp", "credentials", "success", "result-0.json").readText() logger.info { "Content: $contents" } + val cred = MAPPER.readValue(contents) + logger.info { "Parsed: $cred" } // val defaultRegion = getEnvVar("AWS_S3_REGION") // val defaultBucket = getEnvVar("AWS_S3_BUCKET_NAME") "$cloudDetails to $outputDirectory" diff --git a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/model/Credential.kt b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/model/Credential.kt new file mode 100644 index 000000000..760c2edd5 --- /dev/null +++ b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/model/Credential.kt @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: Apache-2.0 + Copyright 2023 Atlan Pte. Ltd. */ +package com.atlan.pkg.model + +import com.atlan.model.assets.Connection +import com.fasterxml.jackson.annotation.JsonAutoDetect + +/** + * Captures the details of credentials once they have been retrieved from the vault. + * + * @param authType connectivity option that was selected + * @param host hostname that was provided as part of sensitive credentials (if any) + * @param port port that was provided as part of sensitive credentials (if any) + * @param username sensitive details that were captured as a username + * @param password sensitive details that were captured as a password + * @param extra any additional sensitive details captured as part of the credential + * @param connector the source defined in the credential config + * @param connectorConfigName name of the credential configmap + * @param connectorType the connectorType defined in the credential config + * @param description TBC + * @param connection TBC + * @param id unique identifier (GUID) of this credential + * @param name name of the workflow associated with this credential + * @param isActive whether the credential is active (true) or not (false) + * @param level TBC + * @param metadata TBC + * @param tenantId TBC + * @param createdBy user who created the credential + * @param createdAt time in milliseconds (epoch) at which the user created the credential + * @param updatedAt time in milliseconds (epoch) at which a user last updated the credential + * @param version unique name for the version of the credential + */ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +data class Credential( + val authType: String, + val host: String?, + val port: Int?, + val username: String?, + val password: String?, + val extra: Map?, + val connector: String, + val connectorConfigName: String, + val connectorType: String, + val description: String?, + val connection: Connection?, + val id: String, + val name: String, + val isActive: Boolean, + val level: Any?, + val metadata: Any?, + val tenantId: String, + val createdBy: String, + val createdAt: Long, + val updatedAt: Long, + val version: String, +) diff --git a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/GCSSync.kt b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/GCSSync.kt index d284a4947..8d2a03e97 100644 --- a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/GCSSync.kt +++ b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/GCSSync.kt @@ -26,7 +26,7 @@ class GCSSync( ) : ObjectStorageSyncer { private val storage = if (credentials != null) { StorageOptions.newBuilder().setProjectId(projectId) - .setCredentials(GoogleCredentials.fromStream(FileInputStream(credentials))) + .setCredentials(GoogleCredentials.fromStream(credentials.byteInputStream())) .build().service } else { StorageOptions.newBuilder().setProjectId(projectId).build().service diff --git a/package-toolkit/runtime/src/main/resources/csa-connectors-objectstore.pkl b/package-toolkit/runtime/src/main/resources/csa-connectors-objectstore.pkl new file mode 100644 index 000000000..1e8dddeba --- /dev/null +++ b/package-toolkit/runtime/src/main/resources/csa-connectors-objectstore.pkl @@ -0,0 +1,169 @@ +/* SPDX-License-Identifier: Apache-2.0 + Copyright 2024 Atlan Pte. Ltd. */ +amends "../../../../config/build/resources/main/Credential.pkl" +import "../../../../config/build/resources/main/Config.pkl" + +// TODO: should generate strongly-typed class(es?) to retrieve these details, too (i.e. knowing what maps to username vs password, etc) +name = "csa-connectors-objectstore" +source = "Object Store" +icon = "http://assets.atlan.com/assets/ph-shapes-light.svg" +helpdesk = "https://solutions.atlan.com" +logo = "http://assets.atlan.com/assets/ph-shapes-light.svg" +connectorType = "rest" +optionsTitle = "Cloud object store" +options { + ["s3"] { + title = "S3" + helpText = "Details for accessing information from Amazon S3." + inputs { + ["username"] = new Config.TextInput { + title = "AWS access key" + required = false + helpText = "Enter your AWS access key." + placeholderText = "" + width = 4 + } + ["password"] = new Config.PasswordInput { + title = "AWS secret key" + required = false + helpText = "Enter your AWS secret key." + width = 4 + } + ["extra"] = new Config.NestedInput { + title = "S3 details" + inputs { + ["region"] = new Config.TextInput { + title = "Region" + required = false + helpText = "Enter your AWS region." + placeholderText = "us-west-1" + width = 4 + } + ["s3_bucket"] = new Config.TextInput { + title = "Bucket" + required = false + helpText = "Enter the bucket from which to retrieve the object store object(s)." + placeholderText = "bucket-name" + width = 4 + } + ["s3_prefix"] = new Config.TextInput { + title = "Prefix (path)" + required = false + helpText = "Enter the prefix (path) within the bucket from which to retrieve the object(s)." + placeholderText = "path/to/files" + width = 4 + } + ["s3_key"] = new Config.TextInput { + title = "Object key (filename)" + required = false + helpText = "Enter the object key (filename), including its extension, within the bucket and prefix." + placeholderText = "file.csv" + width = 4 + } + } + } + } + } + ["gcs"] { + title = "GCS" + helpText = "Details for accessing information from Google Cloud Storage (GCS)." + inputs { + ["username"] = new Config.TextInput { + title = "Project ID" + required = false + helpText = "Enter the ID of the GCP project." + placeholderText = "" + width = 4 + } + ["password"] = new Config.TextInput { + title = "Service account JSON" + required = false + helpText = "Enter the JSON for your service account credentials." + placeholderText = "{\"type\":\"service_account\",\"project_id\":\"gcs-connector-399418\", ..." + width = 8 + } + ["extra"] = new Config.NestedInput { + title = "GCS details" + inputs { + ["gcs_bucket"] = new Config.TextInput { + title = "Bucket" + required = false + helpText = "Enter the bucket from which to retrieve the object store object(s)." + placeholderText = "bucket-name" + width = 4 + } + ["gcs_prefix"] = new Config.TextInput { + title = "Prefix (path)" + required = false + helpText = "Enter the prefix (path) within the bucket from which to retrieve the object(s)." + placeholderText = "path/to/files" + width = 4 + } + ["gcs_key"] = new Config.TextInput { + title = "Object key (filename)" + required = false + helpText = "Enter the object key (filename), including its extension, within the bucket and prefix." + placeholderText = "file.csv" + width = 4 + } + } + } + } + } + ["adls"] { + title = "ADLS" + helpText = "Details for access information from Azure Data Lake Storage (ADLS)." + inputs { + ["username"] = new Config.TextInput { + title = "Azure client ID" + required = false + helpText = "Enter the unique application (client) ID assigned to your app by Azure AD when the app was registered." + width = 4 + } + ["password"] = new Config.TextInput { + title = "Azure client secret" + required = false + helpText = "Enter your client secret." + width = 4 + } + ["extra"] = new Config.NestedInput { + title = "ADLS details" + inputs { + ["azure_tenant_id"] = new Config.TextInput { + title = "Azure tenant ID" + required = false + helpText = "Enter the unique identifier of the Azure Active Directory instance." + width = 4 + } + ["storage_account_name"] = new Config.TextInput { + title = "Storage account name" + required = false + helpText = "Enter the name of your storage account." + width = 4 + } + ["adls_container"] = new Config.TextInput { + title = "Container" + required = false + helpText = "Enter the container from which to retrieve the object store object(s)." + placeholderText = "container-name" + width = 4 + } + ["adls_prefix"] = new Config.TextInput { + title = "Directory" + required = false + helpText = "Enter the directory (path) within the container from which to retrieve the object(s)." + placeholderText = "path/to/files" + width = 4 + } + ["adls_key"] = new Config.TextInput { + title = "Object key (filename)" + required = false + helpText = "Enter the object key (filename), including its extension, within the container and prefix." + placeholderText = "file.csv" + width = 4 + } + } + } + } + } +} diff --git a/samples/packages/relational-assets-builder/src/main/resources/package.pkl b/samples/packages/relational-assets-builder/src/main/resources/package.pkl index f5607f915..55c748410 100644 --- a/samples/packages/relational-assets-builder/src/main/resources/package.pkl +++ b/samples/packages/relational-assets-builder/src/main/resources/package.pkl @@ -3,7 +3,6 @@ amends "modulepath:/Config.pkl" import "pkl:semver" import "modulepath:/BuildInfo.pkl" -import "modulepath:/Connectors.pkl" packageId = "@csa/relational-assets-builder" packageName = "Relational Assets Builder" @@ -131,124 +130,3 @@ uiConfig { } } } - -credentialConfig { - name = "csa-connectors-objectstore" - source = Connectors.S3 - icon = "http://assets.atlan.com/assets/ph-shapes-light.svg" - helpdesk = "https://solutions.atlan.com" - logo = "http://assets.atlan.com/assets/ph-shapes-light.svg" - connectorType = "rest" - commonInputs { - ["object_store_bucket"] = new TextInput { - title = "Bucket or container" - required = false - helpText = "Enter the bucket or container from which to retrieve the object store object(s)." - placeholderText = "bucket-name" - width = 4 - } - ["object_store_prefix"] = new TextInput { - title = "Prefix (path)" - required = false - helpText = "Enter the prefix (path) within the bucket from which to retrieve the object(s)." - placeholderText = "path/to/files" - width = 4 - } - ["object_store_key"] = new TextInput { - title = "Object key (filename)" - required = false - helpText = "Enter the object key (filename), including its extension, within the bucket and prefix." - placeholderText = "file.csv" - width = 4 - } - } - optionsTitle = "Cloud object store" - options { - ["s3"] = new NestedInput { - title = "S3" - helpText = "Details for accessing information from Amazon S3." - inputs { - ["username"] = new TextInput { - title = "AWS access key" - required = false - helpText = "Enter your AWS access key." - placeholderText = "" - width = 4 - } - ["password"] = new PasswordInput { - title = "AWS secret key" - required = false - helpText = "Enter your AWS secret key." - width = 4 - } - ["extra"] = new NestedInput { - title = "S3 details" - inputs { - ["region"] = new TextInput { - title = "Region" - required = false - helpText = "Enter your AWS region." - placeholderText = "us-west-1" - width = 4 - } - } - } - } - } - ["gcs"] = new NestedInput { - title = "GCS" - helpText = "Details for accessing information from Google Cloud Storage (GCS)." - inputs { - ["username"] = new TextInput { - title = "Project ID" - required = false - helpText = "Enter the ID of the GCP project." - placeholderText = "" - width = 4 - } - ["password"] = new TextInput { - title = "Service account JSON" - required = false - helpText = "Enter the JSON for your service account credentials." - placeholderText = "{\"type\":\"service_account\",\"project_id\":\"gcs-connector-399418\", ..." - width = 8 - } - } - } - ["adls"] = new NestedInput { - title = "ADLS" - helpText = "Details for access information from Azure Data Lake Storage (ADLS)." - inputs { - ["username"] = new TextInput { - title = "Azure client ID" - required = false - helpText = "Enter the unique application (client) ID assigned to your app by Azure AD when the app was registered." - width = 4 - } - ["password"] = new TextInput { - title = "Azure client secret" - required = false - helpText = "Enter your client secret." - width = 4 - } - ["extra"] = new NestedInput { - title = "ADLS details" - inputs { - ["azure_tenant_id"] = new TextInput { - title = "Azure tenant ID" - required = false - helpText = "Enter the unique identifier of the Azure Active Directory instance." - width = 4 - } - ["storage_account_name"] = new TextInput { - title = "Storage account name" - required = false - helpText = "Enter the name of your storage account." - width = 4 - } - } - } - } - } - } -} diff --git a/samples/typedefs/build.gradle.kts b/samples/typedefs/build.gradle.kts index c26f4c96f..f1b42826d 100644 --- a/samples/typedefs/build.gradle.kts +++ b/samples/typedefs/build.gradle.kts @@ -3,13 +3,19 @@ plugins { id("com.atlan.kotlin-custom-typedef") } +tasks { + assemble { + dependsOn("genPklTypedefs") + } +} + pkl { evaluators { register("genPklTypedefs") { sourceModules.add("src/main/resources/MultiDimensionalDataset.pkl") modulePath.from(file("../../typedef-toolkit/model/src/main/resources")) outputFormat.set("json") - multipleFileOutputDir.set(layout.projectDirectory.dir("build/generated")) + multipleFileOutputDir.set(layout.projectDirectory.dir("build")) } } } diff --git a/samples/typedefs/src/main/resources/MultiDimensionalDataset.pkl b/samples/typedefs/src/main/resources/MultiDimensionalDataset.pkl index c1b2bd24d..a5fcb4140 100644 --- a/samples/typedefs/src/main/resources/MultiDimensionalDataset.pkl +++ b/samples/typedefs/src/main/resources/MultiDimensionalDataset.pkl @@ -12,6 +12,29 @@ local qualifiedName = "QualifiedName" local supertypeName = "MultiDimensionalDataset" local attrName = "Name" +local cube = new Icon { + name = "CubeGray" + nameActive = "Cube" + svg = "cube-light-gray.svg" + svgActive = "cube-light.svg" +} +local cubeDimension = new Icon { + name = "CubeDimensionGray" + nameActive = "CubeDimension" + svg = "square-half-light-gray.svg" + svgActive = "square-half-light.svg" +} +local schema = new Icon { + name = "SchemaGray" + nameActive = "Schema" + svg = "schema-gray.svg" + svgActive = "schema.svg" +} +local pivotTable = new Icon { + name = "PivotTable" + svg = "pivot-table.svg" +} + shared { supertypeDefinition { name = supertypeName @@ -19,6 +42,7 @@ shared { superTypes { "Catalog" } attributes { ["\(a)\(attrName)"] { + label = "Cube" description = "Simple name of the cube in which this asset exists, or empty if it is itself a cube." type = "string" indexAs = "both" @@ -29,6 +53,7 @@ shared { indexAs = "keyword" } ["\(a)\(dimension)\(attrName)"] { + label = "Dimension" description = "Simple name of the cube dimension in which this asset exists, or empty if it is itself a dimension." type = "string" indexAs = "both" @@ -39,6 +64,7 @@ shared { indexAs = "keyword" } ["\(a)\(hierarchy)\(attrName)"] { + label = "Hierarchy" description = "Simple name of the dimension hierarchy in which this asset exists, or empty if it is itself a hierarchy." type = "string" indexAs = "both" @@ -50,24 +76,58 @@ shared { } } } + ui { + svgName = "MultiDimensional.svg" + filters { + [t] { + attribute = "\(a)\(qualifiedName)" + } + ["\(t)\(dimension)"] { + attribute = "\(a)\(dimension)\(qualifiedName)" + } + } + breadcrumb { + [t] { + q = "\(a)\(qualifiedName)" + n = "\(a)\(attrName)" + } + ["\(t)\(dimension)"] { + q = "\(a)\(dimension)\(qualifiedName)" + n = "\(a)\(dimension)\(attrName)" + } + ["\(t)\(hierarchy)"] { + q = "\(a)\(hierarchy)\(qualifiedName)" + n = "\(a)\(hierarchy)\(attrName)" + } + } + } } customTypes { [t] { + label = t + icon = cube description = "Instance of a cube in Atlan." attributes { ["\(a)\(dimension)Count"] { + label = dimension.decapitalize() description = "Number of dimensions in the cube." type = "long" + childCount = true } } } ["\(t)\(dimension)"] { + label = dimension + icon = cubeDimension description = "Instance of a cube dimension in Atlan." + parentQualifiedName = "\(a)\(qualifiedName)" attributes { ["\(a)\(hierarchy)Count"] { + label = hierarchy.decapitalize() description = "Number of hierarchies in the cube dimension." type = "long" + childCount = true } } relationships { @@ -88,11 +148,17 @@ customTypes { } } ["\(t)\(hierarchy)"] { + label = hierarchy + labelPlural = "Hierarchies" + icon = schema description = "Instance of a cube hierarchy in Atlan." + parentQualifiedName = "\(a)\(dimension)\(qualifiedName)" attributes { ["\(a)\(field)Count"] { + label = field.decapitalize() description = "Number of total fields in the cube hierarchy." type = "long" + childCount = true } } relationships { @@ -113,9 +179,13 @@ customTypes { } } ["\(t)\(field)"] { + label = field + icon = pivotTable description = "Instance of a cube field in Atlan." + parentQualifiedName = "\(a)\(hierarchy)\(qualifiedName)" attributes { ["\(a)Parent\(field)Name"] { + label = "Parent field" description = "Name of the parent field in which this field is nested." type = "string" indexAs = "both" @@ -126,17 +196,21 @@ customTypes { indexAs = "keyword" } ["\(a)\(field)Level"] { + label = "Level" description = "Level of the field in the cube hierarchy." type = "long" } ["\(a)\(field)MeasureExpression"] { + label = "Measure expression" description = "Expression used to calculate this measure." type = "string" indexAs = "both" } ["\(a)Sub\(field)Count"] { + label = "sub\(field.decapitalize())" description = "Number of sub-fields that are direct children of this field." type = "long" + childCount = true } } relationships { diff --git a/typedef-toolkit/model/src/test/kotlin/CanonicalExampleTest.kt b/typedef-toolkit/model/src/test/kotlin/CanonicalExampleTest.kt index fa1d2e649..57390587a 100644 --- a/typedef-toolkit/model/src/test/kotlin/CanonicalExampleTest.kt +++ b/typedef-toolkit/model/src/test/kotlin/CanonicalExampleTest.kt @@ -88,7 +88,7 @@ object CanonicalExampleTest { assertEquals("Table", table.superTypes[1]) assertEquals(1, table.attributeDefs.size) assertEquals("customRatings", table.attributeDefs[0].name) - assertEquals("Ratings for the CustomTable asset from the source system.", table.attributeDefs!![0].description) + assertEquals("Ratings for the CustomTable asset from the source system.", table.attributeDefs[0].description) assertEquals("array", table.attributeDefs[0].typeName) } diff --git a/typedef-toolkit/model/src/test/kotlin/ModelUnitTest.kt b/typedef-toolkit/model/src/test/kotlin/ModelUnitTest.kt index fbe5cce21..3374914c2 100644 --- a/typedef-toolkit/model/src/test/kotlin/ModelUnitTest.kt +++ b/typedef-toolkit/model/src/test/kotlin/ModelUnitTest.kt @@ -117,6 +117,6 @@ class ModelUnitTest { } private fun getAttribute(model: Model): Model.AttributeDef { - return model.shared.supertypeDefinition.attributeDefs?.get(0)!! + return model.shared.supertypeDefinition.attributeDefs[0] } } From 80fbe4b00844e777478a3f1a904143e7aeacbde7 Mon Sep 17 00:00:00 2001 From: Christopher Grote Date: Fri, 24 May 2024 10:48:50 +0100 Subject: [PATCH 2/3] Adds secure credential-based object store support Signed-off-by: Christopher Grote --- gradle/libs.versions.toml | 2 ++ package-toolkit/runtime/build.gradle.kts | 1 + .../src/main/kotlin/com/atlan/pkg/Utils.kt | 36 ++++++++++++++----- .../kotlin/com/atlan/pkg/model/Credential.kt | 3 +- .../atlan/pkg/objectstore/ADLSCredential.kt | 15 ++++++++ .../com/atlan/pkg/objectstore/ADLSSync.kt | 16 +++++++-- .../atlan/pkg/objectstore/GCSCredential.kt | 13 +++++++ .../com/atlan/pkg/objectstore/S3Credential.kt | 15 ++++++++ .../com/atlan/pkg/objectstore/S3Sync.kt | 22 +++++++++++- 9 files changed, 108 insertions(+), 15 deletions(-) create mode 100644 package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/ADLSCredential.kt create mode 100644 package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/GCSCredential.kt create mode 100644 package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/S3Credential.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4173df9c4..9f4470fd8 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -24,6 +24,7 @@ jakarta-mail = "2.1.3" angus-mail = "2.0.3" pkl = "0.25.3" adls = "12.19.0" +azure = "1.12.1" [libraries] jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson" } @@ -44,6 +45,7 @@ gcs-bom = { module = "com.google.cloud:libraries-bom", version.ref = "gcs" } gcs = { module = "com.google.cloud:google-cloud-storage" } gcs-control = { module = "com.google.cloud:google-cloud-storage-control" } adls = { module = "com.azure:azure-storage-file-datalake", version.ref = "adls" } +azure-identity = { module = "com.azure:azure-identity", version.ref = "azure" } system-stubs = { module = "uk.org.webcompere:system-stubs-testng", version.ref = "system-stubs" } fastcsv = { module = "de.siegmar:fastcsv", version.ref = "fastcsv" } apache-poi = { module = "org.apache.poi:poi", version.ref = "poi" } diff --git a/package-toolkit/runtime/build.gradle.kts b/package-toolkit/runtime/build.gradle.kts index 2f1c3e001..c27d0829b 100644 --- a/package-toolkit/runtime/build.gradle.kts +++ b/package-toolkit/runtime/build.gradle.kts @@ -26,6 +26,7 @@ dependencies { api(libs.awssdk.s3) api(platform(libs.gcs.bom)) api(libs.gcs) + api(libs.azure.identity) api(libs.adls) implementation(libs.sqlite) implementation(libs.simple.java.mail) diff --git a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/Utils.kt b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/Utils.kt index 6b2099d3b..8f6e81d5b 100644 --- a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/Utils.kt +++ b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/Utils.kt @@ -8,9 +8,12 @@ import com.atlan.exception.NotFoundException import com.atlan.model.assets.Connection import com.atlan.model.enums.AssetCreationHandling import com.atlan.pkg.model.Credential +import com.atlan.pkg.objectstore.ADLSCredential import com.atlan.pkg.objectstore.ADLSSync +import com.atlan.pkg.objectstore.GCSCredential import com.atlan.pkg.objectstore.GCSSync import com.atlan.pkg.objectstore.ObjectStorageSyncer +import com.atlan.pkg.objectstore.S3Credential import com.atlan.pkg.objectstore.S3Sync import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue @@ -496,14 +499,29 @@ object Utils { return if (preferUpload) { uploadResult } else { - logger.info { "Cloud details: $cloudDetails" } - val contents = Paths.get("tmp", "credentials", "success", "result-0.json").readText() - logger.info { "Content: $contents" } + val contents = Paths.get("/tmp", "credentials", "success", "result-0.json").readText() val cred = MAPPER.readValue(contents) - logger.info { "Parsed: $cred" } - // val defaultRegion = getEnvVar("AWS_S3_REGION") - // val defaultBucket = getEnvVar("AWS_S3_BUCKET_NAME") - "$cloudDetails to $outputDirectory" + when (cred.authType) { + "s3" -> { + val s3 = S3Credential(cred) + val sync = S3Sync(s3.bucket, s3.region, logger, s3.accessKey, s3.secretKey) + getInputFile(sync, "${s3.objectPrefix}/${s3.objectKey}", outputDirectory) + } + "gcs" -> { + val gcs = GCSCredential(cred) + val sync = GCSSync(gcs.projectId, gcs.bucket, logger, gcs.serviceAccountJson) + getInputFile(sync, "${gcs.objectPrefix}/${gcs.objectKey}", outputDirectory) + } + "adls" -> { + val adls = ADLSCredential(cred) + val sync = ADLSSync(adls.storageAccount, adls.containerName, logger, adls.tenantId, adls.clientId, adls.clientSecret) + getInputFile(sync, "${adls.objectPrefix}/${adls.objectKey}", outputDirectory) + } + else -> { + logger.warn { "Unknown source ${cred.authType} -- skipping." } + "" + } + } } } @@ -555,7 +573,7 @@ object Utils { val sync = GCSSync(gcsProjectId, gcsBucket, logger, gcsCredentials) getInputFile(sync, gcsObjectKey, outputDirectory) } else if (adlsObjectKey.isNotBlank()) { - val sync = ADLSSync(adlsAccountName, adlsContainerName, logger, adlsSasToken) + val sync = ADLSSync(adlsAccountName, adlsContainerName, logger, "", "", "") getInputFile(sync, adlsObjectKey, outputDirectory) } else { "" @@ -625,7 +643,7 @@ object Utils { val sync = GCSSync(gcsProjectId, gcsBucket, logger, gcsCredentials) sync.copyFrom(gcsObjectKey, outputDirectory) } else if (adlsObjectKey.isNotBlank() && adlsObjectKey.endsWith("/")) { - val sync = ADLSSync(adlsAccountName, adlsContainerName, logger, adlsSasToken) + val sync = ADLSSync(adlsAccountName, adlsContainerName, logger, "", "", "") sync.copyFrom(adlsObjectKey, outputDirectory) } else if (s3ObjectKey.isNotBlank() || gcsObjectKey.isNotBlank() || adlsObjectKey.isNotBlank()) { listOf( diff --git a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/model/Credential.kt b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/model/Credential.kt index 760c2edd5..4314fb752 100644 --- a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/model/Credential.kt +++ b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/model/Credential.kt @@ -2,7 +2,6 @@ Copyright 2023 Atlan Pte. Ltd. */ package com.atlan.pkg.model -import com.atlan.model.assets.Connection import com.fasterxml.jackson.annotation.JsonAutoDetect /** @@ -42,7 +41,7 @@ data class Credential( val connectorConfigName: String, val connectorType: String, val description: String?, - val connection: Connection?, + val connection: Any?, val id: String, val name: String, val isActive: Boolean, diff --git a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/ADLSCredential.kt b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/ADLSCredential.kt new file mode 100644 index 000000000..6c000aa75 --- /dev/null +++ b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/ADLSCredential.kt @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: Apache-2.0 + Copyright 2023 Atlan Pte. Ltd. */ +package com.atlan.pkg.objectstore + +import com.atlan.pkg.model.Credential + +data class ADLSCredential(val from: Credential) { + val clientId = from.username ?: "" + val clientSecret = from.password ?: "" + val tenantId = (from.extra?.get("azure_tenant_id") ?: "") as String + val storageAccount = (from.extra?.get("storage_account_name") ?: "") as String + val containerName = (from.extra?.get("adls_container") ?: "") as String + val objectPrefix = (from.extra?.get("adls_prefix") ?: "") as String + val objectKey = (from.extra?.get("adls_key") ?: "") as String +} diff --git a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/ADLSSync.kt b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/ADLSSync.kt index 63edb3aa7..0db110ab8 100644 --- a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/ADLSSync.kt +++ b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/ADLSSync.kt @@ -2,6 +2,7 @@ Copyright 2023 Atlan Pte. Ltd. */ package com.atlan.pkg.objectstore +import com.azure.identity.ClientSecretCredentialBuilder import com.azure.storage.file.datalake.DataLakeServiceClientBuilder import com.azure.storage.file.datalake.models.ListPathsOptions import mu.KLogger @@ -13,17 +14,26 @@ import java.io.File * @param accountName name of the Azure account * @param containerName name of the container in ADLS to use for syncing * @param logger through which to record any problems - * @param sasToken shared access signature (SAS) token + * @param tenantId unique identifier (GUID) of the tenant + * @param clientId unique identifier (GUID) of the client + * @param clientSecret value of the secret for the client (note this is not the GUID of the client secret) */ class ADLSSync( private val accountName: String, private val containerName: String, private val logger: KLogger, - private val sasToken: String, + private val tenantId: String, + private val clientId: String, + private val clientSecret: String, ) : ObjectStorageSyncer { + private val credential = ClientSecretCredentialBuilder() + .tenantId(tenantId) + .clientId(clientId) + .clientSecret(clientSecret) + .build() private val adlsClient = DataLakeServiceClientBuilder() .endpoint("https://$accountName.dfs.core.windows.net") - .sasToken(sasToken) + .credential(credential) .buildClient() /** {@inheritDoc} */ diff --git a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/GCSCredential.kt b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/GCSCredential.kt new file mode 100644 index 000000000..b6446b6dc --- /dev/null +++ b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/GCSCredential.kt @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: Apache-2.0 + Copyright 2023 Atlan Pte. Ltd. */ +package com.atlan.pkg.objectstore + +import com.atlan.pkg.model.Credential + +data class GCSCredential(val from: Credential) { + val projectId = from.username ?: "" + val serviceAccountJson = from.password ?: "" + val bucket = (from.extra?.get("gcs_bucket") ?: "") as String + val objectPrefix = (from.extra?.get("gcs_prefix") ?: "") as String + val objectKey = (from.extra?.get("gcs_key") ?: "") as String +} diff --git a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/S3Credential.kt b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/S3Credential.kt new file mode 100644 index 000000000..eacc216c0 --- /dev/null +++ b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/S3Credential.kt @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: Apache-2.0 + Copyright 2023 Atlan Pte. Ltd. */ +package com.atlan.pkg.objectstore + +import com.atlan.pkg.Utils.getEnvVar +import com.atlan.pkg.model.Credential + +data class S3Credential(val from: Credential) { + val accessKey = from.username ?: "" + val secretKey = from.password ?: "" + val region = (from.extra?.get("region") ?: getEnvVar("AWS_S3_REGION")) as String + val bucket = (from.extra?.get("s3_bucket") ?: getEnvVar("AWS_S3_BUCKET_NAME")) as String + val objectPrefix = (from.extra?.get("s3_prefix") ?: "") as String + val objectKey = (from.extra?.get("s3_key") ?: "") as String +} diff --git a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/S3Sync.kt b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/S3Sync.kt index 21a1125c0..13798659d 100644 --- a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/S3Sync.kt +++ b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/objectstore/S3Sync.kt @@ -3,6 +3,8 @@ package com.atlan.pkg.objectstore import mu.KLogger +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.model.GetObjectRequest @@ -16,14 +18,32 @@ import java.io.File * @param bucketName name of the bucket in S3 to use for syncing * @param region AWS region through which to sync S3 * @param logger through which to record any problems + * @param accessKey (optional) AWS access key, if using as the form of authentication + * @param secretKey (optional) AWS secret key, if using as the form of authentication */ class S3Sync( private val bucketName: String, private val region: String, private val logger: KLogger, + private val accessKey: String = "", + private val secretKey: String = "", ) : ObjectStorageSyncer { - private val s3Client = S3Client.builder().region(Region.of(region)).build() + private val credential = if (accessKey.isNotBlank()) { + AwsBasicCredentials.create(accessKey, secretKey) + } else { + null + } + private val s3Client = if (credential != null) { + S3Client.builder() + .credentialsProvider(StaticCredentialsProvider.create(credential)) + .region(Region.of(region)) + .build() + } else { + S3Client.builder() + .region(Region.of(region)) + .build() + } /** {@inheritDoc} */ override fun copyFrom(prefix: String, localDirectory: String): List { From c62148f95ca7b326d88dd8e3dc8329024a474eff Mon Sep 17 00:00:00 2001 From: Christopher Grote Date: Fri, 24 May 2024 11:05:57 +0100 Subject: [PATCH 3/3] Add missing dependencies for GCS, ADLS Signed-off-by: Christopher Grote --- package-toolkit/runtime/build.gradle.kts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/package-toolkit/runtime/build.gradle.kts b/package-toolkit/runtime/build.gradle.kts index c27d0829b..d8551a295 100644 --- a/package-toolkit/runtime/build.gradle.kts +++ b/package-toolkit/runtime/build.gradle.kts @@ -86,6 +86,7 @@ tasks { include(dependency("commons-logging:commons-logging:.*")) include(dependency("commons-codec:commons-codec:.*")) // GCS + include(dependency("com.google.cloud:google-cloud-storage:.*")) include(dependency("com.google.guava:guava:.*")) include(dependency("com.google.guava:failureaccess:.*")) include(dependency("com.google.guava:listenablefuture:.*")) @@ -147,6 +148,10 @@ tasks { include(dependency("com.google.re2j:re2j:.*")) include(dependency("io.grpc:grpc-rls:.*")) // ADLS + include(dependency("com.azure:azure-identity:.*")) + include(dependency("com.microsoft.azure:msal4j:.*")) + include(dependency("com.microsoft.azure:msal4j-persistence-extension:.*")) + include(dependency("net.java.dev.jna:jna-platform:.*")) include(dependency("com.azure:azure-core-http-netty:.*")) include(dependency("com.azure:azure-core:.*")) include(dependency("com.azure:azure-json:.*"))