From 73243c49080c86aef54574747d13665ebac9da9b Mon Sep 17 00:00:00 2001 From: Louis Christopher Date: Tue, 21 Feb 2023 17:39:20 +0530 Subject: [PATCH 1/6] Release 0.9.2 --- .github/workflows/npm-publish.yml | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index d338f39..c946168 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -13,9 +13,9 @@ jobs: - name: Set up Node.js uses: actions/setup-node@master with: - node-version: 10.0.0 + node-version: 16.13.0 - name: Publish if version has been updated - uses: pascalgn/npm-publish-action@06e0830ea83eea10ed4a62654eeaedafb8bf50fc + uses: pascalgn/npm-publish-action@1.3.9 with: # All of theses inputs are optional tag_name: "v%s" tag_message: "v%s" diff --git a/package.json b/package.json index 3737419..a2a18b5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "argopm", - "version": "0.9.1", + "version": "0.9.2", "description": "Argo package manager", "main": "./lib/index.js", "scripts": { From f3dd23ab5485abd79a56bc1cf123707c058700d4 Mon Sep 17 00:00:00 2001 From: Mrunmayi Date: Tue, 11 Apr 2023 14:20:13 +0530 Subject: [PATCH 2/6] Updating folders setup to build correct default folder structure --- bin/install.js | 56 ++ lib/connectors/models/BI/default.json | 35 + lib/connectors/packages/BI/README.md | 0 .../packages/BI/configmaps/default.yaml | 153 +++++ lib/connectors/packages/BI/index.js | 4 + lib/connectors/packages/BI/package.json | 53 ++ lib/connectors/packages/BI/templates/aws.yaml | 649 ++++++++++++++++++ .../packages/BI/templates/basic+jwt.yaml | 633 +++++++++++++++++ .../packages/BI/templates/oauth.yaml | 581 ++++++++++++++++ .../packages/BI/transformers/default.jinja2 | 19 + .../packages/common-configmap/api.yaml | 166 +++++ .../packages/common-configmap/aws.yaml | 244 +++++++ .../packages/common-configmap/basic.yaml | 151 ++++ .../packages/common-configmap/jdbc-basic.yaml | 180 +++++ lib/connectors/scripts/BI/__init__.py | 0 lib/connectors/scripts/BI/main.py | 36 + lib/connectors/scripts/BI/tests/input/test.py | 18 + lib/init.js | 187 ++++- package.json | 5 +- 19 files changed, 3168 insertions(+), 2 deletions(-) create mode 100644 lib/connectors/models/BI/default.json create mode 100644 lib/connectors/packages/BI/README.md create mode 100644 lib/connectors/packages/BI/configmaps/default.yaml create mode 100644 lib/connectors/packages/BI/index.js create mode 100644 lib/connectors/packages/BI/package.json create mode 100644 lib/connectors/packages/BI/templates/aws.yaml create mode 100644 lib/connectors/packages/BI/templates/basic+jwt.yaml create mode 100644 lib/connectors/packages/BI/templates/oauth.yaml create mode 100644 lib/connectors/packages/BI/transformers/default.jinja2 create mode 100644 lib/connectors/packages/common-configmap/api.yaml create mode 100644 lib/connectors/packages/common-configmap/aws.yaml create mode 100644 lib/connectors/packages/common-configmap/basic.yaml create mode 100644 lib/connectors/packages/common-configmap/jdbc-basic.yaml create mode 100644 lib/connectors/scripts/BI/__init__.py create mode 100644 lib/connectors/scripts/BI/main.py create mode 100644 lib/connectors/scripts/BI/tests/input/test.py diff --git a/bin/install.js b/bin/install.js index 35c4512..3f1f3df 100755 --- a/bin/install.js +++ b/bin/install.js @@ -3,7 +3,9 @@ const { install, installGlobal } = require("../lib/install.js"); const { initHelp, installHelp } = require("../lib/help"); const { uninstall, info, run, list } = require("../lib/index"); const init = require("../lib/init").init; +const cinit = require("../lib/init").cinit; +const yargsInteractive = require("yargs-interactive"); const yargs = require("yargs"); const { cyan, dim, bright } = require("ansicolor"); @@ -13,6 +15,39 @@ const asTable = require("as-table").configure({ dash: bright(cyan("-")), }); +const options = { + name: { + type: "input", + describe: "Enter connector name", + }, + type: { + type: "list", + describe: "Enter connector type", + choices: ["BI", "SQL", "ETL"], + }, + auth: { + type: "checkbox", + describe: "Enter oauth type", + choices: ["basic", "api", "oauth2", "aws", "gcp"], + }, + AssetList: { + type: "input", + describe: "Enter asset names in comma separated list", + }, + models: { + type: "confirm", + describe: "Build models?", + }, + scripts: { + type: "confirm", + describe: "Build scripts?", + }, + linear: { + type: "confirm", + describe: "Create linear task with subtask for package?", + }, +}; + yargs .command({ command: "install ", @@ -151,6 +186,27 @@ yargs }); }, }) + .command({ + command: "c-init [package_name]", + desc: "Initializes an connector Argo package inside the current working directory", + builder: () => + yargsInteractive() + .usage("$0 connector-init [args]") + .interactive(options) + .then((result) => { + console.log(result); + cinit( + "false", + result.name, + result.type, + result.auth, + result.AssetList, + result.models, + result.scripts, + result.linear + ); + }), + }) .command({ command: "list", aliases: ["l"], diff --git a/lib/connectors/models/BI/default.json b/lib/connectors/models/BI/default.json new file mode 100644 index 0000000..eefc78b --- /dev/null +++ b/lib/connectors/models/BI/default.json @@ -0,0 +1,35 @@ +{ + "enumDefs": [], + "entityDefs": [ + { + "superTypes": [ + "NAME" + ], + "name": "ASSET_NAME", + "description": "Atlan ASSET_NAME Asset", + "typeVersion": "1.1", + "serviceType": "atlan", + "attributeDefs": [ + { + "name": "<>", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "skipScrubbing": true, + "isIndexable": true, + "includeInNotification": true, + "indexType": "STRING", + "indexTypeESFields": { + "text": { + "type": "text", + "analyzer": "atlan_text_analyzer" + } + } + } + ] + } + ], + "relationshipDefs": [] + } + \ No newline at end of file diff --git a/lib/connectors/packages/BI/README.md b/lib/connectors/packages/BI/README.md new file mode 100644 index 0000000..e69de29 diff --git a/lib/connectors/packages/BI/configmaps/default.yaml b/lib/connectors/packages/BI/configmaps/default.yaml new file mode 100644 index 0000000..2637460 --- /dev/null +++ b/lib/connectors/packages/BI/configmaps/default.yaml @@ -0,0 +1,153 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlan-NAME +data: + NAME-api: "10" + config: | + { + "properties": { + "connection": { + "type": "string", + "required": false, + "ui": { + "widget": "connection", + "label": "", + "placeholder": "Connection Name", + "connectionOptions": false + } + }, + "publish-mode": { + "type": "string", + "enum": ["production", "test", "dev"], + "default": "production", + "enumNames": ["Production", "Test", "Development"], + "ui": { + "widget": "select", + "label": "Run Mode", + "grid": 4, + "placeholder": "Connection Mode" + } + }, + "credentials-fetch-strategy": { + "type": "string", + "enum": ["k8s_secret", "credential_guid"], + "default": "credential_guid", + "enumNames": ["k8s Secret Key", "Credential Guid"], + "ui": { + "widget": "select", + "label": "Credential Type", + "placeholder": "Credential Type" + } + }, + "credential-guid": { + "type": "string", + "ui": { + "widget": "credential", + "label": "", + "credentialType": "atlan-connectors-NAME", + "placeholder": "Credential Guid", + "hidden": false + } + }, + "credential-kube-secret-name": { + "type": "string", + "ui": { + "label": "Credential Secret Name", + "placeholder": "Credential Secret Name", + "hidden": true + } + }, + "atlas-auth-type": { + "type": "string", + "enum": ["internal", "apikey"], + "default": "internal", + "enumNames": ["Internal", "API Key"], + "ui": { + "widget": "select", + "label": "Atlas Authentication Type", + "placeholder": "Atlas Authentication Type", + "hidden": true + } + }, + "runtime-properties": { + "type": "object", + "ui": { + "label": "Run time properties", + "hidden": true + } + }, + "include-filter": { + "type": "object", + "additionalProperties": { + "type": "array" + }, + "default": "{}", + "ui": { + "widget": "apitree", + "connectorConfigName": "atlan-connectors-NAME", + "credential": "credential-guid", + "label": "Include Metadata", + "description": "Selected assets only will be processed. Exclude gets preference over include.", + "grid": 4 + } + }, + "exclude-filter": { + "type": "object", + "additionalProperties": { + "type": "array" + }, + "default": "{}", + "ui": { + "widget": "apitree", + "connectorConfigName": "atlan-connectors-NAME", + "credential": "credential-guid", + "label": "Exclude Metadata", + "description": "Selected assets will not be processed.", + "grid": 4 + } + } + }, + "anyOf": [ + { + "properties": { + "credentials-fetch-strategy": { + "const": "k8s_secret" + } + }, + "required": ["credential-kube-secret-name"] + }, + { + "properties": { + "credentials-fetch-strategy": { + "const": "credential_guid" + } + }, + "required": ["credential-guid"] + } + ], + "steps": [ + { + "id": "credential", + "title": "Credential", + "description": "Credential Details", + "properties": [ + "credential-guid" + ] + }, + { + "id": "connection", + "title": "Connection", + "description": "Connection Details", + "properties": [ + "connection" + ] + }, + { + "id": "metadata", + "title": "Metadata", + "description": "Metadata", + "properties": ["include-filter", "exclude-filter"] + } + ] + } \ No newline at end of file diff --git a/lib/connectors/packages/BI/index.js b/lib/connectors/packages/BI/index.js new file mode 100644 index 0000000..fbecdc9 --- /dev/null +++ b/lib/connectors/packages/BI/index.js @@ -0,0 +1,4 @@ +function dummy() { + console.log("don't call this dummy."); +} +module.exports = dummy; diff --git a/lib/connectors/packages/BI/package.json b/lib/connectors/packages/BI/package.json new file mode 100644 index 0000000..bbc52ab --- /dev/null +++ b/lib/connectors/packages/BI/package.json @@ -0,0 +1,53 @@ +{ + "name": "@atlan/NAME", + "version": "0.3.6", + "description": "Package to crawl NAME assets and publish to Atlan for discovery", + "keywords": [ + "NAME", + "bi", + "connector", + "crawler" + ], + "main": "index.js", + "scripts": {}, + "author": { + "name": "Atlan", + "email": "hello@atlan.com", + "url": "https://atlan.com" + }, + "repository": { + "type": "git", + "url": "https://github.com/atlanhq/marketplace-packages.git" + }, + "license": "MIT", + "dependencies": { + "@atlan/crawler": "^2.1.5", + "@atlan/sql-parser": "^0.3.16", + "rest-api": "^0.8.5", + "utils": "^0.2.4" + }, + "bugs": { + "url": "https://atlan.com", + "email": "support@atlan.com" + }, + "config": { + "labels": { + "orchestration.atlan.com/verified": "true", + "orchestration.atlan.com/type": "connector", + "orchestration.atlan.com/source": "NAME", + "orchestration.atlan.com/sourceCategory": "bi", + "orchestration.atlan.com/certified": "true" + }, + "annotations": { + "orchestration.atlan.com/name": "NAME Assets", + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/categories": "NAME,crawler", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/NAME.svg", + "orchestration.atlan.com/marketplaceLink": "https://packages.atlan.com/-/web/detail/@atlan/NAME", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/NAME.svg", + "orchestration.atlan.com/docsUrl": "" + } + } +} diff --git a/lib/connectors/packages/BI/templates/aws.yaml b/lib/connectors/packages/BI/templates/aws.yaml new file mode 100644 index 0000000..22034e3 --- /dev/null +++ b/lib/connectors/packages/BI/templates/aws.yaml @@ -0,0 +1,649 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: atlan-NAME +spec: + entrypoint: main + templates: + - name: main + inputs: + parameters: + #credential + - name: credentials-fetch-strategy + value: "credential_guid" + enum: + - "k8s_secret" + - "credential_guid" + - name: credential-kube-secret-name + value: "" + - name: credential-guid + value: "" + #connection atlan object + - name: connection + value: "" + # extraction + - name: runtime-properties + value: "{}" + - name: fetch-folderless-assets + value: "true" + - name: include-filter + value: "{}" + - name: exclude-filter + value: "{}" + #publish mode + - name: publish-mode + value: "production" + enum: + - "production" + - "test" + - "dev" + + #publish credential + - name: atlas-auth-type + value: "internal" + enum: + - "internal" + - "apikey" + + # enable classification match + - name: auto-classification + value: "false" + enum: + - "true" + - "false" + + - name: attachment-confidence-threshold + value: "0.8" + # Scripts + - name: marketplace-scripts-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplaceScriptsBranch" + optional: true + default: "master" + # Revisions + - name: marketplace-packages-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplacePackagesBranch" + optional: true + default: "master" + dag: + tasks: + - name: extract + template: extract-NAME-metadata + arguments: + parameters: + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: connection-qualified-name + value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + - name: output-prefix + value: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/extracted-metadata/{{workflow.name}}" + - name: threads + value: "2" + - name: fetch-folderless-assets + value: "{{inputs.parameters.fetch-folderless-assets}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: process + template: process-NAME-metadata + depends: "extract.Succeeded" + arguments: + parameters: + - name: connection-qualified-name + value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + - name: fetch-folderless-assets + value: "{{inputs.parameters.fetch-folderless-assets}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: publish + depends: "process.Succeeded" + template: publish-NAME-metadata + arguments: + parameters: + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.publish-mode}}" + - name: source + value: "NAME" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: fetch-folderless-assets + value: "{{inputs.parameters.fetch-folderless-assets}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: exclude-workbook-regex + value: "{{inputs.parameters.exclude-workbook-regex}}" + - name: atlan-web-kube-secret + value: "argo-client-creds" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: atlas-api-uri + value: "http://atlas-service-atlas.atlas.svc.cluster.local/api/atlas/v2" + - name: publish-chunk-size + value: "100" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + + - name: extract-NAME-metadata + inputs: + parameters: + - name: credential-guid + - name: connection-qualified-name + - name: output-prefix + value: "argo-artifacts/{{workflow.namespace}}/{{workflow.name}}" + - name: fetch-folderless-assets + - name: include-filter + - name: exclude-filter + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + - name: statsd-global-tags + dag: + tasks: + - name: extract-assets + template: NAME-api + withItems: + - { "url": "https://NAME..amazonaws.com/accounts//analyses", "name": "analysis" } + arguments: + parameters: + - name: url + value: "{{item.url}}" + - name: output-prefix + value: "{{inputs.parameters.output-prefix}}/{{item.name}}" + - name: NAME-request-config + value: | + { + "headers": { + "Content-Type":"application/x-amz-json-1.1" + }, + "json": { + "MaxResults": 100 + } + } + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: output-chunk-size + value: "1000" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + - name: NAME-execution-script + value: | + if state == ExecutionState.RAW_INPUT_PROCESS: + creds_arr = secrets["result-0.json"].split("""\n""") + region = '' + for cred in creds_arr: + LOGGER.debug("cred %s" % cred) + if 'AWS_REGION' in cred: + region = cred.split('=')[-1].replace('"', '') + region = region.strip() + + request_config['url'] = request_config['url'].replace('', region) + + accountid = '' + for cred in creds_arr: + if 'AWS_ACCOUNT_ID' in cred: + accountid = cred.split('=')[-1].replace('"', '') + + request_config['url'] = request_config['url'].replace('', accountid) + + + if state == ExecutionState.OUTPUT_PROCESS: + output = json.loads(output) + + if state == ExecutionState.API_POST: + next_token = response.json().get('NextToken', None) + if not next_token: + stop = True + else: + request_config['json']['NextToken'] = next_token + + if state == ExecutionState.API_FAIL: + failure_handler=FailureHandler.RETRY + + - name: process-NAME-metadata + inputs: + parameters: + - name: connection-qualified-name + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + - name: fetch-folderless-assets + - name: include-filter + - name: exclude-filter + dag: + tasks: + # Process NAME metadata and generate files for transformer + - name: process-metadata + template: process-metadata-template + arguments: + parameters: + - name: output-prefix + value: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/processed-metadata/{{workflow.name}}" + - name: git-kube-secret-name + value: "{{inputs.parameters.git-kube-secret-name}}" + - name: git-kube-ssh-key + value: "{{inputs.parameters.git-kube-ssh-key}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + - name: fetch-folderless-assets + value: "{{inputs.parameters.fetch-folderless-assets}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: connection-qualified-name + value: "{{inputs.parameters.connection-qualified-name}}" + artifacts: + - name: extracted-metadata + s3: + key: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/extracted-metadata/{{workflow.name}}" + + - name: process-metadata-template + inputs: + parameters: + - name: output-prefix + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: fetch-folderless-assets + - name: include-filter + - name: exclude-filter + - name: connection-qualified-name + + artifacts: + - name: extracted-metadata + path: /tmp/extracted-metadata + - name: scripts + path: /tmp/marketplace-scripts + git: + repo: git@github.com:atlanhq/marketplace-scripts + revision: "{{inputs.parameters.marketplace-scripts-revision}}" + insecureIgnoreHostKey: true + depth: 1 + sshPrivateKeySecret: + name: "{{inputs.parameters.git-kube-secret-name}}" + key: "{{inputs.parameters.git-kube-ssh-key}}" + - name: connection-cache + optional: true + path: "/tmp/connection-cache" + s3: + key: "connection-cache" + outputs: + artifacts: + - name: processed-metadata + path: /tmp/processed-metadata + s3: + key: "{{inputs.parameters.output-prefix}}" + archive: + none: { } + container: + image: ghcr.io/atlanhq/marketplace-scripts-base:0.1.10 + imagePullPolicy: IfNotPresent + workingDir: "/tmp/marketplace-scripts" + command: [ "python" ] + args: + - "-m" + - "marketplace_scripts.NAME.metadata" + - "--metadata-prefix" + - "/tmp/extracted-metadata" + - "--cache-prefix" + - "/tmp/connection-cache" + - "--output-prefix" + - "/tmp/processed-metadata" + - "--exclude-filter" + - "{{inputs.parameters.exclude-filter}}" + - "--include-filter" + - "{{inputs.parameters.include-filter}}" + - "--fetch-folderless-assets" + - "{{inputs.parameters.fetch-folderless-assets}}" + - "--connection-qualified-name" + - "{{inputs.parameters.connection-qualified-name}}" + + - name: publish-NAME-metadata + inputs: + parameters: + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + - name: connection + - name: mode + - name: source + - name: atlas-api-uri + - name: heracles-uri + - name: atlan-web-kube-secret + - name: atlas-auth-type + - name: publish-chunk-size + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + dag: + tasks: + # Run atlan-crawler/generic-publish template + - name: publish_metadata + templateRef: + name: atlan-crawler + template: generic-publish + arguments: + artifacts: + - name: data + s3: + key: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/processed-metadata/{{workflow.name}}" + - name: transformer-config + raw: + data: | + { + "external_map": { + "crawler_name": "{{workflow.labels.workflows.argoproj.io/workflow-template}}", + "tenant_id": "{{workflow.namespace}}", + "integration_name": "{{inputs.parameters.source}}", + "workflow_name": "{{workflow.name}}", + "connection_name": "{{=jsonpath(inputs.parameters.connection, '$.attributes.name')}}", + "connection_qn": "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + }, + "output_prefix": "/tmp/entities", + "templates_root": "/tmp/templates", + "transformation_config": [ + { + "input_file_pattern": "/tmp/inputs/ASSET_NAME.json", + "template": "packages/atlan/NAME/transformers/ASSET_NAME.jinja2", + "output_file_prefix": "ASSET_NAME/ASSET_NAME", + "output_chunk_size": 10000 + } + ] + } + parameters: + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.mode}}" + - name: source + value: "{{inputs.parameters.source}}" + - name: raw-input-file-sort + value: "" + - name: raw-input-folder-sort + value: "" + - name: raw-input-file-pattern + value: "**/*.json" + - name: publish-chunk-size + value: "{{inputs.parameters.publish-chunk-size}}" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + + - name: NAME-api + synchronization: + semaphore: + configMapKeyRef: + name: atlan-NAME + key: NAME-api + inputs: + artifacts: + - name: raw-input + path: "/tmp/input/" + optional: true + - name: raw-input-file + path: "/tmp/input/input.json" + optional: true + parameters: + - name: threads + value: "1" + - name: credential-guid + - name: NAME-request-config + value: | + { + "headers": { + "Content-Type":"application/x-amz-json-1.1" + }, + "json": { + "MaxResults": 100 + } + } + - name: paginate + value: "true" + enum: + - "true" + - "false" + - name: NAME-output-key + value: "TableList" + - name: url + - name: NAME-execution-script + value: | + if "{{inputs.parameters.threads}}" != "1": + logger.error("Error! You must pass custom execution script from parent for thread count > 1") + raise Exception("Invalid thread count for offset based pagination!") + if state == ExecutionState.RAW_INPUT_PROCESS: + creds_arr = secrets["result-0.json"].split("""\n""") + region = '' + for cred in creds_arr: + if 'AWS_REGION' in cred: + region = cred.split('=')[-1].replace('"', '') + + request_config['url'] = request_config['url'].replace('', region) + + if state == ExecutionState.OUTPUT_PROCESS: + output=json.loads(output)['{{inputs.parameters.NAME-output-key}}'] + + if state == ExecutionState.API_POST: + next_token = response.json().get('NextToken', None) + if not next_token: + stop = True + else: + request_config['json']['NextToken'] = next_token + + if state == ExecutionState.API_FAIL: + failure_handler=FailureHandler.RETRY + - name: output-chunk-size + value: 0 + - name: raw-input-file-pattern + value: "" + - name: raw-input-paginate + value: "0" + - name: raw-input-multiline + value: "False" + - name: pagination-wait-time + value: "0" + - name: kube-secret-name + value: "argo-client-creds" + - name: client-id-env + value: "login" + - name: client-secret-env + value: "password" + - name: token-url-env + value: "host" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},bot=atlan-NAME" + - name: output-prefix + value: "argo-artifacts/{{workflow.namespace}}/{{workflow.name}}/{{pod.name}}/" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: init-execution-script + value: | + if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): + LOGGER.debug('Heracles is unavailable. Performing retry with back-off') + failure_handler = FailureHandler.RETRY + if state == ExecutionState.OUTPUT_PROCESS: + + credential = json.loads(output) + output = "" + for key, value in credential.items(): + if key == "username": + output += f"""AWS_ACCESS_KEY_ID=\"{str(value)}\"\n""" + elif key == "password": + output += f"""AWS_SECRET_ACCESS_KEY=\"{str(value)}\"\n""" + elif key == "host": + if value is not None: + region = value.split('.')[-3] + output += f"""AWS_REGION=\"{str(region)}\"\n""" + elif key == "extra": + if "region" in value: + region = value.get("region", "") + output += f"""AWS_REGION=\"{str(region)}\"\n""" + if "accountid" in value: + accountid = value.get("accountid", "") + output += f"""AWS_ACCOUNT_ID=\"{str(accountid)}\"\n""" + if "aws_role_arn" in value: + output += f"""AWS_ROLE_ARN=\"{str(value.get("aws_role_arn", ""))}\"\n""" + if "aws_external_id" in value: + output += f"""AWS_EXTERNAL_ID=\"{str(value.get("aws_external_id", ""))}\"\n""" + else: + continue + + if state == ExecutionState.API_POST: + stop = True + outputs: + artifacts: + - name: success + path: "/tmp/rest/success" + s3: + key: "{{inputs.parameters.output-prefix}}/success" + archive: + none: { } + - name: failure + path: "/tmp/rest/failure" + archive: + none: { } + s3: + key: "{{inputs.parameters.output-prefix}}/failure" + parameters: + - name: success-num-files + valueFrom: + path: "/tmp/rest/success/result-gen.txt" + - name: failure-num-files + valueFrom: + path: "/tmp/rest/failure/result-gen.txt" + volumes: + - name: credentials + emptyDir: { } + container: + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "./entrypoint.sh" ] + volumeMounts: + - name: credentials + mountPath: /tmp/credentials + imagePullPolicy: IfNotPresent + args: [ + "python3", "main.py","GET", "{{inputs.parameters.url}}", + "--raw-input-paginate", "{{inputs.parameters.raw-input-paginate}}", + "--raw-input-multiline", "{{inputs.parameters.raw-input-multiline}}", + "--threads", "{{inputs.parameters.threads}}", + "--raw-input-file-pattern", "{{inputs.parameters.raw-input-file-pattern}}", + "--request-config", "{{inputs.parameters.NAME-request-config}}", + "--execution-script", "{{inputs.parameters.NAME-execution-script}}", + "--secrets-path", "/tmp/credentials/success/*.json", + "--auth-type", "aws", + "--auth-aws-service", "NAME", + "--auth-aws-region", "AWS_REGION", + "--output-chunk-size", "{{inputs.parameters.output-chunk-size}}", + "--output-file-prefix", "/tmp/rest", + "--pagination-wait-time", "0", + "--max-retries", "10", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] + initContainers: + - name: fetch-credentials + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "python3", "main.py" ] + env: + - name: OAUTHLIB_INSECURE_TRANSPORT + value: "1" + - name: CLIENT_ID + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-id-env}}" + - name: CLIENT_SECRET + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-secret-env}}" + - name: TOKEN_URL + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.token-url-env}}" + mirrorVolumeMounts: true + args: [ + "GET", + "{{inputs.parameters.heracles-uri}}/credentials/{{inputs.parameters.credential-guid}}/use", + "--raw-input", "{}", + "--raw-input-file-sort", "", + "--raw-input-multiline", "False", + "--execution-script", "{{inputs.parameters.init-execution-script}}", + "--raw-input-paginate", "0", + "--auth-type", "oauth2", + "--auth-oauth2-type", "client_credentials", + "--auth-oauth2-impersonate-user", "{{=sprig.dig('labels', 'workflows', 'argoproj', 'io/creator', '', workflow)}}", + "--auth-oauth2-client-credentials-client-id", "CLIENT_ID", + "--auth-oauth2-client-credentials-secret", "CLIENT_SECRET", + "--auth-oauth2-client-credentials-token-url", "TOKEN_URL", + "--output-chunk-size", "0", + "--output-file-prefix", "/tmp/credentials", + "--pagination-wait-time", "0", + "--max-retries", "10", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] diff --git a/lib/connectors/packages/BI/templates/basic+jwt.yaml b/lib/connectors/packages/BI/templates/basic+jwt.yaml new file mode 100644 index 0000000..ab64710 --- /dev/null +++ b/lib/connectors/packages/BI/templates/basic+jwt.yaml @@ -0,0 +1,633 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: atlan-NAME +spec: + entrypoint: main + templates: + - name: main + inputs: + parameters: + - name: credentials-fetch-strategy + value: "credential_guid" + enum: + - "k8s_secret" + - "credential_guid" + + - name: credential-kube-secret-name + value: "{{workflow.name}}-credential-secret" + + - name: credential-guid + value: "" + + #connection atlan object + - name: connection + + # NAME api version. + - name: api-version + value: "v52.0" + + #publish mode + - name: publish-mode + value: "production" + enum: + - "production" + - "test" + - "dev" + + #publish credential + - name: atlas-auth-type + value: "internal" + enum: + - "internal" + - "apikey" + + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + + - name: auto-classification + value: "false" + enum: + - "true" + - "false" + + - name: attachment-confidence-threshold + value: "0.8" + + - name: fetch-reports + value: "true" + + # Revisions + - name: marketplace-scripts-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplaceScriptsBranch" + optional: true + default: "master" + - name: marketplace-packages-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplacePackagesBranch" + optional: true + default: "master" + + dag: + tasks: + - name: fetch-credentials + templateRef: + name: rest-api + template: oauth2-client-credentials + arguments: + parameters: + - name: method + value: GET + - name: url + value: "http://heracles-service.heracles.svc.cluster.local/credentials/{{inputs.parameters.credential-guid}}/use" + - name: request-config + value: | + { + "headers": { + "user-id": "{{workflow.labels.workflows.argoproj.io/creator}}" + } + } + - name: kube-secret-name + value: "argo-client-creds" + - name: max-retries + value: "10" + - name: client-id-env + value: "login" + - name: client-secret-env + value: "password" + - name: token-url-env + value: "host" + - name: execution-script + value: | + if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): + LOGGER.debug('Heracles is unavailable. Performing retry with back-off') + failure_handler = FailureHandler.RETRY + if state == ExecutionState.OUTPUT_PROCESS: + credential = json.loads(output) + auth_type = credential["authType"] + + if auth_type == "basic": + oauth2_type = "resource_owner_password" + elif auth_type == "jwt": + oauth2_type = "jwt_bearer" + + output = { + "oauth2_type": oauth2_type, + } + + if state == ExecutionState.API_POST: + stop=True + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},bot=atlan-NAME" + - name: auth-type-to-param + dependencies: + - fetch-credentials + templateRef: + name: utils + template: artifact-to-key-param + arguments: + parameters: + - name: key + value: "oauth2_type" + artifacts: + - name: input + from: "{{tasks.fetch-credentials.outputs.artifacts.success}}" + subPath: "result-0.json" + - name: extract + dependencies: + - auth-type-to-param + template: extract-NAME-metadata + arguments: + parameters: + - name: credentials-fetch-strategy + value: "{{inputs.parameters.credentials-fetch-strategy}}" + - name: credential-kube-secret-name + value: "{{inputs.parameters.credential-kube-secret-name}}" + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: oauth2-type + value: "{{tasks.auth-type-to-param.outputs.parameters.output}}" + - name: connection-qualified-name + value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: fetch-reports + value: "{{inputs.parameters.fetch-reports}}" + + - name: process-objects + template: process-NAME-described-assets + dependencies: + - extract + withParam: "{{tasks.extract.outputs.parameters.org-host-name}}" + arguments: + parameters: + - name: host-name + value: "{{item.host}}" + - name: output-prefix + value: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/metadata-extract/{{workflow.name}}/processed-metadata" + - name: process-chunk-size + value: 10000 + artifacts: + - name: input-raw-metadata + s3: + key: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/metadata-extract/{{workflow.name}}/raw-metadata" + archive: + none: { } + - name: scripts + git: + repo: git@github.com:atlanhq/marketplace-scripts + insecureIgnoreHostKey: true + depth: 1 + revision: "{{inputs.parameters.marketplace-scripts-revision}}" + sshPrivateKeySecret: + name: "{{inputs.parameters.git-kube-secret-name}}" + key: "{{inputs.parameters.git-kube-ssh-key}}" + - name: publish + dependencies: + - process-objects + template: publish-NAME-metadata + arguments: + parameters: + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.publish-mode}}" + - name: source + value: "NAME" + - name: atlas-api-uri + value: "http://atlas-service-atlas.atlas.svc.cluster.local/api/atlas/v2" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: processed-data-key + value: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/metadata-extract/{{workflow.name}}/processed-metadata" + - name: atlan-web-kube-secret + value: "argo-client-creds" + - name: publish-chunk-size + value: "25" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + + - name: extract-NAME-metadata + inputs: + parameters: + - name: credentials-fetch-strategy + - name: credential-kube-secret-name + - name: credential-guid + - name: oauth2-type + - name: connection-qualified-name + - name: heracles-uri + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: fetch-reports + outputs: + parameters: + - name: org-host-name + valueFrom: + default: "[]" + parameter: "{{tasks.convert-to-param.outputs.parameters.output}}" + dag: + tasks: + - name: extract-organization + template: NAME-api + arguments: + parameters: + - name: url + value: "/services/data/v52.0/query?q=select%20Fields(All)%20from%20Organization%20limit%201" + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: oauth2-type + value: "{{inputs.parameters.oauth2-type}}" + - name: execution-script + value: | + if state == ExecutionState.RAW_INPUT_PROCESS: + creds_arr = secrets["result-0.json"].split("""\n""") + host = '' + for cred in creds_arr: + if 'HOST' in cred: + host = cred.split('=')[-1].replace('"', '') + request_config['url'] = request_config['url'].replace('', host) + store['host'] = host + + if state == ExecutionState.OUTPUT_PROCESS: + _response = json.loads(output).get('records', []) + + if len(_response) > 0: + output = _response[0] + output['host'] = store['host'] + + if state == ExecutionState.API_POST: + stop = True + + if state == ExecutionState.API_FAIL: failure_handler=FailureHandler.RETRY + - name: output-chunk-size + value: 1 + - name: output-prefix + value: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/metadata-extract/{{workflow.name}}/raw-metadata/described-organization/0" + + - name: process-NAME-metadata + inputs: + parameters: + - name: host-name + - name: output-prefix + - name: process-chunk-size + artifacts: + - name: input-raw-metadata + path: "/tmp/input" + - name: scripts + path: /tmp/marketplace-scripts + outputs: + artifacts: + - name: output + path: "/tmp/output" + s3: + key: "{{inputs.parameters.output-prefix}}" + archive: + none: { } + container: + image: ghcr.io/atlanhq/marketplace-scripts-base:0.1.11 + workingDir: "/tmp/marketplace-scripts" + command: [ "python" ] + args: + - "-m" + - "marketplace_scripts.NAME.main" + - "--host-name" + - "{{inputs.parameters.host-name}}" + - "--input-dir" + - "/tmp/input" + - "--output-prefix" + - "/tmp/output/" + - "--chunk-size" + - "{{inputs.parameters.process-chunk-size}}" + + - name: publish-NAME-metadata + inputs: + parameters: + - name: connection + - name: mode + - name: source + - name: atlas-api-uri + - name: heracles-uri + - name: processed-data-key + - name: atlan-web-kube-secret + - name: atlas-auth-type + - name: publish-chunk-size + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + dag: + tasks: + - name: publish + templateRef: + name: atlan-crawler + template: generic-publish + arguments: + artifacts: + - name: data + s3: + key: "{{inputs.parameters.processed-data-key}}" + - name: transformer-config + raw: + data: | + { + "external_map": { + "crawler_name": "{{workflow.labels.workflows.argoproj.io/workflow-template}}", + "tenant_id": "{{workflow.namespace}}", + "integration_name": "{{inputs.parameters.source}}", + "workflow_name": "{{workflow.name}}", + "connection_name": "{{=jsonpath(inputs.parameters.connection, '$.attributes.name')}}", + "connection_qn": "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + }, + "output_prefix": "/tmp/entities/", + "templates_root": "/tmp/templates/packages", + "transformation_config": [ + { + "input_file_pattern": "/tmp/inputs/organization*.json", + "template": "atlan/NAME/transformers/organization.jinja2", + "output_file_prefix": "organization" + }, + { + "input_file_pattern": "/tmp/inputs/objects*.json", + "template": "atlan/NAME/transformers/object.jinja2", + "output_file_prefix": "objects", + "ignore": true + }, + { + "input_file_pattern": "/tmp/inputs/fields*.json", + "template": "atlan/NAME/transformers/field.jinja2", + "output_file_prefix": "fields", + "ignore": true + }, + { + "input_file_pattern": "/tmp/inputs/reports*.json", + "template": "atlan/NAME/transformers/report.jinja2", + "output_file_prefix": "reports", + "ignore": true + }, + { + "input_file_pattern": "/tmp/inputs/dashboards*.json", + "template": "atlan/NAME/transformers/dashboard.jinja2", + "output_file_prefix": "dashboards", + "ignore": true + } + ] + } + parameters: + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.mode}}" + - name: source + value: "{{inputs.parameters.source}}" + - name: raw-input-file-sort + value: "organization,objects,fields,reports,dashboards" + - name: publish-chunk-size + value: "{{inputs.parameters.publish-chunk-size}}" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + + - name: NAME-api + synchronization: + semaphore: + configMapKeyRef: + name: atlan-NAME + key: api + volumes: + - name: atlan-NAME-workflow-directory + ephemeral: + volumeClaimTemplate: + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 60Gi + - name: credentials + emptyDir: { } + inputs: + artifacts: + - name: raw-input + path: /tmp/input + optional: true + parameters: + - name: credential-guid + - name: oauth2-type + - name: page-size + value: "10" + - name: url + - name: request-config + value: | + { + "headers": { + "Content-Type": "application/x-www-form-urlencoded" + } + } + - name: execution-script + - name: output-chunk-size + value: "0"sourceCategory + - name: kube-secret-name + value: "argo-client-creds" + - name: client-id-env + value: "login" + - name: client-secret-env + value: "password" + - name: token-url-env + value: "host" + - name: raw-input-file-pattern + value: "" + - name: raw-input-paginate + value: 0 + - name: raw-input-multiline + value: "False" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},bot=atlan-NAME" + - name: output-prefix + value: "argo-artifacts/{{workflow.namespace}}/{{workflow.name}}/{{pod.name}}" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: init-execution-script + value: | + if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): + LOGGER.debug('Heracles is unavailable. Performing retry with back-off') + failure_handler = FailureHandler.RETRY + if state == ExecutionState.OUTPUT_PROCESS: + credential = json.loads(output) + auth_type = credential.get('authType', '') + extra_params = credential.get('extra', {}) + is_sandbox = extra_params.get('is_sandbox', False) + token_url = 'test.NAME.com' if is_sandbox else 'login.NAME.com' + + output = f"""USERNAME="{credential.get('username', '')}" + HOST="{credential.get('host', '')}" + TOKEN_URL="https://{token_url}/services/oauth2/token" + CLIENT_ID="{extra_params.get('client_id', '')}" + """ + + if auth_type == 'basic': + password = credential.get('password', '').replace('$', '\$') + output += f"""PASSWORD="{password}" + CLIENT_SECRET="{extra_params.get('client_secret', '')}" + """ + elif auth_type == 'jwt': + output += f"""PRIVATE_KEY="{extra_params.get('private_key', '')}" + """ + + if state == ExecutionState.API_POST: + stop = True + outputs: + artifacts: + - name: success + path: "/tmp/rest/success" + s3: + key: "{{inputs.parameters.output-prefix}}/success" + archive: + none: { } + - name: failure + path: "/tmp/rest/failure" + archive: + none: { } + s3: + key: "{{inputs.parameters.output-prefix}}/failure" + parameters: + - name: success-num-files + valueFrom: + path: "/tmp/rest/success/result-gen.txt" + - name: failure-num-files + valueFrom: + path: "/tmp/rest/failure/result-gen.txt" + container: + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "./entrypoint.sh" ] + volumeMounts: + - name: atlan-NAME-workflow-directory + mountPath: /tmp + - name: credentials + mountPath: /tmp/credentials + imagePullPolicy: IfNotPresent + args: [ + "python3", "main.py", "GET", "{{inputs.parameters.url}}", + "--raw-input-paginate", "{{inputs.parameters.raw-input-paginate}}", + "--raw-input-file-pattern", "{{inputs.parameters.raw-input-file-pattern}}", + "--raw-input-multiline", "{{inputs.parameters.raw-input-multiline}}", + "--request-config", "{{inputs.parameters.request-config}}", + "--execution-script", "{{inputs.parameters.execution-script}}", + "--secrets-path", "/tmp/credentials/success/*.json", + "--auth-type", "oauth2", + "--auth-oauth2-type", "{{inputs.parameters.oauth2-type}}", + "--auth-oauth2-resource-owner-password-client-id", "CLIENT_ID", + "--auth-oauth2-resource-owner-password-secret", "CLIENT_SECRET", + "--auth-oauth2-resource-owner-password-token-url", "TOKEN_URL", + "--auth-oauth2-resource-owner-password-username", "USERNAME", + "--auth-oauth2-resource-owner-password-password", "PASSWORD", + "--auth-oauth2-jwt-bearer-client-id", "CLIENT_ID", + "--auth-oauth2-jwt-bearer-private-key", "PRIVATE_KEY", + "--auth-oauth2-jwt-bearer-token-url", "TOKEN_URL", + "--auth-oauth2-jwt-bearer-username", "USERNAME", + "--auth-oauth2-jwt-bearer-token-expiry", "{{=sprig.unixEpoch(sprig.dateModify('3m', sprig.now()))}}", + "--output-chunk-size", "{{inputs.parameters.output-chunk-size}}", + "--output-file-prefix", "/tmp/rest/", + "--pagination-wait-time", "10", + "--max-retries", "3", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] + + initContainers: + - name: fetch-credentials + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "python3", "main.py" ] + env: + - name: OAUTHLIB_INSECURE_TRANSPORT + value: "1" + - name: CLIENT_ID + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-id-env}}" + - name: CLIENT_SECRET + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-secret-env}}" + - name: TOKEN_URL + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.token-url-env}}" + mirrorVolumeMounts: true + args: [ + "GET", + "{{inputs.parameters.heracles-uri}}/credentials/{{inputs.parameters.credential-guid}}/use", + "--raw-input", "{}", + "--raw-input-file-pattern", "", + "--raw-input-file-sort", "", + "--raw-input-multiline", "f", + "--execution-script", "{{inputs.parameters.init-execution-script}}", + "--raw-input-paginate", "0", + "--auth-type", "oauth2", + "--auth-oauth2-type", "client_credentials", + "--auth-oauth2-impersonate-user", "{{=sprig.dig('labels', 'workflows', 'argoproj', 'io/creator', '', workflow)}}", + "--auth-oauth2-client-credentials-client-id", "CLIENT_ID", + "--auth-oauth2-client-credentials-secret", "CLIENT_SECRET", + "--auth-oauth2-client-credentials-token-url", "TOKEN_URL", + "--output-chunk-size", "0", + "--output-file-prefix", "/tmp/credentials", + "--pagination-wait-time", "0", + "--max-retries", + "10", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] + diff --git a/lib/connectors/packages/BI/templates/oauth.yaml b/lib/connectors/packages/BI/templates/oauth.yaml new file mode 100644 index 0000000..152899f --- /dev/null +++ b/lib/connectors/packages/BI/templates/oauth.yaml @@ -0,0 +1,581 @@ +# Workflow Templates +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: atlan-NAME +spec: + entrypoint: main + templates: + - name: main + inputs: + parameters: + # Credential + - name: credentials-fetch-strategy + value: "credential_guid" + enum: + - "credential_guid" + - "k8s_secret" + - name: credential-guid + value: "" + # Connection Entity + - name: connection + - name: atlas-auth-type + value: "internal" + enum: + - "internal" + - "apikey" + # Include/Exclude Filters + - name: include-filter + value: "{}" + - name: exclude-filter + value: "{}" + # Publish Mode + - name: publish-mode + value: "production" + enum: + - "production" + - "dev" + - "test" + # Revisions + - name: marketplace-scripts-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplaceScriptsBranch" + optional: true + default: "master" + - name: marketplace-packages-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplacePackagesBranch" + optional: true + default: "master" + dag: + tasks: + # Extract metadata from NAME + - name: extract + template: extract-NAME-metadata + arguments: + parameters: + - name: credentials-fetch-strategy + value: "{{inputs.parameters.credentials-fetch-strategy}}" + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: connection-qualified-name + value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + # Process the extracted metadata + - name: process + template: process-NAME-metadata + depends: "extract.Succeeded" + arguments: + parameters: + - name: connection-qualified-name + value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + + - name: publish + depends: "process.Succeeded" + template: publish-NAME-metadata + arguments: + parameters: + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.publish-mode}}" + - name: source + value: "NAME" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: atlan-web-kube-secret + value: "argo-client-creds" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: atlas-api-uri + value: "http://atlas-service-atlas.atlas.svc.cluster.local/api/atlas/v2" + - name: publish-chunk-size + value: "100" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + + - name: extract-NAME-metadata + inputs: + parameters: + - name: credentials-fetch-strategy + - name: credential-guid + - name: connection-qualified-name + - name: include-filter + - name: exclude-filter + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: heracles-uri + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: marketplace-scripts-revision + outputs: + parameters: + - name: auth-type-and-host + valueFrom: + parameter: "{{tasks.auth-type-and-host-to-param.outputs.parameters.output}}" + dag: + tasks: + # Fetch the credential + - name: fetch-credentials + when: "{{inputs.parameters.credentials-fetch-strategy}} == credential_guid" + templateRef: + name: rest-api + template: oauth2-client-credentials + arguments: + parameters: + - name: method + value: GET + - name: url + value: "{{inputs.parameters.heracles-uri}}/credentials/{{inputs.parameters.credential-guid}}/use" + - name: request-config + value: | + { + "headers": { + "user-id": "{{workflow.labels.workflows.argoproj.io/creator}}" + } + } + - name: kube-secret-name + value: "argo-client-creds" + - name: max-retries + value: "10" + - name: client-id-env + value: "login" + - name: client-secret-env + value: "password" + - name: token-url-env + value: "host" + - name: execution-script + value: | + if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): + LOGGER.debug("Heracles is unavailable. Performing retry with back-off") + failure_handler = FailureHandler.RETRY + + if state == ExecutionState.OUTPUT_PROCESS: + credential = json.loads(output) + output = { + "authType": credential["authType"], + "host": credential["host"] + } + + if state == ExecutionState.API_POST: + stop=True + - name: statsd-host + value: "{{inputs.parameters.statsd-host}}" + - name: statsd-port + value: "{{inputs.parameters.statsd-port}}" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + # Convert that `fetch-credentials` output from an artifact to a parameter + - name: auth-type-and-host-to-param + dependencies: + - fetch-credentials + templateRef: + name: utils + template: artifact-to-direct-param + arguments: + artifacts: + - name: input + from: "{{tasks.fetch-credentials.outputs.artifacts.success}}" + subPath: "result-0.json" + # Get all the workbooks + - name: fetch-ASSET_NAME + dependencies: + - auth-type-and-host-to-param + template: api-request + arguments: + parameters: + - name: auth-type + value: "{{=jsonpath(tasks['auth-type-and-host-to-param'].outputs.parameters.output, '$.authType')}}" + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: heracles-uri + value: "{{inputs.parameters.heracles-uri}}" + - name: method + value: GET + - name: url + value: "<>" + - name: request-config + value: | + { + "params": { + "page": 0, + "limit": 100 + } + } + - name: kube-secret-name + value: "argo-client-creds" + - name: output-prefix + value: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/extracted-metadata/{{workflow.name}}/ASSET_NAME" + - name: output-chunk-size + value: "1000" + - name: execution-script + value: | + if state == ExecutionState.RAW_INPUT_PROCESS: + creds_arr = secrets["result-0.json"].split("""\n""") + host = "" + for cred in creds_arr: + if "HOST" in cred: + host = cred.split('=')[-1].replace('"', '') + + url = "{{inputs.parameters.url}}" + url = url.replace("", host) + LOGGER.debug(f"URL: {url}") + request_config["url"] = url + + if state == ExecutionState.OUTPUT_PROCESS: + output = json.loads(output) + + if state == ExecutionState.API_POST: + workbook_response = response.json() + if not workbook_response.get("hasMore"): + stop = True + else: + next_page = workbook_response.get("nextPage") + request_config["params"] = { + "page": next_page, + "limit": 100 + } + if state == ExecutionState.API_FAIL: + if response.status_code == 500: + failure_handler = FailureHandler.NONE + else: + failure_handler = FailureHandler.RETRY + - name: statsd-host + value: "{{inputs.parameters.statsd-host}}" + - name: statsd-port + value: "{{inputs.parameters.statsd-port}}" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + + - name: process-NAME-metadata + inputs: + parameters: + - name: connection-qualified-name + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: marketplace-scripts-revision + dag: + tasks: + - name: process-metadata + template: process-metadata + arguments: + parameters: + - name: output-prefix + value: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/processed-metadata/{{workflow.name}}" + - name: git-kube-secret-name + value: "{{inputs.parameters.git-kube-secret-name}}" + - name: git-kube-ssh-key + value: "{{inputs.parameters.git-kube-ssh-key}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + artifacts: + - name: extracted-metadata + s3: + key: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/extracted-metadata/{{workflow.name}}" + - name: parsed-queries + s3: + key: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/parsed-queries/{{workflow.name}}" + + - name: publish-NAME-metadata + inputs: + parameters: + - name: connection + - name: mode + - name: source + - name: atlas-api-uri + - name: heracles-uri + - name: atlan-web-kube-secret + - name: atlas-auth-type + - name: publish-chunk-size + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + dag: + tasks: + # Run atlan-crawler/publish-bi template + - name: publish + templateRef: + name: atlan-crawler + template: publish-bi + arguments: + artifacts: + - name: data + s3: + key: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/processed-metadata/{{workflow.name}}" + - name: transformer-config + raw: + data: | + { + "external_map": { + "crawler_name": "{{workflow.labels.workflows.argoproj.io/workflow-template}}", + "tenant_id": "{{workflow.namespace}}", + "integration_name": "{{inputs.parameters.source}}", + "workflow_name": "{{workflow.name}}", + "connection_name": "{{=jsonpath(inputs.parameters.connection, '$.attributes.name')}}", + "connection_qn": "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + }, + "output_prefix": "/tmp/entities", + "templates_root": "/tmp/templates/packages", + "transformation_config": [ + { + "input_file_pattern": "/tmp/inputs/<>.json", + "template": "atlan/NAME/transformers/<>.jinja2", + "output_file_prefix": "<>/<>", + "output_chunk_size": 15000 + } + ] + } + parameters: + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.mode}}" + - name: source + value: "{{inputs.parameters.source}}" + - name: raw-input-file-sort + value: "" + - name: raw-input-folder-sort + value: "" + - name: raw-input-file-pattern + value: "**/*.json" + - name: publish-chunk-size + value: "{{inputs.parameters.publish-chunk-size}}" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: hierarchy + value: | + [ + [ + { + "name":"", + "file_pattern": "" + } + ] + ] + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + + - name: NAME-api + synchronization: + semaphore: + configMapKeyRef: + name: atlan-NAME + key: api + volumes: + - name: credentials + emptyDir: { } + inputs: + artifacts: + - name: raw-input + path: /tmp/input + optional: true + - name: raw-input-file + path: "/tmp/input/input.json" + optional: true + parameters: + - name: credential-guid + - name: page-size + value: "10" + - name: method + value: "GET" + - name: url + - name: request-config + value: "{}" + - name: execution-script + - name: output-chunk-size + value: 100 + - name: kube-secret-name + value: "argo-client-creds" + - name: client-id-env + value: "login" + - name: client-secret-env + value: "password" + - name: token-url-env + value: "host" + - name: raw-input-paginate + value: 0 + - name: raw-input-file-pattern + value: "" + - name: raw-input-multiline + value: "False" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},bot=atlan-NAME" + - name: output-prefix + value: "argo-artifacts/{{workflow.namespace}}/{{workflow.name}}/{{pod.name}}" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: init-execution-script + value: | + if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): + LOGGER.debug('Heracles is unavailable. Performing retry with back-off') + failure_handler = FailureHandler.RETRY + + if state == ExecutionState.OUTPUT_PROCESS: + credential = json.loads(output) + output = "" + for key, value in credential.items(): + output += f"""ATLAN_{key.upper()}="{str(value)}"\n""" + + output += f"""ATLAN_TOKEN_URL='https://{credential['host']}/v2/auth/token'\n""" + + if state == ExecutionState.API_POST: + stop = True + outputs: + artifacts: + - name: success + path: "/tmp/rest/success" + s3: + key: "{{inputs.parameters.output-prefix}}" + archive: + none: { } + - name: failure + path: "/tmp/rest/failure" + parameters: + - name: success-num-files + valueFrom: + path: "/tmp/rest/success/result-gen.txt" + - name: failure-num-files + valueFrom: + path: "/tmp/rest/failure/result-gen.txt" + container: + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "./entrypoint.sh" ] + volumeMounts: + - name: credentials + mountPath: /tmp/credentials + env: + - name: OAUTHLIB_INSECURE_TRANSPORT + value: "1" + imagePullPolicy: IfNotPresent + args: [ + "python3", "main.py", "{{inputs.parameters.method}}", "{{inputs.parameters.url}}", + "--request-config", "{{inputs.parameters.request-config}}", + "--raw-input-paginate", "{{inputs.parameters.raw-input-paginate}}", + "--raw-input-file-pattern", "{{inputs.parameters.raw-input-file-pattern}}", + "--raw-input-multiline", "{{inputs.parameters.raw-input-multiline}}", + "--execution-script", "{{inputs.parameters.execution-script}}", + "--secrets-path", "/tmp/credentials/success/*.json", + "--auth-type", "oauth2", + "--auth-oauth2-type", "client_credentials", + "--auth-oauth2-impersonate-user", "", + "--auth-oauth2-client-credentials-client-id", "ATLAN_USERNAME", + "--auth-oauth2-client-credentials-secret", "ATLAN_PASSWORD", + "--auth-oauth2-client-credentials-token-url", "ATLAN_TOKEN_URL", + "--output-chunk-size", "{{inputs.parameters.output-chunk-size}}", + "--output-file-prefix", "/tmp/rest", + "--pagination-wait-time", "10", + "--max-retries", "3", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] + initContainers: + - name: fetch-credentials + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "python3", "main.py" ] + env: + - name: OAUTHLIB_INSECURE_TRANSPORT + value: "1" + - name: CLIENT_ID + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-id-env}}" + - name: CLIENT_SECRET + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-secret-env}}" + - name: TOKEN_URL + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.token-url-env}}" + mirrorVolumeMounts: true + args: [ + "GET", + "{{inputs.parameters.heracles-uri}}/credentials/{{inputs.parameters.credential-guid}}/use", + "--raw-input", "{}", + "--raw-input-file-sort", "", + "--raw-input-multiline", "False", + "--execution-script", "{{inputs.parameters.init-execution-script}}", + "--raw-input-paginate", "0", + "--auth-type", "oauth2", + "--auth-oauth2-type", "client_credentials", + "--auth-oauth2-impersonate-user", "{{=sprig.dig('labels', 'workflows', 'argoproj', 'io/creator', '', workflow)}}", + "--auth-oauth2-client-credentials-client-id", "CLIENT_ID", + "--auth-oauth2-client-credentials-secret", "CLIENT_SECRET", + "--auth-oauth2-client-credentials-token-url", "TOKEN_URL", + "--output-chunk-size", "0", + "--output-file-prefix", "/tmp/credentials", + "--pagination-wait-time", "0", + "--max-retries", "10", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] \ No newline at end of file diff --git a/lib/connectors/packages/BI/transformers/default.jinja2 b/lib/connectors/packages/BI/transformers/default.jinja2 new file mode 100644 index 0000000..1a0d6dc --- /dev/null +++ b/lib/connectors/packages/BI/transformers/default.jinja2 @@ -0,0 +1,19 @@ +{ + "typeName": "ASSET_NAME", + "status": "ACTIVE", + "attributes": { + "connectorName": "{{external_map['integration_name']}}", + "connectionName": "{{external_map['connection_name']}}", + "connectionQualifiedName": "{{external_map['connection_qn']}}", + "lastSyncRunAt": {{now}}, + "lastSyncWorkflowName": "{{external_map['crawler_name']}}", + "lastSyncRun": "{{external_map['workflow_name']}}", + "tenantId": "{{external_map['tenant_id']}}", + + "qualifiedName": "{{external_map['connection_qn']}}/{{record['id']}}", + + "name": {{record.get('name') | tojson}}, + "sourceCreatedAt": "{{convert_date(record['createdAt'], format="%Y-%m-%dT%H:%M:%S.%fZ")}}", + "sourceUpdatedAt": "{{convert_date(record['updatedAt'], format="%Y-%m-%dT%H:%M:%S.%fZ")}}", + } +} \ No newline at end of file diff --git a/lib/connectors/packages/common-configmap/api.yaml b/lib/connectors/packages/common-configmap/api.yaml new file mode 100644 index 0000000..f258942 --- /dev/null +++ b/lib/connectors/packages/common-configmap/api.yaml @@ -0,0 +1,166 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlan-connectors-NAME + labels: + orchestration.atlan.com/version: "1" + orchestration.atlan.com/source: "NAME" +data: + icon: "https://cdn-images-1.medium.com/max/1200/1*8bPQYTZUHcvvk_1fSPwwHw.png" + helpdeskLink: "" + logo: "https://cdn-images-1.medium.com/max/1200/1*8bPQYTZUHcvvk_1fSPwwHw.png" + connector: "NAME" + defaultConnectorType: "rest" + jdbcCredentialTemplate: "{}" + restCredentialTemplate: | + { + "default": [ + { + "id": 1, + "name": "auth", + "curl": "curl --location --request POST 'https://{{host}}/v2/auth/token' --header 'Content-Type:application/x-www-form-urlencoded' --data-urlencode 'grant_type=client_credentials' --data-urlencode 'client_id={{username}}' --data-urlencode 'client_secret={{password}}' --connect-timeout 4" + } + ] + } + odbcCredentialTemplate: "{}" + grpcCredentialTemplate: "{}" + restMetadataTemplate: | + { + "default": [ + { + "id": 1, + "name": "auth", + "curl": "curl --location --request POST 'https://{{host}}/v2/auth/token' --header 'Content-Type:application/x-www-form-urlencoded' --data-urlencode 'grant_type=client_credentials' --data-urlencode 'client_id={{username}}' --data-urlencode 'client_secret={{password}}' --connect-timeout 4" + } + ] + } + restMetadataOutputTransformerTemplate: | + { + + } + odbcTemplate: "{}" + grpcTemplate: "{}" + sageTemplate: | + { + + } + config: | + { + "properties": { + "name": { + "type": "string", + "required": false, + "ui": { + "label": "Name", + "hidden": true, + "placeholder": "Host Name" + } + }, + "connector": { + "type": "string", + "required": false, + "ui": { + "label": "Connector", + "hidden": true, + "placeholder": "Connector" + } + }, + "connectorType": { + "type": "string", + "required": false, + "ui": { + "key": "_host", + "label": "connectorType", + "placeholder": "connectorType", + "hidden": true + } + }, + "host": { + "type": "string", + "required": true, + "enum": ["aws-api.NAMEcomputing.com", "api.NAMEcomputing.com"], + "enumNames": ["aws-api.NAMEcomputing.com - For AWS Hosted Organizations", "api.NAMEcomputing.com - For GCP Hosted Organizations"], + "default": "aws-api.NAMEcomputing.com", + "ui": { + "widget": "select", + "label": "Endpoint", + "grid": 8, + "hidden": false, + "placeholder": "Endpoint" + } + }, + "auth-type": { + "type": "string", + "enum": ["api_token"], + "default": "api_token", + "required": true, + "enumNames": ["API Token"], + "ui": { + "widget": "radio", + "label": "Authentication", + "placeholder": "Credential Type", + "rules": [ + { + "required": true, + "message": "Please enter a valid authentication type" + } + ] + } + }, + "api_token": { + "type": "object", + "properties": { + "username": { + "type": "string", + "required": true, + "ui": { + "label": "Client ID", + "placeholder": "Client ID", + "feedback": true, + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid client Id" + } + ] + } + }, + "password": { + "type": "string", + "required": true, + "ui": { + "widget": "password", + "label": "API Token", + "feedback": true, + "placeholder": "API Token", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid API token" + } + ] + } + } + }, + "ui": { + "widget": "nested", + "label": "Client ID & API Token", + "placeholder": "Credential Type", + "nestedValue": false, + "hidden": true + } + } + }, + "anyOf": [ + { + "properties": { + "auth-type": { + "const": "api_token" + } + }, + "required": ["api_token"] + } + ] + } \ No newline at end of file diff --git a/lib/connectors/packages/common-configmap/aws.yaml b/lib/connectors/packages/common-configmap/aws.yaml new file mode 100644 index 0000000..07f6794 --- /dev/null +++ b/lib/connectors/packages/common-configmap/aws.yaml @@ -0,0 +1,244 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlan-connectors-NAME + labels: + orchestration.atlan.com/version: "1" + orchestration.atlan.com/source: "NAME" +data: + icon: "https://atlan-public.s3.eu-west-1.amazonaws.com/atlan/logos/aws-NAME.png" + helpdeskLink: "https://ask.atlan.com/hc/en-us/articles/4605882724369-How-to-set-up-your-NAME-connection-on-Atlan" + logo: "https://atlan-public.s3.eu-west-1.amazonaws.com/atlan/logos/aws-NAME.png" + connector: "NAME" + defaultConnectorType: "sdk" + jdbcCredentialTemplate: "{}" + restCredentialTemplate: "{}" + odbcCredentialTemplate: "{}" + grpcCredentialTemplate: "{}" + restMetadataTemplate: "" + restMetadataOutputTransformerTemplate: "" + sdkCredentialTemplate: | + [ + { + "id": 1, + "name": "<>", + "source": "AWS", + "action": "AWSNAME.<>", + "params": { {% if authType == "iam" %} "aws_access_key_id": "{{username}}", "aws_secret_access_key": "{{password}}", {% endif %} "region": "{{extra.region}}" {% if authType == "role" and extra.aws_role_arn|length %}, "aws_role_arn": "{{extra.aws_role_arn}}" {% endif %} {% if authType == "role" and extra.aws_external_id|length %}, "aws_external_id": "{{extra.aws_external_id}}" {% endif %} } + } + ] + config: | + { + "properties": { + "name": { + "type": "string", + "required": false, + "ui": { + "label": "Name", + "hidden": true, + "placeholder": "Credential Name" + } + }, + "connector": { + "type": "string", + "required": false, + "ui": { + "label": "Connector", + "hidden": true, + "placeholder": "Connector" + } + }, + "connectorType": { + "type": "string", + "required": false, + "ui": { + "key": "_host", + "label": "connectorType", + "placeholder": "connectorType", + "hidden": true + } + }, + "auth-type": { + "type": "string", + "enum": [ + "iam", + "role" + ], + "default": "iam", + "required": true, + "enumNames": [ + "IAM User", + "IAM Role" + ], + "ui": { + "widget": "radio", + "hidden": false, + "label": "Authentication", + "placeholder": "Credential Type", + "rules": [ + { + "required": true, + "message": "Please enter a valid authentication type" + } + ] + } + }, + "port": { + "type": "number", + "default": 443, + "required": false, + "ui": { + "label": "Port", + "hidden": true, + "placeholder": "Port" + } + }, + "iam": { + "type": "object", + "properties": { + "username": { + "type": "string", + "required": true, + "ui": { + "label": "AWS Access Key", + "placeholder": "Access Key", + "feedback": true, + "help": "AWS Access Key", + "message": "Please enter a valid Access Key", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid Access Key" + } + ] + } + }, + "password": { + "type": "string", + "required": true, + "ui": { + "widget": "password", + "label": "AWS Secret Key", + "feedback": true, + "help": "AWS Secret Key", + "placeholder": "Secret Key", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid Secret Key" + } + ] + } + } + }, + "ui": { + "widget": "nested", + "label": "", + "placeholder": "Credential Type", + "nestedValue": false, + "hidden": false + } + }, + "role":{ + "type": "object", + "properties": { + "extra": { + "type": "object", + "properties": { + "aws_role_arn": { + "type": "string", + "required": false, + "ui": { + "label": "AWS Role ARN", + "hidden": false, + "placeholder": "arn:aws:iam::123456789012:role/roleName", + "help": "AWS role to assume for the connection", + "grid": 4, + "rules": [ + { + "pattern": "^arn:aws:iam::\\d{12}:role/.+", + "message": "Please enter a valid role ARN", + "trigger": "blur" + } + ] + } + }, + "aws_external_id": { + "type": "string", + "required": false, + "ui": { + "label": "External ID", + "widget": "keygen", + "help": "Unique external ID to enable access of your AWS resources to Atlan" + } + } + }, + "ui": { + "widget": "nested", + "label": "", + "header": "Advanced", + "hidden": false + } + } + }, + "ui": { + "widget": "nested", + "label": "", + "placeholder": "Credential Type", + "nestedValue": false, + "hidden": true + } + }, + "extra": { + "type": "object", + "properties": { + "region": { + "type": "string", + "required": true, + "default": "", + "ui": { + "help": "Enter region of NAME setup", + "label": "Region", + "hidden": false, + "placeholder": "us-west-1", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a region" + } + ] + } + } + }, + "ui": { + "widget": "nested", + "label": "", + "header": "Advanced", + "hidden": false + } + } + }, + "anyOf": [ + { + "properties": { + "auth-type": { + "const": "iam" + } + }, + "required": [ + "iam" + ] + }, + { + "properties": { + "auth-type": { + "const": "role" + } + }, + "required": ["role"] + } + ] + } diff --git a/lib/connectors/packages/common-configmap/basic.yaml b/lib/connectors/packages/common-configmap/basic.yaml new file mode 100644 index 0000000..5a60ae4 --- /dev/null +++ b/lib/connectors/packages/common-configmap/basic.yaml @@ -0,0 +1,151 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlan-connectors-NAME + labels: + orchestration.atlan.com/version: "1" + orchestration.atlan.com/source: "NAME" +data: + icon: "<>" + helpdeskLink: "" + logo: "<>" + connector: "NAME" + defaultConnectorType: "rest" + jdbcCredentialTemplate: "{}" + restCredentialTemplate: "{}" + { + "default": [ + { + "id": 1, + "name": "auth", + "curl": "curl --location --request POST 'https://{{host}}/v2/auth/token' --header 'Content-Type:application/x-www-form-urlencoded' --data-urlencode 'grant_type=client_credentials' --data-urlencode 'client_id={{username}}' --data-urlencode 'client_secret={{password}}' --connect-timeout 4" + } + ] + } + odbcCredentialTemplate: "{}" + grpcCredentialTemplate: "{}" + restMetadataTemplate: "{}" + restMetadataOutputTransformerTemplate: "{}" + odbcTemplate: "{}" + grpcTemplate: "{}" + sageTemplate: "{}" + config: | + { + "properties": { + "name": { + "type": "string", + "required": false, + "ui": { + "label": "Name", + "hidden": true, + "placeholder": "Host Name" + } + }, + "connector": { + "type": "string", + "required": false, + "ui": { + "label": "Connector", + "hidden": true, + "placeholder": "Connector" + } + }, + "connectorType": { + "type": "string", + "required": false, + "ui": { + "key": "_host", + "label": "connectorType", + "placeholder": "connectorType", + "hidden": true + } + }, + "host": { + "type": "string", + "required": true, + "enum": ["aws-api.NAMEcomputing.com", "api.NAMEcomputing.com"], + "enumNames": ["aws-api.NAMEcomputing.com - For AWS Hosted Organizations", "api.NAMEcomputing.com - For GCP Hosted Organizations"], + "default": "aws-api.NAMEcomputing.com", + "ui": { + "widget": "select", + "label": "Endpoint", + "grid": 8, + "hidden": false, + "placeholder": "Endpoint" + } + }, + "auth-type": { + "type": "string", + "enum": ["api_token"], + "default": "api_token", + "required": true, + "enumNames": ["API Token"], + "ui": { + "widget": "radio", + "label": "Authentication", + "placeholder": "Credential Type", + "rules": [ + { + "required": true, + "message": "Please enter a valid authentication type" + } + ] + } + }, + "api_token": { + "type": "object", + "properties": { + "username": { + "type": "string", + "required": true, + "ui": { + "label": "Client ID", + "placeholder": "Client ID", + "feedback": true, + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid client Id" + } + ] + } + }, + "password": { + "type": "string", + "required": true, + "ui": { + "widget": "password", + "label": "API Token", + "feedback": true, + "placeholder": "API Token", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid API token" + } + ] + } + } + }, + "ui": { + "widget": "nested", + "label": "Client ID & API Token", + "placeholder": "Credential Type", + "nestedValue": false, + "hidden": true + } + } + }, + "anyOf": [ + { + "properties": { + "auth-type": { + "const": "api_token" + } + }, + "required": ["api_token"] + } + ] + } \ No newline at end of file diff --git a/lib/connectors/packages/common-configmap/jdbc-basic.yaml b/lib/connectors/packages/common-configmap/jdbc-basic.yaml new file mode 100644 index 0000000..4bea4ce --- /dev/null +++ b/lib/connectors/packages/common-configmap/jdbc-basic.yaml @@ -0,0 +1,180 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlan-connectors-NAME + labels: + orchestration.atlan.com/version: "1" + orchestration.atlan.com/source: "NAME" +data: + icon: "https://atlan-public.s3.eu-west-1.amazonaws.com/atlan/logos/NAME.png" + helpdeskLink: "https://docs.atlan.com/integrations/relational-database/NAME" + logo: "https://atlan-public.s3.eu-west-1.amazonaws.com/atlan/logos/NAME.png" + connector: "NAME" + defaultConnectorType: "jdbc" + jdbcCredentialTemplate: | + { + "className": "com.facebook.NAME.jdbc.NAMEDriver", + "jarLink": "https://atlan-public.s3-eu-west-1.amazonaws.com/atlan/jdbc/NAME.tar.gz", + "url": "jdbc:NAME://{{ host }}:{{ port|int }}", + "driverProperties": { "user": "{{ username }}", "password": "{{ password }}" } + } + restCredentialTemplate: "{}" + odbcCredentialTemplate: "{}" + grpcCredentialTemplate: "{}" + restMetadataTemplate: "" + restMetadataOutputTransformerTemplate: "" + sageTemplate: | + { + "catalogsCheck": { + "curls": [ + { + "name": "schemas", + "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' --data-raw '{\"query\": \"show atlan schemas\"}'", + "addCredential": true, + "credentialConnectorType": "jdbc" + } + ], + "responseTemplate": "{{- $includeFilter := dict}} {{- if eq `string` (printf `%T` (index .formData `include-filter`)) }} {{- $includeFilter = index .formData `include-filter` | fromJson}} {{- else }} {{- $includeFilter = index .formData `include-filter` }} {{- end }} {{- $allowedDatabases := list}} {{- $allowedSchemas := list}} {{- $missingObjectName := ``}} {{- $checkSuccess := true }} {{- range $schemaList := .schemas.results }} {{- $allowedDatabases = append $allowedDatabases $schemaList.TABLE_CATALOG }} {{- $allowedSchemas = append $allowedSchemas (print $schemaList.TABLE_CATALOG `.` $schemaList.TABLE_SCHEM )}} {{- end }} {{- range $filteredDb, $filteredSchemas := $includeFilter }} {{- $_db := lower $filteredDb | trimPrefix `^` | trimSuffix `$` }} {{- $checkSuccess = and $checkSuccess (has $_db $allowedDatabases) }} {{- if not (has $_db $allowedDatabases)}} {{- $missingObjectName = (print $_db ` ` `database`)}} {{- end }} {{- range $schmea := $filteredSchemas }} {{- $_schema := lower $schmea | trimPrefix `^` | trimSuffix `$` }} {{- $checkSuccess = and $checkSuccess (has (print $_db `.` (lower $_schema)) $allowedSchemas)}} {{- if not (has (print $_db `.` (lower $_schema)) $allowedSchemas)}} {{- $missingObjectName = (print $_db `.` (lower $_schema) ` ` `schema`)}} {{- end }} {{- end }} {{- end }} {{- $response := dict `successMessage` `` `failureMessage` `` `data` .schemas.results `response` dict }} {{- if $checkSuccess }} {{- $_ := set $response `successMessage` `Check successful` }} {{- else }} {{- $_ := set $response `failureMessage` (print `Check failed for ` $missingObjectName) }} {{- end }} {{- $response | toJson }}" + } + } + config: | + { + "properties": { + "name": { + "type": "string", + "required": false, + "ui": { + "label": "Name", + "hidden": true, + "placeholder": "Host Name" + } + }, + "connector": { + "type": "string", + "required": false, + "ui": { + "label": "Connector", + "hidden": true, + "placeholder": "Connector" + } + }, + "connectorType": { + "type": "string", + "required": false, + "ui": { + "key": "_host", + "label": "connectorType", + "placeholder": "connectorType", + "hidden": true + } + }, + "host": { + "type": "string", + "required": true, + "default": "", + "ui": { + "label": "Host", + "feedback": true, + "help": "Your NAME instance host name", + "BYOCdisabled": true, + "rules": [ + { + "required": true, + "message": "Please enter a valid host name" + } + ], + "grid": 6 + } + }, + "port": { + "type": "number", + "default": 8080, + "required": true, + "ui": { + "label": "Port", + "placeholder": "Port", + "disabled": false, + "help": "Your NAME instance port number", + "grid": 2, + "BYOCdisabled": true, + "rules": [ + { + "required": true, + "message": "Please enter a valid port number" + } + ] + } + }, + "auth-type": { + "type": "string", + "enum": ["basic"], + "default": "basic", + "required": true, + "enumNames": ["Basic"], + "ui": { + "widget": "radio", + "hidden": false, + "label": "Authentication", + "placeholder": "Credential Type", + "rules": [ + { + "required": true, + "message": "Please enter a valid authentication type" + } + ] + } + }, + "basic": { + "type": "object", + "properties": { + "username": { + "type": "string", + "required": true, + "ui": { + "label": "Username", + "placeholder": "Username", + "help": "Database Username", + "feedback": true, + "message": "Please enter a valid username", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid username" + } + ] + } + }, + "password": { + "type": "string", + "required": false, + "ui": { + "widget": "password", + "label": "Password", + "help": "Database Password", + "feedback": false, + "placeholder": "Password", + "grid": 4 + } + } + }, + "ui": { + "widget": "nested", + "label": "Basic Authentication", + "placeholder": "Credential Type", + "nestedValue": false, + "hidden": true + } + } + }, + "anyOf": [ + { + "properties": { + "auth-type": { + "const": "basic" + } + }, + "required": ["basic"] + } + ] + } diff --git a/lib/connectors/scripts/BI/__init__.py b/lib/connectors/scripts/BI/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/connectors/scripts/BI/main.py b/lib/connectors/scripts/BI/main.py new file mode 100644 index 0000000..91ab53a --- /dev/null +++ b/lib/connectors/scripts/BI/main.py @@ -0,0 +1,36 @@ +import argparse +import glob +import logging +import orjson +import os +import re + +from dataclasses import dataclass +from marketplace_scripts.utils import timer, get_value_for_attr +from marketplace_scripts.utils.chunked_output_handler import ChunkedOutputHandler +from sqlitedict import SqliteDict +from typing import List, Dict, Any + +parser = argparse.ArgumentParser() + + +@dataclass +class NAMEScannerResultInputArgs: + output_prefix: str = "/tmp/NAME" + + +class NAMEResult: + def __init__(self, results: SqliteDict, output_prefix: str): + self.output_prefix = output_prefix + self.chunk_size = 10000 + + + +@timer() +def main(): + args = NAMEScannerResultInputArgs(**vars(parser.parse_args())) + os.makedirs(args.output_prefix, exist_ok=True) + + +if __name__ == "__main__": + main() diff --git a/lib/connectors/scripts/BI/tests/input/test.py b/lib/connectors/scripts/BI/tests/input/test.py new file mode 100644 index 0000000..835ee4a --- /dev/null +++ b/lib/connectors/scripts/BI/tests/input/test.py @@ -0,0 +1,18 @@ +import unittest +from pathlib import Path +from shutil import rmtree + + +class CreateNAMEResultTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.object = NAMEResult( + output_prefix="/tmp/output/requests" + ) + + @classmethod + def tearDownClass(cls) -> None: + rmtree("/tmp/output/") + + def test_assets_transformed(self): + #Add test here to call main function \ No newline at end of file diff --git a/lib/init.js b/lib/init.js index 5ca2c50..c2e4271 100644 --- a/lib/init.js +++ b/lib/init.js @@ -2,9 +2,11 @@ const Promise = require("bluebird"); const fs = Promise.promisifyAll(require("fs")); const system = require("system-commands"); const utils = require("./utils"); +const LinearClient = require("@linear/sdk").LinearClient; +const linearClient = new LinearClient(); /** - * Init an Aro package inside the folder + * Init an Argo package inside the folder * Steps: * 1. Check if folder is empty * 2. Package.json should not be present (unless force is set to true) @@ -47,3 +49,186 @@ function replaceInFile(filePath, searchText, replaceText) { return fs.writeFileAsync(filePath, result, "utf8"); }); } + +/* Init an connector Argo package inside the folder + * Steps: + * 1. Check if folder is empty + * 2. Package.json should not be present (unless force is set to true) + * 3. Copy the static folder in the current directory + * 4. Change name to the folder name + * @param {boolean} force + * @param {string} filepath + */ +exports.cinit = async function (force, inputPackageName, inputPackageType, auth, assetList, model, script, linear) { + const dirPath = "/Users/mrunmayi.tripathi/atlan/code/marketplace-packages/packages/atlan/test-package"; + //const dirPath = process.cwd(); + const pathComponents = dirPath.split("/"); + const packageName = inputPackageName || pathComponents[pathComponents.length - 1]; + + if (inputPackageType) { + var assetListArray = assetList.split(","); + if (inputPackageType === "BI") { + await exports.initConnectorBI(force, dirPath, inputPackageName, auth, assetListArray); + } else if (inputPackageType === "SQL") { + //await exports.initConnectorSQL(force, dirPath, inputPackageName, assetListArray); + } + + //Create entry in packages.json + await createPackageEntry(dirPath, inputPackageName); + + //Create linear + if (linear) { + await createLinearWithSubtask(); + } + + //Create models repo folder + if (model) { + await createModelsRepoDirectory(force, dirPath, inputPackageName, inputPackageType, assetList); + } + + //Create scripts repo folder + if (script) { + await createScriptsRepoDirectory(force, dirPath, inputPackageName, inputPackageType, assetList); + console.info("add entry to model"); + } + } else { + return fs + .readdirAsync(dirPath) + .then((files) => { + if (!force && (files.length !== 0 || files.includes("package.json"))) { + throw `Files already present in the ${__dirname}. Run this command again with --force to ignore`; + } + var skeletonPackagePath = `${__dirname}/static/package`; + system(`cp -r ${skeletonPackagePath}/ ${dirPath}/../packages/package.json`); + return system(`cp -r ${skeletonPackagePath}/ ${dirPath}`); + }) + .then((_) => { + return utils.walk(dirPath); + }) + .then((files) => { + return Promise.each(files, function (file) { + return replaceInFile(file, "NAME", packageName); + }); + }) + .then((_) => { + return packageName; + }) + .catch((error) => { + console.log(error); + }); + } +}; + +exports.initConnectorBI = async function (force, dirPath, packageName, auth, assetList) { + return fs + .readdirAsync(dirPath) + .then((files) => { + if (!force && (files.length !== 0 || files.includes("package.json"))) { + throw `Files already present in the ${__dirname}. Run this command again with --force to ignore`; + } + var skeletonPackagePath = `${__dirname}/connectors/BI`; + return system(`cp -r ${skeletonPackagePath}/ ${dirPath}`); + }) + .then((_) => { + return utils.walk(dirPath); + }) + .then((files) => { + return Promise.each(files, function (file) { + if (assetList.length != 0 && file.includes("/transformers/")) { + return BuildTransformerFilesForBI(file, assetList); + } else if (assetList.length != 0 && file.includes("/templates/")) { + return BuildTemplateFiles(packageName, file, auth); + } else { + return replaceInFile(file, "NAME", packageName); + } + }); + }) + .then((_) => { + fs.unlinkSync(`${dirPath}/transformers/default.jinja2`); + return packageName; + }) + .catch((error) => { + console.error(`Error in function initConnectorBI: ${error}`); + console.log(error); + }); +}; + +function BuildTransformerFilesForBI(filePath, assetList) { + return Promise.each(assetList, function (asset) { + const newFilePath = filePath.replace("default", asset); + return fs.readFileAsync(filePath, "utf-8").then((data) => { + data = data.replace("ASSET_NAME", asset); + fs.writeFile(newFilePath, data, (err) => { + if (err) throw err; + console.log("File %s created successfully!", newFilePath); + }); + }); + }); +} + +function BuildTemplateFiles(packageName, filePath, auth) { + var newFilePath = ""; + + if (auth.includes("oauth") && filePath.includes("oauth")) { + newFilePath = filePath; + } else if (auth.includes("basic") && filePath.includes("basic")) { + newFilePath = filePath; + } else if (auth.includes("api") && filePath.includes("api")) { + newFilePath = filePath; + } + if (newFilePath.length) { + return fs.readFileAsync(filePath, "utf-8").then((data) => { + data = data.replace("NAME", packageName); + return fs.writeFile(newFilePath, data, (err) => { + if (err) throw err; + console.log("File %s created successfully!", newFilePath); + }); + }); + } else { + return; + } +} + +async function createLinearWithSubtask() { + const teams = await linearClient.teams(); + let orcTeam = teams.nodes[0]; + teams.nodes.forEach(function (team) { + if (team.key === "ORC") { + orcTeam = team; + } + }); + const issue = await linearClient.createIssue({ teamId: orcTeam.id, title: "Redash Package" }); + console.info("add entry to linear %s", issue); + return; +} + +async function createPackageEntry(dirPath, packageName) { + const filePath = `${dirPath}/../packages/package.json`; + console.info("add connector entry to %s", filePath); + const rawPackageData = await fs.readFileAsync(filePath, "utf-8"); + let packageData = JSON.parse(rawPackageData); + packageData.dependencies[`@atlan/${packageName}`] = "^1.0.0"; + fs.writeFileAsync(filePath, packageData, "utf8"); +} + +async function createModelsRepoDirectory(force, dirPath, packageName, packageType, assetList) { + console.info("Creating folders in models repo"); + const modelsFolderPath = `${dirPath}/../../../models/atlas/entityDefs/Referenceable/Asset/Catalog/BI/${packageName}`; + const files = await fs.readdirAsync(modelsFolderPath); + if (!force && (files.length !== 0 || files.includes("package.json"))) { + throw `Files already present in the ${modelsFolderPath}. Run this command again with --force to ignore`; + } + var skeletonPackagePath = `${__dirname}/connectors/models/${packageType}`; + return system(`cp -r ${skeletonPackagePath}/ ${modelsFolderPath}`); +} + +async function createScriptsRepoDirectory(force, dirPath, packageName, packageType, assetList) { + console.info("Creating folders in scripts repo"); + const scriptsFolderPath = `${dirPath}/../../../marketplace-scripts/marketplace_scripts/${packageName}`; + const files = await fs.readdirAsync(scriptsFolderPath); + if (!force && (files.length !== 0 || files.includes("package.json"))) { + throw `Files already present in the ${scriptsFolderPath}. Run this command again with --force to ignore`; + } + var skeletonPackagePath = `${__dirname}/connectors/scripts/${packageType}`; + return system(`cp -r ${skeletonPackagePath}/ ${scriptsFolderPath}`); +} diff --git a/package.json b/package.json index a2a18b5..df70594 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,8 @@ "dependencies": { "@aws-sdk/client-s3": "^3.45.0", "@kubernetes/client-node": "^0.16.1", + "@linear/sdk": "^2.6.0", + "@notionhq/client": "^2.2.3", "ansicolor": "^1.1.95", "as-table": "^1.0.55", "axios": "^0.24.0", @@ -27,6 +29,7 @@ "stream-buffers": "^3.0.2", "system-commands": "^1.1.7", "yargs": "^17.3.1", + "yargs-interactive": "^3.0.1", "yargs-parser": ">=21.0.0" }, "devDependencies": { @@ -51,4 +54,4 @@ "argopm": "./bin/install.js" }, "preferGlobal": true -} \ No newline at end of file +} From c79b18a4464f7b38a9be1b3a025e608aa82103c4 Mon Sep 17 00:00:00 2001 From: Mrunmayi Date: Tue, 11 Apr 2023 14:26:58 +0530 Subject: [PATCH 3/6] Updating folders setup to build correct default folder structure --- .github/workflows/npm-publish.yml | 4 +-- lib/connectors/models/BI/default.json | 35 ------------------ lib/connectors/scripts/BI/__init__.py | 0 lib/connectors/scripts/BI/main.py | 36 ------------------- lib/connectors/scripts/BI/tests/input/test.py | 18 ---------- 5 files changed, 2 insertions(+), 91 deletions(-) delete mode 100644 lib/connectors/models/BI/default.json delete mode 100644 lib/connectors/scripts/BI/__init__.py delete mode 100644 lib/connectors/scripts/BI/main.py delete mode 100644 lib/connectors/scripts/BI/tests/input/test.py diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index c946168..d338f39 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -13,9 +13,9 @@ jobs: - name: Set up Node.js uses: actions/setup-node@master with: - node-version: 16.13.0 + node-version: 10.0.0 - name: Publish if version has been updated - uses: pascalgn/npm-publish-action@1.3.9 + uses: pascalgn/npm-publish-action@06e0830ea83eea10ed4a62654eeaedafb8bf50fc with: # All of theses inputs are optional tag_name: "v%s" tag_message: "v%s" diff --git a/lib/connectors/models/BI/default.json b/lib/connectors/models/BI/default.json deleted file mode 100644 index eefc78b..0000000 --- a/lib/connectors/models/BI/default.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "enumDefs": [], - "entityDefs": [ - { - "superTypes": [ - "NAME" - ], - "name": "ASSET_NAME", - "description": "Atlan ASSET_NAME Asset", - "typeVersion": "1.1", - "serviceType": "atlan", - "attributeDefs": [ - { - "name": "<>", - "typeName": "string", - "isOptional": true, - "cardinality": "SINGLE", - "isUnique": false, - "skipScrubbing": true, - "isIndexable": true, - "includeInNotification": true, - "indexType": "STRING", - "indexTypeESFields": { - "text": { - "type": "text", - "analyzer": "atlan_text_analyzer" - } - } - } - ] - } - ], - "relationshipDefs": [] - } - \ No newline at end of file diff --git a/lib/connectors/scripts/BI/__init__.py b/lib/connectors/scripts/BI/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/lib/connectors/scripts/BI/main.py b/lib/connectors/scripts/BI/main.py deleted file mode 100644 index 91ab53a..0000000 --- a/lib/connectors/scripts/BI/main.py +++ /dev/null @@ -1,36 +0,0 @@ -import argparse -import glob -import logging -import orjson -import os -import re - -from dataclasses import dataclass -from marketplace_scripts.utils import timer, get_value_for_attr -from marketplace_scripts.utils.chunked_output_handler import ChunkedOutputHandler -from sqlitedict import SqliteDict -from typing import List, Dict, Any - -parser = argparse.ArgumentParser() - - -@dataclass -class NAMEScannerResultInputArgs: - output_prefix: str = "/tmp/NAME" - - -class NAMEResult: - def __init__(self, results: SqliteDict, output_prefix: str): - self.output_prefix = output_prefix - self.chunk_size = 10000 - - - -@timer() -def main(): - args = NAMEScannerResultInputArgs(**vars(parser.parse_args())) - os.makedirs(args.output_prefix, exist_ok=True) - - -if __name__ == "__main__": - main() diff --git a/lib/connectors/scripts/BI/tests/input/test.py b/lib/connectors/scripts/BI/tests/input/test.py deleted file mode 100644 index 835ee4a..0000000 --- a/lib/connectors/scripts/BI/tests/input/test.py +++ /dev/null @@ -1,18 +0,0 @@ -import unittest -from pathlib import Path -from shutil import rmtree - - -class CreateNAMEResultTest(unittest.TestCase): - @classmethod - def setUpClass(cls) -> None: - cls.object = NAMEResult( - output_prefix="/tmp/output/requests" - ) - - @classmethod - def tearDownClass(cls) -> None: - rmtree("/tmp/output/") - - def test_assets_transformed(self): - #Add test here to call main function \ No newline at end of file From b63d6a77af474187d2b23cb0779c2ccc43d19f86 Mon Sep 17 00:00:00 2001 From: Mrunmayi Date: Tue, 11 Apr 2023 14:42:19 +0530 Subject: [PATCH 4/6] Updating folders setup to build correct default folder structure --- bin/install.js | 7 +- lib/connectors/packages/BI/README.md | 0 .../packages/BI/configmaps/default.yaml | 153 ----- lib/connectors/packages/BI/index.js | 4 - lib/connectors/packages/BI/package.json | 53 -- lib/connectors/packages/BI/templates/aws.yaml | 649 ------------------ .../packages/BI/templates/basic+jwt.yaml | 633 ----------------- .../packages/BI/templates/oauth.yaml | 581 ---------------- .../packages/BI/transformers/default.jinja2 | 19 - .../packages/common-configmap/api.yaml | 166 ----- .../packages/common-configmap/aws.yaml | 244 ------- .../packages/common-configmap/basic.yaml | 151 ---- .../packages/common-configmap/jdbc-basic.yaml | 180 ----- lib/init.js | 50 +- 14 files changed, 10 insertions(+), 2880 deletions(-) delete mode 100644 lib/connectors/packages/BI/README.md delete mode 100644 lib/connectors/packages/BI/configmaps/default.yaml delete mode 100644 lib/connectors/packages/BI/index.js delete mode 100644 lib/connectors/packages/BI/package.json delete mode 100644 lib/connectors/packages/BI/templates/aws.yaml delete mode 100644 lib/connectors/packages/BI/templates/basic+jwt.yaml delete mode 100644 lib/connectors/packages/BI/templates/oauth.yaml delete mode 100644 lib/connectors/packages/BI/transformers/default.jinja2 delete mode 100644 lib/connectors/packages/common-configmap/api.yaml delete mode 100644 lib/connectors/packages/common-configmap/aws.yaml delete mode 100644 lib/connectors/packages/common-configmap/basic.yaml delete mode 100644 lib/connectors/packages/common-configmap/jdbc-basic.yaml diff --git a/bin/install.js b/bin/install.js index 3f1f3df..092e00b 100755 --- a/bin/install.js +++ b/bin/install.js @@ -34,10 +34,6 @@ const options = { type: "input", describe: "Enter asset names in comma separated list", }, - models: { - type: "confirm", - describe: "Build models?", - }, scripts: { type: "confirm", describe: "Build scripts?", @@ -187,7 +183,7 @@ yargs }, }) .command({ - command: "c-init [package_name]", + command: "cinit [package_name]", desc: "Initializes an connector Argo package inside the current working directory", builder: () => yargsInteractive() @@ -201,7 +197,6 @@ yargs result.type, result.auth, result.AssetList, - result.models, result.scripts, result.linear ); diff --git a/lib/connectors/packages/BI/README.md b/lib/connectors/packages/BI/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/lib/connectors/packages/BI/configmaps/default.yaml b/lib/connectors/packages/BI/configmaps/default.yaml deleted file mode 100644 index 2637460..0000000 --- a/lib/connectors/packages/BI/configmaps/default.yaml +++ /dev/null @@ -1,153 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: atlan-NAME -data: - NAME-api: "10" - config: | - { - "properties": { - "connection": { - "type": "string", - "required": false, - "ui": { - "widget": "connection", - "label": "", - "placeholder": "Connection Name", - "connectionOptions": false - } - }, - "publish-mode": { - "type": "string", - "enum": ["production", "test", "dev"], - "default": "production", - "enumNames": ["Production", "Test", "Development"], - "ui": { - "widget": "select", - "label": "Run Mode", - "grid": 4, - "placeholder": "Connection Mode" - } - }, - "credentials-fetch-strategy": { - "type": "string", - "enum": ["k8s_secret", "credential_guid"], - "default": "credential_guid", - "enumNames": ["k8s Secret Key", "Credential Guid"], - "ui": { - "widget": "select", - "label": "Credential Type", - "placeholder": "Credential Type" - } - }, - "credential-guid": { - "type": "string", - "ui": { - "widget": "credential", - "label": "", - "credentialType": "atlan-connectors-NAME", - "placeholder": "Credential Guid", - "hidden": false - } - }, - "credential-kube-secret-name": { - "type": "string", - "ui": { - "label": "Credential Secret Name", - "placeholder": "Credential Secret Name", - "hidden": true - } - }, - "atlas-auth-type": { - "type": "string", - "enum": ["internal", "apikey"], - "default": "internal", - "enumNames": ["Internal", "API Key"], - "ui": { - "widget": "select", - "label": "Atlas Authentication Type", - "placeholder": "Atlas Authentication Type", - "hidden": true - } - }, - "runtime-properties": { - "type": "object", - "ui": { - "label": "Run time properties", - "hidden": true - } - }, - "include-filter": { - "type": "object", - "additionalProperties": { - "type": "array" - }, - "default": "{}", - "ui": { - "widget": "apitree", - "connectorConfigName": "atlan-connectors-NAME", - "credential": "credential-guid", - "label": "Include Metadata", - "description": "Selected assets only will be processed. Exclude gets preference over include.", - "grid": 4 - } - }, - "exclude-filter": { - "type": "object", - "additionalProperties": { - "type": "array" - }, - "default": "{}", - "ui": { - "widget": "apitree", - "connectorConfigName": "atlan-connectors-NAME", - "credential": "credential-guid", - "label": "Exclude Metadata", - "description": "Selected assets will not be processed.", - "grid": 4 - } - } - }, - "anyOf": [ - { - "properties": { - "credentials-fetch-strategy": { - "const": "k8s_secret" - } - }, - "required": ["credential-kube-secret-name"] - }, - { - "properties": { - "credentials-fetch-strategy": { - "const": "credential_guid" - } - }, - "required": ["credential-guid"] - } - ], - "steps": [ - { - "id": "credential", - "title": "Credential", - "description": "Credential Details", - "properties": [ - "credential-guid" - ] - }, - { - "id": "connection", - "title": "Connection", - "description": "Connection Details", - "properties": [ - "connection" - ] - }, - { - "id": "metadata", - "title": "Metadata", - "description": "Metadata", - "properties": ["include-filter", "exclude-filter"] - } - ] - } \ No newline at end of file diff --git a/lib/connectors/packages/BI/index.js b/lib/connectors/packages/BI/index.js deleted file mode 100644 index fbecdc9..0000000 --- a/lib/connectors/packages/BI/index.js +++ /dev/null @@ -1,4 +0,0 @@ -function dummy() { - console.log("don't call this dummy."); -} -module.exports = dummy; diff --git a/lib/connectors/packages/BI/package.json b/lib/connectors/packages/BI/package.json deleted file mode 100644 index bbc52ab..0000000 --- a/lib/connectors/packages/BI/package.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "name": "@atlan/NAME", - "version": "0.3.6", - "description": "Package to crawl NAME assets and publish to Atlan for discovery", - "keywords": [ - "NAME", - "bi", - "connector", - "crawler" - ], - "main": "index.js", - "scripts": {}, - "author": { - "name": "Atlan", - "email": "hello@atlan.com", - "url": "https://atlan.com" - }, - "repository": { - "type": "git", - "url": "https://github.com/atlanhq/marketplace-packages.git" - }, - "license": "MIT", - "dependencies": { - "@atlan/crawler": "^2.1.5", - "@atlan/sql-parser": "^0.3.16", - "rest-api": "^0.8.5", - "utils": "^0.2.4" - }, - "bugs": { - "url": "https://atlan.com", - "email": "support@atlan.com" - }, - "config": { - "labels": { - "orchestration.atlan.com/verified": "true", - "orchestration.atlan.com/type": "connector", - "orchestration.atlan.com/source": "NAME", - "orchestration.atlan.com/sourceCategory": "bi", - "orchestration.atlan.com/certified": "true" - }, - "annotations": { - "orchestration.atlan.com/name": "NAME Assets", - "orchestration.atlan.com/allowSchedule": "true", - "orchestration.atlan.com/dependentPackage": "", - "orchestration.atlan.com/emoji": "🚀", - "orchestration.atlan.com/categories": "NAME,crawler", - "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/NAME.svg", - "orchestration.atlan.com/marketplaceLink": "https://packages.atlan.com/-/web/detail/@atlan/NAME", - "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/NAME.svg", - "orchestration.atlan.com/docsUrl": "" - } - } -} diff --git a/lib/connectors/packages/BI/templates/aws.yaml b/lib/connectors/packages/BI/templates/aws.yaml deleted file mode 100644 index 22034e3..0000000 --- a/lib/connectors/packages/BI/templates/aws.yaml +++ /dev/null @@ -1,649 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: atlan-NAME -spec: - entrypoint: main - templates: - - name: main - inputs: - parameters: - #credential - - name: credentials-fetch-strategy - value: "credential_guid" - enum: - - "k8s_secret" - - "credential_guid" - - name: credential-kube-secret-name - value: "" - - name: credential-guid - value: "" - #connection atlan object - - name: connection - value: "" - # extraction - - name: runtime-properties - value: "{}" - - name: fetch-folderless-assets - value: "true" - - name: include-filter - value: "{}" - - name: exclude-filter - value: "{}" - #publish mode - - name: publish-mode - value: "production" - enum: - - "production" - - "test" - - "dev" - - #publish credential - - name: atlas-auth-type - value: "internal" - enum: - - "internal" - - "apikey" - - # enable classification match - - name: auto-classification - value: "false" - enum: - - "true" - - "false" - - - name: attachment-confidence-threshold - value: "0.8" - # Scripts - - name: marketplace-scripts-revision - valueFrom: - configMapKeyRef: - name: atlan-runtime-packages-config - key: "marketplaceScriptsBranch" - optional: true - default: "master" - # Revisions - - name: marketplace-packages-revision - valueFrom: - configMapKeyRef: - name: atlan-runtime-packages-config - key: "marketplacePackagesBranch" - optional: true - default: "master" - dag: - tasks: - - name: extract - template: extract-NAME-metadata - arguments: - parameters: - - name: credential-guid - value: "{{inputs.parameters.credential-guid}}" - - name: connection-qualified-name - value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" - - name: output-prefix - value: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/extracted-metadata/{{workflow.name}}" - - name: threads - value: "2" - - name: fetch-folderless-assets - value: "{{inputs.parameters.fetch-folderless-assets}}" - - name: include-filter - value: "{{inputs.parameters.include-filter}}" - - name: exclude-filter - value: "{{inputs.parameters.exclude-filter}}" - - name: marketplace-scripts-revision - value: "{{inputs.parameters.marketplace-scripts-revision}}" - - name: marketplace-packages-revision - value: "{{inputs.parameters.marketplace-packages-revision}}" - - name: statsd-global-tags - value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" - - name: process - template: process-NAME-metadata - depends: "extract.Succeeded" - arguments: - parameters: - - name: connection-qualified-name - value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" - - name: git-kube-secret-name - value: "git-ssh" - - name: git-kube-ssh-key - value: "private-key" - - name: statsd-host - value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" - - name: statsd-port - value: "9125" - - name: statsd-global-tags - value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" - - name: marketplace-scripts-revision - value: "{{inputs.parameters.marketplace-scripts-revision}}" - - name: marketplace-packages-revision - value: "{{inputs.parameters.marketplace-packages-revision}}" - - name: fetch-folderless-assets - value: "{{inputs.parameters.fetch-folderless-assets}}" - - name: include-filter - value: "{{inputs.parameters.include-filter}}" - - name: exclude-filter - value: "{{inputs.parameters.exclude-filter}}" - - name: publish - depends: "process.Succeeded" - template: publish-NAME-metadata - arguments: - parameters: - - name: marketplace-scripts-revision - value: "{{inputs.parameters.marketplace-scripts-revision}}" - - name: marketplace-packages-revision - value: "{{inputs.parameters.marketplace-packages-revision}}" - - name: connection - value: "{{inputs.parameters.connection}}" - - name: mode - value: "{{inputs.parameters.publish-mode}}" - - name: source - value: "NAME" - - name: atlas-auth-type - value: "{{inputs.parameters.atlas-auth-type}}" - - name: fetch-folderless-assets - value: "{{inputs.parameters.fetch-folderless-assets}}" - - name: include-filter - value: "{{inputs.parameters.include-filter}}" - - name: exclude-filter - value: "{{inputs.parameters.exclude-filter}}" - - name: exclude-workbook-regex - value: "{{inputs.parameters.exclude-workbook-regex}}" - - name: atlan-web-kube-secret - value: "argo-client-creds" - - name: heracles-uri - value: "http://heracles-service.heracles.svc.cluster.local" - - name: atlas-api-uri - value: "http://atlas-service-atlas.atlas.svc.cluster.local/api/atlas/v2" - - name: publish-chunk-size - value: "100" - - name: git-kube-secret-name - value: "git-ssh" - - name: git-kube-ssh-key - value: "private-key" - - name: statsd-host - value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" - - name: statsd-port - value: "9125" - - name: statsd-global-tags - value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" - - - name: extract-NAME-metadata - inputs: - parameters: - - name: credential-guid - - name: connection-qualified-name - - name: output-prefix - value: "argo-artifacts/{{workflow.namespace}}/{{workflow.name}}" - - name: fetch-folderless-assets - - name: include-filter - - name: exclude-filter - - name: marketplace-scripts-revision - - name: marketplace-packages-revision - - name: statsd-global-tags - dag: - tasks: - - name: extract-assets - template: NAME-api - withItems: - - { "url": "https://NAME..amazonaws.com/accounts//analyses", "name": "analysis" } - arguments: - parameters: - - name: url - value: "{{item.url}}" - - name: output-prefix - value: "{{inputs.parameters.output-prefix}}/{{item.name}}" - - name: NAME-request-config - value: | - { - "headers": { - "Content-Type":"application/x-amz-json-1.1" - }, - "json": { - "MaxResults": 100 - } - } - - name: credential-guid - value: "{{inputs.parameters.credential-guid}}" - - name: output-chunk-size - value: "1000" - - name: statsd-global-tags - value: "{{inputs.parameters.statsd-global-tags}}" - - name: NAME-execution-script - value: | - if state == ExecutionState.RAW_INPUT_PROCESS: - creds_arr = secrets["result-0.json"].split("""\n""") - region = '' - for cred in creds_arr: - LOGGER.debug("cred %s" % cred) - if 'AWS_REGION' in cred: - region = cred.split('=')[-1].replace('"', '') - region = region.strip() - - request_config['url'] = request_config['url'].replace('', region) - - accountid = '' - for cred in creds_arr: - if 'AWS_ACCOUNT_ID' in cred: - accountid = cred.split('=')[-1].replace('"', '') - - request_config['url'] = request_config['url'].replace('', accountid) - - - if state == ExecutionState.OUTPUT_PROCESS: - output = json.loads(output) - - if state == ExecutionState.API_POST: - next_token = response.json().get('NextToken', None) - if not next_token: - stop = True - else: - request_config['json']['NextToken'] = next_token - - if state == ExecutionState.API_FAIL: - failure_handler=FailureHandler.RETRY - - - name: process-NAME-metadata - inputs: - parameters: - - name: connection-qualified-name - - name: git-kube-secret-name - - name: git-kube-ssh-key - - name: statsd-host - - name: statsd-port - - name: statsd-global-tags - - name: marketplace-scripts-revision - - name: marketplace-packages-revision - - name: fetch-folderless-assets - - name: include-filter - - name: exclude-filter - dag: - tasks: - # Process NAME metadata and generate files for transformer - - name: process-metadata - template: process-metadata-template - arguments: - parameters: - - name: output-prefix - value: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/processed-metadata/{{workflow.name}}" - - name: git-kube-secret-name - value: "{{inputs.parameters.git-kube-secret-name}}" - - name: git-kube-ssh-key - value: "{{inputs.parameters.git-kube-ssh-key}}" - - name: marketplace-scripts-revision - value: "{{inputs.parameters.marketplace-scripts-revision}}" - - name: marketplace-packages-revision - value: "{{inputs.parameters.marketplace-packages-revision}}" - - name: fetch-folderless-assets - value: "{{inputs.parameters.fetch-folderless-assets}}" - - name: include-filter - value: "{{inputs.parameters.include-filter}}" - - name: exclude-filter - value: "{{inputs.parameters.exclude-filter}}" - - name: connection-qualified-name - value: "{{inputs.parameters.connection-qualified-name}}" - artifacts: - - name: extracted-metadata - s3: - key: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/extracted-metadata/{{workflow.name}}" - - - name: process-metadata-template - inputs: - parameters: - - name: output-prefix - - name: marketplace-scripts-revision - - name: marketplace-packages-revision - - name: git-kube-secret-name - value: "git-ssh" - - name: git-kube-ssh-key - value: "private-key" - - name: fetch-folderless-assets - - name: include-filter - - name: exclude-filter - - name: connection-qualified-name - - artifacts: - - name: extracted-metadata - path: /tmp/extracted-metadata - - name: scripts - path: /tmp/marketplace-scripts - git: - repo: git@github.com:atlanhq/marketplace-scripts - revision: "{{inputs.parameters.marketplace-scripts-revision}}" - insecureIgnoreHostKey: true - depth: 1 - sshPrivateKeySecret: - name: "{{inputs.parameters.git-kube-secret-name}}" - key: "{{inputs.parameters.git-kube-ssh-key}}" - - name: connection-cache - optional: true - path: "/tmp/connection-cache" - s3: - key: "connection-cache" - outputs: - artifacts: - - name: processed-metadata - path: /tmp/processed-metadata - s3: - key: "{{inputs.parameters.output-prefix}}" - archive: - none: { } - container: - image: ghcr.io/atlanhq/marketplace-scripts-base:0.1.10 - imagePullPolicy: IfNotPresent - workingDir: "/tmp/marketplace-scripts" - command: [ "python" ] - args: - - "-m" - - "marketplace_scripts.NAME.metadata" - - "--metadata-prefix" - - "/tmp/extracted-metadata" - - "--cache-prefix" - - "/tmp/connection-cache" - - "--output-prefix" - - "/tmp/processed-metadata" - - "--exclude-filter" - - "{{inputs.parameters.exclude-filter}}" - - "--include-filter" - - "{{inputs.parameters.include-filter}}" - - "--fetch-folderless-assets" - - "{{inputs.parameters.fetch-folderless-assets}}" - - "--connection-qualified-name" - - "{{inputs.parameters.connection-qualified-name}}" - - - name: publish-NAME-metadata - inputs: - parameters: - - name: marketplace-scripts-revision - - name: marketplace-packages-revision - - name: connection - - name: mode - - name: source - - name: atlas-api-uri - - name: heracles-uri - - name: atlan-web-kube-secret - - name: atlas-auth-type - - name: publish-chunk-size - - name: git-kube-secret-name - - name: git-kube-ssh-key - - name: statsd-host - - name: statsd-port - - name: statsd-global-tags - dag: - tasks: - # Run atlan-crawler/generic-publish template - - name: publish_metadata - templateRef: - name: atlan-crawler - template: generic-publish - arguments: - artifacts: - - name: data - s3: - key: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/processed-metadata/{{workflow.name}}" - - name: transformer-config - raw: - data: | - { - "external_map": { - "crawler_name": "{{workflow.labels.workflows.argoproj.io/workflow-template}}", - "tenant_id": "{{workflow.namespace}}", - "integration_name": "{{inputs.parameters.source}}", - "workflow_name": "{{workflow.name}}", - "connection_name": "{{=jsonpath(inputs.parameters.connection, '$.attributes.name')}}", - "connection_qn": "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" - }, - "output_prefix": "/tmp/entities", - "templates_root": "/tmp/templates", - "transformation_config": [ - { - "input_file_pattern": "/tmp/inputs/ASSET_NAME.json", - "template": "packages/atlan/NAME/transformers/ASSET_NAME.jinja2", - "output_file_prefix": "ASSET_NAME/ASSET_NAME", - "output_chunk_size": 10000 - } - ] - } - parameters: - - name: connection - value: "{{inputs.parameters.connection}}" - - name: mode - value: "{{inputs.parameters.mode}}" - - name: source - value: "{{inputs.parameters.source}}" - - name: raw-input-file-sort - value: "" - - name: raw-input-folder-sort - value: "" - - name: raw-input-file-pattern - value: "**/*.json" - - name: publish-chunk-size - value: "{{inputs.parameters.publish-chunk-size}}" - - name: atlas-auth-type - value: "{{inputs.parameters.atlas-auth-type}}" - - name: statsd-global-tags - value: "{{inputs.parameters.statsd-global-tags}}" - - name: marketplace-scripts-revision - value: "{{inputs.parameters.marketplace-scripts-revision}}" - - name: marketplace-packages-revision - value: "{{inputs.parameters.marketplace-packages-revision}}" - - - name: NAME-api - synchronization: - semaphore: - configMapKeyRef: - name: atlan-NAME - key: NAME-api - inputs: - artifacts: - - name: raw-input - path: "/tmp/input/" - optional: true - - name: raw-input-file - path: "/tmp/input/input.json" - optional: true - parameters: - - name: threads - value: "1" - - name: credential-guid - - name: NAME-request-config - value: | - { - "headers": { - "Content-Type":"application/x-amz-json-1.1" - }, - "json": { - "MaxResults": 100 - } - } - - name: paginate - value: "true" - enum: - - "true" - - "false" - - name: NAME-output-key - value: "TableList" - - name: url - - name: NAME-execution-script - value: | - if "{{inputs.parameters.threads}}" != "1": - logger.error("Error! You must pass custom execution script from parent for thread count > 1") - raise Exception("Invalid thread count for offset based pagination!") - if state == ExecutionState.RAW_INPUT_PROCESS: - creds_arr = secrets["result-0.json"].split("""\n""") - region = '' - for cred in creds_arr: - if 'AWS_REGION' in cred: - region = cred.split('=')[-1].replace('"', '') - - request_config['url'] = request_config['url'].replace('', region) - - if state == ExecutionState.OUTPUT_PROCESS: - output=json.loads(output)['{{inputs.parameters.NAME-output-key}}'] - - if state == ExecutionState.API_POST: - next_token = response.json().get('NextToken', None) - if not next_token: - stop = True - else: - request_config['json']['NextToken'] = next_token - - if state == ExecutionState.API_FAIL: - failure_handler=FailureHandler.RETRY - - name: output-chunk-size - value: 0 - - name: raw-input-file-pattern - value: "" - - name: raw-input-paginate - value: "0" - - name: raw-input-multiline - value: "False" - - name: pagination-wait-time - value: "0" - - name: kube-secret-name - value: "argo-client-creds" - - name: client-id-env - value: "login" - - name: client-secret-env - value: "password" - - name: token-url-env - value: "host" - - name: statsd-host - value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" - - name: statsd-port - value: "9125" - - name: statsd-global-tags - value: "workflow={{workflow.name}},bot=atlan-NAME" - - name: output-prefix - value: "argo-artifacts/{{workflow.namespace}}/{{workflow.name}}/{{pod.name}}/" - - name: heracles-uri - value: "http://heracles-service.heracles.svc.cluster.local" - - name: init-execution-script - value: | - if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): - LOGGER.debug('Heracles is unavailable. Performing retry with back-off') - failure_handler = FailureHandler.RETRY - if state == ExecutionState.OUTPUT_PROCESS: - - credential = json.loads(output) - output = "" - for key, value in credential.items(): - if key == "username": - output += f"""AWS_ACCESS_KEY_ID=\"{str(value)}\"\n""" - elif key == "password": - output += f"""AWS_SECRET_ACCESS_KEY=\"{str(value)}\"\n""" - elif key == "host": - if value is not None: - region = value.split('.')[-3] - output += f"""AWS_REGION=\"{str(region)}\"\n""" - elif key == "extra": - if "region" in value: - region = value.get("region", "") - output += f"""AWS_REGION=\"{str(region)}\"\n""" - if "accountid" in value: - accountid = value.get("accountid", "") - output += f"""AWS_ACCOUNT_ID=\"{str(accountid)}\"\n""" - if "aws_role_arn" in value: - output += f"""AWS_ROLE_ARN=\"{str(value.get("aws_role_arn", ""))}\"\n""" - if "aws_external_id" in value: - output += f"""AWS_EXTERNAL_ID=\"{str(value.get("aws_external_id", ""))}\"\n""" - else: - continue - - if state == ExecutionState.API_POST: - stop = True - outputs: - artifacts: - - name: success - path: "/tmp/rest/success" - s3: - key: "{{inputs.parameters.output-prefix}}/success" - archive: - none: { } - - name: failure - path: "/tmp/rest/failure" - archive: - none: { } - s3: - key: "{{inputs.parameters.output-prefix}}/failure" - parameters: - - name: success-num-files - valueFrom: - path: "/tmp/rest/success/result-gen.txt" - - name: failure-num-files - valueFrom: - path: "/tmp/rest/failure/result-gen.txt" - volumes: - - name: credentials - emptyDir: { } - container: - image: ghcr.io/atlanhq/rest-master:165b7e5 - command: [ "./entrypoint.sh" ] - volumeMounts: - - name: credentials - mountPath: /tmp/credentials - imagePullPolicy: IfNotPresent - args: [ - "python3", "main.py","GET", "{{inputs.parameters.url}}", - "--raw-input-paginate", "{{inputs.parameters.raw-input-paginate}}", - "--raw-input-multiline", "{{inputs.parameters.raw-input-multiline}}", - "--threads", "{{inputs.parameters.threads}}", - "--raw-input-file-pattern", "{{inputs.parameters.raw-input-file-pattern}}", - "--request-config", "{{inputs.parameters.NAME-request-config}}", - "--execution-script", "{{inputs.parameters.NAME-execution-script}}", - "--secrets-path", "/tmp/credentials/success/*.json", - "--auth-type", "aws", - "--auth-aws-service", "NAME", - "--auth-aws-region", "AWS_REGION", - "--output-chunk-size", "{{inputs.parameters.output-chunk-size}}", - "--output-file-prefix", "/tmp/rest", - "--pagination-wait-time", "0", - "--max-retries", "10", - "--statsd-host", "{{inputs.parameters.statsd-host}}", - "--statsd-port", "{{inputs.parameters.statsd-port}}", - "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" - ] - initContainers: - - name: fetch-credentials - image: ghcr.io/atlanhq/rest-master:165b7e5 - command: [ "python3", "main.py" ] - env: - - name: OAUTHLIB_INSECURE_TRANSPORT - value: "1" - - name: CLIENT_ID - valueFrom: - secretKeyRef: - name: "{{inputs.parameters.kube-secret-name}}" - key: "{{inputs.parameters.client-id-env}}" - - name: CLIENT_SECRET - valueFrom: - secretKeyRef: - name: "{{inputs.parameters.kube-secret-name}}" - key: "{{inputs.parameters.client-secret-env}}" - - name: TOKEN_URL - valueFrom: - secretKeyRef: - name: "{{inputs.parameters.kube-secret-name}}" - key: "{{inputs.parameters.token-url-env}}" - mirrorVolumeMounts: true - args: [ - "GET", - "{{inputs.parameters.heracles-uri}}/credentials/{{inputs.parameters.credential-guid}}/use", - "--raw-input", "{}", - "--raw-input-file-sort", "", - "--raw-input-multiline", "False", - "--execution-script", "{{inputs.parameters.init-execution-script}}", - "--raw-input-paginate", "0", - "--auth-type", "oauth2", - "--auth-oauth2-type", "client_credentials", - "--auth-oauth2-impersonate-user", "{{=sprig.dig('labels', 'workflows', 'argoproj', 'io/creator', '', workflow)}}", - "--auth-oauth2-client-credentials-client-id", "CLIENT_ID", - "--auth-oauth2-client-credentials-secret", "CLIENT_SECRET", - "--auth-oauth2-client-credentials-token-url", "TOKEN_URL", - "--output-chunk-size", "0", - "--output-file-prefix", "/tmp/credentials", - "--pagination-wait-time", "0", - "--max-retries", "10", - "--statsd-host", "{{inputs.parameters.statsd-host}}", - "--statsd-port", "{{inputs.parameters.statsd-port}}", - "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" - ] diff --git a/lib/connectors/packages/BI/templates/basic+jwt.yaml b/lib/connectors/packages/BI/templates/basic+jwt.yaml deleted file mode 100644 index ab64710..0000000 --- a/lib/connectors/packages/BI/templates/basic+jwt.yaml +++ /dev/null @@ -1,633 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: atlan-NAME -spec: - entrypoint: main - templates: - - name: main - inputs: - parameters: - - name: credentials-fetch-strategy - value: "credential_guid" - enum: - - "k8s_secret" - - "credential_guid" - - - name: credential-kube-secret-name - value: "{{workflow.name}}-credential-secret" - - - name: credential-guid - value: "" - - #connection atlan object - - name: connection - - # NAME api version. - - name: api-version - value: "v52.0" - - #publish mode - - name: publish-mode - value: "production" - enum: - - "production" - - "test" - - "dev" - - #publish credential - - name: atlas-auth-type - value: "internal" - enum: - - "internal" - - "apikey" - - - name: git-kube-secret-name - value: "git-ssh" - - name: git-kube-ssh-key - value: "private-key" - - - name: auto-classification - value: "false" - enum: - - "true" - - "false" - - - name: attachment-confidence-threshold - value: "0.8" - - - name: fetch-reports - value: "true" - - # Revisions - - name: marketplace-scripts-revision - valueFrom: - configMapKeyRef: - name: atlan-runtime-packages-config - key: "marketplaceScriptsBranch" - optional: true - default: "master" - - name: marketplace-packages-revision - valueFrom: - configMapKeyRef: - name: atlan-runtime-packages-config - key: "marketplacePackagesBranch" - optional: true - default: "master" - - dag: - tasks: - - name: fetch-credentials - templateRef: - name: rest-api - template: oauth2-client-credentials - arguments: - parameters: - - name: method - value: GET - - name: url - value: "http://heracles-service.heracles.svc.cluster.local/credentials/{{inputs.parameters.credential-guid}}/use" - - name: request-config - value: | - { - "headers": { - "user-id": "{{workflow.labels.workflows.argoproj.io/creator}}" - } - } - - name: kube-secret-name - value: "argo-client-creds" - - name: max-retries - value: "10" - - name: client-id-env - value: "login" - - name: client-secret-env - value: "password" - - name: token-url-env - value: "host" - - name: execution-script - value: | - if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): - LOGGER.debug('Heracles is unavailable. Performing retry with back-off') - failure_handler = FailureHandler.RETRY - if state == ExecutionState.OUTPUT_PROCESS: - credential = json.loads(output) - auth_type = credential["authType"] - - if auth_type == "basic": - oauth2_type = "resource_owner_password" - elif auth_type == "jwt": - oauth2_type = "jwt_bearer" - - output = { - "oauth2_type": oauth2_type, - } - - if state == ExecutionState.API_POST: - stop=True - - name: statsd-host - value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" - - name: statsd-port - value: "9125" - - name: statsd-global-tags - value: "workflow={{workflow.name}},bot=atlan-NAME" - - name: auth-type-to-param - dependencies: - - fetch-credentials - templateRef: - name: utils - template: artifact-to-key-param - arguments: - parameters: - - name: key - value: "oauth2_type" - artifacts: - - name: input - from: "{{tasks.fetch-credentials.outputs.artifacts.success}}" - subPath: "result-0.json" - - name: extract - dependencies: - - auth-type-to-param - template: extract-NAME-metadata - arguments: - parameters: - - name: credentials-fetch-strategy - value: "{{inputs.parameters.credentials-fetch-strategy}}" - - name: credential-kube-secret-name - value: "{{inputs.parameters.credential-kube-secret-name}}" - - name: credential-guid - value: "{{inputs.parameters.credential-guid}}" - - name: oauth2-type - value: "{{tasks.auth-type-to-param.outputs.parameters.output}}" - - name: connection-qualified-name - value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" - - name: heracles-uri - value: "http://heracles-service.heracles.svc.cluster.local" - - name: git-kube-secret-name - value: "git-ssh" - - name: git-kube-ssh-key - value: "private-key" - - name: statsd-host - value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" - - name: statsd-port - value: "9125" - - name: statsd-global-tags - value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" - - name: fetch-reports - value: "{{inputs.parameters.fetch-reports}}" - - - name: process-objects - template: process-NAME-described-assets - dependencies: - - extract - withParam: "{{tasks.extract.outputs.parameters.org-host-name}}" - arguments: - parameters: - - name: host-name - value: "{{item.host}}" - - name: output-prefix - value: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/metadata-extract/{{workflow.name}}/processed-metadata" - - name: process-chunk-size - value: 10000 - artifacts: - - name: input-raw-metadata - s3: - key: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/metadata-extract/{{workflow.name}}/raw-metadata" - archive: - none: { } - - name: scripts - git: - repo: git@github.com:atlanhq/marketplace-scripts - insecureIgnoreHostKey: true - depth: 1 - revision: "{{inputs.parameters.marketplace-scripts-revision}}" - sshPrivateKeySecret: - name: "{{inputs.parameters.git-kube-secret-name}}" - key: "{{inputs.parameters.git-kube-ssh-key}}" - - name: publish - dependencies: - - process-objects - template: publish-NAME-metadata - arguments: - parameters: - - name: connection - value: "{{inputs.parameters.connection}}" - - name: mode - value: "{{inputs.parameters.publish-mode}}" - - name: source - value: "NAME" - - name: atlas-api-uri - value: "http://atlas-service-atlas.atlas.svc.cluster.local/api/atlas/v2" - - name: atlas-auth-type - value: "{{inputs.parameters.atlas-auth-type}}" - - name: heracles-uri - value: "http://heracles-service.heracles.svc.cluster.local" - - name: processed-data-key - value: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/metadata-extract/{{workflow.name}}/processed-metadata" - - name: atlan-web-kube-secret - value: "argo-client-creds" - - name: publish-chunk-size - value: "25" - - name: git-kube-secret-name - value: "git-ssh" - - name: git-kube-ssh-key - value: "private-key" - - name: statsd-host - value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" - - name: statsd-port - value: "9125" - - name: statsd-global-tags - value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" - - name: marketplace-scripts-revision - value: "{{inputs.parameters.marketplace-scripts-revision}}" - - name: marketplace-packages-revision - value: "{{inputs.parameters.marketplace-packages-revision}}" - - - name: extract-NAME-metadata - inputs: - parameters: - - name: credentials-fetch-strategy - - name: credential-kube-secret-name - - name: credential-guid - - name: oauth2-type - - name: connection-qualified-name - - name: heracles-uri - - name: git-kube-secret-name - - name: git-kube-ssh-key - - name: statsd-host - - name: statsd-port - - name: statsd-global-tags - - name: fetch-reports - outputs: - parameters: - - name: org-host-name - valueFrom: - default: "[]" - parameter: "{{tasks.convert-to-param.outputs.parameters.output}}" - dag: - tasks: - - name: extract-organization - template: NAME-api - arguments: - parameters: - - name: url - value: "/services/data/v52.0/query?q=select%20Fields(All)%20from%20Organization%20limit%201" - - name: credential-guid - value: "{{inputs.parameters.credential-guid}}" - - name: oauth2-type - value: "{{inputs.parameters.oauth2-type}}" - - name: execution-script - value: | - if state == ExecutionState.RAW_INPUT_PROCESS: - creds_arr = secrets["result-0.json"].split("""\n""") - host = '' - for cred in creds_arr: - if 'HOST' in cred: - host = cred.split('=')[-1].replace('"', '') - request_config['url'] = request_config['url'].replace('', host) - store['host'] = host - - if state == ExecutionState.OUTPUT_PROCESS: - _response = json.loads(output).get('records', []) - - if len(_response) > 0: - output = _response[0] - output['host'] = store['host'] - - if state == ExecutionState.API_POST: - stop = True - - if state == ExecutionState.API_FAIL: failure_handler=FailureHandler.RETRY - - name: output-chunk-size - value: 1 - - name: output-prefix - value: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/metadata-extract/{{workflow.name}}/raw-metadata/described-organization/0" - - - name: process-NAME-metadata - inputs: - parameters: - - name: host-name - - name: output-prefix - - name: process-chunk-size - artifacts: - - name: input-raw-metadata - path: "/tmp/input" - - name: scripts - path: /tmp/marketplace-scripts - outputs: - artifacts: - - name: output - path: "/tmp/output" - s3: - key: "{{inputs.parameters.output-prefix}}" - archive: - none: { } - container: - image: ghcr.io/atlanhq/marketplace-scripts-base:0.1.11 - workingDir: "/tmp/marketplace-scripts" - command: [ "python" ] - args: - - "-m" - - "marketplace_scripts.NAME.main" - - "--host-name" - - "{{inputs.parameters.host-name}}" - - "--input-dir" - - "/tmp/input" - - "--output-prefix" - - "/tmp/output/" - - "--chunk-size" - - "{{inputs.parameters.process-chunk-size}}" - - - name: publish-NAME-metadata - inputs: - parameters: - - name: connection - - name: mode - - name: source - - name: atlas-api-uri - - name: heracles-uri - - name: processed-data-key - - name: atlan-web-kube-secret - - name: atlas-auth-type - - name: publish-chunk-size - - name: git-kube-secret-name - - name: git-kube-ssh-key - - name: statsd-host - - name: statsd-port - - name: statsd-global-tags - - name: marketplace-scripts-revision - - name: marketplace-packages-revision - dag: - tasks: - - name: publish - templateRef: - name: atlan-crawler - template: generic-publish - arguments: - artifacts: - - name: data - s3: - key: "{{inputs.parameters.processed-data-key}}" - - name: transformer-config - raw: - data: | - { - "external_map": { - "crawler_name": "{{workflow.labels.workflows.argoproj.io/workflow-template}}", - "tenant_id": "{{workflow.namespace}}", - "integration_name": "{{inputs.parameters.source}}", - "workflow_name": "{{workflow.name}}", - "connection_name": "{{=jsonpath(inputs.parameters.connection, '$.attributes.name')}}", - "connection_qn": "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" - }, - "output_prefix": "/tmp/entities/", - "templates_root": "/tmp/templates/packages", - "transformation_config": [ - { - "input_file_pattern": "/tmp/inputs/organization*.json", - "template": "atlan/NAME/transformers/organization.jinja2", - "output_file_prefix": "organization" - }, - { - "input_file_pattern": "/tmp/inputs/objects*.json", - "template": "atlan/NAME/transformers/object.jinja2", - "output_file_prefix": "objects", - "ignore": true - }, - { - "input_file_pattern": "/tmp/inputs/fields*.json", - "template": "atlan/NAME/transformers/field.jinja2", - "output_file_prefix": "fields", - "ignore": true - }, - { - "input_file_pattern": "/tmp/inputs/reports*.json", - "template": "atlan/NAME/transformers/report.jinja2", - "output_file_prefix": "reports", - "ignore": true - }, - { - "input_file_pattern": "/tmp/inputs/dashboards*.json", - "template": "atlan/NAME/transformers/dashboard.jinja2", - "output_file_prefix": "dashboards", - "ignore": true - } - ] - } - parameters: - - name: connection - value: "{{inputs.parameters.connection}}" - - name: mode - value: "{{inputs.parameters.mode}}" - - name: source - value: "{{inputs.parameters.source}}" - - name: raw-input-file-sort - value: "organization,objects,fields,reports,dashboards" - - name: publish-chunk-size - value: "{{inputs.parameters.publish-chunk-size}}" - - name: atlas-auth-type - value: "{{inputs.parameters.atlas-auth-type}}" - - name: statsd-global-tags - value: "{{inputs.parameters.statsd-global-tags}}" - - name: marketplace-scripts-revision - value: "{{inputs.parameters.marketplace-scripts-revision}}" - - name: marketplace-packages-revision - value: "{{inputs.parameters.marketplace-packages-revision}}" - - - name: NAME-api - synchronization: - semaphore: - configMapKeyRef: - name: atlan-NAME - key: api - volumes: - - name: atlan-NAME-workflow-directory - ephemeral: - volumeClaimTemplate: - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 60Gi - - name: credentials - emptyDir: { } - inputs: - artifacts: - - name: raw-input - path: /tmp/input - optional: true - parameters: - - name: credential-guid - - name: oauth2-type - - name: page-size - value: "10" - - name: url - - name: request-config - value: | - { - "headers": { - "Content-Type": "application/x-www-form-urlencoded" - } - } - - name: execution-script - - name: output-chunk-size - value: "0"sourceCategory - - name: kube-secret-name - value: "argo-client-creds" - - name: client-id-env - value: "login" - - name: client-secret-env - value: "password" - - name: token-url-env - value: "host" - - name: raw-input-file-pattern - value: "" - - name: raw-input-paginate - value: 0 - - name: raw-input-multiline - value: "False" - - name: statsd-host - value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" - - name: statsd-port - value: "9125" - - name: statsd-global-tags - value: "workflow={{workflow.name}},bot=atlan-NAME" - - name: output-prefix - value: "argo-artifacts/{{workflow.namespace}}/{{workflow.name}}/{{pod.name}}" - - name: heracles-uri - value: "http://heracles-service.heracles.svc.cluster.local" - - name: init-execution-script - value: | - if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): - LOGGER.debug('Heracles is unavailable. Performing retry with back-off') - failure_handler = FailureHandler.RETRY - if state == ExecutionState.OUTPUT_PROCESS: - credential = json.loads(output) - auth_type = credential.get('authType', '') - extra_params = credential.get('extra', {}) - is_sandbox = extra_params.get('is_sandbox', False) - token_url = 'test.NAME.com' if is_sandbox else 'login.NAME.com' - - output = f"""USERNAME="{credential.get('username', '')}" - HOST="{credential.get('host', '')}" - TOKEN_URL="https://{token_url}/services/oauth2/token" - CLIENT_ID="{extra_params.get('client_id', '')}" - """ - - if auth_type == 'basic': - password = credential.get('password', '').replace('$', '\$') - output += f"""PASSWORD="{password}" - CLIENT_SECRET="{extra_params.get('client_secret', '')}" - """ - elif auth_type == 'jwt': - output += f"""PRIVATE_KEY="{extra_params.get('private_key', '')}" - """ - - if state == ExecutionState.API_POST: - stop = True - outputs: - artifacts: - - name: success - path: "/tmp/rest/success" - s3: - key: "{{inputs.parameters.output-prefix}}/success" - archive: - none: { } - - name: failure - path: "/tmp/rest/failure" - archive: - none: { } - s3: - key: "{{inputs.parameters.output-prefix}}/failure" - parameters: - - name: success-num-files - valueFrom: - path: "/tmp/rest/success/result-gen.txt" - - name: failure-num-files - valueFrom: - path: "/tmp/rest/failure/result-gen.txt" - container: - image: ghcr.io/atlanhq/rest-master:165b7e5 - command: [ "./entrypoint.sh" ] - volumeMounts: - - name: atlan-NAME-workflow-directory - mountPath: /tmp - - name: credentials - mountPath: /tmp/credentials - imagePullPolicy: IfNotPresent - args: [ - "python3", "main.py", "GET", "{{inputs.parameters.url}}", - "--raw-input-paginate", "{{inputs.parameters.raw-input-paginate}}", - "--raw-input-file-pattern", "{{inputs.parameters.raw-input-file-pattern}}", - "--raw-input-multiline", "{{inputs.parameters.raw-input-multiline}}", - "--request-config", "{{inputs.parameters.request-config}}", - "--execution-script", "{{inputs.parameters.execution-script}}", - "--secrets-path", "/tmp/credentials/success/*.json", - "--auth-type", "oauth2", - "--auth-oauth2-type", "{{inputs.parameters.oauth2-type}}", - "--auth-oauth2-resource-owner-password-client-id", "CLIENT_ID", - "--auth-oauth2-resource-owner-password-secret", "CLIENT_SECRET", - "--auth-oauth2-resource-owner-password-token-url", "TOKEN_URL", - "--auth-oauth2-resource-owner-password-username", "USERNAME", - "--auth-oauth2-resource-owner-password-password", "PASSWORD", - "--auth-oauth2-jwt-bearer-client-id", "CLIENT_ID", - "--auth-oauth2-jwt-bearer-private-key", "PRIVATE_KEY", - "--auth-oauth2-jwt-bearer-token-url", "TOKEN_URL", - "--auth-oauth2-jwt-bearer-username", "USERNAME", - "--auth-oauth2-jwt-bearer-token-expiry", "{{=sprig.unixEpoch(sprig.dateModify('3m', sprig.now()))}}", - "--output-chunk-size", "{{inputs.parameters.output-chunk-size}}", - "--output-file-prefix", "/tmp/rest/", - "--pagination-wait-time", "10", - "--max-retries", "3", - "--statsd-host", "{{inputs.parameters.statsd-host}}", - "--statsd-port", "{{inputs.parameters.statsd-port}}", - "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" - ] - - initContainers: - - name: fetch-credentials - image: ghcr.io/atlanhq/rest-master:165b7e5 - command: [ "python3", "main.py" ] - env: - - name: OAUTHLIB_INSECURE_TRANSPORT - value: "1" - - name: CLIENT_ID - valueFrom: - secretKeyRef: - name: "{{inputs.parameters.kube-secret-name}}" - key: "{{inputs.parameters.client-id-env}}" - - name: CLIENT_SECRET - valueFrom: - secretKeyRef: - name: "{{inputs.parameters.kube-secret-name}}" - key: "{{inputs.parameters.client-secret-env}}" - - name: TOKEN_URL - valueFrom: - secretKeyRef: - name: "{{inputs.parameters.kube-secret-name}}" - key: "{{inputs.parameters.token-url-env}}" - mirrorVolumeMounts: true - args: [ - "GET", - "{{inputs.parameters.heracles-uri}}/credentials/{{inputs.parameters.credential-guid}}/use", - "--raw-input", "{}", - "--raw-input-file-pattern", "", - "--raw-input-file-sort", "", - "--raw-input-multiline", "f", - "--execution-script", "{{inputs.parameters.init-execution-script}}", - "--raw-input-paginate", "0", - "--auth-type", "oauth2", - "--auth-oauth2-type", "client_credentials", - "--auth-oauth2-impersonate-user", "{{=sprig.dig('labels', 'workflows', 'argoproj', 'io/creator', '', workflow)}}", - "--auth-oauth2-client-credentials-client-id", "CLIENT_ID", - "--auth-oauth2-client-credentials-secret", "CLIENT_SECRET", - "--auth-oauth2-client-credentials-token-url", "TOKEN_URL", - "--output-chunk-size", "0", - "--output-file-prefix", "/tmp/credentials", - "--pagination-wait-time", "0", - "--max-retries", - "10", - "--statsd-host", "{{inputs.parameters.statsd-host}}", - "--statsd-port", "{{inputs.parameters.statsd-port}}", - "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" - ] - diff --git a/lib/connectors/packages/BI/templates/oauth.yaml b/lib/connectors/packages/BI/templates/oauth.yaml deleted file mode 100644 index 152899f..0000000 --- a/lib/connectors/packages/BI/templates/oauth.yaml +++ /dev/null @@ -1,581 +0,0 @@ -# Workflow Templates -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: atlan-NAME -spec: - entrypoint: main - templates: - - name: main - inputs: - parameters: - # Credential - - name: credentials-fetch-strategy - value: "credential_guid" - enum: - - "credential_guid" - - "k8s_secret" - - name: credential-guid - value: "" - # Connection Entity - - name: connection - - name: atlas-auth-type - value: "internal" - enum: - - "internal" - - "apikey" - # Include/Exclude Filters - - name: include-filter - value: "{}" - - name: exclude-filter - value: "{}" - # Publish Mode - - name: publish-mode - value: "production" - enum: - - "production" - - "dev" - - "test" - # Revisions - - name: marketplace-scripts-revision - valueFrom: - configMapKeyRef: - name: atlan-runtime-packages-config - key: "marketplaceScriptsBranch" - optional: true - default: "master" - - name: marketplace-packages-revision - valueFrom: - configMapKeyRef: - name: atlan-runtime-packages-config - key: "marketplacePackagesBranch" - optional: true - default: "master" - dag: - tasks: - # Extract metadata from NAME - - name: extract - template: extract-NAME-metadata - arguments: - parameters: - - name: credentials-fetch-strategy - value: "{{inputs.parameters.credentials-fetch-strategy}}" - - name: credential-guid - value: "{{inputs.parameters.credential-guid}}" - - name: connection-qualified-name - value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" - - name: include-filter - value: "{{inputs.parameters.include-filter}}" - - name: exclude-filter - value: "{{inputs.parameters.exclude-filter}}" - - name: git-kube-secret-name - value: "git-ssh" - - name: git-kube-ssh-key - value: "private-key" - - name: heracles-uri - value: "http://heracles-service.heracles.svc.cluster.local" - - name: statsd-host - value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" - - name: statsd-port - value: "9125" - - name: statsd-global-tags - value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" - - name: marketplace-scripts-revision - value: "{{inputs.parameters.marketplace-scripts-revision}}" - # Process the extracted metadata - - name: process - template: process-NAME-metadata - depends: "extract.Succeeded" - arguments: - parameters: - - name: connection-qualified-name - value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" - - name: git-kube-secret-name - value: "git-ssh" - - name: git-kube-ssh-key - value: "private-key" - - name: statsd-host - value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" - - name: statsd-port - value: "9125" - - name: statsd-global-tags - value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" - - name: marketplace-scripts-revision - value: "{{inputs.parameters.marketplace-scripts-revision}}" - - - name: publish - depends: "process.Succeeded" - template: publish-NAME-metadata - arguments: - parameters: - - name: connection - value: "{{inputs.parameters.connection}}" - - name: mode - value: "{{inputs.parameters.publish-mode}}" - - name: source - value: "NAME" - - name: atlas-auth-type - value: "{{inputs.parameters.atlas-auth-type}}" - - name: include-filter - value: "{{inputs.parameters.include-filter}}" - - name: exclude-filter - value: "{{inputs.parameters.exclude-filter}}" - - name: atlan-web-kube-secret - value: "argo-client-creds" - - name: heracles-uri - value: "http://heracles-service.heracles.svc.cluster.local" - - name: atlas-api-uri - value: "http://atlas-service-atlas.atlas.svc.cluster.local/api/atlas/v2" - - name: publish-chunk-size - value: "100" - - name: git-kube-secret-name - value: "git-ssh" - - name: git-kube-ssh-key - value: "private-key" - - name: statsd-host - value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" - - name: statsd-port - value: "9125" - - name: statsd-global-tags - value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" - - name: marketplace-scripts-revision - value: "{{inputs.parameters.marketplace-scripts-revision}}" - - name: marketplace-packages-revision - value: "{{inputs.parameters.marketplace-packages-revision}}" - - - name: extract-NAME-metadata - inputs: - parameters: - - name: credentials-fetch-strategy - - name: credential-guid - - name: connection-qualified-name - - name: include-filter - - name: exclude-filter - - name: git-kube-secret-name - - name: git-kube-ssh-key - - name: heracles-uri - - name: statsd-host - - name: statsd-port - - name: statsd-global-tags - - name: marketplace-scripts-revision - outputs: - parameters: - - name: auth-type-and-host - valueFrom: - parameter: "{{tasks.auth-type-and-host-to-param.outputs.parameters.output}}" - dag: - tasks: - # Fetch the credential - - name: fetch-credentials - when: "{{inputs.parameters.credentials-fetch-strategy}} == credential_guid" - templateRef: - name: rest-api - template: oauth2-client-credentials - arguments: - parameters: - - name: method - value: GET - - name: url - value: "{{inputs.parameters.heracles-uri}}/credentials/{{inputs.parameters.credential-guid}}/use" - - name: request-config - value: | - { - "headers": { - "user-id": "{{workflow.labels.workflows.argoproj.io/creator}}" - } - } - - name: kube-secret-name - value: "argo-client-creds" - - name: max-retries - value: "10" - - name: client-id-env - value: "login" - - name: client-secret-env - value: "password" - - name: token-url-env - value: "host" - - name: execution-script - value: | - if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): - LOGGER.debug("Heracles is unavailable. Performing retry with back-off") - failure_handler = FailureHandler.RETRY - - if state == ExecutionState.OUTPUT_PROCESS: - credential = json.loads(output) - output = { - "authType": credential["authType"], - "host": credential["host"] - } - - if state == ExecutionState.API_POST: - stop=True - - name: statsd-host - value: "{{inputs.parameters.statsd-host}}" - - name: statsd-port - value: "{{inputs.parameters.statsd-port}}" - - name: statsd-global-tags - value: "{{inputs.parameters.statsd-global-tags}}" - # Convert that `fetch-credentials` output from an artifact to a parameter - - name: auth-type-and-host-to-param - dependencies: - - fetch-credentials - templateRef: - name: utils - template: artifact-to-direct-param - arguments: - artifacts: - - name: input - from: "{{tasks.fetch-credentials.outputs.artifacts.success}}" - subPath: "result-0.json" - # Get all the workbooks - - name: fetch-ASSET_NAME - dependencies: - - auth-type-and-host-to-param - template: api-request - arguments: - parameters: - - name: auth-type - value: "{{=jsonpath(tasks['auth-type-and-host-to-param'].outputs.parameters.output, '$.authType')}}" - - name: credential-guid - value: "{{inputs.parameters.credential-guid}}" - - name: heracles-uri - value: "{{inputs.parameters.heracles-uri}}" - - name: method - value: GET - - name: url - value: "<>" - - name: request-config - value: | - { - "params": { - "page": 0, - "limit": 100 - } - } - - name: kube-secret-name - value: "argo-client-creds" - - name: output-prefix - value: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/extracted-metadata/{{workflow.name}}/ASSET_NAME" - - name: output-chunk-size - value: "1000" - - name: execution-script - value: | - if state == ExecutionState.RAW_INPUT_PROCESS: - creds_arr = secrets["result-0.json"].split("""\n""") - host = "" - for cred in creds_arr: - if "HOST" in cred: - host = cred.split('=')[-1].replace('"', '') - - url = "{{inputs.parameters.url}}" - url = url.replace("", host) - LOGGER.debug(f"URL: {url}") - request_config["url"] = url - - if state == ExecutionState.OUTPUT_PROCESS: - output = json.loads(output) - - if state == ExecutionState.API_POST: - workbook_response = response.json() - if not workbook_response.get("hasMore"): - stop = True - else: - next_page = workbook_response.get("nextPage") - request_config["params"] = { - "page": next_page, - "limit": 100 - } - if state == ExecutionState.API_FAIL: - if response.status_code == 500: - failure_handler = FailureHandler.NONE - else: - failure_handler = FailureHandler.RETRY - - name: statsd-host - value: "{{inputs.parameters.statsd-host}}" - - name: statsd-port - value: "{{inputs.parameters.statsd-port}}" - - name: statsd-global-tags - value: "{{inputs.parameters.statsd-global-tags}}" - - - name: process-NAME-metadata - inputs: - parameters: - - name: connection-qualified-name - - name: git-kube-secret-name - - name: git-kube-ssh-key - - name: statsd-host - - name: statsd-port - - name: statsd-global-tags - - name: marketplace-scripts-revision - dag: - tasks: - - name: process-metadata - template: process-metadata - arguments: - parameters: - - name: output-prefix - value: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/processed-metadata/{{workflow.name}}" - - name: git-kube-secret-name - value: "{{inputs.parameters.git-kube-secret-name}}" - - name: git-kube-ssh-key - value: "{{inputs.parameters.git-kube-ssh-key}}" - - name: marketplace-scripts-revision - value: "{{inputs.parameters.marketplace-scripts-revision}}" - artifacts: - - name: extracted-metadata - s3: - key: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/extracted-metadata/{{workflow.name}}" - - name: parsed-queries - s3: - key: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/parsed-queries/{{workflow.name}}" - - - name: publish-NAME-metadata - inputs: - parameters: - - name: connection - - name: mode - - name: source - - name: atlas-api-uri - - name: heracles-uri - - name: atlan-web-kube-secret - - name: atlas-auth-type - - name: publish-chunk-size - - name: git-kube-secret-name - - name: git-kube-ssh-key - - name: statsd-host - - name: statsd-port - - name: statsd-global-tags - - name: marketplace-scripts-revision - - name: marketplace-packages-revision - dag: - tasks: - # Run atlan-crawler/publish-bi template - - name: publish - templateRef: - name: atlan-crawler - template: publish-bi - arguments: - artifacts: - - name: data - s3: - key: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/processed-metadata/{{workflow.name}}" - - name: transformer-config - raw: - data: | - { - "external_map": { - "crawler_name": "{{workflow.labels.workflows.argoproj.io/workflow-template}}", - "tenant_id": "{{workflow.namespace}}", - "integration_name": "{{inputs.parameters.source}}", - "workflow_name": "{{workflow.name}}", - "connection_name": "{{=jsonpath(inputs.parameters.connection, '$.attributes.name')}}", - "connection_qn": "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" - }, - "output_prefix": "/tmp/entities", - "templates_root": "/tmp/templates/packages", - "transformation_config": [ - { - "input_file_pattern": "/tmp/inputs/<>.json", - "template": "atlan/NAME/transformers/<>.jinja2", - "output_file_prefix": "<>/<>", - "output_chunk_size": 15000 - } - ] - } - parameters: - - name: connection - value: "{{inputs.parameters.connection}}" - - name: mode - value: "{{inputs.parameters.mode}}" - - name: source - value: "{{inputs.parameters.source}}" - - name: raw-input-file-sort - value: "" - - name: raw-input-folder-sort - value: "" - - name: raw-input-file-pattern - value: "**/*.json" - - name: publish-chunk-size - value: "{{inputs.parameters.publish-chunk-size}}" - - name: atlas-auth-type - value: "{{inputs.parameters.atlas-auth-type}}" - - name: hierarchy - value: | - [ - [ - { - "name":"", - "file_pattern": "" - } - ] - ] - - name: marketplace-scripts-revision - value: "{{inputs.parameters.marketplace-scripts-revision}}" - - name: marketplace-packages-revision - value: "{{inputs.parameters.marketplace-packages-revision}}" - - name: statsd-global-tags - value: "{{inputs.parameters.statsd-global-tags}}" - - - name: NAME-api - synchronization: - semaphore: - configMapKeyRef: - name: atlan-NAME - key: api - volumes: - - name: credentials - emptyDir: { } - inputs: - artifacts: - - name: raw-input - path: /tmp/input - optional: true - - name: raw-input-file - path: "/tmp/input/input.json" - optional: true - parameters: - - name: credential-guid - - name: page-size - value: "10" - - name: method - value: "GET" - - name: url - - name: request-config - value: "{}" - - name: execution-script - - name: output-chunk-size - value: 100 - - name: kube-secret-name - value: "argo-client-creds" - - name: client-id-env - value: "login" - - name: client-secret-env - value: "password" - - name: token-url-env - value: "host" - - name: raw-input-paginate - value: 0 - - name: raw-input-file-pattern - value: "" - - name: raw-input-multiline - value: "False" - - name: statsd-host - value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" - - name: statsd-port - value: "9125" - - name: statsd-global-tags - value: "workflow={{workflow.name}},bot=atlan-NAME" - - name: output-prefix - value: "argo-artifacts/{{workflow.namespace}}/{{workflow.name}}/{{pod.name}}" - - name: heracles-uri - value: "http://heracles-service.heracles.svc.cluster.local" - - name: init-execution-script - value: | - if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): - LOGGER.debug('Heracles is unavailable. Performing retry with back-off') - failure_handler = FailureHandler.RETRY - - if state == ExecutionState.OUTPUT_PROCESS: - credential = json.loads(output) - output = "" - for key, value in credential.items(): - output += f"""ATLAN_{key.upper()}="{str(value)}"\n""" - - output += f"""ATLAN_TOKEN_URL='https://{credential['host']}/v2/auth/token'\n""" - - if state == ExecutionState.API_POST: - stop = True - outputs: - artifacts: - - name: success - path: "/tmp/rest/success" - s3: - key: "{{inputs.parameters.output-prefix}}" - archive: - none: { } - - name: failure - path: "/tmp/rest/failure" - parameters: - - name: success-num-files - valueFrom: - path: "/tmp/rest/success/result-gen.txt" - - name: failure-num-files - valueFrom: - path: "/tmp/rest/failure/result-gen.txt" - container: - image: ghcr.io/atlanhq/rest-master:165b7e5 - command: [ "./entrypoint.sh" ] - volumeMounts: - - name: credentials - mountPath: /tmp/credentials - env: - - name: OAUTHLIB_INSECURE_TRANSPORT - value: "1" - imagePullPolicy: IfNotPresent - args: [ - "python3", "main.py", "{{inputs.parameters.method}}", "{{inputs.parameters.url}}", - "--request-config", "{{inputs.parameters.request-config}}", - "--raw-input-paginate", "{{inputs.parameters.raw-input-paginate}}", - "--raw-input-file-pattern", "{{inputs.parameters.raw-input-file-pattern}}", - "--raw-input-multiline", "{{inputs.parameters.raw-input-multiline}}", - "--execution-script", "{{inputs.parameters.execution-script}}", - "--secrets-path", "/tmp/credentials/success/*.json", - "--auth-type", "oauth2", - "--auth-oauth2-type", "client_credentials", - "--auth-oauth2-impersonate-user", "", - "--auth-oauth2-client-credentials-client-id", "ATLAN_USERNAME", - "--auth-oauth2-client-credentials-secret", "ATLAN_PASSWORD", - "--auth-oauth2-client-credentials-token-url", "ATLAN_TOKEN_URL", - "--output-chunk-size", "{{inputs.parameters.output-chunk-size}}", - "--output-file-prefix", "/tmp/rest", - "--pagination-wait-time", "10", - "--max-retries", "3", - "--statsd-host", "{{inputs.parameters.statsd-host}}", - "--statsd-port", "{{inputs.parameters.statsd-port}}", - "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" - ] - initContainers: - - name: fetch-credentials - image: ghcr.io/atlanhq/rest-master:165b7e5 - command: [ "python3", "main.py" ] - env: - - name: OAUTHLIB_INSECURE_TRANSPORT - value: "1" - - name: CLIENT_ID - valueFrom: - secretKeyRef: - name: "{{inputs.parameters.kube-secret-name}}" - key: "{{inputs.parameters.client-id-env}}" - - name: CLIENT_SECRET - valueFrom: - secretKeyRef: - name: "{{inputs.parameters.kube-secret-name}}" - key: "{{inputs.parameters.client-secret-env}}" - - name: TOKEN_URL - valueFrom: - secretKeyRef: - name: "{{inputs.parameters.kube-secret-name}}" - key: "{{inputs.parameters.token-url-env}}" - mirrorVolumeMounts: true - args: [ - "GET", - "{{inputs.parameters.heracles-uri}}/credentials/{{inputs.parameters.credential-guid}}/use", - "--raw-input", "{}", - "--raw-input-file-sort", "", - "--raw-input-multiline", "False", - "--execution-script", "{{inputs.parameters.init-execution-script}}", - "--raw-input-paginate", "0", - "--auth-type", "oauth2", - "--auth-oauth2-type", "client_credentials", - "--auth-oauth2-impersonate-user", "{{=sprig.dig('labels', 'workflows', 'argoproj', 'io/creator', '', workflow)}}", - "--auth-oauth2-client-credentials-client-id", "CLIENT_ID", - "--auth-oauth2-client-credentials-secret", "CLIENT_SECRET", - "--auth-oauth2-client-credentials-token-url", "TOKEN_URL", - "--output-chunk-size", "0", - "--output-file-prefix", "/tmp/credentials", - "--pagination-wait-time", "0", - "--max-retries", "10", - "--statsd-host", "{{inputs.parameters.statsd-host}}", - "--statsd-port", "{{inputs.parameters.statsd-port}}", - "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" - ] \ No newline at end of file diff --git a/lib/connectors/packages/BI/transformers/default.jinja2 b/lib/connectors/packages/BI/transformers/default.jinja2 deleted file mode 100644 index 1a0d6dc..0000000 --- a/lib/connectors/packages/BI/transformers/default.jinja2 +++ /dev/null @@ -1,19 +0,0 @@ -{ - "typeName": "ASSET_NAME", - "status": "ACTIVE", - "attributes": { - "connectorName": "{{external_map['integration_name']}}", - "connectionName": "{{external_map['connection_name']}}", - "connectionQualifiedName": "{{external_map['connection_qn']}}", - "lastSyncRunAt": {{now}}, - "lastSyncWorkflowName": "{{external_map['crawler_name']}}", - "lastSyncRun": "{{external_map['workflow_name']}}", - "tenantId": "{{external_map['tenant_id']}}", - - "qualifiedName": "{{external_map['connection_qn']}}/{{record['id']}}", - - "name": {{record.get('name') | tojson}}, - "sourceCreatedAt": "{{convert_date(record['createdAt'], format="%Y-%m-%dT%H:%M:%S.%fZ")}}", - "sourceUpdatedAt": "{{convert_date(record['updatedAt'], format="%Y-%m-%dT%H:%M:%S.%fZ")}}", - } -} \ No newline at end of file diff --git a/lib/connectors/packages/common-configmap/api.yaml b/lib/connectors/packages/common-configmap/api.yaml deleted file mode 100644 index f258942..0000000 --- a/lib/connectors/packages/common-configmap/api.yaml +++ /dev/null @@ -1,166 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: atlan-connectors-NAME - labels: - orchestration.atlan.com/version: "1" - orchestration.atlan.com/source: "NAME" -data: - icon: "https://cdn-images-1.medium.com/max/1200/1*8bPQYTZUHcvvk_1fSPwwHw.png" - helpdeskLink: "" - logo: "https://cdn-images-1.medium.com/max/1200/1*8bPQYTZUHcvvk_1fSPwwHw.png" - connector: "NAME" - defaultConnectorType: "rest" - jdbcCredentialTemplate: "{}" - restCredentialTemplate: | - { - "default": [ - { - "id": 1, - "name": "auth", - "curl": "curl --location --request POST 'https://{{host}}/v2/auth/token' --header 'Content-Type:application/x-www-form-urlencoded' --data-urlencode 'grant_type=client_credentials' --data-urlencode 'client_id={{username}}' --data-urlencode 'client_secret={{password}}' --connect-timeout 4" - } - ] - } - odbcCredentialTemplate: "{}" - grpcCredentialTemplate: "{}" - restMetadataTemplate: | - { - "default": [ - { - "id": 1, - "name": "auth", - "curl": "curl --location --request POST 'https://{{host}}/v2/auth/token' --header 'Content-Type:application/x-www-form-urlencoded' --data-urlencode 'grant_type=client_credentials' --data-urlencode 'client_id={{username}}' --data-urlencode 'client_secret={{password}}' --connect-timeout 4" - } - ] - } - restMetadataOutputTransformerTemplate: | - { - - } - odbcTemplate: "{}" - grpcTemplate: "{}" - sageTemplate: | - { - - } - config: | - { - "properties": { - "name": { - "type": "string", - "required": false, - "ui": { - "label": "Name", - "hidden": true, - "placeholder": "Host Name" - } - }, - "connector": { - "type": "string", - "required": false, - "ui": { - "label": "Connector", - "hidden": true, - "placeholder": "Connector" - } - }, - "connectorType": { - "type": "string", - "required": false, - "ui": { - "key": "_host", - "label": "connectorType", - "placeholder": "connectorType", - "hidden": true - } - }, - "host": { - "type": "string", - "required": true, - "enum": ["aws-api.NAMEcomputing.com", "api.NAMEcomputing.com"], - "enumNames": ["aws-api.NAMEcomputing.com - For AWS Hosted Organizations", "api.NAMEcomputing.com - For GCP Hosted Organizations"], - "default": "aws-api.NAMEcomputing.com", - "ui": { - "widget": "select", - "label": "Endpoint", - "grid": 8, - "hidden": false, - "placeholder": "Endpoint" - } - }, - "auth-type": { - "type": "string", - "enum": ["api_token"], - "default": "api_token", - "required": true, - "enumNames": ["API Token"], - "ui": { - "widget": "radio", - "label": "Authentication", - "placeholder": "Credential Type", - "rules": [ - { - "required": true, - "message": "Please enter a valid authentication type" - } - ] - } - }, - "api_token": { - "type": "object", - "properties": { - "username": { - "type": "string", - "required": true, - "ui": { - "label": "Client ID", - "placeholder": "Client ID", - "feedback": true, - "grid": 4, - "rules": [ - { - "required": true, - "message": "Please enter a valid client Id" - } - ] - } - }, - "password": { - "type": "string", - "required": true, - "ui": { - "widget": "password", - "label": "API Token", - "feedback": true, - "placeholder": "API Token", - "grid": 4, - "rules": [ - { - "required": true, - "message": "Please enter a valid API token" - } - ] - } - } - }, - "ui": { - "widget": "nested", - "label": "Client ID & API Token", - "placeholder": "Credential Type", - "nestedValue": false, - "hidden": true - } - } - }, - "anyOf": [ - { - "properties": { - "auth-type": { - "const": "api_token" - } - }, - "required": ["api_token"] - } - ] - } \ No newline at end of file diff --git a/lib/connectors/packages/common-configmap/aws.yaml b/lib/connectors/packages/common-configmap/aws.yaml deleted file mode 100644 index 07f6794..0000000 --- a/lib/connectors/packages/common-configmap/aws.yaml +++ /dev/null @@ -1,244 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: atlan-connectors-NAME - labels: - orchestration.atlan.com/version: "1" - orchestration.atlan.com/source: "NAME" -data: - icon: "https://atlan-public.s3.eu-west-1.amazonaws.com/atlan/logos/aws-NAME.png" - helpdeskLink: "https://ask.atlan.com/hc/en-us/articles/4605882724369-How-to-set-up-your-NAME-connection-on-Atlan" - logo: "https://atlan-public.s3.eu-west-1.amazonaws.com/atlan/logos/aws-NAME.png" - connector: "NAME" - defaultConnectorType: "sdk" - jdbcCredentialTemplate: "{}" - restCredentialTemplate: "{}" - odbcCredentialTemplate: "{}" - grpcCredentialTemplate: "{}" - restMetadataTemplate: "" - restMetadataOutputTransformerTemplate: "" - sdkCredentialTemplate: | - [ - { - "id": 1, - "name": "<>", - "source": "AWS", - "action": "AWSNAME.<>", - "params": { {% if authType == "iam" %} "aws_access_key_id": "{{username}}", "aws_secret_access_key": "{{password}}", {% endif %} "region": "{{extra.region}}" {% if authType == "role" and extra.aws_role_arn|length %}, "aws_role_arn": "{{extra.aws_role_arn}}" {% endif %} {% if authType == "role" and extra.aws_external_id|length %}, "aws_external_id": "{{extra.aws_external_id}}" {% endif %} } - } - ] - config: | - { - "properties": { - "name": { - "type": "string", - "required": false, - "ui": { - "label": "Name", - "hidden": true, - "placeholder": "Credential Name" - } - }, - "connector": { - "type": "string", - "required": false, - "ui": { - "label": "Connector", - "hidden": true, - "placeholder": "Connector" - } - }, - "connectorType": { - "type": "string", - "required": false, - "ui": { - "key": "_host", - "label": "connectorType", - "placeholder": "connectorType", - "hidden": true - } - }, - "auth-type": { - "type": "string", - "enum": [ - "iam", - "role" - ], - "default": "iam", - "required": true, - "enumNames": [ - "IAM User", - "IAM Role" - ], - "ui": { - "widget": "radio", - "hidden": false, - "label": "Authentication", - "placeholder": "Credential Type", - "rules": [ - { - "required": true, - "message": "Please enter a valid authentication type" - } - ] - } - }, - "port": { - "type": "number", - "default": 443, - "required": false, - "ui": { - "label": "Port", - "hidden": true, - "placeholder": "Port" - } - }, - "iam": { - "type": "object", - "properties": { - "username": { - "type": "string", - "required": true, - "ui": { - "label": "AWS Access Key", - "placeholder": "Access Key", - "feedback": true, - "help": "AWS Access Key", - "message": "Please enter a valid Access Key", - "grid": 4, - "rules": [ - { - "required": true, - "message": "Please enter a valid Access Key" - } - ] - } - }, - "password": { - "type": "string", - "required": true, - "ui": { - "widget": "password", - "label": "AWS Secret Key", - "feedback": true, - "help": "AWS Secret Key", - "placeholder": "Secret Key", - "grid": 4, - "rules": [ - { - "required": true, - "message": "Please enter a valid Secret Key" - } - ] - } - } - }, - "ui": { - "widget": "nested", - "label": "", - "placeholder": "Credential Type", - "nestedValue": false, - "hidden": false - } - }, - "role":{ - "type": "object", - "properties": { - "extra": { - "type": "object", - "properties": { - "aws_role_arn": { - "type": "string", - "required": false, - "ui": { - "label": "AWS Role ARN", - "hidden": false, - "placeholder": "arn:aws:iam::123456789012:role/roleName", - "help": "AWS role to assume for the connection", - "grid": 4, - "rules": [ - { - "pattern": "^arn:aws:iam::\\d{12}:role/.+", - "message": "Please enter a valid role ARN", - "trigger": "blur" - } - ] - } - }, - "aws_external_id": { - "type": "string", - "required": false, - "ui": { - "label": "External ID", - "widget": "keygen", - "help": "Unique external ID to enable access of your AWS resources to Atlan" - } - } - }, - "ui": { - "widget": "nested", - "label": "", - "header": "Advanced", - "hidden": false - } - } - }, - "ui": { - "widget": "nested", - "label": "", - "placeholder": "Credential Type", - "nestedValue": false, - "hidden": true - } - }, - "extra": { - "type": "object", - "properties": { - "region": { - "type": "string", - "required": true, - "default": "", - "ui": { - "help": "Enter region of NAME setup", - "label": "Region", - "hidden": false, - "placeholder": "us-west-1", - "grid": 4, - "rules": [ - { - "required": true, - "message": "Please enter a region" - } - ] - } - } - }, - "ui": { - "widget": "nested", - "label": "", - "header": "Advanced", - "hidden": false - } - } - }, - "anyOf": [ - { - "properties": { - "auth-type": { - "const": "iam" - } - }, - "required": [ - "iam" - ] - }, - { - "properties": { - "auth-type": { - "const": "role" - } - }, - "required": ["role"] - } - ] - } diff --git a/lib/connectors/packages/common-configmap/basic.yaml b/lib/connectors/packages/common-configmap/basic.yaml deleted file mode 100644 index 5a60ae4..0000000 --- a/lib/connectors/packages/common-configmap/basic.yaml +++ /dev/null @@ -1,151 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: atlan-connectors-NAME - labels: - orchestration.atlan.com/version: "1" - orchestration.atlan.com/source: "NAME" -data: - icon: "<>" - helpdeskLink: "" - logo: "<>" - connector: "NAME" - defaultConnectorType: "rest" - jdbcCredentialTemplate: "{}" - restCredentialTemplate: "{}" - { - "default": [ - { - "id": 1, - "name": "auth", - "curl": "curl --location --request POST 'https://{{host}}/v2/auth/token' --header 'Content-Type:application/x-www-form-urlencoded' --data-urlencode 'grant_type=client_credentials' --data-urlencode 'client_id={{username}}' --data-urlencode 'client_secret={{password}}' --connect-timeout 4" - } - ] - } - odbcCredentialTemplate: "{}" - grpcCredentialTemplate: "{}" - restMetadataTemplate: "{}" - restMetadataOutputTransformerTemplate: "{}" - odbcTemplate: "{}" - grpcTemplate: "{}" - sageTemplate: "{}" - config: | - { - "properties": { - "name": { - "type": "string", - "required": false, - "ui": { - "label": "Name", - "hidden": true, - "placeholder": "Host Name" - } - }, - "connector": { - "type": "string", - "required": false, - "ui": { - "label": "Connector", - "hidden": true, - "placeholder": "Connector" - } - }, - "connectorType": { - "type": "string", - "required": false, - "ui": { - "key": "_host", - "label": "connectorType", - "placeholder": "connectorType", - "hidden": true - } - }, - "host": { - "type": "string", - "required": true, - "enum": ["aws-api.NAMEcomputing.com", "api.NAMEcomputing.com"], - "enumNames": ["aws-api.NAMEcomputing.com - For AWS Hosted Organizations", "api.NAMEcomputing.com - For GCP Hosted Organizations"], - "default": "aws-api.NAMEcomputing.com", - "ui": { - "widget": "select", - "label": "Endpoint", - "grid": 8, - "hidden": false, - "placeholder": "Endpoint" - } - }, - "auth-type": { - "type": "string", - "enum": ["api_token"], - "default": "api_token", - "required": true, - "enumNames": ["API Token"], - "ui": { - "widget": "radio", - "label": "Authentication", - "placeholder": "Credential Type", - "rules": [ - { - "required": true, - "message": "Please enter a valid authentication type" - } - ] - } - }, - "api_token": { - "type": "object", - "properties": { - "username": { - "type": "string", - "required": true, - "ui": { - "label": "Client ID", - "placeholder": "Client ID", - "feedback": true, - "grid": 4, - "rules": [ - { - "required": true, - "message": "Please enter a valid client Id" - } - ] - } - }, - "password": { - "type": "string", - "required": true, - "ui": { - "widget": "password", - "label": "API Token", - "feedback": true, - "placeholder": "API Token", - "grid": 4, - "rules": [ - { - "required": true, - "message": "Please enter a valid API token" - } - ] - } - } - }, - "ui": { - "widget": "nested", - "label": "Client ID & API Token", - "placeholder": "Credential Type", - "nestedValue": false, - "hidden": true - } - } - }, - "anyOf": [ - { - "properties": { - "auth-type": { - "const": "api_token" - } - }, - "required": ["api_token"] - } - ] - } \ No newline at end of file diff --git a/lib/connectors/packages/common-configmap/jdbc-basic.yaml b/lib/connectors/packages/common-configmap/jdbc-basic.yaml deleted file mode 100644 index 4bea4ce..0000000 --- a/lib/connectors/packages/common-configmap/jdbc-basic.yaml +++ /dev/null @@ -1,180 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: atlan-connectors-NAME - labels: - orchestration.atlan.com/version: "1" - orchestration.atlan.com/source: "NAME" -data: - icon: "https://atlan-public.s3.eu-west-1.amazonaws.com/atlan/logos/NAME.png" - helpdeskLink: "https://docs.atlan.com/integrations/relational-database/NAME" - logo: "https://atlan-public.s3.eu-west-1.amazonaws.com/atlan/logos/NAME.png" - connector: "NAME" - defaultConnectorType: "jdbc" - jdbcCredentialTemplate: | - { - "className": "com.facebook.NAME.jdbc.NAMEDriver", - "jarLink": "https://atlan-public.s3-eu-west-1.amazonaws.com/atlan/jdbc/NAME.tar.gz", - "url": "jdbc:NAME://{{ host }}:{{ port|int }}", - "driverProperties": { "user": "{{ username }}", "password": "{{ password }}" } - } - restCredentialTemplate: "{}" - odbcCredentialTemplate: "{}" - grpcCredentialTemplate: "{}" - restMetadataTemplate: "" - restMetadataOutputTransformerTemplate: "" - sageTemplate: | - { - "catalogsCheck": { - "curls": [ - { - "name": "schemas", - "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' --data-raw '{\"query\": \"show atlan schemas\"}'", - "addCredential": true, - "credentialConnectorType": "jdbc" - } - ], - "responseTemplate": "{{- $includeFilter := dict}} {{- if eq `string` (printf `%T` (index .formData `include-filter`)) }} {{- $includeFilter = index .formData `include-filter` | fromJson}} {{- else }} {{- $includeFilter = index .formData `include-filter` }} {{- end }} {{- $allowedDatabases := list}} {{- $allowedSchemas := list}} {{- $missingObjectName := ``}} {{- $checkSuccess := true }} {{- range $schemaList := .schemas.results }} {{- $allowedDatabases = append $allowedDatabases $schemaList.TABLE_CATALOG }} {{- $allowedSchemas = append $allowedSchemas (print $schemaList.TABLE_CATALOG `.` $schemaList.TABLE_SCHEM )}} {{- end }} {{- range $filteredDb, $filteredSchemas := $includeFilter }} {{- $_db := lower $filteredDb | trimPrefix `^` | trimSuffix `$` }} {{- $checkSuccess = and $checkSuccess (has $_db $allowedDatabases) }} {{- if not (has $_db $allowedDatabases)}} {{- $missingObjectName = (print $_db ` ` `database`)}} {{- end }} {{- range $schmea := $filteredSchemas }} {{- $_schema := lower $schmea | trimPrefix `^` | trimSuffix `$` }} {{- $checkSuccess = and $checkSuccess (has (print $_db `.` (lower $_schema)) $allowedSchemas)}} {{- if not (has (print $_db `.` (lower $_schema)) $allowedSchemas)}} {{- $missingObjectName = (print $_db `.` (lower $_schema) ` ` `schema`)}} {{- end }} {{- end }} {{- end }} {{- $response := dict `successMessage` `` `failureMessage` `` `data` .schemas.results `response` dict }} {{- if $checkSuccess }} {{- $_ := set $response `successMessage` `Check successful` }} {{- else }} {{- $_ := set $response `failureMessage` (print `Check failed for ` $missingObjectName) }} {{- end }} {{- $response | toJson }}" - } - } - config: | - { - "properties": { - "name": { - "type": "string", - "required": false, - "ui": { - "label": "Name", - "hidden": true, - "placeholder": "Host Name" - } - }, - "connector": { - "type": "string", - "required": false, - "ui": { - "label": "Connector", - "hidden": true, - "placeholder": "Connector" - } - }, - "connectorType": { - "type": "string", - "required": false, - "ui": { - "key": "_host", - "label": "connectorType", - "placeholder": "connectorType", - "hidden": true - } - }, - "host": { - "type": "string", - "required": true, - "default": "", - "ui": { - "label": "Host", - "feedback": true, - "help": "Your NAME instance host name", - "BYOCdisabled": true, - "rules": [ - { - "required": true, - "message": "Please enter a valid host name" - } - ], - "grid": 6 - } - }, - "port": { - "type": "number", - "default": 8080, - "required": true, - "ui": { - "label": "Port", - "placeholder": "Port", - "disabled": false, - "help": "Your NAME instance port number", - "grid": 2, - "BYOCdisabled": true, - "rules": [ - { - "required": true, - "message": "Please enter a valid port number" - } - ] - } - }, - "auth-type": { - "type": "string", - "enum": ["basic"], - "default": "basic", - "required": true, - "enumNames": ["Basic"], - "ui": { - "widget": "radio", - "hidden": false, - "label": "Authentication", - "placeholder": "Credential Type", - "rules": [ - { - "required": true, - "message": "Please enter a valid authentication type" - } - ] - } - }, - "basic": { - "type": "object", - "properties": { - "username": { - "type": "string", - "required": true, - "ui": { - "label": "Username", - "placeholder": "Username", - "help": "Database Username", - "feedback": true, - "message": "Please enter a valid username", - "grid": 4, - "rules": [ - { - "required": true, - "message": "Please enter a valid username" - } - ] - } - }, - "password": { - "type": "string", - "required": false, - "ui": { - "widget": "password", - "label": "Password", - "help": "Database Password", - "feedback": false, - "placeholder": "Password", - "grid": 4 - } - } - }, - "ui": { - "widget": "nested", - "label": "Basic Authentication", - "placeholder": "Credential Type", - "nestedValue": false, - "hidden": true - } - } - }, - "anyOf": [ - { - "properties": { - "auth-type": { - "const": "basic" - } - }, - "required": ["basic"] - } - ] - } diff --git a/lib/init.js b/lib/init.js index c2e4271..c469dfc 100644 --- a/lib/init.js +++ b/lib/init.js @@ -2,8 +2,8 @@ const Promise = require("bluebird"); const fs = Promise.promisifyAll(require("fs")); const system = require("system-commands"); const utils = require("./utils"); -const LinearClient = require("@linear/sdk").LinearClient; -const linearClient = new LinearClient(); +//const LinearClient = require("@linear/sdk").LinearClient; +//const linearClient = new LinearClient(); /** * Init an Argo package inside the folder @@ -59,7 +59,7 @@ function replaceInFile(filePath, searchText, replaceText) { * @param {boolean} force * @param {string} filepath */ -exports.cinit = async function (force, inputPackageName, inputPackageType, auth, assetList, model, script, linear) { +exports.cinit = async function (force, inputPackageName, inputPackageType, auth, assetList, script, linear) { const dirPath = "/Users/mrunmayi.tripathi/atlan/code/marketplace-packages/packages/atlan/test-package"; //const dirPath = process.cwd(); const pathComponents = dirPath.split("/"); @@ -78,18 +78,7 @@ exports.cinit = async function (force, inputPackageName, inputPackageType, auth, //Create linear if (linear) { - await createLinearWithSubtask(); - } - - //Create models repo folder - if (model) { - await createModelsRepoDirectory(force, dirPath, inputPackageName, inputPackageType, assetList); - } - - //Create scripts repo folder - if (script) { - await createScriptsRepoDirectory(force, dirPath, inputPackageName, inputPackageType, assetList); - console.info("add entry to model"); + //await createLinearWithSubtask(inputPackageName); } } else { return fs @@ -188,8 +177,8 @@ function BuildTemplateFiles(packageName, filePath, auth) { return; } } - -async function createLinearWithSubtask() { +/* +async function createLinearWithSubtask(inputPackageName) { const teams = await linearClient.teams(); let orcTeam = teams.nodes[0]; teams.nodes.forEach(function (team) { @@ -197,10 +186,11 @@ async function createLinearWithSubtask() { orcTeam = team; } }); - const issue = await linearClient.createIssue({ teamId: orcTeam.id, title: "Redash Package" }); + const issue = await linearClient.createIssue({ teamId: orcTeam.id, title: `${inputPackageName} Package` }); console.info("add entry to linear %s", issue); return; } +*/ async function createPackageEntry(dirPath, packageName) { const filePath = `${dirPath}/../packages/package.json`; @@ -208,27 +198,5 @@ async function createPackageEntry(dirPath, packageName) { const rawPackageData = await fs.readFileAsync(filePath, "utf-8"); let packageData = JSON.parse(rawPackageData); packageData.dependencies[`@atlan/${packageName}`] = "^1.0.0"; - fs.writeFileAsync(filePath, packageData, "utf8"); -} - -async function createModelsRepoDirectory(force, dirPath, packageName, packageType, assetList) { - console.info("Creating folders in models repo"); - const modelsFolderPath = `${dirPath}/../../../models/atlas/entityDefs/Referenceable/Asset/Catalog/BI/${packageName}`; - const files = await fs.readdirAsync(modelsFolderPath); - if (!force && (files.length !== 0 || files.includes("package.json"))) { - throw `Files already present in the ${modelsFolderPath}. Run this command again with --force to ignore`; - } - var skeletonPackagePath = `${__dirname}/connectors/models/${packageType}`; - return system(`cp -r ${skeletonPackagePath}/ ${modelsFolderPath}`); -} - -async function createScriptsRepoDirectory(force, dirPath, packageName, packageType, assetList) { - console.info("Creating folders in scripts repo"); - const scriptsFolderPath = `${dirPath}/../../../marketplace-scripts/marketplace_scripts/${packageName}`; - const files = await fs.readdirAsync(scriptsFolderPath); - if (!force && (files.length !== 0 || files.includes("package.json"))) { - throw `Files already present in the ${scriptsFolderPath}. Run this command again with --force to ignore`; - } - var skeletonPackagePath = `${__dirname}/connectors/scripts/${packageType}`; - return system(`cp -r ${skeletonPackagePath}/ ${scriptsFolderPath}`); + fs.writeFileAsync(filePath, JSON.stringify(packageData), "utf8"); } From ebd53540c3b66e472a05e5d97bd7d5cf8550aaf1 Mon Sep 17 00:00:00 2001 From: Mrunmayi Date: Tue, 11 Apr 2023 14:42:53 +0530 Subject: [PATCH 5/6] Updating folders setup to build correct default folder structure --- lib/connectors/BI/README.md | 0 lib/connectors/BI/configmaps/default.yaml | 153 +++++ lib/connectors/BI/index.js | 4 + lib/connectors/BI/package.json | 53 ++ lib/connectors/BI/templates/aws.yaml | 649 ++++++++++++++++++ lib/connectors/BI/templates/basic+jwt.yaml | 633 +++++++++++++++++ lib/connectors/BI/templates/oauth.yaml | 581 ++++++++++++++++ lib/connectors/BI/transformers/default.jinja2 | 19 + lib/connectors/common-configmap/api.yaml | 166 +++++ lib/connectors/common-configmap/aws.yaml | 244 +++++++ lib/connectors/common-configmap/basic.yaml | 151 ++++ .../common-configmap/jdbc-basic.yaml | 180 +++++ 12 files changed, 2833 insertions(+) create mode 100644 lib/connectors/BI/README.md create mode 100644 lib/connectors/BI/configmaps/default.yaml create mode 100644 lib/connectors/BI/index.js create mode 100644 lib/connectors/BI/package.json create mode 100644 lib/connectors/BI/templates/aws.yaml create mode 100644 lib/connectors/BI/templates/basic+jwt.yaml create mode 100644 lib/connectors/BI/templates/oauth.yaml create mode 100644 lib/connectors/BI/transformers/default.jinja2 create mode 100644 lib/connectors/common-configmap/api.yaml create mode 100644 lib/connectors/common-configmap/aws.yaml create mode 100644 lib/connectors/common-configmap/basic.yaml create mode 100644 lib/connectors/common-configmap/jdbc-basic.yaml diff --git a/lib/connectors/BI/README.md b/lib/connectors/BI/README.md new file mode 100644 index 0000000..e69de29 diff --git a/lib/connectors/BI/configmaps/default.yaml b/lib/connectors/BI/configmaps/default.yaml new file mode 100644 index 0000000..2637460 --- /dev/null +++ b/lib/connectors/BI/configmaps/default.yaml @@ -0,0 +1,153 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlan-NAME +data: + NAME-api: "10" + config: | + { + "properties": { + "connection": { + "type": "string", + "required": false, + "ui": { + "widget": "connection", + "label": "", + "placeholder": "Connection Name", + "connectionOptions": false + } + }, + "publish-mode": { + "type": "string", + "enum": ["production", "test", "dev"], + "default": "production", + "enumNames": ["Production", "Test", "Development"], + "ui": { + "widget": "select", + "label": "Run Mode", + "grid": 4, + "placeholder": "Connection Mode" + } + }, + "credentials-fetch-strategy": { + "type": "string", + "enum": ["k8s_secret", "credential_guid"], + "default": "credential_guid", + "enumNames": ["k8s Secret Key", "Credential Guid"], + "ui": { + "widget": "select", + "label": "Credential Type", + "placeholder": "Credential Type" + } + }, + "credential-guid": { + "type": "string", + "ui": { + "widget": "credential", + "label": "", + "credentialType": "atlan-connectors-NAME", + "placeholder": "Credential Guid", + "hidden": false + } + }, + "credential-kube-secret-name": { + "type": "string", + "ui": { + "label": "Credential Secret Name", + "placeholder": "Credential Secret Name", + "hidden": true + } + }, + "atlas-auth-type": { + "type": "string", + "enum": ["internal", "apikey"], + "default": "internal", + "enumNames": ["Internal", "API Key"], + "ui": { + "widget": "select", + "label": "Atlas Authentication Type", + "placeholder": "Atlas Authentication Type", + "hidden": true + } + }, + "runtime-properties": { + "type": "object", + "ui": { + "label": "Run time properties", + "hidden": true + } + }, + "include-filter": { + "type": "object", + "additionalProperties": { + "type": "array" + }, + "default": "{}", + "ui": { + "widget": "apitree", + "connectorConfigName": "atlan-connectors-NAME", + "credential": "credential-guid", + "label": "Include Metadata", + "description": "Selected assets only will be processed. Exclude gets preference over include.", + "grid": 4 + } + }, + "exclude-filter": { + "type": "object", + "additionalProperties": { + "type": "array" + }, + "default": "{}", + "ui": { + "widget": "apitree", + "connectorConfigName": "atlan-connectors-NAME", + "credential": "credential-guid", + "label": "Exclude Metadata", + "description": "Selected assets will not be processed.", + "grid": 4 + } + } + }, + "anyOf": [ + { + "properties": { + "credentials-fetch-strategy": { + "const": "k8s_secret" + } + }, + "required": ["credential-kube-secret-name"] + }, + { + "properties": { + "credentials-fetch-strategy": { + "const": "credential_guid" + } + }, + "required": ["credential-guid"] + } + ], + "steps": [ + { + "id": "credential", + "title": "Credential", + "description": "Credential Details", + "properties": [ + "credential-guid" + ] + }, + { + "id": "connection", + "title": "Connection", + "description": "Connection Details", + "properties": [ + "connection" + ] + }, + { + "id": "metadata", + "title": "Metadata", + "description": "Metadata", + "properties": ["include-filter", "exclude-filter"] + } + ] + } \ No newline at end of file diff --git a/lib/connectors/BI/index.js b/lib/connectors/BI/index.js new file mode 100644 index 0000000..fbecdc9 --- /dev/null +++ b/lib/connectors/BI/index.js @@ -0,0 +1,4 @@ +function dummy() { + console.log("don't call this dummy."); +} +module.exports = dummy; diff --git a/lib/connectors/BI/package.json b/lib/connectors/BI/package.json new file mode 100644 index 0000000..bbc52ab --- /dev/null +++ b/lib/connectors/BI/package.json @@ -0,0 +1,53 @@ +{ + "name": "@atlan/NAME", + "version": "0.3.6", + "description": "Package to crawl NAME assets and publish to Atlan for discovery", + "keywords": [ + "NAME", + "bi", + "connector", + "crawler" + ], + "main": "index.js", + "scripts": {}, + "author": { + "name": "Atlan", + "email": "hello@atlan.com", + "url": "https://atlan.com" + }, + "repository": { + "type": "git", + "url": "https://github.com/atlanhq/marketplace-packages.git" + }, + "license": "MIT", + "dependencies": { + "@atlan/crawler": "^2.1.5", + "@atlan/sql-parser": "^0.3.16", + "rest-api": "^0.8.5", + "utils": "^0.2.4" + }, + "bugs": { + "url": "https://atlan.com", + "email": "support@atlan.com" + }, + "config": { + "labels": { + "orchestration.atlan.com/verified": "true", + "orchestration.atlan.com/type": "connector", + "orchestration.atlan.com/source": "NAME", + "orchestration.atlan.com/sourceCategory": "bi", + "orchestration.atlan.com/certified": "true" + }, + "annotations": { + "orchestration.atlan.com/name": "NAME Assets", + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/categories": "NAME,crawler", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/NAME.svg", + "orchestration.atlan.com/marketplaceLink": "https://packages.atlan.com/-/web/detail/@atlan/NAME", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/NAME.svg", + "orchestration.atlan.com/docsUrl": "" + } + } +} diff --git a/lib/connectors/BI/templates/aws.yaml b/lib/connectors/BI/templates/aws.yaml new file mode 100644 index 0000000..22034e3 --- /dev/null +++ b/lib/connectors/BI/templates/aws.yaml @@ -0,0 +1,649 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: atlan-NAME +spec: + entrypoint: main + templates: + - name: main + inputs: + parameters: + #credential + - name: credentials-fetch-strategy + value: "credential_guid" + enum: + - "k8s_secret" + - "credential_guid" + - name: credential-kube-secret-name + value: "" + - name: credential-guid + value: "" + #connection atlan object + - name: connection + value: "" + # extraction + - name: runtime-properties + value: "{}" + - name: fetch-folderless-assets + value: "true" + - name: include-filter + value: "{}" + - name: exclude-filter + value: "{}" + #publish mode + - name: publish-mode + value: "production" + enum: + - "production" + - "test" + - "dev" + + #publish credential + - name: atlas-auth-type + value: "internal" + enum: + - "internal" + - "apikey" + + # enable classification match + - name: auto-classification + value: "false" + enum: + - "true" + - "false" + + - name: attachment-confidence-threshold + value: "0.8" + # Scripts + - name: marketplace-scripts-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplaceScriptsBranch" + optional: true + default: "master" + # Revisions + - name: marketplace-packages-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplacePackagesBranch" + optional: true + default: "master" + dag: + tasks: + - name: extract + template: extract-NAME-metadata + arguments: + parameters: + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: connection-qualified-name + value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + - name: output-prefix + value: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/extracted-metadata/{{workflow.name}}" + - name: threads + value: "2" + - name: fetch-folderless-assets + value: "{{inputs.parameters.fetch-folderless-assets}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: process + template: process-NAME-metadata + depends: "extract.Succeeded" + arguments: + parameters: + - name: connection-qualified-name + value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + - name: fetch-folderless-assets + value: "{{inputs.parameters.fetch-folderless-assets}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: publish + depends: "process.Succeeded" + template: publish-NAME-metadata + arguments: + parameters: + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.publish-mode}}" + - name: source + value: "NAME" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: fetch-folderless-assets + value: "{{inputs.parameters.fetch-folderless-assets}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: exclude-workbook-regex + value: "{{inputs.parameters.exclude-workbook-regex}}" + - name: atlan-web-kube-secret + value: "argo-client-creds" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: atlas-api-uri + value: "http://atlas-service-atlas.atlas.svc.cluster.local/api/atlas/v2" + - name: publish-chunk-size + value: "100" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + + - name: extract-NAME-metadata + inputs: + parameters: + - name: credential-guid + - name: connection-qualified-name + - name: output-prefix + value: "argo-artifacts/{{workflow.namespace}}/{{workflow.name}}" + - name: fetch-folderless-assets + - name: include-filter + - name: exclude-filter + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + - name: statsd-global-tags + dag: + tasks: + - name: extract-assets + template: NAME-api + withItems: + - { "url": "https://NAME..amazonaws.com/accounts//analyses", "name": "analysis" } + arguments: + parameters: + - name: url + value: "{{item.url}}" + - name: output-prefix + value: "{{inputs.parameters.output-prefix}}/{{item.name}}" + - name: NAME-request-config + value: | + { + "headers": { + "Content-Type":"application/x-amz-json-1.1" + }, + "json": { + "MaxResults": 100 + } + } + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: output-chunk-size + value: "1000" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + - name: NAME-execution-script + value: | + if state == ExecutionState.RAW_INPUT_PROCESS: + creds_arr = secrets["result-0.json"].split("""\n""") + region = '' + for cred in creds_arr: + LOGGER.debug("cred %s" % cred) + if 'AWS_REGION' in cred: + region = cred.split('=')[-1].replace('"', '') + region = region.strip() + + request_config['url'] = request_config['url'].replace('', region) + + accountid = '' + for cred in creds_arr: + if 'AWS_ACCOUNT_ID' in cred: + accountid = cred.split('=')[-1].replace('"', '') + + request_config['url'] = request_config['url'].replace('', accountid) + + + if state == ExecutionState.OUTPUT_PROCESS: + output = json.loads(output) + + if state == ExecutionState.API_POST: + next_token = response.json().get('NextToken', None) + if not next_token: + stop = True + else: + request_config['json']['NextToken'] = next_token + + if state == ExecutionState.API_FAIL: + failure_handler=FailureHandler.RETRY + + - name: process-NAME-metadata + inputs: + parameters: + - name: connection-qualified-name + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + - name: fetch-folderless-assets + - name: include-filter + - name: exclude-filter + dag: + tasks: + # Process NAME metadata and generate files for transformer + - name: process-metadata + template: process-metadata-template + arguments: + parameters: + - name: output-prefix + value: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/processed-metadata/{{workflow.name}}" + - name: git-kube-secret-name + value: "{{inputs.parameters.git-kube-secret-name}}" + - name: git-kube-ssh-key + value: "{{inputs.parameters.git-kube-ssh-key}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + - name: fetch-folderless-assets + value: "{{inputs.parameters.fetch-folderless-assets}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: connection-qualified-name + value: "{{inputs.parameters.connection-qualified-name}}" + artifacts: + - name: extracted-metadata + s3: + key: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/extracted-metadata/{{workflow.name}}" + + - name: process-metadata-template + inputs: + parameters: + - name: output-prefix + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: fetch-folderless-assets + - name: include-filter + - name: exclude-filter + - name: connection-qualified-name + + artifacts: + - name: extracted-metadata + path: /tmp/extracted-metadata + - name: scripts + path: /tmp/marketplace-scripts + git: + repo: git@github.com:atlanhq/marketplace-scripts + revision: "{{inputs.parameters.marketplace-scripts-revision}}" + insecureIgnoreHostKey: true + depth: 1 + sshPrivateKeySecret: + name: "{{inputs.parameters.git-kube-secret-name}}" + key: "{{inputs.parameters.git-kube-ssh-key}}" + - name: connection-cache + optional: true + path: "/tmp/connection-cache" + s3: + key: "connection-cache" + outputs: + artifacts: + - name: processed-metadata + path: /tmp/processed-metadata + s3: + key: "{{inputs.parameters.output-prefix}}" + archive: + none: { } + container: + image: ghcr.io/atlanhq/marketplace-scripts-base:0.1.10 + imagePullPolicy: IfNotPresent + workingDir: "/tmp/marketplace-scripts" + command: [ "python" ] + args: + - "-m" + - "marketplace_scripts.NAME.metadata" + - "--metadata-prefix" + - "/tmp/extracted-metadata" + - "--cache-prefix" + - "/tmp/connection-cache" + - "--output-prefix" + - "/tmp/processed-metadata" + - "--exclude-filter" + - "{{inputs.parameters.exclude-filter}}" + - "--include-filter" + - "{{inputs.parameters.include-filter}}" + - "--fetch-folderless-assets" + - "{{inputs.parameters.fetch-folderless-assets}}" + - "--connection-qualified-name" + - "{{inputs.parameters.connection-qualified-name}}" + + - name: publish-NAME-metadata + inputs: + parameters: + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + - name: connection + - name: mode + - name: source + - name: atlas-api-uri + - name: heracles-uri + - name: atlan-web-kube-secret + - name: atlas-auth-type + - name: publish-chunk-size + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + dag: + tasks: + # Run atlan-crawler/generic-publish template + - name: publish_metadata + templateRef: + name: atlan-crawler + template: generic-publish + arguments: + artifacts: + - name: data + s3: + key: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/processed-metadata/{{workflow.name}}" + - name: transformer-config + raw: + data: | + { + "external_map": { + "crawler_name": "{{workflow.labels.workflows.argoproj.io/workflow-template}}", + "tenant_id": "{{workflow.namespace}}", + "integration_name": "{{inputs.parameters.source}}", + "workflow_name": "{{workflow.name}}", + "connection_name": "{{=jsonpath(inputs.parameters.connection, '$.attributes.name')}}", + "connection_qn": "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + }, + "output_prefix": "/tmp/entities", + "templates_root": "/tmp/templates", + "transformation_config": [ + { + "input_file_pattern": "/tmp/inputs/ASSET_NAME.json", + "template": "packages/atlan/NAME/transformers/ASSET_NAME.jinja2", + "output_file_prefix": "ASSET_NAME/ASSET_NAME", + "output_chunk_size": 10000 + } + ] + } + parameters: + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.mode}}" + - name: source + value: "{{inputs.parameters.source}}" + - name: raw-input-file-sort + value: "" + - name: raw-input-folder-sort + value: "" + - name: raw-input-file-pattern + value: "**/*.json" + - name: publish-chunk-size + value: "{{inputs.parameters.publish-chunk-size}}" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + + - name: NAME-api + synchronization: + semaphore: + configMapKeyRef: + name: atlan-NAME + key: NAME-api + inputs: + artifacts: + - name: raw-input + path: "/tmp/input/" + optional: true + - name: raw-input-file + path: "/tmp/input/input.json" + optional: true + parameters: + - name: threads + value: "1" + - name: credential-guid + - name: NAME-request-config + value: | + { + "headers": { + "Content-Type":"application/x-amz-json-1.1" + }, + "json": { + "MaxResults": 100 + } + } + - name: paginate + value: "true" + enum: + - "true" + - "false" + - name: NAME-output-key + value: "TableList" + - name: url + - name: NAME-execution-script + value: | + if "{{inputs.parameters.threads}}" != "1": + logger.error("Error! You must pass custom execution script from parent for thread count > 1") + raise Exception("Invalid thread count for offset based pagination!") + if state == ExecutionState.RAW_INPUT_PROCESS: + creds_arr = secrets["result-0.json"].split("""\n""") + region = '' + for cred in creds_arr: + if 'AWS_REGION' in cred: + region = cred.split('=')[-1].replace('"', '') + + request_config['url'] = request_config['url'].replace('', region) + + if state == ExecutionState.OUTPUT_PROCESS: + output=json.loads(output)['{{inputs.parameters.NAME-output-key}}'] + + if state == ExecutionState.API_POST: + next_token = response.json().get('NextToken', None) + if not next_token: + stop = True + else: + request_config['json']['NextToken'] = next_token + + if state == ExecutionState.API_FAIL: + failure_handler=FailureHandler.RETRY + - name: output-chunk-size + value: 0 + - name: raw-input-file-pattern + value: "" + - name: raw-input-paginate + value: "0" + - name: raw-input-multiline + value: "False" + - name: pagination-wait-time + value: "0" + - name: kube-secret-name + value: "argo-client-creds" + - name: client-id-env + value: "login" + - name: client-secret-env + value: "password" + - name: token-url-env + value: "host" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},bot=atlan-NAME" + - name: output-prefix + value: "argo-artifacts/{{workflow.namespace}}/{{workflow.name}}/{{pod.name}}/" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: init-execution-script + value: | + if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): + LOGGER.debug('Heracles is unavailable. Performing retry with back-off') + failure_handler = FailureHandler.RETRY + if state == ExecutionState.OUTPUT_PROCESS: + + credential = json.loads(output) + output = "" + for key, value in credential.items(): + if key == "username": + output += f"""AWS_ACCESS_KEY_ID=\"{str(value)}\"\n""" + elif key == "password": + output += f"""AWS_SECRET_ACCESS_KEY=\"{str(value)}\"\n""" + elif key == "host": + if value is not None: + region = value.split('.')[-3] + output += f"""AWS_REGION=\"{str(region)}\"\n""" + elif key == "extra": + if "region" in value: + region = value.get("region", "") + output += f"""AWS_REGION=\"{str(region)}\"\n""" + if "accountid" in value: + accountid = value.get("accountid", "") + output += f"""AWS_ACCOUNT_ID=\"{str(accountid)}\"\n""" + if "aws_role_arn" in value: + output += f"""AWS_ROLE_ARN=\"{str(value.get("aws_role_arn", ""))}\"\n""" + if "aws_external_id" in value: + output += f"""AWS_EXTERNAL_ID=\"{str(value.get("aws_external_id", ""))}\"\n""" + else: + continue + + if state == ExecutionState.API_POST: + stop = True + outputs: + artifacts: + - name: success + path: "/tmp/rest/success" + s3: + key: "{{inputs.parameters.output-prefix}}/success" + archive: + none: { } + - name: failure + path: "/tmp/rest/failure" + archive: + none: { } + s3: + key: "{{inputs.parameters.output-prefix}}/failure" + parameters: + - name: success-num-files + valueFrom: + path: "/tmp/rest/success/result-gen.txt" + - name: failure-num-files + valueFrom: + path: "/tmp/rest/failure/result-gen.txt" + volumes: + - name: credentials + emptyDir: { } + container: + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "./entrypoint.sh" ] + volumeMounts: + - name: credentials + mountPath: /tmp/credentials + imagePullPolicy: IfNotPresent + args: [ + "python3", "main.py","GET", "{{inputs.parameters.url}}", + "--raw-input-paginate", "{{inputs.parameters.raw-input-paginate}}", + "--raw-input-multiline", "{{inputs.parameters.raw-input-multiline}}", + "--threads", "{{inputs.parameters.threads}}", + "--raw-input-file-pattern", "{{inputs.parameters.raw-input-file-pattern}}", + "--request-config", "{{inputs.parameters.NAME-request-config}}", + "--execution-script", "{{inputs.parameters.NAME-execution-script}}", + "--secrets-path", "/tmp/credentials/success/*.json", + "--auth-type", "aws", + "--auth-aws-service", "NAME", + "--auth-aws-region", "AWS_REGION", + "--output-chunk-size", "{{inputs.parameters.output-chunk-size}}", + "--output-file-prefix", "/tmp/rest", + "--pagination-wait-time", "0", + "--max-retries", "10", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] + initContainers: + - name: fetch-credentials + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "python3", "main.py" ] + env: + - name: OAUTHLIB_INSECURE_TRANSPORT + value: "1" + - name: CLIENT_ID + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-id-env}}" + - name: CLIENT_SECRET + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-secret-env}}" + - name: TOKEN_URL + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.token-url-env}}" + mirrorVolumeMounts: true + args: [ + "GET", + "{{inputs.parameters.heracles-uri}}/credentials/{{inputs.parameters.credential-guid}}/use", + "--raw-input", "{}", + "--raw-input-file-sort", "", + "--raw-input-multiline", "False", + "--execution-script", "{{inputs.parameters.init-execution-script}}", + "--raw-input-paginate", "0", + "--auth-type", "oauth2", + "--auth-oauth2-type", "client_credentials", + "--auth-oauth2-impersonate-user", "{{=sprig.dig('labels', 'workflows', 'argoproj', 'io/creator', '', workflow)}}", + "--auth-oauth2-client-credentials-client-id", "CLIENT_ID", + "--auth-oauth2-client-credentials-secret", "CLIENT_SECRET", + "--auth-oauth2-client-credentials-token-url", "TOKEN_URL", + "--output-chunk-size", "0", + "--output-file-prefix", "/tmp/credentials", + "--pagination-wait-time", "0", + "--max-retries", "10", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] diff --git a/lib/connectors/BI/templates/basic+jwt.yaml b/lib/connectors/BI/templates/basic+jwt.yaml new file mode 100644 index 0000000..ab64710 --- /dev/null +++ b/lib/connectors/BI/templates/basic+jwt.yaml @@ -0,0 +1,633 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: atlan-NAME +spec: + entrypoint: main + templates: + - name: main + inputs: + parameters: + - name: credentials-fetch-strategy + value: "credential_guid" + enum: + - "k8s_secret" + - "credential_guid" + + - name: credential-kube-secret-name + value: "{{workflow.name}}-credential-secret" + + - name: credential-guid + value: "" + + #connection atlan object + - name: connection + + # NAME api version. + - name: api-version + value: "v52.0" + + #publish mode + - name: publish-mode + value: "production" + enum: + - "production" + - "test" + - "dev" + + #publish credential + - name: atlas-auth-type + value: "internal" + enum: + - "internal" + - "apikey" + + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + + - name: auto-classification + value: "false" + enum: + - "true" + - "false" + + - name: attachment-confidence-threshold + value: "0.8" + + - name: fetch-reports + value: "true" + + # Revisions + - name: marketplace-scripts-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplaceScriptsBranch" + optional: true + default: "master" + - name: marketplace-packages-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplacePackagesBranch" + optional: true + default: "master" + + dag: + tasks: + - name: fetch-credentials + templateRef: + name: rest-api + template: oauth2-client-credentials + arguments: + parameters: + - name: method + value: GET + - name: url + value: "http://heracles-service.heracles.svc.cluster.local/credentials/{{inputs.parameters.credential-guid}}/use" + - name: request-config + value: | + { + "headers": { + "user-id": "{{workflow.labels.workflows.argoproj.io/creator}}" + } + } + - name: kube-secret-name + value: "argo-client-creds" + - name: max-retries + value: "10" + - name: client-id-env + value: "login" + - name: client-secret-env + value: "password" + - name: token-url-env + value: "host" + - name: execution-script + value: | + if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): + LOGGER.debug('Heracles is unavailable. Performing retry with back-off') + failure_handler = FailureHandler.RETRY + if state == ExecutionState.OUTPUT_PROCESS: + credential = json.loads(output) + auth_type = credential["authType"] + + if auth_type == "basic": + oauth2_type = "resource_owner_password" + elif auth_type == "jwt": + oauth2_type = "jwt_bearer" + + output = { + "oauth2_type": oauth2_type, + } + + if state == ExecutionState.API_POST: + stop=True + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},bot=atlan-NAME" + - name: auth-type-to-param + dependencies: + - fetch-credentials + templateRef: + name: utils + template: artifact-to-key-param + arguments: + parameters: + - name: key + value: "oauth2_type" + artifacts: + - name: input + from: "{{tasks.fetch-credentials.outputs.artifacts.success}}" + subPath: "result-0.json" + - name: extract + dependencies: + - auth-type-to-param + template: extract-NAME-metadata + arguments: + parameters: + - name: credentials-fetch-strategy + value: "{{inputs.parameters.credentials-fetch-strategy}}" + - name: credential-kube-secret-name + value: "{{inputs.parameters.credential-kube-secret-name}}" + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: oauth2-type + value: "{{tasks.auth-type-to-param.outputs.parameters.output}}" + - name: connection-qualified-name + value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: fetch-reports + value: "{{inputs.parameters.fetch-reports}}" + + - name: process-objects + template: process-NAME-described-assets + dependencies: + - extract + withParam: "{{tasks.extract.outputs.parameters.org-host-name}}" + arguments: + parameters: + - name: host-name + value: "{{item.host}}" + - name: output-prefix + value: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/metadata-extract/{{workflow.name}}/processed-metadata" + - name: process-chunk-size + value: 10000 + artifacts: + - name: input-raw-metadata + s3: + key: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/metadata-extract/{{workflow.name}}/raw-metadata" + archive: + none: { } + - name: scripts + git: + repo: git@github.com:atlanhq/marketplace-scripts + insecureIgnoreHostKey: true + depth: 1 + revision: "{{inputs.parameters.marketplace-scripts-revision}}" + sshPrivateKeySecret: + name: "{{inputs.parameters.git-kube-secret-name}}" + key: "{{inputs.parameters.git-kube-ssh-key}}" + - name: publish + dependencies: + - process-objects + template: publish-NAME-metadata + arguments: + parameters: + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.publish-mode}}" + - name: source + value: "NAME" + - name: atlas-api-uri + value: "http://atlas-service-atlas.atlas.svc.cluster.local/api/atlas/v2" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: processed-data-key + value: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/metadata-extract/{{workflow.name}}/processed-metadata" + - name: atlan-web-kube-secret + value: "argo-client-creds" + - name: publish-chunk-size + value: "25" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + + - name: extract-NAME-metadata + inputs: + parameters: + - name: credentials-fetch-strategy + - name: credential-kube-secret-name + - name: credential-guid + - name: oauth2-type + - name: connection-qualified-name + - name: heracles-uri + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: fetch-reports + outputs: + parameters: + - name: org-host-name + valueFrom: + default: "[]" + parameter: "{{tasks.convert-to-param.outputs.parameters.output}}" + dag: + tasks: + - name: extract-organization + template: NAME-api + arguments: + parameters: + - name: url + value: "/services/data/v52.0/query?q=select%20Fields(All)%20from%20Organization%20limit%201" + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: oauth2-type + value: "{{inputs.parameters.oauth2-type}}" + - name: execution-script + value: | + if state == ExecutionState.RAW_INPUT_PROCESS: + creds_arr = secrets["result-0.json"].split("""\n""") + host = '' + for cred in creds_arr: + if 'HOST' in cred: + host = cred.split('=')[-1].replace('"', '') + request_config['url'] = request_config['url'].replace('', host) + store['host'] = host + + if state == ExecutionState.OUTPUT_PROCESS: + _response = json.loads(output).get('records', []) + + if len(_response) > 0: + output = _response[0] + output['host'] = store['host'] + + if state == ExecutionState.API_POST: + stop = True + + if state == ExecutionState.API_FAIL: failure_handler=FailureHandler.RETRY + - name: output-chunk-size + value: 1 + - name: output-prefix + value: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/metadata-extract/{{workflow.name}}/raw-metadata/described-organization/0" + + - name: process-NAME-metadata + inputs: + parameters: + - name: host-name + - name: output-prefix + - name: process-chunk-size + artifacts: + - name: input-raw-metadata + path: "/tmp/input" + - name: scripts + path: /tmp/marketplace-scripts + outputs: + artifacts: + - name: output + path: "/tmp/output" + s3: + key: "{{inputs.parameters.output-prefix}}" + archive: + none: { } + container: + image: ghcr.io/atlanhq/marketplace-scripts-base:0.1.11 + workingDir: "/tmp/marketplace-scripts" + command: [ "python" ] + args: + - "-m" + - "marketplace_scripts.NAME.main" + - "--host-name" + - "{{inputs.parameters.host-name}}" + - "--input-dir" + - "/tmp/input" + - "--output-prefix" + - "/tmp/output/" + - "--chunk-size" + - "{{inputs.parameters.process-chunk-size}}" + + - name: publish-NAME-metadata + inputs: + parameters: + - name: connection + - name: mode + - name: source + - name: atlas-api-uri + - name: heracles-uri + - name: processed-data-key + - name: atlan-web-kube-secret + - name: atlas-auth-type + - name: publish-chunk-size + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + dag: + tasks: + - name: publish + templateRef: + name: atlan-crawler + template: generic-publish + arguments: + artifacts: + - name: data + s3: + key: "{{inputs.parameters.processed-data-key}}" + - name: transformer-config + raw: + data: | + { + "external_map": { + "crawler_name": "{{workflow.labels.workflows.argoproj.io/workflow-template}}", + "tenant_id": "{{workflow.namespace}}", + "integration_name": "{{inputs.parameters.source}}", + "workflow_name": "{{workflow.name}}", + "connection_name": "{{=jsonpath(inputs.parameters.connection, '$.attributes.name')}}", + "connection_qn": "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + }, + "output_prefix": "/tmp/entities/", + "templates_root": "/tmp/templates/packages", + "transformation_config": [ + { + "input_file_pattern": "/tmp/inputs/organization*.json", + "template": "atlan/NAME/transformers/organization.jinja2", + "output_file_prefix": "organization" + }, + { + "input_file_pattern": "/tmp/inputs/objects*.json", + "template": "atlan/NAME/transformers/object.jinja2", + "output_file_prefix": "objects", + "ignore": true + }, + { + "input_file_pattern": "/tmp/inputs/fields*.json", + "template": "atlan/NAME/transformers/field.jinja2", + "output_file_prefix": "fields", + "ignore": true + }, + { + "input_file_pattern": "/tmp/inputs/reports*.json", + "template": "atlan/NAME/transformers/report.jinja2", + "output_file_prefix": "reports", + "ignore": true + }, + { + "input_file_pattern": "/tmp/inputs/dashboards*.json", + "template": "atlan/NAME/transformers/dashboard.jinja2", + "output_file_prefix": "dashboards", + "ignore": true + } + ] + } + parameters: + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.mode}}" + - name: source + value: "{{inputs.parameters.source}}" + - name: raw-input-file-sort + value: "organization,objects,fields,reports,dashboards" + - name: publish-chunk-size + value: "{{inputs.parameters.publish-chunk-size}}" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + + - name: NAME-api + synchronization: + semaphore: + configMapKeyRef: + name: atlan-NAME + key: api + volumes: + - name: atlan-NAME-workflow-directory + ephemeral: + volumeClaimTemplate: + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 60Gi + - name: credentials + emptyDir: { } + inputs: + artifacts: + - name: raw-input + path: /tmp/input + optional: true + parameters: + - name: credential-guid + - name: oauth2-type + - name: page-size + value: "10" + - name: url + - name: request-config + value: | + { + "headers": { + "Content-Type": "application/x-www-form-urlencoded" + } + } + - name: execution-script + - name: output-chunk-size + value: "0"sourceCategory + - name: kube-secret-name + value: "argo-client-creds" + - name: client-id-env + value: "login" + - name: client-secret-env + value: "password" + - name: token-url-env + value: "host" + - name: raw-input-file-pattern + value: "" + - name: raw-input-paginate + value: 0 + - name: raw-input-multiline + value: "False" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},bot=atlan-NAME" + - name: output-prefix + value: "argo-artifacts/{{workflow.namespace}}/{{workflow.name}}/{{pod.name}}" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: init-execution-script + value: | + if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): + LOGGER.debug('Heracles is unavailable. Performing retry with back-off') + failure_handler = FailureHandler.RETRY + if state == ExecutionState.OUTPUT_PROCESS: + credential = json.loads(output) + auth_type = credential.get('authType', '') + extra_params = credential.get('extra', {}) + is_sandbox = extra_params.get('is_sandbox', False) + token_url = 'test.NAME.com' if is_sandbox else 'login.NAME.com' + + output = f"""USERNAME="{credential.get('username', '')}" + HOST="{credential.get('host', '')}" + TOKEN_URL="https://{token_url}/services/oauth2/token" + CLIENT_ID="{extra_params.get('client_id', '')}" + """ + + if auth_type == 'basic': + password = credential.get('password', '').replace('$', '\$') + output += f"""PASSWORD="{password}" + CLIENT_SECRET="{extra_params.get('client_secret', '')}" + """ + elif auth_type == 'jwt': + output += f"""PRIVATE_KEY="{extra_params.get('private_key', '')}" + """ + + if state == ExecutionState.API_POST: + stop = True + outputs: + artifacts: + - name: success + path: "/tmp/rest/success" + s3: + key: "{{inputs.parameters.output-prefix}}/success" + archive: + none: { } + - name: failure + path: "/tmp/rest/failure" + archive: + none: { } + s3: + key: "{{inputs.parameters.output-prefix}}/failure" + parameters: + - name: success-num-files + valueFrom: + path: "/tmp/rest/success/result-gen.txt" + - name: failure-num-files + valueFrom: + path: "/tmp/rest/failure/result-gen.txt" + container: + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "./entrypoint.sh" ] + volumeMounts: + - name: atlan-NAME-workflow-directory + mountPath: /tmp + - name: credentials + mountPath: /tmp/credentials + imagePullPolicy: IfNotPresent + args: [ + "python3", "main.py", "GET", "{{inputs.parameters.url}}", + "--raw-input-paginate", "{{inputs.parameters.raw-input-paginate}}", + "--raw-input-file-pattern", "{{inputs.parameters.raw-input-file-pattern}}", + "--raw-input-multiline", "{{inputs.parameters.raw-input-multiline}}", + "--request-config", "{{inputs.parameters.request-config}}", + "--execution-script", "{{inputs.parameters.execution-script}}", + "--secrets-path", "/tmp/credentials/success/*.json", + "--auth-type", "oauth2", + "--auth-oauth2-type", "{{inputs.parameters.oauth2-type}}", + "--auth-oauth2-resource-owner-password-client-id", "CLIENT_ID", + "--auth-oauth2-resource-owner-password-secret", "CLIENT_SECRET", + "--auth-oauth2-resource-owner-password-token-url", "TOKEN_URL", + "--auth-oauth2-resource-owner-password-username", "USERNAME", + "--auth-oauth2-resource-owner-password-password", "PASSWORD", + "--auth-oauth2-jwt-bearer-client-id", "CLIENT_ID", + "--auth-oauth2-jwt-bearer-private-key", "PRIVATE_KEY", + "--auth-oauth2-jwt-bearer-token-url", "TOKEN_URL", + "--auth-oauth2-jwt-bearer-username", "USERNAME", + "--auth-oauth2-jwt-bearer-token-expiry", "{{=sprig.unixEpoch(sprig.dateModify('3m', sprig.now()))}}", + "--output-chunk-size", "{{inputs.parameters.output-chunk-size}}", + "--output-file-prefix", "/tmp/rest/", + "--pagination-wait-time", "10", + "--max-retries", "3", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] + + initContainers: + - name: fetch-credentials + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "python3", "main.py" ] + env: + - name: OAUTHLIB_INSECURE_TRANSPORT + value: "1" + - name: CLIENT_ID + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-id-env}}" + - name: CLIENT_SECRET + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-secret-env}}" + - name: TOKEN_URL + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.token-url-env}}" + mirrorVolumeMounts: true + args: [ + "GET", + "{{inputs.parameters.heracles-uri}}/credentials/{{inputs.parameters.credential-guid}}/use", + "--raw-input", "{}", + "--raw-input-file-pattern", "", + "--raw-input-file-sort", "", + "--raw-input-multiline", "f", + "--execution-script", "{{inputs.parameters.init-execution-script}}", + "--raw-input-paginate", "0", + "--auth-type", "oauth2", + "--auth-oauth2-type", "client_credentials", + "--auth-oauth2-impersonate-user", "{{=sprig.dig('labels', 'workflows', 'argoproj', 'io/creator', '', workflow)}}", + "--auth-oauth2-client-credentials-client-id", "CLIENT_ID", + "--auth-oauth2-client-credentials-secret", "CLIENT_SECRET", + "--auth-oauth2-client-credentials-token-url", "TOKEN_URL", + "--output-chunk-size", "0", + "--output-file-prefix", "/tmp/credentials", + "--pagination-wait-time", "0", + "--max-retries", + "10", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] + diff --git a/lib/connectors/BI/templates/oauth.yaml b/lib/connectors/BI/templates/oauth.yaml new file mode 100644 index 0000000..152899f --- /dev/null +++ b/lib/connectors/BI/templates/oauth.yaml @@ -0,0 +1,581 @@ +# Workflow Templates +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: atlan-NAME +spec: + entrypoint: main + templates: + - name: main + inputs: + parameters: + # Credential + - name: credentials-fetch-strategy + value: "credential_guid" + enum: + - "credential_guid" + - "k8s_secret" + - name: credential-guid + value: "" + # Connection Entity + - name: connection + - name: atlas-auth-type + value: "internal" + enum: + - "internal" + - "apikey" + # Include/Exclude Filters + - name: include-filter + value: "{}" + - name: exclude-filter + value: "{}" + # Publish Mode + - name: publish-mode + value: "production" + enum: + - "production" + - "dev" + - "test" + # Revisions + - name: marketplace-scripts-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplaceScriptsBranch" + optional: true + default: "master" + - name: marketplace-packages-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplacePackagesBranch" + optional: true + default: "master" + dag: + tasks: + # Extract metadata from NAME + - name: extract + template: extract-NAME-metadata + arguments: + parameters: + - name: credentials-fetch-strategy + value: "{{inputs.parameters.credentials-fetch-strategy}}" + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: connection-qualified-name + value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + # Process the extracted metadata + - name: process + template: process-NAME-metadata + depends: "extract.Succeeded" + arguments: + parameters: + - name: connection-qualified-name + value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + + - name: publish + depends: "process.Succeeded" + template: publish-NAME-metadata + arguments: + parameters: + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.publish-mode}}" + - name: source + value: "NAME" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: atlan-web-kube-secret + value: "argo-client-creds" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: atlas-api-uri + value: "http://atlas-service-atlas.atlas.svc.cluster.local/api/atlas/v2" + - name: publish-chunk-size + value: "100" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + + - name: extract-NAME-metadata + inputs: + parameters: + - name: credentials-fetch-strategy + - name: credential-guid + - name: connection-qualified-name + - name: include-filter + - name: exclude-filter + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: heracles-uri + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: marketplace-scripts-revision + outputs: + parameters: + - name: auth-type-and-host + valueFrom: + parameter: "{{tasks.auth-type-and-host-to-param.outputs.parameters.output}}" + dag: + tasks: + # Fetch the credential + - name: fetch-credentials + when: "{{inputs.parameters.credentials-fetch-strategy}} == credential_guid" + templateRef: + name: rest-api + template: oauth2-client-credentials + arguments: + parameters: + - name: method + value: GET + - name: url + value: "{{inputs.parameters.heracles-uri}}/credentials/{{inputs.parameters.credential-guid}}/use" + - name: request-config + value: | + { + "headers": { + "user-id": "{{workflow.labels.workflows.argoproj.io/creator}}" + } + } + - name: kube-secret-name + value: "argo-client-creds" + - name: max-retries + value: "10" + - name: client-id-env + value: "login" + - name: client-secret-env + value: "password" + - name: token-url-env + value: "host" + - name: execution-script + value: | + if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): + LOGGER.debug("Heracles is unavailable. Performing retry with back-off") + failure_handler = FailureHandler.RETRY + + if state == ExecutionState.OUTPUT_PROCESS: + credential = json.loads(output) + output = { + "authType": credential["authType"], + "host": credential["host"] + } + + if state == ExecutionState.API_POST: + stop=True + - name: statsd-host + value: "{{inputs.parameters.statsd-host}}" + - name: statsd-port + value: "{{inputs.parameters.statsd-port}}" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + # Convert that `fetch-credentials` output from an artifact to a parameter + - name: auth-type-and-host-to-param + dependencies: + - fetch-credentials + templateRef: + name: utils + template: artifact-to-direct-param + arguments: + artifacts: + - name: input + from: "{{tasks.fetch-credentials.outputs.artifacts.success}}" + subPath: "result-0.json" + # Get all the workbooks + - name: fetch-ASSET_NAME + dependencies: + - auth-type-and-host-to-param + template: api-request + arguments: + parameters: + - name: auth-type + value: "{{=jsonpath(tasks['auth-type-and-host-to-param'].outputs.parameters.output, '$.authType')}}" + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: heracles-uri + value: "{{inputs.parameters.heracles-uri}}" + - name: method + value: GET + - name: url + value: "<>" + - name: request-config + value: | + { + "params": { + "page": 0, + "limit": 100 + } + } + - name: kube-secret-name + value: "argo-client-creds" + - name: output-prefix + value: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/extracted-metadata/{{workflow.name}}/ASSET_NAME" + - name: output-chunk-size + value: "1000" + - name: execution-script + value: | + if state == ExecutionState.RAW_INPUT_PROCESS: + creds_arr = secrets["result-0.json"].split("""\n""") + host = "" + for cred in creds_arr: + if "HOST" in cred: + host = cred.split('=')[-1].replace('"', '') + + url = "{{inputs.parameters.url}}" + url = url.replace("", host) + LOGGER.debug(f"URL: {url}") + request_config["url"] = url + + if state == ExecutionState.OUTPUT_PROCESS: + output = json.loads(output) + + if state == ExecutionState.API_POST: + workbook_response = response.json() + if not workbook_response.get("hasMore"): + stop = True + else: + next_page = workbook_response.get("nextPage") + request_config["params"] = { + "page": next_page, + "limit": 100 + } + if state == ExecutionState.API_FAIL: + if response.status_code == 500: + failure_handler = FailureHandler.NONE + else: + failure_handler = FailureHandler.RETRY + - name: statsd-host + value: "{{inputs.parameters.statsd-host}}" + - name: statsd-port + value: "{{inputs.parameters.statsd-port}}" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + + - name: process-NAME-metadata + inputs: + parameters: + - name: connection-qualified-name + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: marketplace-scripts-revision + dag: + tasks: + - name: process-metadata + template: process-metadata + arguments: + parameters: + - name: output-prefix + value: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/processed-metadata/{{workflow.name}}" + - name: git-kube-secret-name + value: "{{inputs.parameters.git-kube-secret-name}}" + - name: git-kube-ssh-key + value: "{{inputs.parameters.git-kube-ssh-key}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + artifacts: + - name: extracted-metadata + s3: + key: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/extracted-metadata/{{workflow.name}}" + - name: parsed-queries + s3: + key: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/parsed-queries/{{workflow.name}}" + + - name: publish-NAME-metadata + inputs: + parameters: + - name: connection + - name: mode + - name: source + - name: atlas-api-uri + - name: heracles-uri + - name: atlan-web-kube-secret + - name: atlas-auth-type + - name: publish-chunk-size + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + dag: + tasks: + # Run atlan-crawler/publish-bi template + - name: publish + templateRef: + name: atlan-crawler + template: publish-bi + arguments: + artifacts: + - name: data + s3: + key: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/processed-metadata/{{workflow.name}}" + - name: transformer-config + raw: + data: | + { + "external_map": { + "crawler_name": "{{workflow.labels.workflows.argoproj.io/workflow-template}}", + "tenant_id": "{{workflow.namespace}}", + "integration_name": "{{inputs.parameters.source}}", + "workflow_name": "{{workflow.name}}", + "connection_name": "{{=jsonpath(inputs.parameters.connection, '$.attributes.name')}}", + "connection_qn": "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + }, + "output_prefix": "/tmp/entities", + "templates_root": "/tmp/templates/packages", + "transformation_config": [ + { + "input_file_pattern": "/tmp/inputs/<>.json", + "template": "atlan/NAME/transformers/<>.jinja2", + "output_file_prefix": "<>/<>", + "output_chunk_size": 15000 + } + ] + } + parameters: + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.mode}}" + - name: source + value: "{{inputs.parameters.source}}" + - name: raw-input-file-sort + value: "" + - name: raw-input-folder-sort + value: "" + - name: raw-input-file-pattern + value: "**/*.json" + - name: publish-chunk-size + value: "{{inputs.parameters.publish-chunk-size}}" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: hierarchy + value: | + [ + [ + { + "name":"", + "file_pattern": "" + } + ] + ] + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + + - name: NAME-api + synchronization: + semaphore: + configMapKeyRef: + name: atlan-NAME + key: api + volumes: + - name: credentials + emptyDir: { } + inputs: + artifacts: + - name: raw-input + path: /tmp/input + optional: true + - name: raw-input-file + path: "/tmp/input/input.json" + optional: true + parameters: + - name: credential-guid + - name: page-size + value: "10" + - name: method + value: "GET" + - name: url + - name: request-config + value: "{}" + - name: execution-script + - name: output-chunk-size + value: 100 + - name: kube-secret-name + value: "argo-client-creds" + - name: client-id-env + value: "login" + - name: client-secret-env + value: "password" + - name: token-url-env + value: "host" + - name: raw-input-paginate + value: 0 + - name: raw-input-file-pattern + value: "" + - name: raw-input-multiline + value: "False" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},bot=atlan-NAME" + - name: output-prefix + value: "argo-artifacts/{{workflow.namespace}}/{{workflow.name}}/{{pod.name}}" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: init-execution-script + value: | + if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): + LOGGER.debug('Heracles is unavailable. Performing retry with back-off') + failure_handler = FailureHandler.RETRY + + if state == ExecutionState.OUTPUT_PROCESS: + credential = json.loads(output) + output = "" + for key, value in credential.items(): + output += f"""ATLAN_{key.upper()}="{str(value)}"\n""" + + output += f"""ATLAN_TOKEN_URL='https://{credential['host']}/v2/auth/token'\n""" + + if state == ExecutionState.API_POST: + stop = True + outputs: + artifacts: + - name: success + path: "/tmp/rest/success" + s3: + key: "{{inputs.parameters.output-prefix}}" + archive: + none: { } + - name: failure + path: "/tmp/rest/failure" + parameters: + - name: success-num-files + valueFrom: + path: "/tmp/rest/success/result-gen.txt" + - name: failure-num-files + valueFrom: + path: "/tmp/rest/failure/result-gen.txt" + container: + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "./entrypoint.sh" ] + volumeMounts: + - name: credentials + mountPath: /tmp/credentials + env: + - name: OAUTHLIB_INSECURE_TRANSPORT + value: "1" + imagePullPolicy: IfNotPresent + args: [ + "python3", "main.py", "{{inputs.parameters.method}}", "{{inputs.parameters.url}}", + "--request-config", "{{inputs.parameters.request-config}}", + "--raw-input-paginate", "{{inputs.parameters.raw-input-paginate}}", + "--raw-input-file-pattern", "{{inputs.parameters.raw-input-file-pattern}}", + "--raw-input-multiline", "{{inputs.parameters.raw-input-multiline}}", + "--execution-script", "{{inputs.parameters.execution-script}}", + "--secrets-path", "/tmp/credentials/success/*.json", + "--auth-type", "oauth2", + "--auth-oauth2-type", "client_credentials", + "--auth-oauth2-impersonate-user", "", + "--auth-oauth2-client-credentials-client-id", "ATLAN_USERNAME", + "--auth-oauth2-client-credentials-secret", "ATLAN_PASSWORD", + "--auth-oauth2-client-credentials-token-url", "ATLAN_TOKEN_URL", + "--output-chunk-size", "{{inputs.parameters.output-chunk-size}}", + "--output-file-prefix", "/tmp/rest", + "--pagination-wait-time", "10", + "--max-retries", "3", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] + initContainers: + - name: fetch-credentials + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "python3", "main.py" ] + env: + - name: OAUTHLIB_INSECURE_TRANSPORT + value: "1" + - name: CLIENT_ID + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-id-env}}" + - name: CLIENT_SECRET + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-secret-env}}" + - name: TOKEN_URL + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.token-url-env}}" + mirrorVolumeMounts: true + args: [ + "GET", + "{{inputs.parameters.heracles-uri}}/credentials/{{inputs.parameters.credential-guid}}/use", + "--raw-input", "{}", + "--raw-input-file-sort", "", + "--raw-input-multiline", "False", + "--execution-script", "{{inputs.parameters.init-execution-script}}", + "--raw-input-paginate", "0", + "--auth-type", "oauth2", + "--auth-oauth2-type", "client_credentials", + "--auth-oauth2-impersonate-user", "{{=sprig.dig('labels', 'workflows', 'argoproj', 'io/creator', '', workflow)}}", + "--auth-oauth2-client-credentials-client-id", "CLIENT_ID", + "--auth-oauth2-client-credentials-secret", "CLIENT_SECRET", + "--auth-oauth2-client-credentials-token-url", "TOKEN_URL", + "--output-chunk-size", "0", + "--output-file-prefix", "/tmp/credentials", + "--pagination-wait-time", "0", + "--max-retries", "10", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] \ No newline at end of file diff --git a/lib/connectors/BI/transformers/default.jinja2 b/lib/connectors/BI/transformers/default.jinja2 new file mode 100644 index 0000000..1a0d6dc --- /dev/null +++ b/lib/connectors/BI/transformers/default.jinja2 @@ -0,0 +1,19 @@ +{ + "typeName": "ASSET_NAME", + "status": "ACTIVE", + "attributes": { + "connectorName": "{{external_map['integration_name']}}", + "connectionName": "{{external_map['connection_name']}}", + "connectionQualifiedName": "{{external_map['connection_qn']}}", + "lastSyncRunAt": {{now}}, + "lastSyncWorkflowName": "{{external_map['crawler_name']}}", + "lastSyncRun": "{{external_map['workflow_name']}}", + "tenantId": "{{external_map['tenant_id']}}", + + "qualifiedName": "{{external_map['connection_qn']}}/{{record['id']}}", + + "name": {{record.get('name') | tojson}}, + "sourceCreatedAt": "{{convert_date(record['createdAt'], format="%Y-%m-%dT%H:%M:%S.%fZ")}}", + "sourceUpdatedAt": "{{convert_date(record['updatedAt'], format="%Y-%m-%dT%H:%M:%S.%fZ")}}", + } +} \ No newline at end of file diff --git a/lib/connectors/common-configmap/api.yaml b/lib/connectors/common-configmap/api.yaml new file mode 100644 index 0000000..f258942 --- /dev/null +++ b/lib/connectors/common-configmap/api.yaml @@ -0,0 +1,166 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlan-connectors-NAME + labels: + orchestration.atlan.com/version: "1" + orchestration.atlan.com/source: "NAME" +data: + icon: "https://cdn-images-1.medium.com/max/1200/1*8bPQYTZUHcvvk_1fSPwwHw.png" + helpdeskLink: "" + logo: "https://cdn-images-1.medium.com/max/1200/1*8bPQYTZUHcvvk_1fSPwwHw.png" + connector: "NAME" + defaultConnectorType: "rest" + jdbcCredentialTemplate: "{}" + restCredentialTemplate: | + { + "default": [ + { + "id": 1, + "name": "auth", + "curl": "curl --location --request POST 'https://{{host}}/v2/auth/token' --header 'Content-Type:application/x-www-form-urlencoded' --data-urlencode 'grant_type=client_credentials' --data-urlencode 'client_id={{username}}' --data-urlencode 'client_secret={{password}}' --connect-timeout 4" + } + ] + } + odbcCredentialTemplate: "{}" + grpcCredentialTemplate: "{}" + restMetadataTemplate: | + { + "default": [ + { + "id": 1, + "name": "auth", + "curl": "curl --location --request POST 'https://{{host}}/v2/auth/token' --header 'Content-Type:application/x-www-form-urlencoded' --data-urlencode 'grant_type=client_credentials' --data-urlencode 'client_id={{username}}' --data-urlencode 'client_secret={{password}}' --connect-timeout 4" + } + ] + } + restMetadataOutputTransformerTemplate: | + { + + } + odbcTemplate: "{}" + grpcTemplate: "{}" + sageTemplate: | + { + + } + config: | + { + "properties": { + "name": { + "type": "string", + "required": false, + "ui": { + "label": "Name", + "hidden": true, + "placeholder": "Host Name" + } + }, + "connector": { + "type": "string", + "required": false, + "ui": { + "label": "Connector", + "hidden": true, + "placeholder": "Connector" + } + }, + "connectorType": { + "type": "string", + "required": false, + "ui": { + "key": "_host", + "label": "connectorType", + "placeholder": "connectorType", + "hidden": true + } + }, + "host": { + "type": "string", + "required": true, + "enum": ["aws-api.NAMEcomputing.com", "api.NAMEcomputing.com"], + "enumNames": ["aws-api.NAMEcomputing.com - For AWS Hosted Organizations", "api.NAMEcomputing.com - For GCP Hosted Organizations"], + "default": "aws-api.NAMEcomputing.com", + "ui": { + "widget": "select", + "label": "Endpoint", + "grid": 8, + "hidden": false, + "placeholder": "Endpoint" + } + }, + "auth-type": { + "type": "string", + "enum": ["api_token"], + "default": "api_token", + "required": true, + "enumNames": ["API Token"], + "ui": { + "widget": "radio", + "label": "Authentication", + "placeholder": "Credential Type", + "rules": [ + { + "required": true, + "message": "Please enter a valid authentication type" + } + ] + } + }, + "api_token": { + "type": "object", + "properties": { + "username": { + "type": "string", + "required": true, + "ui": { + "label": "Client ID", + "placeholder": "Client ID", + "feedback": true, + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid client Id" + } + ] + } + }, + "password": { + "type": "string", + "required": true, + "ui": { + "widget": "password", + "label": "API Token", + "feedback": true, + "placeholder": "API Token", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid API token" + } + ] + } + } + }, + "ui": { + "widget": "nested", + "label": "Client ID & API Token", + "placeholder": "Credential Type", + "nestedValue": false, + "hidden": true + } + } + }, + "anyOf": [ + { + "properties": { + "auth-type": { + "const": "api_token" + } + }, + "required": ["api_token"] + } + ] + } \ No newline at end of file diff --git a/lib/connectors/common-configmap/aws.yaml b/lib/connectors/common-configmap/aws.yaml new file mode 100644 index 0000000..07f6794 --- /dev/null +++ b/lib/connectors/common-configmap/aws.yaml @@ -0,0 +1,244 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlan-connectors-NAME + labels: + orchestration.atlan.com/version: "1" + orchestration.atlan.com/source: "NAME" +data: + icon: "https://atlan-public.s3.eu-west-1.amazonaws.com/atlan/logos/aws-NAME.png" + helpdeskLink: "https://ask.atlan.com/hc/en-us/articles/4605882724369-How-to-set-up-your-NAME-connection-on-Atlan" + logo: "https://atlan-public.s3.eu-west-1.amazonaws.com/atlan/logos/aws-NAME.png" + connector: "NAME" + defaultConnectorType: "sdk" + jdbcCredentialTemplate: "{}" + restCredentialTemplate: "{}" + odbcCredentialTemplate: "{}" + grpcCredentialTemplate: "{}" + restMetadataTemplate: "" + restMetadataOutputTransformerTemplate: "" + sdkCredentialTemplate: | + [ + { + "id": 1, + "name": "<>", + "source": "AWS", + "action": "AWSNAME.<>", + "params": { {% if authType == "iam" %} "aws_access_key_id": "{{username}}", "aws_secret_access_key": "{{password}}", {% endif %} "region": "{{extra.region}}" {% if authType == "role" and extra.aws_role_arn|length %}, "aws_role_arn": "{{extra.aws_role_arn}}" {% endif %} {% if authType == "role" and extra.aws_external_id|length %}, "aws_external_id": "{{extra.aws_external_id}}" {% endif %} } + } + ] + config: | + { + "properties": { + "name": { + "type": "string", + "required": false, + "ui": { + "label": "Name", + "hidden": true, + "placeholder": "Credential Name" + } + }, + "connector": { + "type": "string", + "required": false, + "ui": { + "label": "Connector", + "hidden": true, + "placeholder": "Connector" + } + }, + "connectorType": { + "type": "string", + "required": false, + "ui": { + "key": "_host", + "label": "connectorType", + "placeholder": "connectorType", + "hidden": true + } + }, + "auth-type": { + "type": "string", + "enum": [ + "iam", + "role" + ], + "default": "iam", + "required": true, + "enumNames": [ + "IAM User", + "IAM Role" + ], + "ui": { + "widget": "radio", + "hidden": false, + "label": "Authentication", + "placeholder": "Credential Type", + "rules": [ + { + "required": true, + "message": "Please enter a valid authentication type" + } + ] + } + }, + "port": { + "type": "number", + "default": 443, + "required": false, + "ui": { + "label": "Port", + "hidden": true, + "placeholder": "Port" + } + }, + "iam": { + "type": "object", + "properties": { + "username": { + "type": "string", + "required": true, + "ui": { + "label": "AWS Access Key", + "placeholder": "Access Key", + "feedback": true, + "help": "AWS Access Key", + "message": "Please enter a valid Access Key", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid Access Key" + } + ] + } + }, + "password": { + "type": "string", + "required": true, + "ui": { + "widget": "password", + "label": "AWS Secret Key", + "feedback": true, + "help": "AWS Secret Key", + "placeholder": "Secret Key", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid Secret Key" + } + ] + } + } + }, + "ui": { + "widget": "nested", + "label": "", + "placeholder": "Credential Type", + "nestedValue": false, + "hidden": false + } + }, + "role":{ + "type": "object", + "properties": { + "extra": { + "type": "object", + "properties": { + "aws_role_arn": { + "type": "string", + "required": false, + "ui": { + "label": "AWS Role ARN", + "hidden": false, + "placeholder": "arn:aws:iam::123456789012:role/roleName", + "help": "AWS role to assume for the connection", + "grid": 4, + "rules": [ + { + "pattern": "^arn:aws:iam::\\d{12}:role/.+", + "message": "Please enter a valid role ARN", + "trigger": "blur" + } + ] + } + }, + "aws_external_id": { + "type": "string", + "required": false, + "ui": { + "label": "External ID", + "widget": "keygen", + "help": "Unique external ID to enable access of your AWS resources to Atlan" + } + } + }, + "ui": { + "widget": "nested", + "label": "", + "header": "Advanced", + "hidden": false + } + } + }, + "ui": { + "widget": "nested", + "label": "", + "placeholder": "Credential Type", + "nestedValue": false, + "hidden": true + } + }, + "extra": { + "type": "object", + "properties": { + "region": { + "type": "string", + "required": true, + "default": "", + "ui": { + "help": "Enter region of NAME setup", + "label": "Region", + "hidden": false, + "placeholder": "us-west-1", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a region" + } + ] + } + } + }, + "ui": { + "widget": "nested", + "label": "", + "header": "Advanced", + "hidden": false + } + } + }, + "anyOf": [ + { + "properties": { + "auth-type": { + "const": "iam" + } + }, + "required": [ + "iam" + ] + }, + { + "properties": { + "auth-type": { + "const": "role" + } + }, + "required": ["role"] + } + ] + } diff --git a/lib/connectors/common-configmap/basic.yaml b/lib/connectors/common-configmap/basic.yaml new file mode 100644 index 0000000..5a60ae4 --- /dev/null +++ b/lib/connectors/common-configmap/basic.yaml @@ -0,0 +1,151 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlan-connectors-NAME + labels: + orchestration.atlan.com/version: "1" + orchestration.atlan.com/source: "NAME" +data: + icon: "<>" + helpdeskLink: "" + logo: "<>" + connector: "NAME" + defaultConnectorType: "rest" + jdbcCredentialTemplate: "{}" + restCredentialTemplate: "{}" + { + "default": [ + { + "id": 1, + "name": "auth", + "curl": "curl --location --request POST 'https://{{host}}/v2/auth/token' --header 'Content-Type:application/x-www-form-urlencoded' --data-urlencode 'grant_type=client_credentials' --data-urlencode 'client_id={{username}}' --data-urlencode 'client_secret={{password}}' --connect-timeout 4" + } + ] + } + odbcCredentialTemplate: "{}" + grpcCredentialTemplate: "{}" + restMetadataTemplate: "{}" + restMetadataOutputTransformerTemplate: "{}" + odbcTemplate: "{}" + grpcTemplate: "{}" + sageTemplate: "{}" + config: | + { + "properties": { + "name": { + "type": "string", + "required": false, + "ui": { + "label": "Name", + "hidden": true, + "placeholder": "Host Name" + } + }, + "connector": { + "type": "string", + "required": false, + "ui": { + "label": "Connector", + "hidden": true, + "placeholder": "Connector" + } + }, + "connectorType": { + "type": "string", + "required": false, + "ui": { + "key": "_host", + "label": "connectorType", + "placeholder": "connectorType", + "hidden": true + } + }, + "host": { + "type": "string", + "required": true, + "enum": ["aws-api.NAMEcomputing.com", "api.NAMEcomputing.com"], + "enumNames": ["aws-api.NAMEcomputing.com - For AWS Hosted Organizations", "api.NAMEcomputing.com - For GCP Hosted Organizations"], + "default": "aws-api.NAMEcomputing.com", + "ui": { + "widget": "select", + "label": "Endpoint", + "grid": 8, + "hidden": false, + "placeholder": "Endpoint" + } + }, + "auth-type": { + "type": "string", + "enum": ["api_token"], + "default": "api_token", + "required": true, + "enumNames": ["API Token"], + "ui": { + "widget": "radio", + "label": "Authentication", + "placeholder": "Credential Type", + "rules": [ + { + "required": true, + "message": "Please enter a valid authentication type" + } + ] + } + }, + "api_token": { + "type": "object", + "properties": { + "username": { + "type": "string", + "required": true, + "ui": { + "label": "Client ID", + "placeholder": "Client ID", + "feedback": true, + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid client Id" + } + ] + } + }, + "password": { + "type": "string", + "required": true, + "ui": { + "widget": "password", + "label": "API Token", + "feedback": true, + "placeholder": "API Token", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid API token" + } + ] + } + } + }, + "ui": { + "widget": "nested", + "label": "Client ID & API Token", + "placeholder": "Credential Type", + "nestedValue": false, + "hidden": true + } + } + }, + "anyOf": [ + { + "properties": { + "auth-type": { + "const": "api_token" + } + }, + "required": ["api_token"] + } + ] + } \ No newline at end of file diff --git a/lib/connectors/common-configmap/jdbc-basic.yaml b/lib/connectors/common-configmap/jdbc-basic.yaml new file mode 100644 index 0000000..4bea4ce --- /dev/null +++ b/lib/connectors/common-configmap/jdbc-basic.yaml @@ -0,0 +1,180 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlan-connectors-NAME + labels: + orchestration.atlan.com/version: "1" + orchestration.atlan.com/source: "NAME" +data: + icon: "https://atlan-public.s3.eu-west-1.amazonaws.com/atlan/logos/NAME.png" + helpdeskLink: "https://docs.atlan.com/integrations/relational-database/NAME" + logo: "https://atlan-public.s3.eu-west-1.amazonaws.com/atlan/logos/NAME.png" + connector: "NAME" + defaultConnectorType: "jdbc" + jdbcCredentialTemplate: | + { + "className": "com.facebook.NAME.jdbc.NAMEDriver", + "jarLink": "https://atlan-public.s3-eu-west-1.amazonaws.com/atlan/jdbc/NAME.tar.gz", + "url": "jdbc:NAME://{{ host }}:{{ port|int }}", + "driverProperties": { "user": "{{ username }}", "password": "{{ password }}" } + } + restCredentialTemplate: "{}" + odbcCredentialTemplate: "{}" + grpcCredentialTemplate: "{}" + restMetadataTemplate: "" + restMetadataOutputTransformerTemplate: "" + sageTemplate: | + { + "catalogsCheck": { + "curls": [ + { + "name": "schemas", + "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' --data-raw '{\"query\": \"show atlan schemas\"}'", + "addCredential": true, + "credentialConnectorType": "jdbc" + } + ], + "responseTemplate": "{{- $includeFilter := dict}} {{- if eq `string` (printf `%T` (index .formData `include-filter`)) }} {{- $includeFilter = index .formData `include-filter` | fromJson}} {{- else }} {{- $includeFilter = index .formData `include-filter` }} {{- end }} {{- $allowedDatabases := list}} {{- $allowedSchemas := list}} {{- $missingObjectName := ``}} {{- $checkSuccess := true }} {{- range $schemaList := .schemas.results }} {{- $allowedDatabases = append $allowedDatabases $schemaList.TABLE_CATALOG }} {{- $allowedSchemas = append $allowedSchemas (print $schemaList.TABLE_CATALOG `.` $schemaList.TABLE_SCHEM )}} {{- end }} {{- range $filteredDb, $filteredSchemas := $includeFilter }} {{- $_db := lower $filteredDb | trimPrefix `^` | trimSuffix `$` }} {{- $checkSuccess = and $checkSuccess (has $_db $allowedDatabases) }} {{- if not (has $_db $allowedDatabases)}} {{- $missingObjectName = (print $_db ` ` `database`)}} {{- end }} {{- range $schmea := $filteredSchemas }} {{- $_schema := lower $schmea | trimPrefix `^` | trimSuffix `$` }} {{- $checkSuccess = and $checkSuccess (has (print $_db `.` (lower $_schema)) $allowedSchemas)}} {{- if not (has (print $_db `.` (lower $_schema)) $allowedSchemas)}} {{- $missingObjectName = (print $_db `.` (lower $_schema) ` ` `schema`)}} {{- end }} {{- end }} {{- end }} {{- $response := dict `successMessage` `` `failureMessage` `` `data` .schemas.results `response` dict }} {{- if $checkSuccess }} {{- $_ := set $response `successMessage` `Check successful` }} {{- else }} {{- $_ := set $response `failureMessage` (print `Check failed for ` $missingObjectName) }} {{- end }} {{- $response | toJson }}" + } + } + config: | + { + "properties": { + "name": { + "type": "string", + "required": false, + "ui": { + "label": "Name", + "hidden": true, + "placeholder": "Host Name" + } + }, + "connector": { + "type": "string", + "required": false, + "ui": { + "label": "Connector", + "hidden": true, + "placeholder": "Connector" + } + }, + "connectorType": { + "type": "string", + "required": false, + "ui": { + "key": "_host", + "label": "connectorType", + "placeholder": "connectorType", + "hidden": true + } + }, + "host": { + "type": "string", + "required": true, + "default": "", + "ui": { + "label": "Host", + "feedback": true, + "help": "Your NAME instance host name", + "BYOCdisabled": true, + "rules": [ + { + "required": true, + "message": "Please enter a valid host name" + } + ], + "grid": 6 + } + }, + "port": { + "type": "number", + "default": 8080, + "required": true, + "ui": { + "label": "Port", + "placeholder": "Port", + "disabled": false, + "help": "Your NAME instance port number", + "grid": 2, + "BYOCdisabled": true, + "rules": [ + { + "required": true, + "message": "Please enter a valid port number" + } + ] + } + }, + "auth-type": { + "type": "string", + "enum": ["basic"], + "default": "basic", + "required": true, + "enumNames": ["Basic"], + "ui": { + "widget": "radio", + "hidden": false, + "label": "Authentication", + "placeholder": "Credential Type", + "rules": [ + { + "required": true, + "message": "Please enter a valid authentication type" + } + ] + } + }, + "basic": { + "type": "object", + "properties": { + "username": { + "type": "string", + "required": true, + "ui": { + "label": "Username", + "placeholder": "Username", + "help": "Database Username", + "feedback": true, + "message": "Please enter a valid username", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid username" + } + ] + } + }, + "password": { + "type": "string", + "required": false, + "ui": { + "widget": "password", + "label": "Password", + "help": "Database Password", + "feedback": false, + "placeholder": "Password", + "grid": 4 + } + } + }, + "ui": { + "widget": "nested", + "label": "Basic Authentication", + "placeholder": "Credential Type", + "nestedValue": false, + "hidden": true + } + } + }, + "anyOf": [ + { + "properties": { + "auth-type": { + "const": "basic" + } + }, + "required": ["basic"] + } + ] + } From fd94ac9ddb8f38d4d19b307d1397205ac545c1d7 Mon Sep 17 00:00:00 2001 From: Mrunmayi Date: Wed, 12 Apr 2023 06:38:10 +0530 Subject: [PATCH 6/6] Updating folders setup to build correct default folder structure --- bin/install.js | 73 ++++++++++++++++++++++++++------------------------ lib/init.js | 23 +++++++++++----- 2 files changed, 54 insertions(+), 42 deletions(-) diff --git a/bin/install.js b/bin/install.js index 092e00b..2b9ed9e 100755 --- a/bin/install.js +++ b/bin/install.js @@ -3,7 +3,7 @@ const { install, installGlobal } = require("../lib/install.js"); const { initHelp, installHelp } = require("../lib/help"); const { uninstall, info, run, list } = require("../lib/index"); const init = require("../lib/init").init; -const cinit = require("../lib/init").cinit; +const connectorInit = require("../lib/init").connectorInit; const yargsInteractive = require("yargs-interactive"); const yargs = require("yargs"); @@ -16,6 +16,9 @@ const asTable = require("as-table").configure({ }); const options = { + interactive: { + default: true, + }, name: { type: "input", describe: "Enter connector name", @@ -34,10 +37,6 @@ const options = { type: "input", describe: "Enter asset names in comma separated list", }, - scripts: { - type: "confirm", - describe: "Build scripts?", - }, linear: { type: "confirm", describe: "Create linear task with subtask for package?", @@ -169,39 +168,43 @@ yargs command: "init [package_name]", desc: "Initializes an Argo package inside the current working directory", builder: (yargs) => - yargs.option("force", { - alias: "f", - type: "boolean", - description: "Force the command", - default: true, - }), + yargs + .option("force", { + alias: "f", + type: "boolean", + description: "Force the command", + default: true, + }) + .option("connector", { + alias: "connect", + type: "boolean", + description: "want to add connector the command", + default: false, + }), handler: (argv) => { - init(argv.force, argv.package_name).then((packageName) => { - const re = new RegExp("NAME", "g"); - console.log(initHelp.replace(re, packageName)); - }); + if (argv.connector) { + yargsInteractive() + .usage("$0 [args]") + .interactive(options) + .then((result) => { + connectorInit( + argv.force, + result.name, + result.type, + result.auth, + result.AssetList, + result.scripts, + result.linear + ); + }); + } else { + init(argv.force, argv.package_name).then((packageName) => { + const re = new RegExp("NAME", "g"); + console.log(initHelp.replace(re, packageName)); + }); + } }, }) - .command({ - command: "cinit [package_name]", - desc: "Initializes an connector Argo package inside the current working directory", - builder: () => - yargsInteractive() - .usage("$0 connector-init [args]") - .interactive(options) - .then((result) => { - console.log(result); - cinit( - "false", - result.name, - result.type, - result.auth, - result.AssetList, - result.scripts, - result.linear - ); - }), - }) .command({ command: "list", aliases: ["l"], diff --git a/lib/init.js b/lib/init.js index c469dfc..c6278b8 100644 --- a/lib/init.js +++ b/lib/init.js @@ -59,7 +59,7 @@ function replaceInFile(filePath, searchText, replaceText) { * @param {boolean} force * @param {string} filepath */ -exports.cinit = async function (force, inputPackageName, inputPackageType, auth, assetList, script, linear) { +exports.connectorInit = async function (force, inputPackageName, inputPackageType, auth, assetList, script, linear) { const dirPath = "/Users/mrunmayi.tripathi/atlan/code/marketplace-packages/packages/atlan/test-package"; //const dirPath = process.cwd(); const pathComponents = dirPath.split("/"); @@ -192,11 +192,20 @@ async function createLinearWithSubtask(inputPackageName) { } */ +/* + Add entry to package.json + This step requires dirPath from marketplace packages repo inside atlan/ folder +*/ async function createPackageEntry(dirPath, packageName) { - const filePath = `${dirPath}/../packages/package.json`; - console.info("add connector entry to %s", filePath); - const rawPackageData = await fs.readFileAsync(filePath, "utf-8"); - let packageData = JSON.parse(rawPackageData); - packageData.dependencies[`@atlan/${packageName}`] = "^1.0.0"; - fs.writeFileAsync(filePath, JSON.stringify(packageData), "utf8"); + try { + const filePath = `${dirPath}/../packages/package.json`; + const rawPackageData = await fs.readFileAsync(filePath, "utf-8"); + console.info(rawPackageData); + let packageData = JSON.parse(rawPackageData); + packageData.dependencies[`@atlan/${packageName}`] = "^1.0.0"; + fs.writeFileAsync(filePath, JSON.stringify(packageData), "utf8"); + console.info("add connector entry to %s", filePath); + } catch (e) { + console.log("error", e); + } }