diff --git a/bin/install.js b/bin/install.js index 35c4512..2b9ed9e 100755 --- a/bin/install.js +++ b/bin/install.js @@ -3,7 +3,9 @@ const { install, installGlobal } = require("../lib/install.js"); const { initHelp, installHelp } = require("../lib/help"); const { uninstall, info, run, list } = require("../lib/index"); const init = require("../lib/init").init; +const connectorInit = require("../lib/init").connectorInit; +const yargsInteractive = require("yargs-interactive"); const yargs = require("yargs"); const { cyan, dim, bright } = require("ansicolor"); @@ -13,6 +15,34 @@ const asTable = require("as-table").configure({ dash: bright(cyan("-")), }); +const options = { + interactive: { + default: true, + }, + name: { + type: "input", + describe: "Enter connector name", + }, + type: { + type: "list", + describe: "Enter connector type", + choices: ["BI", "SQL", "ETL"], + }, + auth: { + type: "checkbox", + describe: "Enter oauth type", + choices: ["basic", "api", "oauth2", "aws", "gcp"], + }, + AssetList: { + type: "input", + describe: "Enter asset names in comma separated list", + }, + linear: { + type: "confirm", + describe: "Create linear task with subtask for package?", + }, +}; + yargs .command({ command: "install ", @@ -138,17 +168,41 @@ yargs command: "init [package_name]", desc: "Initializes an Argo package inside the current working directory", builder: (yargs) => - yargs.option("force", { - alias: "f", - type: "boolean", - description: "Force the command", - default: true, - }), + yargs + .option("force", { + alias: "f", + type: "boolean", + description: "Force the command", + default: true, + }) + .option("connector", { + alias: "connect", + type: "boolean", + description: "want to add connector the command", + default: false, + }), handler: (argv) => { - init(argv.force, argv.package_name).then((packageName) => { - const re = new RegExp("NAME", "g"); - console.log(initHelp.replace(re, packageName)); - }); + if (argv.connector) { + yargsInteractive() + .usage("$0 [args]") + .interactive(options) + .then((result) => { + connectorInit( + argv.force, + result.name, + result.type, + result.auth, + result.AssetList, + result.scripts, + result.linear + ); + }); + } else { + init(argv.force, argv.package_name).then((packageName) => { + const re = new RegExp("NAME", "g"); + console.log(initHelp.replace(re, packageName)); + }); + } }, }) .command({ diff --git a/lib/connectors/BI/README.md b/lib/connectors/BI/README.md new file mode 100644 index 0000000..e69de29 diff --git a/lib/connectors/BI/configmaps/default.yaml b/lib/connectors/BI/configmaps/default.yaml new file mode 100644 index 0000000..2637460 --- /dev/null +++ b/lib/connectors/BI/configmaps/default.yaml @@ -0,0 +1,153 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlan-NAME +data: + NAME-api: "10" + config: | + { + "properties": { + "connection": { + "type": "string", + "required": false, + "ui": { + "widget": "connection", + "label": "", + "placeholder": "Connection Name", + "connectionOptions": false + } + }, + "publish-mode": { + "type": "string", + "enum": ["production", "test", "dev"], + "default": "production", + "enumNames": ["Production", "Test", "Development"], + "ui": { + "widget": "select", + "label": "Run Mode", + "grid": 4, + "placeholder": "Connection Mode" + } + }, + "credentials-fetch-strategy": { + "type": "string", + "enum": ["k8s_secret", "credential_guid"], + "default": "credential_guid", + "enumNames": ["k8s Secret Key", "Credential Guid"], + "ui": { + "widget": "select", + "label": "Credential Type", + "placeholder": "Credential Type" + } + }, + "credential-guid": { + "type": "string", + "ui": { + "widget": "credential", + "label": "", + "credentialType": "atlan-connectors-NAME", + "placeholder": "Credential Guid", + "hidden": false + } + }, + "credential-kube-secret-name": { + "type": "string", + "ui": { + "label": "Credential Secret Name", + "placeholder": "Credential Secret Name", + "hidden": true + } + }, + "atlas-auth-type": { + "type": "string", + "enum": ["internal", "apikey"], + "default": "internal", + "enumNames": ["Internal", "API Key"], + "ui": { + "widget": "select", + "label": "Atlas Authentication Type", + "placeholder": "Atlas Authentication Type", + "hidden": true + } + }, + "runtime-properties": { + "type": "object", + "ui": { + "label": "Run time properties", + "hidden": true + } + }, + "include-filter": { + "type": "object", + "additionalProperties": { + "type": "array" + }, + "default": "{}", + "ui": { + "widget": "apitree", + "connectorConfigName": "atlan-connectors-NAME", + "credential": "credential-guid", + "label": "Include Metadata", + "description": "Selected assets only will be processed. Exclude gets preference over include.", + "grid": 4 + } + }, + "exclude-filter": { + "type": "object", + "additionalProperties": { + "type": "array" + }, + "default": "{}", + "ui": { + "widget": "apitree", + "connectorConfigName": "atlan-connectors-NAME", + "credential": "credential-guid", + "label": "Exclude Metadata", + "description": "Selected assets will not be processed.", + "grid": 4 + } + } + }, + "anyOf": [ + { + "properties": { + "credentials-fetch-strategy": { + "const": "k8s_secret" + } + }, + "required": ["credential-kube-secret-name"] + }, + { + "properties": { + "credentials-fetch-strategy": { + "const": "credential_guid" + } + }, + "required": ["credential-guid"] + } + ], + "steps": [ + { + "id": "credential", + "title": "Credential", + "description": "Credential Details", + "properties": [ + "credential-guid" + ] + }, + { + "id": "connection", + "title": "Connection", + "description": "Connection Details", + "properties": [ + "connection" + ] + }, + { + "id": "metadata", + "title": "Metadata", + "description": "Metadata", + "properties": ["include-filter", "exclude-filter"] + } + ] + } \ No newline at end of file diff --git a/lib/connectors/BI/index.js b/lib/connectors/BI/index.js new file mode 100644 index 0000000..fbecdc9 --- /dev/null +++ b/lib/connectors/BI/index.js @@ -0,0 +1,4 @@ +function dummy() { + console.log("don't call this dummy."); +} +module.exports = dummy; diff --git a/lib/connectors/BI/package.json b/lib/connectors/BI/package.json new file mode 100644 index 0000000..bbc52ab --- /dev/null +++ b/lib/connectors/BI/package.json @@ -0,0 +1,53 @@ +{ + "name": "@atlan/NAME", + "version": "0.3.6", + "description": "Package to crawl NAME assets and publish to Atlan for discovery", + "keywords": [ + "NAME", + "bi", + "connector", + "crawler" + ], + "main": "index.js", + "scripts": {}, + "author": { + "name": "Atlan", + "email": "hello@atlan.com", + "url": "https://atlan.com" + }, + "repository": { + "type": "git", + "url": "https://github.com/atlanhq/marketplace-packages.git" + }, + "license": "MIT", + "dependencies": { + "@atlan/crawler": "^2.1.5", + "@atlan/sql-parser": "^0.3.16", + "rest-api": "^0.8.5", + "utils": "^0.2.4" + }, + "bugs": { + "url": "https://atlan.com", + "email": "support@atlan.com" + }, + "config": { + "labels": { + "orchestration.atlan.com/verified": "true", + "orchestration.atlan.com/type": "connector", + "orchestration.atlan.com/source": "NAME", + "orchestration.atlan.com/sourceCategory": "bi", + "orchestration.atlan.com/certified": "true" + }, + "annotations": { + "orchestration.atlan.com/name": "NAME Assets", + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/categories": "NAME,crawler", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/NAME.svg", + "orchestration.atlan.com/marketplaceLink": "https://packages.atlan.com/-/web/detail/@atlan/NAME", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/NAME.svg", + "orchestration.atlan.com/docsUrl": "" + } + } +} diff --git a/lib/connectors/BI/templates/aws.yaml b/lib/connectors/BI/templates/aws.yaml new file mode 100644 index 0000000..22034e3 --- /dev/null +++ b/lib/connectors/BI/templates/aws.yaml @@ -0,0 +1,649 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: atlan-NAME +spec: + entrypoint: main + templates: + - name: main + inputs: + parameters: + #credential + - name: credentials-fetch-strategy + value: "credential_guid" + enum: + - "k8s_secret" + - "credential_guid" + - name: credential-kube-secret-name + value: "" + - name: credential-guid + value: "" + #connection atlan object + - name: connection + value: "" + # extraction + - name: runtime-properties + value: "{}" + - name: fetch-folderless-assets + value: "true" + - name: include-filter + value: "{}" + - name: exclude-filter + value: "{}" + #publish mode + - name: publish-mode + value: "production" + enum: + - "production" + - "test" + - "dev" + + #publish credential + - name: atlas-auth-type + value: "internal" + enum: + - "internal" + - "apikey" + + # enable classification match + - name: auto-classification + value: "false" + enum: + - "true" + - "false" + + - name: attachment-confidence-threshold + value: "0.8" + # Scripts + - name: marketplace-scripts-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplaceScriptsBranch" + optional: true + default: "master" + # Revisions + - name: marketplace-packages-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplacePackagesBranch" + optional: true + default: "master" + dag: + tasks: + - name: extract + template: extract-NAME-metadata + arguments: + parameters: + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: connection-qualified-name + value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + - name: output-prefix + value: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/extracted-metadata/{{workflow.name}}" + - name: threads + value: "2" + - name: fetch-folderless-assets + value: "{{inputs.parameters.fetch-folderless-assets}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: process + template: process-NAME-metadata + depends: "extract.Succeeded" + arguments: + parameters: + - name: connection-qualified-name + value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + - name: fetch-folderless-assets + value: "{{inputs.parameters.fetch-folderless-assets}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: publish + depends: "process.Succeeded" + template: publish-NAME-metadata + arguments: + parameters: + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.publish-mode}}" + - name: source + value: "NAME" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: fetch-folderless-assets + value: "{{inputs.parameters.fetch-folderless-assets}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: exclude-workbook-regex + value: "{{inputs.parameters.exclude-workbook-regex}}" + - name: atlan-web-kube-secret + value: "argo-client-creds" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: atlas-api-uri + value: "http://atlas-service-atlas.atlas.svc.cluster.local/api/atlas/v2" + - name: publish-chunk-size + value: "100" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + + - name: extract-NAME-metadata + inputs: + parameters: + - name: credential-guid + - name: connection-qualified-name + - name: output-prefix + value: "argo-artifacts/{{workflow.namespace}}/{{workflow.name}}" + - name: fetch-folderless-assets + - name: include-filter + - name: exclude-filter + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + - name: statsd-global-tags + dag: + tasks: + - name: extract-assets + template: NAME-api + withItems: + - { "url": "https://NAME..amazonaws.com/accounts//analyses", "name": "analysis" } + arguments: + parameters: + - name: url + value: "{{item.url}}" + - name: output-prefix + value: "{{inputs.parameters.output-prefix}}/{{item.name}}" + - name: NAME-request-config + value: | + { + "headers": { + "Content-Type":"application/x-amz-json-1.1" + }, + "json": { + "MaxResults": 100 + } + } + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: output-chunk-size + value: "1000" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + - name: NAME-execution-script + value: | + if state == ExecutionState.RAW_INPUT_PROCESS: + creds_arr = secrets["result-0.json"].split("""\n""") + region = '' + for cred in creds_arr: + LOGGER.debug("cred %s" % cred) + if 'AWS_REGION' in cred: + region = cred.split('=')[-1].replace('"', '') + region = region.strip() + + request_config['url'] = request_config['url'].replace('', region) + + accountid = '' + for cred in creds_arr: + if 'AWS_ACCOUNT_ID' in cred: + accountid = cred.split('=')[-1].replace('"', '') + + request_config['url'] = request_config['url'].replace('', accountid) + + + if state == ExecutionState.OUTPUT_PROCESS: + output = json.loads(output) + + if state == ExecutionState.API_POST: + next_token = response.json().get('NextToken', None) + if not next_token: + stop = True + else: + request_config['json']['NextToken'] = next_token + + if state == ExecutionState.API_FAIL: + failure_handler=FailureHandler.RETRY + + - name: process-NAME-metadata + inputs: + parameters: + - name: connection-qualified-name + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + - name: fetch-folderless-assets + - name: include-filter + - name: exclude-filter + dag: + tasks: + # Process NAME metadata and generate files for transformer + - name: process-metadata + template: process-metadata-template + arguments: + parameters: + - name: output-prefix + value: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/processed-metadata/{{workflow.name}}" + - name: git-kube-secret-name + value: "{{inputs.parameters.git-kube-secret-name}}" + - name: git-kube-ssh-key + value: "{{inputs.parameters.git-kube-ssh-key}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + - name: fetch-folderless-assets + value: "{{inputs.parameters.fetch-folderless-assets}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: connection-qualified-name + value: "{{inputs.parameters.connection-qualified-name}}" + artifacts: + - name: extracted-metadata + s3: + key: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/extracted-metadata/{{workflow.name}}" + + - name: process-metadata-template + inputs: + parameters: + - name: output-prefix + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: fetch-folderless-assets + - name: include-filter + - name: exclude-filter + - name: connection-qualified-name + + artifacts: + - name: extracted-metadata + path: /tmp/extracted-metadata + - name: scripts + path: /tmp/marketplace-scripts + git: + repo: git@github.com:atlanhq/marketplace-scripts + revision: "{{inputs.parameters.marketplace-scripts-revision}}" + insecureIgnoreHostKey: true + depth: 1 + sshPrivateKeySecret: + name: "{{inputs.parameters.git-kube-secret-name}}" + key: "{{inputs.parameters.git-kube-ssh-key}}" + - name: connection-cache + optional: true + path: "/tmp/connection-cache" + s3: + key: "connection-cache" + outputs: + artifacts: + - name: processed-metadata + path: /tmp/processed-metadata + s3: + key: "{{inputs.parameters.output-prefix}}" + archive: + none: { } + container: + image: ghcr.io/atlanhq/marketplace-scripts-base:0.1.10 + imagePullPolicy: IfNotPresent + workingDir: "/tmp/marketplace-scripts" + command: [ "python" ] + args: + - "-m" + - "marketplace_scripts.NAME.metadata" + - "--metadata-prefix" + - "/tmp/extracted-metadata" + - "--cache-prefix" + - "/tmp/connection-cache" + - "--output-prefix" + - "/tmp/processed-metadata" + - "--exclude-filter" + - "{{inputs.parameters.exclude-filter}}" + - "--include-filter" + - "{{inputs.parameters.include-filter}}" + - "--fetch-folderless-assets" + - "{{inputs.parameters.fetch-folderless-assets}}" + - "--connection-qualified-name" + - "{{inputs.parameters.connection-qualified-name}}" + + - name: publish-NAME-metadata + inputs: + parameters: + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + - name: connection + - name: mode + - name: source + - name: atlas-api-uri + - name: heracles-uri + - name: atlan-web-kube-secret + - name: atlas-auth-type + - name: publish-chunk-size + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + dag: + tasks: + # Run atlan-crawler/generic-publish template + - name: publish_metadata + templateRef: + name: atlan-crawler + template: generic-publish + arguments: + artifacts: + - name: data + s3: + key: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/processed-metadata/{{workflow.name}}" + - name: transformer-config + raw: + data: | + { + "external_map": { + "crawler_name": "{{workflow.labels.workflows.argoproj.io/workflow-template}}", + "tenant_id": "{{workflow.namespace}}", + "integration_name": "{{inputs.parameters.source}}", + "workflow_name": "{{workflow.name}}", + "connection_name": "{{=jsonpath(inputs.parameters.connection, '$.attributes.name')}}", + "connection_qn": "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + }, + "output_prefix": "/tmp/entities", + "templates_root": "/tmp/templates", + "transformation_config": [ + { + "input_file_pattern": "/tmp/inputs/ASSET_NAME.json", + "template": "packages/atlan/NAME/transformers/ASSET_NAME.jinja2", + "output_file_prefix": "ASSET_NAME/ASSET_NAME", + "output_chunk_size": 10000 + } + ] + } + parameters: + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.mode}}" + - name: source + value: "{{inputs.parameters.source}}" + - name: raw-input-file-sort + value: "" + - name: raw-input-folder-sort + value: "" + - name: raw-input-file-pattern + value: "**/*.json" + - name: publish-chunk-size + value: "{{inputs.parameters.publish-chunk-size}}" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + + - name: NAME-api + synchronization: + semaphore: + configMapKeyRef: + name: atlan-NAME + key: NAME-api + inputs: + artifacts: + - name: raw-input + path: "/tmp/input/" + optional: true + - name: raw-input-file + path: "/tmp/input/input.json" + optional: true + parameters: + - name: threads + value: "1" + - name: credential-guid + - name: NAME-request-config + value: | + { + "headers": { + "Content-Type":"application/x-amz-json-1.1" + }, + "json": { + "MaxResults": 100 + } + } + - name: paginate + value: "true" + enum: + - "true" + - "false" + - name: NAME-output-key + value: "TableList" + - name: url + - name: NAME-execution-script + value: | + if "{{inputs.parameters.threads}}" != "1": + logger.error("Error! You must pass custom execution script from parent for thread count > 1") + raise Exception("Invalid thread count for offset based pagination!") + if state == ExecutionState.RAW_INPUT_PROCESS: + creds_arr = secrets["result-0.json"].split("""\n""") + region = '' + for cred in creds_arr: + if 'AWS_REGION' in cred: + region = cred.split('=')[-1].replace('"', '') + + request_config['url'] = request_config['url'].replace('', region) + + if state == ExecutionState.OUTPUT_PROCESS: + output=json.loads(output)['{{inputs.parameters.NAME-output-key}}'] + + if state == ExecutionState.API_POST: + next_token = response.json().get('NextToken', None) + if not next_token: + stop = True + else: + request_config['json']['NextToken'] = next_token + + if state == ExecutionState.API_FAIL: + failure_handler=FailureHandler.RETRY + - name: output-chunk-size + value: 0 + - name: raw-input-file-pattern + value: "" + - name: raw-input-paginate + value: "0" + - name: raw-input-multiline + value: "False" + - name: pagination-wait-time + value: "0" + - name: kube-secret-name + value: "argo-client-creds" + - name: client-id-env + value: "login" + - name: client-secret-env + value: "password" + - name: token-url-env + value: "host" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},bot=atlan-NAME" + - name: output-prefix + value: "argo-artifacts/{{workflow.namespace}}/{{workflow.name}}/{{pod.name}}/" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: init-execution-script + value: | + if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): + LOGGER.debug('Heracles is unavailable. Performing retry with back-off') + failure_handler = FailureHandler.RETRY + if state == ExecutionState.OUTPUT_PROCESS: + + credential = json.loads(output) + output = "" + for key, value in credential.items(): + if key == "username": + output += f"""AWS_ACCESS_KEY_ID=\"{str(value)}\"\n""" + elif key == "password": + output += f"""AWS_SECRET_ACCESS_KEY=\"{str(value)}\"\n""" + elif key == "host": + if value is not None: + region = value.split('.')[-3] + output += f"""AWS_REGION=\"{str(region)}\"\n""" + elif key == "extra": + if "region" in value: + region = value.get("region", "") + output += f"""AWS_REGION=\"{str(region)}\"\n""" + if "accountid" in value: + accountid = value.get("accountid", "") + output += f"""AWS_ACCOUNT_ID=\"{str(accountid)}\"\n""" + if "aws_role_arn" in value: + output += f"""AWS_ROLE_ARN=\"{str(value.get("aws_role_arn", ""))}\"\n""" + if "aws_external_id" in value: + output += f"""AWS_EXTERNAL_ID=\"{str(value.get("aws_external_id", ""))}\"\n""" + else: + continue + + if state == ExecutionState.API_POST: + stop = True + outputs: + artifacts: + - name: success + path: "/tmp/rest/success" + s3: + key: "{{inputs.parameters.output-prefix}}/success" + archive: + none: { } + - name: failure + path: "/tmp/rest/failure" + archive: + none: { } + s3: + key: "{{inputs.parameters.output-prefix}}/failure" + parameters: + - name: success-num-files + valueFrom: + path: "/tmp/rest/success/result-gen.txt" + - name: failure-num-files + valueFrom: + path: "/tmp/rest/failure/result-gen.txt" + volumes: + - name: credentials + emptyDir: { } + container: + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "./entrypoint.sh" ] + volumeMounts: + - name: credentials + mountPath: /tmp/credentials + imagePullPolicy: IfNotPresent + args: [ + "python3", "main.py","GET", "{{inputs.parameters.url}}", + "--raw-input-paginate", "{{inputs.parameters.raw-input-paginate}}", + "--raw-input-multiline", "{{inputs.parameters.raw-input-multiline}}", + "--threads", "{{inputs.parameters.threads}}", + "--raw-input-file-pattern", "{{inputs.parameters.raw-input-file-pattern}}", + "--request-config", "{{inputs.parameters.NAME-request-config}}", + "--execution-script", "{{inputs.parameters.NAME-execution-script}}", + "--secrets-path", "/tmp/credentials/success/*.json", + "--auth-type", "aws", + "--auth-aws-service", "NAME", + "--auth-aws-region", "AWS_REGION", + "--output-chunk-size", "{{inputs.parameters.output-chunk-size}}", + "--output-file-prefix", "/tmp/rest", + "--pagination-wait-time", "0", + "--max-retries", "10", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] + initContainers: + - name: fetch-credentials + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "python3", "main.py" ] + env: + - name: OAUTHLIB_INSECURE_TRANSPORT + value: "1" + - name: CLIENT_ID + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-id-env}}" + - name: CLIENT_SECRET + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-secret-env}}" + - name: TOKEN_URL + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.token-url-env}}" + mirrorVolumeMounts: true + args: [ + "GET", + "{{inputs.parameters.heracles-uri}}/credentials/{{inputs.parameters.credential-guid}}/use", + "--raw-input", "{}", + "--raw-input-file-sort", "", + "--raw-input-multiline", "False", + "--execution-script", "{{inputs.parameters.init-execution-script}}", + "--raw-input-paginate", "0", + "--auth-type", "oauth2", + "--auth-oauth2-type", "client_credentials", + "--auth-oauth2-impersonate-user", "{{=sprig.dig('labels', 'workflows', 'argoproj', 'io/creator', '', workflow)}}", + "--auth-oauth2-client-credentials-client-id", "CLIENT_ID", + "--auth-oauth2-client-credentials-secret", "CLIENT_SECRET", + "--auth-oauth2-client-credentials-token-url", "TOKEN_URL", + "--output-chunk-size", "0", + "--output-file-prefix", "/tmp/credentials", + "--pagination-wait-time", "0", + "--max-retries", "10", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] diff --git a/lib/connectors/BI/templates/basic+jwt.yaml b/lib/connectors/BI/templates/basic+jwt.yaml new file mode 100644 index 0000000..ab64710 --- /dev/null +++ b/lib/connectors/BI/templates/basic+jwt.yaml @@ -0,0 +1,633 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: atlan-NAME +spec: + entrypoint: main + templates: + - name: main + inputs: + parameters: + - name: credentials-fetch-strategy + value: "credential_guid" + enum: + - "k8s_secret" + - "credential_guid" + + - name: credential-kube-secret-name + value: "{{workflow.name}}-credential-secret" + + - name: credential-guid + value: "" + + #connection atlan object + - name: connection + + # NAME api version. + - name: api-version + value: "v52.0" + + #publish mode + - name: publish-mode + value: "production" + enum: + - "production" + - "test" + - "dev" + + #publish credential + - name: atlas-auth-type + value: "internal" + enum: + - "internal" + - "apikey" + + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + + - name: auto-classification + value: "false" + enum: + - "true" + - "false" + + - name: attachment-confidence-threshold + value: "0.8" + + - name: fetch-reports + value: "true" + + # Revisions + - name: marketplace-scripts-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplaceScriptsBranch" + optional: true + default: "master" + - name: marketplace-packages-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplacePackagesBranch" + optional: true + default: "master" + + dag: + tasks: + - name: fetch-credentials + templateRef: + name: rest-api + template: oauth2-client-credentials + arguments: + parameters: + - name: method + value: GET + - name: url + value: "http://heracles-service.heracles.svc.cluster.local/credentials/{{inputs.parameters.credential-guid}}/use" + - name: request-config + value: | + { + "headers": { + "user-id": "{{workflow.labels.workflows.argoproj.io/creator}}" + } + } + - name: kube-secret-name + value: "argo-client-creds" + - name: max-retries + value: "10" + - name: client-id-env + value: "login" + - name: client-secret-env + value: "password" + - name: token-url-env + value: "host" + - name: execution-script + value: | + if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): + LOGGER.debug('Heracles is unavailable. Performing retry with back-off') + failure_handler = FailureHandler.RETRY + if state == ExecutionState.OUTPUT_PROCESS: + credential = json.loads(output) + auth_type = credential["authType"] + + if auth_type == "basic": + oauth2_type = "resource_owner_password" + elif auth_type == "jwt": + oauth2_type = "jwt_bearer" + + output = { + "oauth2_type": oauth2_type, + } + + if state == ExecutionState.API_POST: + stop=True + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},bot=atlan-NAME" + - name: auth-type-to-param + dependencies: + - fetch-credentials + templateRef: + name: utils + template: artifact-to-key-param + arguments: + parameters: + - name: key + value: "oauth2_type" + artifacts: + - name: input + from: "{{tasks.fetch-credentials.outputs.artifacts.success}}" + subPath: "result-0.json" + - name: extract + dependencies: + - auth-type-to-param + template: extract-NAME-metadata + arguments: + parameters: + - name: credentials-fetch-strategy + value: "{{inputs.parameters.credentials-fetch-strategy}}" + - name: credential-kube-secret-name + value: "{{inputs.parameters.credential-kube-secret-name}}" + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: oauth2-type + value: "{{tasks.auth-type-to-param.outputs.parameters.output}}" + - name: connection-qualified-name + value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: fetch-reports + value: "{{inputs.parameters.fetch-reports}}" + + - name: process-objects + template: process-NAME-described-assets + dependencies: + - extract + withParam: "{{tasks.extract.outputs.parameters.org-host-name}}" + arguments: + parameters: + - name: host-name + value: "{{item.host}}" + - name: output-prefix + value: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/metadata-extract/{{workflow.name}}/processed-metadata" + - name: process-chunk-size + value: 10000 + artifacts: + - name: input-raw-metadata + s3: + key: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/metadata-extract/{{workflow.name}}/raw-metadata" + archive: + none: { } + - name: scripts + git: + repo: git@github.com:atlanhq/marketplace-scripts + insecureIgnoreHostKey: true + depth: 1 + revision: "{{inputs.parameters.marketplace-scripts-revision}}" + sshPrivateKeySecret: + name: "{{inputs.parameters.git-kube-secret-name}}" + key: "{{inputs.parameters.git-kube-ssh-key}}" + - name: publish + dependencies: + - process-objects + template: publish-NAME-metadata + arguments: + parameters: + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.publish-mode}}" + - name: source + value: "NAME" + - name: atlas-api-uri + value: "http://atlas-service-atlas.atlas.svc.cluster.local/api/atlas/v2" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: processed-data-key + value: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/metadata-extract/{{workflow.name}}/processed-metadata" + - name: atlan-web-kube-secret + value: "argo-client-creds" + - name: publish-chunk-size + value: "25" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + + - name: extract-NAME-metadata + inputs: + parameters: + - name: credentials-fetch-strategy + - name: credential-kube-secret-name + - name: credential-guid + - name: oauth2-type + - name: connection-qualified-name + - name: heracles-uri + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: fetch-reports + outputs: + parameters: + - name: org-host-name + valueFrom: + default: "[]" + parameter: "{{tasks.convert-to-param.outputs.parameters.output}}" + dag: + tasks: + - name: extract-organization + template: NAME-api + arguments: + parameters: + - name: url + value: "/services/data/v52.0/query?q=select%20Fields(All)%20from%20Organization%20limit%201" + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: oauth2-type + value: "{{inputs.parameters.oauth2-type}}" + - name: execution-script + value: | + if state == ExecutionState.RAW_INPUT_PROCESS: + creds_arr = secrets["result-0.json"].split("""\n""") + host = '' + for cred in creds_arr: + if 'HOST' in cred: + host = cred.split('=')[-1].replace('"', '') + request_config['url'] = request_config['url'].replace('', host) + store['host'] = host + + if state == ExecutionState.OUTPUT_PROCESS: + _response = json.loads(output).get('records', []) + + if len(_response) > 0: + output = _response[0] + output['host'] = store['host'] + + if state == ExecutionState.API_POST: + stop = True + + if state == ExecutionState.API_FAIL: failure_handler=FailureHandler.RETRY + - name: output-chunk-size + value: 1 + - name: output-prefix + value: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/metadata-extract/{{workflow.name}}/raw-metadata/described-organization/0" + + - name: process-NAME-metadata + inputs: + parameters: + - name: host-name + - name: output-prefix + - name: process-chunk-size + artifacts: + - name: input-raw-metadata + path: "/tmp/input" + - name: scripts + path: /tmp/marketplace-scripts + outputs: + artifacts: + - name: output + path: "/tmp/output" + s3: + key: "{{inputs.parameters.output-prefix}}" + archive: + none: { } + container: + image: ghcr.io/atlanhq/marketplace-scripts-base:0.1.11 + workingDir: "/tmp/marketplace-scripts" + command: [ "python" ] + args: + - "-m" + - "marketplace_scripts.NAME.main" + - "--host-name" + - "{{inputs.parameters.host-name}}" + - "--input-dir" + - "/tmp/input" + - "--output-prefix" + - "/tmp/output/" + - "--chunk-size" + - "{{inputs.parameters.process-chunk-size}}" + + - name: publish-NAME-metadata + inputs: + parameters: + - name: connection + - name: mode + - name: source + - name: atlas-api-uri + - name: heracles-uri + - name: processed-data-key + - name: atlan-web-kube-secret + - name: atlas-auth-type + - name: publish-chunk-size + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + dag: + tasks: + - name: publish + templateRef: + name: atlan-crawler + template: generic-publish + arguments: + artifacts: + - name: data + s3: + key: "{{inputs.parameters.processed-data-key}}" + - name: transformer-config + raw: + data: | + { + "external_map": { + "crawler_name": "{{workflow.labels.workflows.argoproj.io/workflow-template}}", + "tenant_id": "{{workflow.namespace}}", + "integration_name": "{{inputs.parameters.source}}", + "workflow_name": "{{workflow.name}}", + "connection_name": "{{=jsonpath(inputs.parameters.connection, '$.attributes.name')}}", + "connection_qn": "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + }, + "output_prefix": "/tmp/entities/", + "templates_root": "/tmp/templates/packages", + "transformation_config": [ + { + "input_file_pattern": "/tmp/inputs/organization*.json", + "template": "atlan/NAME/transformers/organization.jinja2", + "output_file_prefix": "organization" + }, + { + "input_file_pattern": "/tmp/inputs/objects*.json", + "template": "atlan/NAME/transformers/object.jinja2", + "output_file_prefix": "objects", + "ignore": true + }, + { + "input_file_pattern": "/tmp/inputs/fields*.json", + "template": "atlan/NAME/transformers/field.jinja2", + "output_file_prefix": "fields", + "ignore": true + }, + { + "input_file_pattern": "/tmp/inputs/reports*.json", + "template": "atlan/NAME/transformers/report.jinja2", + "output_file_prefix": "reports", + "ignore": true + }, + { + "input_file_pattern": "/tmp/inputs/dashboards*.json", + "template": "atlan/NAME/transformers/dashboard.jinja2", + "output_file_prefix": "dashboards", + "ignore": true + } + ] + } + parameters: + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.mode}}" + - name: source + value: "{{inputs.parameters.source}}" + - name: raw-input-file-sort + value: "organization,objects,fields,reports,dashboards" + - name: publish-chunk-size + value: "{{inputs.parameters.publish-chunk-size}}" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + + - name: NAME-api + synchronization: + semaphore: + configMapKeyRef: + name: atlan-NAME + key: api + volumes: + - name: atlan-NAME-workflow-directory + ephemeral: + volumeClaimTemplate: + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 60Gi + - name: credentials + emptyDir: { } + inputs: + artifacts: + - name: raw-input + path: /tmp/input + optional: true + parameters: + - name: credential-guid + - name: oauth2-type + - name: page-size + value: "10" + - name: url + - name: request-config + value: | + { + "headers": { + "Content-Type": "application/x-www-form-urlencoded" + } + } + - name: execution-script + - name: output-chunk-size + value: "0"sourceCategory + - name: kube-secret-name + value: "argo-client-creds" + - name: client-id-env + value: "login" + - name: client-secret-env + value: "password" + - name: token-url-env + value: "host" + - name: raw-input-file-pattern + value: "" + - name: raw-input-paginate + value: 0 + - name: raw-input-multiline + value: "False" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},bot=atlan-NAME" + - name: output-prefix + value: "argo-artifacts/{{workflow.namespace}}/{{workflow.name}}/{{pod.name}}" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: init-execution-script + value: | + if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): + LOGGER.debug('Heracles is unavailable. Performing retry with back-off') + failure_handler = FailureHandler.RETRY + if state == ExecutionState.OUTPUT_PROCESS: + credential = json.loads(output) + auth_type = credential.get('authType', '') + extra_params = credential.get('extra', {}) + is_sandbox = extra_params.get('is_sandbox', False) + token_url = 'test.NAME.com' if is_sandbox else 'login.NAME.com' + + output = f"""USERNAME="{credential.get('username', '')}" + HOST="{credential.get('host', '')}" + TOKEN_URL="https://{token_url}/services/oauth2/token" + CLIENT_ID="{extra_params.get('client_id', '')}" + """ + + if auth_type == 'basic': + password = credential.get('password', '').replace('$', '\$') + output += f"""PASSWORD="{password}" + CLIENT_SECRET="{extra_params.get('client_secret', '')}" + """ + elif auth_type == 'jwt': + output += f"""PRIVATE_KEY="{extra_params.get('private_key', '')}" + """ + + if state == ExecutionState.API_POST: + stop = True + outputs: + artifacts: + - name: success + path: "/tmp/rest/success" + s3: + key: "{{inputs.parameters.output-prefix}}/success" + archive: + none: { } + - name: failure + path: "/tmp/rest/failure" + archive: + none: { } + s3: + key: "{{inputs.parameters.output-prefix}}/failure" + parameters: + - name: success-num-files + valueFrom: + path: "/tmp/rest/success/result-gen.txt" + - name: failure-num-files + valueFrom: + path: "/tmp/rest/failure/result-gen.txt" + container: + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "./entrypoint.sh" ] + volumeMounts: + - name: atlan-NAME-workflow-directory + mountPath: /tmp + - name: credentials + mountPath: /tmp/credentials + imagePullPolicy: IfNotPresent + args: [ + "python3", "main.py", "GET", "{{inputs.parameters.url}}", + "--raw-input-paginate", "{{inputs.parameters.raw-input-paginate}}", + "--raw-input-file-pattern", "{{inputs.parameters.raw-input-file-pattern}}", + "--raw-input-multiline", "{{inputs.parameters.raw-input-multiline}}", + "--request-config", "{{inputs.parameters.request-config}}", + "--execution-script", "{{inputs.parameters.execution-script}}", + "--secrets-path", "/tmp/credentials/success/*.json", + "--auth-type", "oauth2", + "--auth-oauth2-type", "{{inputs.parameters.oauth2-type}}", + "--auth-oauth2-resource-owner-password-client-id", "CLIENT_ID", + "--auth-oauth2-resource-owner-password-secret", "CLIENT_SECRET", + "--auth-oauth2-resource-owner-password-token-url", "TOKEN_URL", + "--auth-oauth2-resource-owner-password-username", "USERNAME", + "--auth-oauth2-resource-owner-password-password", "PASSWORD", + "--auth-oauth2-jwt-bearer-client-id", "CLIENT_ID", + "--auth-oauth2-jwt-bearer-private-key", "PRIVATE_KEY", + "--auth-oauth2-jwt-bearer-token-url", "TOKEN_URL", + "--auth-oauth2-jwt-bearer-username", "USERNAME", + "--auth-oauth2-jwt-bearer-token-expiry", "{{=sprig.unixEpoch(sprig.dateModify('3m', sprig.now()))}}", + "--output-chunk-size", "{{inputs.parameters.output-chunk-size}}", + "--output-file-prefix", "/tmp/rest/", + "--pagination-wait-time", "10", + "--max-retries", "3", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] + + initContainers: + - name: fetch-credentials + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "python3", "main.py" ] + env: + - name: OAUTHLIB_INSECURE_TRANSPORT + value: "1" + - name: CLIENT_ID + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-id-env}}" + - name: CLIENT_SECRET + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-secret-env}}" + - name: TOKEN_URL + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.token-url-env}}" + mirrorVolumeMounts: true + args: [ + "GET", + "{{inputs.parameters.heracles-uri}}/credentials/{{inputs.parameters.credential-guid}}/use", + "--raw-input", "{}", + "--raw-input-file-pattern", "", + "--raw-input-file-sort", "", + "--raw-input-multiline", "f", + "--execution-script", "{{inputs.parameters.init-execution-script}}", + "--raw-input-paginate", "0", + "--auth-type", "oauth2", + "--auth-oauth2-type", "client_credentials", + "--auth-oauth2-impersonate-user", "{{=sprig.dig('labels', 'workflows', 'argoproj', 'io/creator', '', workflow)}}", + "--auth-oauth2-client-credentials-client-id", "CLIENT_ID", + "--auth-oauth2-client-credentials-secret", "CLIENT_SECRET", + "--auth-oauth2-client-credentials-token-url", "TOKEN_URL", + "--output-chunk-size", "0", + "--output-file-prefix", "/tmp/credentials", + "--pagination-wait-time", "0", + "--max-retries", + "10", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] + diff --git a/lib/connectors/BI/templates/oauth.yaml b/lib/connectors/BI/templates/oauth.yaml new file mode 100644 index 0000000..152899f --- /dev/null +++ b/lib/connectors/BI/templates/oauth.yaml @@ -0,0 +1,581 @@ +# Workflow Templates +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: atlan-NAME +spec: + entrypoint: main + templates: + - name: main + inputs: + parameters: + # Credential + - name: credentials-fetch-strategy + value: "credential_guid" + enum: + - "credential_guid" + - "k8s_secret" + - name: credential-guid + value: "" + # Connection Entity + - name: connection + - name: atlas-auth-type + value: "internal" + enum: + - "internal" + - "apikey" + # Include/Exclude Filters + - name: include-filter + value: "{}" + - name: exclude-filter + value: "{}" + # Publish Mode + - name: publish-mode + value: "production" + enum: + - "production" + - "dev" + - "test" + # Revisions + - name: marketplace-scripts-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplaceScriptsBranch" + optional: true + default: "master" + - name: marketplace-packages-revision + valueFrom: + configMapKeyRef: + name: atlan-runtime-packages-config + key: "marketplacePackagesBranch" + optional: true + default: "master" + dag: + tasks: + # Extract metadata from NAME + - name: extract + template: extract-NAME-metadata + arguments: + parameters: + - name: credentials-fetch-strategy + value: "{{inputs.parameters.credentials-fetch-strategy}}" + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: connection-qualified-name + value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + # Process the extracted metadata + - name: process + template: process-NAME-metadata + depends: "extract.Succeeded" + arguments: + parameters: + - name: connection-qualified-name + value: "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + + - name: publish + depends: "process.Succeeded" + template: publish-NAME-metadata + arguments: + parameters: + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.publish-mode}}" + - name: source + value: "NAME" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: include-filter + value: "{{inputs.parameters.include-filter}}" + - name: exclude-filter + value: "{{inputs.parameters.exclude-filter}}" + - name: atlan-web-kube-secret + value: "argo-client-creds" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: atlas-api-uri + value: "http://atlas-service-atlas.atlas.svc.cluster.local/api/atlas/v2" + - name: publish-chunk-size + value: "100" + - name: git-kube-secret-name + value: "git-ssh" + - name: git-kube-ssh-key + value: "private-key" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},connector=NAME,package=atlan-NAME,connection={{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}},template={{workflow.labels.workflows.argoproj.io/workflow-template}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + + - name: extract-NAME-metadata + inputs: + parameters: + - name: credentials-fetch-strategy + - name: credential-guid + - name: connection-qualified-name + - name: include-filter + - name: exclude-filter + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: heracles-uri + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: marketplace-scripts-revision + outputs: + parameters: + - name: auth-type-and-host + valueFrom: + parameter: "{{tasks.auth-type-and-host-to-param.outputs.parameters.output}}" + dag: + tasks: + # Fetch the credential + - name: fetch-credentials + when: "{{inputs.parameters.credentials-fetch-strategy}} == credential_guid" + templateRef: + name: rest-api + template: oauth2-client-credentials + arguments: + parameters: + - name: method + value: GET + - name: url + value: "{{inputs.parameters.heracles-uri}}/credentials/{{inputs.parameters.credential-guid}}/use" + - name: request-config + value: | + { + "headers": { + "user-id": "{{workflow.labels.workflows.argoproj.io/creator}}" + } + } + - name: kube-secret-name + value: "argo-client-creds" + - name: max-retries + value: "10" + - name: client-id-env + value: "login" + - name: client-secret-env + value: "password" + - name: token-url-env + value: "host" + - name: execution-script + value: | + if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): + LOGGER.debug("Heracles is unavailable. Performing retry with back-off") + failure_handler = FailureHandler.RETRY + + if state == ExecutionState.OUTPUT_PROCESS: + credential = json.loads(output) + output = { + "authType": credential["authType"], + "host": credential["host"] + } + + if state == ExecutionState.API_POST: + stop=True + - name: statsd-host + value: "{{inputs.parameters.statsd-host}}" + - name: statsd-port + value: "{{inputs.parameters.statsd-port}}" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + # Convert that `fetch-credentials` output from an artifact to a parameter + - name: auth-type-and-host-to-param + dependencies: + - fetch-credentials + templateRef: + name: utils + template: artifact-to-direct-param + arguments: + artifacts: + - name: input + from: "{{tasks.fetch-credentials.outputs.artifacts.success}}" + subPath: "result-0.json" + # Get all the workbooks + - name: fetch-ASSET_NAME + dependencies: + - auth-type-and-host-to-param + template: api-request + arguments: + parameters: + - name: auth-type + value: "{{=jsonpath(tasks['auth-type-and-host-to-param'].outputs.parameters.output, '$.authType')}}" + - name: credential-guid + value: "{{inputs.parameters.credential-guid}}" + - name: heracles-uri + value: "{{inputs.parameters.heracles-uri}}" + - name: method + value: GET + - name: url + value: "<>" + - name: request-config + value: | + { + "params": { + "page": 0, + "limit": 100 + } + } + - name: kube-secret-name + value: "argo-client-creds" + - name: output-prefix + value: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/extracted-metadata/{{workflow.name}}/ASSET_NAME" + - name: output-chunk-size + value: "1000" + - name: execution-script + value: | + if state == ExecutionState.RAW_INPUT_PROCESS: + creds_arr = secrets["result-0.json"].split("""\n""") + host = "" + for cred in creds_arr: + if "HOST" in cred: + host = cred.split('=')[-1].replace('"', '') + + url = "{{inputs.parameters.url}}" + url = url.replace("", host) + LOGGER.debug(f"URL: {url}") + request_config["url"] = url + + if state == ExecutionState.OUTPUT_PROCESS: + output = json.loads(output) + + if state == ExecutionState.API_POST: + workbook_response = response.json() + if not workbook_response.get("hasMore"): + stop = True + else: + next_page = workbook_response.get("nextPage") + request_config["params"] = { + "page": next_page, + "limit": 100 + } + if state == ExecutionState.API_FAIL: + if response.status_code == 500: + failure_handler = FailureHandler.NONE + else: + failure_handler = FailureHandler.RETRY + - name: statsd-host + value: "{{inputs.parameters.statsd-host}}" + - name: statsd-port + value: "{{inputs.parameters.statsd-port}}" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + + - name: process-NAME-metadata + inputs: + parameters: + - name: connection-qualified-name + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: marketplace-scripts-revision + dag: + tasks: + - name: process-metadata + template: process-metadata + arguments: + parameters: + - name: output-prefix + value: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/processed-metadata/{{workflow.name}}" + - name: git-kube-secret-name + value: "{{inputs.parameters.git-kube-secret-name}}" + - name: git-kube-ssh-key + value: "{{inputs.parameters.git-kube-ssh-key}}" + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + artifacts: + - name: extracted-metadata + s3: + key: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/extracted-metadata/{{workflow.name}}" + - name: parsed-queries + s3: + key: "argo-artifacts/{{inputs.parameters.connection-qualified-name}}/parsed-queries/{{workflow.name}}" + + - name: publish-NAME-metadata + inputs: + parameters: + - name: connection + - name: mode + - name: source + - name: atlas-api-uri + - name: heracles-uri + - name: atlan-web-kube-secret + - name: atlas-auth-type + - name: publish-chunk-size + - name: git-kube-secret-name + - name: git-kube-ssh-key + - name: statsd-host + - name: statsd-port + - name: statsd-global-tags + - name: marketplace-scripts-revision + - name: marketplace-packages-revision + dag: + tasks: + # Run atlan-crawler/publish-bi template + - name: publish + templateRef: + name: atlan-crawler + template: publish-bi + arguments: + artifacts: + - name: data + s3: + key: "argo-artifacts/{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}/processed-metadata/{{workflow.name}}" + - name: transformer-config + raw: + data: | + { + "external_map": { + "crawler_name": "{{workflow.labels.workflows.argoproj.io/workflow-template}}", + "tenant_id": "{{workflow.namespace}}", + "integration_name": "{{inputs.parameters.source}}", + "workflow_name": "{{workflow.name}}", + "connection_name": "{{=jsonpath(inputs.parameters.connection, '$.attributes.name')}}", + "connection_qn": "{{=jsonpath(inputs.parameters.connection, '$.attributes.qualifiedName')}}" + }, + "output_prefix": "/tmp/entities", + "templates_root": "/tmp/templates/packages", + "transformation_config": [ + { + "input_file_pattern": "/tmp/inputs/<>.json", + "template": "atlan/NAME/transformers/<>.jinja2", + "output_file_prefix": "<>/<>", + "output_chunk_size": 15000 + } + ] + } + parameters: + - name: connection + value: "{{inputs.parameters.connection}}" + - name: mode + value: "{{inputs.parameters.mode}}" + - name: source + value: "{{inputs.parameters.source}}" + - name: raw-input-file-sort + value: "" + - name: raw-input-folder-sort + value: "" + - name: raw-input-file-pattern + value: "**/*.json" + - name: publish-chunk-size + value: "{{inputs.parameters.publish-chunk-size}}" + - name: atlas-auth-type + value: "{{inputs.parameters.atlas-auth-type}}" + - name: hierarchy + value: | + [ + [ + { + "name":"", + "file_pattern": "" + } + ] + ] + - name: marketplace-scripts-revision + value: "{{inputs.parameters.marketplace-scripts-revision}}" + - name: marketplace-packages-revision + value: "{{inputs.parameters.marketplace-packages-revision}}" + - name: statsd-global-tags + value: "{{inputs.parameters.statsd-global-tags}}" + + - name: NAME-api + synchronization: + semaphore: + configMapKeyRef: + name: atlan-NAME + key: api + volumes: + - name: credentials + emptyDir: { } + inputs: + artifacts: + - name: raw-input + path: /tmp/input + optional: true + - name: raw-input-file + path: "/tmp/input/input.json" + optional: true + parameters: + - name: credential-guid + - name: page-size + value: "10" + - name: method + value: "GET" + - name: url + - name: request-config + value: "{}" + - name: execution-script + - name: output-chunk-size + value: 100 + - name: kube-secret-name + value: "argo-client-creds" + - name: client-id-env + value: "login" + - name: client-secret-env + value: "password" + - name: token-url-env + value: "host" + - name: raw-input-paginate + value: 0 + - name: raw-input-file-pattern + value: "" + - name: raw-input-multiline + value: "False" + - name: statsd-host + value: "prometheus-statsd-exporter.monitoring.svc.cluster.local" + - name: statsd-port + value: "9125" + - name: statsd-global-tags + value: "workflow={{workflow.name}},bot=atlan-NAME" + - name: output-prefix + value: "argo-artifacts/{{workflow.namespace}}/{{workflow.name}}/{{pod.name}}" + - name: heracles-uri + value: "http://heracles-service.heracles.svc.cluster.local" + - name: init-execution-script + value: | + if state == ExecutionState.API_FAIL and (response.status_code >= 500 or response.status_code in {400}): + LOGGER.debug('Heracles is unavailable. Performing retry with back-off') + failure_handler = FailureHandler.RETRY + + if state == ExecutionState.OUTPUT_PROCESS: + credential = json.loads(output) + output = "" + for key, value in credential.items(): + output += f"""ATLAN_{key.upper()}="{str(value)}"\n""" + + output += f"""ATLAN_TOKEN_URL='https://{credential['host']}/v2/auth/token'\n""" + + if state == ExecutionState.API_POST: + stop = True + outputs: + artifacts: + - name: success + path: "/tmp/rest/success" + s3: + key: "{{inputs.parameters.output-prefix}}" + archive: + none: { } + - name: failure + path: "/tmp/rest/failure" + parameters: + - name: success-num-files + valueFrom: + path: "/tmp/rest/success/result-gen.txt" + - name: failure-num-files + valueFrom: + path: "/tmp/rest/failure/result-gen.txt" + container: + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "./entrypoint.sh" ] + volumeMounts: + - name: credentials + mountPath: /tmp/credentials + env: + - name: OAUTHLIB_INSECURE_TRANSPORT + value: "1" + imagePullPolicy: IfNotPresent + args: [ + "python3", "main.py", "{{inputs.parameters.method}}", "{{inputs.parameters.url}}", + "--request-config", "{{inputs.parameters.request-config}}", + "--raw-input-paginate", "{{inputs.parameters.raw-input-paginate}}", + "--raw-input-file-pattern", "{{inputs.parameters.raw-input-file-pattern}}", + "--raw-input-multiline", "{{inputs.parameters.raw-input-multiline}}", + "--execution-script", "{{inputs.parameters.execution-script}}", + "--secrets-path", "/tmp/credentials/success/*.json", + "--auth-type", "oauth2", + "--auth-oauth2-type", "client_credentials", + "--auth-oauth2-impersonate-user", "", + "--auth-oauth2-client-credentials-client-id", "ATLAN_USERNAME", + "--auth-oauth2-client-credentials-secret", "ATLAN_PASSWORD", + "--auth-oauth2-client-credentials-token-url", "ATLAN_TOKEN_URL", + "--output-chunk-size", "{{inputs.parameters.output-chunk-size}}", + "--output-file-prefix", "/tmp/rest", + "--pagination-wait-time", "10", + "--max-retries", "3", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] + initContainers: + - name: fetch-credentials + image: ghcr.io/atlanhq/rest-master:165b7e5 + command: [ "python3", "main.py" ] + env: + - name: OAUTHLIB_INSECURE_TRANSPORT + value: "1" + - name: CLIENT_ID + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-id-env}}" + - name: CLIENT_SECRET + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.client-secret-env}}" + - name: TOKEN_URL + valueFrom: + secretKeyRef: + name: "{{inputs.parameters.kube-secret-name}}" + key: "{{inputs.parameters.token-url-env}}" + mirrorVolumeMounts: true + args: [ + "GET", + "{{inputs.parameters.heracles-uri}}/credentials/{{inputs.parameters.credential-guid}}/use", + "--raw-input", "{}", + "--raw-input-file-sort", "", + "--raw-input-multiline", "False", + "--execution-script", "{{inputs.parameters.init-execution-script}}", + "--raw-input-paginate", "0", + "--auth-type", "oauth2", + "--auth-oauth2-type", "client_credentials", + "--auth-oauth2-impersonate-user", "{{=sprig.dig('labels', 'workflows', 'argoproj', 'io/creator', '', workflow)}}", + "--auth-oauth2-client-credentials-client-id", "CLIENT_ID", + "--auth-oauth2-client-credentials-secret", "CLIENT_SECRET", + "--auth-oauth2-client-credentials-token-url", "TOKEN_URL", + "--output-chunk-size", "0", + "--output-file-prefix", "/tmp/credentials", + "--pagination-wait-time", "0", + "--max-retries", "10", + "--statsd-host", "{{inputs.parameters.statsd-host}}", + "--statsd-port", "{{inputs.parameters.statsd-port}}", + "--statsd-global-tags", "{{inputs.parameters.statsd-global-tags}}" + ] \ No newline at end of file diff --git a/lib/connectors/BI/transformers/default.jinja2 b/lib/connectors/BI/transformers/default.jinja2 new file mode 100644 index 0000000..1a0d6dc --- /dev/null +++ b/lib/connectors/BI/transformers/default.jinja2 @@ -0,0 +1,19 @@ +{ + "typeName": "ASSET_NAME", + "status": "ACTIVE", + "attributes": { + "connectorName": "{{external_map['integration_name']}}", + "connectionName": "{{external_map['connection_name']}}", + "connectionQualifiedName": "{{external_map['connection_qn']}}", + "lastSyncRunAt": {{now}}, + "lastSyncWorkflowName": "{{external_map['crawler_name']}}", + "lastSyncRun": "{{external_map['workflow_name']}}", + "tenantId": "{{external_map['tenant_id']}}", + + "qualifiedName": "{{external_map['connection_qn']}}/{{record['id']}}", + + "name": {{record.get('name') | tojson}}, + "sourceCreatedAt": "{{convert_date(record['createdAt'], format="%Y-%m-%dT%H:%M:%S.%fZ")}}", + "sourceUpdatedAt": "{{convert_date(record['updatedAt'], format="%Y-%m-%dT%H:%M:%S.%fZ")}}", + } +} \ No newline at end of file diff --git a/lib/connectors/common-configmap/api.yaml b/lib/connectors/common-configmap/api.yaml new file mode 100644 index 0000000..f258942 --- /dev/null +++ b/lib/connectors/common-configmap/api.yaml @@ -0,0 +1,166 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlan-connectors-NAME + labels: + orchestration.atlan.com/version: "1" + orchestration.atlan.com/source: "NAME" +data: + icon: "https://cdn-images-1.medium.com/max/1200/1*8bPQYTZUHcvvk_1fSPwwHw.png" + helpdeskLink: "" + logo: "https://cdn-images-1.medium.com/max/1200/1*8bPQYTZUHcvvk_1fSPwwHw.png" + connector: "NAME" + defaultConnectorType: "rest" + jdbcCredentialTemplate: "{}" + restCredentialTemplate: | + { + "default": [ + { + "id": 1, + "name": "auth", + "curl": "curl --location --request POST 'https://{{host}}/v2/auth/token' --header 'Content-Type:application/x-www-form-urlencoded' --data-urlencode 'grant_type=client_credentials' --data-urlencode 'client_id={{username}}' --data-urlencode 'client_secret={{password}}' --connect-timeout 4" + } + ] + } + odbcCredentialTemplate: "{}" + grpcCredentialTemplate: "{}" + restMetadataTemplate: | + { + "default": [ + { + "id": 1, + "name": "auth", + "curl": "curl --location --request POST 'https://{{host}}/v2/auth/token' --header 'Content-Type:application/x-www-form-urlencoded' --data-urlencode 'grant_type=client_credentials' --data-urlencode 'client_id={{username}}' --data-urlencode 'client_secret={{password}}' --connect-timeout 4" + } + ] + } + restMetadataOutputTransformerTemplate: | + { + + } + odbcTemplate: "{}" + grpcTemplate: "{}" + sageTemplate: | + { + + } + config: | + { + "properties": { + "name": { + "type": "string", + "required": false, + "ui": { + "label": "Name", + "hidden": true, + "placeholder": "Host Name" + } + }, + "connector": { + "type": "string", + "required": false, + "ui": { + "label": "Connector", + "hidden": true, + "placeholder": "Connector" + } + }, + "connectorType": { + "type": "string", + "required": false, + "ui": { + "key": "_host", + "label": "connectorType", + "placeholder": "connectorType", + "hidden": true + } + }, + "host": { + "type": "string", + "required": true, + "enum": ["aws-api.NAMEcomputing.com", "api.NAMEcomputing.com"], + "enumNames": ["aws-api.NAMEcomputing.com - For AWS Hosted Organizations", "api.NAMEcomputing.com - For GCP Hosted Organizations"], + "default": "aws-api.NAMEcomputing.com", + "ui": { + "widget": "select", + "label": "Endpoint", + "grid": 8, + "hidden": false, + "placeholder": "Endpoint" + } + }, + "auth-type": { + "type": "string", + "enum": ["api_token"], + "default": "api_token", + "required": true, + "enumNames": ["API Token"], + "ui": { + "widget": "radio", + "label": "Authentication", + "placeholder": "Credential Type", + "rules": [ + { + "required": true, + "message": "Please enter a valid authentication type" + } + ] + } + }, + "api_token": { + "type": "object", + "properties": { + "username": { + "type": "string", + "required": true, + "ui": { + "label": "Client ID", + "placeholder": "Client ID", + "feedback": true, + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid client Id" + } + ] + } + }, + "password": { + "type": "string", + "required": true, + "ui": { + "widget": "password", + "label": "API Token", + "feedback": true, + "placeholder": "API Token", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid API token" + } + ] + } + } + }, + "ui": { + "widget": "nested", + "label": "Client ID & API Token", + "placeholder": "Credential Type", + "nestedValue": false, + "hidden": true + } + } + }, + "anyOf": [ + { + "properties": { + "auth-type": { + "const": "api_token" + } + }, + "required": ["api_token"] + } + ] + } \ No newline at end of file diff --git a/lib/connectors/common-configmap/aws.yaml b/lib/connectors/common-configmap/aws.yaml new file mode 100644 index 0000000..07f6794 --- /dev/null +++ b/lib/connectors/common-configmap/aws.yaml @@ -0,0 +1,244 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlan-connectors-NAME + labels: + orchestration.atlan.com/version: "1" + orchestration.atlan.com/source: "NAME" +data: + icon: "https://atlan-public.s3.eu-west-1.amazonaws.com/atlan/logos/aws-NAME.png" + helpdeskLink: "https://ask.atlan.com/hc/en-us/articles/4605882724369-How-to-set-up-your-NAME-connection-on-Atlan" + logo: "https://atlan-public.s3.eu-west-1.amazonaws.com/atlan/logos/aws-NAME.png" + connector: "NAME" + defaultConnectorType: "sdk" + jdbcCredentialTemplate: "{}" + restCredentialTemplate: "{}" + odbcCredentialTemplate: "{}" + grpcCredentialTemplate: "{}" + restMetadataTemplate: "" + restMetadataOutputTransformerTemplate: "" + sdkCredentialTemplate: | + [ + { + "id": 1, + "name": "<>", + "source": "AWS", + "action": "AWSNAME.<>", + "params": { {% if authType == "iam" %} "aws_access_key_id": "{{username}}", "aws_secret_access_key": "{{password}}", {% endif %} "region": "{{extra.region}}" {% if authType == "role" and extra.aws_role_arn|length %}, "aws_role_arn": "{{extra.aws_role_arn}}" {% endif %} {% if authType == "role" and extra.aws_external_id|length %}, "aws_external_id": "{{extra.aws_external_id}}" {% endif %} } + } + ] + config: | + { + "properties": { + "name": { + "type": "string", + "required": false, + "ui": { + "label": "Name", + "hidden": true, + "placeholder": "Credential Name" + } + }, + "connector": { + "type": "string", + "required": false, + "ui": { + "label": "Connector", + "hidden": true, + "placeholder": "Connector" + } + }, + "connectorType": { + "type": "string", + "required": false, + "ui": { + "key": "_host", + "label": "connectorType", + "placeholder": "connectorType", + "hidden": true + } + }, + "auth-type": { + "type": "string", + "enum": [ + "iam", + "role" + ], + "default": "iam", + "required": true, + "enumNames": [ + "IAM User", + "IAM Role" + ], + "ui": { + "widget": "radio", + "hidden": false, + "label": "Authentication", + "placeholder": "Credential Type", + "rules": [ + { + "required": true, + "message": "Please enter a valid authentication type" + } + ] + } + }, + "port": { + "type": "number", + "default": 443, + "required": false, + "ui": { + "label": "Port", + "hidden": true, + "placeholder": "Port" + } + }, + "iam": { + "type": "object", + "properties": { + "username": { + "type": "string", + "required": true, + "ui": { + "label": "AWS Access Key", + "placeholder": "Access Key", + "feedback": true, + "help": "AWS Access Key", + "message": "Please enter a valid Access Key", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid Access Key" + } + ] + } + }, + "password": { + "type": "string", + "required": true, + "ui": { + "widget": "password", + "label": "AWS Secret Key", + "feedback": true, + "help": "AWS Secret Key", + "placeholder": "Secret Key", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid Secret Key" + } + ] + } + } + }, + "ui": { + "widget": "nested", + "label": "", + "placeholder": "Credential Type", + "nestedValue": false, + "hidden": false + } + }, + "role":{ + "type": "object", + "properties": { + "extra": { + "type": "object", + "properties": { + "aws_role_arn": { + "type": "string", + "required": false, + "ui": { + "label": "AWS Role ARN", + "hidden": false, + "placeholder": "arn:aws:iam::123456789012:role/roleName", + "help": "AWS role to assume for the connection", + "grid": 4, + "rules": [ + { + "pattern": "^arn:aws:iam::\\d{12}:role/.+", + "message": "Please enter a valid role ARN", + "trigger": "blur" + } + ] + } + }, + "aws_external_id": { + "type": "string", + "required": false, + "ui": { + "label": "External ID", + "widget": "keygen", + "help": "Unique external ID to enable access of your AWS resources to Atlan" + } + } + }, + "ui": { + "widget": "nested", + "label": "", + "header": "Advanced", + "hidden": false + } + } + }, + "ui": { + "widget": "nested", + "label": "", + "placeholder": "Credential Type", + "nestedValue": false, + "hidden": true + } + }, + "extra": { + "type": "object", + "properties": { + "region": { + "type": "string", + "required": true, + "default": "", + "ui": { + "help": "Enter region of NAME setup", + "label": "Region", + "hidden": false, + "placeholder": "us-west-1", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a region" + } + ] + } + } + }, + "ui": { + "widget": "nested", + "label": "", + "header": "Advanced", + "hidden": false + } + } + }, + "anyOf": [ + { + "properties": { + "auth-type": { + "const": "iam" + } + }, + "required": [ + "iam" + ] + }, + { + "properties": { + "auth-type": { + "const": "role" + } + }, + "required": ["role"] + } + ] + } diff --git a/lib/connectors/common-configmap/basic.yaml b/lib/connectors/common-configmap/basic.yaml new file mode 100644 index 0000000..5a60ae4 --- /dev/null +++ b/lib/connectors/common-configmap/basic.yaml @@ -0,0 +1,151 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlan-connectors-NAME + labels: + orchestration.atlan.com/version: "1" + orchestration.atlan.com/source: "NAME" +data: + icon: "<>" + helpdeskLink: "" + logo: "<>" + connector: "NAME" + defaultConnectorType: "rest" + jdbcCredentialTemplate: "{}" + restCredentialTemplate: "{}" + { + "default": [ + { + "id": 1, + "name": "auth", + "curl": "curl --location --request POST 'https://{{host}}/v2/auth/token' --header 'Content-Type:application/x-www-form-urlencoded' --data-urlencode 'grant_type=client_credentials' --data-urlencode 'client_id={{username}}' --data-urlencode 'client_secret={{password}}' --connect-timeout 4" + } + ] + } + odbcCredentialTemplate: "{}" + grpcCredentialTemplate: "{}" + restMetadataTemplate: "{}" + restMetadataOutputTransformerTemplate: "{}" + odbcTemplate: "{}" + grpcTemplate: "{}" + sageTemplate: "{}" + config: | + { + "properties": { + "name": { + "type": "string", + "required": false, + "ui": { + "label": "Name", + "hidden": true, + "placeholder": "Host Name" + } + }, + "connector": { + "type": "string", + "required": false, + "ui": { + "label": "Connector", + "hidden": true, + "placeholder": "Connector" + } + }, + "connectorType": { + "type": "string", + "required": false, + "ui": { + "key": "_host", + "label": "connectorType", + "placeholder": "connectorType", + "hidden": true + } + }, + "host": { + "type": "string", + "required": true, + "enum": ["aws-api.NAMEcomputing.com", "api.NAMEcomputing.com"], + "enumNames": ["aws-api.NAMEcomputing.com - For AWS Hosted Organizations", "api.NAMEcomputing.com - For GCP Hosted Organizations"], + "default": "aws-api.NAMEcomputing.com", + "ui": { + "widget": "select", + "label": "Endpoint", + "grid": 8, + "hidden": false, + "placeholder": "Endpoint" + } + }, + "auth-type": { + "type": "string", + "enum": ["api_token"], + "default": "api_token", + "required": true, + "enumNames": ["API Token"], + "ui": { + "widget": "radio", + "label": "Authentication", + "placeholder": "Credential Type", + "rules": [ + { + "required": true, + "message": "Please enter a valid authentication type" + } + ] + } + }, + "api_token": { + "type": "object", + "properties": { + "username": { + "type": "string", + "required": true, + "ui": { + "label": "Client ID", + "placeholder": "Client ID", + "feedback": true, + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid client Id" + } + ] + } + }, + "password": { + "type": "string", + "required": true, + "ui": { + "widget": "password", + "label": "API Token", + "feedback": true, + "placeholder": "API Token", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid API token" + } + ] + } + } + }, + "ui": { + "widget": "nested", + "label": "Client ID & API Token", + "placeholder": "Credential Type", + "nestedValue": false, + "hidden": true + } + } + }, + "anyOf": [ + { + "properties": { + "auth-type": { + "const": "api_token" + } + }, + "required": ["api_token"] + } + ] + } \ No newline at end of file diff --git a/lib/connectors/common-configmap/jdbc-basic.yaml b/lib/connectors/common-configmap/jdbc-basic.yaml new file mode 100644 index 0000000..4bea4ce --- /dev/null +++ b/lib/connectors/common-configmap/jdbc-basic.yaml @@ -0,0 +1,180 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlan-connectors-NAME + labels: + orchestration.atlan.com/version: "1" + orchestration.atlan.com/source: "NAME" +data: + icon: "https://atlan-public.s3.eu-west-1.amazonaws.com/atlan/logos/NAME.png" + helpdeskLink: "https://docs.atlan.com/integrations/relational-database/NAME" + logo: "https://atlan-public.s3.eu-west-1.amazonaws.com/atlan/logos/NAME.png" + connector: "NAME" + defaultConnectorType: "jdbc" + jdbcCredentialTemplate: | + { + "className": "com.facebook.NAME.jdbc.NAMEDriver", + "jarLink": "https://atlan-public.s3-eu-west-1.amazonaws.com/atlan/jdbc/NAME.tar.gz", + "url": "jdbc:NAME://{{ host }}:{{ port|int }}", + "driverProperties": { "user": "{{ username }}", "password": "{{ password }}" } + } + restCredentialTemplate: "{}" + odbcCredentialTemplate: "{}" + grpcCredentialTemplate: "{}" + restMetadataTemplate: "" + restMetadataOutputTransformerTemplate: "" + sageTemplate: | + { + "catalogsCheck": { + "curls": [ + { + "name": "schemas", + "curl": "curl --location --request POST 'http://heka-service.heka.svc.cluster.local/credential/test' --header 'Content-Type: application/json' --data-raw '{\"query\": \"show atlan schemas\"}'", + "addCredential": true, + "credentialConnectorType": "jdbc" + } + ], + "responseTemplate": "{{- $includeFilter := dict}} {{- if eq `string` (printf `%T` (index .formData `include-filter`)) }} {{- $includeFilter = index .formData `include-filter` | fromJson}} {{- else }} {{- $includeFilter = index .formData `include-filter` }} {{- end }} {{- $allowedDatabases := list}} {{- $allowedSchemas := list}} {{- $missingObjectName := ``}} {{- $checkSuccess := true }} {{- range $schemaList := .schemas.results }} {{- $allowedDatabases = append $allowedDatabases $schemaList.TABLE_CATALOG }} {{- $allowedSchemas = append $allowedSchemas (print $schemaList.TABLE_CATALOG `.` $schemaList.TABLE_SCHEM )}} {{- end }} {{- range $filteredDb, $filteredSchemas := $includeFilter }} {{- $_db := lower $filteredDb | trimPrefix `^` | trimSuffix `$` }} {{- $checkSuccess = and $checkSuccess (has $_db $allowedDatabases) }} {{- if not (has $_db $allowedDatabases)}} {{- $missingObjectName = (print $_db ` ` `database`)}} {{- end }} {{- range $schmea := $filteredSchemas }} {{- $_schema := lower $schmea | trimPrefix `^` | trimSuffix `$` }} {{- $checkSuccess = and $checkSuccess (has (print $_db `.` (lower $_schema)) $allowedSchemas)}} {{- if not (has (print $_db `.` (lower $_schema)) $allowedSchemas)}} {{- $missingObjectName = (print $_db `.` (lower $_schema) ` ` `schema`)}} {{- end }} {{- end }} {{- end }} {{- $response := dict `successMessage` `` `failureMessage` `` `data` .schemas.results `response` dict }} {{- if $checkSuccess }} {{- $_ := set $response `successMessage` `Check successful` }} {{- else }} {{- $_ := set $response `failureMessage` (print `Check failed for ` $missingObjectName) }} {{- end }} {{- $response | toJson }}" + } + } + config: | + { + "properties": { + "name": { + "type": "string", + "required": false, + "ui": { + "label": "Name", + "hidden": true, + "placeholder": "Host Name" + } + }, + "connector": { + "type": "string", + "required": false, + "ui": { + "label": "Connector", + "hidden": true, + "placeholder": "Connector" + } + }, + "connectorType": { + "type": "string", + "required": false, + "ui": { + "key": "_host", + "label": "connectorType", + "placeholder": "connectorType", + "hidden": true + } + }, + "host": { + "type": "string", + "required": true, + "default": "", + "ui": { + "label": "Host", + "feedback": true, + "help": "Your NAME instance host name", + "BYOCdisabled": true, + "rules": [ + { + "required": true, + "message": "Please enter a valid host name" + } + ], + "grid": 6 + } + }, + "port": { + "type": "number", + "default": 8080, + "required": true, + "ui": { + "label": "Port", + "placeholder": "Port", + "disabled": false, + "help": "Your NAME instance port number", + "grid": 2, + "BYOCdisabled": true, + "rules": [ + { + "required": true, + "message": "Please enter a valid port number" + } + ] + } + }, + "auth-type": { + "type": "string", + "enum": ["basic"], + "default": "basic", + "required": true, + "enumNames": ["Basic"], + "ui": { + "widget": "radio", + "hidden": false, + "label": "Authentication", + "placeholder": "Credential Type", + "rules": [ + { + "required": true, + "message": "Please enter a valid authentication type" + } + ] + } + }, + "basic": { + "type": "object", + "properties": { + "username": { + "type": "string", + "required": true, + "ui": { + "label": "Username", + "placeholder": "Username", + "help": "Database Username", + "feedback": true, + "message": "Please enter a valid username", + "grid": 4, + "rules": [ + { + "required": true, + "message": "Please enter a valid username" + } + ] + } + }, + "password": { + "type": "string", + "required": false, + "ui": { + "widget": "password", + "label": "Password", + "help": "Database Password", + "feedback": false, + "placeholder": "Password", + "grid": 4 + } + } + }, + "ui": { + "widget": "nested", + "label": "Basic Authentication", + "placeholder": "Credential Type", + "nestedValue": false, + "hidden": true + } + } + }, + "anyOf": [ + { + "properties": { + "auth-type": { + "const": "basic" + } + }, + "required": ["basic"] + } + ] + } diff --git a/lib/init.js b/lib/init.js index 5ca2c50..c6278b8 100644 --- a/lib/init.js +++ b/lib/init.js @@ -2,9 +2,11 @@ const Promise = require("bluebird"); const fs = Promise.promisifyAll(require("fs")); const system = require("system-commands"); const utils = require("./utils"); +//const LinearClient = require("@linear/sdk").LinearClient; +//const linearClient = new LinearClient(); /** - * Init an Aro package inside the folder + * Init an Argo package inside the folder * Steps: * 1. Check if folder is empty * 2. Package.json should not be present (unless force is set to true) @@ -47,3 +49,163 @@ function replaceInFile(filePath, searchText, replaceText) { return fs.writeFileAsync(filePath, result, "utf8"); }); } + +/* Init an connector Argo package inside the folder + * Steps: + * 1. Check if folder is empty + * 2. Package.json should not be present (unless force is set to true) + * 3. Copy the static folder in the current directory + * 4. Change name to the folder name + * @param {boolean} force + * @param {string} filepath + */ +exports.connectorInit = async function (force, inputPackageName, inputPackageType, auth, assetList, script, linear) { + const dirPath = "/Users/mrunmayi.tripathi/atlan/code/marketplace-packages/packages/atlan/test-package"; + //const dirPath = process.cwd(); + const pathComponents = dirPath.split("/"); + const packageName = inputPackageName || pathComponents[pathComponents.length - 1]; + + if (inputPackageType) { + var assetListArray = assetList.split(","); + if (inputPackageType === "BI") { + await exports.initConnectorBI(force, dirPath, inputPackageName, auth, assetListArray); + } else if (inputPackageType === "SQL") { + //await exports.initConnectorSQL(force, dirPath, inputPackageName, assetListArray); + } + + //Create entry in packages.json + await createPackageEntry(dirPath, inputPackageName); + + //Create linear + if (linear) { + //await createLinearWithSubtask(inputPackageName); + } + } else { + return fs + .readdirAsync(dirPath) + .then((files) => { + if (!force && (files.length !== 0 || files.includes("package.json"))) { + throw `Files already present in the ${__dirname}. Run this command again with --force to ignore`; + } + var skeletonPackagePath = `${__dirname}/static/package`; + system(`cp -r ${skeletonPackagePath}/ ${dirPath}/../packages/package.json`); + return system(`cp -r ${skeletonPackagePath}/ ${dirPath}`); + }) + .then((_) => { + return utils.walk(dirPath); + }) + .then((files) => { + return Promise.each(files, function (file) { + return replaceInFile(file, "NAME", packageName); + }); + }) + .then((_) => { + return packageName; + }) + .catch((error) => { + console.log(error); + }); + } +}; + +exports.initConnectorBI = async function (force, dirPath, packageName, auth, assetList) { + return fs + .readdirAsync(dirPath) + .then((files) => { + if (!force && (files.length !== 0 || files.includes("package.json"))) { + throw `Files already present in the ${__dirname}. Run this command again with --force to ignore`; + } + var skeletonPackagePath = `${__dirname}/connectors/BI`; + return system(`cp -r ${skeletonPackagePath}/ ${dirPath}`); + }) + .then((_) => { + return utils.walk(dirPath); + }) + .then((files) => { + return Promise.each(files, function (file) { + if (assetList.length != 0 && file.includes("/transformers/")) { + return BuildTransformerFilesForBI(file, assetList); + } else if (assetList.length != 0 && file.includes("/templates/")) { + return BuildTemplateFiles(packageName, file, auth); + } else { + return replaceInFile(file, "NAME", packageName); + } + }); + }) + .then((_) => { + fs.unlinkSync(`${dirPath}/transformers/default.jinja2`); + return packageName; + }) + .catch((error) => { + console.error(`Error in function initConnectorBI: ${error}`); + console.log(error); + }); +}; + +function BuildTransformerFilesForBI(filePath, assetList) { + return Promise.each(assetList, function (asset) { + const newFilePath = filePath.replace("default", asset); + return fs.readFileAsync(filePath, "utf-8").then((data) => { + data = data.replace("ASSET_NAME", asset); + fs.writeFile(newFilePath, data, (err) => { + if (err) throw err; + console.log("File %s created successfully!", newFilePath); + }); + }); + }); +} + +function BuildTemplateFiles(packageName, filePath, auth) { + var newFilePath = ""; + + if (auth.includes("oauth") && filePath.includes("oauth")) { + newFilePath = filePath; + } else if (auth.includes("basic") && filePath.includes("basic")) { + newFilePath = filePath; + } else if (auth.includes("api") && filePath.includes("api")) { + newFilePath = filePath; + } + if (newFilePath.length) { + return fs.readFileAsync(filePath, "utf-8").then((data) => { + data = data.replace("NAME", packageName); + return fs.writeFile(newFilePath, data, (err) => { + if (err) throw err; + console.log("File %s created successfully!", newFilePath); + }); + }); + } else { + return; + } +} +/* +async function createLinearWithSubtask(inputPackageName) { + const teams = await linearClient.teams(); + let orcTeam = teams.nodes[0]; + teams.nodes.forEach(function (team) { + if (team.key === "ORC") { + orcTeam = team; + } + }); + const issue = await linearClient.createIssue({ teamId: orcTeam.id, title: `${inputPackageName} Package` }); + console.info("add entry to linear %s", issue); + return; +} +*/ + +/* + Add entry to package.json + This step requires dirPath from marketplace packages repo inside atlan/ folder +*/ +async function createPackageEntry(dirPath, packageName) { + try { + const filePath = `${dirPath}/../packages/package.json`; + const rawPackageData = await fs.readFileAsync(filePath, "utf-8"); + console.info(rawPackageData); + let packageData = JSON.parse(rawPackageData); + packageData.dependencies[`@atlan/${packageName}`] = "^1.0.0"; + fs.writeFileAsync(filePath, JSON.stringify(packageData), "utf8"); + console.info("add connector entry to %s", filePath); + } catch (e) { + console.log("error", e); + } +} diff --git a/package.json b/package.json index 3737419..df70594 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "argopm", - "version": "0.9.1", + "version": "0.9.2", "description": "Argo package manager", "main": "./lib/index.js", "scripts": { @@ -14,6 +14,8 @@ "dependencies": { "@aws-sdk/client-s3": "^3.45.0", "@kubernetes/client-node": "^0.16.1", + "@linear/sdk": "^2.6.0", + "@notionhq/client": "^2.2.3", "ansicolor": "^1.1.95", "as-table": "^1.0.55", "axios": "^0.24.0", @@ -27,6 +29,7 @@ "stream-buffers": "^3.0.2", "system-commands": "^1.1.7", "yargs": "^17.3.1", + "yargs-interactive": "^3.0.1", "yargs-parser": ">=21.0.0" }, "devDependencies": { @@ -51,4 +54,4 @@ "argopm": "./bin/install.js" }, "preferGlobal": true -} \ No newline at end of file +}