diff --git a/Tasks/Common/awsConnectionParameters.ts b/Tasks/Common/awsConnectionParameters.ts new file mode 100644 index 00000000..9b39f9e7 --- /dev/null +++ b/Tasks/Common/awsConnectionParameters.ts @@ -0,0 +1,283 @@ +/*! + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: MIT + */ + +import STS = require('aws-sdk/clients/sts') +import AWS = require('aws-sdk/global') +import HttpsProxyAgent = require('https-proxy-agent') +import { format, parse, Url } from 'url' +import tl = require('vsts-task-lib/task') + +// Task variable names that can be used to supply the AWS credentials +// to a task (in addition to using a service endpoint, or environment +// variables, or EC2 instance metadata) +export const awsAccessKeyIdVariable: string = 'AWS.AccessKeyID' +export const awsSecretAccessKeyVariable: string = 'AWS.SecretAccessKey' +export const awsSessionTokenVariable: string = 'AWS.SessionToken' + +// Task variable name that can be used to supply the region setting to +// a task. +export const awsRegionVariable: string = 'AWS.Region' + +// default session name to apply to the generated credentials if not overridden +// in the endpoint definition +export const defaultRoleSessionName: string = 'aws-vsts-tools' +// The minimum duration, 15mins, should be enough for a task +export const minDuration: number = 900 +export const maxduration: number = 3600 +// To have a longer duration, users can set this variable in their build or +// release definitions to the required duration (in seconds, min 900 max 3600). +export const roleCredentialMaxDurationVariableName: string = 'aws.rolecredential.maxduration' + +export interface AWSConnectionParameters { + // pre-formatted url string, or vsts-task-lib/ProxyConfiguration + proxyConfiguration: string | tl.ProxyConfiguration + // If set, the task should expect to receive temporary session credentials + // scoped to the role. + AssumeRoleARN: string + // Optional diagnostic logging switches + logRequestData: boolean + logResponseData: boolean + // Original task credentials configured by the task user; if we are in assume-role + // mode, these credentials were used to generate the temporary credential + // fields above + awsEndpointAuth: tl.EndpointAuthorization +} + +// Discover any configured proxy setup, first using the Agent.ProxyUrl and related variables. +// If those are not set, fall back to checking HTTP(s)_PROXY that some customers are using +// instead. If HTTP(s)_PROXY is in use we deconstruct the url to make up a ProxyConfiguration +// instance, and then reform to configure the SDK. This allows us to work with either approach. +function completeProxySetup(connectionParamaters: AWSConnectionParameters): void { + if (!connectionParamaters.proxyConfiguration) { + return + } + + let proxy: Url + try { + if (typeof connectionParamaters.proxyConfiguration === 'string') { + proxy = parse(connectionParamaters.proxyConfiguration) + } else { + const config = connectionParamaters.proxyConfiguration as tl.ProxyConfiguration + proxy = parse(config.proxyUrl) + if (config.proxyUsername || config.proxyPassword) { + proxy.auth = `${config.proxyUsername}:${config.proxyPassword}` + } + } + + // do not want any auth in the logged url + tl.debug(`Configuring task for proxy host ${proxy.host}, protocol ${proxy.protocol}`) + AWS.config.update({ + // tslint:disable-next-line: no-unsafe-any + httpOptions: { agent: new HttpsProxyAgent(format(proxy)) } + }) + } catch (err) { + console.error(`Failed to process proxy configuration, error ${err}`) + } +} + +export function buildConnectionParameters(): AWSConnectionParameters { + const connectionParameters: AWSConnectionParameters = { + logRequestData: tl.getBoolInput('logRequest', false), + logResponseData: tl.getBoolInput('logResponse', false), + proxyConfiguration: tl.getHttpProxyConfiguration() || process.env.HTTPS_PROXY || process.env.HTTP_PROXY, + AssumeRoleARN: undefined, + awsEndpointAuth: undefined + } + + completeProxySetup(connectionParameters) + + return connectionParameters +} + +// Unpacks credentials from the specified endpoint configuration, if defined +function attemptEndpointCredentialConfiguration( + awsparams: AWSConnectionParameters, + endpointName: string): AWS.Credentials { + + if (!endpointName) { + return undefined + } + + awsparams.awsEndpointAuth = tl.getEndpointAuthorization(endpointName, false) + console.log(`...configuring AWS credentials from service endpoint '${endpointName}'`) + + const accessKey = awsparams.awsEndpointAuth.parameters.username + const secretKey = awsparams.awsEndpointAuth.parameters.password + + awsparams.AssumeRoleARN = awsparams.awsEndpointAuth.parameters.assumeRoleArn + if (!awsparams.AssumeRoleARN) { + console.log('...endpoint defines standard access/secret key credentials') + + return new AWS.Credentials({ + accessKeyId: accessKey, + secretAccessKey: secretKey + }) + } + + console.log(`...endpoint defines role-based credentials for role ${awsparams.AssumeRoleARN}.`) + + const externalId: string = awsparams.awsEndpointAuth.parameters.externalId + let roleSessionName: string = awsparams.awsEndpointAuth.parameters.roleSessionName + if (!roleSessionName) { + roleSessionName = defaultRoleSessionName + } + let duration: number = minDuration + + const customDurationVariable = tl.getVariable(roleCredentialMaxDurationVariableName) + if (customDurationVariable) { + try { + const customDuration = parseInt(customDurationVariable, 10) + if (customDuration >= minDuration && customDuration <= maxduration) { + throw new RangeError( + `Invalid credential duration '${customDurationVariable}',` + + ` minimum is ${minDuration}seconds, max ${maxduration}seconds`) + } else { + duration = customDuration + } + } catch (err) { + console.warn( + `...ignoring invalid custom ${roleCredentialMaxDurationVariableName} setting due to error: ${err}`) + } + } + + const masterCredentials = new AWS.Credentials({ + accessKeyId: accessKey, + secretAccessKey: secretKey + }) + const options: STS.AssumeRoleRequest = { + RoleArn: awsparams.AssumeRoleARN, + DurationSeconds: duration, + RoleSessionName: roleSessionName + } + if (externalId) { + options.ExternalId = externalId + } + + return new AWS.TemporaryCredentials(options, masterCredentials) +} + +// credentials can also be set, using their environment variable names, +// as variables set on the task or build - these do not appear to be +// visible as environment vars for the AWS Node.js sdk to auto-recover +// so treat as if a service endpoint had been set and return a credentials +// instance. +function attemptCredentialConfigurationFromVariables(): AWS.Credentials { + const accessKey = tl.getVariable(awsAccessKeyIdVariable) + if (!accessKey) { + return undefined + } + + const secretKey = tl.getVariable(awsSecretAccessKeyVariable) + if (!secretKey) { + throw new Error ( + 'AWS access key ID present in task variables but secret key value is missing; ' + + 'cannot configure task credentials.') + } + + const token = tl.getVariable(awsSessionTokenVariable) + + console.log('...configuring AWS credentials from task variables') + + return new AWS.Credentials({ + accessKeyId: accessKey, + secretAccessKey: secretKey, + sessionToken: token + }) +} + +// Probes for credentials to be used by the executing task. Credentials +// can be configured as a service endpoint (of type 'AWS'), or specified +// using task variables. If we don't discover credentials inside the +// Team Services environment we will assume they are set as either +// environment variables on the build host (or, if the build host is +// an EC2 instance, in instance metadata). +export async function getCredentials(awsParams: AWSConnectionParameters): Promise { + + console.log('Configuring credentials for task') + + const credentials = attemptEndpointCredentialConfiguration(awsParams, tl.getInput('awsCredentials', false)) + || attemptCredentialConfigurationFromVariables() + if (credentials) { + return credentials + } + + // at this point user either has to have credentials in environment vars or + // ec2 instance metadata + console.log('No credentials configured.' + + 'The task will attempt to use credentials found in the build host environment.') + + return undefined +} + +async function queryRegionFromMetadata(): Promise { + // SDK doesn't support discovery of region from EC2 instance metadata, so do it manually. We set + // a timeout so that if the build host isn't EC2, we don't hang forever + return new Promise((resolve, reject) => { + const metadataService = new AWS.MetadataService() + metadataService.httpOptions.timeout = 5000 + metadataService.request('/latest/dynamic/instance-identity/document', (err, data) => { + try { + if (err) { + throw err + } + + console.log('...received instance identity document from metadata') + const identity = JSON.parse(data) + // tslint:disable-next-line: no-unsafe-any + if (identity.region) { + // tslint:disable-next-line: no-unsafe-any + resolve(identity.region) + } else { + throw new Error('...region value not found in instance identity metadata') + } + } catch (err) { + reject(err) + } + } + ) + }) +} + +export async function getRegion(): Promise { + console.log('Configuring region for task') + + let region = tl.getInput('regionName', false) + if (region) { + // lowercase it because the picker we know have can return mixed case + // data if the user typed in a region whose prefix (US- etc) exists + // already in the friendly text + region = region.toLowerCase() + console.log(`...configured to use region ${region}, defined in task.`) + + return region + } + + region = tl.getVariable(awsRegionVariable) + if (region) { + console.log(`...configured to use region ${region}, defined in task variable ${awsRegionVariable}.`) + + return region + } + + if (process.env.AWS_REGION) { + region = process.env.AWS_REGION + console.log(`...configured to use region ${region}, defined in environment variable.`) + + return region + } + + try { + region = await queryRegionFromMetadata() + console.log(`...configured to use region ${region}, from EC2 instance metadata.`) + + return region + } catch (err) { + console.log(`...error: failed to query EC2 instance metadata for region - ${err}`) + } + + console.log('No region specified in the task configuration or environment') + + return undefined +} diff --git a/Tasks/Common/sdkutils/defaultClients.ts b/Tasks/Common/defaultClients.ts similarity index 88% rename from Tasks/Common/sdkutils/defaultClients.ts rename to Tasks/Common/defaultClients.ts index 6f42fd52..f11136af 100644 --- a/Tasks/Common/sdkutils/defaultClients.ts +++ b/Tasks/Common/defaultClients.ts @@ -4,8 +4,8 @@ */ import { S3 } from 'aws-sdk/clients/all' +import { SdkUtils } from 'Common/sdkutils' import { AWSConnectionParameters } from './awsConnectionParameters' -import { SdkUtils } from './sdkutils' export async function createDefaultS3Client( connectionParams: AWSConnectionParameters, @@ -16,5 +16,5 @@ export async function createDefaultS3Client( s3ForcePathStyle: forcePathStyle } - return await SdkUtils.createAndConfigureSdkClient(S3, s3Opts, connectionParams, logger) + return await SdkUtils.createAndConfigureSdkClient(S3, s3Opts, connectionParams, logger) as S3 } diff --git a/Tasks/Common/s3.ts b/Tasks/Common/s3.ts new file mode 100644 index 00000000..3988b03e --- /dev/null +++ b/Tasks/Common/s3.ts @@ -0,0 +1,193 @@ + +/*! + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: MIT + */ + +import { S3 } from 'aws-sdk/clients/all' + +// known mime types as recognized by the AWS SDK for .NET and +// AWS Toolkit for Visual Studio +export const knownMimeTypes: Map = new Map([ + [ '.ai', 'application/postscript' ], + [ '.aif', 'audio/x-aiff' ], + [ '.aifc', 'audio/x-aiff' ], + [ '.aiff', 'audio/x-aiff' ], + [ '.asc', 'text/plain' ], + [ '.au', 'audio/basic' ], + [ '.avi', 'video/x-msvideo' ], + [ '.bcpio', 'application/x-bcpio' ], + [ '.bin', 'application/octet-stream' ], + [ '.c', 'text/plain' ], + [ '.cc', 'text/plain' ], + [ '.ccad', 'application/clariscad' ], + [ '.cdf', 'application/x-netcdf' ], + [ '.class', 'application/octet-stream' ], + [ '.cpio', 'application/x-cpio' ], + [ '.cpp', 'text/plain' ], + [ '.cpt', 'application/mac-compactpro' ], + [ '.cs', 'text/plain' ], + [ '.csh', 'application/x-csh' ], + [ '.css', 'text/css' ], + [ '.dcr', 'application/x-director' ], + [ '.dir', 'application/x-director' ], + [ '.dms', 'application/octet-stream' ], + [ '.doc', 'application/msword' ], + [ '.docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' ], + [ '.dot', 'application/msword' ], + [ '.drw', 'application/drafting' ], + [ '.dvi', 'application/x-dvi' ], + [ '.dwg', 'application/acad' ], + [ '.dxf', 'application/dxf' ], + [ '.dxr', 'application/x-director' ], + [ '.eps', 'application/postscript' ], + [ '.etx', 'text/x-setext' ], + [ '.exe', 'application/octet-stream' ], + [ '.ez', 'application/andrew-inset' ], + [ '.f', 'text/plain' ], + [ '.f90', 'text/plain' ], + [ '.fli', 'video/x-fli' ], + [ '.gif', 'image/gif' ], + [ '.gtar', 'application/x-gtar' ], + [ '.gz', 'application/x-gzip' ], + [ '.h', 'text/plain' ], + [ '.hdf', 'application/x-hdf' ], + [ '.hh', 'text/plain' ], + [ '.hqx', 'application/mac-binhex40' ], + [ '.htm', 'text/html' ], + [ '.html', 'text/html' ], + [ '.ice', 'x-conference/x-cooltalk' ], + [ '.ief', 'image/ief' ], + [ '.iges', 'model/iges' ], + [ '.igs', 'model/iges' ], + [ '.ips', 'application/x-ipscript' ], + [ '.ipx', 'application/x-ipix' ], + [ '.jpe', 'image/jpeg' ], + [ '.jpeg', 'image/jpeg' ], + [ '.jpg', 'image/jpeg' ], + [ '.js', 'application/x-javascript' ], + [ '.kar', 'audio/midi' ], + [ '.latex', 'application/x-latex' ], + [ '.lha', 'application/octet-stream' ], + [ '.lsp', 'application/x-lisp' ], + [ '.lzh', 'application/octet-stream' ], + [ '.m', 'text/plain' ], + [ '.m3u8', 'application/x-mpegURL' ], + [ '.man', 'application/x-troff-man' ], + [ '.me', 'application/x-troff-me' ], + [ '.mesh', 'model/mesh' ], + [ '.mid', 'audio/midi' ], + [ '.midi', 'audio/midi' ], + [ '.mime', 'www/mime' ], + [ '.mov', 'video/quicktime' ], + [ '.movie', 'video/x-sgi-movie' ], + [ '.mp2', 'audio/mpeg' ], + [ '.mp3', 'audio/mpeg' ], + [ '.mpe', 'video/mpeg' ], + [ '.mpeg', 'video/mpeg' ], + [ '.mpg', 'video/mpeg' ], + [ '.mpga', 'audio/mpeg' ], + [ '.ms', 'application/x-troff-ms' ], + [ '.msi', 'application/x-ole-storage' ], + [ '.msh', 'model/mesh' ], + [ '.nc', 'application/x-netcdf' ], + [ '.oda', 'application/oda' ], + [ '.pbm', 'image/x-portable-bitmap' ], + [ '.pdb', 'chemical/x-pdb' ], + [ '.pdf', 'application/pdf' ], + [ '.pgm', 'image/x-portable-graymap' ], + [ '.pgn', 'application/x-chess-pgn' ], + [ '.png', 'image/png' ], + [ '.pnm', 'image/x-portable-anymap' ], + [ '.pot', 'application/mspowerpoint' ], + [ '.ppm', 'image/x-portable-pixmap' ], + [ '.pps', 'application/mspowerpoint' ], + [ '.ppt', 'application/mspowerpoint' ], + [ '.pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation' ], + [ '.ppz', 'application/mspowerpoint' ], + [ '.pre', 'application/x-freelance' ], + [ '.prt', 'application/pro_eng' ], + [ '.ps', 'application/postscript' ], + [ '.qt', 'video/quicktime' ], + [ '.ra', 'audio/x-realaudio' ], + [ '.ram', 'audio/x-pn-realaudio' ], + [ '.ras', 'image/cmu-raster' ], + [ '.rgb', 'image/x-rgb' ], + [ '.rm', 'audio/x-pn-realaudio' ], + [ '.roff', 'application/x-troff' ], + [ '.rpm', 'audio/x-pn-realaudio-plugin' ], + [ '.rtf', 'text/rtf' ], + [ '.rtx', 'text/richtext' ], + [ '.scm', 'application/x-lotusscreencam' ], + [ '.set', 'application/set' ], + [ '.sgm', 'text/sgml' ], + [ '.sgml', 'text/sgml' ], + [ '.sh', 'application/x-sh' ], + [ '.shar', 'application/x-shar' ], + [ '.silo', 'model/mesh' ], + [ '.sit', 'application/x-stuffit' ], + [ '.skd', 'application/x-koan' ], + [ '.skm', 'application/x-koan' ], + [ '.skp', 'application/x-koan' ], + [ '.skt', 'application/x-koan' ], + [ '.smi', 'application/smil' ], + [ '.smil', 'application/smil' ], + [ '.snd', 'audio/basic' ], + [ '.sol', 'application/solids' ], + [ '.spl', 'application/x-futuresplash' ], + [ '.src', 'application/x-wais-source' ], + [ '.step', 'application/STEP' ], + [ '.stl', 'application/SLA' ], + [ '.stp', 'application/STEP' ], + [ '.sv4cpio', 'application/x-sv4cpio' ], + [ '.sv4crc', 'application/x-sv4crc' ], + [ '.svg', 'image/svg+xml' ], + [ '.swf', 'application/x-shockwave-flash' ], + [ '.t', 'application/x-troff' ], + [ '.tar', 'application/x-tar' ], + [ '.tcl', 'application/x-tcl' ], + [ '.tex', 'application/x-tex' ], + [ '.tif', 'image/tiff' ], + [ '.tiff', 'image/tiff' ], + [ '.tr', 'application/x-troff' ], + [ '.ts', 'video/MP2T' ], + [ '.tsi', 'audio/TSP-audio' ], + [ '.tsp', 'application/dsptype' ], + [ '.tsv', 'text/tab-separated-values' ], + [ '.txt', 'text/plain' ], + [ '.unv', 'application/i-deas' ], + [ '.ustar', 'application/x-ustar' ], + [ '.vcd', 'application/x-cdlink' ], + [ '.vda', 'application/vda' ], + [ '.vrml', 'model/vrml' ], + [ '.wav', 'audio/x-wav' ], + [ '.wrl', 'model/vrml' ], + [ '.xbm', 'image/x-xbitmap' ], + [ '.xlc', 'application/vnd.ms-excel' ], + [ '.xll', 'application/vnd.ms-excel' ], + [ '.xlm', 'application/vnd.ms-excel' ], + [ '.xls', 'application/vnd.ms-excel' ], + [ '.xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ], + [ '.xlw', 'application/vnd.ms-excel' ], + [ '.xml', 'text/xml' ], + [ '.xpm', 'image/x-xpixmap' ], + [ '.xwd', 'image/x-xwindowdump' ], + [ '.xyz', 'chemical/x-pdb' ], + [ '.zip', 'application/zip' ], + [ '.m4v', 'video/x-m4v' ], + [ '.webm', 'video/webm' ], + [ '.ogv', 'video/ogv' ], + [ '.xap', 'application/x-silverlight-app' ], + [ '.mp4', 'video/mp4' ], + [ '.wmv', 'video/x-ms-wmv' ] +]) + +export async function testBucketExists(s3Client: S3, bucketName: string): Promise { + try { + await s3Client.headBucket({ Bucket: bucketName}).promise() + + return true + } catch (err) { + return false + } +} diff --git a/Tasks/Common/sdkutils.ts b/Tasks/Common/sdkutils.ts new file mode 100644 index 00000000..aaa115ad --- /dev/null +++ b/Tasks/Common/sdkutils.ts @@ -0,0 +1,6 @@ +/*! + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: MIT + */ + +export { SdkUtils } from 'sdkutils/sdkutils' diff --git a/Tasks/Common/sdkutils/awsConnectionParameters.ts b/Tasks/Common/sdkutils/awsConnectionParameters.ts deleted file mode 100644 index 8c4fcd85..00000000 --- a/Tasks/Common/sdkutils/awsConnectionParameters.ts +++ /dev/null @@ -1,7 +0,0 @@ -/*! - * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: MIT - */ - -// TODO see this github issue for tracking removal of this https://github.com/aws/aws-vsts-tools/issues/166 -export { AWSTaskParametersBase as AWSConnectionParameters } from './awsTaskParametersBase' diff --git a/Tasks/Common/sdkutils/awsTaskParametersBase.ts b/Tasks/Common/sdkutils/awsTaskParametersBase.ts index a05d75de..c98956c2 100644 --- a/Tasks/Common/sdkutils/awsTaskParametersBase.ts +++ b/Tasks/Common/sdkutils/awsTaskParametersBase.ts @@ -8,46 +8,56 @@ import { parse, format, Url } from 'url'; import STS = require('aws-sdk/clients/sts'); import AWS = require('aws-sdk/global'); import HttpsProxyAgent = require('https-proxy-agent'); - -export class AWSTaskParametersBase { +import { + AWSConnectionParameters, + awsRegionVariable, + defaultRoleSessionName, + minDuration, + maxduration, + roleCredentialMaxDurationVariableName, + awsAccessKeyIdVariable, + awsSecretAccessKeyVariable, + awsSessionTokenVariable } from '../awsConnectionParameters'; + +export class AWSTaskParametersBase implements AWSConnectionParameters{ // Task variable names that can be used to supply the AWS credentials // to a task (in addition to using a service endpoint, or environment // variables, or EC2 instance metadata) - private static readonly awsAccessKeyIdVariable: string = 'AWS.AccessKeyID'; - private static readonly awsSecretAccessKeyVariable: string = 'AWS.SecretAccessKey'; - private static readonly awsSessionTokenVariable: string = 'AWS.SessionToken'; + private static readonly awsAccessKeyIdVariable: string = awsAccessKeyIdVariable; + private static readonly awsSecretAccessKeyVariable: string = awsSecretAccessKeyVariable; + private static readonly awsSessionTokenVariable: string = awsSessionTokenVariable; // Task variable name that can be used to supply the region setting to // a task. - private static readonly awsRegionVariable: string = 'AWS.Region'; + private static readonly awsRegionVariable: string = awsRegionVariable; // pre-formatted url string, or vsts-task-lib/ProxyConfiguration - public readonly proxyConfiguration: string | tl.ProxyConfiguration; + public proxyConfiguration: string | tl.ProxyConfiguration; // If set, the task should expect to receive temporary session credentials // scoped to the role. public AssumeRoleARN: string; // Optional diagnostic logging switches - public readonly logRequestData: boolean; - public readonly logResponseData: boolean; + public logRequestData: boolean; + public logResponseData: boolean; // Original task credentials configured by the task user; if we are in assume-role // mode, these credentials were used to generate the temporary credential // fields above - protected awsEndpointAuth: tl.EndpointAuthorization; + public awsEndpointAuth: tl.EndpointAuthorization; // default session name to apply to the generated credentials if not overridden // in the endpoint definition - protected readonly defaultRoleSessionName: string = 'aws-vsts-tools'; + public readonly defaultRoleSessionName: string = defaultRoleSessionName; // The minimum duration, 15mins, should be enough for a task - protected readonly minDuration: number = 900; - protected readonly maxduration: number = 3600; - protected readonly defaultRoleDuration: number = this.minDuration; + public readonly minDuration: number = minDuration; + public readonly maxduration: number = maxduration; + public readonly defaultRoleDuration: number = this.minDuration; // To have a longer duration, users can set this variable in their build or // release definitions to the required duration (in seconds, min 900 max 3600). - protected readonly roleCredentialMaxDurationVariableName: string = 'aws.rolecredential.maxduration'; + public readonly roleCredentialMaxDurationVariableName: string = roleCredentialMaxDurationVariableName; public constructor() { this.logRequestData = tl.getBoolInput('logRequest', false); diff --git a/Tasks/Common/sdkutils/sdkutils.ts b/Tasks/Common/sdkutils/sdkutils.ts index 02f8df98..6c2b1e45 100644 --- a/Tasks/Common/sdkutils/sdkutils.ts +++ b/Tasks/Common/sdkutils/sdkutils.ts @@ -9,7 +9,7 @@ import AWS = require('aws-sdk/global') import fs = require('fs') import path = require('path') import tl = require('vsts-task-lib/task') -import { AWSTaskParametersBase } from './awsTaskParametersBase' +import { AWSConnectionParameters, getCredentials, getRegion } from 'Common/awsConnectionParameters'; export abstract class SdkUtils { @@ -69,7 +69,7 @@ export abstract class SdkUtils { // to the task log. public static async createAndConfigureSdkClient(awsService: any, awsServiceOpts: any, - taskParams: AWSTaskParametersBase, + taskParams: AWSConnectionParameters, logger: (msg: string) => void): Promise { awsService.prototype.customizeRequests((request) => { @@ -137,22 +137,22 @@ export abstract class SdkUtils { // instance metadata if (awsServiceOpts) { if (!awsServiceOpts.credentials) { - const credentials = await taskParams.getCredentials(); + const credentials = await await getCredentials(taskParams) if (credentials) { awsServiceOpts.credentials = await credentials.getPromise(); } } if (!awsServiceOpts.region) { - awsServiceOpts.region = await taskParams.getRegion() + awsServiceOpts.region = await getRegion() } return new awsService(awsServiceOpts) } - const credentials = await taskParams.getCredentials(); + const credentials = await getCredentials(taskParams) return new awsService({ credentials: credentials ? credentials.getPromise() : undefined, - region: await taskParams.getRegion() + region: await getRegion() }) } diff --git a/Tasks/S3Download/DownloadTaskOperations.ts b/Tasks/S3Download/DownloadTaskOperations.ts index 3cf11dea..e9921292 100644 --- a/Tasks/S3Download/DownloadTaskOperations.ts +++ b/Tasks/S3Download/DownloadTaskOperations.ts @@ -3,13 +3,14 @@ * SPDX-License-Identifier: MIT */ +import { S3 } from 'aws-sdk/clients/all' import * as fs from 'fs' import * as mm from 'minimatch' import * as path from 'path' import * as tl from 'vsts-task-lib/task' -import { aes256AlgorithmValue, customerManagedKeyValue, TaskParameters } from './DownloadTaskParameters' -import S3 = require('aws-sdk/clients/s3') +import { testBucketExists } from 'Common/s3' +import { aes256AlgorithmValue, customerManagedKeyValue, TaskParameters } from './DownloadTaskParameters' export class TaskOperations { public constructor( @@ -19,7 +20,7 @@ export class TaskOperations { } public async execute(): Promise { - const exists = await this.testBucketExists(this.taskParameters.bucketName) + const exists = await testBucketExists(this.s3Client, this.taskParameters.bucketName) if (!exists) { throw new Error(tl.loc('BucketNotExist', this.taskParameters.bucketName)) } @@ -29,16 +30,6 @@ export class TaskOperations { console.log(tl.loc('TaskCompleted')) } - private async testBucketExists(bucketName: string): Promise { - try { - await this.s3Client.headBucket({ Bucket: bucketName }).promise() - - return true - } catch (err) { - return false - } - } - private async downloadFiles() { let msgSource: string @@ -122,7 +113,8 @@ export class TaskOperations { } const allKeys: string[] = [] - let nextToken: string + // tslint:disable-next-line:no-unnecessary-initializer + let nextToken: string | undefined = undefined do { const params: S3.ListObjectsRequest = { Bucket: this.taskParameters.bucketName, @@ -132,13 +124,15 @@ export class TaskOperations { try { const s3Data = await this.s3Client.listObjects(params).promise() nextToken = s3Data.NextMarker - s3Data.Contents.forEach((s3object) => { - // AWS IDE toolkits can cause 0 byte 'folder markers' to be in the bucket, - // filter those out - if (!s3object.Key.endsWith('_$folder$')) { - allKeys.push(s3object.Key) - } - }) + if (s3Data.Contents) { + s3Data.Contents.forEach((s3object) => { + // AWS IDE toolkits can cause 0 byte 'folder markers' to be in the bucket, + // filter those out + if (!s3object.Key.endsWith('_$folder$')) { + allKeys.push(s3object.Key) + } + }) + } } catch (err) { console.error(err) nextToken = undefined diff --git a/Tasks/S3Download/DownloadTaskParameters.ts b/Tasks/S3Download/DownloadTaskParameters.ts index c7dcfa98..89a30393 100644 --- a/Tasks/S3Download/DownloadTaskParameters.ts +++ b/Tasks/S3Download/DownloadTaskParameters.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: MIT */ -import { AWSConnectionParameters } from 'sdkutils/awsConnectionParameters' +import { AWSConnectionParameters, buildConnectionParameters } from 'Common/awsConnectionParameters' import tl = require('vsts-task-lib/task') // options for Server-side encryption Key Management @@ -26,7 +26,7 @@ export interface TaskParameters { export function buildTaskParameters(): TaskParameters { const parameters: TaskParameters = { - awsConnectionParameters: new AWSConnectionParameters(), + awsConnectionParameters: buildConnectionParameters(), bucketName: tl.getInput('bucketName', true), sourceFolder: tl.getPathInput('sourceFolder', false, false), targetFolder: tl.getPathInput('targetFolder', true, false), diff --git a/Tasks/S3Download/S3Download.ts b/Tasks/S3Download/S3Download.ts index 0f16e171..a4d28d8f 100644 --- a/Tasks/S3Download/S3Download.ts +++ b/Tasks/S3Download/S3Download.ts @@ -5,7 +5,7 @@ import tl = require('vsts-task-lib/task') -import { createDefaultS3Client } from 'sdkutils/defaultClients' +import { createDefaultS3Client } from 'Common/defaultClients' import { SdkUtils } from 'sdkutils/sdkutils' import { TaskOperations } from './DownloadTaskOperations' diff --git a/Tasks/S3Download/package.json b/Tasks/S3Download/package.json index 22d6ab7b..96ad6721 100644 --- a/Tasks/S3Download/package.json +++ b/Tasks/S3Download/package.json @@ -3,9 +3,6 @@ "version": "1.1.8", "description": "Download content from an AWS Simple Storage Service (S3) Bucket", "main": "S3Download.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, "author": "", "license": "MIT", "dependencies": { diff --git a/Tasks/S3Download/tsconfig.json b/Tasks/S3Download/tsconfig.json deleted file mode 100644 index beeb3d52..00000000 --- a/Tasks/S3Download/tsconfig.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "compilerOptions": { - "target": "ES6", - "module": "commonjs", - "sourceMap": true, - "alwaysStrict": true - }, - "exclude": [ - "node_modules" - ] -} diff --git a/Tasks/S3Upload/S3Upload.ts b/Tasks/S3Upload/S3Upload.ts index bb20da20..0c04ac9d 100644 --- a/Tasks/S3Upload/S3Upload.ts +++ b/Tasks/S3Upload/S3Upload.ts @@ -1,32 +1,32 @@ -/* - Copyright 2017-2018 Amazon.com, Inc. and its affiliates. All Rights Reserved. - * - * Licensed under the MIT License. See the LICENSE accompanying this file - * for the specific language governing permissions and limitations under - * the License. - */ +/*! + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: MIT + */ -import tl = require('vsts-task-lib/task'); -import path = require('path'); +import tl = require('vsts-task-lib/task') -import { SdkUtils } from 'sdkutils/sdkutils'; +import { SdkUtils } from 'sdkutils/sdkutils' -import { TaskParameters } from './helpers/UploadTaskParameters'; -import { TaskOperations } from './helpers/UploadTaskOperations'; +import { getRegion } from 'Common/awsConnectionParameters' +import { createDefaultS3Client } from 'Common/defaultClients' +import { TaskOperations } from './UploadTaskOperations' +import { buildTaskParameters } from './UploadTaskParameters' -function run(): Promise { +async function run(): Promise { + SdkUtils.readResources() + const taskParameters = buildTaskParameters() + const s3 = await createDefaultS3Client( + taskParameters.awsConnectionParameters, + taskParameters.forcePathStyleAddressing, + tl.debug) + const region = await getRegion() - const taskManifestFile = path.join(__dirname, 'task.json'); - tl.setResourcePath(taskManifestFile); - SdkUtils.setSdkUserAgentFromManifest(taskManifestFile); - - const taskParameters = new TaskParameters(); - return new TaskOperations(taskParameters).execute(); + return new TaskOperations(s3, region, taskParameters).execute() } // run run().then((result) => tl.setResult(tl.TaskResult.Succeeded, '') ).catch((error) => - tl.setResult(tl.TaskResult.Failed, error) -); + tl.setResult(tl.TaskResult.Failed, `${error}`) +) diff --git a/Tasks/S3Upload/UploadTaskOperations.ts b/Tasks/S3Upload/UploadTaskOperations.ts new file mode 100644 index 00000000..10f41101 --- /dev/null +++ b/Tasks/S3Upload/UploadTaskOperations.ts @@ -0,0 +1,162 @@ +/*! + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: MIT + */ + +import S3 = require('aws-sdk/clients/s3') +import { knownMimeTypes, testBucketExists } from 'Common/s3' +import fs = require('fs') +import path = require('path') +import tl = require('vsts-task-lib/task') +import { awsKeyManagementValue, + customerKeyManagementValue, + noKeyManagementValue, + TaskParameters } from './UploadTaskParameters' + +export class TaskOperations { + public constructor( + public readonly s3Client: S3, + public readonly region: string, + public readonly taskParameters: TaskParameters + ) { + } + + public async execute(): Promise { + if (this.taskParameters.createBucket) { + await this.createBucketIfNotExist( + this.taskParameters.bucketName, + this.region) + } else { + const exists = await testBucketExists(this.s3Client, this.taskParameters.bucketName) + if (!exists) { + throw new Error(tl.loc('BucketNotExistNoAutocreate', this.taskParameters.bucketName)) + } + } + + await this.uploadFiles() + console.log(tl.loc('TaskCompleted')) + } + + public findMatchingFiles(taskParameters: TaskParameters): string[] { + console.log(`Searching ${taskParameters.sourceFolder} for files to upload`) + taskParameters.sourceFolder = path.normalize(taskParameters.sourceFolder) + const allPaths = tl.find(taskParameters.sourceFolder) // default find options (follow sym links) + tl.debug(tl.loc('AllPaths', allPaths)) + const matchedPaths = tl.match( + allPaths, + taskParameters.globExpressions, + taskParameters.sourceFolder) // default match options + tl.debug(tl.loc('MatchedPaths', matchedPaths)) + const matchedFiles = matchedPaths.filter((itemPath) => !tl.stats(itemPath).isDirectory()) + tl.debug(tl.loc('MatchedFiles', matchedFiles)) + tl.debug(tl.loc('FoundNFiles', matchedFiles.length)) + + return matchedFiles + } + + private async createBucketIfNotExist(bucketName: string, region: string): Promise { + const exists = await testBucketExists(this.s3Client, bucketName) + if (exists) { + return + } + + try { + console.log(tl.loc('BucketNotExistCreating', bucketName, region)) + await this.s3Client.createBucket({ Bucket: bucketName }).promise() + console.log(tl.loc('BucketCreated')) + } catch (err) { + console.log(tl.loc('CreateBucketFailure'), err) + throw err + } + } + + private async uploadFiles() { + + let msgTarget: string + if (this.taskParameters.targetFolder) { + msgTarget = this.taskParameters.targetFolder + } else { + msgTarget = '/' + } + console.log(tl.loc( + 'UploadingFiles', + this.taskParameters.sourceFolder, + msgTarget, + this.taskParameters.bucketName)) + + const matchedFiles = this.findMatchingFiles(this.taskParameters) + for (const matchedFile of matchedFiles) { + let relativePath = matchedFile.substring(this.taskParameters.sourceFolder.length) + if (relativePath.startsWith(path.sep)) { + relativePath = relativePath.substr(1) + } + let targetPath = relativePath + + if (this.taskParameters.flattenFolders) { + const flatFileName = path.basename(matchedFile) + if (this.taskParameters.targetFolder) { + targetPath = path.join(this.taskParameters.targetFolder, flatFileName) + } else { + targetPath = flatFileName + } + } else { + if (this.taskParameters.targetFolder) { + targetPath = path.join(this.taskParameters.targetFolder, relativePath) + } else { + targetPath = relativePath + } + } + + const targetDir = path.dirname(targetPath) + targetPath = targetPath.replace(/\\/g, '/') + const stats = fs.lstatSync(matchedFile) + if (!stats.isDirectory()) { + const fileBuffer = fs.createReadStream(matchedFile) + try { + let contentType: string + if (this.taskParameters.contentType) { + contentType = this.taskParameters.contentType + } else { + contentType = knownMimeTypes.get(path.extname(matchedFile)) + if (!contentType) { + contentType = 'application/octet-stream' + } + } + console.log(tl.loc('UploadingFile', matchedFile, contentType)) + + const request: S3.PutObjectRequest = { + Bucket: this.taskParameters.bucketName, + Key: targetPath, + Body: fileBuffer, + ACL: this.taskParameters.filesAcl, + ContentType: contentType, + StorageClass: this.taskParameters.storageClass + } + switch (this.taskParameters.keyManagement) { + case noKeyManagementValue: + break + + case awsKeyManagementValue: { + request.ServerSideEncryption = this.taskParameters.encryptionAlgorithm + request.SSEKMSKeyId = this.taskParameters.kmsMasterKeyId + break + } + + case customerKeyManagementValue: { + request.SSECustomerAlgorithm = this.taskParameters.encryptionAlgorithm + request.SSECustomerKey = this.taskParameters.customerKey + break + } + } + + const response: S3.ManagedUpload.SendData = await this.s3Client.upload(request).promise() + console.log(tl.loc('FileUploadCompleted', matchedFile, targetPath)) + } catch (err) { + console.error(tl.loc('FileUploadFailed'), err) + throw err + } + } + } + } + +} diff --git a/Tasks/S3Upload/UploadTaskParameters.ts b/Tasks/S3Upload/UploadTaskParameters.ts new file mode 100644 index 00000000..722cc2ca --- /dev/null +++ b/Tasks/S3Upload/UploadTaskParameters.ts @@ -0,0 +1,86 @@ +/*! + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: MIT + */ + +import { AWSConnectionParameters, buildConnectionParameters } from 'Common/awsConnectionParameters' +import tl = require('vsts-task-lib/task') + + // options for Server-side encryption Key Management; 'none' disables SSE +export const noKeyManagementValue: string = 'none' +export const awsKeyManagementValue: string = 'awsManaged' +export const customerKeyManagementValue: string = 'customerManaged' + +// options for encryption algorithm when key management is set to 'aws'; +// customer managed keys always use AES256 +export const awskmsAlgorithmValue: string = 'KMS' // translated to aws:kms when used in api call +export const aes256AlgorithmValue: string = 'AES256' + +export interface TaskParameters { + awsConnectionParameters: AWSConnectionParameters + bucketName: string + sourceFolder: string + targetFolder: string + flattenFolders: boolean + overwrite: boolean + globExpressions: string[] + filesAcl: string + createBucket: boolean + contentType: string + contentEncoding: string + forcePathStyleAddressing: boolean + storageClass: string + keyManagement: string + encryptionAlgorithm: string + kmsMasterKeyId: string + customerKey: Buffer +} + +export function buildTaskParameters(): TaskParameters { + const parameters: TaskParameters = { + awsConnectionParameters: buildConnectionParameters(), + bucketName: tl.getInput('bucketName', true), + overwrite: tl.getBoolInput('overwrite', false), + flattenFolders: tl.getBoolInput('flattenFolders', false), + sourceFolder: tl.getPathInput('sourceFolder', true, true), + targetFolder: tl.getInput('targetFolder', false), + globExpressions: tl.getDelimitedInput('globExpressions', '\n', true), + filesAcl: tl.getInput('filesAcl', false), + createBucket: tl.getBoolInput('createBucket'), + contentType: tl.getInput('contentType', false), + contentEncoding: tl.getInput('contentEncoding', false), + forcePathStyleAddressing: tl.getBoolInput('forcePathStyleAddressing', false), + storageClass: tl.getInput('storageClass', false), + keyManagement: undefined, + encryptionAlgorithm: undefined, + kmsMasterKeyId: undefined, + customerKey: undefined + } + if (!parameters.storageClass) { + parameters.storageClass = 'STANDARD' + } + parameters.keyManagement = tl.getInput('keyManagement', false) + if (parameters.keyManagement && parameters.keyManagement !== noKeyManagementValue) { + switch (parameters.keyManagement) { + case awsKeyManagementValue: { + const algorithm = tl.getInput('encryptionAlgorithm', true) + if (algorithm === awskmsAlgorithmValue) { + parameters.encryptionAlgorithm = 'aws:kms' + } else { + parameters.encryptionAlgorithm = aes256AlgorithmValue + } + parameters.kmsMasterKeyId = tl.getInput('kmsMasterKeyId', algorithm === awskmsAlgorithmValue) + break + } + + case customerKeyManagementValue: { + parameters.encryptionAlgorithm = aes256AlgorithmValue + const customerKey = tl.getInput('customerKey', true) + parameters.customerKey = Buffer.from(customerKey, 'hex') + break + } + } + } + + return parameters +} diff --git a/Tasks/S3Upload/helpers/UploadTaskOperations.ts b/Tasks/S3Upload/helpers/UploadTaskOperations.ts deleted file mode 100644 index 5a02a165..00000000 --- a/Tasks/S3Upload/helpers/UploadTaskOperations.ts +++ /dev/null @@ -1,351 +0,0 @@ -/* - Copyright 2017-2018 Amazon.com, Inc. and its affiliates. All Rights Reserved. - * - * Licensed under the MIT License. See the LICENSE accompanying this file - * for the specific language governing permissions and limitations under - * the License. - */ - -import tl = require('vsts-task-lib/task'); -import path = require('path'); -import fs = require('fs'); -import AWS = require('aws-sdk/global'); -import S3 = require('aws-sdk/clients/s3'); -import { AWSError } from 'aws-sdk/lib/error'; -import { SdkUtils } from 'sdkutils/sdkutils'; -import { TaskParameters } from './UploadTaskParameters'; - -export class TaskOperations { - - public constructor( - public readonly taskParameters: TaskParameters - ) { - } - - public async execute(): Promise { - await this.createServiceClients(); - - if (this.taskParameters.createBucket) { - await this.createBucketIfNotExist(this.taskParameters.bucketName, await this.taskParameters.getRegion()); - } else { - const exists = await this.testBucketExists(this.taskParameters.bucketName); - if (!exists) { - throw new Error(tl.loc('BucketNotExistNoAutocreate', this.taskParameters.bucketName)); - } - } - - await this.uploadFiles(); - console.log(tl.loc('TaskCompleted')); - } - - private s3Client: S3; - - private async createServiceClients(): Promise { - - const s3Opts: S3.ClientConfiguration = { - apiVersion: '2006-03-01', - s3ForcePathStyle: this.taskParameters.forcePathStyleAddressing - }; - this.s3Client = await SdkUtils.createAndConfigureSdkClient(S3, s3Opts, this.taskParameters, tl.debug); - } - - private async createBucketIfNotExist(bucketName: string, region: string) : Promise { - const exists = await this.testBucketExists(bucketName); - if (exists) { - return; - } - - try { - console.log(tl.loc('BucketNotExistCreating', bucketName, region)); - await this.s3Client.createBucket({ Bucket: bucketName }).promise(); - console.log(tl.loc('BucketCreated')); - } catch (err) { - console.log(tl.loc('CreateBucketFailure'), err); - throw err; - } - } - - private async testBucketExists(bucketName: string): Promise { - try { - await this.s3Client.headBucket({ Bucket: bucketName}).promise(); - return true; - } catch (err) { - return false; - } - } - - private async uploadFiles() { - - let msgTarget: string; - if (this.taskParameters.targetFolder) { - msgTarget = this.taskParameters.targetFolder; - } else { - msgTarget = '/'; - } - console.log(tl.loc('UploadingFiles', this.taskParameters.sourceFolder, msgTarget, this.taskParameters.bucketName)); - - const matchedFiles = this.findFiles(); - for (let i = 0; i < matchedFiles.length; i++) { - const matchedFile = matchedFiles[i]; - let relativePath = matchedFile.substring(this.taskParameters.sourceFolder.length); - if (relativePath.startsWith(path.sep)) { - relativePath = relativePath.substr(1); - } - let targetPath = relativePath; - - if (this.taskParameters.flattenFolders) { - const flatFileName = path.basename(matchedFile); - if (this.taskParameters.targetFolder) { - targetPath = path.join(this.taskParameters.targetFolder, flatFileName); - } else { - targetPath = flatFileName; - } - } else { - if (this.taskParameters.targetFolder) { - targetPath = path.join(this.taskParameters.targetFolder, relativePath); - } else { - targetPath = relativePath; - } - } - - const targetDir = path.dirname(targetPath); - targetPath = targetPath.replace(/\\/g, '/'); - const stats = fs.lstatSync(matchedFile); - if (!stats.isDirectory()) { - const fileBuffer = fs.createReadStream(matchedFile); - try { - let contentType: string; - if (this.taskParameters.contentType) { - contentType = this.taskParameters.contentType; - } else { - contentType = TaskOperations.knownMimeTypes.get(path.extname(matchedFile)); - if (!contentType) { - contentType = 'application/octet-stream'; - } - } - console.log(tl.loc('UploadingFile', matchedFile, contentType)); - - const request: S3.PutObjectRequest = { - Bucket: this.taskParameters.bucketName, - Key: targetPath, - Body: fileBuffer, - ACL: this.taskParameters.filesAcl, - ContentType: contentType, - StorageClass: this.taskParameters.storageClass - }; - switch (this.taskParameters.keyManagement) { - case TaskParameters.noKeyManagementValue: - break; - case TaskParameters.awsKeyManagementValue: { - request.ServerSideEncryption = this.taskParameters.encryptionAlgorithm; - request.SSEKMSKeyId = this.taskParameters.kmsMasterKeyId; - } - break; - - case TaskParameters.customerKeyManagementValue: { - request.SSECustomerAlgorithm = this.taskParameters.encryptionAlgorithm; - request.SSECustomerKey = this.taskParameters.customerKey; - } - break; - } - - const response: S3.ManagedUpload.SendData = await this.s3Client.upload(request).promise(); - console.log(tl.loc('FileUploadCompleted', matchedFile, targetPath)); - } catch (err) { - console.error(tl.loc('FileUploadFailed'), err); - throw err; - } - } - } - } - - // known mime types as recognized by the AWS SDK for .NET and - // AWS Toolkit for Visual Studio - private static knownMimeTypes: Map = new Map([ - [ '.ai', 'application/postscript' ], - [ '.aif', 'audio/x-aiff' ], - [ '.aifc', 'audio/x-aiff' ], - [ '.aiff', 'audio/x-aiff' ], - [ '.asc', 'text/plain' ], - [ '.au', 'audio/basic' ], - [ '.avi', 'video/x-msvideo' ], - [ '.bcpio', 'application/x-bcpio' ], - [ '.bin', 'application/octet-stream' ], - [ '.c', 'text/plain' ], - [ '.cc', 'text/plain' ], - [ '.ccad', 'application/clariscad' ], - [ '.cdf', 'application/x-netcdf' ], - [ '.class', 'application/octet-stream' ], - [ '.cpio', 'application/x-cpio' ], - [ '.cpp', 'text/plain' ], - [ '.cpt', 'application/mac-compactpro' ], - [ '.cs', 'text/plain' ], - [ '.csh', 'application/x-csh' ], - [ '.css', 'text/css' ], - [ '.dcr', 'application/x-director' ], - [ '.dir', 'application/x-director' ], - [ '.dms', 'application/octet-stream' ], - [ '.doc', 'application/msword' ], - [ '.docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' ], - [ '.dot', 'application/msword' ], - [ '.drw', 'application/drafting' ], - [ '.dvi', 'application/x-dvi' ], - [ '.dwg', 'application/acad' ], - [ '.dxf', 'application/dxf' ], - [ '.dxr', 'application/x-director' ], - [ '.eps', 'application/postscript' ], - [ '.etx', 'text/x-setext' ], - [ '.exe', 'application/octet-stream' ], - [ '.ez', 'application/andrew-inset' ], - [ '.f', 'text/plain' ], - [ '.f90', 'text/plain' ], - [ '.fli', 'video/x-fli' ], - [ '.gif', 'image/gif' ], - [ '.gtar', 'application/x-gtar' ], - [ '.gz', 'application/x-gzip' ], - [ '.h', 'text/plain' ], - [ '.hdf', 'application/x-hdf' ], - [ '.hh', 'text/plain' ], - [ '.hqx', 'application/mac-binhex40' ], - [ '.htm', 'text/html' ], - [ '.html', 'text/html' ], - [ '.ice', 'x-conference/x-cooltalk' ], - [ '.ief', 'image/ief' ], - [ '.iges', 'model/iges' ], - [ '.igs', 'model/iges' ], - [ '.ips', 'application/x-ipscript' ], - [ '.ipx', 'application/x-ipix' ], - [ '.jpe', 'image/jpeg' ], - [ '.jpeg', 'image/jpeg' ], - [ '.jpg', 'image/jpeg' ], - [ '.js', 'application/x-javascript' ], - [ '.kar', 'audio/midi' ], - [ '.latex', 'application/x-latex' ], - [ '.lha', 'application/octet-stream' ], - [ '.lsp', 'application/x-lisp' ], - [ '.lzh', 'application/octet-stream' ], - [ '.m', 'text/plain' ], - [ '.m3u8', 'application/x-mpegURL' ], - [ '.man', 'application/x-troff-man' ], - [ '.me', 'application/x-troff-me' ], - [ '.mesh', 'model/mesh' ], - [ '.mid', 'audio/midi' ], - [ '.midi', 'audio/midi' ], - [ '.mime', 'www/mime' ], - [ '.mov', 'video/quicktime' ], - [ '.movie', 'video/x-sgi-movie' ], - [ '.mp2', 'audio/mpeg' ], - [ '.mp3', 'audio/mpeg' ], - [ '.mpe', 'video/mpeg' ], - [ '.mpeg', 'video/mpeg' ], - [ '.mpg', 'video/mpeg' ], - [ '.mpga', 'audio/mpeg' ], - [ '.ms', 'application/x-troff-ms' ], - [ '.msi', 'application/x-ole-storage' ], - [ '.msh', 'model/mesh' ], - [ '.nc', 'application/x-netcdf' ], - [ '.oda', 'application/oda' ], - [ '.pbm', 'image/x-portable-bitmap' ], - [ '.pdb', 'chemical/x-pdb' ], - [ '.pdf', 'application/pdf' ], - [ '.pgm', 'image/x-portable-graymap' ], - [ '.pgn', 'application/x-chess-pgn' ], - [ '.png', 'image/png' ], - [ '.pnm', 'image/x-portable-anymap' ], - [ '.pot', 'application/mspowerpoint' ], - [ '.ppm', 'image/x-portable-pixmap' ], - [ '.pps', 'application/mspowerpoint' ], - [ '.ppt', 'application/mspowerpoint' ], - [ '.pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation' ], - [ '.ppz', 'application/mspowerpoint' ], - [ '.pre', 'application/x-freelance' ], - [ '.prt', 'application/pro_eng' ], - [ '.ps', 'application/postscript' ], - [ '.qt', 'video/quicktime' ], - [ '.ra', 'audio/x-realaudio' ], - [ '.ram', 'audio/x-pn-realaudio' ], - [ '.ras', 'image/cmu-raster' ], - [ '.rgb', 'image/x-rgb' ], - [ '.rm', 'audio/x-pn-realaudio' ], - [ '.roff', 'application/x-troff' ], - [ '.rpm', 'audio/x-pn-realaudio-plugin' ], - [ '.rtf', 'text/rtf' ], - [ '.rtx', 'text/richtext' ], - [ '.scm', 'application/x-lotusscreencam' ], - [ '.set', 'application/set' ], - [ '.sgm', 'text/sgml' ], - [ '.sgml', 'text/sgml' ], - [ '.sh', 'application/x-sh' ], - [ '.shar', 'application/x-shar' ], - [ '.silo', 'model/mesh' ], - [ '.sit', 'application/x-stuffit' ], - [ '.skd', 'application/x-koan' ], - [ '.skm', 'application/x-koan' ], - [ '.skp', 'application/x-koan' ], - [ '.skt', 'application/x-koan' ], - [ '.smi', 'application/smil' ], - [ '.smil', 'application/smil' ], - [ '.snd', 'audio/basic' ], - [ '.sol', 'application/solids' ], - [ '.spl', 'application/x-futuresplash' ], - [ '.src', 'application/x-wais-source' ], - [ '.step', 'application/STEP' ], - [ '.stl', 'application/SLA' ], - [ '.stp', 'application/STEP' ], - [ '.sv4cpio', 'application/x-sv4cpio' ], - [ '.sv4crc', 'application/x-sv4crc' ], - [ '.svg', 'image/svg+xml' ], - [ '.swf', 'application/x-shockwave-flash' ], - [ '.t', 'application/x-troff' ], - [ '.tar', 'application/x-tar' ], - [ '.tcl', 'application/x-tcl' ], - [ '.tex', 'application/x-tex' ], - [ '.tif', 'image/tiff' ], - [ '.tiff', 'image/tiff' ], - [ '.tr', 'application/x-troff' ], - [ '.ts', 'video/MP2T' ], - [ '.tsi', 'audio/TSP-audio' ], - [ '.tsp', 'application/dsptype' ], - [ '.tsv', 'text/tab-separated-values' ], - [ '.txt', 'text/plain' ], - [ '.unv', 'application/i-deas' ], - [ '.ustar', 'application/x-ustar' ], - [ '.vcd', 'application/x-cdlink' ], - [ '.vda', 'application/vda' ], - [ '.vrml', 'model/vrml' ], - [ '.wav', 'audio/x-wav' ], - [ '.wrl', 'model/vrml' ], - [ '.xbm', 'image/x-xbitmap' ], - [ '.xlc', 'application/vnd.ms-excel' ], - [ '.xll', 'application/vnd.ms-excel' ], - [ '.xlm', 'application/vnd.ms-excel' ], - [ '.xls', 'application/vnd.ms-excel' ], - [ '.xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ], - [ '.xlw', 'application/vnd.ms-excel' ], - [ '.xml', 'text/xml' ], - [ '.xpm', 'image/x-xpixmap' ], - [ '.xwd', 'image/x-xwindowdump' ], - [ '.xyz', 'chemical/x-pdb' ], - [ '.zip', 'application/zip' ], - [ '.m4v', 'video/x-m4v' ], - [ '.webm', 'video/webm' ], - [ '.ogv', 'video/ogv' ], - [ '.xap', 'application/x-silverlight-app' ], - [ '.mp4', 'video/mp4' ], - [ '.wmv', 'video/x-ms-wmv' ] - ]); - - private findFiles(): string[] { - console.log(`Searching ${this.taskParameters.sourceFolder} for files to upload`); - this.taskParameters.sourceFolder = path.normalize(this.taskParameters.sourceFolder); - const allPaths = tl.find(this.taskParameters.sourceFolder); // default find options (follow sym links) - tl.debug(tl.loc('AllPaths', allPaths)); - const matchedPaths = tl.match(allPaths, this.taskParameters.globExpressions, this.taskParameters.sourceFolder); // default match options - tl.debug(tl.loc('MatchedPaths', matchedPaths)); - const matchedFiles = matchedPaths.filter((itemPath) => !tl.stats(itemPath).isDirectory()); // filter-out directories - tl.debug(tl.loc('MatchedFiles', matchedFiles)); - tl.debug(tl.loc('FoundNFiles', matchedFiles.length)); - return matchedFiles; - } - -} diff --git a/Tasks/S3Upload/helpers/UploadTaskParameters.ts b/Tasks/S3Upload/helpers/UploadTaskParameters.ts deleted file mode 100644 index 6facf306..00000000 --- a/Tasks/S3Upload/helpers/UploadTaskParameters.ts +++ /dev/null @@ -1,84 +0,0 @@ -/* - Copyright 2017-2018 Amazon.com, Inc. and its affiliates. All Rights Reserved. - * - * Licensed under the MIT License. See the LICENSE accompanying this file - * for the specific language governing permissions and limitations under - * the License. - */ - -import tl = require('vsts-task-lib/task'); -import { AWSTaskParametersBase } from 'sdkutils/awsTaskParametersBase'; - -export class TaskParameters extends AWSTaskParametersBase { - - // options for Server-side encryption Key Management; 'none' disables SSE - public static readonly noKeyManagementValue: string = 'none'; - public static readonly awsKeyManagementValue: string = 'awsManaged'; - public static readonly customerKeyManagementValue: string = 'customerManaged'; - - // options for encryption algorithm when key management is set to 'aws'; - // customer managed keys always use AES256 - public static readonly awskmsAlgorithmValue: string = 'KMS'; // translated to aws:kms when used in api call - public static readonly aes256AlgorithmValue: string = 'AES256'; - - public bucketName: string; - public sourceFolder: string; - public targetFolder: string; - public flattenFolders: boolean; - public overwrite: boolean; - public globExpressions: string[]; - public filesAcl: string; - public createBucket: boolean; - public contentType: string; - public forcePathStyleAddressing: boolean; - public storageClass: string; - public keyManagement: string; - public encryptionAlgorithm: string; - public kmsMasterKeyId: string; - public customerKey: Buffer; - - constructor() { - super(); - try { - this.bucketName = tl.getInput('bucketName', true); - this.overwrite = tl.getBoolInput('overwrite', false); - this.flattenFolders = tl.getBoolInput('flattenFolders', false); - this.sourceFolder = tl.getPathInput('sourceFolder', true, true); - this.targetFolder = tl.getInput('targetFolder', false); - this.globExpressions = tl.getDelimitedInput('globExpressions', '\n', true); - this.filesAcl = tl.getInput('filesAcl', false); - this.createBucket = tl.getBoolInput('createBucket'); - this.contentType = tl.getInput('contentType', false); - this.forcePathStyleAddressing = tl.getBoolInput('forcePathStyleAddressing', false); - this.storageClass = tl.getInput('storageClass', false); - if (!this.storageClass) { - this.storageClass = 'STANDARD'; - } - - this.keyManagement = tl.getInput('keyManagement', false); - if (this.keyManagement && this.keyManagement !== TaskParameters.noKeyManagementValue) { - switch (this.keyManagement) { - case TaskParameters.awsKeyManagementValue: { - const algorithm = tl.getInput('encryptionAlgorithm', true); - if (algorithm === TaskParameters.awskmsAlgorithmValue) { - this.encryptionAlgorithm = 'aws:kms'; - } else { - this.encryptionAlgorithm = TaskParameters.aes256AlgorithmValue; - } - this.kmsMasterKeyId = tl.getInput('kmsMasterKeyId', algorithm === TaskParameters.awskmsAlgorithmValue); - } - break; - - case TaskParameters.customerKeyManagementValue: { - this.encryptionAlgorithm = TaskParameters.aes256AlgorithmValue; - const customerKey = tl.getInput('customerKey', true); - this.customerKey = Buffer.from(customerKey, 'hex'); - } - break; - } - } - } catch (error) { - throw new Error(error.message); - } - } -} diff --git a/Tasks/S3Upload/package.json b/Tasks/S3Upload/package.json index 36c79ab8..700756fa 100644 --- a/Tasks/S3Upload/package.json +++ b/Tasks/S3Upload/package.json @@ -9,8 +9,6 @@ "author": "", "license": "MIT", "dependencies": { - "aws-sdk": "2.279.1", - "sdkutils": "file:../../_build/Tasks/Common/sdkutils", - "vsts-task-lib": "^2.0.6" + "sdkutils": "file:../../_build/Tasks/Common/sdkutils" } } diff --git a/Tasks/S3Upload/task.json b/Tasks/S3Upload/task.json index 64fe2503..a3d88f5f 100644 --- a/Tasks/S3Upload/task.json +++ b/Tasks/S3Upload/task.json @@ -183,6 +183,15 @@ "helpMarkDown": "Sets a custom content type for the uploaded files. If a custom content type is not specified the task will apply built-in defaults for common file types (html, css, js, image files etc). This parameter can be used to override the built-in defaults.\n\n__Note:__ that any value is applied to __all__ files processed by the task.", "groupName": "advanced" }, + { + "name": "contentEncoding", + "type": "string", + "label": "Content Encoding", + "defaultValue": "", + "required": false, + "helpMarkDown": "Specifies the content encoding for the uploaded object(s). For more information go to [http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11](http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11).\n\n__Note:__ that any value is applied to __all__ files processed by the task.", + "groupName": "advanced" + }, { "name": "storageClass", "type": "pickList", diff --git a/Tasks/S3Upload/tsconfig.json b/Tasks/S3Upload/tsconfig.json deleted file mode 100644 index beeb3d52..00000000 --- a/Tasks/S3Upload/tsconfig.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "compilerOptions": { - "target": "ES6", - "module": "commonjs", - "sourceMap": true, - "alwaysStrict": true - }, - "exclude": [ - "node_modules" - ] -} diff --git a/package.json b/package.json index dcdc4f8e..5fefec1d 100644 --- a/package.json +++ b/package.json @@ -73,6 +73,7 @@ ], "moduleNameMapper": { "^sdkutils/(.*)": "/Tasks/Common/sdkutils/$1.ts", + "^Common/(.*)": "/Tasks/Common/$1.ts", "^aws-sdk/(.*)": "/node_modules/aws-sdk/$1.js" }, "transform": { diff --git a/tests/TaskTests/S3Download/S3Download-test.ts b/tests/TaskTests/S3Download/S3Download-test.ts index 6db17896..b86b2593 100644 --- a/tests/TaskTests/S3Download/S3Download-test.ts +++ b/tests/TaskTests/S3Download/S3Download-test.ts @@ -20,17 +20,18 @@ describe('S3 Download', () => { bucketName: '', sourceFolder: '', targetFolder: '', - globExpressions: undefined, + globExpressions: [], overwrite: false, forcePathStyleAddressing: false, flattenFolders: false, keyManagement: '', - customerKey: undefined + customerKey: Buffer.from([]) } const headBucketResponse = { promise: function() { } } + const listObjectsResponse = { promise: function() { return { NextMarker: undefined, Contents: undefined} @@ -60,12 +61,12 @@ describe('S3 Download', () => { test('Creates a TaskOperation', () => { const taskParameters = baseTaskParameters - expect(new TaskOperations(undefined, taskParameters)).not.toBeNull() + expect(new TaskOperations(new S3(), taskParameters)).not.toBeNull() }) test('Handles not being able to connect to a bucket', async () => { const s3 = new S3({ region: 'us-east-1' }) - s3.headBucket = jest.fn()((params, cb) => { throw new Error('doesn\'t exist dummy') }) + s3.headBucket = jest.fn()((params: any, cb: any) => { throw new Error('doesn\'t exist dummy') }) const taskParameters = baseTaskParameters const taskOperation = new TaskOperations(s3, taskParameters) expect.assertions(1) diff --git a/tests/TaskTests/S3Upload/S3Upload-test.ts b/tests/TaskTests/S3Upload/S3Upload-test.ts new file mode 100644 index 00000000..03ee4021 --- /dev/null +++ b/tests/TaskTests/S3Upload/S3Upload-test.ts @@ -0,0 +1,124 @@ +/*! + * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: MIT + */ + +import { S3 } from 'aws-sdk' +import { SdkUtils } from '../../../Tasks/Common/sdkutils/sdkutils' +import { TaskOperations } from '../../../Tasks/S3Upload/UploadTaskOperations' +import { TaskParameters } from '../../../Tasks/S3Upload/UploadTaskParameters' + +// unsafe any's is how jest mocking works, so this needs to be disabled for all test files +// tslint:disable: no-unsafe-any +jest.mock('aws-sdk') + +describe('S3 Upload', () => { + const baseTaskParameters: TaskParameters = { + awsConnectionParameters: undefined, + bucketName: '', + sourceFolder: '', + targetFolder: '', + flattenFolders: false, + overwrite: false, + globExpressions: [], + filesAcl: '', + createBucket: false, + contentType: '', + contentEncoding: '', + forcePathStyleAddressing: false, + storageClass: '', + keyManagement: '', + encryptionAlgorithm: '', + kmsMasterKeyId: '', + customerKey: Buffer.from([]) + } + + const connectionParameters = { + proxyConfiguration: '', + logRequestData: true, + logResponseData: true, + AssumeRoleARN: '', + awsEndpointAuth: undefined + } + + const headBucketResponse = { + promise: function() { } + } + + const headBucketResponseFails = { + promise: function() { throw new Error('doesn\'t exist') } + } + + const createBucketResponse = { + promise: function() { throw new Error('create called') } + } + + const validateUpload = { + promise: function() { return undefined } + } + + // TODO https://github.com/aws/aws-vsts-tools/issues/167 + beforeAll(() => { + SdkUtils.readResourcesFromRelativePath('../../../_build/Tasks/S3Upload/task.json') + }) + + test('Creates a TaskOperation', () => { + const taskParameters = baseTaskParameters + expect(new TaskOperations(new S3(), '', taskParameters)).not.toBeNull() + }) + + test('Handles bucket not existing (and not being able to create one)', async () => { + const s3 = new S3({ region: 'us-east-1' }) as any + s3.headBucket = jest.fn()((params: any, cb: any) => headBucketResponseFails) + const taskParameters = baseTaskParameters + const taskOperation = new TaskOperations(s3, '', taskParameters) + expect.assertions(1) + await taskOperation.execute().catch((e) => { expect(e.message).toContain('not exist') }) + }) + + test('Tries and fails to create bucket when told to', async () => { + const s3 = new S3({ region: 'us-east-1' }) as any + s3.headBucket = jest.fn((params: any, cb: any) => headBucketResponseFails) + s3.createBucket = jest.fn((params: any, cb: any ) => createBucketResponse) + const taskParameters = {...baseTaskParameters} + taskParameters.awsConnectionParameters = connectionParameters + taskParameters.createBucket = true + taskParameters.bucketName = 'potato' + const taskOperation = new TaskOperations(s3, '', taskParameters) + expect.assertions(1) + await taskOperation.execute().catch((e) => { expect(e.message).toContain('create called') }) + }) + + test('Finds matching files', () => { + const s3 = new S3({ region: 'us-east-1' }) + const taskParameters = {...baseTaskParameters} + taskParameters.bucketName = 'potato' + taskParameters.sourceFolder = __dirname + taskParameters.globExpressions = [ '*.ts' ] + const taskOperation = new TaskOperations(s3, '', taskParameters) + const results = taskOperation.findMatchingFiles(taskParameters) + // expect it to find this file only + expect(results.length).toBe(1) + }) + + test('Happy path uplaods a found file', async () => { + const s3 = new S3({ region: 'us-east-1' }) as any + s3.headBucket = jest.fn((params: any, cb: any) => headBucketResponse) + s3.upload = jest.fn((params: S3.PutObjectRequest, cb: any) => { + expect(params.Bucket).toBe('potato') + expect(params.Key).toContain('ts') + + return validateUpload + }) + const taskParameters = {...baseTaskParameters} + taskParameters.awsConnectionParameters = connectionParameters + taskParameters.createBucket = true + taskParameters.bucketName = 'potato' + taskParameters.sourceFolder = __dirname + taskParameters.globExpressions = [ '*.ts' ] + const taskOperation = new TaskOperations(s3, '', taskParameters) + expect.assertions(3) + await taskOperation.execute() + expect(s3.upload.mock.calls.length).toBe(1) + }) +}) diff --git a/tsconfig.json b/tsconfig.json index 343e0d36..89f7b075 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,8 +9,9 @@ //"strict": true, "paths": { "aws-sdk*": [ "node_modules/aws-sdk/*" ], - "sdkutils*": [ "Tasks/Common/sdkutils/*" ], - "beanstalkutils*": [ "Tasks/Common/beanstalkutils*" ], + "Common*": [ "Tasks/Common*" ], + "sdkutils*": [ "Tasks/Common/sdkutils*" ], + "beanstalkutils*": [ "Tasks/Common/beanstalkutils*" ], "cloudformationutils*": [ "Tasks/Common/cloudformationutils*" ] }, "rootDirs" : [ diff --git a/tslint.yaml b/tslint.yaml index 6a586f3a..e268dd94 100644 --- a/tslint.yaml +++ b/tslint.yaml @@ -16,12 +16,14 @@ linterOptions: - Tasks/CloudFormationDeleteStack/** - Tasks/CloudFormationExecuteChangeSet/** - Tasks/CodeDeployDeployApplication/** - - Tasks/Common/** + - Tasks/Common/beanstalkutils/** + - Tasks/Common/sdkutils/awsTaskParametersBase.ts + - Tasks/Common/sdkutils/sdkutils.ts + - Tasks/Common/sdkutils/cloudformationutils.ts - Tasks/ECRPushImage/** - Tasks/LambdaDeployFunction/** - Tasks/LambdaInvokeFunction/** - Tasks/LambdaNETCoreDeploy/** - - Tasks/S3Upload/** - Tasks/SecretsManagerCreateOrUpdateSecret/** - Tasks/SecretsManagerGetSecret/** - Tasks/SendMessage/** diff --git a/webpack.config.js b/webpack.config.js index c6ada698..003d3c1c 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -11,10 +11,11 @@ module.exports = { }, resolve: { alias: { - "aws-sdk": path.resolve(__dirname, 'node_modules/aws-sdk'), - "sdkutils": path.resolve(__dirname, '_build/Tasks/Common/sdkutils'), - "beanstalkutils": path.resolve(__dirname, '_build/Tasks/Common/beanstalkutils'), - } + "aws-sdk": path.resolve(__dirname, 'node_modules/aws-sdk'), + "sdkutils": path.resolve(__dirname, '_build/Tasks/Common/sdkutils'), + "Common": path.resolve(__dirname, '_build/Tasks/Common'), + "beanstalkutils": path.resolve(__dirname, '_build/Tasks/Common/beanstalkutils') + } }, externals: { "vsts-task-lib/task": 'require("vsts-task-lib/task")'