Skip to content

Commit

Permalink
refactor(Console): Dynamically resolve latest version of the extension
Browse files Browse the repository at this point in the history
  • Loading branch information
@medikoo
medikoo committed Mar 14, 2022
1 parent dd421cc commit c28c48c
Show file tree
Hide file tree
Showing 9 changed files with 115 additions and 67 deletions.
107 changes: 74 additions & 33 deletions lib/classes/console.js
View file
Expand Up @@ -4,18 +4,25 @@ const _ = require('lodash');
const d = require('d');
const lazy = require('d/lazy');
const path = require('path');
const os = require('os');
const fsp = require('fs').promises;
const fetch = require('node-fetch');
const tar = require('tar');
const filesize = require('filesize');
const provisionTmpDir = require('process-utils/tmpdir/provision');
const resolvePackageVersionMetadata = require('npm-registry-utilities/resolve-version-metadata');
const log = require('@serverless/utils/log').log.get('console');
const isAuthenticated = require('@serverless/dashboard-plugin/lib/is-authenticated');
const { getPlatformClientWithAccessKey } = require('@serverless/dashboard-plugin/lib/client-utils');
const ServerlessError = require('../serverless-error');
const ensureExists = require('../utils/ensure-exists');
const safeMoveFile = require('../utils/fs/safe-move-file');
const { setBucketName } = require('../plugins/aws/lib/set-bucket-name');
const { uploadZipFile } = require('../plugins/aws/lib/upload-zip-file');

const supportedCommands = new Set(['deploy', 'deploy function', 'package', 'remove', 'rollback']);
const devVersionTimeBase = new Date(2022, 1, 17).getTime();
const extensionCachePath = path.resolve(os.homedir(), '.serverless/aws-lambda-otel-extension');

class Console {
constructor(serverless) {
Expand Down Expand Up @@ -230,90 +237,97 @@ class Console {
overrideSettings({ otelIngestionToken, extensionLayerVersionPostfix, service, stage }) {
Object.defineProperties(this, {
deferredOtelIngestionToken: d(Promise.resolve(otelIngestionToken)),
extensionLayerVersionPostfix: d(extensionLayerVersionPostfix),
deferredExtensionLayerVersionPostfix: d(Promise.resolve(extensionLayerVersionPostfix)),
service: d('cew', service),
stage: d('cew', stage),
});
}

compileOtelExtensionLayer() {
log.debug('compile extension resource (%s)', this.extensionLayerName);
async compileOtelExtensionLayer() {
const layerName = await this.deferredExtensionLayerName;
log.debug('compile extension resource (%s)', layerName);
this.serverless.service.provider.compiledCloudFormationTemplate.Resources[
this.provider.naming.getConsoleExtensionLayerLogicalId()
] = {
Type: 'AWS::Lambda::LayerVersion',
Properties: {
Content: {
S3Bucket: { Ref: 'ServerlessDeploymentBucket' },
S3Key: `${this.serverless.service.package.artifactsS3KeyDirname}/${this.extensionLayerFilename}`,
S3Key: `${this.serverless.service.package.artifactsS3KeyDirname}/${await this
.deferredExtensionLayerBasename}`,
},
LayerName: this.extensionLayerName,
LayerName: layerName,
},
};
}

async packageOtelExtensionLayer() {
log.debug('copy extension file (%s) to package directory', this.extensionLayerFilename);
const layerFilename = await this.deferredExtensionLayerFilename;
log.debug('copy extension file (%s) to package directory', layerFilename);
await fsp.copyFile(
require.resolve('@serverless/aws-lambda-otel-extension-dist/extension.zip'),
path.join(this.serverless.serviceDir, '.serverless', this.extensionLayerFilename)
layerFilename,
path.join(
this.serverless.serviceDir,
'.serverless',
await this.deferredExtensionLayerBasename
)
);
}

async ensureLayerVersion() {
const layerName = await this.deferredExtensionLayerName;
let layerVersionMeta = (
await this.provider.request('Lambda', 'listLayerVersions', {
LayerName: this.extensionLayerName,
LayerName: layerName,
})
).LayerVersions[0];
if (!layerVersionMeta) {
log.debug('publish layer version (%s)', this.extensionLayerName);
log.debug('publish layer version (%s)', layerName);
await this.uploadOtelExtensionLayer({ readFromTheSource: true });
await setBucketName.call(this);
await this.provider.request('Lambda', 'publishLayerVersion', {
LayerName: this.extensionLayerName,
LayerName: layerName,
Content: {
S3Bucket: this.bucketName,
S3Key: `${this.serverless.service.package.artifactsS3KeyDirname}/${this.extensionLayerFilename}`,
S3Key: `${this.serverless.service.package.artifactsS3KeyDirname}/${await this
.deferredExtensionLayerBasename}`,
},
});
layerVersionMeta = (
await this.provider.request('Lambda', 'listLayerVersions', {
LayerName: this.extensionLayerName,
LayerName: layerName,
})
).LayerVersions[0];
} else {
log.debug('layer version already published (%s)', this.extensionLayerName);
log.debug('layer version already published (%s)', layerName);
}
log.debug('retrieved layer version arn (%s)', layerVersionMeta.LayerVersionArn);
return layerVersionMeta.LayerVersionArn;
}

async uploadOtelExtensionLayer(options = {}) {
log.debug(
'check if extension file (%s) is already uploaded to S3',
this.extensionLayerFilename
);
const layerBasename = await this.deferredExtensionLayerBasename;
log.debug('check if extension file (%s) is already uploaded to S3', layerBasename);
await setBucketName.call(this);
try {
await this.provider.request('S3', 'headObject', {
Bucket: this.bucketName,
Key: `${this.serverless.service.package.artifactsS3KeyDirname}/${this.extensionLayerFilename}`,
Key: `${this.serverless.service.package.artifactsS3KeyDirname}/${layerBasename}`,
});
// Extension layer is already available at S3, skip
log.debug('extension file is already uploaded to S3');
return;
} catch (error) {
if (error.code !== 'AWS_S3_HEAD_OBJECT_NOT_FOUND') throw error;
const filename = options.readFromTheSource
? require.resolve('@serverless/aws-lambda-otel-extension-dist/extension.zip')
: path.join(this.packagePath, this.extensionLayerFilename);
? await this.deferredExtensionLayerFilename
: path.join(this.packagePath, layerBasename);
const stats = await fsp.stat(filename);
log.info(`Uploading console otel extension file to S3 (${filesize(stats.size)})`);
await uploadZipFile.call(this, {
filename,
s3KeyDirname: this.serverless.service.package.artifactsS3KeyDirname,
basename: this.extensionLayerFilename,
basename: layerBasename,
});
}
}
Expand All @@ -338,19 +352,46 @@ Object.defineProperties(
deferredOtelIngestionToken: d(function () {
return this.createOtelIngestionToken();
}),
extensionLayerName: d(function () {
return `sls-console-otel-extension-${this.extensionLayerVersionPostfix.replace(/\./g, '-')}`;
deferredExtensionLayerFilename: d(async () => {
if (process.env.SLS_OTEL_LAYER_FILENAME) {
log.debug('target extension filename (overriden): %s', process.env.SLS_OTEL_LAYER_FILENAME);
return process.env.SLS_OTEL_LAYER_FILENAME;
}
const extensionVersionMetadata = await resolvePackageVersionMetadata(
'@serverless/aws-lambda-otel-extension-dist',
'^0.1'
);
log.debug('target extension version: %s', extensionVersionMetadata.version);
const extensionArtifactFilename = path.resolve(
extensionCachePath,
`${extensionVersionMetadata.version}.zip`
);
await ensureExists(extensionArtifactFilename, async () => {
log.debug('resolving extension layer from npm registry');
const tmpDir = await provisionTmpDir();
const response = await fetch(extensionVersionMetadata.dist.tarball);
await new Promise((resolve, reject) => {
const stream = response.body.pipe(tar.x({ cwd: tmpDir, strip: 1 }));
stream.on('error', reject);
stream.on('end', resolve);
});
await safeMoveFile(path.resolve(tmpDir, 'extension.zip'), extensionArtifactFilename);
});
return extensionArtifactFilename;
}),
deferredExtensionLayerName: d(async function () {
return `sls-console-otel-extension-${(
await this.deferredExtensionLayerVersionPostfix
).replace(/\./g, '-')}`;
}),
extensionLayerFilename: d(function () {
return `sls-otel.${this.extensionLayerVersionPostfix}.zip`;
deferredExtensionLayerBasename: d(async function () {
return `sls-otel.${await this.deferredExtensionLayerVersionPostfix}.zip`;
}),
extensionLayerVersionPostfix: d(() => {
if (process.env.SLS_OTEL_LAYER_VERSION) return process.env.SLS_OTEL_LAYER_VERSION;
const installedVersion =
require('@serverless/aws-lambda-otel-extension-dist/package').version;
if (installedVersion) return installedVersion;
// If we link to package in repository, then there's no version exposed
return (Date.now() - devVersionTimeBase).toString(32);
deferredExtensionLayerVersionPostfix: d(async function () {
if (process.env.SLS_OTEL_LAYER_FILENAME) {
return (Date.now() - devVersionTimeBase).toString(32);
}
return path.basename(await this.deferredExtensionLayerFilename, '.zip');
}),
})
);
Expand Down
16 changes: 10 additions & 6 deletions lib/plugins/aws/package/compile/functions.js
View file
Expand Up @@ -471,14 +471,18 @@ class AwsCompileFunctions {
}
// Include all referenced layer code in the version id hash
const layerArtifactPaths = [];
const consoleExtensionLayerName =
this.console.isEnabled && (await this.console.deferredExtensionLayerName);
const consoleExtensionBasename =
this.console.isEnabled && (await this.console.deferredExtensionLayerBasename);
layerConfigurations.forEach((layer) => {
if (!layer.name && layer.properties.LayerName === this.console.extensionLayerName) {
if (
!layer.name &&
consoleExtensionLayerName &&
layer.properties.LayerName === consoleExtensionLayerName
) {
layerArtifactPaths.push(
path.join(
this.serverless.serviceDir,
'.serverless',
this.serverless.console.extensionLayerFilename
)
path.join(this.serverless.serviceDir, '.serverless', consoleExtensionBasename)
);
return;
}
Expand Down
4 changes: 3 additions & 1 deletion lib/plugins/aws/package/compile/layers.js
View file
Expand Up @@ -170,7 +170,9 @@ class AwsCompileLayers {
this.compileLayer(layerName).then(() => this.compareWithLastLayer(layerName))
)
);
if (this.serverless.console.isEnabled) this.serverless.console.compileOtelExtensionLayer();
if (this.serverless.console.isEnabled) {
await this.serverless.console.compileOtelExtensionLayer();
}
}

cfLambdaLayerTemplate() {
Expand Down
2 changes: 1 addition & 1 deletion lib/plugins/aws/package/lib/save-service-state.js
View file
Expand Up @@ -38,7 +38,7 @@ module.exports = {
state.console = {
schemaVersion: this.console.stateSchemaVersion,
otelIngestionToken: await this.console.deferredOtelIngestionToken,
extensionLayerVersionPostfix: this.console.extensionLayerVersionPostfix,
extensionLayerVersionPostfix: await this.console.deferredExtensionLayerVersionPostfix,
service: this.console.service,
stage: this.console.stage,
orgId: this.console.orgId,
Expand Down
2 changes: 0 additions & 2 deletions lib/utils/telemetry/generate-payload.js
View file
Expand Up @@ -184,8 +184,6 @@ module.exports = ({
return {
'serverless': require('../../../package').version,
'@serverless/dashboard-plugin': require('@serverless/dashboard-plugin/package').version,
'@serverless/aws-lambda-otel-extension-dist':
require('@serverless/aws-lambda-otel-extension-dist/package').version,
};
})();

Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Expand Up @@ -22,7 +22,6 @@
"sls": "./bin/serverless.js"
},
"dependencies": {
"@serverless/aws-lambda-otel-extension-dist": "^0.1.9",
"@serverless/dashboard-plugin": "^6.1.5",
"@serverless/platform-client": "^4.3.2",
"@serverless/utils": "^6.0.3",
Expand Down Expand Up @@ -59,6 +58,7 @@
"memoizee": "^0.4.15",
"micromatch": "^4.0.4",
"node-fetch": "^2.6.7",
"npm-registry-utilities": "^1.0.0",
"object-hash": "^2.2.0",
"open": "^7.4.2",
"path2": "^0.1.0",
Expand Down
2 changes: 0 additions & 2 deletions scripts/pkg/config.js
View file
Expand Up @@ -20,8 +20,6 @@ module.exports = {
// Dashboard wrappers
'../../node_modules/@serverless/dashboard-plugin/sdk-js/dist/index.js',
'../../node_modules/@serverless/dashboard-plugin/sdk-py',
// Console extension
'../../node_modules/@serverless/aws-lambda-otel-extension-dist',
// Ensure npm is bundled as a dependency
'../../node_modules/npm/bin/npm-cli.js',
// Below module is not automatically traced by pkg, we need to point it manually
Expand Down
45 changes: 26 additions & 19 deletions test/unit/lib/classes/console.test.js
View file
Expand Up @@ -5,6 +5,7 @@ const sinon = require('sinon');
const path = require('path');
const fsp = require('fs').promises;
const _ = require('lodash');
const fetch = require('node-fetch');
const log = require('log').get('serverless:test');
const runServerless = require('../../../utils/run-serverless');
const getRequire = require('../../../../lib/utils/get-require');
Expand All @@ -17,7 +18,7 @@ const createFetchStub = () => {
const requests = [];
return {
requests,
stub: sinon.stub().callsFake(async (url, { method }) => {
stub: sinon.stub().callsFake(async (url, { method } = { method: 'GET' }) => {
log.debug('fetch request %s %o', url, method);
if (url.includes('/org/')) {
if (method.toUpperCase() === 'GET') {
Expand Down Expand Up @@ -48,7 +49,8 @@ const createFetchStub = () => {
return { ok: true, text: async () => '' };
}
}
throw new Error('Unexpected request');
if (url.startsWith('https://registry.npmjs.org')) return fetch(url, { method });
throw new Error(`Unexpected request: ${url} method: ${method}`);
}),
};
};
Expand Down Expand Up @@ -188,15 +190,24 @@ describe('test/unit/lib/classes/console.test.js', () => {
awsNaming.getConsoleExtensionLayerLogicalId()
);
await fsp.access(
path.resolve(servicePath, '.serverless', serverless.console.extensionLayerFilename)
path.resolve(
servicePath,
'.serverless',
await serverless.console.deferredExtensionLayerBasename
)
);
});

it('should upload extension layer to S3', () => {
it('should upload extension layer to S3', async () => {
const consoleExtensionLayerBasename = await serverless.console
.deferredExtensionLayerBasename;
log.debug(
'layer basename: %s, s3Keys: %o',
consoleExtensionLayerBasename,
uploadStub.args.map(([{ Key: s3Key }]) => s3Key)
);
expect(
uploadStub.args.some(([{ Key: s3Key }]) =>
s3Key.endsWith(serverless.console.extensionLayerFilename)
)
uploadStub.args.some(([{ Key: s3Key }]) => s3Key.endsWith(consoleExtensionLayerBasename))
).to.be.true;
});

Expand Down Expand Up @@ -259,12 +270,10 @@ describe('test/unit/lib/classes/console.test.js', () => {
expect(consoleDeploy.serviceId).to.equal(consolePackage.serviceId);
});

it('should upload extension layer to S3', () => {
expect(
uploadStub.args.some(([{ Key: s3Key }]) =>
s3Key.endsWith(consoleDeploy.extensionLayerFilename)
)
).to.be.true;
it('should upload extension layer to S3', async () => {
const extensionLayerFilename = await consoleDeploy.deferredExtensionLayerBasename;
expect(uploadStub.args.some(([{ Key: s3Key }]) => s3Key.endsWith(extensionLayerFilename))).to
.be.true;
});

it('should activate otel ingestion token', () => {
Expand Down Expand Up @@ -326,12 +335,10 @@ describe('test/unit/lib/classes/console.test.js', () => {
expect(fnVariables).to.have.property('AWS_LAMBDA_EXEC_WRAPPER');
});

it('should upload extension layer to S3', () => {
expect(
uploadStub.args.some(([{ Key: s3Key }]) =>
s3Key.endsWith(serverless.console.extensionLayerFilename)
)
).to.be.true;
it('should upload extension layer to S3', async () => {
const extensionLayerFilename = await serverless.console.deferredExtensionLayerBasename;
expect(uploadStub.args.some(([{ Key: s3Key }]) => s3Key.endsWith(extensionLayerFilename))).to
.be.true;
});

it('should activate otel ingestion token', () => {
Expand Down
2 changes: 0 additions & 2 deletions test/unit/lib/utils/telemetry/generate-payload.test.js
View file
Expand Up @@ -14,8 +14,6 @@ const fixtures = require('../../../../fixtures/programmatic');
const versions = {
'serverless': require('../../../../../package').version,
'@serverless/dashboard-plugin': require('@serverless/dashboard-plugin/package').version,
'@serverless/aws-lambda-otel-extension-dist':
require('@serverless/aws-lambda-otel-extension-dist/package').version,
};

const getGeneratePayload = () =>
Expand Down

0 comments on commit c28c48c

Please sign in to comment.