diff --git a/packages/generate/src/extended-docker-syntax.ts b/packages/generate/src/extended-docker-syntax.ts index f74378a0..179040a7 100644 --- a/packages/generate/src/extended-docker-syntax.ts +++ b/packages/generate/src/extended-docker-syntax.ts @@ -29,7 +29,7 @@ async function applyExtendedDockerSyntaxCopy(step: string, pkg: Package): Promis const [_, fromStage, chown, ifExistsFlag, slimFlag, filesList, destination] = step.match(isCopy)!; if (fromStage) { const [_, fromStageName] = fromStage.match(/--from=(\S*)/) ?? []; - const isLocalStage = pkg.dockerFile!.find(x => x.originalName === fromStageName); + const isLocalStage = pkg.dockerFile!.stages.find(x => x.originalName === fromStageName); if (isLocalStage) { return `COPY --from=${isLocalStage.name} ${chown || ''} ${filesList} ${destination}`; } diff --git a/packages/generate/src/iterate-dependencies.ts b/packages/generate/src/iterate-dependencies.ts index 6cb2d5c5..43084387 100644 --- a/packages/generate/src/iterate-dependencies.ts +++ b/packages/generate/src/iterate-dependencies.ts @@ -3,7 +3,7 @@ import { PackageGraph } from '@lerna/package-graph'; import { runTopologically } from '@lerna/run-topologically'; import { IGenerateArgs } from './args'; import { Package } from './package'; -import { DockerStage } from './read-dockerfile'; +import { Dockerfile } from './read-dockerfile'; export async function iterateDependencies( args: IGenerateArgs, @@ -11,7 +11,7 @@ export async function iterateDependencies( packageGraph: PackageGraph, concurrency: number, rejectCycles?: boolean, - defaultDockerFile?: DockerStage[], + defaultDockerFile?: Dockerfile, ): Promise { const packages: Package[] = []; await runTopologically( diff --git a/packages/generate/src/lerna-command.ts b/packages/generate/src/lerna-command.ts index 9c4ff1a6..01612562 100644 --- a/packages/generate/src/lerna-command.ts +++ b/packages/generate/src/lerna-command.ts @@ -32,11 +32,13 @@ export class Dockerize extends Command { defaultDockerFile = await readDockerfile(templateDockerFileName); } - const baseStage = baseDockerFile[baseDockerFile.length - 1]; + const baseStage = baseDockerFile.stages[baseDockerFile.stages.length - 1]; baseStage.name = 'base'; - const result: string[] = []; - for (let baseStage of baseDockerFile) { + const result: string[] = [ + ...baseDockerFile.preStage, + ]; + for (let baseStage of baseDockerFile.stages) { result.push(getDockerFileFromInstruction(baseStage.baseImage, baseStage.name, baseStage.platform)); result.push(...baseStage.stepsBeforeInstall); if (baseStage.install) { @@ -93,7 +95,7 @@ export class Dockerize extends Command { // overwrite final docker stage if (finalDockerfileName) { - finalStages = await readDockerfile(finalDockerfileName); + finalStages = (await readDockerfile(finalDockerfileName)).stages; } for (let finalStage of finalStages) { diff --git a/packages/generate/src/package.ts b/packages/generate/src/package.ts index 31ef94dd..c7649c85 100644 --- a/packages/generate/src/package.ts +++ b/packages/generate/src/package.ts @@ -1,6 +1,6 @@ import { PackageGraphNode } from '@lerna/package-graph'; import { Package as LernaPackage } from '@lerna/package'; -import { DockerStage, readDockerfile } from './read-dockerfile'; +import { Dockerfile, DockerStage, readDockerfile } from './read-dockerfile'; import { promises } from 'fs'; import { join as joinPath, relative } from 'path'; import { getDependenciesTransitive } from './get-dependencies-transitive'; @@ -15,7 +15,7 @@ export type PackageMap = Map; export class Package { - dockerFile?: DockerStage[]; + dockerFile?: Dockerfile; constructor( public name: string, @@ -46,7 +46,7 @@ export class Package { return undefined; } - async loadDockerfile(defaultDockerFile?: DockerStage[]): Promise { + async loadDockerfile(defaultDockerFile?: Dockerfile): Promise { const dockerFileName = await this.findDockerfile(); if (!dockerFileName && !defaultDockerFile) { throw new Error(`No Dockerfile for the package ${this.name} and no default docker file was found!`); @@ -58,7 +58,10 @@ export class Package { getLogger().info(`using custom dockerfile for package ${this.name}`); this.dockerFile = await readDockerfile(dockerFileName); } - this.dockerFile = this.dockerFile!.map((stage, i) => this.scopeDockerStage(stage, i)); + this.dockerFile = { + stages: this.dockerFile!.stages.map((stage, i) => this.scopeDockerStage(stage, i)), + preStage: this.dockerFile!.preStage, + }; return this.dockerFile!; } @@ -76,7 +79,7 @@ export class Package { if (!this.args.addPrepareStages) { return this.getBuildStageName(); } - return this.dockerFile[this.dockerFile.length - 1].prepareStageName; + return this.dockerFile.stages[this.dockerFile.stages.length - 1].prepareStageName; } getBuildStageName(): string | undefined { @@ -87,7 +90,7 @@ export class Package { if (!this.dockerFile) { return undefined; } - return this.dockerFile[this.dockerFile.length - 1]; + return this.dockerFile.stages[this.dockerFile.stages.length - 1]; } stageHasInstall(stage?: DockerStage): boolean { @@ -102,9 +105,9 @@ export class Package { return []; } const result: string[] = []; - for (let stage of this.dockerFile) { + for (let stage of this.dockerFile.stages) { let baseImage = stage.baseImage; - const baseImageIsLocalStage = this.dockerFile!.find(x => x.originalName === baseImage); + const baseImageIsLocalStage = this.dockerFile!.stages.find(x => x.originalName === baseImage); if (baseImageIsLocalStage) { baseImage = baseImageIsLocalStage.name!; } diff --git a/packages/generate/src/read-dockerfile.ts b/packages/generate/src/read-dockerfile.ts index f4f643b0..3d1215ca 100644 --- a/packages/generate/src/read-dockerfile.ts +++ b/packages/generate/src/read-dockerfile.ts @@ -19,12 +19,18 @@ export interface DockerStage { install?: IInstallOptions; } +export interface Dockerfile { + stages: DockerStage[]; + preStage: string[]; +} + -export async function readDockerfile(path: string): Promise { +export async function readDockerfile(path: string): Promise { const dockerfile = (await promises.readFile(path)).toString(); const steps = splitInSteps(dockerfile); const result = []; let currentStep = 0; + let preStage: string[] = []; while (true) { const readStageResult = readStage(steps, currentStep); if (!readStageResult) { @@ -32,19 +38,26 @@ export async function readDockerfile(path: string): Promise { } currentStep = readStageResult.endIndex + 1; result.push(readStageResult.stage); + if (preStage.length === 0 && readStageResult.preStage.length > 0) { + preStage = readStageResult.preStage; + } } if (result.length === 0) { getLogger().warn(`The dockerfile '${path}' appears to be empty.`); } - return result; + return { + preStage: preStage, + stages: result, + }; } -export function readStage(steps: string[], startIndex: number): { stage: DockerStage; endIndex: number } | undefined { +export function readStage(steps: string[], startIndex: number): { stage: DockerStage; endIndex: number; preStage: string[] } | undefined { let i = startIndex; let baseImage; let stageName; let platform; - const isStageFromClause = /(FROM|from)(\s--platform=(\S+))? ([a-zA-Z0-9:_\-@.\/]*)( as ([a-zA-Z0-9:_-]*))?/; + const preStage = []; + const isStageFromClause = /(FROM|from)(\s--platform=(\S+))? ([a-zA-Z0-9:_\-@.\/${}]*)( as ([a-zA-Z0-9:_-]*))?/; for (; ; i++) { if (i >= steps.length) { return undefined; @@ -57,6 +70,7 @@ export function readStage(steps: string[], startIndex: number): { stage: DockerS i++; break; } + preStage.push(steps[i]); } const stepsBeforeInstall: string[] = []; const stepsAfterInstall: string[] = []; @@ -93,6 +107,7 @@ export function readStage(steps: string[], startIndex: number): { stage: DockerS stepsAfterInstall, install: installHit, }, + preStage, }; } diff --git a/packages/generate/test/read-dockerfile.spec.ts b/packages/generate/test/read-dockerfile.spec.ts index 424bcde4..83d557d5 100644 --- a/packages/generate/test/read-dockerfile.spec.ts +++ b/packages/generate/test/read-dockerfile.spec.ts @@ -21,7 +21,10 @@ describe('readDockerfile', () => { getLogger().level = 'error'; const result = await readDockerfile('Dockerfile'); getLogger().level = oldLogLevel; - expect(result).toEqual([]); + expect(result).toEqual({ + stages: [], + preStage: [], + }); }); it('should return one stage from dockerfile', async function (this: ReadDockerfileThisContext) { @@ -32,18 +35,51 @@ describe('readDockerfile', () => { 'RUN npm run prepare', ].join('\n')); const result = await readDockerfile('Dockerfile'); - expect(result).toEqual([{ - baseImage: 'nginx:latest', - platform: undefined, - name: undefined, - stepsBeforeInstall: ['COPY ./file ./somewhere'], - stepsAfterInstall: ['RUN npm run prepare'], - install: { - ci: undefined, - ignoreScripts: false, - onlyProduction: false, - }, - }]); + expect(result).toEqual({ + preStage: [], + stages: [{ + baseImage: 'nginx:latest', + platform: undefined, + name: undefined, + stepsBeforeInstall: ['COPY ./file ./somewhere'], + stepsAfterInstall: ['RUN npm run prepare'], + install: { + ci: undefined, + ignoreScripts: false, + onlyProduction: false, + }, + }], + }); + }); + + it('should return one stage and pre stage args with from dockerfile', async function (this: ReadDockerfileThisContext) { + this.fsReadFile.and.resolveTo([ + 'ARG FOO=bar', + 'ARG version=1.0.0', + 'FROM nginx:${version}', + 'COPY ./file ./somewhere', + 'RUN npm i', + 'RUN npm run prepare', + ].join('\n')); + const result = await readDockerfile('Dockerfile'); + expect(result).toEqual({ + preStage: [ + 'ARG FOO=bar', + 'ARG version=1.0.0', + ], + stages: [{ + baseImage: 'nginx:${version}', + platform: undefined, + name: undefined, + stepsBeforeInstall: ['COPY ./file ./somewhere'], + stepsAfterInstall: ['RUN npm run prepare'], + install: { + ci: undefined, + ignoreScripts: false, + onlyProduction: false, + }, + }], + }); }); it('should read stage with custom registry base image', async function (this: ReadDockerfileThisContext) { @@ -51,14 +87,17 @@ describe('readDockerfile', () => { 'FROM some.registry.com/somewhere/something/nginx:latest', ].join('\n')); const result = await readDockerfile('Dockerfile'); - expect(result).toEqual([{ - baseImage: 'some.registry.com/somewhere/something/nginx:latest', - platform: undefined, - name: undefined, - stepsBeforeInstall: [], - stepsAfterInstall: [], - install: undefined, - }]); + expect(result).toEqual({ + preStage: [], + stages: [{ + baseImage: 'some.registry.com/somewhere/something/nginx:latest', + platform: undefined, + name: undefined, + stepsBeforeInstall: [], + stepsAfterInstall: [], + install: undefined, + }], + }); }); it('should read stage with custom platform', async function (this: ReadDockerfileThisContext) { @@ -66,14 +105,17 @@ describe('readDockerfile', () => { 'FROM --platform=arm/64 nginx:latest', ].join('\n')); const result = await readDockerfile('Dockerfile'); - expect(result).toEqual([{ - baseImage: 'nginx:latest', - platform: 'arm/64', - name: undefined, - stepsBeforeInstall: [], - stepsAfterInstall: [], - install: undefined, - }]); + expect(result).toEqual({ + preStage: [], + stages: [{ + baseImage: 'nginx:latest', + platform: 'arm/64', + name: undefined, + stepsBeforeInstall: [], + stepsAfterInstall: [], + install: undefined, + }], + }); }); it('should return one stage from dockerfile with yarn install', async function (this: ReadDockerfileThisContext) { @@ -84,18 +126,21 @@ describe('readDockerfile', () => { 'RUN npm run prepare', ].join('\n')); const result = await readDockerfile('Dockerfile'); - expect(result).toEqual([{ - baseImage: 'nginx:latest', - platform: undefined, - name: undefined, - stepsBeforeInstall: ['COPY ./file ./somewhere'], - stepsAfterInstall: ['RUN npm run prepare'], - install: { - ci: undefined, - ignoreScripts: false, - onlyProduction: false, - }, - }]); + expect(result).toEqual({ + preStage: [], + stages: [{ + baseImage: 'nginx:latest', + platform: undefined, + name: undefined, + stepsBeforeInstall: ['COPY ./file ./somewhere'], + stepsAfterInstall: ['RUN npm run prepare'], + install: { + ci: undefined, + ignoreScripts: false, + onlyProduction: false, + }, + }], + }); }); it('should return one stage from dockerfile with install parameters', async function (this: ReadDockerfileThisContext) { @@ -106,18 +151,21 @@ describe('readDockerfile', () => { 'RUN npm run prepare', ].join('\n')); const result = await readDockerfile('Dockerfile'); - expect(result).toEqual([{ - baseImage: 'nginx:latest', - platform: undefined, - name: undefined, - stepsBeforeInstall: ['COPY ./file ./somewhere'], - stepsAfterInstall: ['RUN npm run prepare'], - install: { - ci: true, - ignoreScripts: true, - onlyProduction: true, - }, - }]); + expect(result).toEqual({ + preStage: [], + stages: [{ + baseImage: 'nginx:latest', + platform: undefined, + name: undefined, + stepsBeforeInstall: ['COPY ./file ./somewhere'], + stepsAfterInstall: ['RUN npm run prepare'], + install: { + ci: true, + ignoreScripts: true, + onlyProduction: true, + }, + }], + }); }); it('should return one stage from dockerfile with ci install parameter', async function (this: ReadDockerfileThisContext) { @@ -128,18 +176,21 @@ describe('readDockerfile', () => { 'RUN npm run prepare', ].join('\n')); const result = await readDockerfile('Dockerfile'); - expect(result).toEqual([{ - baseImage: 'nginx:latest', - platform: undefined, - name: undefined, - stepsBeforeInstall: ['COPY ./file ./somewhere'], - stepsAfterInstall: ['RUN npm run prepare'], - install: { - ci: true, - ignoreScripts: false, - onlyProduction: false, - }, - }]); + expect(result).toEqual({ + preStage: [], + stages: [{ + baseImage: 'nginx:latest', + platform: undefined, + name: undefined, + stepsBeforeInstall: ['COPY ./file ./somewhere'], + stepsAfterInstall: ['RUN npm run prepare'], + install: { + ci: true, + ignoreScripts: false, + onlyProduction: false, + }, + }], + }); }); it('should pass npm i of dependencies into dockerfile', async function (this: ReadDockerfileThisContext) { @@ -150,18 +201,21 @@ describe('readDockerfile', () => { 'RUN npm i lerna', ].join('\n')); const result = await readDockerfile('Dockerfile'); - expect(result).toEqual([{ - baseImage: 'nginx:latest', - platform: undefined, - name: undefined, - stepsBeforeInstall: ['COPY ./file ./somewhere'], - stepsAfterInstall: ['RUN npm i lerna'], - install: { - ci: true, - ignoreScripts: false, - onlyProduction: false, - }, - }]); + expect(result).toEqual({ + preStage: [], + stages: [{ + baseImage: 'nginx:latest', + platform: undefined, + name: undefined, + stepsBeforeInstall: ['COPY ./file ./somewhere'], + stepsAfterInstall: ['RUN npm i lerna'], + install: { + ci: true, + ignoreScripts: false, + onlyProduction: false, + }, + }], + }); }); it('should return multiple stages from dockerfile', async function (this: ReadDockerfileThisContext) { @@ -175,32 +229,35 @@ describe('readDockerfile', () => { 'ENTRYPOINT ["entrypoint.sh"]', ].join('\n')); const result = await readDockerfile('Dockerfile'); - expect(result).toEqual([ - { - baseImage: 'node:14', - platform: undefined, - name: 'build', - stepsBeforeInstall: ['COPY ./file ./somewhere'], - stepsAfterInstall: ['RUN npm run build'], - install: { - ci: undefined, - ignoreScripts: false, - onlyProduction: false, + expect(result).toEqual({ + preStage: [], + stages: [ + { + baseImage: 'node:14', + platform: undefined, + name: 'build', + stepsBeforeInstall: ['COPY ./file ./somewhere'], + stepsAfterInstall: ['RUN npm run build'], + install: { + ci: undefined, + ignoreScripts: false, + onlyProduction: false, + }, }, - }, - { - baseImage: 'nginx:latest', - platform: undefined, - name: undefined, - stepsBeforeInstall: [], - stepsAfterInstall: ['ENTRYPOINT ["entrypoint.sh"]'], - install: { - ci: undefined, - ignoreScripts: false, - onlyProduction: false, + { + baseImage: 'nginx:latest', + platform: undefined, + name: undefined, + stepsBeforeInstall: [], + stepsAfterInstall: ['ENTRYPOINT ["entrypoint.sh"]'], + install: { + ci: undefined, + ignoreScripts: false, + onlyProduction: false, + }, }, - }, - ]); + ], + }); }); }); diff --git a/packages/lerna-dockerize/test/integration/integration.spec.ts b/packages/lerna-dockerize/test/integration/integration.spec.ts index 39919b62..146ce83e 100644 --- a/packages/lerna-dockerize/test/integration/integration.spec.ts +++ b/packages/lerna-dockerize/test/integration/integration.spec.ts @@ -6,7 +6,7 @@ import { deleteIfExists } from './delete-if-exists'; describe('integration', () => - ['simple', 'config-file', 'if_exists', 'custom_dockerfile', 'hoist', 'split-stages', 'npm_i_args'] + ['simple', 'config-file', 'if_exists', 'custom_dockerfile', 'hoist', 'split-stages', 'npm_i_args', 'pre-stage'] .forEach(testCase => describe(testCase, () => { afterAll(async function cleanup() { diff --git a/packages/lerna-dockerize/test/integration/pre-stage/Dockerfile.base b/packages/lerna-dockerize/test/integration/pre-stage/Dockerfile.base new file mode 100644 index 00000000..4d820a00 --- /dev/null +++ b/packages/lerna-dockerize/test/integration/pre-stage/Dockerfile.base @@ -0,0 +1,4 @@ +ARG NODE_VERSION=16 +FROM node:${NODE_VERSION} as base +COPY ./package.json ./ +RUN npm install diff --git a/packages/lerna-dockerize/test/integration/pre-stage/Dockerfile.expected b/packages/lerna-dockerize/test/integration/pre-stage/Dockerfile.expected new file mode 100644 index 00000000..cbc989b7 --- /dev/null +++ b/packages/lerna-dockerize/test/integration/pre-stage/Dockerfile.expected @@ -0,0 +1,42 @@ +ARG NODE_VERSION=16 +FROM node:${NODE_VERSION} as base +COPY ./package.json ./ +RUN npm install +# Package a +FROM base as a-build +WORKDIR /app/packages/a +COPY packages/a/package-slim.json package.json +WORKDIR /app/ +RUN npx lerna bootstrap --scope=a --includeDependencies +WORKDIR /app/packages/a +COPY packages/a/package.json ./ +RUN npm run build +# Package b +FROM base as b-build +WORKDIR /app/packages/b +COPY packages/b/package-slim.json package.json +WORKDIR /app/ +COPY --from=a-build /app/packages/a/package.json /app/packages/a/ +RUN npx lerna bootstrap --scope=b --includeDependencies +COPY --from=a-build /app/packages/a/ /app/packages/a/ +WORKDIR /app/packages/b +COPY packages/b/package.json ./ +RUN npm run build +# Package c +FROM base as c-build +WORKDIR /app/packages/c +COPY packages/c/package-slim.json package.json +WORKDIR /app/ +COPY --from=b-build /app/packages/b/package.json /app/packages/b/ +COPY --from=a-build /app/packages/a/package.json /app/packages/a/ +RUN npx lerna bootstrap --scope=c --includeDependencies +COPY --from=b-build /app/packages/b/ /app/packages/b/ +COPY --from=a-build /app/packages/a/ /app/packages/a/ +WORKDIR /app/packages/c +COPY packages/c/package.json ./ +RUN npm run build +# final stage +FROM base +COPY --from=a-build /app/packages/a /app/packages/a +COPY --from=b-build /app/packages/b /app/packages/b +COPY --from=c-build /app/packages/c /app/packages/c \ No newline at end of file diff --git a/packages/lerna-dockerize/test/integration/pre-stage/Dockerfile.template b/packages/lerna-dockerize/test/integration/pre-stage/Dockerfile.template new file mode 100644 index 00000000..ebca3af7 --- /dev/null +++ b/packages/lerna-dockerize/test/integration/pre-stage/Dockerfile.template @@ -0,0 +1,7 @@ +FROM base as build + +COPY --slim ./package.json ./ +RUN npm install + +COPY ./package.json ./ +RUN npm run build diff --git a/packages/lerna-dockerize/test/integration/pre-stage/Readme.md b/packages/lerna-dockerize/test/integration/pre-stage/Readme.md new file mode 100644 index 00000000..5c68cef7 --- /dev/null +++ b/packages/lerna-dockerize/test/integration/pre-stage/Readme.md @@ -0,0 +1,7 @@ +# Simple project setup + +This project setup contains 3 packages, who are dependant of each other. The `dockerfile.base` and `Dockerfile.template` are used by `lerna-dockerize` to generate the out coming dockerfile. + +`lerna-dockerize` is started over an npm command inside the root package.json. + +The file `Dockerfile.template` will be used for each service. It uses the `--slim` flag from the extendet Dockerfile syntax. diff --git a/packages/lerna-dockerize/test/integration/pre-stage/lerna.json b/packages/lerna-dockerize/test/integration/pre-stage/lerna.json new file mode 100644 index 00000000..2b3450d0 --- /dev/null +++ b/packages/lerna-dockerize/test/integration/pre-stage/lerna.json @@ -0,0 +1,6 @@ +{ + "packages": [ + "packages/*" + ], + "version": "1.0.0" +} diff --git a/packages/lerna-dockerize/test/integration/pre-stage/package.json b/packages/lerna-dockerize/test/integration/pre-stage/package.json new file mode 100644 index 00000000..22a28c59 --- /dev/null +++ b/packages/lerna-dockerize/test/integration/pre-stage/package.json @@ -0,0 +1,11 @@ +{ + "name": "test", + "version": "1.0.0", + "dependencies": { + "lerna": "^4.0.0", + "lerna-dockerize": "latest" + }, + "scripts": { + "lerna-dockerize": "lerna-dockerize --logLevel error --templateDockerfileName Dockerfile.template" + } +} diff --git a/packages/lerna-dockerize/test/integration/pre-stage/packages/a/Dockerfile b/packages/lerna-dockerize/test/integration/pre-stage/packages/a/Dockerfile new file mode 100644 index 00000000..84267ac9 --- /dev/null +++ b/packages/lerna-dockerize/test/integration/pre-stage/packages/a/Dockerfile @@ -0,0 +1,9 @@ +ARG SHOULD_BE_IGNORED + +FROM base as build + +COPY --slim ./package.json ./ +RUN npm install + +COPY ./package.json ./ +RUN npm run build diff --git a/packages/lerna-dockerize/test/integration/pre-stage/packages/a/package.json b/packages/lerna-dockerize/test/integration/pre-stage/packages/a/package.json new file mode 100644 index 00000000..5693ad05 --- /dev/null +++ b/packages/lerna-dockerize/test/integration/pre-stage/packages/a/package.json @@ -0,0 +1,4 @@ +{ + "name": "a", + "version": "1.0.0" +} diff --git a/packages/lerna-dockerize/test/integration/pre-stage/packages/b/package.json b/packages/lerna-dockerize/test/integration/pre-stage/packages/b/package.json new file mode 100644 index 00000000..e88763c6 --- /dev/null +++ b/packages/lerna-dockerize/test/integration/pre-stage/packages/b/package.json @@ -0,0 +1,7 @@ +{ + "name": "b", + "version": "1.0.0", + "dependencies": { + "a": "1.0.0" + } +} diff --git a/packages/lerna-dockerize/test/integration/pre-stage/packages/c/package.json b/packages/lerna-dockerize/test/integration/pre-stage/packages/c/package.json new file mode 100644 index 00000000..c8b4abe2 --- /dev/null +++ b/packages/lerna-dockerize/test/integration/pre-stage/packages/c/package.json @@ -0,0 +1,7 @@ +{ + "name": "c", + "version": "1.0.0", + "dependencies": { + "b": "1.0.0" + } +}