From 1ea7d44f086083aa093fa4a90f07d8af62b90b70 Mon Sep 17 00:00:00 2001 From: delvedor Date: Thu, 19 May 2022 14:19:19 +0200 Subject: [PATCH 1/4] Add make.sh workflows --- .ci/Dockerfile | 2 +- .ci/make.mjs | 89 +++++++++++++++++++++++++ .ci/make.sh | 174 +++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 6 +- 4 files changed, 268 insertions(+), 3 deletions(-) create mode 100644 .ci/make.mjs create mode 100755 .ci/make.sh diff --git a/.ci/Dockerfile b/.ci/Dockerfile index 9e3716246..81f8ae8f2 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -1,4 +1,4 @@ -ARG NODE_JS_VERSION=10 +ARG NODE_JS_VERSION=16 FROM node:${NODE_JS_VERSION} # Create app directory diff --git a/.ci/make.mjs b/.ci/make.mjs new file mode 100644 index 000000000..d67d54947 --- /dev/null +++ b/.ci/make.mjs @@ -0,0 +1,89 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* global $ argv */ + +'use strict' + +import 'zx/globals' + +import { readFile, writeFile } from 'fs/promises' +import assert from 'assert' +import { join } from 'desm' +import semver from 'semver' + +assert(typeof argv.task === 'string', 'Missing task parameter') + +switch (argv.task) { + case 'release': + release(argv._).catch(onError) + break + case 'bump': + bump(argv._).catch(onError) + break + default: + console.log(`Unknown task: ${argv.task}`) + process.exit(1) +} + +async function release (args) { + assert(args.length === 2, 'Release task expects two parameters') + let [version, outputFolder] = args + const packageJson = JSON.parse(await readFile( + join(import.meta.url, '..', 'package.json'), + 'utf8' + )) + + if (process.env.WORKFLOW === 'snapshot' && !version.endsWith('SNAPSHOT')) { + version = `${version}-SNAPSHOT` + } + + await bump([version]) + + await $`npm run build` + await $`npm pack` + await $`zip elasticsearch-js-${version}.zip elastic-elasticsearch-${packageJson.version}.tgz` + await $`rm elastic-elasticsearch-${packageJson.version}.tgz` + await $`mv ${join(import.meta.url, '..', `elasticsearch-js-${version}.zip`)} ${join(import.meta.url, '..', outputFolder, `elasticsearch-js-${version}.zip`)}` +} + +async function bump (args) { + assert(args.length === 2, 'Bump task expects one parameter') + const [version] = args + const packageJson = JSON.parse(await readFile( + join(import.meta.url, '..', 'package.json'), + 'utf8' + )) + + const cleanVersion = semver.clean(version.includes('SNAPSHOT') ? version.split('-')[0] : version) + assert(semver.valid(cleanVersion)) + packageJson.version = cleanVersion + packageJson.versionCanary = `${cleanVersion}-canary.0` + + await writeFile( + join(import.meta.url, '..', 'package.json'), + JSON.stringify(packageJson, null, 2), + 'utf8' + ) +} + +function onError (err) { + console.log(err) + process.exit(1) +} diff --git a/.ci/make.sh b/.ci/make.sh new file mode 100755 index 000000000..7433f342f --- /dev/null +++ b/.ci/make.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash + +# ------------------------------------------------------- # +# +# Skeleton for common build entry script for all elastic +# clients. Needs to be adapted to individual client usage. +# +# Must be called: ./.ci/make.sh +# +# Version: 1.1.0 +# +# Targets: +# --------------------------- +# assemble : build client artefacts with version +# bump : bump client internals to version +# codegen : generate endpoints +# docsgen : generate documentation +# examplegen : generate the doc examples +# clean : clean workspace +# +# ------------------------------------------------------- # + +# ------------------------------------------------------- # +# Bootstrap +# ------------------------------------------------------- # + +script_path=$(dirname "$(realpath -s "$0")") +repo=$(realpath "$script_path/../") + +# shellcheck disable=SC1090 +CMD=$1 +TASK=$1 +TASK_ARGS=() +VERSION=$2 +STACK_VERSION=$VERSION +NODE_JS_VERSION=16 +WORKFLOW=${WORKFLOW-staging} +set -euo pipefail + +product="elastic/elasticsearch-js" +output_folder=".ci/output" +codegen_folder=".ci/output" +OUTPUT_DIR="$repo/${output_folder}" +REPO_BINDING="${OUTPUT_DIR}:/sln/${output_folder}" +mkdir -p "$OUTPUT_DIR" + +echo -e "\033[34;1mINFO:\033[0m PRODUCT ${product}\033[0m" +echo -e "\033[34;1mINFO:\033[0m VERSION ${STACK_VERSION}\033[0m" +echo -e "\033[34;1mINFO:\033[0m OUTPUT_DIR ${OUTPUT_DIR}\033[0m" + +# ------------------------------------------------------- # +# Parse Command +# ------------------------------------------------------- # + +case $CMD in + clean) + echo -e "\033[36;1mTARGET: clean workspace $output_folder\033[0m" + rm -rf "$output_folder" + echo -e "\033[32;1mdone.\033[0m" + exit 0 + ;; + assemble) + if [ -v $VERSION ]; then + echo -e "\033[31;1mTARGET: assemble -> missing version parameter\033[0m" + exit 1 + fi + echo -e "\033[36;1mTARGET: assemble artefact $VERSION\033[0m" + TASK=release + TASK_ARGS=("$VERSION" "$output_folder") + ;; + codegen) + if [ -v $VERSION ]; then + echo -e "\033[31;1mTARGET: codegen -> missing version parameter\033[0m" + exit 1 + fi + echo -e "\033[36;1mTARGET: codegen API v$VERSION\033[0m" + TASK=codegen + # VERSION is BRANCH here for now + TASK_ARGS=("$VERSION" "$codegen_folder") + ;; + docsgen) + if [ -v $VERSION ]; then + echo -e "\033[31;1mTARGET: docsgen -> missing version parameter\033[0m" + exit 1 + fi + echo -e "\033[36;1mTARGET: generate docs for $VERSION\033[0m" + TASK=codegen + # VERSION is BRANCH here for now + TASK_ARGS=("$VERSION" "$codegen_folder") + ;; + examplesgen) + echo -e "\033[36;1mTARGET: generate examples\033[0m" + TASK=codegen + # VERSION is BRANCH here for now + TASK_ARGS=("$VERSION" "$codegen_folder") + ;; + bump) + if [ -v $VERSION ]; then + echo -e "\033[31;1mTARGET: bump -> missing version parameter\033[0m" + exit 1 + fi + echo -e "\033[36;1mTARGET: bump to version $VERSION\033[0m" + TASK=bump + # VERSION is BRANCH here for now + TASK_ARGS=("$VERSION") + ;; + *) + echo -e "\nUsage:\n\t $CMD is not supported right now\n" + exit 1 +esac + + +# ------------------------------------------------------- # +# Build Container +# ------------------------------------------------------- # + +echo -e "\033[34;1mINFO: building $product container\033[0m" + +docker build \ + --file .ci/Dockerfile \ + --tag ${product} \ + --build-arg NODE_JS_VERSION=${NODE_JS_VERSION} \ + --build-arg USER_ID="$(id -u)" \ + --build-arg GROUP_ID="$(id -g)" \ + . + +# ------------------------------------------------------- # +# Run the Container +# ------------------------------------------------------- # + +echo -e "\033[34;1mINFO: running $product container\033[0m" + +docker run \ + --volume $repo:/usr/src/app \ + --volume /usr/src/app/node_modules \ + --env "WORKFLOW=${WORKFLOW}" \ + --name make-elasticsearch-js \ + --rm \ + $product \ + node .ci/make.mjs --task $TASK ${TASK_ARGS[*]} + +# ------------------------------------------------------- # +# Post Command tasks & checks +# ------------------------------------------------------- # + +if [[ "$CMD" == "assemble" ]]; then + if compgen -G ".ci/output/*" > /dev/null; then + echo -e "\033[32;1mTARGET: successfully assembled client v$VERSION\033[0m" + else + echo -e "\033[31;1mTARGET: assemble failed, empty workspace!\033[0m" + exit 1 + fi +fi + +if [[ "$CMD" == "bump" ]]; then + if [ -n "$(git status --porcelain)" ]; then + echo -e "\033[32;1mTARGET: successfully bumped client v$VERSION\033[0m" + else + echo -e "\033[31;1mTARGET: failed bumped client v$VERSION\033[0m" + exit 1 + fi +fi + +if [[ "$CMD" == "codegen" ]]; then + echo "TODO" +fi + +if [[ "$CMD" == "docsgen" ]]; then + echo "TODO" +fi + +if [[ "$CMD" == "examplesgen" ]]; then + echo "TODO" +fi diff --git a/package.json b/package.json index 259cd7c9b..7141a2eb1 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "@types/stoppable": "^1.1.1", "@types/tap": "^15.0.7", "cross-zip": "^4.0.0", + "desm": "^1.2.0", "fast-deep-equal": "^3.1.3", "into-stream": "^7.0.0", "js-yaml": "^4.1.0", @@ -76,7 +77,8 @@ "ts-standard": "^11.0.0", "typescript": "^4.6.4", "workq": "^3.0.0", - "xmlbuilder2": "^3.0.2" + "xmlbuilder2": "^3.0.2", + "zx": "^6.1.0" }, "dependencies": { "@elastic/transport": "^8.2.0", @@ -89,4 +91,4 @@ "coverage": false, "check-coverage": false } -} +} \ No newline at end of file From 2cdc78c9ac96282fac1be64463addef8c951160b Mon Sep 17 00:00:00 2001 From: delvedor Date: Thu, 19 May 2022 14:25:42 +0200 Subject: [PATCH 2/4] Nits --- .ci/make.mjs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.ci/make.mjs b/.ci/make.mjs index d67d54947..cd5a9cad3 100644 --- a/.ci/make.mjs +++ b/.ci/make.mjs @@ -45,10 +45,6 @@ switch (argv.task) { async function release (args) { assert(args.length === 2, 'Release task expects two parameters') let [version, outputFolder] = args - const packageJson = JSON.parse(await readFile( - join(import.meta.url, '..', 'package.json'), - 'utf8' - )) if (process.env.WORKFLOW === 'snapshot' && !version.endsWith('SNAPSHOT')) { version = `${version}-SNAPSHOT` @@ -56,6 +52,11 @@ async function release (args) { await bump([version]) + const packageJson = JSON.parse(await readFile( + join(import.meta.url, '..', 'package.json'), + 'utf8' + )) + await $`npm run build` await $`npm pack` await $`zip elasticsearch-js-${version}.zip elastic-elasticsearch-${packageJson.version}.tgz` @@ -64,7 +65,7 @@ async function release (args) { } async function bump (args) { - assert(args.length === 2, 'Bump task expects one parameter') + assert(args.length === 1, 'Bump task expects one parameter') const [version] = args const packageJson = JSON.parse(await readFile( join(import.meta.url, '..', 'package.json'), From 76704d42403ba730974668ebc85f0672821b6e25 Mon Sep 17 00:00:00 2001 From: delvedor Date: Thu, 19 May 2022 14:57:25 +0200 Subject: [PATCH 3/4] Add codegen target --- .ci/make.mjs | 28 ++++++++++++++++++++++++++++ .ci/make.sh | 12 +++++++++--- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/.ci/make.mjs b/.ci/make.mjs index cd5a9cad3..ef1445e8f 100644 --- a/.ci/make.mjs +++ b/.ci/make.mjs @@ -37,6 +37,9 @@ switch (argv.task) { case 'bump': bump(argv._).catch(onError) break + case 'codegen': + codegen(argv._).catch(onError) + break default: console.log(`Unknown task: ${argv.task}`) process.exit(1) @@ -84,6 +87,31 @@ async function bump (args) { ) } +// this command can only be executed locally for now +async function codegen (args) { + assert(args.length === 1, 'Bump task expects one parameter') + const clientGeneratorPath = join(import.meta.url, '..', '..', 'elastic-client-generator-js') + const [version] = args + + const isGeneratorCloned = await $`[[ -d ${clientGeneratorPath} ]]`.exitCode === 0 + assert(isGeneratorCloned, 'You must clone the elastic-client-generator-js first') + + await $`npm install --prefix ${clientGeneratorPath}` + // this command will take a while! + if (version === 'main') { + await $`npm run elasticsearch --prefix ${clientGeneratorPath} -- --version main` + } else { + await $`npm run elasticsearch --prefix ${clientGeneratorPath} -- --version ${version.split('.').slice(0, 2).join('.')}` + } + await $`npm run lint --prefix ${clientGeneratorPath}` + + await $`rm -rf ${join(import.meta.url, '..', 'src', 'api')}` + await $`mkdir ${join(import.meta.url, '..', 'src', 'api')}` + await $`cp -R ${join(import.meta.url, '..', '..', 'elastic-client-generator-js', 'output')}/* ${join(import.meta.url, '..', 'src', 'api')}` + await $`mv ${join(import.meta.url, '..', 'src', 'api', 'reference.asciidoc')} ${join(import.meta.url, '..', 'docs', 'reference.asciidoc')}` + await $`npm run build` +} + function onError (err) { console.log(err) process.exit(1) diff --git a/.ci/make.sh b/.ci/make.sh index 7433f342f..39755599f 100755 --- a/.ci/make.sh +++ b/.ci/make.sh @@ -26,6 +26,7 @@ script_path=$(dirname "$(realpath -s "$0")") repo=$(realpath "$script_path/../") +generator=$(realpath "$script_path/../../elastic-client-generator-js") # shellcheck disable=SC1090 CMD=$1 @@ -39,7 +40,6 @@ set -euo pipefail product="elastic/elasticsearch-js" output_folder=".ci/output" -codegen_folder=".ci/output" OUTPUT_DIR="$repo/${output_folder}" REPO_BINDING="${OUTPUT_DIR}:/sln/${output_folder}" mkdir -p "$OUTPUT_DIR" @@ -76,7 +76,7 @@ case $CMD in echo -e "\033[36;1mTARGET: codegen API v$VERSION\033[0m" TASK=codegen # VERSION is BRANCH here for now - TASK_ARGS=("$VERSION" "$codegen_folder") + TASK_ARGS=("$VERSION") ;; docsgen) if [ -v $VERSION ]; then @@ -132,6 +132,7 @@ echo -e "\033[34;1mINFO: running $product container\033[0m" docker run \ --volume $repo:/usr/src/app \ + --volume $generator:/usr/src/elastic-client-generator-js \ --volume /usr/src/app/node_modules \ --env "WORKFLOW=${WORKFLOW}" \ --name make-elasticsearch-js \ @@ -162,7 +163,12 @@ if [[ "$CMD" == "bump" ]]; then fi if [[ "$CMD" == "codegen" ]]; then - echo "TODO" + if [ -n "$(git status --porcelain)" ]; then + echo -e "\033[32;1mTARGET: successfully generated client v$VERSION\033[0m" + else + echo -e "\033[31;1mTARGET: failed generating client v$VERSION\033[0m" + exit 1 + fi fi if [[ "$CMD" == "docsgen" ]]; then From f65b2416b1a9e3e9aedc4b17c04afc11ce1b6c55 Mon Sep 17 00:00:00 2001 From: delvedor Date: Thu, 19 May 2022 15:06:05 +0200 Subject: [PATCH 4/4] Bump test matrix as well --- .ci/make.mjs | 7 +++++++ .ci/test-matrix.yml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.ci/make.mjs b/.ci/make.mjs index ef1445e8f..1695a4869 100644 --- a/.ci/make.mjs +++ b/.ci/make.mjs @@ -85,6 +85,13 @@ async function bump (args) { JSON.stringify(packageJson, null, 2), 'utf8' ) + + const testMatrix = await readFile(join(import.meta.url, 'test-matrix.yml'), 'utf8') + await writeFile( + join(import.meta.url, 'test-matrix.yml'), + testMatrix.replace(/STACK_VERSION:\s+\- "[0-9]+[0-9\.]*[0-9](?:\-SNAPSHOT)?"/, `STACK_VERSION:\n - "${cleanVersion}-SNAPSHOT"`), // eslint-disable-line + 'utf8' + ) } // this command can only be executed locally for now diff --git a/.ci/test-matrix.yml b/.ci/test-matrix.yml index 6b491ee9c..d29ab7347 100644 --- a/.ci/test-matrix.yml +++ b/.ci/test-matrix.yml @@ -1,6 +1,6 @@ --- STACK_VERSION: - - 8.3.0-SNAPSHOT + - "8.3.0-SNAPSHOT" NODE_JS_VERSION: - 18