buildx and other things

CHANGELOG.md

@@ -1,3 +1,25 @@
+## v3.21.0-beta.15 - [April 20, 2024](https://github.com/lando/core/releases/tag/v3.21.0-beta.15)
+
+### New Features
+
+* Added `buildx` toggle to `l337` service `image` key
+* Added build `args` support to `l337` service `image` key
+* Added `ssh` support to `l337` service `image` key
+* Changed default `l337` service builder to `buildx`
+* Improved `api: 4` image build errors and handling
+
+### Bug Fixes
+
+* Fixed bug causing `healthy` info to not persist correctly
+* Fixed inconsistent container shutdown by switching from `kill` to `stop`
+* Fixed inconsistent error display in `dc2` `listr` renderer
+
+### Internal
+
+* Changed default `healthy` info from `true` to `unknown`
+* Changed `api: 4` service info to provide a `state` key
+* Added `app.updateComposeCache()` and `app.v4.updateComposeCache` for better metadata consistencu
+
 ## v3.21.0-beta.14 - [April 10, 2024](https://github.com/lando/core/releases/tag/v3.21.0-beta.14)
 
 ### New Features
@@ -9,7 +31,7 @@
 * Updated tested Docker Desktop range to `<=4.29`
 * Updated tested Docker Engine range to `<27`
 
-### Bug Fixes Features
+### Bug Fixes
 
 * Fixed bug where `lando update` check failures were failing silently
 * Fixed bug where `GITHUB_TOKEN` was being used, if set, to get `lando update` info

app.js

@@ -20,6 +20,20 @@ module.exports = async (app, lando) => {
   // Build step locl files
   app.preLockfile = `${app.name}.build.lock`;
   app.postLockfile = `${app.name}.post-build.lock`;
+  // add compose cache updated
+  app.updateComposeCache = () => {
+    lando.cache.set(app.composeCache, {
+      name: app.name,
+      project: app.project,
+      compose: app.compose,
+      containers: app.containers,
+      root: app.root,
+      info: app.info,
+      overrides: {
+        tooling: app._coreToolingOverrides,
+      },
+    }, {persist: true});
+  };
 
   // Add v4 stuff to the app object
   app.v4 = {};
@@ -30,6 +44,23 @@ module.exports = async (app, lando) => {
   app.v4.services = [];
   app.v4.composeCache = `${app.name}.compose.cache`;
 
+  // Add compose cache v4 updaters
+  // add compose cache updated
+  app.v4.updateComposeCache = () => {
+    lando.cache.set(app.v4.composeCache, {
+      name: app.name,
+      project: app.project,
+      compose: app.compose,
+      containers: app.containers,
+      root: app.root,
+      info: app.info,
+      mounts: require('./utils/get-mounts')(_.get(app, 'v4.services', {})),
+      overrides: {
+        tooling: app._coreToolingOverrides,
+      },
+    }, {persist: true});
+  };
+
   // front load top level networks
   app.v4.addNetworks = (data = {}) => {
     app.add({

components/docker-engine.js

@@ -19,26 +19,35 @@ const mergePromise = require('../utils/merge-promise');
 class DockerEngine extends Dockerode {
   static name = 'docker-engine';
   static cspace = 'docker-engine';
-  static debug = require('debug')('docker-engine');
   static config = {};
+  static debug = require('debug')('docker-engine');
+  static builder = require('../utils/get-docker-x')();
+  static orchestrator = require('../utils/get-compose-x')();
   // @NOTE: is wsl accurate here?
   // static supportedPlatforms = ['linux', 'wsl'];
 
-  constructor(config, {debug = DockerEngine.debug} = {}) {
+  constructor(config, {
+    builder = DockerEngine.buildx,
+    debug = DockerEngine.debug,
+    orchestrator = DockerEngine.orchestrator,
+  } = {}) {
     super(config);
+    this.builder = builder;
     this.debug = debug;
+    this.orchestrator = orchestrator;
   }
 
   /*
-   * This is intended for building images
-   * This is a wrapper around Dockerode.build that provides either an await or return implementation.
+   * this is the legacy rest API image builder eg NOT buildx
+   * this is a wrapper around Dockerode.build that provides either an await or return implementation.
    *
    * @param {*} command
    * @param {*} param1
    */
   build(dockerfile,
     {
       tag,
+      buildArgs = {},
       attach = false,
       context = path.join(require('os').tmpdir(), nanoid()),
       id = tag,
@@ -105,13 +114,13 @@ class DockerEngine extends Dockerode {
     // error if no dockerfile exits
     if (!fs.existsSync(dockerfile)) throw new Error(`${dockerfile} does not exist`);
 
+    // extend debugger in appropriate way
+    const debug = id ? this.debug.extend(id) : this.debug.extend('docker-engine:build');
     // collect some args we can merge into promise resolution
     // @TODO: obscure auth?
     const args = {command: 'dockerode buildImage', args: {dockerfile, tag, sources}};
     // create an event emitter we can pass into the promisifier
     const builder = new EventEmitter();
-    // extend debugger in appropriate way
-    const debug = id ? this.debug.extend(id) : this.debug.extend('docker-engine:build');
 
     // ensure context dir exists
     fs.mkdirSync(context, {recursive: true});
@@ -130,13 +139,156 @@ class DockerEngine extends Dockerode {
 
     // call the parent
     // @TODO: consider other opts? https://docs.docker.com/engine/api/v1.43/#tag/Image/operation/ImageBuild args?
-    this.debug('building image %o from %o', tag, context);
-    super.buildImage({context, src: fs.readdirSync(context)}, {forcerm: true, t: tag}, callbackHandler);
+    debug('building image %o from %o writh build-args %o', tag, context, buildArgs);
+    super.buildImage(
+      {
+        context,
+        src: fs.readdirSync(context),
+      },
+      {
+        buildargs: JSON.stringify(buildArgs),
+        forcerm: true,
+        t: tag,
+      },
+      callbackHandler,
+    );
 
     // make this a hybrid async func and return
     return mergePromise(builder, awaitHandler);
   }
 
+  /*
+   * this is the buildx image builder
+   *
+   * unfortunately dockerode does not have an endpoint for this
+   * see: https://github.com/apocas/dockerode/issues/601
+   *
+   * so we are invoking the cli directly
+   *
+   * @param {*} command
+   * @param {*} param1
+   */
+  buildx(dockerfile,
+    {
+      tag,
+      buildArgs = {},
+      context = path.join(require('os').tmpdir(), nanoid()),
+      id = tag,
+      ignoreReturnCode = false,
+      sshKeys = [],
+      sshSocket = false,
+      sources = [],
+      stderr = '',
+      stdout = '',
+    } = {}) {
+    // handles the promisification of the merged return
+    const awaitHandler = async () => {
+      return new Promise((resolve, reject) => {
+        // handle resolve/reject
+        buildxer.on('done', ({code, stdout, stderr}) => {
+          debug('command %o done with code %o', args, code);
+          resolve(makeSuccess(merge({}, args, code, stdout, stderr)));
+        });
+        buildxer.on('error', error => {
+          debug('command %o error %o', args, error?.message);
+          reject(error);
+        });
+      });
+    };
+
+    // error if no dockerfile
+    if (!dockerfile) throw new Error('you must pass a dockerfile into buildx');
+    // error if no dockerfile exits
+    if (!fs.existsSync(dockerfile)) throw new Error(`${dockerfile} does not exist`);
+
+    // extend debugger in appropriate way
+    const debug = id ? this.debug.extend(id) : this.debug.extend('docker-engine:buildx');
+
+    // build initial buildx command
+    const args = {
+      command: this.builder,
+      args: [
+        'buildx',
+        'build',
+        `--file=${dockerfile}`,
+        '--progress=plain',
+        `--tag=${tag}`,
+        context,
+      ],
+    };
+
+    // add any needed build args into the command
+    for (const [key, value] of Object.entries(buildArgs)) args.args.push(`--build-arg=${key}=${value}`);
+
+    // if we have sshKeys then lets pass those in
+    if (sshKeys.length > 0) {
+      // ensure we have good keys
+      sshKeys = require('../utils/get-passphraseless-keys')(sshKeys);
+      // first add all the keys with id "keys"
+      args.args.push(`--ssh=keys=${sshKeys.join(',')}`);
+      // then add each key separately with its name as the key
+      for (const key of sshKeys) args.args.push(`--ssh=${path.basename(key)}=${key}`);
+      // log
+      debug('passing in ssh keys %o', sshKeys);
+    }
+
+    // if we have an sshAuth socket then add that as well
+    if (sshSocket && fs.existsSync(sshSocket)) {
+      args.args.push(`--ssh=agent=${sshSocket}`);
+      debug('passing in ssh agent socket %o', sshSocket);
+    }
+
+    // get builder
+    // @TODO: consider other opts? https://docs.docker.com/reference/cli/docker/buildx/build/ args?
+    // secrets?
+    // gha cache-from/to?
+    const buildxer = require('../utils/run-command')(args.command, args.args, {debug});
+
+    // augment buildxer with more events so it has the same interface as build
+    buildxer.stdout.on('data', data => {
+      buildxer.emit('data', data);
+      buildxer.emit('progress', data);
+      for (const line of data.toString().trim().split('\n')) debug(line);
+      stdout += data;
+    });
+    buildxer.stderr.on('data', data => {
+      buildxer.emit('data', data);
+      buildxer.emit('progress', data);
+      for (const line of data.toString().trim().split('\n')) debug(line);
+      stderr += data;
+    });
+    buildxer.on('close', code => {
+      // if code is non-zero and we arent ignoring then reject here
+      if (code !== 0 && !ignoreReturnCode) {
+        buildxer.emit('error', require('../utils/get-buildx-error')({code, stdout, stderr}));
+      // otherwise return done
+      } else {
+        buildxer.emit('done', {code, stdout, stderr});
+        buildxer.emit('finished', {code, stdout, stderr});
+        buildxer.emit('success', {code, stdout, stderr});
+      }
+    });
+
+    // ensure context dir exists
+    fs.mkdirSync(context, {recursive: true});
+
+    // move other sources into the build context
+    for (const source of sources) {
+      fs.copySync(source.source, path.join(context, source.destination));
+      debug('copied %o into build context %o', source.source, path.join(context, source.destination));
+    }
+
+    // copy the dockerfile to the correct place
+    fs.copySync(dockerfile, path.join(context, 'Dockerfile'));
+    debug('copied Imagefile from %o to %o', dockerfile, path.join(context, 'Dockerfile'));
+
+    // debug
+    debug('buildxing image %o from %o with build-args', tag, context, buildArgs);
+
+    // return merger
+    return mergePromise(buildxer, awaitHandler);
+  }
+
   /*
    * A helper method that automatically will build the image needed for the run command
    * NOTE: this is only available as async/await so you cannot return directly and access events

components/error.js

@@ -0,0 +1,34 @@
+'use strict';
+
+/**
+ *
+ */
+class LandoError extends Error {
+  static id = 'error';
+  static debug = require('debug')('@lando/core:error');
+
+  /*
+   */
+  constructor(message, {
+    all = '',
+    args = [],
+    code = 1,
+    command = '',
+    stdout = '',
+    stderr = '',
+    short,
+  } = {}) {
+    super(message);
+
+    // add other metadata
+    this.all = all;
+    this.args = args;
+    this.code = code;
+    this.command = command;
+    this.short = short ?? message;
+    this.stdout = stdout;
+    this.stderr = stderr;
+  }
+}
+
+module.exports = LandoError;