Add image creator

docs/imagecustomizer/developer-guide.md

@@ -47,7 +47,7 @@ sudo go test -C ./toolkit/tools ./...
 2. Download the test RPM files:
 
    ```bash
-   ./toolkit/tools/pkg/imagecustomizerlib/testdata/testrpms/download-test-rpms.sh
+   ./toolkit/tools/internal/testutils/testrpms/download-test-utils.sh
    ```
 
 3. Run the tests:

toolkit/scripts/tools.mk

@@ -34,6 +34,7 @@ go_tool_list = \
 	imager \
 	isomaker \
 	osmodifier \
+	imagecreator \
 
 # For each utility "util", create a "out/tools/util" target which references code in "tools/util/"
 go_tool_targets = $(foreach target,$(go_tool_list),$(TOOL_BINS_DIR)/$(target))

toolkit/tools/imagecreator/main.go

@@ -0,0 +1,55 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+// Tool to create and install images
+
+package main
+
+import (
+	"log"
+	"maps"
+	"strings"
+
+	"github.com/alecthomas/kong"
+	"github.com/microsoft/azurelinux/toolkit/tools/imagecustomizerapi"
+	"github.com/microsoft/azurelinux/toolkit/tools/internal/exekong"
+	"github.com/microsoft/azurelinux/toolkit/tools/internal/logger"
+	"github.com/microsoft/azurelinux/toolkit/tools/internal/ptrutils"
+	"github.com/microsoft/azurelinux/toolkit/tools/pkg/imagecreatorlib"
+	"github.com/microsoft/azurelinux/toolkit/tools/pkg/imagecustomizerlib"
+)
+
+type ImageCreatorCmd struct {
+	BuildDir          string   `name:"build-dir" help:"Directory to run build out of." required:""`
+	ConfigFile        string   `name:"config-file" help:"Path of the image creator config file." required:""`
+	RpmSources        []string `name:"rpm-source" help:"Path to a RPM repo config file or a directory containing RPMs." required:""`
+	ToolsTar          string   `name:"tools-file" help:"Path to tdnf worker tarball" required:""`
+	OutputImageFile   string   `name:"output-image-file" help:"Path to write the customized image to."`
+	OutputImageFormat string   `name:"output-image-format" placeholder:"(vhd|vhd-fixed|vhdx|qcow2|raw)" help:"Format of output image." enum:"${imageformat}" default:""`
+	exekong.LogFlags
+}
+
+func main() {
+	cli := &ImageCreatorCmd{}
+
+	vars := kong.Vars{
+		"imageformat": strings.Join(imagecustomizerapi.SupportedImageFormatTypesImageCreator(), ",") + ",",
+		"version":     imagecustomizerlib.ToolVersion,
+	}
+	maps.Copy(vars, exekong.KongVars)
+
+	_ = kong.Parse(cli,
+		vars,
+		kong.HelpOptions{
+			Compact:   true,
+			FlagsLast: true,
+		},
+		kong.UsageOnError())
+
+	logger.InitBestEffort(ptrutils.PtrTo(cli.LogFlags.AsLoggerFlags()))
+
+	err := imagecreatorlib.CreateImageWithConfigFile(cli.BuildDir, cli.ConfigFile, cli.RpmSources, cli.ToolsTar, cli.OutputImageFile, cli.OutputImageFormat)
+	if err != nil {
+		log.Fatalf("image creation failed:\n%v", err)
+	}
+}

toolkit/tools/imagecustomizerapi/imageFormatType.go

@@ -37,6 +37,14 @@ var supportedImageFormatTypes = []string{
 	string(ImageFormatTypeCosi),
 }
 
+var supportedImageFormatTypesImageCreator = []string{
+	string(ImageFormatTypeVhd),
+	string(ImageFormatVhdTypeFixed),
+	string(ImageFormatTypeVhdx),
+	string(ImageFormatTypeQcow2),
+	string(ImageFormatTypeRaw),
+}
+
 func (ft ImageFormatType) IsValid() error {
 	if ft != ImageFormatTypeNone && !slices.Contains(SupportedImageFormatTypes(), string(ft)) {
 		return fmt.Errorf("invalid image format type (%s)", ft)
@@ -49,3 +57,7 @@ func (ft ImageFormatType) IsValid() error {
 func SupportedImageFormatTypes() []string {
 	return supportedImageFormatTypes
 }
+
+func SupportedImageFormatTypesImageCreator() []string {
+	return supportedImageFormatTypesImageCreator
+}

toolkit/tools/pkg/imagecustomizerlib/createuuid.go → toolkit/tools/internal/randomization/createuuid.go

@@ -1,27 +1,23 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-package imagecustomizerlib
+package randomization
 
 import (
 	"crypto/rand"
 	"encoding/hex"
 	"fmt"
-	"path/filepath"
 	"strings"
 
-	"github.com/microsoft/azurelinux/toolkit/tools/internal/file"
 	"github.com/microsoft/azurelinux/toolkit/tools/internal/logger"
-	"github.com/microsoft/azurelinux/toolkit/tools/internal/sliceutils"
 )
 
 const (
-	UuidSize                   uint32 = 16
-	ImageCustomizerReleasePath        = "etc/image-customizer-release"
+	UuidSize uint32 = 16
 )
 
 // Create the uuid and return byte array and string representation
-func createUuid() ([UuidSize]byte, string, error) {
+func CreateUuid() ([UuidSize]byte, string, error) {
 	uuid, err := generateRandom128BitNumber()
 	if err != nil {
 		return uuid, "", err
@@ -54,34 +50,7 @@ func convertUuidToString(uuid [UuidSize]byte) string {
 	return uuidStr
 }
 
-func extractImageUUID(imageConnection *ImageConnection) ([UuidSize]byte, string, error) {
-	var emptyUuid [UuidSize]byte
-
-	releasePath := filepath.Join(imageConnection.Chroot().RootDir(), ImageCustomizerReleasePath)
-	data, err := file.Read(releasePath)
-	if err != nil {
-		return emptyUuid, "", fmt.Errorf("failed to read %s:\n%w", releasePath, err)
-	}
-
-	lines := strings.Split(string(data), "\n")
-	line, found := sliceutils.FindValueFunc(lines, func(line string) bool {
-		return strings.HasPrefix(line, "IMAGE_UUID=")
-	})
-	if !found {
-		return emptyUuid, "", fmt.Errorf("IMAGE_UUID not found in %s", releasePath)
-	}
-
-	uuidStr := strings.Trim(strings.TrimPrefix(line, "IMAGE_UUID="), `"`)
-
-	parsed, err := parseUuidString(uuidStr)
-	if err != nil {
-		return emptyUuid, "", fmt.Errorf("failed to parse IMAGE_UUID (%s):\n%w", uuidStr, err)
-	}
-
-	return parsed, uuidStr, nil
-}
-
-func parseUuidString(s string) ([UuidSize]byte, error) {
+func ParseUuidString(s string) ([UuidSize]byte, error) {
 	var uuid [UuidSize]byte
 
 	parts := strings.Split(s, "-")

toolkit/tools/internal/safechroot/safechroot.go

@@ -670,7 +670,6 @@ func (c *Chroot) unmountAndRemove(leaveOnDisk, lazyUnmount bool) (err error) {
 			umountErr := unix.Unmount(fullPath, unmountFlags)
 			return umountErr
 		}, totalAttempts, retryDuration, 2.0)
-
 		if err != nil {
 			err = fmt.Errorf("failed to unmount (%s):\n%w", fullPath, err)
 			return
@@ -687,23 +686,23 @@ func (c *Chroot) unmountAndRemove(leaveOnDisk, lazyUnmount bool) (err error) {
 // defaultMountPoints returns a new copy of the default mount points used by a functional chroot
 func defaultMountPoints() []*MountPoint {
 	return []*MountPoint{
-		&MountPoint{
+		{
 			target: "/dev",
 			fstype: "devtmpfs",
 		},
-		&MountPoint{
+		{
 			target: "/proc",
 			fstype: "proc",
 		},
-		&MountPoint{
+		{
 			target: "/sys",
 			fstype: "sysfs",
 		},
-		&MountPoint{
+		{
 			target: "/run",
 			fstype: "tmpfs",
 		},
-		&MountPoint{
+		{
 			target: "/dev/pts",
 			fstype: "devpts",
 			data:   "gid=5,mode=620",

toolkit/tools/internal/testutils/testrpms/create-tools-file.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+CONTAINER_IMAGE="$1"
+TOOLS_FILE="$2"
+
+if [[ -z "$CONTAINER_IMAGE" || -z "$TOOLS_FILE" ]]; then
+  echo "Usage: $0 <container_image> <tools_file>"
+  exit 1
+fi
+
+docker create --name temp-container "$CONTAINER_IMAGE"
+docker export temp-container | gzip > "$TOOLS_FILE"
+docker rm temp-container

toolkit/tools/pkg/imagecustomizerlib/testdata/testrpms/download-test-rpms.sh → toolkit/tools/internal/testutils/testrpms/download-test-rpms.sh

@@ -1,41 +1,21 @@
-#!/usr/bin/env bash
-set -eu
+#!/bin/bash
 
-SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
-CONTAINER_TAG="imagecustomizertestrpms:latest"
-DOCKERFILE_DIR="$SCRIPT_DIR/downloader"
-
-AZURELINUX_2_CONTAINER_IMAGE="mcr.microsoft.com/cbl-mariner/base/core:2.0"
-
-IMAGE_VERSION="2.0"
+set -e
+CONTAINER_IMAGE="$1"
+IMAGE_CREATOR="$2"
 
-while getopts "t:" flag
-do
-    case "${flag}" in
-        t) IMAGE_VERSION="$OPTARG";;
-        h) ;;&
-        ?)
-            echo "Usage: download-test-rpms.sh [-t IMAGE_VERSION]"
-            echo ""
-            echo "Args:"
-            echo "  -t IMAGE_VERSION   The Azure Image version to download the RPMs for."
-            echo "  -h Show help"
-            exit 1;;
-    esac
-done
 
-case "${IMAGE_VERSION}" in
-  2.0)
-    CONTAINER_IMAGE="$AZURELINUX_2_CONTAINER_IMAGE"
-    ;;
-  *)
-    echo "error: unsupported Azure Linux version: $IMAGE_VERSION"
-    exit 1;;
-esac
-
-set -x
+# Check if the required arguments are provided
+if [[ -z "$CONTAINER_IMAGE" || -z "$IMAGE_CREATOR" ]]; then
+  echo "Usage: $0 <container_image> <image_creator>"
+  echo "Example: $0 mcr.microsoft.com/azurelinux/base/core:3.0 true"
+  exit 1
+fi
 
+SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
+DOCKERFILE_DIR="$SCRIPT_DIR/downloader"
 DOWNLOADER_RPMS_DIRS="$SCRIPT_DIR/downloadedrpms"
+CONTAINER_TAG="imagecustomizertestrpms:latest"
 OUT_DIR="$DOWNLOADER_RPMS_DIRS/$IMAGE_VERSION"
 REPO_WITH_KEY_FILE="$DOWNLOADER_RPMS_DIRS/rpms-$IMAGE_VERSION-withkey.repo"
 REPO_NO_KEY_FILE="$DOWNLOADER_RPMS_DIRS/rpms-$IMAGE_VERSION-nokey.repo"
@@ -44,7 +24,8 @@ mkdir -p "$OUT_DIR"
 
 # Build a container image that contains the RPMs.
 docker build \
-  --build-arg "baseimage=$AZURELINUX_2_CONTAINER_IMAGE" \
+  --build-arg "baseimage=$CONTAINER_IMAGE" \
+  --build-arg "imagecreator=$IMAGE_CREATOR" \
   --tag "$CONTAINER_TAG" \
   "$DOCKERFILE_DIR"
 

toolkit/tools/internal/testutils/testrpms/download-test-utils.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -eu
+
+SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
+
+AZURELINUX_2_CONTAINER_IMAGE="mcr.microsoft.com/cbl-mariner/base/core:2.0"
+AZURELINUX_3_CONTAINER_IMAGE="mcr.microsoft.com/azurelinux/base/core:3.0"
+
+IMAGE_VERSION="2.0"
+
+IMAGE_CREATOR="false"
+TOOLS_FILE="$SCRIPT_DIR/tools.tar.gz"
+
+
+while getopts "s:t:" flag
+do
+    case "${flag}" in
+        s) IMAGE_CREATOR="$OPTARG";;
+        t) IMAGE_VERSION="$OPTARG";;
+        h) ;;&
+        ?)
+            echo "Usage: download-test-utils.sh [-t IMAGE_VERSION] [-s IMAGE_CREATOR]"
+            echo ""
+            echo "Args:"
+            echo "  -t IMAGE_VERSION   The Azure Image version to download the RPMs for."
+            echo "  -s IMAGE_CREATOR   If set to true, the script will create a tar.gz file with the tools and download the rpms needed to test imagecreator."
+            echo "  -h Show help"
+            exit 1;;
+    esac
+done
+
+
+case "${IMAGE_VERSION}" in
+  3.0)
+    CONTAINER_IMAGE="$AZURELINUX_3_CONTAINER_IMAGE"
+    ;;
+  2.0)
+    CONTAINER_IMAGE="$AZURELINUX_2_CONTAINER_IMAGE"
+    ;;  
+  *)
+    echo "error: unsupported Azure Linux version: $IMAGE_VERSION"
+    exit 1;;
+esac
+
+set -x
+
+# call the script to create the tools file if requested
+if [ "$IMAGE_CREATOR" = "true" ]; then
+  echo "Creating tools file: $TOOLS_FILE"
+  $SCRIPT_DIR/create-tools-file.sh "$CONTAINER_IMAGE" "$TOOLS_FILE"
+  echo "Tools file created successfully."
+else
+  echo "Skipping tools file creation."
+fi
+
+# call the script to download the rpms
+echo "Downloading test rpms for Azure Linux version: $IMAGE_VERSION"
+$SCRIPT_DIR/download-test-rpms.sh "$CONTAINER_IMAGE" "$IMAGE_CREATOR" 
+echo "Test rpms downloaded successfully."

toolkit/tools/internal/testutils/testrpms/downloader/Dockerfile

@@ -0,0 +1,24 @@
+ARG baseimage
+FROM ${baseimage}
+ARG imagecreator
+
+RUN tdnf update -y && \
+   tdnf install -y dnf dnf-plugins-core createrepo_c
+
+RUN dnf download -y --resolve --alldeps --destdir /downloadedrpms \
+   jq golang 
+
+RUN if [ "${imagecreator}" = "true" ]; then \
+
+    dnf download -y --resolve --alldeps --destdir /downloadedrpms \
+    azurelinux-release \
+    azurelinux-repos azurelinux-rpm-macros bash dbus dracut-hostonly e2fsprogs filesystem \
+    grub2 grub2-efi-binary iana-etc iproute iputils irqbalance \
+    ncurses-libs openssl rpm rpm-libs shadow-utils shim sudo \
+    systemd systemd-networkd systemd-resolved systemd-udev tdnf \
+    tdnf-plugin-repogpgcheck util-linux zlib kernel initramfs ; \
+
+    fi
+
+# Add repo metadata, so that the directory can be used in a .repo file.
+RUN createrepo --compatibility --update /downloadedrpms