Add image creator

This PR adds a new tool image creator that creates seed images
Added a new module for image creator
Added a new share testutils module

follow-up PRs:
1. validation logic and tests for image creator configs, tests to check
the actual contents of the image. e.g. installed packages, grub.cfg
files, etc.
2. Add cli argument for release version to support 2.0/3.0/4.0 AZL
packages installation, packagesnapshot time
3. Support for fedora 
4. doc updates to run imagecreator and test 


### **Checklist**
- [ ] Tests added/updated
- [ ] Documentation updated (if needed)
- [ ] Code conforms to style guidelines

Author: himaja-kesari

docs/imagecustomizer/developer-guide.md

@@ -47,7 +47,7 @@ sudo go test -C ./toolkit/tools ./...
 2. Download the test RPM files:
 
    ```bash
-   ./toolkit/tools/pkg/imagecustomizerlib/testdata/testrpms/download-test-rpms.sh
+   ./toolkit/tools/internal/testutils/testrpms/download-test-utils.sh
    ```
 
 3. Run the tests:

toolkit/scripts/tools.mk

@@ -34,6 +34,7 @@ go_tool_list = \
 	imager \
 	isomaker \
 	osmodifier \
+	imagecreator \
 
 # For each utility "util", create a "out/tools/util" target which references code in "tools/util/"
 go_tool_targets = $(foreach target,$(go_tool_list),$(TOOL_BINS_DIR)/$(target))

toolkit/tools/imagecreator/main.go

@@ -0,0 +1,55 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+// Tool to create and install images
+
+package main
+
+import (
+	"log"
+	"maps"
+	"strings"
+
+	"github.com/alecthomas/kong"
+	"github.com/microsoft/azurelinux/toolkit/tools/imagecustomizerapi"
+	"github.com/microsoft/azurelinux/toolkit/tools/internal/exekong"
+	"github.com/microsoft/azurelinux/toolkit/tools/internal/logger"
+	"github.com/microsoft/azurelinux/toolkit/tools/internal/ptrutils"
+	"github.com/microsoft/azurelinux/toolkit/tools/pkg/imagecreatorlib"
+	"github.com/microsoft/azurelinux/toolkit/tools/pkg/imagecustomizerlib"
+)
+
+type ImageCreatorCmd struct {
+	BuildDir          string   `name:"build-dir" help:"Directory to run build out of." required:""`
+	ConfigFile        string   `name:"config-file" help:"Path of the image creator config file." required:""`
+	RpmSources        []string `name:"rpm-source" help:"Path to a RPM repo config file or a directory containing RPMs." required:""`
+	ToolsTar          string   `name:"tools-file" help:"Path to tdnf worker tarball" required:""`
+	OutputImageFile   string   `name:"output-image-file" help:"Path to write the customized image to."`
+	OutputImageFormat string   `name:"output-image-format" placeholder:"(vhd|vhd-fixed|vhdx|qcow2|raw)" help:"Format of output image." enum:"${imageformat}" default:""`
+	exekong.LogFlags
+}
+
+func main() {
+	cli := &ImageCreatorCmd{}
+
+	vars := kong.Vars{
+		"imageformat": strings.Join(imagecustomizerapi.SupportedImageFormatTypesImageCreator(), ",") + ",",
+		"version":     imagecustomizerlib.ToolVersion,
+	}
+	maps.Copy(vars, exekong.KongVars)
+
+	_ = kong.Parse(cli,
+		vars,
+		kong.HelpOptions{
+			Compact:   true,
+			FlagsLast: true,
+		},
+		kong.UsageOnError())
+
+	logger.InitBestEffort(ptrutils.PtrTo(cli.LogFlags.AsLoggerFlags()))
+
+	err := imagecreatorlib.CreateImageWithConfigFile(cli.BuildDir, cli.ConfigFile, cli.RpmSources, cli.ToolsTar, cli.OutputImageFile, cli.OutputImageFormat)
+	if err != nil {
+		log.Fatalf("image creation failed:\n%v", err)
+	}
+}

toolkit/tools/imagecustomizerapi/imageFormatType.go

@@ -37,6 +37,14 @@ var supportedImageFormatTypes = []string{
 	string(ImageFormatTypeCosi),
 }
 
+var supportedImageFormatTypesImageCreator = []string{
+	string(ImageFormatTypeVhd),
+	string(ImageFormatVhdTypeFixed),
+	string(ImageFormatTypeVhdx),
+	string(ImageFormatTypeQcow2),
+	string(ImageFormatTypeRaw),
+}
+
 func (ft ImageFormatType) IsValid() error {
 	if ft != ImageFormatTypeNone && !slices.Contains(SupportedImageFormatTypes(), string(ft)) {
 		return fmt.Errorf("invalid image format type (%s)", ft)
@@ -49,3 +57,7 @@ func (ft ImageFormatType) IsValid() error {
 func SupportedImageFormatTypes() []string {
 	return supportedImageFormatTypes
 }
+
+func SupportedImageFormatTypesImageCreator() []string {
+	return supportedImageFormatTypesImageCreator
+}

toolkit/tools/pkg/imagecustomizerlib/createuuid.go renamed to toolkit/tools/internal/randomization/createuuid.go

@@ -1,27 +1,23 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-package imagecustomizerlib
+package randomization
 
 import (
 	"crypto/rand"
 	"encoding/hex"
 	"fmt"
-	"path/filepath"
 	"strings"
 
-	"github.com/microsoft/azurelinux/toolkit/tools/internal/file"
 	"github.com/microsoft/azurelinux/toolkit/tools/internal/logger"
-	"github.com/microsoft/azurelinux/toolkit/tools/internal/sliceutils"
 )
 
 const (
-	UuidSize                   uint32 = 16
-	ImageCustomizerReleasePath        = "etc/image-customizer-release"
+	UuidSize uint32 = 16
 )
 
 // Create the uuid and return byte array and string representation
-func createUuid() ([UuidSize]byte, string, error) {
+func CreateUuid() ([UuidSize]byte, string, error) {
 	uuid, err := generateRandom128BitNumber()
 	if err != nil {
 		return uuid, "", err
@@ -54,34 +50,7 @@ func convertUuidToString(uuid [UuidSize]byte) string {
 	return uuidStr
 }
 
-func extractImageUUID(imageConnection *ImageConnection) ([UuidSize]byte, string, error) {
-	var emptyUuid [UuidSize]byte
-
-	releasePath := filepath.Join(imageConnection.Chroot().RootDir(), ImageCustomizerReleasePath)
-	data, err := file.Read(releasePath)
-	if err != nil {
-		return emptyUuid, "", fmt.Errorf("failed to read %s:\n%w", releasePath, err)
-	}
-
-	lines := strings.Split(string(data), "\n")
-	line, found := sliceutils.FindValueFunc(lines, func(line string) bool {
-		return strings.HasPrefix(line, "IMAGE_UUID=")
-	})
-	if !found {
-		return emptyUuid, "", fmt.Errorf("IMAGE_UUID not found in %s", releasePath)
-	}
-
-	uuidStr := strings.Trim(strings.TrimPrefix(line, "IMAGE_UUID="), `"`)
-
-	parsed, err := parseUuidString(uuidStr)
-	if err != nil {
-		return emptyUuid, "", fmt.Errorf("failed to parse IMAGE_UUID (%s):\n%w", uuidStr, err)
-	}
-
-	return parsed, uuidStr, nil
-}
-
-func parseUuidString(s string) ([UuidSize]byte, error) {
+func ParseUuidString(s string) ([UuidSize]byte, error) {
 	var uuid [UuidSize]byte
 
 	parts := strings.Split(s, "-")

toolkit/tools/internal/safechroot/safechroot.go

@@ -670,7 +670,6 @@ func (c *Chroot) unmountAndRemove(leaveOnDisk, lazyUnmount bool) (err error) {
 			umountErr := unix.Unmount(fullPath, unmountFlags)
 			return umountErr
 		}, totalAttempts, retryDuration, 2.0)
-
 		if err != nil {
 			err = fmt.Errorf("failed to unmount (%s):\n%w", fullPath, err)
 			return
@@ -687,23 +686,23 @@ func (c *Chroot) unmountAndRemove(leaveOnDisk, lazyUnmount bool) (err error) {
 // defaultMountPoints returns a new copy of the default mount points used by a functional chroot
 func defaultMountPoints() []*MountPoint {
 	return []*MountPoint{
-		&MountPoint{
+		{
 			target: "/dev",
 			fstype: "devtmpfs",
 		},
-		&MountPoint{
+		{
 			target: "/proc",
 			fstype: "proc",
 		},
-		&MountPoint{
+		{
 			target: "/sys",
 			fstype: "sysfs",
 		},
-		&MountPoint{
+		{
 			target: "/run",
 			fstype: "tmpfs",
 		},
-		&MountPoint{
+		{
 			target: "/dev/pts",
 			fstype: "devpts",
 			data:   "gid=5,mode=620",

toolkit/tools/internal/testutils/testrpms/create-tools-file.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+CONTAINER_IMAGE="$1"
+TOOLS_FILE="$2"
+
+if [[ -z "$CONTAINER_IMAGE" || -z "$TOOLS_FILE" ]]; then
+  echo "Usage: $0 <container_image> <tools_file>"
+  exit 1
+fi
+
+docker create --name temp-container "$CONTAINER_IMAGE"
+docker export temp-container | gzip > "$TOOLS_FILE"
+docker rm temp-container

toolkit/tools/pkg/imagecustomizerlib/testdata/testrpms/download-test-rpms.sh renamed to toolkit/tools/internal/testutils/testrpms/download-test-rpms.sh

@@ -1,41 +1,21 @@
-#!/usr/bin/env bash
-set -eu
+#!/bin/bash
 
-SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
-CONTAINER_TAG="imagecustomizertestrpms:latest"
-DOCKERFILE_DIR="$SCRIPT_DIR/downloader"
-
-AZURELINUX_2_CONTAINER_IMAGE="mcr.microsoft.com/cbl-mariner/base/core:2.0"
-
-IMAGE_VERSION="2.0"
+set -e
+CONTAINER_IMAGE="$1"
+IMAGE_VERSION="$2"
+IMAGE_CREATOR="$3"
 
-while getopts "t:" flag
-do
-    case "${flag}" in
-        t) IMAGE_VERSION="$OPTARG";;
-        h) ;;&
-        ?)
-            echo "Usage: download-test-rpms.sh [-t IMAGE_VERSION]"
-            echo ""
-            echo "Args:"
-            echo "  -t IMAGE_VERSION   The Azure Image version to download the RPMs for."
-            echo "  -h Show help"
-            exit 1;;
-    esac
-done
-
-case "${IMAGE_VERSION}" in
-  2.0)
-    CONTAINER_IMAGE="$AZURELINUX_2_CONTAINER_IMAGE"
-    ;;
-  *)
-    echo "error: unsupported Azure Linux version: $IMAGE_VERSION"
-    exit 1;;
-esac
-
-set -x
+# Check if the required arguments are provided
+if [[ -z "$CONTAINER_IMAGE" || -z "$IMAGE_VERSION" || -z "$IMAGE_CREATOR" ]]; then
+  echo "Usage: $0 <container_image> <image_version> <image_creator>"
+  echo "Example: $0 mcr.microsoft.com/azurelinux/base/core:3.0 3.0 true"
+  exit 1
+fi
 
+SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
+DOCKERFILE_DIR="$SCRIPT_DIR/downloader"
 DOWNLOADER_RPMS_DIRS="$SCRIPT_DIR/downloadedrpms"
+CONTAINER_TAG="imagecustomizertestrpms:latest"
 OUT_DIR="$DOWNLOADER_RPMS_DIRS/$IMAGE_VERSION"
 REPO_WITH_KEY_FILE="$DOWNLOADER_RPMS_DIRS/rpms-$IMAGE_VERSION-withkey.repo"
 REPO_NO_KEY_FILE="$DOWNLOADER_RPMS_DIRS/rpms-$IMAGE_VERSION-nokey.repo"
@@ -44,7 +24,8 @@ mkdir -p "$OUT_DIR"
 
 # Build a container image that contains the RPMs.
 docker build \
-  --build-arg "baseimage=$AZURELINUX_2_CONTAINER_IMAGE" \
+  --build-arg "baseimage=$CONTAINER_IMAGE" \
+  --build-arg "imagecreator=$IMAGE_CREATOR" \
   --tag "$CONTAINER_TAG" \
   "$DOCKERFILE_DIR"
 

toolkit/tools/internal/testutils/testrpms/download-test-utils.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -eu
+
+SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
+
+AZURELINUX_2_CONTAINER_IMAGE="mcr.microsoft.com/cbl-mariner/base/core:2.0"
+AZURELINUX_3_CONTAINER_IMAGE="mcr.microsoft.com/azurelinux/base/core:3.0"
+
+IMAGE_VERSION="2.0"
+
+IMAGE_CREATOR="false"
+TOOLS_FILE="$SCRIPT_DIR/tools.tar.gz"
+
+
+while getopts "s:t:" flag
+do
+    case "${flag}" in
+        s) IMAGE_CREATOR="$OPTARG";;
+        t) IMAGE_VERSION="$OPTARG";;
+        h) ;;&
+        ?)
+            echo "Usage: download-test-utils.sh [-t IMAGE_VERSION] [-s IMAGE_CREATOR]"
+            echo ""
+            echo "Args:"
+            echo "  -t IMAGE_VERSION   The Azure Image version to download the RPMs for."
+            echo "  -s IMAGE_CREATOR   If set to true, the script will create a tar.gz file with the tools and download the rpms needed to test imagecreator."
+            echo "  -h Show help"
+            exit 1;;
+    esac
+done
+
+
+case "${IMAGE_VERSION}" in
+  3.0)
+    CONTAINER_IMAGE="$AZURELINUX_3_CONTAINER_IMAGE"
+    ;;
+  2.0)
+    CONTAINER_IMAGE="$AZURELINUX_2_CONTAINER_IMAGE"
+    ;;  
+  *)
+    echo "error: unsupported Azure Linux version: $IMAGE_VERSION"
+    exit 1;;
+esac
+
+set -x
+
+# call the script to create the tools file if requested
+if [ "$IMAGE_CREATOR" = "true" ]; then
+  echo "Creating tools file: $TOOLS_FILE"
+  $SCRIPT_DIR/create-tools-file.sh "$CONTAINER_IMAGE" "$TOOLS_FILE"
+  echo "Tools file created successfully."
+else
+  echo "Skipping tools file creation."
+fi
+
+# call the script to download the rpms
+echo "Downloading test rpms for Azure Linux version: $IMAGE_VERSION"
+$SCRIPT_DIR/download-test-rpms.sh "$CONTAINER_IMAGE"  "$IMAGE_VERSION" "$IMAGE_CREATOR" 
+echo "Test rpms downloaded successfully."

toolkit/tools/internal/testutils/testrpms/downloader/Dockerfile

@@ -0,0 +1,24 @@
+ARG baseimage
+FROM ${baseimage}
+ARG imagecreator
+
+RUN tdnf update -y && \
+   tdnf install -y dnf dnf-plugins-core createrepo_c
+
+RUN dnf download -y --resolve --alldeps --destdir /downloadedrpms \
+   jq golang 
+   
+RUN if [ "${imagecreator}" = "true" ]; then \
+
+    dnf download -y --resolve --alldeps --destdir /downloadedrpms \
+    azurelinux-release \
+    azurelinux-repos azurelinux-rpm-macros bash dbus dracut-hostonly e2fsprogs filesystem \
+    grub2 grub2-efi-binary iana-etc iproute iputils irqbalance \
+    ncurses-libs openssl rpm rpm-libs shadow-utils shim sudo \
+    systemd systemd-networkd systemd-resolved systemd-udev tdnf \
+    tdnf-plugin-repogpgcheck util-linux zlib kernel initramfs ; \
+
+    fi
+
+# Add repo metadata, so that the directory can be used in a .repo file.
+RUN createrepo --compatibility --update /downloadedrpms