From 3669eaf4b18b1f9571cc4c4f6f0536ef07c065f8 Mon Sep 17 00:00:00 2001 From: Philippe Vanhaesendonck Date: Mon, 25 Mar 2024 14:34:51 +0100 Subject: [PATCH] refactor(olit): :recycle: libvirt/libguestfs refactor Signed-off-by: Philippe Vanhaesendonck --- oracle-linux-image-tools/.shellcheckrc | 2 + oracle-linux-image-tools/CHANGELOG.md | 88 +++ oracle-linux-image-tools/README.md | 151 ++-- oracle-linux-image-tools/bin/build-image.sh | 673 +++++++----------- oracle-linux-image-tools/bin/common.sh | 166 ++++- oracle-linux-image-tools/bin/mkpasswd.py | 33 - oracle-linux-image-tools/bin/mnt-img.sh | 270 ------- .../bin/provision-common.sh | 191 +++++ oracle-linux-image-tools/bin/provision.sh | 93 +-- .../cloud/azure/image-scripts.sh | 30 +- .../cloud/azure/provision.sh | 26 +- .../cloud/none/image-scripts.sh | 41 +- .../cloud/oci/env.properties | 2 + .../cloud/oci/image-scripts.sh | 34 +- .../oci/ol7-slim/files/oci-yum-repo-mapper | 191 +++++ .../files/oci-yum-repo-mapper.service | 17 + .../cloud/oci/ol8-aarch64 | 1 + .../oci/ol8-slim/files/oci-yum-repo-mapper | 240 +++++++ .../files/oci-yum-repo-mapper.service | 17 + .../cloud/oci/ol8-slim/provision.sh | 30 + .../cloud/oci/ol9-aarch64 | 1 + .../oci/ol9-slim/files/oci-yum-repo-mapper | 206 ++++++ .../files/oci-yum-repo-mapper.service | 20 + .../cloud/oci/provision.sh | 26 +- .../cloud/olvm/image-scripts.sh | 44 +- .../cloud/olvm/mk-envelope.py | 6 +- .../cloud/olvm/provision.sh | 16 +- .../cloud/ovm/env.properties | 3 + .../cloud/ovm/image-scripts.sh | 32 +- .../cloud/ovm/mk-envelope-template.sh | 2 +- .../cloud/ovm/ol7-slim/provision.sh | 4 +- .../cloud/ovm/ol8-slim/env.properties | 1 - .../cloud/ovm/ol8-slim/image-scripts.sh | 4 +- .../cloud/ovm/ol8-slim/provision.sh | 10 +- .../cloud/ovm/ol9-slim/image-scripts.sh | 4 +- .../cloud/ovm/ol9-slim/provision.sh | 12 +- .../cloud/ovm/provision.sh | 10 +- .../cloud/utm/env.properties | 6 + .../cloud/utm/image-scripts.sh | 34 +- .../cloud/utm/provision.sh | 10 +- .../cloud/vagrant-libvirt/image-scripts.sh | 65 +- .../cloud/vagrant-libvirt/provision.sh | 8 +- .../files/vagrant-common.sh | 25 +- .../cloud/vagrant-virtualbox/image-scripts.sh | 144 ++-- .../cloud/vagrant-virtualbox/mk-envelope.py | 583 +++++++++++++++ .../cloud/vagrant-virtualbox/provision.sh | 36 +- .../custom/template/files/custom.txt | 2 +- .../custom/template/image-scripts.sh | 57 +- .../custom/template/provision.sh | 8 +- .../distr/ol7-slim/env.properties | 22 +- .../distr/ol7-slim/image-scripts.sh | 64 +- .../distr/ol7-slim/ol7-ks.cfg | 2 + .../distr/ol7-slim/provision.sh | 255 ++----- .../distr/ol8-aarch64/env.properties | 26 +- .../distr/ol8-aarch64/image-scripts.sh | 81 +-- .../distr/ol8-aarch64/ol8-aarch64-ks.cfg | 2 +- .../distr/ol8-aarch64/provision.sh | 240 ++----- .../distr/ol8-slim/env.properties | 22 +- .../distr/ol8-slim/image-scripts.sh | 58 +- .../distr/ol8-slim/provision.sh | 273 ++----- .../distr/ol9-aarch64/env.properties | 26 +- .../distr/ol9-aarch64/image-scripts.sh | 62 +- .../distr/ol9-aarch64/ol9-aarch64-ks.cfg | 6 - .../distr/ol9-aarch64/provision.sh | 230 +----- .../distr/ol9-slim/env.properties | 22 +- .../distr/ol9-slim/image-scripts.sh | 54 +- .../distr/ol9-slim/ol9-ks.cfg | 6 - .../distr/ol9-slim/provision.sh | 235 ++---- oracle-linux-image-tools/env.properties | 93 ++- .../env.properties.defaults | 46 +- .../images/.gitattributes | 1 + oracle-linux-image-tools/images/olit.png | 3 + .../packer-template/build.pkr.hcl | 30 - .../packer-template/qemu-aarch64.pkr.hcl | 88 --- .../packer-template/qemu-x86-64.pkr.hcl | 32 - .../packer-template/variables.pkr.hcl | 122 ---- .../packer-template/virtualbox-x86-64.pkr.hcl | 38 - 77 files changed, 2921 insertions(+), 2893 deletions(-) create mode 100644 oracle-linux-image-tools/.shellcheckrc create mode 100644 oracle-linux-image-tools/CHANGELOG.md delete mode 100755 oracle-linux-image-tools/bin/mkpasswd.py delete mode 100755 oracle-linux-image-tools/bin/mnt-img.sh create mode 100755 oracle-linux-image-tools/bin/provision-common.sh create mode 100755 oracle-linux-image-tools/cloud/oci/ol7-slim/files/oci-yum-repo-mapper create mode 100644 oracle-linux-image-tools/cloud/oci/ol7-slim/files/oci-yum-repo-mapper.service create mode 120000 oracle-linux-image-tools/cloud/oci/ol8-aarch64 create mode 100755 oracle-linux-image-tools/cloud/oci/ol8-slim/files/oci-yum-repo-mapper create mode 100644 oracle-linux-image-tools/cloud/oci/ol8-slim/files/oci-yum-repo-mapper.service create mode 100755 oracle-linux-image-tools/cloud/oci/ol8-slim/provision.sh create mode 120000 oracle-linux-image-tools/cloud/oci/ol9-aarch64 create mode 100755 oracle-linux-image-tools/cloud/oci/ol9-slim/files/oci-yum-repo-mapper create mode 100644 oracle-linux-image-tools/cloud/oci/ol9-slim/files/oci-yum-repo-mapper.service create mode 100644 oracle-linux-image-tools/cloud/utm/env.properties create mode 100755 oracle-linux-image-tools/cloud/vagrant-virtualbox/mk-envelope.py create mode 100644 oracle-linux-image-tools/images/.gitattributes create mode 100644 oracle-linux-image-tools/images/olit.png delete mode 100644 oracle-linux-image-tools/packer-template/build.pkr.hcl delete mode 100644 oracle-linux-image-tools/packer-template/qemu-aarch64.pkr.hcl delete mode 100644 oracle-linux-image-tools/packer-template/qemu-x86-64.pkr.hcl delete mode 100644 oracle-linux-image-tools/packer-template/variables.pkr.hcl delete mode 100644 oracle-linux-image-tools/packer-template/virtualbox-x86-64.pkr.hcl diff --git a/oracle-linux-image-tools/.shellcheckrc b/oracle-linux-image-tools/.shellcheckrc new file mode 100644 index 0000000..c56116d --- /dev/null +++ b/oracle-linux-image-tools/.shellcheckrc @@ -0,0 +1,2 @@ +# Allow opening any 'source'd file, even if not specified as input +external-sources=true diff --git a/oracle-linux-image-tools/CHANGELOG.md b/oracle-linux-image-tools/CHANGELOG.md new file mode 100644 index 0000000..1394746 --- /dev/null +++ b/oracle-linux-image-tools/CHANGELOG.md @@ -0,0 +1,88 @@ +# Release Notes + +## March, 2024 + +Major refactoring of the scripts, reducing dependencies on third parties. +The build tools mainly rely on [`qemu-kvm`](http://www.qemu.org/), using [`libvirt`](https://libvirt.org/) and [`libguestfs`](https://libguestfs.org/). + +As it is a breaking change, previous version has been archived under the `olit-legacy` branch (unmaintained). + +### Refactor + +The overall build process has been changed. In previous releases we had: + +1. Create image from distribution ISO and a kickstart file +1. Customize the image by running provision scripts inside the running VM +1. Cleanup the environment by mounting the image filesystems on the host; + the outcome is a raw disk image +1. Package the image for the target cloud + +As of this release, we have: + +1. Create image: unchanged +1. Use [`virt-customize`](customize) to run the provisioning scripts +1. Use [`virt-sysprep`](https://libguestfs.org/virt-sysprep.1.html)/[`virt-sparsify`](https://libguestfs.org/virt-sparsify.1.html) for the cleanup; + the outcome is a compressed qcow2 image +1. Package image: unchanged + +Notable code changes: + +- Drop support for [VirtualBox](https://www.virtualbox.org/) as **builder** (you can still create Vagrant VirtualBox **images**) +- `image-scripts.sh` `::seal()` functions obsolete; code moved to `provision.sh` `::cleanup()` functions. + We don't need anymore a separate _offline_ cleanup as `virt-customize` doesn't actually run the built VM. +- Simplify `provision.sh` `::cleanup()` functions as most parts are now handled by `virt-sysprep` operations. +- Add `image-scripts.sh` `::customize_args()` and `::sysprep_args()` hooks to inject command line parameters for `virt-customize` and `virt-sysprep`. +- Root privileges are no longer required on the build host. +- Root access to the image VM is no longer needed at built time. Root password and/or ssh public key can still be set for the image; parameters have been renamed to ensure configuration is secure by default. +- Move common code to the `common.sh` and `provision-common.sh` libraries. +- QCOW2 image files now have the `qcow2` extension instead of `qcow`. + +### Configuration variables + +Changes to the configuration variables. +See the corresponding `env.properties` files for more details. + +New variables + +- Generic + - `INSTALL_WAIT_TIME`: configurable timeout for initial image creation + - `OS_VARIANT` (optional): OS variant used when creating the image + - `BOOT_MODE`: OS boot mode (`bios` or `efi`) + - `BOOT_COMMAND_SERIAL_CONSOLE`: kernel parameters to enable serial console + - `BOOT_LOCATION`(optional): kernel and initrd location on the distribution media + - `ROOT_PASSWORD` (optional, default: locked): password for the root account in the generated image + - `ROOT_SSH_KEY` (optional): public ssh key the root account in the generated image + - `PERMIT_ROOT_LOGIN` (Default: prohibit-password): default policy for ssh root login + - `CACHE_DIR` (Default: `.cache` in workspace directory): location of ISO images cache +- utm cloud + - `OPC_PASSWORD`: password for the `opc` user for UTM builds + +Changed variables + +- `DISTR`: is now mandatory +- `ISO_LABEL`: is now optional +- `BOOT_COMMAND`: array of kernel parameters instead of a string + +Obsolete variables + +- `LOCK_ROOT`, `SSH_KEY_FILE`, `SSH_PASSWORD`: root access to the image is not needed anymore, see new `ROOT_PASSWORD`, `ROOT_SSH_KEY` if root access to the generated image is needed +- `X2APIC` +- `PACKER`, `PACKER_BUILD_OPTIONS`, `PACKER_BUILDER` +- `QEMU_BINARY` + +### New features + +- sshd `PermitRootLogin` parameter is now `prohibit-password` by default for all images (instead of `yes` for OL7/OL8) +- update azure cloud for OL9 + +### Bug fixes + +- Wrong pattern matching in bash regular expressions when validating variables +- TERM variable in serial console configuration for OVM not escaped properly +- Wrong swap page size for aarch64 builds when host is running UEK6 kernel +- Workaround for OL8 cloud-init issue in OCI +- Setup OCI yum mirrors for OCI images + +### Documentation + +README file updated diff --git a/oracle-linux-image-tools/README.md b/oracle-linux-image-tools/README.md index e3ea61c..a2577bd 100755 --- a/oracle-linux-image-tools/README.md +++ b/oracle-linux-image-tools/README.md @@ -4,87 +4,101 @@ This repository provides tools to build Oracle Linux images for cloud deployment. -The images built by these tools are based on distribution flavours and target packages. -Image building is accomplished using Packer to build images from the Oracle Linux ISO using Oracle VM VirtualBox or QEMU/KVM builders. +__Note__: as of March 2024 the scripts have been refactored and introduce breaking changes. See [CHANGELOG](CHANGELOG.md) for details. -The tool currently supports: +The tools are architected around _distribution flavours_ and _target clouds_. +They currently support: - Distributions: - - Oracle Linux 7 update 9 -- Slim (x86_64) - - Oracle Linux 8 update 9 -- Slim (x86_64 and aarch64) - __Note__: for aarch64, only Generic, OCI and UTM clouds are supported - - Oracle Linux 9 update 3 -- Slim (x86_64 and aarch64) - __Note__: for aarch64, only Generic, OCI and UTM clouds are supported + - Oracle Linux 7 update 9 -- Slim (x86_64 only) + - Oracle Linux 8 update 9 -- Slim (x86_64 and aarch64) + - Oracle Linux 9 update 3 -- Slim (x86_64 and aarch64) - Clouds: - - Microsoft Azure cloud + - Microsoft Azure cloud (x86_64) Target packages: WALinuxAgent Image format: VHD - - Oracle Cloud Infrastructure (OCI) + - Oracle Cloud Infrastructure (OCI) (x86_64 and aarch64) Target packages: qemu-guest-agent / cloud-init Image format: QCOW2 __Note__: no specific OCI tools are actually installed; this image can be used in any cloud-init based environment. - - Oracle Linux Virtualization Manager (OLVM) + - Oracle Linux Virtualization Manager (OLVM) (x86_64) Target packages: qemu-guest-agent / cloud-init Image format: OLVM OVA - - Oracle VM Server (OVM) + - Oracle VM Server (OVM) (x86_64) Target packages: oracle-template-config + vmapi Image format: OVM OVA - - Vagrant (VirtualBox provider - requires VirtualBox for the build) + - Vagrant (VirtualBox provider) (x86_64) Target packages: VirtualBox guest additions Image format: box - - Vagrant (libvirt provider) + - Vagrant (libvirt provider) (x86_64) Target packages: nfs-utils Image format: box - - UTM ([UTM for macOS](https://mac.getutm.app/)) + - UTM ([UTM for macOS](https://mac.getutm.app/)) (aarch64) Target packages: none Image format: utm __Note__: only for aarch64 distributions - - Generic (No cloud setup) + - Generic (No cloud setup) (x86_64 and aarch64) Target packages: none Image format: VirtualBox OVA or QCOW2 (depending on the builder used) -Additional information is available in the [Building (Small) Oracle Linux Images For The Cloud](https://blogs.oracle.com/linux/post/building-small-oracle-linux-images-for-the-cloud) blog post. +## Requirements + +### Overview + +The tools require a Linux host supporting [KVM](https://linux-kvm.org) virtualization with the following installed: + +- [`qemu-kvm`](http://www.qemu.org/) (Including `qemu-img`) +- [`libvirt`](https://libvirt.org/) +- [`virt-install`](https://virt-manager.org/) +- [`libguestfs`](https://libguestfs.org/) (including tools) + +Additionally: + +- the host architecture must match the architecture of the target image (e.g.: an `aarch64` host is needed to build `aarch64` images) +- the host kernel must support the filesystem used in the guest (e.g.: the host kernel must support `btrfs` to build an image with a `btrfs` filesystem) + +For building [HashiCorp Vagrant](https://vagrantup.com/) boxes for the libvirt provider, download the [`create_box.sh`](https://github.com/vagrant-libvirt/vagrant-libvirt/blob/master/tools/create_box.sh) third party script from the [`vagrant-libvirt`](https://github.com/vagrant-libvirt/vagrant-libvirt) project or install [Vagrant](https://vagrantup.com/) and the [`vagrant-libvirt`](https://github.com/vagrant-libvirt/vagrant-libvirt) plugin. + +### Oracle Linux 8 + +```shell +dnf module install virt +dnf install qemu-img libguestfs-tools virt-install +dnf install zip # For UTM images +``` + +### Oracle Linux 9 + +```shell +dnf install libvirt qemu-kvm libguestfs +dnf install qemu-img guestfs-tools virt-install +dnf install zip # For UTM images +``` ## Build instructions -The build script requires a Linux environment and has been tested on Oracle Linux 7 and 8. - -1. Install either QEMU or VirtualBox (VirtualBox required for Vagrant VirtualBox images): - - Oracle Linux 7: - `yum --enablerepo=ol7_kvm_utils group install "Virtualization Host"` - or - `yum --enablerepo=ol7_developer install VirtualBox-7.0` - - Oracle Linux 8: - `dnf module install virt` - or - `dnf --enablerepo=ol8_developer install VirtualBox-7.0` -1. Install `kpartx` and `qemu-img` to manipulate the artifacts - - Oracle Linux 7: - `yum --enablerepo=ol7_kvm_utils install kpartx qemu-img` - - Oracle Linux 8: - `dnf install kpartx qemu-img` -1. Install packer: - - Oracle Linux 7: - `yum --enablerepo=ol7_developer install packer` - - Oracle Linux 8: Download and install Packer from [HashiCorp](https://www.packer.io/downloads/) -1. Cloud specific requirements: - - For `Vagrant` box (VirtualBox provider), install [HashiCorp Vagrant](https://vagrantup.com/) - - For `Vagrant` box (libvirt provider), download the [`create_box.sh`](https://github.com/vagrant-libvirt/vagrant-libvirt/blob/master/tools/create_box.sh) third party script from the [`vagrant-libvirt`](https://github.com/vagrant-libvirt/vagrant-libvirt) project or install [HashiCorp Vagrant](https://vagrantup.com/) and the [`vagrant-libvirt`](https://github.com/vagrant-libvirt/vagrant-libvirt) plugin -1. Clone this repo: - `git clone https://github.com/oracle/oracle-linux.git` -1. The build script need root privileges during the build. - Ensure `sudo` is properly configured for your build user +The image builder does not require system privileges and should not be run as root. + +1. Clone this repo: + + ```shell + git clone https://github.com/oracle/oracle-linux.git + ``` + 1. Set up a separate workspace directory where the image will be built. Ensure there is enough free space in the workspace partition, the builder will need up the two times the image size. 1. Configure your build environment in the `env.properties` file (or in a copy). Minimal configuration: - `WORKSPACE`: path of your workspace directory - `ISO_URL`: location of the Oracle Linux distribution ISO - - `ISO_CHECKSUM`: checksum for the ISO file. As from packer 1.6.0, you can prepend the checksum type (see [packer documentation](https://www.packer.io/docs/builders/virtualbox/iso#iso_checksum)) - - `CLOUD`: cloud target (azure, oci, olvm, ovm or none) - - `PACKER_BUILDER`: builder used by packer (virtualbox-iso.x86-64 or qemu.x86-64) -1. Run the builder: - `./bin/build-image.sh --env ENV_PROPERTY_FILE` + - `ISO_CHECKSUM`: checksum for the ISO file + Checksums files are available on the [Verify Oracle Linux Downloads](https://linux.oracle.com/security/gpg/) page + - `CLOUD`: cloud target (azure, oci, olvm, ovm, utm, vagrant-libvirt, vagrant-virtualbox or none) +1. Run the builder as a non-privileged user: + + ```shell + ./bin/build-image.sh --env ENV_PROPERTY_FILE` + ``` ## Advanced configuration @@ -96,7 +110,7 @@ In that case, you will have to provide an URL to an installation tree and option Example for an Oracle Linux 9 using the UEK boot ISO: ```Shell -ISO_URL="https://yum.oracle.com/ISOS/OracleLinux/OL9/u2/x86_64/OracleLinux-R9-U2-x86_64-boot-uek.iso" +ISO_URL="https://yum.oracle.com/ISOS/OracleLinux/OL9/u3/x86_64/OracleLinux-R9-U3-x86_64-boot-uek.iso" REPO_URL="https://yum.oracle.com/repo/OracleLinux/OL9/baseos/latest/x86_64" REPO[AppStream]="https://yum.oracle.com/repo/OracleLinux/OL9/appstream/x86_64" REPO[ol9_UEKR7]="https://yum.oracle.com/repo/OracleLinux/OL9/UEKR7/x86_64" @@ -127,8 +141,10 @@ For a given Oracle Linux distribution and target Cloud, the following properties - Local `env.properties` file (passed as parameter to the builder) Files are processed in that order. -Changes should be made to a local env.properties file which can override any definition made in an upstream property file. -Relevant parameters are documented in the distributed [`env.properties`](env.properties) file. +Changes should be made to the local env.properties file which will override any definition made in an upstream property file. +Relevant parameters are documented in the distributed [`env.properties`](env.properties) sample file. + +![File layout and process](images/olit.png) ## Cloud specific notes @@ -140,11 +156,11 @@ This can be done from the Console, or using the [Command Line Interface (CLI)](h ```shell # Upload in the Object Storage Bucket oci os object put -bn my_bucket \ - --file /workspace/OL7U9_x86_64-oci-b0/OL7U9_x86_64-oci-b0.qcow + --file /workspace/OL7U9_x86_64-oci-b0/OL7U9_x86_64-oci-b0.qcow2 # Import as Custom image oci compute image import from-object -bn my_bucket \ --namespace my_namespace \ - --name OL7U9_x86_64-oci-b0.qcow \ + --name OL7U9_x86_64-oci-b0.qcow2 \ --display-name MyImage \ --launch-mode PARAVIRTUALIZED \ --source-image-type QCOW2 @@ -171,7 +187,7 @@ For cloud-init support, you will need to specify `CLOUD_INIT="Yes"` in your `env The `build-image` script relies on the following directory structure: -- distr: directory for all Oracle Linux distribution +- distr: directory for all Oracle Linux distributions - _distribution name_ - env.properties: distribution parameters - _name_-ks.cfg: kickstart file for the distribution @@ -203,13 +219,11 @@ The builder will process the directories in the following order: - distr::kickstart - cloud_distr::kickstart - custom::kickstart -1. Select a packer configuration file and customise it. The following hooks are called if defined: - - distr::packer_conf - - cloud_distr::packer_conf - - custom::packer_conf -1. Stage files from the _files_ directories. These files are copied during provisioning in `/tmp/packer_files` in the VM. -1. Run packer to provision the VM image. - During provisioning the `provision.sh` scripts run in the following order: +1. Stage files from the _files_ directories. These files are copied during provisioning in `PROVISION_DIR` in the VM. +1. Run `virt-install` to create the image as described in the kickstart file. +1. Run `virt-customize` to actually provision the image. + The optional `::customize_args` hooks in the `image_scripts.sh` files are invoked to provide additional arguments to `virt-customize`. + The `provision.sh` scripts run in the following order: - distr::provision - cloud::provision - cloud_distr::provision @@ -218,23 +232,14 @@ The builder will process the directories in the following order: - cloud_distr::cleanup - cloud::cleanup - distr::cleanup - - distr::seal[^1] -1. Image cleanup: the generated image is mounted on the host and the `image-scripts` scripts are run[^1]: - - custom::cleanup - - cloud_distr::cleanup - - cloud::cleanup - - distr::cleanup +1. Run `virt-sysprep` to _seal_ the image (final cleanup). + The optional `::sysprep_args` hooks in the `image_scripts.sh` files are invoked to provide additional arguments to `virt-sysprep`. 1. Image packaging: the generated image is packaged in its final format. Only the first script found is executed: - custom::image_package - cloud_distr::image_package - cloud::image_package -[^1]: `provision` `seal` vs. `image-scripts` `cleanup`. -These functions have the same purpose: _seal_ the image before packaging. -The difference is that the former runs in the VM while the latter runs on the host. -Sealing on the host might be more efficient, but when it is not possible to mount the image disk on the host, in-VM sealing can be used. When no `image-scripts` `cleanup` are defined, no attempt will be made to mount the filesystem on the host. - ## Feedback Please provide feedback of any kind via GitHub issues on this repository. diff --git a/oracle-linux-image-tools/bin/build-image.sh b/oracle-linux-image-tools/bin/build-image.sh index 0ebcf14..6b9448c 100755 --- a/oracle-linux-image-tools/bin/build-image.sh +++ b/oracle-linux-image-tools/bin/build-image.sh @@ -3,7 +3,7 @@ # # Create minimal Oracle Linux images # -# Copyright (c) 2019, 2022 Oracle and/or its affiliates. +# Copyright (c) 2019, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl. # @@ -14,21 +14,19 @@ # # Constants -readonly PGM=$(basename "$0") -readonly BIN_DIR=$( cd "$(dirname "$0")" ; pwd -P ) -readonly REPO_DIR=$(dirname "${BIN_DIR}") +PGM=$(basename "$0") +BIN_DIR=$( cd "$(dirname "$0")" ; pwd -P ) +REPO_DIR=$(dirname "${BIN_DIR}") +readonly PGM BIN_DIR REPO_DIR readonly DISTR_DIR="${REPO_DIR}/distr" readonly CLOUD_DIR="${REPO_DIR}/cloud" readonly CUSTOM_DIR="${REPO_DIR}/custom" -readonly TEMPLATE_DIR="${REPO_DIR}/packer-template" readonly ENV_FILE="env.properties" readonly ENV_FILE_DEFAULTS="${REPO_DIR}/${ENV_FILE}.defaults" readonly FILES_DIR="files" -readonly PACKER_FILES="packer_files" -readonly SHUTDOWN_CMD="shutdown -P now; init 0" +readonly PROVISION_DIR="provision.d" readonly PROVISION_SCRIPT="provision.sh" readonly IMAGE_SCRIPTS="image-scripts.sh" -readonly MOUNT_IMAGE="${BIN_DIR}/mnt-img.sh" # Exit on error set -e @@ -36,7 +34,7 @@ set -e ####################################### # Print usage message and exit # Globals: -# PGM REPO_DIR ENV_FILE +# ENV_FILE, PGM, REPO_DIR # Arguments: # None # Returns: @@ -49,58 +47,18 @@ usage() { exit 1 } -####################################### -# Print header -# Globals: -# PGM -# Arguments: -# None -# Returns: -# None -####################################### -echo_header() { - echo "$(tput setaf 2)+++ ${PGM}: $*$(tput sgr 0)" -} - -####################################### -# Print message -# Globals: -# PGM -# Arguments: -# None -# Returns: -# None -####################################### -echo_message() { - echo " ${PGM}: $*" -} - -####################################### -# Print error message and exit -# Globals: -# PGM -# Arguments: -# message -# Returns: -# None -####################################### -error() { - echo "$(tput setaf 1)+++ ${PGM}: $*$(tput sgr 0)" >&2 - exit 1 -} - ####################################### # Parse arguments # Exit on error. # Globals: -# ENV_FILE LOCAL_ENV_FILE REPO_DIR +# ENV_FILE, LOCAL_ENV_FILE, REPO_DIR # Arguments: # Command line # Returns: # None ####################################### parse_args() { - echo_header "Parse arguments" + common::echo_header "Parse arguments" LOCAL_ENV_FILE="${REPO_DIR}/${ENV_FILE}" while [[ $# -gt 0 ]]; do @@ -129,7 +87,8 @@ parse_args() { ####################################### # Load environment variables # Globals: -# LOCAL_ENV_FILE GLOBAL_ENV_FILE DISTR CLOUD WORKSPACE PACKER_FILES +# CLOUD, CLOUD_DIR, CUSTOM, CUSTOM_DIR, DISTR, DISTR_DIR, ENV_FILES +# ENV_FILE_DEFAULTS, LOCAL_ENV_FILE, WORKSPACE # Loaded environment files... # Arguments: # None @@ -137,80 +96,68 @@ parse_args() { # None ####################################### load_env() { - echo_header "Load environment" - - local file + common::echo_header "Load environment" [[ -e "${ENV_FILE_DEFAULTS}" ]] || - error "missing default env file '${ENV_FILE_DEFAULTS}'" + common::error "missing default env file '${ENV_FILE_DEFAULTS}'" source "${ENV_FILE_DEFAULTS}" [[ -e "${LOCAL_ENV_FILE}" ]] || - error "env file '${LOCAL_ENV_FILE}' does not exists" + common::error "env file '${LOCAL_ENV_FILE}' does not exists" source "${LOCAL_ENV_FILE}" # Check for minimal environment - [[ -z "${WORKSPACE}" ]] && error "no workspace directory defined" + [[ -z "${WORKSPACE}" ]] && common::error "no workspace directory defined" [[ -d "${WORKSPACE}" ]] || - error "workspace directory '${WORKSPACE}' does not exist" - [[ -z "${DISTR}" ]] && error "no distribution defined" - [[ -d "${DISTR_DIR}/${DISTR}" ]] || error "no such distribution: ${DISTR}" - [[ -z "${CLOUD}" ]] && error "no cloud defined" - [[ -d "${CLOUD_DIR}/${CLOUD}" ]] || error "no such cloud: ${CLOUD}" + common::error "workspace directory '${WORKSPACE}' does not exist" + [[ -z "${DISTR}" ]] && common::error "no distribution defined" + [[ -d "${DISTR_DIR}/${DISTR}" ]] || common::error "no such distribution: ${DISTR}" + [[ -z "${CLOUD}" ]] && common::error "no cloud defined" + [[ -d "${CLOUD_DIR}/${CLOUD}" ]] || common::error "no such cloud: ${CLOUD}" [[ -z "${CUSTOM}" ]] && - error "no custom project defined (use 'none' for no customization)" + common::error "no custom project defined (use 'none' for no customization)" [[ "${CUSTOM}" == "none" || -d "${CUSTOM_DIR}/${CUSTOM}" ]] || - error "no such custom project: ${CUSTOM}" - - # Generate global env file - rm -rf "${WORKSPACE:?}/${PACKER_FILES}" - mkdir "${WORKSPACE}/${PACKER_FILES}" - readonly GLOBAL_ENV_FILE="${WORKSPACE}/${PACKER_FILES}/${ENV_FILE}" - for file in \ - "${ENV_FILE_DEFAULTS}" \ - "${DISTR_DIR}/${DISTR}/${ENV_FILE}" \ - "${CLOUD_DIR}/${CLOUD}/${ENV_FILE}" \ - "${CLOUD_DIR}/${CLOUD}/${DISTR}/${ENV_FILE}" \ - "${CUSTOM_DIR}/${CUSTOM}/${ENV_FILE}" \ + common::error "no such custom project: ${CUSTOM}" + + # Load all environment files + # LOCAL_ENV_FILE is re-loaded as it overrides all defaults + local env_file + declare -ag ENV_FILES + ENV_FILES=( + "${ENV_FILE_DEFAULTS}" + "${DISTR_DIR}/${DISTR}/${ENV_FILE}" + "${CLOUD_DIR}/${CLOUD}/${ENV_FILE}" + "${CLOUD_DIR}/${CLOUD}/${DISTR}/${ENV_FILE}" + "${CUSTOM_DIR}/${CUSTOM}/${ENV_FILE}" "${LOCAL_ENV_FILE}" - do - [[ -r "${file}" ]] && cat "${file}" >> "${GLOBAL_ENV_FILE}" + ) + readonly ENV_FILES + for env_file in "${ENV_FILES[@]}"; do + [[ -r "${env_file}" ]] && source "${env_file}" done - # Load it - source "${GLOBAL_ENV_FILE}" - readonly WORKSPACE DISTR CLOUD CUSTOM # Basic validation - [[ ${PACKER_BUILDER} =~ ^(virtualbox-iso\.|qemu\.) ]] || - error "Invalid PACKER_BUILDER: ${PACKER_BUILDER}" - [[ ${PACKER_BUILDER} =~ ^qemu\. && -n ${QEMU_BINARY} && ! -x ${QEMU_BINARY} ]] && - error "QEMU binary ${QEMU_BINARY} not found" - - [[ -z "${ISO_URL}" ]] && error "missing ISO URL" - [[ -z "${ISO_CHECKSUM}" ]] && error "missing ISO checksum" + [[ -z "${ISO_URL}" ]] && common::error "missing ISO URL" + [[ ${ISO_URL%%:*} =~ ^((https?)|(file))$ ]] || common::error "invalid ISO URL: ${ISO_URL}" + [[ -z "${ISO_CHECKSUM}" ]] && common::error "missing ISO checksum" + [[ ${#ISO_CHECKSUM} -eq 40 || ${#ISO_CHECKSUM} -eq 64 ]] || + common::error "ISO_CHECKSUM must be SHA1 or SHA256" readonly ISO_URL ISO_CHECKSUM - [[ -z "${SSH_PASSWORD}" && -z "${SSH_KEY_FILE}" ]] && - error "need at least ssh key or password" - if [[ -n "${SSH_KEY_FILE}" ]]; then - if [[ -r "${SSH_KEY_FILE}.pub" ]]; then - SSH_PUB_KEY=$(cat "${SSH_KEY_FILE}.pub") - else - error "missing public key file: ${SSH_KEY_FILE}.pub" - fi - fi - [[ -n "${SSH_PASSWORD}" || -r "${SSH_KEY_FILE}" ]] || - error "missing private key file: ${SSH_KEY_FILE}" - readonly SSH_PASSWORD SSH_KEY_FILE SSH_PUB_KEY - - [[ "${LOCK_ROOT,,}" =~ ^(yes)|(no)$ ]] || error "LOCK_ROOT must be yes or no" - readonly LOCK_ROOT + [[ ${ROOT_PASSWORD} =~ ^((file:)|(password:)|(locked)) ]] || + common::error "invalid root password selector: ${ROOT_PASSWORD}" + [[ -z ${ROOT_SSH_KEY} || ${ROOT_SSH_KEY} =~ ^((file:)|(string:)) ]] || + common::error "invalid root ssh key selector: ${ROOT_SSH_KEY}" + [[ ${PERMIT_ROOT_LOGIN,,} =~ ^((yes)|(prohibit-password)|(forced-commands-only)|(no))$ ]] || + common::error "invalid root login selector: ${PERMIT_ROOT_LOGIN}" + readonly ROOT_PASSWORD ROOT_SSH_KEY PERMIT_ROOT_LOGIN # Attempt to derive DISTR_NAME from the iso image name, otherwise fall back # to the configured name local distr_name + # Note: OL7 media have space in the label which needs to be escaped # shellcheck disable=SC2001 distr_name=$(sed -e 's/^.*OracleLinux-R\([[:digit:]]\)-U\([[:digit:]]\+\)\(-Server\)\?-\([^-]\+\)\(-dvd\)\?\(-[[:digit:]]\+\)\?\.iso$/OL\1U\2_\4/' <<< "${ISO_URL}") if [[ $distr_name =~ ^OL[6789]U ]]; then @@ -218,7 +165,7 @@ load_env() { fi [[ -z "${DISTR_NAME}" && -z "${BUILD_NUMBER}" ]] && - error "missing distribution name / build number" + common::error "missing distribution name / build number" if [[ -z "${VM_NAME}" ]]; then VM_NAME="${DISTR_NAME}-${CLOUD}-b${BUILD_NUMBER}" fi @@ -226,24 +173,42 @@ load_env() { readonly DISTR_NAME BUILD_NUMBER VM_NAME [[ -e "${WORKSPACE}/${VM_NAME}" ]] && - error "${WORKSPACE}/${VM_NAME} already exists" + common::error "${WORKSPACE}/${VM_NAME} already exists" + + [[ ${DISK_SIZE_GB} =~ ^[0-9]+$ ]] || common::error "disk size is not numeric" + readonly DISK_SIZE_GB - [[ ${DISK_SIZE_GB} =~ ^[0-9]+$ ]] || error "disk size is not numeric" - DISK_SIZE_MB=$(( DISK_SIZE_GB * 1024 )) - readonly DISK_SIZE_GB DISK_SIZE_MB + [[ ${INSTALL_WAIT_TIME} =~ ^[0-9]+$ ]] || common::error "install wait time is not numeric" + readonly INSTALL_WAIT_TIME - [[ "${SETUP_SWAP,,}" =~ ^(yes)|(no)$ ]] || error "SETUP_SWAP must be yes or no" + [[ "${SETUP_SWAP,,}" =~ ^((yes)|(no))$ ]] || common::error "SETUP_SWAP must be yes or no" readonly SETUP_SWAP - [[ "${SELINUX,,}" =~ ^(enforcing)|(permissive)|(disabled)$ ]] || error "SELINUX must be enforcing, permissive or disabled" + [[ "${SELINUX,,}" =~ ^((enforcing)|(permissive)|(disabled))$ ]] || common::error "SELINUX must be enforcing, permissive or disabled" readonly SELINUX - [[ "${X2APIC,,}" =~ ^(on)|(off)$ ]] || error "X2APIC must be on or off" - readonly X2APIC="${X2APIC,,}" - - [[ "${SERIAL_CONSOLE,,}" =~ ^(yes)|(no)$ ]] || error "SERIAL_CONSOLE must be yes or no" + [[ "${SERIAL_CONSOLE,,}" =~ ^((yes)|(no))$ ]] || common::error "SERIAL_CONSOLE must be yes or no" readonly SERIAL_CONSOLE + [[ "${BOOT_MODE,,}" =~ ^((bios)|(uefi))$ ]] || common::error "BOOT_MODE must be bios or uefi" + readonly BOOT_MODE + + if [[ -z ${OS_VARIANT} ]]; then + OS_VARIANT=$(osinfo-query os --fields=short-id vendor="Oracle America" | + grep "ol${ORACLE_RELEASE}\." | + tail -1 | + sed -e 's/ //g') + fi + [[ -z ${OS_VARIANT} ]] && common::error "can't determine OS_VARIANT; you must define it in your environment file" + osinfo-query os --fields=short-id short-id="${OS_VARIANT}" >/dev/null || common::error "Invalid OS_VARIANT" + readonly OS_VARIANT + + [[ -z ${CACHE_DIR} ]] && common::error "CACHE_DIR must be specified" + ( cd "${WORKSPACE}" && [[ -d $(dirname "${CACHE_DIR}") ]] ) || + common::error "parent directory of CACHE_DIR must exists" + CACHE_PATH="$(cd "${WORKSPACE}" && cd "$(dirname "${CACHE_DIR}")" && pwd -P)/$(basename "${CACHE_DIR}")" + readonly CACHE_PATH + # Source image scripts if [[ -r "${DISTR_DIR}/${DISTR}/${IMAGE_SCRIPTS}" ]]; then source "${DISTR_DIR}/${DISTR}/${IMAGE_SCRIPTS}" @@ -274,9 +239,10 @@ load_env() { } ####################################### -# Stage required files for packer +# Stage required files for provisioning # Globals: -# DISTR CLOUD WORKSPACE PACKER_FILES +# CLOUD, CLOUD_DIR, CUSTOM, CUSTOM_DIR, DISTR, DISTR_DIR, ENV_FILES, FILES_DIR +# GLOBAL_ENV_FILE, PROVISION_DIR, PROVISION_SCRIPT, VM_NAME, WORKSPACE # Loaded environment files... # Arguments: # None @@ -284,355 +250,274 @@ load_env() { # None ####################################### stage_files() { - echo_header "Stage Packer files" + common::echo_header "Stage provisioning files" + + local provision_path="${WORKSPACE}/${VM_NAME}/${PROVISION_DIR}" + + mkdir "${provision_path}" + + # Global environment file + readonly GLOBAL_ENV_FILE="${provision_path}/${ENV_FILE}" + for env_file in "${ENV_FILES[@]}"; do + [[ -r "${env_file}" ]] && cat "${env_file}" >> "${GLOBAL_ENV_FILE}" + done + + # Main provisionning script + cp "${BIN_DIR}/provision.sh" "${provision_path}/" + cp "${BIN_DIR}/provision-common.sh" "${provision_path}/" # Cloud files into cloud subdir - mkdir -p "${WORKSPACE}/${PACKER_FILES}/cloud/distr" + mkdir -p "${provision_path}/cloud/distr" if [[ -d "${CLOUD_DIR}/${CLOUD}/${FILES_DIR}" ]]; then - cp -RL "${CLOUD_DIR}/${CLOUD}/${FILES_DIR}/." "${WORKSPACE}/${PACKER_FILES}/cloud" + cp -RL "${CLOUD_DIR}/${CLOUD}/${FILES_DIR}/." "${provision_path}/cloud" fi if [[ -d "${CLOUD_DIR}/${CLOUD}/${DISTR}/${FILES_DIR}" ]]; then - cp -RL "${CLOUD_DIR}/${CLOUD}/${DISTR}/${FILES_DIR}/." "${WORKSPACE}/${PACKER_FILES}/cloud/distr" + cp -RL "${CLOUD_DIR}/${CLOUD}/${DISTR}/${FILES_DIR}/." "${provision_path}/cloud/distr" fi # Distr files into distr subdir - mkdir "${WORKSPACE}/${PACKER_FILES}/distr" + mkdir "${provision_path}/distr" if [[ -d "${DISTR_DIR}/${DISTR}/${FILES_DIR}" ]]; then - cp -RL "${DISTR_DIR}/${DISTR}/${FILES_DIR}/." "${WORKSPACE}/${PACKER_FILES}/distr" + cp -RL "${DISTR_DIR}/${DISTR}/${FILES_DIR}/." "${provision_path}/distr" fi # Custom files into custom subdir - mkdir "${WORKSPACE}/${PACKER_FILES}/custom" + mkdir "${provision_path}/custom" if [[ -d "${CUSTOM_DIR}/${CUSTOM}/${FILES_DIR}" ]]; then - cp -RL "${CUSTOM_DIR}/${CUSTOM}/${FILES_DIR}/." "${WORKSPACE}/${PACKER_FILES}/custom" + cp -RL "${CUSTOM_DIR}/${CUSTOM}/${FILES_DIR}/." "${provision_path}/custom" fi # Provisioners if [[ -r "${CLOUD_DIR}/${CLOUD}/${PROVISION_SCRIPT}" ]]; then - cp "${CLOUD_DIR}/${CLOUD}/${PROVISION_SCRIPT}" "${WORKSPACE}/${PACKER_FILES}/cloud/" + cp "${CLOUD_DIR}/${CLOUD}/${PROVISION_SCRIPT}" "${provision_path}/cloud/" fi if [[ -r "${CLOUD_DIR}/${CLOUD}/${DISTR}/${PROVISION_SCRIPT}" ]]; then - cp "${CLOUD_DIR}/${CLOUD}/${DISTR}/${PROVISION_SCRIPT}" "${WORKSPACE}/${PACKER_FILES}/cloud/distr/" + cp "${CLOUD_DIR}/${CLOUD}/${DISTR}/${PROVISION_SCRIPT}" "${provision_path}/cloud/distr/" fi if [[ -r "${DISTR_DIR}/${DISTR}/${PROVISION_SCRIPT}" ]]; then - cp "${DISTR_DIR}/${DISTR}/${PROVISION_SCRIPT}" "${WORKSPACE}/${PACKER_FILES}/distr/" + cp "${DISTR_DIR}/${DISTR}/${PROVISION_SCRIPT}" "${provision_path}/distr/" fi if [[ -r "${CUSTOM_DIR}/${CUSTOM}/${PROVISION_SCRIPT}" ]]; then - cp "${CUSTOM_DIR}/${CUSTOM}/${PROVISION_SCRIPT}" "${WORKSPACE}/${PACKER_FILES}/custom" + cp "${CUSTOM_DIR}/${CUSTOM}/${PROVISION_SCRIPT}" "${provision_path}/custom" fi } ####################################### # Stage kickstart file # Globals: -# BIN_DIR -# DISTR_DIR DISTR -# KS_FILE -# SETUP_SWAP -# SSH_PASSWORD SSH_PUB_KEY -# WORKSPACE +# DISTR, DISTR_DIR, KS_FILE, REPO, REPO_URL, SETUP_SWAP, VM_NAME, WORKSPACE # Arguments: # None # Returns: # None ####################################### stage_kickstart() { - echo_header "Stage kickstart file" + common::echo_header "Stage kickstart file" - local hash + local ks_path="${WORKSPACE}/${VM_NAME}/${KS_FILE}" - cp "${DISTR_DIR}/${DISTR}/"*-ks.cfg "${WORKSPACE}/${KS_FILE}" - - if [[ -z "${SSH_PASSWORD}" ]]; then - sed -i -e 's/^rootpw .*$/rootpw --lock/' "${WORKSPACE}/${KS_FILE}" - else - # hash=$("${BIN_DIR}/mkpasswd-osx.py" "${SSH_PASSWORD}") - hash=$("${BIN_DIR}/mkpasswd.py" "${SSH_PASSWORD}") - # '/' is a valid character in a SHA512 hash salt! - sed -i -e 's!^rootpw .*$!rootpw --iscrypted '"${hash}"'!' \ - "${WORKSPACE}/${KS_FILE}" - fi - - if [[ -n "${SSH_PUB_KEY}" ]]; then - sed -i -e \ - $'s!^rootpw .*$!&\\\nsshkey --username root "'"${SSH_PUB_KEY}"'"!' \ - "${WORKSPACE}/${KS_FILE}" - fi + cp "${DISTR_DIR}/${DISTR}/"*-ks.cfg "${ks_path}" if [[ "${SETUP_SWAP,,}" = "no" ]]; then - sed -i -e '/^part swap /d' "${WORKSPACE}/${KS_FILE}" + sed -i -e '/^part swap /d' "${ks_path}" fi if [[ -n "${REPO_URL}" ]]; then sed -i -e \ '/^# URL to an installation tree/a url --url "'"${REPO_URL}"'"' \ - "${WORKSPACE}/${KS_FILE}" + "${ks_path}" fi local ks_repo for ks_repo in "${!REPO[@]}"; do sed -i -e \ '/^# Additional yum repositories/a repo --name "'"${ks_repo}"'" --baseurl "'"${REPO[${ks_repo}]}"'"' \ - "${WORKSPACE}/${KS_FILE}" + "${ks_path}" done # Kickstart fixups at distr / cloud_distr level if [[ "$(type -t distr::kickstart)" = 'function' ]]; then - distr::kickstart "${WORKSPACE}/${KS_FILE}" + distr::kickstart "${ks_path}" fi if [[ "$(type -t cloud_distr::kickstart)" = 'function' ]]; then - cloud_distr::kickstart "${WORKSPACE}/${KS_FILE}" + cloud_distr::kickstart "${ks_path}" fi if [[ "$(type -t custom::kickstart)" = 'function' ]]; then - custom::kickstart "${WORKSPACE}/${KS_FILE}" + custom::kickstart "${ks_path}" fi } ####################################### -# Generate Packer config file +# Create Oracle Linux image +# The outcome is a qcow2 file ${WORKSPACE}/${VM_NAME}/${VM_NAME}.qcow2 with +# OL installed based on the generated kickstart file. # Globals: -# DISK_SIZE_MB MEM_SIZE CPU_NUM -# ISO_URL ISO_CHECKSUM -# KS_FILE -# QEMU_BINARY -# SERIAL_CONSOLE -# SHUTDOWN_CMD -# SSH_PASSWORD SSH_KEY_FILE -# VM_NAME -# WORKSPACE PACKER_FILES BIN_DIR PROVISION_SCRIPT -# X2APIC +# BOOT_COMMAND, BOOT_COMMAND_SERIAL_CONSOLE, BOOT_LOCATION, BOOT_MODE +# CPU_NUM, DISK_SIZE_MB, ISO_CHECKSUM, ISO_PATH, KS_FILE +# MEM_SIZE, SERIAL_CONSOLE, VM_NAME, WORKSPACE # Arguments: # None # Returns: # None ####################################### -packer_conf() { - echo_header "Generate Packer configuration file" - - local q='"' - # KS_CONFIG is expanded in BOOT_COMMAND - # shellcheck disable=SC2034 - local KS_CONFIG="http://{{ .HTTPIP }}:{{ .HTTPPort }}/${KS_FILE}" - # shellcheck disable=SC2034 - local CONSOLE="" - local vbox_manage="" - local qemu_serial_console="" +image_create() { + common::echo_header "Install Oracle Linux" + + # retrieve disk label -- alternatively: isoinfo -d -i + local ISO_LABEL + # shellcheck disable=SC2034,SC2153 + ISO_LABEL=$(file "${ISO_PATH}" | sed -e "s/.* '\(.*\)' .*/\1/" -e 's/ /\\x20/g') + + declare -ga virt_install_args + # Set Serial conole if [[ "${SERIAL_CONSOLE,,}" = "yes" ]]; then - # shellcheck disable=SC2034 - CONSOLE=" console=tty0 console=ttyS0" - vbox_manage='["modifyvm", "{{.Name}}", "--uart1", "0x3f8", 4, "--uartmode1", "file", "'"${WORKSPACE}/${VM_NAME}"'/serial-console.txt"],' - qemu_serial_console='[ "-serial", "file:'"${WORKSPACE}/${VM_NAME}"'/serial-console.txt" ]' - fi - # VirtualBox 7 requires flag to allow guest to reach host - if common::is_vbox ; then - if [[ $(vboxmanage --version) =~ ^7\. ]]; then - vbox_manage+='["modifyvm", "{{.Name}}", "--nat-localhostreachable1", "on"],' - fi - fi - - cat > "${WORKSPACE}/${VM_NAME}.pkrvars.hcl" <<-EOF - # Variables file for ${VM_NAME} - workspace = "${WORKSPACE}" - iso_url = "${ISO_URL}" - iso_checksum = "${ISO_CHECKSUM}" - vm_name = "${VM_NAME}" - disk_size = ${DISK_SIZE_MB} - memory = ${MEM_SIZE} - cpus = ${CPU_NUM} - ${SSH_PASSWORD:+ssh_password = ${q}$SSH_PASSWORD${q}} - ${SSH_KEY_FILE:+ssh_private_key_file = ${q}$SSH_KEY_FILE${q}} - boot_command = [ - $(eval echo -e "\"$(printf ' \\"%s\\",\\n' "${BOOT_COMMAND[@]}")\"") - ] - shutdown_command = "${SHUTDOWN_CMD}" - vbox_manage = [ ${vbox_manage} ] - x2apic = "${X2APIC}" - ${QEMU_BINARY:+qemu_binary = ${q}$QEMU_BINARY${q}} - qemu_args = [ ${qemu_serial_console} ] - packer_files = "${WORKSPACE}/${PACKER_FILES}" - provision_script = "${BIN_DIR}/${PROVISION_SCRIPT}" - EOF - - # Packer config specific to distr / cloud / cloud_distr level - if [[ "$(type -t distr::packer_conf)" = 'function' ]]; then - distr::packer_conf "${WORKSPACE}/${VM_NAME}.pkrvars.hcl" - fi - if [[ "$(type -t cloud::packer_conf)" = 'function' ]]; then - cloud::packer_conf "${WORKSPACE}/${VM_NAME}.pkrvars.hcl" - fi - if [[ "$(type -t cloud_distr::packer_conf)" = 'function' ]]; then - cloud_distr::packer_conf "${WORKSPACE}/${VM_NAME}.pkrvars.hcl" - fi - if [[ "$(type -t custom::packer_conf)" = 'function' ]]; then - custom::packer_conf "${WORKSPACE}/${VM_NAME}.pkrvars.hcl" + BOOT_COMMAND+=( "${BOOT_COMMAND_SERIAL_CONSOLE[@]}" ) + else + virt_install_args+=(--wait "${INSTALL_WAIT_TIME}" --noautoconsole) + fi + + if [[ ${BOOT_MODE,,} == uefi ]]; then + virt_install_args+=(--boot uefi) fi + + # Dereference symlink as virt-instal/qemu don't like these + local iso_path + iso_path=$(realpath -e "${ISO_PATH}") + + local location + if [[ -n ${BOOT_LOCATION} ]]; then + location=",kernel=${BOOT_LOCATION}/vmlinuz,initrd=${BOOT_LOCATION}/initrd.img" + fi + + # shellcheck disable=SC2294 + virt-install --os-type linux --os-variant "${OS_VARIANT}" --name "${VM_NAME}" \ + --cpus "${CPU_NUM}" --memory "${MEM_SIZE}" \ + --controller "scsi,model=virtio-scsi" \ + --disk "path=${WORKSPACE}/${VM_NAME}/${VM_NAME}.qcow2,size=${DISK_SIZE_GB},bus=scsi,cache=unsafe" \ + --network default \ + --graphics none \ + --location "${iso_path}${location}" \ + --initrd-inject="${WORKSPACE}/${VM_NAME}/${KS_FILE}" \ + --extra-args="$(eval echo "${BOOT_COMMAND[@]}")" \ + --transient \ + "${virt_install_args[@]}" } ####################################### -# Run packer +# Customize Oracle Linux: run provisionning scripts +# Uses libguestfs to update the ${WORKSPACE}/${VM_NAME}/${VM_NAME}.qcow2 # Globals: -# PACKER_BUILDER -# SERIAL_CONSOLE -# VM_NAME -# WORKSPACE +# BUILD_INFO, MEM_SIZE, PROVISION_DIR, PROVISION_SCRIPT, SELINUX, VM_NAME, WORKSPACE # Arguments: # None # Returns: # None ####################################### -run_packer() { - echo_header "Run Packer" - - cd "${WORKSPACE}" - local packer_status serial_pid - - if [[ ${SERIAL_CONSOLE,,} == "yes" ]]; then - # Print serial console output ([try to] suppress escape sequences) - echo_message "Monitoring serial console" - # shellcheck disable=SC2155 - local monitor="$(shopt -po monitor)" - set -m - ( - while [[ ! -f "${WORKSPACE}/${VM_NAME}/serial-console.txt" ]]; do - sleep 10 - done - sleep 10 - echo "$(tput setaf 4)--- serial console: Showing serial console output$(tput sgr 0)" - tail -f "${WORKSPACE}/${VM_NAME}/serial-console.txt" | - sed -e "s,\x1B\[[0-9;]*[a-zA-Z],,g" \ - -e "s/\x0D//g" \ - -e '/^$/d' \ - -e "s/^/$(tput setaf 4) serial console: /" \ - -e "s/\$/$(tput sgr 0)/" - )& - serial_pid=$! - eval "${monitor}" - fi - - # shellcheck disable=SC2155 - local errexit="$(shopt -po errexit)" - set +e - # shellcheck disable=SC2086 - "${PACKER}" build -only "${PACKER_BUILDER}" ${PACKER_BUILD_OPTIONS} \ - -var-file="${VM_NAME}.pkrvars.hcl" \ - "${TEMPLATE_DIR}" - packer_status=$? - eval "${errexit}" - - if [[ -n "${serial_pid}" ]]; then - echo_message "Stop monitoring serial console" - kill -- -${serial_pid} - fi - - [[ ${packer_status} -ne 0 ]] && error "Packer didn't complete successfully" - - [[ -r "${WORKSPACE}/${VM_NAME}/${VM_NAME}.ova" || -r "${WORKSPACE}/${VM_NAME}/System.img" ]] || - error "Packer didn't built the image" +image_provision() { + common::echo_header "Run provisioning scripts" + + local virt_customize_args=() + if [[ ${SELINUX,,} != disabled ]]; then + virt_customize_args+=(--selinux-relabel) + fi + + # Cloud / Custom specific parameters + if [[ "$(type -t cloud::customize_args)" = 'function' ]]; then + cloud::customize_args virt_customize_args + fi + if [[ "$(type -t cloud_distr::customize_args)" = 'function' ]]; then + cloud_distr::customize_args virt_customize_args + fi + if [[ "$(type -t custom::customize_args)" = 'function' ]]; then + custom::customize_args virt_customize_args + fi + + # `run` will run a /bin/sh, therefore we use `run-command` + virt-customize --copy-in "${WORKSPACE}/${VM_NAME}/${PROVISION_DIR}":/tmp/ \ + --run-command "/bin/bash /tmp/${PROVISION_DIR}/${PROVISION_SCRIPT}" \ + -a "${WORKSPACE}/${VM_NAME}/${VM_NAME}.qcow2" \ + --memsize "${MEM_SIZE}" \ + "${virt_customize_args[@]}" + + virt-copy-out /tmp/builder.log "${WORKSPACE}/${VM_NAME}/" \ + -a "${WORKSPACE}/${VM_NAME}/${VM_NAME}.qcow2" + + virt-copy-out "${BUILD_INFO}" "${WORKSPACE}/${VM_NAME}/" \ + -a "${WORKSPACE}/${VM_NAME}/${VM_NAME}.qcow2" + + local build_info_dir + build_info_dir=$(basename "${BUILD_INFO}") + mv "${WORKSPACE}/${VM_NAME}/${build_info_dir}"/* "${WORKSPACE}/${VM_NAME}/" + rmdir "${WORKSPACE}/${VM_NAME}/${build_info_dir}" } ####################################### -# Cleanup actions run directly on the image -# We prune work images as soon as possible to limit disk space usage +# Cleanup the image +# Run sysprep / sparsify the ${WORKSPACE}/${VM_NAME}/${VM_NAME}.qcow2 image # Globals: -# PACKER_BUILDER -# WORKSPACE VM_NAME -# MOUNT_IMAGE -# Loaded environment files used in invoked modules +# BUILD_INFO, ROOT_PASSWORD, ROOT_SSH_KEY, SELINUX, VM_NAME, WORKSPACE # Arguments: # None # Returns: # None ####################################### image_cleanup() { - echo_header "Cleanup image" - - local boot_fs root_fs - local mnt="${WORKSPACE}/${VM_NAME}/mnt" - - echo_message "Extract image and convert to raw format" - cd "${WORKSPACE}/${VM_NAME}" - if common::is_vbox ; then - tar -xf "${VM_NAME}.ova" - mv "${VM_NAME}.ova" System.ova - mv -f "${VM_NAME}"-disk*.vmdk System.vmdk - vbox-img convert \ - --srcfilename System.vmdk \ - --dstfilename System.img \ - --srcformat VMDK \ - --dstformat RAW - rm -f System.vmdk - fi - - # Run cleanup scripts - if [[ "$(type -t custom::image_cleanup)" = 'function' || - "$(type -t cloud_distr::image_cleanup)" = 'function' || - "$(type -t cloud::image_cleanup)" = 'function' || - "$(type -t distr::image_cleanup)" = 'function' ]]; then - # Only mount the image if we have an image_cleanup function defined - echo_message "Loopback mount image" - # Loopback mount the image - # We will have the following subdirectories: - # - 1: /boot - # - 2: root filesystem (/) - # In case of a btrfs filesystem, / will be in root subvolume - # Should /boot be part of the btrfs volume we then have: - # - 1: btrfs volume with boot and root subvolumes - rm -rf "${mnt}" - mkdir "${mnt}" - sudo "${MOUNT_IMAGE}" System.img "${mnt}" - if [[ $(stat -f -c "%T" "${mnt}/1") = "btrfs" ]]; then - # Both / and /boot are on BTRFS - boot_fs="${mnt}/1/boot" - root_fs="${mnt}/1/root" - else - boot_fs="${mnt}/1" - if [[ $(stat -f -c "%T" "${mnt}/2") = "btrfs" ]]; then - root_fs="${mnt}/2/root" + common::echo_header "Cleanup" + + local virt_sysprep_args=() + if [[ ${SELINUX,,} != disabled ]]; then + virt_sysprep_args+=(--selinux-relabel) + fi + + # Root access + virt_sysprep_args+=(--root-password "${ROOT_PASSWORD}" ) + if [[ -n ${ROOT_SSH_KEY} ]]; then + virt_sysprep_args+=(--ssh-inject "root:${ROOT_SSH_KEY}" ) + fi + + # Cloud / Custom specific parameters + if [[ "$(type -t cloud::sysprep_args)" = 'function' ]]; then + cloud::sysprep_args virt_sysprep_args + fi + if [[ "$(type -t cloud_distr::sysprep_args)" = 'function' ]]; then + cloud_distr::sysprep_args virt_sysprep_args + fi + if [[ "$(type -t custom::sysprep_args)" = 'function' ]]; then + custom::sysprep_args virt_sysprep_args + fi + + virt-sysprep --delete "${BUILD_INFO}" \ + --truncate /etc/machine-id \ + -a "${WORKSPACE}/${VM_NAME}/${VM_NAME}.qcow2" \ + "${virt_sysprep_args[@]}" + + if [[ ${SELINUX,,} != disabled ]]; then + common::echo_message "SELinux relabel non-root filesystems" + eval "$(guestfish -a "${WORKSPACE}/${VM_NAME}/${VM_NAME}.qcow2" -i --selinux --listen)" + local -a mounts + mapfile -t mounts < <(guestfish --remote mountpoints | awk '{print $2}') + local mount + for mount in "${mounts[@]}"; do + if [[ ${mount} =~ ^(/|(/boot/efi))$ ]]; then + common::echo_message " skipping ${mount}" else - root_fs="${mnt}/2" + common::echo_message " relabelling ${mount}" + guestfish --remote \ + selinux-relabel /etc/selinux/targeted/contexts/files/file_contexts \ + "${mount}" fi - fi - - # Basic check to see if we have the "right" partitions mounted - if [[ ! -d "${root_fs}/etc" || ! -d "${boot_fs}/grub2" ]]; then - sudo "${MOUNT_IMAGE}" -u System.img - rm -rf "${mnt}" - error "Loopback mount failed" - fi - - # Run cleanup scripts - if [[ "$(type -t custom::image_cleanup)" = 'function' ]]; then - echo_message "Run custom cleanup" - custom::image_cleanup "${root_fs}" "${boot_fs}" - fi - if [[ "$(type -t cloud_distr::image_cleanup)" = 'function' ]]; then - echo_message "Run cloud distribution cleanup" - cloud_distr::image_cleanup "${root_fs}" "${boot_fs}" - fi - if [[ "$(type -t cloud::image_cleanup)" = 'function' ]]; then - echo_message "Run cloud cleanup" - cloud::image_cleanup "${root_fs}" "${boot_fs}" - fi - if [[ "$(type -t distr::image_cleanup)" = 'function' ]]; then - echo_message "Run distribution cleanup" - distr::image_cleanup "${root_fs}" "${boot_fs}" - fi - - # Ensure we are still in the image directory - cd "${WORKSPACE}/${VM_NAME}" - # unmount and trim image - echo_message "Unmount and trim image" - sudo -- bash -c ' - sync; sync; sync; - fstrim "'"${boot_fs}"'"; - fstrim "'"${root_fs}"'"; - ' - sudo "${MOUNT_IMAGE}" -u System.img - rm -rf "${mnt}" - - cp --sparse=always System.img System.img.sparse - mv -f System.img.sparse System.img - fi - - echo_message "Package image" + done + guestfish --remote quit + fi + + common::echo_message "Sparsify image" + local tmpdir + tmpdir=$(mktemp -d -p "${WORKSPACE}/${VM_NAME}") + virt-sparsify --compress --tmp "${tmpdir}" \ + "${WORKSPACE}/${VM_NAME}/${VM_NAME}.qcow2" \ + "${WORKSPACE}/${VM_NAME}/${VM_NAME}-sparse.qcow2" + rm -r "${tmpdir}" + mv "${WORKSPACE}/${VM_NAME}/${VM_NAME}-sparse.qcow2" "${WORKSPACE}/${VM_NAME}/${VM_NAME}.qcow2" + + common::echo_message "Package image" if [[ "$(type -t custom::image_package)" = 'function' ]]; then custom::image_package elif [[ "$(type -t cloud_distr::image_package)" = 'function' ]]; then @@ -640,51 +525,43 @@ image_cleanup() { elif [[ "$(type -t cloud::image_package)" = 'function' ]]; then cloud::image_package else - error "No packaging script found" - fi - - if common::is_vbox ; then - rm "${WORKSPACE}/${VM_NAME}/${VM_NAME}.ovf" - rm "${WORKSPACE}/${VM_NAME}/System.ova" + common::error "No packaging script found" fi } ####################################### -# Cleanup workspace -- we do not remove the packer cache! +# Cleanup workspace -- we do not remove the ISO cache! # Globals: -# WORKSPACE VM_NAME -# GLOBAL_ENV_FILE -# KS_FILE -# PACKER_FILES +# GLOBAL_ENV_FILE, PROVISION_DIR, VM_NAME, WORKSPACE # Arguments: # None # Returns: # None ####################################### -cleanup() { - echo_header "Cleanup Workspace" +workspace_cleanup() { + common::echo_header "Cleanup Workspace" - mv "${WORKSPACE}/${KS_FILE}" "${WORKSPACE}/${VM_NAME}" - mv "${WORKSPACE}/${VM_NAME}.pkrvars.hcl" "${WORKSPACE}/${VM_NAME}" mv "${GLOBAL_ENV_FILE}" "${WORKSPACE}/${VM_NAME}" - rm -rf "${WORKSPACE:?}/${PACKER_FILES}" + rm -rf "${WORKSPACE:?}/${VM_NAME}/${PROVISION_DIR}" } ####################################### # Main ####################################### main () { - parse_args "$@" source "${BIN_DIR}/common.sh" + parse_args "$@" load_env + common::retrieve_iso "${ISO_URL}" "${ISO_CHECKSUM}" ISO_PATH + mkdir "${WORKSPACE}/${VM_NAME}" stage_files stage_kickstart - packer_conf - run_packer + image_create + image_provision image_cleanup - cleanup - echo_header "All done" - echo_header "Image available in ${WORKSPACE}/${VM_NAME}" + workspace_cleanup + common::echo_header "All done" + common::echo_header "Image available in ${WORKSPACE}/${VM_NAME}" } main "$@" diff --git a/oracle-linux-image-tools/bin/common.sh b/oracle-linux-image-tools/bin/common.sh index ab889a3..25e5348 100755 --- a/oracle-linux-image-tools/bin/common.sh +++ b/oracle-linux-image-tools/bin/common.sh @@ -1,32 +1,52 @@ #!/usr/bin/env bash - -# shellcheck disable=SC1090 # -# Create minimal Oracle Linux images +# Common function for the image builder script # -# Copyright (c) 2022 Oracle and/or its affiliates. +# Copyright (c) 2022, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl. # -# Description: creates minimal Oracle Linux images which can be used in cloud -# environment. -# # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. # ####################################### -# Return 1 (true) if packaged using VirtualBox +# Print header +# Globals: +# PGM +# Arguments: +# None +# Returns: +# None +####################################### +common::echo_header() { + echo "$(tput setaf 2)+++ ${PGM}: $*$(tput sgr 0)" +} + +####################################### +# Print message # Globals: -# PACKER_BUILDER +# PGM # Arguments: # None # Returns: -# 1 -- packaged via VirtualBox -# 0 -- packaged via QEMU +# None ####################################### -common::is_vbox() { - test "${PACKER_BUILDER}" = "virtualbox-iso.x86-64" - return +common::echo_message() { + echo " ${PGM}: $*" +} + +####################################### +# Print error message and exit +# Globals: +# PGM +# Arguments: +# message +# Returns: +# None +####################################### +common::error() { + echo "$(tput setaf 1)+++ ${PGM}: $*$(tput sgr 0)" >&2 + exit 1 } ####################################### @@ -61,7 +81,7 @@ common::make_ova() { # Globals: # None # Arguments: -# 1: output file name (including .qcow extension) +# 1: output file name (including .qcow2 extension) # -: implicit use of `System.img` # Returns: # - $output file generated @@ -74,43 +94,125 @@ common::convert_to_qcow2() { } ####################################### -# Convert disk image to VMDK format +# Convert disk image (QCOW2) to VMDK format # Globals: -# None +# VM_NAME, WORKSPACE # Arguments: # 1: output file name (including .vmdk extension) -# -: implicit use of `System.img` +# -: implicit use of `${WORKSPACE}/${VM_NAME}/${VM_NAME}.qcow2` # Returns: # - $output file generated -# - System.img file removed +# - ${VM_NAME}.qcow2 file removed ####################################### common::convert_to_vmdk() { local output=${1:?- ***error*** \'output\' not set} - if common::is_vbox ; then - vboxmanage convertfromraw System.img --format VMDK "${output}" --variant Stream - else - qemu-img convert -f raw -O vmdk -o subformat=streamOptimized System.img "${output}" - fi - rm System.img + local input="${WORKSPACE}/${VM_NAME}/${VM_NAME}.qcow2" + qemu-img convert -f qcow2 -O vmdk -o subformat=streamOptimized "${input}" "${output}" + rm "${input}" } + ####################################### -# Convert disk image to VHD format +# Fix VMDK header +# In VirtualBox, grub2 is extremely slow if BIOS geometries are not defined +# in the VMDK header. +# We also add UUIDs for the sake of completeness. # Globals: # None # Arguments: +# 1: vmdk file to patch +# 2: file uuid +# Returns: +# vmdk file patched in-place +####################################### +common::fix_vmdk_header() { + local vmdk=${1:?- ***error*** \'vmdk\' not set} + local uuid=${2:?- ***error*** \'uuid\' not set} + local cylinders heads sectors + local disk_descriptor_file patch_file + + disk_descriptor_file=$(mktemp -t ddb-XXXXXXXX.txt) + patch_file=$(mktemp -t patch-XXXXXXXX.txt) + + dd if="${vmdk}" of="${disk_descriptor_file}" bs=1 skip=512 count=1024 + + cylinders=$(grep -a ddb.geometry.cylinders "${disk_descriptor_file}" | sed -e 's/.* = //') + heads=$(grep -a ddb.geometry.heads "${disk_descriptor_file}" | sed -e 's/.* = //') + sectors=$(grep -a ddb.geometry.sectors "${disk_descriptor_file}" | sed -e 's/.* = //') + cat >"${patch_file}" <<-EOF + ddb.geometry.biosCylinders=${cylinders} + ddb.geometry.biosHeads=${heads} + ddb.geometry.biosSectors=${sectors} + ddb.uuid.image="${uuid}" + ddb.uuid.parent="00000000-0000-0000-0000-000000000000" + ddb.uuid.modification="00000000-0000-0000-0000-000000000000" + ddb.uuid.parentmodification="00000000-0000-0000-0000-000000000000" + EOF + sed -i -e "/ddb.geometry.sectors/r ${patch_file}" "${disk_descriptor_file}" + + dd conv=notrunc,nocreat if="${disk_descriptor_file}" of="${vmdk}" bs=1 seek=512 count=1024 + + rm "${disk_descriptor_file}" "${patch_file}" +} +####################################### +# Convert disk image to VHD format +# Globals: +# VM_NAME, WORKSPACE +# Arguments: # 1: output file name (including .vhd extension) -# -: implicit use of `System.img` +# -: implicit use of `${WORKSPACE}/${VM_NAME}/${VM_NAME}.qcow2` # Returns: # - $output file generated -# - System.img file removed +# - ${VM_NAME}.qcow2 file removed ####################################### common::convert_to_vhd() { local output=${1:?- ***error*** \'output\' not set} - if common::is_vbox ; then - vboxmanage convertfromraw System.img --format VHD "${output}" + local input="${WORKSPACE}/${VM_NAME}/${VM_NAME}.qcow2" + qemu-img convert -f qcow2 -O vpc -o subformat=dynamic "${input}" "${output}" + rm "${input}" +} + +####################################### +# Retrieve installation media +# Exit if the file cannot be retrieved +# Globals: +# CACHE_PATH +# Arguments: +# ISO URL +# ISO Checksum (SHA1 or SHA256) +# Name of a variable which will contain the path to the downloaded file +# Returns: +# Path to downloaded file +####################################### +common::retrieve_iso() { + local iso_url="$1" + local iso_checksum="$2" + local -n iso_path="$3" + common::echo_header "Retrieve installation media $(basename "${iso_url}")" + + mkdir -p "${CACHE_PATH}" || common::error "can't create ${CACHE_PATH}" + + # Build a cache file name with a hash from the URL to avoid conflict with + # different images having the same name (typically boot ISOs) + iso_path="${CACHE_PATH}/$(echo -n "${iso_url}" | sha1sum | cut -d ' ' -f 1)-$(basename "${iso_url}")" + + local checksum + if [[ -f ${iso_path} || -L ${iso_path} ]]; then + common::echo_message "using cached file ${iso_path}" + elif [[ ${iso_url%%:*} == "file" ]]; then + common::echo_message "using local file ${iso_url#file:}" + [[ -f ${iso_url#file:} ]] || common::error "file does not exists" + ln -s "${iso_url#file:}" "${iso_path}" else - qemu-img convert -f raw -O vpc -o subformat=dynamic System.img "${output}" + common::echo_message "downloading ${iso_url}" + curl -L -s -o "${iso_path}" "${iso_url}" || common::error "can't retrieve ${iso_url}" fi - rm System.img + local iso_checksum_type="sha1sum" + [[ ${#iso_checksum} -eq 64 ]] && iso_checksum_type="sha256sum" + checksum=$(${iso_checksum_type} "${iso_path}" | cut -d ' ' -f 1) + [[ ${checksum} != "${iso_checksum}" ]] && + common::echo_message "checksum mismatch. Expected ${iso_checksum}, got ${checksum}" && + common::echo_message "after fixing the issue, you may have to remove the cached file ${iso_path}" && + common::error "terminating" + true } diff --git a/oracle-linux-image-tools/bin/mkpasswd.py b/oracle-linux-image-tools/bin/mkpasswd.py deleted file mode 100755 index 7a81a41..0000000 --- a/oracle-linux-image-tools/bin/mkpasswd.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 - -""" -Simple wrapper to generate SHA512 hash for a password. - -Copyright (c) 2019-2022 Oracle and/or its affiliates. -Licensed under the Universal Permissive License v 1.0 as shown at -https://oss.oracle.com/licenses/upl. - -Description: Python < 3.3 does not have crypt.mksalt() we mimmic -https://github.com/python/cpython/blob/master/Lib/crypt.py - -DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. -""" - -from crypt import crypt -from random import SystemRandom -from string import ascii_letters, digits -from sys import argv, exit - - -def mksalt(): - """Generate SHA512 salt.""" - sr = SystemRandom() - return '$6$' + ''.join(sr.choice(ascii_letters + digits + './') - for char in range(16)) - - -if len(argv) != 2: - print("Usage: " + argv[0] + " password") - exit(1) - -print(crypt(argv[1], mksalt())) diff --git a/oracle-linux-image-tools/bin/mnt-img.sh b/oracle-linux-image-tools/bin/mnt-img.sh deleted file mode 100755 index 79f62be..0000000 --- a/oracle-linux-image-tools/bin/mnt-img.sh +++ /dev/null @@ -1,270 +0,0 @@ -#!/usr/bin/env bash -# -# Mount VM images -# -# Copyright (c) 2019,2020 Oracle and/or its affiliates. -# Licensed under the Universal Permissive License v 1.0 as shown at -# https://oss.oracle.com/licenses/upl. -# -# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. -# -################################################################################ -# The utility is a wrapper script that can map the partitions from vm image, -# and mount them to local directories. -# Usage: mnt-img.sh -# Options: -u umount image file. -# -l display all mounted images -# -c umount all images -# -v show version info. -# -d display tracing msgs. -# -h show this usage info. -################################################################################ - -case "$ORACLE_TRACE" in - T) set -x;; - *) ;; -esac - -Usage() { - cat < -Options: -u umount image file. - -l display all mounted images - -c umount all images - -v show version info. - -d display tracing msgs. - -h show this usage info. -EOF -} - -Error() { - echo "ERROR: $1">&2 - echo - [ "x$2" = "xu" ] && Usage - exit 1 -} - -umount_img() { - f=$1 - lp_device=$(grep "MOUNTED IMAGE" "$f"| awk -F: '{print $2}'| awk '{print $2}') - image_file=$(grep "MOUNTED IMAGE" "$f"| awk -F: '{print $2}'| awk '{print $1}') - mounteddirs=$(grep "MOUNTED DIR" "$f" | awk -F: '{print $2}' | awk '{print $2}') - lvmgroups=$(grep "LVM GROUP" "$f" | awk -F: '{print $2}') - if [ -f "$image_file" ]; then - # umount first - for dir_name in $mounteddirs; do - if [ -d "$dir_name" ] && grep -q "$dir_name" /proc/mounts; then - if ! umount "$dir_name"; then - Error "Unable to umount $dir_name." - fi - fi - # umount done, remove entry from database - ftmp=$(mktemp /tmp/mntimg.XXXXX) - grep -v "$dir_name" "$f" > "$ftmp" - /bin/mv -f "$ftmp" "$f" - done - for vg_name in $lvmgroups; do - vgchange -an "$vg_name" >/dev/null - done - # unmount any automounted paths - for p in $(kpartx -l "$lp_device" |awk '{print $1}') - do - umount "/dev/mapper/$p" 2>/dev/null - done - kpartx -d "$lp_device" - losetup -d "$lp_device" - fi - rm -f "$f" -} - -list_mnt() { - f=$1 - lp_device=$(grep "MOUNTED IMAGE" "$f"| awk -F: '{print $2}'| awk '{print $2}') - image_file=$(grep "MOUNTED IMAGE" "$f"| awk -F: '{print $2}'| awk '{print $1}') - if [ -f "$image_file" ]; then - echo "--------------------------------------------------------------------" - grep "MOUNTED IMAGE" "$f" - # umount first - grep "MOUNTED DIR" "$f"| while read -r line; do - echo " |- $line" - done - grep "LVM GROUP" "$f" - fi -} - - -##################################################################### - -# define and env variables - -VERSION=1.0.0b2 - -#default ACTION=mount -ACTION=mount - -WORK_HOME=~/.mntimg -ID=0 - -# main ############################################################### -# handle arguments -while getopts ucldvh OPTION; do - case "$OPTION" in - u) - ACTION=umount - ;; - c) - ACTION=umountall - ;; - l) - ACTION=listall - ;; - d) - export ORACLE_TRACE=T - set -v -x - ;; - v) - echo $VERSION && exit 0 - ;; - h) - Usage && exit 0 - ;; - *) - Error "Wrong argument" u - ;; - esac -done -shift $((OPTIND - 1)) -IMAGE_FILE="$1" -shift -MOUNT_POINT="$1" - -# check arguments -if [[ $ACTION = mount || $ACTION = umount ]]; then - if [ ! -f "$IMAGE_FILE" ]; then - Error "Image file $IMAGE_FILE does NOT exist" u - fi - - if ! file "$IMAGE_FILE" | grep -q 'partition\|DOS/MBR'; then - Error "Image file does NOT seem to have partitions." - fi -fi - -if [ $ACTION = mount ]; then - if [ ! -d "$MOUNT_POINT" ]; then - Error "Mount point directory $MOUNT_POINT does NOT exist" u - fi -fi - - -# create working dir -mkdir -p $WORK_HOME - -case "$ACTION" in -########################################################################## -# action mount -mount) - -IMAGE_FILE=$(readlink -f "$IMAGE_FILE") -MOUNT_POINT=$(readlink -f "$MOUNT_POINT") -ID=$(echo "$IMAGE_FILE"| md5sum | awk '{print $1}') -if [ -f "$WORK_HOME/$ID" ]; then - Error "$IMAGE_FILE has already been mounted" -fi -touch $WORK_HOME/"$ID" -#echo "Mounted Image:$IMAGE_FILE>$WORK_HOME/$ID" - -# setup loop -LPDEVICE=$(losetup -f) -if ! losetup -f "$IMAGE_FILE"; then - Error "setup loop device failed." -fi -# lock -echo "MOUNTED IMAGE:$IMAGE_FILE $LPDEVICE" > $WORK_HOME/"$ID" - -# Get host volume groups -host_vgs=$(vgs --noheadings -o vg_name) -guest_vg=0 - -# map device -if ! kpartx -a "$LPDEVICE" &>/dev/null; then - losetup -d "$LPDEVICE" - Error "kpartx map $LPDEVICE fail." -fi - -mapped_devices=$(kpartx -l "$LPDEVICE" | grep "$LPDEVICE" | awk '{print $1}') -ct=1 -for d in $mapped_devices; do - # mount ext3, ext4, btrfs, xfs partitions - if file -sL /dev/mapper/"$d" | grep -E -q -i "ext3|ext4|btrfs|xfs"; then - mkdir -p "$MOUNT_POINT"/$ct - mount /dev/mapper/"$d" "$MOUNT_POINT"/$ct - echo "MOUNTED DIR:/dev/mapper/$d $MOUNT_POINT/$ct" |tee -a $WORK_HOME/"$ID" - ct=$((ct+1)) - elif file -sL /dev/mapper/"$d" | grep -E -q -i "lvm2"; then - echo "LVM detected" - guest_vg=1 - pvscan --cache /dev/mapper/"$d" - else - echo "/dev/mapper/$d not mounted - Unknown filesystem:" - file -sL /dev/mapper/"$d" - fi -done - -if [[ ${guest_vg} == 1 ]]; then - # Image has LVM - # This only works for simple setups (where the whole VG is on the device) - # Scan for Volume Groups - all_vgs=$(vgscan | grep "Found volume group" | sed -e 's/^[^"]*"\([^"]*\)".*$/\1/') - for vg in ${all_vgs}; do - # Only consider new VGs - if ! grep -w -q "${vg}" <<<"${host_vgs}"; then - # Ensure the VG is active and register it - vgchange -ay "${vg}" - echo "LVM GROUP:${vg}" | tee -a "${WORK_HOME}/${ID}" - lvscan >/dev/null - # Search for filesystems in the VG - for fs in "/dev/${vg}"/*; do - if file -sL "${fs}" | grep -E -q -i "ext3|ext4|btrfs|xfs"; then - mkdir -p "${MOUNT_POINT}/${ct}" - mount "${fs}" "${MOUNT_POINT}/${ct}" - echo "MOUNTED DIR:${fs} ${MOUNT_POINT}/${ct}" | tee -a "${WORK_HOME}/${ID}" - ct=$((ct+1)) - else - echo "${fs} not mounted - Unknown filesystem:" - file -sL "${fs}" - fi - done - fi - done -fi - -;; - -############################################################################ -umountall) -for f in "$WORK_HOME"/*; do - umount_img "$f" -done -;; -############################################################################ -# action umount -umount) -IMAGE_FILE=$(readlink -f "$IMAGE_FILE") -ID=$(echo "$IMAGE_FILE"| md5sum | awk '{print $1}') -if [ ! -f "$WORK_HOME/$ID" ]; then - Error "$IMAGE_FILE has not been mounted" -fi -umount_img $WORK_HOME/"$ID" -;; -############################################################################ -# list all mounted images -listall) -for f in "$WORK_HOME"/*; do - list_mnt "$f" -done -;; -########################################################################## -*) - Error "unexpected error" u -;; -esac diff --git a/oracle-linux-image-tools/bin/provision-common.sh b/oracle-linux-image-tools/bin/provision-common.sh new file mode 100755 index 0000000..d19e06b --- /dev/null +++ b/oracle-linux-image-tools/bin/provision-common.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# +# Common functions for provisionning the VM +# +# Copyright (c) 2024 Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# https://oss.oracle.com/licenses/upl. +# + +####################################### +# Print header +####################################### +common::echo_header() { + echo "=== $* ===" +} + +####################################### +# Print message +####################################### +common::echo_message() { + echo "--- $* ---" +} + +####################################### +# Print error message and exit +####################################### +common::error() { + echo "--- $* ---" >&2 + exit 1 +} + +####################################### +# Print kickstart log +# Globals: +# None +# Arguments: +# None +# Returns: +# None +####################################### +common::ks_log() { + if [[ -f "/root/ks-post.log" ]]; then + common::echo_message "Kickstart post log - Start" + cat /root/ks-post.log + rm /root/ks-post.log + common::echo_message "Kickstart post log - End" + fi +} + +####################################### +# Return latest installed kernel +# Globals: +# None +# Arguments: +# "kernel" or "kernel-uek" +# Returns: +# full version of the latest installed kernel +####################################### +common::latest_kernel() { + local kernel=$1 + rpm -qa --qf "%{VERSION}-%{RELEASE}.%{ARCH}\n" "${kernel}" "${kernel}-core" | sort -V | tail -1 +} + +####################################### +# Return default kernel +# Return the kernel which will be used by default for this VM +# Globals: +# KERNEL +# Arguments: +# none +# Returns: +# full version of the default kernel +####################################### +common::default_kernel() { + local default_kernel + default_kernel=$(grubby --default-kernel) + echo "${default_kernel#*vmlinuz-}" +} + +####################################### +# Remove RHCK or UEK kernels +# Globals: +# None +# Arguments: +# "kernel" or "kernel-uek" +# kernel version to keep (optional) +# Returns: +# None +####################################### +common::remove_kernels() { + local kernel="$1" + local keep="$2" + local -a packages + + for flavor in "" "-core" "-modules" "-devel"; do + if [[ -z ${keep} ]]; then + mapfile -t packages < <(rpm -qa "${kernel}${flavor}") + else + mapfile -t packages < <(rpm -qa "${kernel}${flavor}" | grep -v "${keep}") + fi + if [[ ${#packages[@]} -gt 0 ]]; then + distr::remove_rpms "${packages[@]}" + fi + done +} + +####################################### +# Common distribution cleanup +# Globals: +# BUILD_INFO, EXCLUDE_DOCS, ORACLE_RELEASE +# Arguments: +# None +# Returns: +# None +####################################### +common::distr_cleanup() { + common::echo_message "Stoppping services" + systemctl stop rsyslog || true + systemctl stop auditd || true + + if [[ ${ORACLE_RELEASE} -lt 9 ]]; then + common::echo_message "Remove leftover firewall rules" + if [[ -f /etc/sysconfig/iptables ]]; then + sed -i -e '/-p 50/d' /etc/sysconfig/iptables + sed -i -e '/-p 51/d' /etc/sysconfig/iptables + sed -i -e '/--dport 5353/d' /etc/sysconfig/iptables + sed -i -e '/--dport 631/d' /etc/sysconfig/iptables + fi + if [[ -f /etc/sysconfig/ip6tables ]]; then + sed -i -e '/-p 50/d' /etc/sysconfig/ip6tables + sed -i -e '/-p 51/d' /etc/sysconfig/ip6tables + sed -i -e '/--dport 5353/d' /etc/sysconfig/ip6tables + sed -i -e '/--dport 631/d' /etc/sysconfig/ip6tables + fi + fi + + common::echo_message "Package manager cleanup" + local pm + if [[ ${ORACLE_RELEASE} -lt 8 ]]; then + pm="yum" + else + pm="dnf" + fi + ${pm} -q repolist > "${BUILD_INFO}/repolist.txt" + : > /etc/${pm}/vars/ociregion + echo "oracle.com" > /etc/${pm}/vars/ocidomain + rm -rf /var/cache/${pm}/* + rm -rf /var/lib/${pm}/* + find /etc/ -name "./*.uln-*" -exec rm -rf {} \; + + common::echo_message "Cleanup resolver files" + : > /etc/resolv.conf + /bin/rm -f /etc/resolv.conf.* + + # Rebuild rpmdb to save some space + rpm --rebuilddb + + # Remove man and info pages + common::echo_message "Exclude documentation: ${EXCLUDE_DOCS^^}" + if [[ "${EXCLUDE_DOCS,,}" = "minimal" ]]; then + rm -rf /usr/share/{man,info} + fi + + common::echo_message "Misc cleanup" + if [ -d /root/.vnc ]; then + /bin/rm -f /root/.vnc/*.log + /bin/rm -f /root/.vnc/passwd + fi + rm -f /root/.viminfo + rm -rf /.autorelabel + rm -rf /poweroff + rm -f /etc/udev/rules.d/70-persistent-cd.rules + find /etc/ -name "*.old" -exec rm -f {} \; + rm -rf /var/lib/NetworkManager + rm -rf /lost+found/* + rm -rf /root/.vbox_version + rm -rf /root/.gemrc /root/.gem + export HISTSIZE=0 + + common::echo_message "Save list of installed packages" + rpm -qa --qf "%{name}.%{arch}\n" | sort -u > "${BUILD_INFO}/pkglist.txt" + rpm -qa --qf '"%{NAME}","%{EPOCHNUM}","%{VERSION}","%{RELEASE}","%{ARCH}"\n' | sort > "${BUILD_INFO}/pkglist.csv" + common::default_kernel > "${BUILD_INFO}/kernel.txt" + + common::echo_message "lvm-system-devices" + # OL8 virt-sysprep doesn't have this module + rm -f /etc/lvm/devices/system.devices + + history -c + swapoff -a +} diff --git a/oracle-linux-image-tools/bin/provision.sh b/oracle-linux-image-tools/bin/provision.sh index dc5bd6b..87450ae 100755 --- a/oracle-linux-image-tools/bin/provision.sh +++ b/oracle-linux-image-tools/bin/provision.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # shellcheck disable=SC1090 # -# Packer main provisioning script +# Main provisioning script # -# Copyright (c) 2019, 2022 Oracle and/or its affiliates. +# Copyright (c) 2019, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl. # @@ -18,39 +18,15 @@ set -e # Constants -readonly PACKER_FILES="/tmp/packer_files" -readonly ENV_FILE="${PACKER_FILES}/env.properties" +readonly PROVISION_DIR="/tmp/provision.d" +readonly ENV_FILE="${PROVISION_DIR}/env.properties" # shellcheck disable=SC2034 readonly YUM_VERBOSE="-d1" -####################################### -# Echo header / message convenience functions -# Globals: -# None -# Arguments: -# None -# Returns: -# None -####################################### -echo_header() { - echo "=== $* ===" -} - -echo_message() { - echo "--- $* ---" -} - -echo_error() { - echo "--- $* ---" >&2 - exit 1 -} - ####################################### # Load environment variables and provisioning scripts # Globals: -# ENV_FILE -# PACKER_FILES -# PROXY_URL +# ENV_FILE, PROVISION_DIR, PROXY_URL # Loaded environment files... # Arguments: # None @@ -71,14 +47,9 @@ load_env() { export sftp_proxy="${PROXY_URL}" fi - for dir in \ - "${PACKER_FILES}/distr" \ - "${PACKER_FILES}/cloud" \ - "${PACKER_FILES}/cloud/distr" \ - "${PACKER_FILES}/custom" - do - if [[ -r "${dir}/provision.sh" ]]; then - source "${dir}/provision.sh" + for dir in distr cloud cloud/distr custom; do + if [[ -r "${PROVISION_DIR}/${dir}/provision.sh" ]]; then + source "${PROVISION_DIR}/${dir}/provision.sh" fi done } @@ -87,74 +58,48 @@ load_env() { # provision ####################################### provision () { - echo_header "Load environment" + common::echo_header "Load environment" load_env if [[ "$(type -t distr::provision)" = 'function' ]]; then - echo_header "Run distribution provisioner" + common::echo_header "Run distribution provisioner" distr::provision fi if [[ "$(type -t cloud::provision)" = 'function' ]]; then - echo_header "Run cloud provisioner" + common::echo_header "Run cloud provisioner" cloud::provision fi if [[ "$(type -t cloud_distr::provision)" = 'function' ]]; then - echo_header "Run cloud distribution provisioner" + common::echo_header "Run cloud distribution provisioner" cloud_distr::provision fi if [[ "$(type -t custom::provision)" = 'function' ]]; then - echo_header "Run custom provisioner" + common::echo_header "Run custom provisioner" custom::provision fi if [[ "$(type -t custom::cleanup)" = 'function' ]]; then - echo_header "Run custom cleanup" + common::echo_header "Run custom cleanup" custom::cleanup fi if [[ "$(type -t cloud_distr::cleanup)" = 'function' ]]; then - echo_header "Run cloud distribution cleanup" + common::echo_header "Run cloud distribution cleanup" cloud_distr::cleanup fi if [[ "$(type -t cloud::cleanup)" = 'function' ]]; then - echo_header "Run cloud cleanup" + common::echo_header "Run cloud cleanup" cloud::cleanup fi if [[ "$(type -t distr::cleanup)" = 'function' ]]; then - echo_header "Run distribution cleanup" + common::echo_header "Run distribution cleanup" distr::cleanup fi } -####################################### -# seal -####################################### -seal () { - echo_header "Load environment" - load_env - if [[ "$(type -t distr::seal)" = 'function' ]]; then - echo_header "Seal VM image" - distr::seal - else - echo_message "No seal function defined" - fi -} - ####################################### # Main ####################################### main () { - if [[ -z ${OLIT_ACTION} ]]; then - echo_error "OLIT_ACTION undefined" - fi - case "${OLIT_ACTION}" in - provision) - provision - ;; - seal) - seal - ;; - *) - echo_error "Unexpected action: ${OLIT_ACTION}" - ;; - esac + source "${PROVISION_DIR}/provision-common.sh" + provision } main "$@" diff --git a/oracle-linux-image-tools/cloud/azure/image-scripts.sh b/oracle-linux-image-tools/cloud/azure/image-scripts.sh index 0bc44f0..1cec421 100755 --- a/oracle-linux-image-tools/cloud/azure/image-scripts.sh +++ b/oracle-linux-image-tools/cloud/azure/image-scripts.sh @@ -2,33 +2,21 @@ # # Cleanup and package image for Azure # -# Copyright (c) 2019, 2022 Oracle and/or its affiliates. +# Copyright (c) 2019, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # -# Description: this module provides 2 functions: -# cloud::image_cleanup: cloud specific actions to cleanup the image -# This function is optional +# Description: this module provides the following functions which are run on +# the host: +# cloud::validate: called at the very begining to validate project parameters +# (optional) +# cloud::customize_args: arguments to pass to virt-customize (optional) +# cloud::sysprep_args: arguments to pass to virt-sysprep (optional) # cloud::image_package: Package the raw image for the target cloud. -# This function must be defined either at cloud or cloud/distribution level -# +# This function must be defined either at cloud or cloud/distribution level# # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. # -####################################### -# Cleanup actions run directly on the image -# Globals: -# None -# Arguments: -# root filesystem directory -# boot filesystem directory -# Returns: -# None -####################################### -# cloud::image_cleanup() { -# : -# } - ####################################### # Image packaging # Globals: @@ -39,5 +27,5 @@ # None ####################################### cloud::image_package() { - common::convert_to_vhd "${VM_NAME}.vhd" + common::convert_to_vhd "${WORKSPACE}/${VM_NAME}/${VM_NAME}.vhd" } diff --git a/oracle-linux-image-tools/cloud/azure/provision.sh b/oracle-linux-image-tools/cloud/azure/provision.sh index d6d09c1..5caa9b8 100755 --- a/oracle-linux-image-tools/cloud/azure/provision.sh +++ b/oracle-linux-image-tools/cloud/azure/provision.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # -# Packer provisioning script for Azure +# Provisioning script for Azure # -# Copyright (c) 2019 Oracle and/or its affiliates. +# Copyright (c) 2019, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # @@ -17,7 +17,7 @@ ####################################### # Install the Microsoft Azure Linux Agent # Globals: -# YUM_VERBOSE +# ORACLE_RELEASE, RESCUE_KERNEL, YUM_VERBOSE # Arguments: # None # Returns: @@ -25,12 +25,12 @@ ####################################### cloud::install_WALinuxAgent() { - echo_message "Install Microsoft Azure Linux Agent" + common::echo_message "Install Microsoft Azure Linux Agent" if [[ "${ORACLE_RELEASE}" = "7" ]]; then yum install -y "${YUM_VERBOSE}" parted python-pyasn1 hypervkvpd yum install -y "${YUM_VERBOSE}" --enablerepo ol7_addons WALinuxAgent dnsmasq yum remove -y "${YUM_VERBOSE}" dracut-config-rescue - elif [[ "${ORACLE_RELEASE}" = "8" ]]; then + elif [[ "${ORACLE_RELEASE}" =~ ^[89]$ ]]; then dnf install -y parted hypervkvpd dnf install -y WALinuxAgent dnsmasq if [[ -z "${RESCUE_KERNEL}" || "${RESCUE_KERNEL,,}" = "no" ]]; then @@ -51,7 +51,7 @@ cloud::install_WALinuxAgent() ####################################### # Configuration of the Azure image # Globals: -# AZURE_DBLICENSE +# AZURE_DBLICENSE, ORACLE_RELEASE # Arguments: # None # Returns: @@ -59,7 +59,7 @@ cloud::install_WALinuxAgent() ####################################### cloud::azure_cfg() { - echo_message "Configure networking" + common::echo_message "Configure networking" # Simple eth0 config, again not hard-coded to the build hardware cat > /etc/sysconfig/network-scripts/ifcfg-eth0 <<-EOF DEVICE=eth0 @@ -84,22 +84,22 @@ cloud::azure_cfg() SUBSYSTEM=="net", DRIVERS=="hv_pci", ACTION=="add", ENV{NM_UNMANAGED}="1" EOF - echo_message "Disable unneeded services" + common::echo_message "Disable unneeded services" systemctl disable wpa_supplicant || true systemctl disable iptables || true systemctl disable ip6tables || true - echo_message "Enable required services" + common::echo_message "Enable required services" systemctl enable network || true systemctl enable dnsmasq || true - echo_message "Configure grub" + common::echo_message "Configure grub" grubby --update-kernel=ALL --args="console=tty1 console=ttyS0,115200n8 earlyprintk=ttyS0,115200 rootdelay=300 net.ifnames=0" sed -i 's/^\(GRUB_CMDLINE_LINUX\)=".*"$/\1="console=tty1 console=ttyS0,115200n8 earlyprintk=ttyS0,115200 rootdelay=300 net.ifnames=0"/g' /etc/default/grub sed -i 's/^#\(ClientAliveInterval\).*$/\1 180/g' /etc/ssh/sshd_config - echo_message "Update EULA" - cp "/tmp/packer_files/cloud/${AZURE_DBLICENSE}" /usr/share/oraclelinux-release/EULA - cp "/tmp/packer_files/cloud/${AZURE_DBLICENSE}" /usr/share/eula/eula.en_US + common::echo_message "Update EULA" + cp "${PROVISION_DIR}/cloud/${AZURE_DBLICENSE}" /usr/share/oraclelinux-release/EULA + cp "${PROVISION_DIR}/cloud/${AZURE_DBLICENSE}" /usr/share/eula/eula.en_US } ####################################### diff --git a/oracle-linux-image-tools/cloud/none/image-scripts.sh b/oracle-linux-image-tools/cloud/none/image-scripts.sh index b78efd2..7230bca 100755 --- a/oracle-linux-image-tools/cloud/none/image-scripts.sh +++ b/oracle-linux-image-tools/cloud/none/image-scripts.sh @@ -2,13 +2,16 @@ # # Cleanup and package image for the "None" image # -# Copyright (c) 2019, 2022 Oracle and/or its affiliates. +# Copyright (c) 2019, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # -# Description: this module provides 2 functions: -# cloud::image_cleanup: cloud specific actions to cleanup the image -# This function is optional +# Description: this module provides the following functions which are run on +# the host: +# cloud::validate: called at the very begining to validate project parameters +# (optional) +# cloud::customize_args: arguments to pass to virt-customize (optional) +# cloud::sysprep_args: arguments to pass to virt-sysprep (optional) # cloud::image_package: Package the raw image for the target cloud. # This function must be defined either at cloud or cloud/distribution level # @@ -16,39 +19,15 @@ # ####################################### -# Cleanup actions run directly on the image +# Image packaging: do nothing +# (Provide stub as we need a packaging fnction) # Globals: # None # Arguments: -# root filesystem directory -# boot filesystem directory -# Returns: -# None -####################################### -# cloud::image_cleanup() { -# : -# } - -####################################### -# Image packaging: -# For VistualBox we convert back to VMDK and re-create the OVA file -# For qemu we convert to a qcow2 file -# Globals: -# PACKER_BUILDER -# VM_NAME -# Arguments: # None # Returns: # None ####################################### cloud::image_package() { - if common::is_vbox ; then - local vmdk - vmdk=$(grep "ovf:href" "${VM_NAME}.ovf" | sed -r -e 's/.*ovf:href="([^"]+)".*/\1/') - common::convert_to_vmdk "${vmdk}" - common::make_manifest "${VM_NAME}.ovf" "${vmdk}" > "${VM_NAME}.mf" - common::make_ova "${VM_NAME}.ovf" "${VM_NAME}.mf" "${vmdk}" - else - common::convert_to_qcow2 "${VM_NAME}.qcow" - fi + : } diff --git a/oracle-linux-image-tools/cloud/oci/env.properties b/oracle-linux-image-tools/cloud/oci/env.properties index d677c1e..75edb40 100644 --- a/oracle-linux-image-tools/cloud/oci/env.properties +++ b/oracle-linux-image-tools/cloud/oci/env.properties @@ -6,3 +6,5 @@ CLOUD_INIT="Yes" # Cloud-init user (Empty variable means keep package default -- cloud-user) CLOUD_USER="opc" +# Install OCI repo mapper script +OCI_REPO_MAPPER="yes" diff --git a/oracle-linux-image-tools/cloud/oci/image-scripts.sh b/oracle-linux-image-tools/cloud/oci/image-scripts.sh index cc55173..c9284f4 100755 --- a/oracle-linux-image-tools/cloud/oci/image-scripts.sh +++ b/oracle-linux-image-tools/cloud/oci/image-scripts.sh @@ -2,37 +2,39 @@ # # Cleanup and package image for OCI # -# Copyright (c) 2020, 2022 Oracle and/or its affiliates. +# Copyright (c) 2020, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # -# Description: this module provides 2 functions: -# cloud::image_cleanup: cloud specific actions to cleanup the image -# This function is optional +# Description: this module provides the following functions which are run on +# the host: +# cloud::validate: called at the very begining to validate project parameters +# (optional) +# cloud::customize_args: arguments to pass to virt-customize (optional) +# cloud::sysprep_args: arguments to pass to virt-sysprep (optional) # cloud::image_package: Package the raw image for the target cloud. -# This function must be defined either at cloud or cloud/distribution level -# +# This function must be defined either at cloud or cloud/distribution level# # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. # ####################################### -# Cleanup actions run directly on the image +# Parameter validation # Globals: -# None +# OLVM_TEMPLATE # Arguments: -# root filesystem directory -# boot filesystem directory +# None # Returns: # None ####################################### -# cloud::image_cleanup() { -# : -# } +cloud::validate() { + [[ "${OCI_REPO_MAPPER,,}" =~ ^((yes)|(no))$ ]] || common::error "OCI_REPO_MAPPER must be Yes or No" + readonly OCI_REPO_MAPPER +} ####################################### -# Image packaging - creates a PVM and PVHVM OVA +# Image packaging - nothing needs to be done # Globals: -# VM_NAME +# None # Arguments: # None # Returns: @@ -40,5 +42,5 @@ ####################################### cloud::image_package() { # We only need a QCOW2 file - common::convert_to_qcow2 "${VM_NAME}.qcow" + : } diff --git a/oracle-linux-image-tools/cloud/oci/ol7-slim/files/oci-yum-repo-mapper b/oracle-linux-image-tools/cloud/oci/ol7-slim/files/oci-yum-repo-mapper new file mode 100755 index 0000000..0b52d6d --- /dev/null +++ b/oracle-linux-image-tools/cloud/oci/ol7-slim/files/oci-yum-repo-mapper @@ -0,0 +1,191 @@ +#!/bin/bash +# +# Copyright © 2019 Oracle Corp., Inc. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl +# + +function retry_command() { + + retry_attempts=5 + retry_interval_sec=2 + while [ "$retry_attempts" -gt 0 ]; do + command_success=true + "$@" || { command_success=false; } + if [ "$command_success" == false ]; then + ((retry_attempts--)) + echo "$(date): Error occurred running command $@. Will retry in $retry_interval_sec seconds" >>"$log_file" + sleep $retry_interval_sec + else + break + fi + done + + # Check if there is an issue running the command after all retry_attempts + if [ "$command_success" == false ]; then + echo "$(date): ERROR: failed to execute command '$@' (Retried $retry_attempts times)" >>"$log_file" + return 1 + fi +} + +function repo_mapper_func() { + + first_boot=${1} + mirror_yes=0 + pubyum_yes=0 + + if [[ "$first_boot" == "FirstBoot" ]]; then + String="First Boot" + else + String="Instance Reboot or Service Restart" + fi + + # Begin: Process OCI Regions. + # Possibilities: + # a) yum mirror is accessible. + # b) no yum mirror, but public yum is accessible + # c) neither yum mirror nor public yum accessible. + if [[ -n "$imds_realm" ]]; then + # a) Is yum mirror responsive? + if curl -L -sfm 25 https://yum."${region}"."${domain}" &>/dev/null; then + mirror_yes=1 + echo ".$region" > $ociregion_file + echo "$domain" > $ocidomain_file + echo "$(date): $String seen: OCI realm detected: $imds_realm. Yum mirror connectivity succeeded. Overwriting $ociregion_file to: .$region. Overwriting $ocidomain_file to: $domain" >> $log_file + + # b) No yum mirror. Is public yum accessible? + elif curl -L -sfm 25 https://yum.oracle.com &>/dev/null; then + pubyum_yes=1 + echo "" > $ociregion_file + echo "oracle.com" > $ocidomain_file + echo "$(date): $String seen: OCI realm detected: $imds_realm. Yum mirror connectivity failed. Public yum connectivity succeeded. Overwriting $ociregion_file to: \"\". Overwriting $ocidomain_file to: $(cat "$ocidomain_file")" >> $log_file + + fi + + # c) If both, yum mirror and public yum did not respond + # maybe service gateway not yet setup OR an outage. + # Setup similar to yum mirror available scenario. + if [[ ("$mirror_yes" == 0) && ("$pubyum_yes" == 0) ]]; then + echo ".$region" > $ociregion_file + echo "$domain" > $ocidomain_file + echo "$(date): $String seen: OCI realm detected: $imds_realm. Yum mirror connectivity failed. Public yum connectivity failed. Overwriting $ociregion_file to: .$region. Overwriting $ocidomain_file to: $domain" >> $log_file + + fi + + # End: Process OCI Regions. + + fi + + if [[ "$first_boot" == "FirstBoot" ]]; then + + if [[ "$pubyum_yes" == 1 ]]; then + # Disable repos that are not available on public yum + rpm -q --quiet ksplice-release-el${ol_version} && + yum-config-manager --disable ol${ol_version}_ksplice &>/dev/null + + rpm -q --quiet oci-included-release-el${ol_version} && + yum-config-manager --disable ol${ol_version}_oci_included &>/dev/null + else + # Disable the ksplice-uptrack repo if installed. + rpm -q --quiet ksplice-uptrack-release && + yum-config-manager --disable ksplice-uptrack &>/dev/null + + fi + fi +} + +# Initialize +log_file=/var/log/yum-repo-mapper.log +region_file=/etc/yum/vars/region +ociregion_file=/etc/yum/vars/ociregion +ocidomain_file=/etc/yum/vars/ocidomain +ol_version=7 + +# Instance First Boot Tracker +yum_populated=/var/lib/oci-yum-repo-mapper-firstrun + +# Dynamically set mirror domain and region from IMDSv2 Attributes +if [[ -f "/sys/class/dmi/id/chassis_asset_tag" ]] && grep -q "OracleCloud.com" /sys/class/dmi/id/chassis_asset_tag; then + imds_domain=$(retry_command curl -H "Authorization:Bearer Oracle" -sfm 25 http://169.254.169.254/opc/v2/instance/ 2>/dev/null | jq -r '.regionInfo.realmDomainComponent') + imds_region=$(retry_command curl -H "Authorization:Bearer Oracle" -sfm 25 http://169.254.169.254/opc/v2/instance/ 2>/dev/null | jq -r '.regionInfo.regionIdentifier') + imds_realm=$(retry_command curl -H "Authorization:Bearer Oracle" -sfm 25 http://169.254.169.254/opc/v2/instance/ 2>/dev/null | jq -r '.regionInfo.realmKey') +fi +# OCI-Region +if [[ -n "$imds_realm" ]]; then + domain="oci.$imds_domain" + region="$imds_region" + ociregion=".$region" + +# Non-OCI Region: yum.oracle.com +else + domain="oracle.com" + region="" + ociregion="" +fi + +# Instance First boot Processing +if [[ ! -f "$yum_populated" ]]; then + echo "$region" > $region_file + echo "$(date): First boot detected in $imds_realm realm. $region_file set to: $region" >> $log_file + echo "$(date): Metadata derived values: Domain: $imds_domain, Region: $imds_region" >> $log_file + + # No OCI region found, fallback to public yum. + if [[ -z "$region" ]]; then + echo "$(date): Null region retrieved from metadata service: $region" >> $log_file + # yum variable ociregion needs to be null to fall back to public yum repo + echo "" > $ociregion_file + echo "$(date): $ociregion_file set to: \"\"" >> $log_file + # yum domain ocidomain should be oracle.com to fall back to public yum repo + echo "oracle.com" > $ocidomain_file + echo "$(date): $ocidomain_file set to: $(cat "$ocidomain_file")" >> $log_file + + # Disable repos that are not available on public yum + rpm -q --quiet ksplice-release-el${ol_version} && + yum-config-manager --disable ol${ol_version}_ksplice &>/dev/null + + rpm -q --quiet oci-included-release-el${ol_version} && + yum-config-manager --disable ol${ol_version}_oci_included &>/dev/null + else + repo_mapper_func "FirstBoot" + + fi + + # /etc/yum/vars populated + touch "$yum_populated" + +# per boot +else + + # Do not overwrite the region variables unless the region has changed + # since first boot of the instance or the ociregion variable was previously + # null (having failed the OCI yum mirror connectivity test in first boot.) + # Overwriting region to be null in subsequent reboots of the instance or on + # failure to connect to the OCI yum mirror will result in broken repo URLs + # if the user has enabled any OCI specific repos. + + echo "$(date): Instance Reboot or Service restart detected in $imds_realm realm" >> $log_file >> $log_file + echo "$(date): Metadata derived values: Domain: $imds_domain, Region: $imds_region" >> $log_file >> $log_file + + if [ -n "$region" ]; then + current_region="" + [ -f "$region_file" ] && current_region=$(cat "$region_file") + current_ociregion="" + [ -f "$ociregion_file" ] && current_ociregion=$(cat "$ociregion_file") + current_ocidomain="" + [ -f "$ocidomain_file" ] && current_ocidomain=$(cat "$ocidomain_file") + + # Account for IMDSv2 metadata changes per boot + if [ "$region" != "$current_region" ] || [ "$ociregion" != "$current_ociregion" ] || [ "$domain" != "$current_ocidomain" ] || [ -z "$current_ociregion" ]; then + echo "$region" > $region_file + echo "$(date): Service restart or instance reboot detected. Overwriting $region_file to: $region" >> $log_file + + # Function to determine the yum settings. + repo_mapper_func "Reboot" + fi + fi +fi +echo "$(date): Instance set to: " >> $log_file +echo "$(date): Domain: $(cat "$ocidomain_file")" >> $log_file +echo "$(date): Region: $(cat "$region_file")" >> $log_file +echo "$(date): Ociregion: $(cat "$ociregion_file")" >> $log_file +echo "$(date): oci-yum-repo-mapper service done: $(date)" >> $log_file + diff --git a/oracle-linux-image-tools/cloud/oci/ol7-slim/files/oci-yum-repo-mapper.service b/oracle-linux-image-tools/cloud/oci/ol7-slim/files/oci-yum-repo-mapper.service new file mode 100644 index 0000000..5270a90 --- /dev/null +++ b/oracle-linux-image-tools/cloud/oci/ol7-slim/files/oci-yum-repo-mapper.service @@ -0,0 +1,17 @@ +# Service config file for Oracle Yum Region Configuration. + +[Unit] +Description=Oracle Cloud Infrastructure Yum Region Setting Service +After=network-online.target +Requires=network-online.target +Before=cloud-config.service + +[Service] +PermissionsStartOnly=True +ExecStart=/usr/sbin/oci-yum-repo-mapper +Type=oneshot +RemainAfterExit=yes +TimeoutStartSec=8min + +[Install] +WantedBy=multi-user.target diff --git a/oracle-linux-image-tools/cloud/oci/ol8-aarch64 b/oracle-linux-image-tools/cloud/oci/ol8-aarch64 new file mode 120000 index 0000000..9aa0a23 --- /dev/null +++ b/oracle-linux-image-tools/cloud/oci/ol8-aarch64 @@ -0,0 +1 @@ +ol8-slim \ No newline at end of file diff --git a/oracle-linux-image-tools/cloud/oci/ol8-slim/files/oci-yum-repo-mapper b/oracle-linux-image-tools/cloud/oci/ol8-slim/files/oci-yum-repo-mapper new file mode 100755 index 0000000..edb1f09 --- /dev/null +++ b/oracle-linux-image-tools/cloud/oci/ol8-slim/files/oci-yum-repo-mapper @@ -0,0 +1,240 @@ +#!/bin/bash +# +# Copyright (C) 2022 Oracle. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation, version 2. This program is distributed in the hope that it will +# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the GNU +# General Public License along with this program; if not, write to the Free +# Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 021110-1307, USA. + +log () { + logger -t "${0##*/}" "$*" +} + +log "oci-yum-repo-mapper service start: $(date)" + +function retry_command() { + + retry_attempts=5 + retry_interval_sec=2 + while [[ "$retry_attempts" -gt 0 ]]; do + command_success=true + "$@" || { command_success=false; } + if [[ "$command_success" == false ]]; then + (( retry_attempts-- )) + log "Error occurred running command $@. Will retry in $retry_interval_sec seconds" + sleep $retry_interval_sec + else + break + fi + done + + # Check if there is an issue running the command after all retry_attempts + if [[ "$command_success" == false ]]; then + log "ERROR: failed to execute command '$@' (Retried $retry_attempts times)" + return 1 + fi +} + +function repo_mapper_func() { + + first_boot=${1} + mirror_yes=0 + pubyum_yes=0 + + if [[ "$first_boot" == "FirstBoot" ]]; then + String="First Boot" + else + String="Instance Reboot or Service Restart" + fi + + # Begin: Process OCI Regions. + # Possibilities: + # a) yum mirror is available. + # b) no yum mirror, but public yum is available + # c) neither yum mirror nor public yum available. + if [[ -n "$imds_realm" ]]; then + # a) Is yum mirror responsive? + if curl -L -sfm 25 https://yum."${region}"."${domain}" &>/dev/null; then + mirror_yes=1 + echo ".$region" > $ociregion_file + echo "$domain" > $ocidomain_file + log "$(date): $String seen: OCI realm detected: $imds_realm. Yum mirror connectivity succeeded. Overwriting $ociregion_file to: .$region. Overwriting $ocidomain_file to: $domain" + + # b) No yum mirror. Is public yum accessible? + elif curl -L -sfm 25 https://yum.oracle.com &>/dev/null; then + pubyum_yes=1 + echo "" > $ociregion_file + echo "oracle.com" > $ocidomain_file + log "$(date): $String seen: OCI realm detected: $imds_realm. Yum mirror connectivity failed. Public yum connectivity succeeded. Overwriting $ociregion_file to: \"\". Overwriting $ocidomain_file to: $(cat "$ocidomain_file")" + + fi + + # c) If both, yum mirror and public yum did not respond + # maybe service gateway not yet setup OR an outage. + # Setup similar to yum mirror available scenario. + if [[ ("$mirror_yes" == 0) && ("$pubyum_yes" == 0) ]]; then + echo ".$region" > $ociregion_file + echo "$domain" > $ocidomain_file + log "$(date): $String seen: OCI realm detected: $imds_realm. Yum mirror connectivity failed. Public yum connectivity failed. Overwriting $ociregion_file to: .$region. Overwriting $ocidomain_file to: $domain" + + fi + + # End: Process OCI Regions. + fi + + if [[ "$first_boot" == "FirstBoot" ]]; then + + if [[ "$pubyum_yes" == 1 ]]; then + # Disable repos that are not available on public yum + # Ksplice repo is not available on yum + rpm -q --quiet ksplice-release-el${ol_version} && + dnf config-manager --set-disabled ol${ol_version}_ksplice &>/dev/null + + # oci_included repo is not available on yum + rpm -q --quiet oci-included-release-el${ol_version} && + dnf config-manager --set-disabled ol${ol_version}_oci_included &>/dev/null + # oci-release repo is not available on yum + rpm -q oci-release-el${ol_version} &>/dev/null && + dnf config-manager --set-disabled ol${ol_version}_oci &>/dev/null + + else + # If installed, disable ksplice-uptrack-release + rpm -q --quiet ksplice-uptrack-release && + dnf config-manager --set-disabled ksplice-uptrack &>/dev/null + fi + fi +} + +region_file=/etc/dnf/vars/region +ociregion_file=/etc/dnf/vars/ociregion +ocidomain_file=/etc/dnf/vars/ocidomain +ol_version=8 + +yum_populated=/var/lib/oci-linux-config/yum_populated +mkdir -p /var/lib/oci-linux-config + +# Decide which IMDS endpoint: IPv4 or IPv6 +IMDS_endpointv4=http://169.254.169.254 + +# Default, use IPv4 +IMDS_endpoint=$IMDS_endpointv4 + +# Check IPv4 first. +log "Determining responsive IMDS endpoint" +curl -H "Authorization:Bearer Oracle" -sfm 3 $IMDS_endpoint/opc/v2/instance/ 2>&1 > /dev/null +if [ "$?" -ne 0 ]; then + IMDS_endpointv6=http://[fd00:c1::a9fe:a9fe] + log "Checking IPv6 IMDS endpoint: $IMDS_endpointv6" + + # Is IPv6 responsive? + curl -6 -H "Authorization:Bearer Oracle" -sfm 3 $IMDS_endpointv6/opc/v2/instance/ 2>&1 > /dev/null + if [ "$?" -eq 0 ]; then + IMDS_endpoint=" -6 $IMDS_endpointv6" + else + log "WARNING: Neither IPv4 nor IPv6 IMDS endpoint responded." + log "Defaulting to IPv4 endpoint." + fi +fi + +log " $(date): Using IMDS endpoint: $IMDS_endpoint" + +# Dynamically set mirror domain and region from IMDSv2 Attributes +if [[ -f "/sys/class/dmi/id/chassis_asset_tag" ]] && grep -q "OracleCloud.com" /sys/class/dmi/id/chassis_asset_tag; then + imds_domain=$(retry_command curl -H "Authorization:Bearer Oracle" -sfm 25 $IMDS_endpoint/opc/v2/instance/ 2>/dev/null | jq -r '.regionInfo.realmDomainComponent') + imds_region=$(retry_command curl -H "Authorization:Bearer Oracle" -sfm 25 $IMDS_endpoint/opc/v2/instance/ 2>/dev/null | jq -r '.regionInfo.regionIdentifier') + imds_realm=$(retry_command curl -H "Authorization:Bearer Oracle" -sfm 25 $IMDS_endpoint/opc/v2/instance/ 2>/dev/null | jq -r '.regionInfo.realmKey') +fi + +# OCI-Region. +if [[ -n "$imds_realm" ]]; then + domain="oci.$imds_domain" + region="$imds_region" + ociregion=".$region" + +# Non-OCI Region: yum.oracle.com +else + domain="oracle.com" + region="" + ociregion="" +fi + +# first boot +if [[ ! -f "$yum_populated" ]]; then + + echo "$region" > $region_file + log "First boot detected in $imds_realm realm. $region_file set to: $region" + log "Metadata derived values: Domain: $imds_domain, Region: $imds_region" + + # Metadata service not available. + if [[ -z "$region" ]]; then + log "Null Region retrieved from metadata service: $region" + # yum variable ociregion needs to be null to fall back to public yum repo + echo "" > $ociregion_file + log "$ociregion_file set to: \"\"" + # yum domain ocidomain should be oracle.com to fall back to public yum repo + echo "oracle.com" > $ocidomain_file + log "$ocidomain_file set to: $(cat "$ocidomain_file")" + + # Ksplice repo is not available on yum + rpm -q ksplice-release-el${ol_version} &>/dev/null && + dnf config-manager --set-disabled ol${ol_version}_ksplice &>/dev/null + + # oci_included repo is not available on yum + rpm -q oci-included-release-el${ol_version} &>/dev/null && + dnf config-manager --set-disabled ol${ol_version}_oci_included &>/dev/null + + # oci-release repo is not available on yum + rpm -q oci-release-el${ol_version} &>/dev/null && + dnf config-manager --set-disabled ol${ol_version}_oci &>/dev/null + + else + repo_mapper_func "FirstBoot" + + fi + + # /etc/yum/vars populated + touch "$yum_populated" + +# per boot +else + # Do not overwrite the region variables unless the region has changed + # since first boot of the instance or the ociregion variable was previously + # null (having failed the OCI yum mirror connectivity test) + # Overwriting region to be null in subsequent reboots of the instance or on + # failure to connect to the OCI yum mirror will result in broken repo URLs + # if the user has enabled any OCI specific repos. + + log "Instance Reboot or Service restart detected in $imds_realm realm" + log "Metadata derived values: Domain: $imds_domain, Region: $imds_region" + + if [[ -n "$region" ]]; then + current_region="" + [[ -f "$region_file" ]] && current_region=$(cat "$region_file") + current_ociregion="" + [[ -f "$ociregion_file" ]] && current_ociregion=$(cat "$ociregion_file") + current_ocidomain="" + [[ -f "$ocidomain_file" ]] && current_ocidomain=$(cat "$ocidomain_file") + + # Account for IDMSv2 metadata changes per boot + if [[ "$region" != "$current_region" ]] || [[ "$ociregion" != "$current_ociregion" ]] || [[ "$domain" != "$current_ocidomain" ]] || [[ -z "$current_ociregion" ]]; then + echo "$region" > $region_file + log "Service restart or instance reboot detected. Overwriting $region_file to: $region" + + # Function to determine the yum settings. + repo_mapper_func "Reboot" + fi + fi +fi + +# Log yum config settings. +log "Instance set to: " +log " Domain: $(cat "$ocidomain_file")" +log " Region: $(cat "$region_file")" +log " Ociregion: $(cat "$ociregion_file")" +log "oci-yum-repo-mapper service done: $(date)" diff --git a/oracle-linux-image-tools/cloud/oci/ol8-slim/files/oci-yum-repo-mapper.service b/oracle-linux-image-tools/cloud/oci/ol8-slim/files/oci-yum-repo-mapper.service new file mode 100644 index 0000000..5270a90 --- /dev/null +++ b/oracle-linux-image-tools/cloud/oci/ol8-slim/files/oci-yum-repo-mapper.service @@ -0,0 +1,17 @@ +# Service config file for Oracle Yum Region Configuration. + +[Unit] +Description=Oracle Cloud Infrastructure Yum Region Setting Service +After=network-online.target +Requires=network-online.target +Before=cloud-config.service + +[Service] +PermissionsStartOnly=True +ExecStart=/usr/sbin/oci-yum-repo-mapper +Type=oneshot +RemainAfterExit=yes +TimeoutStartSec=8min + +[Install] +WantedBy=multi-user.target diff --git a/oracle-linux-image-tools/cloud/oci/ol8-slim/provision.sh b/oracle-linux-image-tools/cloud/oci/ol8-slim/provision.sh new file mode 100755 index 0000000..eb01009 --- /dev/null +++ b/oracle-linux-image-tools/cloud/oci/ol8-slim/provision.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# Provisioning script for OCI / OL8 +# +# Copyright (c) 2024 Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# https://oss.oracle.com/licenses/upl +# +# Description: OCI/OL8 specific provisioning. +# +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. +# + + +####################################### +# Provisioning module +# Globals: +# None +# Arguments: +# None +# Returns: +# None +####################################### +cloud_distr::provision() +{ + # There is an issue with the Oracle cloud-init datasource in cloud-init 23.x + # We need to ensure network is already up before running cloud-init. + sed -i 's/^\(GRUB_CMDLINE_LINUX\)="\(.*\)"$/\1="\2 rd.neednet"/' /etc/default/grub + grub2-mkconfig -o /boot/grub2/grub.cfg +} diff --git a/oracle-linux-image-tools/cloud/oci/ol9-aarch64 b/oracle-linux-image-tools/cloud/oci/ol9-aarch64 new file mode 120000 index 0000000..c0de194 --- /dev/null +++ b/oracle-linux-image-tools/cloud/oci/ol9-aarch64 @@ -0,0 +1 @@ +ol9-slim \ No newline at end of file diff --git a/oracle-linux-image-tools/cloud/oci/ol9-slim/files/oci-yum-repo-mapper b/oracle-linux-image-tools/cloud/oci/ol9-slim/files/oci-yum-repo-mapper new file mode 100755 index 0000000..4eb77a4 --- /dev/null +++ b/oracle-linux-image-tools/cloud/oci/ol9-slim/files/oci-yum-repo-mapper @@ -0,0 +1,206 @@ +#!/usr/bin/bash +# +# Copyright (C) 2022 Oracle Corp., Inc. All rights reserved. +# + +log () { + logger -t "${0##*/}" "$*" +} + +log "oci-yum-repo-mapper service start: $(date)" + +function retry_command() { + + retry_attempts=5 + retry_interval_sec=2 + while [[ "$retry_attempts" -gt 0 ]]; do + command_success=true + "$@" || { command_success=false; } + if [[ "$command_success" == false ]]; then + (( retry_attempts-- )) + log "Error occurred running command $@. Will retry in $retry_interval_sec seconds" + sleep $retry_interval_sec + else + break + fi + done + + # Check if there is an issue running the command after all retry_attempts + if [[ "$command_success" == false ]]; then + log "ERROR: failed to execute command '$@' (Retried $retry_attempts times)" + return 1 + fi +} + +function repo_mapper_func() { + + first_boot=${1} + mirror_yes=0 + pubyum_yes=0 + + if [[ "$first_boot" == "FirstBoot" ]]; then + String="First Boot" + else + String="Instance Reboot or Service Restart" + fi + + # Begin: Process OCI Regions. + # Possibilities: + # a) yum mirror is available. + # b) no yum mirror, but public yum is available + # c) neither yum mirror nor public yum available. + if [[ -n "$imds_realm" ]]; then + # a) Is yum mirror responsive? + if curl -L -sfm 25 https://yum."${region}"."${domain}" &>/dev/null; then + mirror_yes=1 + echo ".$region" > $ociregion_file + echo "$domain" > $ocidomain_file + log "$(date): $String seen: OCI realm detected: $imds_realm. Yum mirror connectivity succeeded. Overwriting $ociregion_file to: .$region. Overwriting $ocidomain_file to: $domain" + + # b) No yum mirror. Is public yum accessible? + elif curl -L -sfm 25 https://yum.oracle.com &>/dev/null; then + pubyum_yes=1 + echo "" > $ociregion_file + echo "oracle.com" > $ocidomain_file + log "$(date): $String seen: OCI realm detected: $imds_realm. Yum mirror connectivity failed. Public yum connectivity succeeded. Overwriting $ociregion_file to: \"\". Overwriting $ocidomain_file to: $(cat "$ocidomain_file")" + + fi + + # c) If both, yum mirror and public yum did not respond + # maybe service gateway not yet setup OR an outage. + # Setup similar to yum mirror available scenario. + if [[ ("$mirror_yes" == 0) && ("$pubyum_yes" == 0) ]]; then + echo ".$region" > $ociregion_file + echo "$domain" > $ocidomain_file + log "$(date): $String seen: OCI realm detected: $imds_realm. Yum mirror connectivity failed. Public yum connectivity failed. Overwriting $ociregion_file to: .$region. Overwriting $ocidomain_file to: $domain" + + fi + + # End: Process OCI Regions. + fi + + if [[ "$first_boot" == "FirstBoot" ]]; then + + if [[ "$pubyum_yes" == 1 ]]; then + # Disable repos that are not available on public yum + # Ksplice repo is not available on yum + rpm -q --quiet ksplice-release-el${ol_version} && + dnf config-manager --set-disabled ol${ol_version}_ksplice &>/dev/null + + # oci_included repo is not available on yum + rpm -q --quiet oci-included-release-el${ol_version} && + dnf config-manager --set-disabled ol${ol_version}_oci_included &>/dev/null + # oci-release repo is not available on yum + rpm -q oci-release-el${ol_version} &>/dev/null && + dnf config-manager --set-disabled ol${ol_version}_oci &>/dev/null + + else + # If installed, disable ksplice-uptrack-release + rpm -q --quiet ksplice-uptrack-release && + dnf config-manager --set-disabled ksplice-uptrack &>/dev/null + fi + fi +} + +region_file=/etc/dnf/vars/region +ociregion_file=/etc/dnf/vars/ociregion +ocidomain_file=/etc/dnf/vars/ocidomain +ol_version=9 + +yum_populated=/var/lib/oci-linux-config/yum_populated +mkdir -p /var/lib/oci-linux-config + +# Dynamically set mirror domain and region from IMDSv2 Attributes +if [[ -f "/sys/class/dmi/id/chassis_asset_tag" ]] && grep -q "OracleCloud.com" /sys/class/dmi/id/chassis_asset_tag; then + imds_domain=$(retry_command curl -H "Authorization:Bearer Oracle" -sfm 25 http://169.254.169.254/opc/v2/instance/ 2>/dev/null | jq -r '.regionInfo.realmDomainComponent') + imds_region=$(retry_command curl -H "Authorization:Bearer Oracle" -sfm 25 http://169.254.169.254/opc/v2/instance/ 2>/dev/null | jq -r '.regionInfo.regionIdentifier') + imds_realm=$(retry_command curl -H "Authorization:Bearer Oracle" -sfm 25 http://169.254.169.254/opc/v2/instance/ 2>/dev/null | jq -r '.regionInfo.realmKey') +fi + +# OCI-Region +if [[ -n "$imds_realm" ]]; then + domain="oci.$imds_domain" + region="$imds_region" + ociregion=".$region" + +# Non-OCI Region: yum.oracle.com +else + domain="oracle.com" + region="" + ociregion="" +fi + +# first boot +if [[ ! -f "$yum_populated" ]]; then + + echo "$region" > $region_file + log "First boot detected in $imds_realm realm. $region_file set to: $region" + log "Metadata derived values: Domain: $imds_domain, Region: $imds_region" + + # Metadata service not available. + if [[ -z "$region" ]]; then + log "Null region retrieved from metadata service: $region" + # yum variable ociregion needs to be null to fall back to public yum repo + echo "" > $ociregion_file + log "$ociregion_file set to: \"\"" + # yum domain ocidomain should be oracle.com to fall back to public yum repo + echo "oracle.com" > $ocidomain_file + log "$ocidomain_file set to: $(cat "$ocidomain_file")" + + # Ksplice repo is not available on yum + rpm -q ksplice-release-el${ol_version} &>/dev/null && + dnf config-manager --set-disabled ol${ol_version}_ksplice &>/dev/null + + # oci_included repo is not available on yum + rpm -q oci-included-release-el${ol_version} &>/dev/null && + dnf config-manager --set-disabled ol${ol_version}_oci_included &>/dev/null + + # oci-release repo is not available on yum + rpm -q oci-release-el${ol_version} &>/dev/null && + dnf config-manager --set-disabled ol${ol_version}_oci &>/dev/null + + else + repo_mapper_func "FirstBoot" + + fi + + # /etc/yum/vars populated + touch "$yum_populated" + +# per boot +else + # Do not overwrite the region variables unless the region has changed + # since first boot of the instance or the ociregion variable was previously + # null (having failed the OCI yum mirror connectivity test) + # Overwriting region to be null in subsequent reboots of the instance or on + # failure to connect to the OCI yum mirror will result in broken repo URLs + # if the user has enabled any OCI specific repos. + + log "Instance Reboot or Service restart detected in $imds_realm realm" + log "Metadata derived values: Domain: $imds_domain, Region: $imds_region" + + if [[ -n "$region" ]]; then + current_region="" + [[ -f "$region_file" ]] && current_region=$(cat "$region_file") + current_ociregion="" + [[ -f "$ociregion_file" ]] && current_ociregion=$(cat "$ociregion_file") + current_ocidomain="" + [[ -f "$ocidomain_file" ]] && current_ocidomain=$(cat "$ocidomain_file") + + # Account for IDMSv2 metadata changes per boot + if [[ "$region" != "$current_region" ]] || [[ "$ociregion" != "$current_ociregion" ]] || [[ "$domain" != "$current_ocidomain" ]] || [[ -z "$current_ociregion" ]]; then + echo "$region" > $region_file + log "Service restart or instance reboot detected. Overwriting $region_file to: $region" + + # Function to determine the yum settings. + repo_mapper_func "Reboot" + fi + fi +fi + +# Log yum config settings. +log "Instance set to: " +log " Domain: $(cat "$ocidomain_file")" +log " Region: $(cat "$region_file")" +log " Ociregion: $(cat "$ociregion_file")" +log "oci-yum-repo-mapper service done: $(date)" diff --git a/oracle-linux-image-tools/cloud/oci/ol9-slim/files/oci-yum-repo-mapper.service b/oracle-linux-image-tools/cloud/oci/ol9-slim/files/oci-yum-repo-mapper.service new file mode 100644 index 0000000..301d00c --- /dev/null +++ b/oracle-linux-image-tools/cloud/oci/ol9-slim/files/oci-yum-repo-mapper.service @@ -0,0 +1,20 @@ +# Service config file for Oracle Yum Region Configuration. + +[Unit] +Description=Oracle Cloud Infrastructure Yum Region Setting Service +After=network-online.target +Requires=network-online.target +Before=cloud-config.service + +[Service] +PermissionsStartOnly=True +ExecStart=/usr/sbin/oci-yum-repo-mapper +Type=simple +RemainAfterExit=yes + +# Restart the process when it fails. +Restart=always +RestartSec=15 + +[Install] +WantedBy=multi-user.target diff --git a/oracle-linux-image-tools/cloud/oci/provision.sh b/oracle-linux-image-tools/cloud/oci/provision.sh index a3a6438..2935a1d 100755 --- a/oracle-linux-image-tools/cloud/oci/provision.sh +++ b/oracle-linux-image-tools/cloud/oci/provision.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash # -# Packer provisioning script for OCI +# Provisioning script for OCI # -# Copyright (c) 2020 Oracle and/or its affiliates. +# Copyright (c) 2020, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # -# Description: OVM specific provisioning. This module provides 2 functions, +# Description: OCI specific provisioning. This module provides 2 functions, # both are optional. # cloud::provision: provision the instance # cloud::cleanup: instance cleanup before shutdown @@ -17,7 +17,7 @@ ####################################### # Configure OCI instance # Globals: -# None +# YUM_VERBOSE # Arguments: # None # Returns: @@ -25,7 +25,7 @@ ####################################### cloud::config() { - echo_message "Setup network" + common::echo_message "Setup network" # simple eth0 configuration cat > /etc/sysconfig/network-scripts/ifcfg-eth0 <<-EOF DEVICE="eth0" @@ -37,6 +37,14 @@ cloud::config() IPV6INIT="no" PERSISTENT_DHCLIENT="1" EOF + + if [[ "${OCI_REPO_MAPPER,,}" =~ "yes" ]]; then + common::echo_message "Install repo mapper scripts" + yum install -y "${YUM_VERBOSE}" jq + cp "${PROVISION_DIR}/cloud/distr/oci-yum-repo-mapper" /usr/sbin + cp "${PROVISION_DIR}/cloud/distr/oci-yum-repo-mapper.service" /usr/lib/systemd/system + systemctl enable oci-yum-repo-mapper.service + fi } ####################################### @@ -50,14 +58,14 @@ cloud::config() ####################################### cloud::install_agent() { - echo_message "Install guest agent" + common::echo_message "Install guest agent" yum install -y "${YUM_VERBOSE}" qemu-guest-agent } ####################################### # Install cloud-init, use CLOUD_USER if specified # Globals: -# YUM_VERBOSE +# CLOUD_INIT, CLOUD_USER, YUM_VERBOSE # Arguments: # None # Returns: @@ -65,7 +73,7 @@ cloud::install_agent() ####################################### cloud::cloud_init() { - echo_message "Install cloud-init: ${CLOUD_INIT^^}" + common::echo_message "Install cloud-init: ${CLOUD_INIT^^}" if [[ "${CLOUD_INIT,,}" = "yes" ]]; then # Disable cloud-init during installation # cloud-init-generator is run at install time and generates systemd @@ -73,7 +81,7 @@ cloud::cloud_init() # image mounted! mkdir /etc/cloud touch /etc/cloud/cloud-init.disabled - yum install -y "${YUM_VERBOSE}" cloud-init + yum install -y "${YUM_VERBOSE}" cloud-init cloud-utils-growpart rm /etc/cloud/cloud-init.disabled cat > /etc/cloud/cloud.cfg.d/90_ol.cfg <<-EOF # Provide sensible defaults for OL - see Orabug 34821447 diff --git a/oracle-linux-image-tools/cloud/olvm/image-scripts.sh b/oracle-linux-image-tools/cloud/olvm/image-scripts.sh index fa4a146..2bacb55 100755 --- a/oracle-linux-image-tools/cloud/olvm/image-scripts.sh +++ b/oracle-linux-image-tools/cloud/olvm/image-scripts.sh @@ -6,10 +6,12 @@ # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # -# Description: this module provides 3 functions: -# cloud::validate: optional parameter validation -# cloud::image_cleanup: cloud specific actions to cleanup the image -# This function is optional +# Description: this module provides the following functions which are run on +# the host: +# cloud::validate: called at the very begining to validate project paramters +# (optional) +# cloud::customize_args: arguments to pass to virt-customize (optional) +# cloud::sysprep_args: arguments to pass to virt-sysprep (optional) # cloud::image_package: Package the raw image for the target cloud. # This function must be defined either at cloud or cloud/distribution level # @@ -26,28 +28,15 @@ # None ####################################### cloud::validate() { - [[ "${OLVM_TEMPLATE,,}" =~ ^(yes)|(no)$ ]] || error "OLVM_TEMPLATE must be Yes or No" + [[ "${OLVM_TEMPLATE,,}" =~ ^((yes)|(no))$ ]] || common::error "OLVM_TEMPLATE must be Yes or No" readonly OLVM_TEMPLATE } -####################################### -# Cleanup actions run directly on the image -# Globals: -# None -# Arguments: -# root filesystem directory -# boot filesystem directory -# Returns: -# None -####################################### -# cloud::image_cleanup() { -# : -# } - ####################################### # Image packaging - creates an OVA # Globals: -# CLOUD_DIR CLOUD DISTR_NAME OLVM_TEMPLATE +# BUILD_NUMBER, CLOUD, CLOUD_DIR, CPU_NUM, CUSTOM_SCRIPT, DISK_SIZE_GB, DISTR_NAME +# MEM_SIZE, OLVM_TEMPLATE, VM_NAME # Arguments: # None # Returns: @@ -60,9 +49,7 @@ cloud::image_package() { local build_upd="${DISTR_NAME#*U}" local build_upd="${build_upd%%_*}" local extra_args=() - local package_filename vmdk - - common::convert_to_qcow2 System.qcow + local package_filename href if [[ "${OLVM_TEMPLATE,,}" = "yes" ]]; then extra_args+=("--template") @@ -75,19 +62,22 @@ cloud::image_package() { extra_args+=("--script" "${CUSTOM_SCRIPT}") fi + pushd "${WORKSPACE}/${VM_NAME}" || common::error "can't cd to image directory" ${mk_envelope} "${extra_args[@]}" \ -r "${build_rel}" \ -u "${build_upd##U}" \ -v "${BUILD_NUMBER}" \ -s "${DISK_SIZE_GB}" \ - -i System.qcow \ + -i "${VM_NAME}.qcow2" \ -c "${CPU_NUM}" \ -m "${MEM_SIZE}" \ >"${package_filename}.ovf" - vmdk=$(grep "ovf:href" "${package_filename}.ovf" | sed -r -e 's/.*ovf:href="([^"]+)".*/\1/') + href=$(grep "ovf:href" "${package_filename}.ovf" | sed -r -e 's/.*ovf:href="([^"]+)".*/\1/') + + mv "${VM_NAME}.qcow2" "${href}" - mv System.qcow "${vmdk}" + common::make_ova "${package_filename}.ovf" "${href}" - common::make_ova "${package_filename}.ovf" "${vmdk}" + popd || common::error "can't pop directory" } diff --git a/oracle-linux-image-tools/cloud/olvm/mk-envelope.py b/oracle-linux-image-tools/cloud/olvm/mk-envelope.py index a85a1b5..2e42f2e 100755 --- a/oracle-linux-image-tools/cloud/olvm/mk-envelope.py +++ b/oracle-linux-image-tools/cloud/olvm/mk-envelope.py @@ -99,7 +99,7 @@ def parse_args(): help='Image size in GB, e.g. 10') parser.add_argument('-i', '--image', - default='System.qcow', + default='System.qcow2', help='Image file name') parser.add_argument('-t', '--template', @@ -126,7 +126,7 @@ def generate_ovf(args): # Image capacity and size on disk disk_capacity = args.size * 1024 * 1024 * 1024 file_size = stat(args.image).st_size - # The imported disk image will be an uncompressed qcow file + # The imported disk image will be an uncompressed qcow2 file uncompressed = args.image + ".uncompressed" call(["qemu-img", "convert", "-O", "qcow2", args.image, uncompressed]) disk_size = stat(uncompressed).st_size @@ -200,7 +200,7 @@ def generate_ovf(args): 'ovf:diskId': disk_uuid, # Image size 'ovf:capacity': str(disk_capacity), - # Size on disk of the uncompressed qcow file + # Size on disk of the uncompressed qcow2 file 'ovf:populatedSize': str(disk_size), # Ref to file (should be the "id" of fileref) 'ovf:fileRef': file_uuid, diff --git a/oracle-linux-image-tools/cloud/olvm/provision.sh b/oracle-linux-image-tools/cloud/olvm/provision.sh index 0279135..aab4f40 100755 --- a/oracle-linux-image-tools/cloud/olvm/provision.sh +++ b/oracle-linux-image-tools/cloud/olvm/provision.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash # -# Packer provisioning script for OLVM +# Provisioning script for OLVM # -# Copyright (c) 2020 Oracle and/or its affiliates. +# Copyright (c) 2020, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # -# Description: OVM specific provisioning. This module provides 2 functions, +# Description: OLVM specific provisioning. This module provides 2 functions, # both are optional. # cloud::provision: provision the instance # cloud::cleanup: instance cleanup before shutdown @@ -25,7 +25,7 @@ ####################################### cloud::config() { - echo_message "Setup network" + common::echo_message "Setup network" # simple eth0 configuration cat > /etc/sysconfig/network-scripts/ifcfg-eth0 <<-EOF DEVICE="eth0" @@ -50,14 +50,14 @@ cloud::config() ####################################### cloud::install_agent() { - echo_message "Install guest agent" + common::echo_message "Install guest agent" yum install -y "${YUM_VERBOSE}" qemu-guest-agent } ####################################### # Install cloud-init, use CLOUD_USER if specified # Globals: -# YUM_VERBOSE +# CLOUD_INIT, CLOUD_USER, YUM_VERBOSE # Arguments: # None # Returns: @@ -65,7 +65,7 @@ cloud::install_agent() ####################################### cloud::cloud_init() { - echo_message "Install cloud-init: ${CLOUD_INIT^^}" + common::echo_message "Install cloud-init: ${CLOUD_INIT^^}" if [[ "${CLOUD_INIT,,}" = "yes" ]]; then # Disable cloud-init during installation # cloud-init-generator is run at install time and generates systemd @@ -73,7 +73,7 @@ cloud::cloud_init() # image mounted! mkdir /etc/cloud touch /etc/cloud/cloud-init.disabled - yum install -y "${YUM_VERBOSE}" cloud-init + yum install -y "${YUM_VERBOSE}" cloud-init cloud-utils-growpart rm /etc/cloud/cloud-init.disabled cat > /etc/cloud/cloud.cfg.d/90_ol.cfg <<-EOF # Provide sensible defaults for OL - see Orabug 34821447 diff --git a/oracle-linux-image-tools/cloud/ovm/env.properties b/oracle-linux-image-tools/cloud/ovm/env.properties index 974e1f0..3714e3c 100755 --- a/oracle-linux-image-tools/cloud/ovm/env.properties +++ b/oracle-linux-image-tools/cloud/ovm/env.properties @@ -4,3 +4,6 @@ # Image version (as set in the OVF file) IMAGE_VERSION="1.0" + +# Allow password based root login +PERMIT_ROOT_LOGIN=yes diff --git a/oracle-linux-image-tools/cloud/ovm/image-scripts.sh b/oracle-linux-image-tools/cloud/ovm/image-scripts.sh index 7e11e79..bd0b14a 100755 --- a/oracle-linux-image-tools/cloud/ovm/image-scripts.sh +++ b/oracle-linux-image-tools/cloud/ovm/image-scripts.sh @@ -2,14 +2,16 @@ # # Cleanup and package image for OVM # -# Copyright (c) 2019, 2022 Oracle and/or its affiliates. +# Copyright (c) 2019, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # -# Description: this module provides 3 functions: -# cloud::validate: optional parameter validation -# cloud::image_cleanup: cloud specific actions to cleanup the image -# This function is optional +# Description: this module provides the following functions which are run on +# the host: +# cloud::validate: called at the very begining to validate project paramters +# (optional) +# cloud::customize_args: arguments to pass to virt-cutomize (optional) +# cloud::sysprep_args: arguments to pass to virt-sysprep (optional) # cloud::image_package: Package the raw image for the target cloud. # This function must be defined either at cloud or cloud/distribution level # @@ -29,37 +31,24 @@ cloud::validate() { : } -####################################### -# Cleanup actions run directly on the image -# Globals: -# None -# Arguments: -# root filesystem directory -# boot filesystem directory -# Returns: -# None -####################################### -# cloud::image_cleanup() { -# : -# } - ####################################### # Image packaging - creates a PVM and PVHVM OVA # Globals: -# CLOUD_DIR CLOUD DISTR_NAME IMAGE_VERSION +# CLOUD, CLOUD_DIR, DISK_SIZE_GB, DISTR_NAME, IMAGE_VERSION, VM_NAME # Arguments: # None # Returns: # None ####################################### cloud::image_package() { - common::convert_to_vmdk System.vmdk + common::convert_to_vmdk "${WORKSPACE}/${VM_NAME}/System.vmdk" # Decompose Build Name into Release/update/platform local build_rel="${DISTR_NAME%U*}" local build_upd="${DISTR_NAME#*U}" local build_upd="${build_upd%%_*}" + pushd "${WORKSPACE}/${VM_NAME}" || common::error "can't cd to image directory" "${CLOUD_DIR}/${CLOUD}/mk-envelope.sh" \ -r "${build_rel}" \ -u "${build_upd##U}" \ @@ -69,4 +58,5 @@ cloud::image_package() { common::make_manifest "${VM_NAME}.ovf" System.vmdk >"${VM_NAME}.mf" common::make_ova "${VM_NAME}.ovf" "${VM_NAME}.mf" System.vmdk + popd || common::error "can't pop directory" } diff --git a/oracle-linux-image-tools/cloud/ovm/mk-envelope-template.sh b/oracle-linux-image-tools/cloud/ovm/mk-envelope-template.sh index b9d7c2f..c5a9039 100755 --- a/oracle-linux-image-tools/cloud/ovm/mk-envelope-template.sh +++ b/oracle-linux-image-tools/cloud/ovm/mk-envelope-template.sh @@ -4,7 +4,7 @@ # Creates OVF Envelope template for OL templates # This script requires the open-ovf tools # -# Copyright (c) 2019,2020 Oracle and/or its affiliates. +# Copyright (c) 2019, 2020 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # diff --git a/oracle-linux-image-tools/cloud/ovm/ol7-slim/provision.sh b/oracle-linux-image-tools/cloud/ovm/ol7-slim/provision.sh index b4a88ab..7f4a00f 100755 --- a/oracle-linux-image-tools/cloud/ovm/ol7-slim/provision.sh +++ b/oracle-linux-image-tools/cloud/ovm/ol7-slim/provision.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Packer provisioning script for OVM on OL7 +# Provisioning script for OVM on OL7 # # Copyright (c) 2019 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at @@ -24,7 +24,7 @@ # None ####################################### cloud_distr::serial_cfg() { - cat > /usr/lib/systemd/system/serial_console.service <<-EOF + cat > /usr/lib/systemd/system/serial_console.service <<-'EOF' # This file is part of systemd. # # systemd is free software; you can redistribute it and/or modify it diff --git a/oracle-linux-image-tools/cloud/ovm/ol8-slim/env.properties b/oracle-linux-image-tools/cloud/ovm/ol8-slim/env.properties index fc2f2be..4737cec 100755 --- a/oracle-linux-image-tools/cloud/ovm/ol8-slim/env.properties +++ b/oracle-linux-image-tools/cloud/ovm/ol8-slim/env.properties @@ -7,4 +7,3 @@ EXTRA_KERNEL="no" # Keep kernel-modules packages for OVM. readonly KERNEL_MODULES="yes" -readonly diff --git a/oracle-linux-image-tools/cloud/ovm/ol8-slim/image-scripts.sh b/oracle-linux-image-tools/cloud/ovm/ol8-slim/image-scripts.sh index 8deb758..da2d13e 100755 --- a/oracle-linux-image-tools/cloud/ovm/ol8-slim/image-scripts.sh +++ b/oracle-linux-image-tools/cloud/ovm/ol8-slim/image-scripts.sh @@ -2,7 +2,7 @@ # # Validate parameters # -# Copyright (c) 2021 Oracle and/or its affiliates. +# Copyright (c) 2021, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # @@ -22,6 +22,6 @@ # None ####################################### cloud_distr::validate() { - [[ "${EXTRA_KERNEL,,}" =~ ^(yes)|(no)$ ]] || error "EXTRA_KERNEL must be yes or no" + [[ "${EXTRA_KERNEL,,}" =~ ^((yes)|(no))$ ]] || common::error "EXTRA_KERNEL must be yes or no" readonly EXTRA_KERNEL } diff --git a/oracle-linux-image-tools/cloud/ovm/ol8-slim/provision.sh b/oracle-linux-image-tools/cloud/ovm/ol8-slim/provision.sh index 12b5da3..2e4f701 100644 --- a/oracle-linux-image-tools/cloud/ovm/ol8-slim/provision.sh +++ b/oracle-linux-image-tools/cloud/ovm/ol8-slim/provision.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # -# Packer provisioning script for OVM on OL8 +# Provisioning script for OVM on OL8 # -# Copyright (c) 2020, 2021, Oracle and/or its affiliates. +# Copyright (c) 2020, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # @@ -24,7 +24,7 @@ # None ####################################### cloud_distr::serial_cfg() { - cat > /usr/lib/systemd/system/serial_console.service <<-EOF + cat > /usr/lib/systemd/system/serial_console.service <<-'EOF' # This file is part of systemd. # # systemd is free software; you can redistribute it and/or modify it @@ -96,10 +96,10 @@ cloud_distr::additional_kernel() { dnf config-manager --set-enabled "ol8_UEKR${UEK_RELEASE}" fi - echo_message "Adding kernel: ${kernel}" + common::echo_message "Adding kernel: ${kernel}" dnf install -y ${kernel} kernel_version=$(rpm -q ${kernel} --qf "%{VERSION}-%{RELEASE}.%{ARCH}") - echo_message "Installed kernel: ${kernel_version}" + common::echo_message "Installed kernel: ${kernel_version}" # Regenerate initrd ${DRACUT_CMD} -f "/boot/initramfs-${kernel_version}.img" "${kernel_version}" diff --git a/oracle-linux-image-tools/cloud/ovm/ol9-slim/image-scripts.sh b/oracle-linux-image-tools/cloud/ovm/ol9-slim/image-scripts.sh index bec514d..773bcdd 100755 --- a/oracle-linux-image-tools/cloud/ovm/ol9-slim/image-scripts.sh +++ b/oracle-linux-image-tools/cloud/ovm/ol9-slim/image-scripts.sh @@ -2,7 +2,7 @@ # # Validate parameters # -# Copyright (c) 2023 Oracle and/or its affiliates. +# Copyright (c) 2023, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # @@ -22,6 +22,6 @@ # None ####################################### cloud_distr::validate() { - [[ "${EXTRA_KERNEL,,}" =~ ^(yes)|(no)$ ]] || error "EXTRA_KERNEL must be yes or no" + [[ "${EXTRA_KERNEL,,}" =~ ^((yes)|(no))$ ]] || common::error "EXTRA_KERNEL must be yes or no" readonly EXTRA_KERNEL } diff --git a/oracle-linux-image-tools/cloud/ovm/ol9-slim/provision.sh b/oracle-linux-image-tools/cloud/ovm/ol9-slim/provision.sh index 8e2f5b5..b36dc40 100644 --- a/oracle-linux-image-tools/cloud/ovm/ol9-slim/provision.sh +++ b/oracle-linux-image-tools/cloud/ovm/ol9-slim/provision.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # -# Packer provisioning script for OVM on OL9 +# Provisioning script for OVM on OL9 # -# Copyright (c) 2023, Oracle and/or its affiliates. +# Copyright (c) 2023, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # @@ -24,7 +24,7 @@ # None ####################################### cloud_distr::serial_cfg() { - cat > /usr/lib/systemd/system/serial_console.service <<-EOF + cat > /usr/lib/systemd/system/serial_console.service <<-'EOF' # This file is part of systemd. # # systemd is free software; you can redistribute it and/or modify it @@ -74,7 +74,7 @@ cloud_distr::serial_cfg() { # This will install RHCK if UEK is already there or the opposite # Assumes that we have a single kernel installed # Globals: -# DRACUT_CMD, KERNEL, KERNEL_MODULES, UEK_RELEASE +# DRACUT_CMD, KERNEL, KERNEL_MODULES # Arguments: # None # Returns: @@ -96,10 +96,10 @@ cloud_distr::additional_kernel() { dnf config-manager --set-enabled "ol9_UEKR7" fi - echo_message "Adding kernel: ${kernel}" + common::echo_message "Adding kernel: ${kernel}" dnf install -y ${kernel} kernel_version=$(rpm -q ${kernel} --qf "%{VERSION}-%{RELEASE}.%{ARCH}") - echo_message "Installed kernel: ${kernel_version}" + common::echo_message "Installed kernel: ${kernel_version}" # Regenerate initrd ${DRACUT_CMD} -f "/boot/initramfs-${kernel_version}.img" "${kernel_version}" diff --git a/oracle-linux-image-tools/cloud/ovm/provision.sh b/oracle-linux-image-tools/cloud/ovm/provision.sh index 9937f25..59b1564 100755 --- a/oracle-linux-image-tools/cloud/ovm/provision.sh +++ b/oracle-linux-image-tools/cloud/ovm/provision.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # -# Packer provisioning script for OVM +# Provisioning script for OVM # -# Copyright (c) 2019,2020 Oracle and/or its affiliates. +# Copyright (c) 2019, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # @@ -49,7 +49,7 @@ cloud::default_grub() ####################################### cloud::ovm_cfg() { - echo_message "Setup network" + common::echo_message "Setup network" # simple eth0 config, again not hard-coded to the build hardware cat > /etc/sysconfig/network-scripts/ifcfg-eth0 <<-EOF DEVICE="eth0" @@ -62,7 +62,7 @@ cloud::ovm_cfg() PERSISTENT_DHCLIENT="1" EOF - echo_message 'Configure grub' + common::echo_message 'Configure grub' cloud::default_grub GRUB_TIMEOUT 10 # GRUB_HIDDEN_MENU_QUIET: for historical reason, not used anymore... cloud::default_grub GRUB_HIDDEN_MENU_QUIET false @@ -85,7 +85,7 @@ cloud::ovm_cfg() ####################################### cloud::install_vmapilibxenstore() { - echo_message "Install OVM API and LibXenStore" + common::echo_message "Install OVM API and LibXenStore" if [[ "${ORACLE_RELEASE}" = "7" ]]; then yum install --enablerepo ol7_addons -y "${YUM_VERBOSE}" \ libovmapi \ diff --git a/oracle-linux-image-tools/cloud/utm/env.properties b/oracle-linux-image-tools/cloud/utm/env.properties new file mode 100644 index 0000000..c4e0e11 --- /dev/null +++ b/oracle-linux-image-tools/cloud/utm/env.properties @@ -0,0 +1,6 @@ +# Default properties for the UTM cloud. +# Do NOT change anything in this file, customisation must be done in separate +# env file. + +# Password for the OPC user (mandatory) +# OPC_PASSWORD= diff --git a/oracle-linux-image-tools/cloud/utm/image-scripts.sh b/oracle-linux-image-tools/cloud/utm/image-scripts.sh index 767940f..983e141 100755 --- a/oracle-linux-image-tools/cloud/utm/image-scripts.sh +++ b/oracle-linux-image-tools/cloud/utm/image-scripts.sh @@ -2,20 +2,42 @@ # # Cleanup and package image for the "None" image # -# Copyright (c) 2022 Oracle and/or its affiliates. +# Copyright (c) 2022, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # +# Description: this module provides the following functions which are run on +# the host: +# cloud::validate: called at the very begining to validate project parameters +# (optional) +# cloud::customize_args: arguments to pass to virt-customize (optional) +# cloud::sysprep_args: arguments to pass to virt-sysprep (optional) +# cloud::image_package: Package the raw image for the target cloud. +# This function must be defined either at cloud or cloud/distribution level +# # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. # +####################################### +# Parameter validation +# Globals: +# OPC_PASSWORD +# Arguments: +# None +# Returns: +# None +####################################### +cloud::validate() { + [[ -z "${OPC_PASSWORD}" ]] && common::error "missing OPC_PASSWORD" + readonly OPC_PASSWORD +} + ####################################### # Image packaging: # For VistualBox we convert back to VMDK and re-create the OVA file # For qemu we convert to a qcow2 file # Globals: -# CLOUD_DIR CLOUD -# VM_NAME +# CLOUD, CLOUD_DIR, VM_NAME # Arguments: # None # Returns: @@ -24,15 +46,17 @@ cloud::image_package() { local utm_dir uuid utm_dir="${VM_NAME}.utm" + pushd "${WORKSPACE}/${VM_NAME}" || common::error "can't cd to image directory" mkdir -p "${utm_dir}/Images" - common::convert_to_qcow2 "${utm_dir}/Images/${VM_NAME}.qcow2" + mv "${VM_NAME}.qcow2" "${utm_dir}/Images/${VM_NAME}.qcow2" cp "${CLOUD_DIR}/${CLOUD}/Penguin.png" "${utm_dir}" uuid=$(python3 -c "import uuid; print(str(uuid.uuid4()).upper())") sed \ -e "s/image.qcow2/${VM_NAME}.qcow2/" \ - -e "s!opc/opc!opc/${SSH_PASSWORD}!" \ + -e "s!opc/opc!opc/${OPC_PASSWORD}!" \ -e "s/00000000-0000-0000-0000-000000000000/${uuid}/" \ "${CLOUD_DIR}/${CLOUD}/config.plist" > "${utm_dir}/config.plist" zip -r "${utm_dir}.zip" "${utm_dir}" rm -rf "${utm_dir}" + popd || common::error "can't pop directory" } diff --git a/oracle-linux-image-tools/cloud/utm/provision.sh b/oracle-linux-image-tools/cloud/utm/provision.sh index dc720d7..6f73baa 100755 --- a/oracle-linux-image-tools/cloud/utm/provision.sh +++ b/oracle-linux-image-tools/cloud/utm/provision.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash # -# Packer provisioning script for UTM +# Provisioning script for UTM # -# Copyright (c) 2022 Oracle and/or its affiliates. +# Copyright (c) 2022, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # -# Description: UTM specific provisioning.n +# Description: UTM specific provisioning. # # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. # @@ -14,7 +14,7 @@ ####################################### # Provisioning module # Globals: -# None +# OPC_PASSWORD # Arguments: # None # Returns: @@ -22,7 +22,7 @@ ####################################### cloud::provision() { - hash=$(/usr/libexec/platform-python -c "import crypt; print(crypt.crypt('${SSH_PASSWORD}', crypt.METHOD_SHA512))") + hash=$(/usr/libexec/platform-python -c "import crypt; print(crypt.crypt('${OPC_PASSWORD}', crypt.METHOD_SHA512))") useradd opc -c "Oracle Public Cloud User" -G wheel -m -p "${hash}" passwd -e opc echo "%opc ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/opc diff --git a/oracle-linux-image-tools/cloud/vagrant-libvirt/image-scripts.sh b/oracle-linux-image-tools/cloud/vagrant-libvirt/image-scripts.sh index e5ae90e..a9b7fe6 100755 --- a/oracle-linux-image-tools/cloud/vagrant-libvirt/image-scripts.sh +++ b/oracle-linux-image-tools/cloud/vagrant-libvirt/image-scripts.sh @@ -6,9 +6,12 @@ # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # -# Description: this module provides 2 functions: -# cloud::image_cleanup: cloud specific actions to cleanup the image -# This function is optional +# Description: this module provides the following functions which are run on +# the host: +# cloud::validate: called at the very begining to validate project parameters +# (optional) +# cloud::customize_args: arguments to pass to virt-customize (optional) +# cloud::sysprep_args: arguments to pass to virt-sysprep (optional) # cloud::image_package: Package the raw image for the target cloud. # This function must be defined either at cloud or cloud/distribution level # @@ -18,37 +21,39 @@ ####################################### # Parameter validation # Globals: -# VAGRANT_LIBVIRT_BOX_SCRIPT +# VAGRANT_LIBVIRT_BOX_SCRIPT VAGRANT_DEVELOPER_REPOS # Arguments: # None # Returns: # None ####################################### cloud::validate() { - [[ -n ${VAGRANT_LIBVIRT_BOX_SCRIPT} && -x ${VAGRANT_LIBVIRT_BOX_SCRIPT} ]] || error "missing vagrant box_create script" - [[ ${VAGRANT_LIBVIRT_CPU_NUM} =~ ^[0-9]*$ ]] || error "vagrant cpu count is not numeric" - [[ ${VAGRANT_LIBVIRT_MEM_SIZE} =~ ^[0-9]*$ ]] || error "vagrant memory is not numeric" - [[ ${VAGRANT_DEVELOPER_REPOS,,} =~ ^(yes)|(no)$ ]] || error "VAGRANT_DEVELOPER_REPOS must be Yes or No" + [[ -n ${VAGRANT_LIBVIRT_BOX_SCRIPT} && -x ${VAGRANT_LIBVIRT_BOX_SCRIPT} ]] || + common::error "missing vagrant box_create script" + [[ ${VAGRANT_DEVELOPER_REPOS,,} =~ ^((yes)|(no))$ ]] || + common::error "VAGRANT_DEVELOPER_REPOS must be Yes or No" } ####################################### -# Cleanup actions run directly on the image +# virt-sysprep arguments # Globals: # None # Arguments: -# root filesystem directory -# boot filesystem directory +# virt-sysprep argument nameref # Returns: # None ####################################### -# cloud::image_cleanup() { -# : -# } +cloud::sysprep_args() { + declare -n sysprep_args="$1" + # Default insecure vagrant key + sysprep_args+=( --ssh-inject "vagrant:string:ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key" ) +} ####################################### # Image packaging: generate box using vagrant tool # Globals: -# VM_NAME +# CPU_NUM, MEM_SIZE, VAGRANT_LIBVIRT_BOX_SCRIPT, VAGRANT_LIBVIRT_CPU_NUM +# VAGRANT_LIBVIRT_MEM_SIZE, VM_NAME # Arguments: # None # Returns: @@ -58,22 +63,22 @@ cloud::image_package() { local cpus="${VAGRANT_LIBVIRT_CPU_NUM:-$CPU_NUM}" local memory="${VAGRANT_LIBVIRT_MEM_SIZE:-$MEM_SIZE}" - common::convert_to_qcow2 "${VM_NAME}.qcow" - + pushd "${WORKSPACE}/${VM_NAME}" || common::error "can't cd to image directory" # Defaults for the box cat > Vagrantfile <<-EOF - config.vm.provider :libvirt do |libvirt| - libvirt.memory = ${memory} - libvirt.cpus = ${cpus} - libvirt.features = ['apic', 'acpi'] - libvirt.video_vram = 16384 - end + config.vm.provider :libvirt do |libvirt| + libvirt.memory = ${memory} + libvirt.cpus = ${cpus} + libvirt.features = ['apic', 'acpi'] + libvirt.video_vram = 16384 + end - config.vm.synced_folder ".", "/vagrant", - type: "nfs", - nfs_version: 3, - nfs_udp: false -EOF - ${VAGRANT_LIBVIRT_BOX_SCRIPT} "${VM_NAME}.qcow" "${VM_NAME}.box" Vagrantfile - rm "${VM_NAME}.qcow" Vagrantfile + config.vm.synced_folder ".", "/vagrant", + type: "nfs", + nfs_version: 3, + nfs_udp: false + EOF + ${VAGRANT_LIBVIRT_BOX_SCRIPT} "${VM_NAME}.qcow2" "${VM_NAME}.box" Vagrantfile + rm "${VM_NAME}.qcow2" Vagrantfile + popd || common::error "can't pop directory" } diff --git a/oracle-linux-image-tools/cloud/vagrant-libvirt/provision.sh b/oracle-linux-image-tools/cloud/vagrant-libvirt/provision.sh index 5e3f6f8..0cf1d8b 100755 --- a/oracle-linux-image-tools/cloud/vagrant-libvirt/provision.sh +++ b/oracle-linux-image-tools/cloud/vagrant-libvirt/provision.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # -# Packer provisioning script for Vagrant-libvirt +# Provisioning script for Vagrant-libvirt # -# Copyright (c) 2020 Oracle and/or its affiliates. +# Copyright (c) 2020, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # @@ -16,7 +16,7 @@ # Load vagrant common scripts # shellcheck disable=SC1091 -source /tmp/packer_files/cloud/vagrant-common.sh +source "${PROVISION_DIR}/cloud/vagrant-common.sh" ####################################### # Configure Vagrant instance @@ -43,7 +43,7 @@ cloud::config() ####################################### cloud::install_agent() { - echo_message "Install NFS client" + common::echo_message "Install NFS client" yum install -y "${YUM_VERBOSE}" nfs-utils } diff --git a/oracle-linux-image-tools/cloud/vagrant-virtualbox/files/vagrant-common.sh b/oracle-linux-image-tools/cloud/vagrant-virtualbox/files/vagrant-common.sh index ac43fca..1fc98e8 100755 --- a/oracle-linux-image-tools/cloud/vagrant-virtualbox/files/vagrant-common.sh +++ b/oracle-linux-image-tools/cloud/vagrant-virtualbox/files/vagrant-common.sh @@ -2,7 +2,7 @@ # # Common scripts for vagrant provisioners # -# Copyright (c) 2020, 2022 Oracle and/or its affiliates. +# Copyright (c) 2020, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # @@ -12,7 +12,7 @@ ####################################### # Configure Vagrant instance # Globals: -# ORACLE_RELEASE, UEK_RELEASE +# DRACUT_CMD, ORACLE_RELEASE, UEK_RELEASE, YUM_VERBOSE # Arguments: # None # Returns: @@ -20,7 +20,7 @@ ####################################### vagrant::config() { - echo_message "Configure Vagrant" + common::echo_message "Configure Vagrant" # Add vagrant user /usr/sbin/groupadd vagrant /usr/sbin/useradd vagrant -g vagrant -G wheel @@ -58,12 +58,7 @@ vagrant::config() OPTIONS="-u0" EOF - # Default insecure vagrant key - mkdir -p /home/vagrant/.ssh - chmod 0700 /home/vagrant/.ssh - echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key" >> /home/vagrant/.ssh/authorized_keys - chmod 600 /home/vagrant/.ssh/authorized_keys - chown -R vagrant:vagrant /home/vagrant/.ssh + # Default insecure vagrant key is inserted by virt-sysprep # Fix for issue #76, regular users can gain admin privileges via su ex -s /etc/pam.d/su <<'EOF' @@ -90,7 +85,7 @@ EOF # Blacklist the floppy module to avoid probing timeouts echo blacklist floppy > /etc/modprobe.d/nofloppy.conf - chcon -u system_u -r object_r -t modules_conf_t /etc/modprobe.d/nofloppy.conf + chcon system_u:object_r:modules_conf_t:s0 /etc/modprobe.d/nofloppy.conf # Customize the initramfs if [[ "${ORACLE_RELEASE}" != "9" && "${UEK_RELEASE}" != "7" ]]; then @@ -102,9 +97,9 @@ EOF echo 'omit_drivers+=" floppy "' > /etc/dracut.conf.d/nofloppy.conf restorecon /etc/dracut.conf.d/nofloppy.conf # Regenerate initrd - local current_kernel - current_kernel=$(uname -r) - ${DRACUT_CMD} -f "/boot/initramfs-${current_kernel}.img" "${current_kernel}" + local default_kernel + default_kernel=$(common::default_kernel) + ${DRACUT_CMD} -f "/boot/initramfs-${default_kernel}.img" "${default_kernel}" # Disabling firewalld on vagrant boxes if [[ "${ORACLE_RELEASE}" = "6" ]]; then @@ -113,7 +108,7 @@ EOF service ip6tables stop chkconfig ip6tables off else - systemctl disable firewalld --now + systemctl disable firewalld fi # Install additional release packages and enable repos @@ -161,7 +156,7 @@ For additional packages, updates, documentation and community help, see: ####################################### # Cleanup module # Globals: -# ORACLE_RELEASE RESCUE_KERNEL +# RESCUE_KERNEL # Arguments: # None # Returns: diff --git a/oracle-linux-image-tools/cloud/vagrant-virtualbox/image-scripts.sh b/oracle-linux-image-tools/cloud/vagrant-virtualbox/image-scripts.sh index 4917d9b..f30b4dd 100755 --- a/oracle-linux-image-tools/cloud/vagrant-virtualbox/image-scripts.sh +++ b/oracle-linux-image-tools/cloud/vagrant-virtualbox/image-scripts.sh @@ -2,13 +2,16 @@ # # Cleanup and package image for the "vagrant-virtualbox" image # -# Copyright (c) 2020, 2022 Oracle and/or its affiliates. +# Copyright (c) 2020, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # -# Description: this module provides 2 functions: -# cloud::image_cleanup: cloud specific actions to cleanup the image -# This function is optional +# Description: this module provides the following functions which are run on +# the host: +# cloud::validate: called at the very begining to validate project parameters +# (optional) +# cloud::customize_args: arguments to pass to virt-customize (optional) +# cloud::sysprep_args: arguments to pass to virt-sysprep (optional) # cloud::image_package: Package the raw image for the target cloud. # This function must be defined either at cloud or cloud/distribution level # @@ -26,103 +29,128 @@ # None ####################################### cloud::validate() { - [[ ${VAGRANT_VIRTUALBOX_CPU_NUM} =~ ^[0-9]*$ ]] || error "vagrant cpu count is not numeric" - [[ ${VAGRANT_VIRTUALBOX_MEM_SIZE} =~ ^[0-9]*$ ]] || error "vagrant memory is not numeric" - [[ ${VAGRANT_VIRTUALBOX_EXTRA_DISK_GB} =~ ^[0-9]*$ ]] || error "vagrant disk size is not numeric" - [[ ${VAGRANT_DEVELOPER_REPOS,,} =~ ^(yes)|(no)$ ]] || error "VAGRANT_DEVELOPER_REPOS must be Yes or No" + [[ ${VAGRANT_VIRTUALBOX_CPU_NUM} =~ ^[0-9]*$ ]] || common::error "vagrant cpu count is not numeric" + [[ ${VAGRANT_VIRTUALBOX_MEM_SIZE} =~ ^[0-9]*$ ]] || common::error "vagrant memory is not numeric" + [[ ${VAGRANT_VIRTUALBOX_EXTRA_DISK_GB} =~ ^[0-9]*$ ]] || common::error "vagrant disk size is not numeric" + [[ ${VAGRANT_DEVELOPER_REPOS,,} =~ ^((yes)|(no))$ ]] || common::error "VAGRANT_DEVELOPER_REPOS must be Yes or No" + [[ -z "${VAGRANT_GUEST_ADDITIONS_URL}" ]] && common::error "missing VirtualBox GA ISO URL" + [[ ${VAGRANT_GUEST_ADDITIONS_URL%%:*} =~ ^((https?)|(file))$ ]] || common::error "invalid VirtualBox GA ISO URL: ${VAGRANT_GUEST_ADDITIONS_URL}" + [[ -z "${VAGRANT_GUEST_ADDITIONS_SHA256}" ]] && common::error "missing VirtualBox GA ISO checksum" + [[ ${#VAGRANT_GUEST_ADDITIONS_SHA256} -eq 64 ]] || common::error "VAGRANT_GUEST_ADDITIONS_SHA256 must be SHA256" + readonly VAGRANT_GUEST_ADDITIONS_URL VAGRANT_GUEST_ADDITIONS_SHA256 + # Retriece GA during validation to "fail fast" + declare -g VAGRANT_GUEST_ADDITIONS_PATH + common::retrieve_iso "${VAGRANT_GUEST_ADDITIONS_URL}" "${VAGRANT_GUEST_ADDITIONS_SHA256}" VAGRANT_GUEST_ADDITIONS_PATH + readonly VAGRANT_GUEST_ADDITIONS_PATH } ####################################### -# Packer configuration +# virt-customize arguments # Globals: -# VAGRANT_GUEST_ADDITIONS_URL +# VAGRANT_GUEST_ADDITIONS_PATH # VAGRANT_GUEST_ADDITIONS_SHA256 +# VAGRANT_GUEST_ADDITIONS_URL # Arguments: -# Packer configuration file +# virt-customize argument nameref # Returns: # None ####################################### -cloud::packer_conf() { +cloud::customize_args() { + declare -n customize_args="$1" if [[ -n "${VAGRANT_GUEST_ADDITIONS_URL}" && -n "${VAGRANT_GUEST_ADDITIONS_SHA256}" ]]; then - cat >>"$1" <<-EOF - guest_additions_url = "${VAGRANT_GUEST_ADDITIONS_URL}" - guest_additions_sha256 = "${VAGRANT_GUEST_ADDITIONS_SHA256}" - EOF + local iso_path + iso_path=$(realpath "${VAGRANT_GUEST_ADDITIONS_PATH}") + customize_args+=( --attach "${iso_path}" ) fi } ####################################### -# Cleanup actions run directly on the image +# virt-sysprep arguments # Globals: # None # Arguments: -# root filesystem directory -# boot filesystem directory +# virt-sysprep argument nameref # Returns: # None ####################################### -# cloud::image_cleanup() { -# : -# } +cloud::sysprep_args() { + declare -n sysprep_args="$1" + # Default insecure vagrant key + sysprep_args+=( --ssh-inject "vagrant:string:ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key" ) +} ####################################### # Image packaging: generate box using vagrant tool # Globals: +# CLOUD, CLOUD_DIR # ORACLE_RELEASE -# VM_NAME, VAGRANT_VIRTUALBOX_CPU, VAGRANT_VIRTUALBOX_MEMORY, +# VAGRANT_VIRTUALBOX_CPU, VAGRANT_VIRTUALBOX_MEM_SIZE, +# CPU_NUM, MEM_SIZE # VAGRANT_VIRTUALBOX_EXTRA_DISK_GB +# VM_NAME, WORKSPACE # Arguments: # None # Returns: # None ####################################### cloud::image_package() { + local mk_envelope="${CLOUD_DIR}/${CLOUD}/mk-envelope.py" local cpu="${VAGRANT_VIRTUALBOX_CPU_NUM:-$CPU_NUM}" local memory="${VAGRANT_VIRTUALBOX_MEM_SIZE:-$MEM_SIZE}" - if [[ "${ORACLE_RELEASE}" =~ ^[89]$ ]]; then - # For OL8/OL9 as we don't have image_cleanup (we use distr::seal), we can - # import directrly the saved OVA file. - rm System.img - vboxmanage import System.ova \ - --vsys 0 --vmname "${VM_NAME}" \ - --vsys 0 --ostype "Oracle_64" \ - --vsys 0 --cpus "$cpu" \ - --vsys 0 --memory "$memory" - else - # convert back to VMDK - local vmdk - vmdk=$(grep "ovf:href" "${VM_NAME}.ovf" | sed -r -e 's/.*ovf:href="([^"]+)".*/\1/') - common::convert_to_vmdk "${vmdk}" - # re-create the OVA file - common::make_ova "${VM_NAME}.ovf" "${vmdk}" - # Import in VirtualBox and adjust cpu/memory for the box - vboxmanage import "${VM_NAME}.ova" \ - --vsys 0 --vmname "${VM_NAME}" \ - --vsys 0 --ostype "Oracle_64" \ - --vsys 0 --cpus "$cpu" \ - --vsys 0 --memory "$memory" - rm "${VM_NAME}.ova" + local -a extra_disk=() + local -a file_list=( + ./Vagrantfile + ./box-disk001.vmdk + ./box.ovf + ./metadata.json + ) + + common::convert_to_vmdk "${WORKSPACE}/${VM_NAME}/box-disk001.vmdk" + if [[ -n ${VAGRANT_VIRTUALBOX_EXTRA_DISK_GB} ]]; then + qemu-img create -f vmdk "${WORKSPACE}/${VM_NAME}/box-disk002.vmdk" "${VAGRANT_VIRTUALBOX_EXTRA_DISK_GB}G" + extra_disk=( --extra-image "${WORKSPACE}/${VM_NAME}/box-disk002.vmdk" --extra-size "${VAGRANT_VIRTUALBOX_EXTRA_DISK_GB}") + file_list+=(./box-disk002.vmdk) fi - # Add additional disk - if [[ -n $VAGRANT_VIRTUALBOX_EXTRA_DISK_GB ]]; then - local disk_size_mb=$(( VAGRANT_VIRTUALBOX_EXTRA_DISK_GB * 1024 )) - vboxmanage createhd --filename ./extra_disk.vdi --size $disk_size_mb --format VDI --variant fixed - vboxmanage storageattach "${VM_NAME}" --storagectl "SATA Controller" --port 1 --device 0 --type hdd --medium ./extra_disk.vdi + + ${mk_envelope} --name "${VM_NAME}" --cpu "${cpu}" --memory "${memory}" \ + --image "${WORKSPACE}/${VM_NAME}/box-disk001.vmdk" --size "${DISK_SIZE_GB}" \ + "${extra_disk[@]}" > "${WORKSPACE}/${VM_NAME}/box.ovf" + + # Fix vmdk header + local disk_uuid + disk_uuid=$(grep '"vmdisk1"' "${WORKSPACE}/${VM_NAME}/box.ovf" | sed -e 's!.*vbox:uuid="\([^"]*\)".*!\1!') + common::fix_vmdk_header "${WORKSPACE}/${VM_NAME}/box-disk001.vmdk" "${disk_uuid}" + if [[ -n ${VAGRANT_VIRTUALBOX_EXTRA_DISK_GB} ]]; then + disk_uuid=$(grep '"vmdisk2"' "${WORKSPACE}/${VM_NAME}/box.ovf" | sed -e 's!.*vbox:uuid="\([^"]*\)".*!\1!') + common::fix_vmdk_header "${WORKSPACE}/${VM_NAME}/box-disk002.vmdk" "${disk_uuid}" fi - # Create the box + + cat > "${WORKSPACE}/${VM_NAME}/Vagrantfile" <<-EOF + Vagrant::Config.run do |config| + # This Vagrantfile is auto-generated to contain the MAC address of the box. + # Custom configuration should be placed in the actual \`Vagrantfile\` in this box. + config.vm.base_mac = "080027D25971" + end + EOF + if [[ "${ORACLE_RELEASE}" =~ ^[89]$ ]]; then # For the latest uek kernels (UEK7) we install kernel-uek-core which only has virtio drivers... - cat > Vagrantfile <<-EOF + mkdir "${WORKSPACE}/${VM_NAME}/include" + cat > "${WORKSPACE}/${VM_NAME}/include/_Vagrantfile" <<-EOF Vagrant.configure("2") do |config| config.vm.provider :virtualbox do |v| v.default_nic_type = "virtio" end end EOF - vagrant package --base "${VM_NAME}" --output "${VM_NAME}.box" --vagrantfile Vagrantfile - rm -rf Vagrantfile .vagrant - else - vagrant package --base "${VM_NAME}" --output "${VM_NAME}.box" + file_list+=(./include) fi - vboxmanage unregistervm "${VM_NAME}" --delete + + echo -n '{"provider":"virtualbox"}' >"${WORKSPACE}/${VM_NAME}/metadata.json" + + tar czvf "${WORKSPACE}/${VM_NAME}/${VM_NAME}.box" \ + -C "${WORKSPACE}/${VM_NAME}" \ + --remove-files \ + "${file_list[@]}" + } diff --git a/oracle-linux-image-tools/cloud/vagrant-virtualbox/mk-envelope.py b/oracle-linux-image-tools/cloud/vagrant-virtualbox/mk-envelope.py new file mode 100755 index 0000000..acf374b --- /dev/null +++ b/oracle-linux-image-tools/cloud/vagrant-virtualbox/mk-envelope.py @@ -0,0 +1,583 @@ +#!/usr/bin/env python3 + +""" +Generate VirtualBox OVF file. + +Copyright (c) 2020, 2024 Oracle and/or its affiliates. +Licensed under the Universal Permissive License v 1.0 as shown at +https://oss.oracle.com/licenses/upl + +DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. +""" + +import argparse +from datetime import datetime, timezone +import os.path +import random +import uuid +from xml.dom.minidom import Document + +# OS Id and type +OS_ID = 109 # 109 is OL +OS_TYPE = "Oracle_64" + + +class OvfDocument(Document): + """Add convenience method for element creation.""" + + def createOvfElement( + self, name, attr=None, text=None, parent=None, text_elements=None # noqa: N802 + ): + """Create element with optional attributes and text.""" + element = self.createElement(name) + + if parent: + parent.appendChild(element) + + if attr: + for key, value in attr.items(): + element.setAttribute(key, value) + + if text: + element.appendChild(self.createTextNode(text)) + + if text_elements: + for key, value in text_elements.items(): + self.createOvfElement(key, parent=element, text=value) + + return element + + +def get_uuid(): + """Return UUID as a string.""" + return str(uuid.uuid4()) + + +def parse_args(): + """Parse arguments.""" + parser = argparse.ArgumentParser( + description="Generate an OLVM OVF and package in an OVA library.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("-n", "--name", required=True, help="Image name") + parser.add_argument("-c", "--cpu", type=int, default=1, help="Number of VCPU") + parser.add_argument( + "-m", "--memory", type=int, default=1024, help="Memory size in MB" + ) + parser.add_argument("-i", "--image", required=True, help="Image file name") + parser.add_argument( + "-s", "--size", type=int, required=True, help="Image size in GB, e.g. 10" + ) + parser.add_argument("--extra-image", help="Optional extra image file name") + parser.add_argument( + "--extra-size", type=int, help="Optional extra image size in GB, e.g. 10" + ) + + args = parser.parse_args() + + if not os.path.isfile(args.image): + parser.error("Image file does not exists.") + + if (args.extra_image and not args.extra_size) or (not args.extra_image and args.extra_size): + parser.error("Extra image and extra size must both specify or omitted") + if args.extra_image and not os.path.isfile(args.extra_image): + parser.error("Extra image file does not exists.") + + return args + + +def generate_ovf(args): + """Generate the OVF document.""" + # Image capacity and size on disk + disk_capacity = args.size * 1024 * 1024 * 1024 + + # Random UUIDs + disk_uuid = get_uuid() + extra_disk_uuid = get_uuid() + machine_uuid = get_uuid() + mac_address = "080027{:02x}{:02x}{:02x}".format( + random.randint(0, 0xFF), random.randint(0, 0xFF), random.randint(0, 0xFF) + ) + # Timestamp for objects + iso_time = ( + datetime.now(tz=timezone.utc) + .isoformat(timespec="seconds") + .replace("+00:00", "Z") + ) + + # Files/disks references + file_ref = "file" + disk_id = "vmdisk" + + document = OvfDocument() + + # Envelope + namespaces = { + "xmlns": "http://schemas.dmtf.org/ovf/envelope/1", + "xmlns:ovf": "http://schemas.dmtf.org/ovf/envelope/1", + "xmlns:rasd": ( + "http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/" + "CIM_ResourceAllocationSettingData" + ), + "xmlns:vssd": ( + "http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/" + "CIM_VirtualSystemSettingData" + ), + "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "xmlns:vbox": "http://www.virtualbox.org/ovf/machine", + } + envelope = document.createOvfElement( + "Envelope", + parent=document, + attr={ + "ovf:version": "1.0", + "xml:lang": "en-US", + **namespaces, + }, + ) + + # Envelope / References + reference = document.createOvfElement("References", parent=envelope) + + # Envelope / References / File + document.createOvfElement( + "File", + parent=reference, + attr={ + "ovf:id": f"{file_ref}1", + "ovf:href": os.path.basename(args.image), + }, + ) + if args.extra_image: + document.createOvfElement( + "File", + parent=reference, + attr={ + "ovf:id": f"{file_ref}2", + "ovf:href": os.path.basename(args.extra_image), + }, + ) + + # Envelope / Disk Section + disk_section = document.createOvfElement("DiskSection", parent=envelope) + document.createOvfElement( + "Info", + parent=disk_section, + text="List of the virtual disks used in the package", + ) + + # Envelope / Disk Section / Disk + document.createOvfElement( + "Disk", + parent=disk_section, + attr={ + # Image size + "ovf:capacity": str(disk_capacity), + # UUID for this disk + "ovf:diskId": f"{disk_id}1", + # Ref to file (should be the "id" of fileref) + "ovf:fileRef": f"{file_ref}1", + "ovf:format": "http://www.vmware.com/interfaces/specifications/vmdk.html#streamOptimized", + "vbox:uuid": disk_uuid, + }, + ) + if args.extra_image: + document.createOvfElement( + "Disk", + parent=disk_section, + attr={ + # Image size + "ovf:capacity": str(disk_capacity), + # UUID for this disk + "ovf:diskId": f"{disk_id}2", + # Ref to file (should be the "id" of fileref) + "ovf:fileRef": f"{file_ref}2", + "ovf:format": "http://www.vmware.com/interfaces/specifications/vmdk.html", + "vbox:uuid": extra_disk_uuid, + }, + ) + + # Envelope / Network Section + network_section = document.createOvfElement("NetworkSection", parent=envelope) + document.createOvfElement( + "Info", parent=network_section, text="Logical networks used in the package" + ) + document.createOvfElement( + "Network", + parent=network_section, + attr={ + "ovf:name": "NAT", + }, + text_elements={"Description": "Logical network used by this appliance."}, + ) + + # Envelope / Virtual System + virtual_system = document.createOvfElement( + "VirtualSystem", + parent=envelope, + attr={ + "ovf:id": args.name, + }, + ) + document.createOvfElement("Info", parent=virtual_system, text="A virtual machine") + + # Envelope / Virtual System / Operating System Section + os_section = document.createOvfElement( + "OperatingSystemSection", + parent=virtual_system, + attr={ + "ovf:id": str(OS_ID), + }, + ) + + # Envelope / Virtual System / Operating System Section / Info + document.createOvfElement( + "Info", parent=os_section, text="The kind of installed guest operating system" + ) + + # Envelope / Virtual System / Operating System Section / Description + document.createOvfElement( + "Description", + parent=os_section, + text=OS_TYPE, + ) + + # Envelope / Virtual System / Operating System Section / OSType + document.createOvfElement( + "vbox:OSType", + parent=os_section, + attr={ + "ovf:required": "false", + }, + text=OS_TYPE, + ) + + # Envelope / Virtual System / Virtual Hardware Section + vh_section = document.createOvfElement( + "VirtualHardwareSection", parent=virtual_system + ) + + # Envelope / Virtual System / Virtual Hardware Section / Info + document.createOvfElement( + "Info", + parent=vh_section, + text="Virtual hardware requirements for a virtual machine", + ) + + # Envelope / Virtual System / Virtual Hardware Section / System + document.createOvfElement( + "System", + parent=vh_section, + text_elements={ + "vssd:ElementName": "Virtual Hardware Family", + "vssd:InstanceID": "0", + "vssd:VirtualSystemIdentifier": args.name, + "vssd:VirtualSystemType": "virtualbox-2.2", + }, + ) + + # Envelope / Virtual System / Virtual Hardware Section / Item CPU + instance_id = 1 + document.createOvfElement( + "Item", + parent=vh_section, + text_elements={ + "rasd:Caption": "{} virtual CPU".format(args.cpu), + "rasd:Description": "Number of virtual CPUs", + "rasd:ElementName": "{} virtual CPU".format(args.cpu), + "rasd:InstanceID": str(instance_id), + "rasd:ResourceType": "3", + "rasd:VirtualQuantity": str(args.cpu), + }, + ) + + # Envelope / Virtual System / Virtual Hardware Section / Item Memory + instance_id += 1 + document.createOvfElement( + "Item", + parent=vh_section, + text_elements={ + "rasd:AllocationUnits": "MegaBytes", + "rasd:Caption": "{} MB of memory".format(args.memory), + "rasd:Description": "Memory Size", + "rasd:ElementName": "{} MB of memory".format(args.memory), + "rasd:InstanceID": str(instance_id), + "rasd:ResourceType": "4", + "rasd:VirtualQuantity": str(args.memory), + }, + ) + + # Envelope / Virtual System / Virtual Hardware Section / IDE Controller 0 + instance_id += 1 + document.createOvfElement( + "Item", + parent=vh_section, + text_elements={ + "rasd:Address": "0", + "rasd:Caption": "ideController0", + "rasd:Description": "IDE Controller", + "rasd:ElementName": "ideController0", + "rasd:InstanceID": str(instance_id), + "rasd:ResourceSubType": "PIIX4", + "rasd:ResourceType": "5", + }, + ) + + # Envelope / Virtual System / Virtual Hardware Section / IDE Controller 1 + instance_id += 1 + document.createOvfElement( + "Item", + parent=vh_section, + text_elements={ + "rasd:Address": "1", + "rasd:Caption": "ideController1", + "rasd:Description": "IDE Controller", + "rasd:ElementName": "ideController1", + "rasd:InstanceID": str(instance_id), + "rasd:ResourceSubType": "PIIX4", + "rasd:ResourceType": "5", + }, + ) + + # Envelope / Virtual System / Virtual Hardware Section / SATA Controller 0 + instance_id += 1 + document.createOvfElement( + "Item", + parent=vh_section, + text_elements={ + "rasd:Address": "0", + "rasd:Caption": "sataController0", + "rasd:Description": "SATA Controller", + "rasd:ElementName": "sataController0", + "rasd:InstanceID": str(instance_id), + "rasd:ResourceSubType": "AHCI", + "rasd:ResourceType": "20", + }, + ) + + # Envelope / Virtual System / Virtual Hardware Section / Disk + instance_id += 1 + document.createOvfElement( + "Item", + parent=vh_section, + text_elements={ + "rasd:AddressOnParent": "0", + "rasd:Caption": "disk1", + "rasd:Description": "Disk Image", + "rasd:ElementName": "disk1", + "rasd:HostResource": f"/disk/{disk_id}1", + "rasd:InstanceID": str(instance_id), + "rasd:Parent": "5", + "rasd:ResourceType": "17", + }, + ) + if args.extra_image: + instance_id += 1 + document.createOvfElement( + "Item", + parent=vh_section, + text_elements={ + "rasd:AddressOnParent": "1", + "rasd:Caption": "disk2", + "rasd:Description": "Disk Image", + "rasd:ElementName": "disk2", + "rasd:HostResource": f"/disk/{disk_id}2", + "rasd:InstanceID": str(instance_id), + "rasd:Parent": "5", + "rasd:ResourceType": "17", + }, + ) + + # Envelope / Virtual System / Virtual Hardware Section / Item Network + instance_id += 1 + document.createOvfElement( + "Item", + parent=vh_section, + text_elements={ + "rasd:AutomaticAllocation": "true", + "rasd:Caption": "Ethernet adapter on 'NAT'", + "rasd:Connection": "NAT", + "rasd:ElementName": "Ethernet adapter on 'NAT'", + "rasd:InstanceID": str(instance_id), + "rasd:ResourceType": "10", + }, + ) + + # Envelope / Virtual System / Machine Section + machine_section = document.createOvfElement( + "vbox:Machine", + parent=virtual_system, + attr={ + "ovf:required": "false", + "version": "1.19-linux", + "uuid": f"{{{machine_uuid}}}", + "name": args.name, + "OSType": OS_TYPE, + "snapshotFolder": "Snapshots", + "lastStateChange": iso_time, + }, + ) + + # Envelope / Virtual System / Machine Section / Info + document.createOvfElement( + "ovf:Info", + parent=machine_section, + text="Complete VirtualBox machine configuration in VirtualBox format", + ) + + # Envelope / Virtual System / Machine Section / Hardware + ms_hardware = document.createOvfElement( + "Hardware", + parent=machine_section, + ) + + ms_hardware_cpu = document.createOvfElement( + "CPU", parent=ms_hardware, attr={"count": str(args.cpu)} + ) + document.createOvfElement("PAE", parent=ms_hardware_cpu, attr={"enabled": "true"}) + document.createOvfElement( + "LongMode", parent=ms_hardware_cpu, attr={"enabled": "true"} + ) + document.createOvfElement( + "X2APIC", parent=ms_hardware_cpu, attr={"enabled": "true"} + ) + document.createOvfElement( + "HardwareVirtExLargePages", parent=ms_hardware_cpu, attr={"enabled": "true"} + ) + + document.createOvfElement( + "Memory", parent=ms_hardware, attr={"RAMSize": str(args.memory)} + ) + + ms_hardware_boot = document.createOvfElement("Boot", parent=ms_hardware) + document.createOvfElement( + "Order", parent=ms_hardware_boot, attr={"position": "1", "device": "HardDisk"} + ) + document.createOvfElement( + "Order", parent=ms_hardware_boot, attr={"position": "2", "device": "DVD"} + ) + document.createOvfElement( + "Order", parent=ms_hardware_boot, attr={"position": "3", "device": "None"} + ) + document.createOvfElement( + "Order", parent=ms_hardware_boot, attr={"position": "4", "device": "None"} + ) + + document.createOvfElement("Display", parent=ms_hardware, attr={"VRAMSize": "4"}) + + ms_hardware_rd = document.createOvfElement( + "RemoteDisplay", parent=ms_hardware, attr={"enabled": "true"} + ) + ms_hardware_rd_vrde = document.createOvfElement( + "VRDEProperties", parent=ms_hardware_rd + ) + document.createOvfElement( + "Property", + parent=ms_hardware_rd_vrde, + attr={"name": "TCP/Address", "value": "127.0.0.1"}, + ) + document.createOvfElement( + "Property", + parent=ms_hardware_rd_vrde, + attr={"name": "TCP/Ports", "value": "5905"}, + ) + + ms_hardware_bios = document.createOvfElement("BIOS", parent=ms_hardware) + document.createOvfElement( + "IOAPIC", parent=ms_hardware_bios, attr={"enabled": "true"} + ) + document.createOvfElement( + "SmbiosUuidLittleEndian", parent=ms_hardware_bios, attr={"enabled": "true"} + ) + + document.createOvfElement( + "NAT", + attr={"localhost-reachable": "true"}, + parent=document.createOvfElement( + "Adapter", + attr={ + "slot": "0", + "enabled": "true", + "MACAddress": mac_address, + "type": "virtio", + }, + parent=document.createOvfElement("Network", parent=ms_hardware), + ), + ) + + document.createOvfElement( + "AudioAdapter", parent=ms_hardware, attr={"driver": "Null"} + ) + + document.createOvfElement("Clipboard", parent=ms_hardware) + + ms_hardware_storage = document.createOvfElement( + "StorageControllers", parent=ms_hardware + ) + document.createOvfElement( + "StorageController", + parent=ms_hardware_storage, + attr={ + "name": "IDE Controller", + "type": "PIIX4", + "PortCount": "2", + "useHostIOCache": "true", + "Bootable": "true", + }, + ) + ms_hardware_storage_sata = document.createOvfElement( + "StorageController", + parent=ms_hardware_storage, + attr={ + "name": "SATA Controller", + "type": "AHCI", + "PortCount": "2" if args.extra_image else "1", + "useHostIOCache": "false", + "Bootable": "true", + "IDE0MasterEmulationPort": "0", + "IDE0SlaveEmulationPort": "1", + "IDE1MasterEmulationPort": "2", + "IDE1SlaveEmulationPort": "3", + }, + ) + document.createOvfElement( + "Image", + attr={"uuid": f"{{{disk_uuid}}}"}, + parent=document.createOvfElement( + "AttachedDevice", + attr={ + "type": "HardDisk", + "hotpluggable": "false", + "port": "0", + "device": "0", + }, + parent=ms_hardware_storage_sata, + ), + ) + if args.extra_image: + document.createOvfElement( + "Image", + attr={"uuid": f"{{{extra_disk_uuid}}}"}, + parent=document.createOvfElement( + "AttachedDevice", + attr={ + "type": "HardDisk", + "hotpluggable": "false", + "port": "1", + "device": "0", + }, + parent=ms_hardware_storage_sata, + ), + ) + + return document.toprettyxml(indent=" ") + + +def main(): + """Make envelope.""" + args = parse_args() + print(generate_ovf(args), end="") + + +if __name__ == "__main__": + main() diff --git a/oracle-linux-image-tools/cloud/vagrant-virtualbox/provision.sh b/oracle-linux-image-tools/cloud/vagrant-virtualbox/provision.sh index 4fbfc7b..0c34d12 100755 --- a/oracle-linux-image-tools/cloud/vagrant-virtualbox/provision.sh +++ b/oracle-linux-image-tools/cloud/vagrant-virtualbox/provision.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # -# Packer provisioning script for Vagrant-VirtualBox +# Provisioning script for Vagrant-VirtualBox # -# Copyright (c) 2020, 2022 Oracle and/or its affiliates. +# Copyright (c) 2020, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # @@ -16,7 +16,7 @@ # Load vagrant common scripts # shellcheck disable=SC1091 -source /tmp/packer_files/cloud/vagrant-common.sh +source "${PROVISION_DIR}/cloud/vagrant-common.sh" ####################################### # Configure Vagrant instance @@ -35,7 +35,7 @@ cloud::config() ####################################### # Install Virtualbox guest agent # Globals: -# KERNEL, YUM_VERBOSE +# YUM_VERBOSE # Arguments: # None # Returns: @@ -43,7 +43,7 @@ cloud::config() ####################################### cloud::install_agent() { - echo_message "Install guest agent" + common::echo_message "Install Guest Additions" local additions="/mnt/VBoxLinuxAdditions.run" yum install -y "${YUM_VERBOSE}" make gcc bzip2 tar @@ -54,32 +54,36 @@ cloud::install_agent() fi # Orabug 34811820 for OL8 UEK7 -- for the current install - case $(uname -r) in + case $(common::default_kernel) in 5.15.0-*.el8uek*) export PATH="/opt/rh/gcc-toolset-11/root/usr/bin:$PATH" esac - # Search for guest additions on cd devices - for cdrom in /dev/sr*; do - if mount -o ro "${cdrom}" /mnt; then + # Search for guest additions ISO -- it is typically labeled VBox_... + # Note: use "blkid -s" as "--match-tag" is not suported on OL7 + local label + for label in $(/sbin/blkid -s LABEL -o value | grep VBox_); do + if mount -o ro LABEL="${label}" /mnt; then if [[ -f ${additions} ]]; then # Found! break else - echo_message "No guest additions on ${cdrom}" + common::echo_message "No guest additions on ${label}" + umount /mnt fi else - echo_message "No media in ${cdrom}" + common::echo_message "Cannot mount ${label}" fi done - [[ -f ${additions} ]] || echo_error "Guest additions not found" + [[ -f ${additions} ]] || common::error "Guest additions not found" + # Installation will fail when running in libguestfs environment sh "${additions}" || : umount /mnt # Orabug 34811820 for OL8 UEK7 -- for subsequent rebuilds - case $(uname -r) in + case $(common::default_kernel) in 5.15.0-*.el8uek*) # shellcheck disable=SC2016 sed -i '/PATH=$PATH/a PATH="/opt/rh/gcc-toolset-11/root/usr/bin:$PATH"' /usr/sbin/rcvboxadd @@ -88,6 +92,12 @@ cloud::install_agent() done esac + # Ensure modules are built for the target kernel + if [[ $(uname -r) != $(common::default_kernel) ]]; then + common::echo_message "Building Guest Additions for $(common::default_kernel)" + /sbin/rcvboxadd quicksetup "$(common::default_kernel)" + fi + } ####################################### diff --git a/oracle-linux-image-tools/custom/template/files/custom.txt b/oracle-linux-image-tools/custom/template/files/custom.txt index c231780..d4a6c7c 100644 --- a/oracle-linux-image-tools/custom/template/files/custom.txt +++ b/oracle-linux-image-tools/custom/template/files/custom.txt @@ -1,3 +1,3 @@ Custom file example -This file will be uploaded in the /tmp/packer_files/custom/ directory inside +This file will be uploaded in the /tmp/provision.d/custom/ directory inside the VM diff --git a/oracle-linux-image-tools/custom/template/image-scripts.sh b/oracle-linux-image-tools/custom/template/image-scripts.sh index 1f1d9fb..c1c64e8 100755 --- a/oracle-linux-image-tools/custom/template/image-scripts.sh +++ b/oracle-linux-image-tools/custom/template/image-scripts.sh @@ -6,8 +6,7 @@ # the host: # custom::validate: called at the very begining to validate project paramters # custom::kickstart: allow changes in the kickstart file -# custom::packer_conf: allow changes in the packer configuration file -# custom::image_cleanup: image cleanup actions after build completes +# custom::customize_args: arguments to pass to virt-cutomize (optional) # custom::image_package: package image in final format (override the cloud # image_package function!) # All functions are optional @@ -24,8 +23,8 @@ # None ####################################### custom::validate() { - echo_message "Running ${FUNCNAME[0]}" - # [[ "${CUSTOM_YES_NO_PARAMETER,,}" =~ ^(yes)|(no)$ ]] || error "CUSTOM_YES_NO_PARAMETER must be yes or no" + common::echo_message "Running ${FUNCNAME[0]}" + # [[ "${CUSTOM_YES_NO_PARAMETER,,}" =~ ^((yes)|(no))$ ]] || common::error "CUSTOM_YES_NO_PARAMETER must be yes or no" # readonly CUSTOM_YES_NO_PARAMETER } @@ -39,51 +38,11 @@ custom::validate() { # None ####################################### custom::kickstart() { - echo_message "Running ${FUNCNAME[0]}" + common::echo_message "Running ${FUNCNAME[0]}" # local ks_file="$1" # Use `sed` to modify the kickstart file } -####################################### -# Packer configuration -# Globals: -# All build parameters are available -# Arguments: -# Packer configuration file -# Returns: -# None -####################################### -custom::packer_conf() { - echo_message "Running ${FUNCNAME[0]}" - # local packer_file="$1" - # Use `sed` to modify the packer configuration file -} - -####################################### -# Cleanup actions run directly on the image -# Globals: -# All build parameters are available -# Arguments: -# root filesystem directory -# boot filesystem directory -# Returns: -# None -####################################### -custom::image_cleanup() { - local root_fs="$1" - local boot_fs="$2" - - echo_message "Running ${FUNCNAME[0]}" - - # Ensure we don't blindly cleanup local host! - [[ -z ${root_fs} ]] && error "Undefined root filesystem" - [[ -z ${boot_fs} ]] && error "Undefined boot filesystem" - - # Cleanup actions -- use `chroot` to execute actions in the context of - # the image. - # See examples in distr image-scripts.sh files -} - ####################################### # Image packaging # Warning: when this function is defined it will orverride the cloud @@ -98,11 +57,5 @@ custom::image_cleanup() { # None ####################################### # custom::image_package() { -# if common::is_vbox ; then -# # VirtualBox Packer builder -# : -# else -# # QEmu Packer builder -# : -# fi +# : # } diff --git a/oracle-linux-image-tools/custom/template/provision.sh b/oracle-linux-image-tools/custom/template/provision.sh index 1efdeda..4a9d173 100644 --- a/oracle-linux-image-tools/custom/template/provision.sh +++ b/oracle-linux-image-tools/custom/template/provision.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Packer provisioning script template for custom projects +# Provisioning script template for custom projects # # Description: provision an custom image. This module provides 2 functions, # which are run inside the VM: @@ -19,8 +19,8 @@ # None ####################################### custom::provision() { - echo_message "Running ${FUNCNAME[0]} for ${PROJECT_NAME} (${DISTR}/${CLOUD})" - cat /tmp/packer_files/custom/custom.txt + common::echo_message "Running ${FUNCNAME[0]} for ${PROJECT_NAME} (${DISTR}/${CLOUD})" + cat "${PROVISION_DIR}/custom/custom.txt" } ####################################### @@ -33,5 +33,5 @@ custom::provision() { # None ####################################### custom::cleanup() { - echo_message "Running ${FUNCNAME[0]}" + common::echo_message "Running ${FUNCNAME[0]}" } diff --git a/oracle-linux-image-tools/distr/ol7-slim/env.properties b/oracle-linux-image-tools/distr/ol7-slim/env.properties index f041e67..dd64eb2 100644 --- a/oracle-linux-image-tools/distr/ol7-slim/env.properties +++ b/oracle-linux-image-tools/distr/ol7-slim/env.properties @@ -14,9 +14,29 @@ SETUP_SWAP="yes" # Root filesystem: btrfs or xfs ROOT_FS="xfs" +# Location of the kernel and initrd on the distribution image. +BOOT_LOCATION="isolinux" + +# Boot mode +BOOT_MODE="bios" # Boot command # Variables MUST be escaped as they are evaluated at build time. -BOOT_COMMAND=( '${CONSOLE} text ks=${KS_CONFIG} setup_swap=${SETUP_SWAP} ' ) +# Following variables are available: +# KS_FILE: name of the kickstart file which will be in the root directory +# ISO_LABEL: label of the ISO image +BOOT_COMMAND=( + 'inst.text' + 'inst.ks=file:/${KS_FILE}' + 'inst.geoloc=0' + 'inst.stage2=hd:LABEL=${ISO_LABEL}' + 'net.ifnames=0' +) +# Additional parameters to enable serial console +BOOT_COMMAND_SERIAL_CONSOLE=( + 'console=tty0' + 'console=ttyS0' + 'inst.notmux' +) # Kernel: uek, rhck or modrhck KERNEL="uek" diff --git a/oracle-linux-image-tools/distr/ol7-slim/image-scripts.sh b/oracle-linux-image-tools/distr/ol7-slim/image-scripts.sh index c5e8aa1..ecdb3d4 100755 --- a/oracle-linux-image-tools/distr/ol7-slim/image-scripts.sh +++ b/oracle-linux-image-tools/distr/ol7-slim/image-scripts.sh @@ -2,13 +2,14 @@ # # image scripts for OL7 # -# Copyright (c) 2019,2022 Oracle and/or its affiliates. +# Copyright (c) 2019, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # -# Description: this module provides a single function: -# distr::image_cleanup: distribution specific actions to cleanup the image -# This function is optional +# Description: this module provides the following function: +# distr::validate: basic parameter validation +# distr::kickstart: hook for kickstart file updates +# All functions are optional # # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. # @@ -16,26 +17,28 @@ ####################################### # Validate distribution parameters # Globals: -# ROOT_FS TMP_IN_TMPFS UEK_RELEASE LINUX_FIRMWARE STRIP_LOCALES EXCLUDE_DOCS +# LINUX_FIRMWARE, ROOT_FS, TMP_IN_TMPFS, UEK_RELEASE +# STRIP_LOCALES, EXCLUDE_DOCS # Arguments: # None # Returns: # None ####################################### distr::validate() { - [[ "${ROOT_FS,,}" =~ ^(xfs)|(btrfs)|(lvm)$ ]] || error "ROOT_FS must be xfs, btrfs or lvm" - [[ "${TMP_IN_TMPFS,,}" =~ ^(yes)|(no)$ ]] || error "TMP_IN_TMPFS must be yes or no" - [[ "${UEK_RELEASE}" =~ ^[56]$ ]] || error "UEK_RELEASE must be 5 or 6" - [[ "${LINUX_FIRMWARE,,}" =~ ^(yes)|(no)$ ]] || error "LINUX_FIRMWARE must be yes or no" - [[ "${STRIP_LOCALES,,}" =~ ^(yes)|(no)$ ]] || error "STRIP_LOCALES must be yes or no" - [[ "${EXCLUDE_DOCS,,}" =~ ^(yes)|(no)|(minimal)$ ]] || error "EXCLUDE_DOCS must be yes, no or minimal" + [[ "${ROOT_FS,,}" =~ ^((xfs)|(btrfs)|(lvm))$ ]] || common::error "ROOT_FS must be xfs, btrfs or lvm" + [[ "${TMP_IN_TMPFS,,}" =~ ^((yes)|(no))$ ]] || common::error "TMP_IN_TMPFS must be yes or no" + [[ "${UEK_RELEASE}" =~ ^[56]$ ]] || common::error "UEK_RELEASE must be 5 or 6" + [[ "${LINUX_FIRMWARE,,}" =~ ^((yes)|(no))$ ]] || common::error "LINUX_FIRMWARE must be yes or no" + [[ "${STRIP_LOCALES,,}" =~ ^((yes)|(no))$ ]] || common::error "STRIP_LOCALES must be yes or no" + [[ "${EXCLUDE_DOCS,,}" =~ ^((yes)|(no)|(minimal))$ ]] || common::error "EXCLUDE_DOCS must be yes, no or minimal" readonly ROOT_FS TMP_IN_TMPFS UEK_RELEASE LINUX_FIRMWARE STRIP_LOCALES EXCLUDE_DOCS } ####################################### # Kickcstart fixup # Globals: -# ROOT_FS +# KERNEL, ROOT_FS, UEK_RELEASE +# EXCLUDE_DOCS, STRIP_LOCALES, TMP_IN_TMPFS # Arguments: # kickstart file name # Returns: @@ -78,40 +81,5 @@ logvol / --fstype=\"xfs\" --vgname=vg_main --size=4096 --name=lv_root --gr fi # /tmp in tmpfs - sed -i -e "s!^TMP_IN_TMPFS=no!TMP_IN_TMPFS=$TMP_IN_TMPFS!" "${ks_file}" -} - -####################################### -# Cleanup actions run directly on the image -# Globals: -# WORKSPACE VM_NAME BUILD_INFO -# Arguments: -# root filesystem directory -# boot filesystem directory -# Returns: -# None -####################################### -distr::image_cleanup() { - local root_fs="$1" - local boot_fs="$2" - - # Ensure we don't blindly cleanup local host! - [[ -z ${root_fs} ]] && error "Undefined root filesystem" - [[ -z ${boot_fs} ]] && error "Undefined boot filesystem" - - if [[ -n ${BUILD_INFO} && -d "${root_fs}${BUILD_INFO}" ]]; then - find "${root_fs}${BUILD_INFO}" -type f -exec cp {} "${WORKSPACE}/${VM_NAME}/" \; - fi - - sudo chroot "${root_fs}" /bin/bash <<-EOF - : > /var/log/wtmp - : > /var/log/lastlog - rm -f /var/log/audit/audit.log - rm -f /var/log/tuned/tuned.log - rm -rf /root/.gemrc /root/.gem - rm -rf /var/spool/root /var/spool/mail/root - rm -rf /var/lib/NetworkManager - rm -rf /var/tmp/* - [[ -n "${BUILD_INFO}" ]] && rm -rf "${BUILD_INFO}" - EOF + sed -i -e "s!^TMP_IN_TMPFS=no!TMP_IN_TMPFS=${TMP_IN_TMPFS}!" "${ks_file}" } diff --git a/oracle-linux-image-tools/distr/ol7-slim/ol7-ks.cfg b/oracle-linux-image-tools/distr/ol7-slim/ol7-ks.cfg index bc6e499..117c615 100644 --- a/oracle-linux-image-tools/distr/ol7-slim/ol7-ks.cfg +++ b/oracle-linux-image-tools/distr/ol7-slim/ol7-ks.cfg @@ -220,6 +220,8 @@ rpm -qa | grep '^kernel' | sort kernel="kernel" yum_options="-d1" +# We should not have "ol7_ociyum_config" repo +yum-config-manager --disable ol7_ociyum_config >/dev/null 2>&1 # Ensure we have the latest release file yum upgrade -y "${yum_options}" oraclelinux-release-el7 diff --git a/oracle-linux-image-tools/distr/ol7-slim/provision.sh b/oracle-linux-image-tools/distr/ol7-slim/provision.sh index a32f6b4..719dc76 100644 --- a/oracle-linux-image-tools/distr/ol7-slim/provision.sh +++ b/oracle-linux-image-tools/distr/ol7-slim/provision.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # -# Packer provisioning script for OL7 +# Provisioning script for OL7 # -# Copyright (c) 2019,2022 Oracle and/or its affiliates. +# Copyright (c) 2019, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # @@ -30,48 +30,49 @@ distr::remove_rpms() { yum -C -y "${YUM_VERBOSE}" remove "$@" --setopt="clean_requirements_on_remove=1" } -####################################### -# Print kickstart log -# Globals: -# Arguments: -# None -# Returns: -# None -####################################### -distr::ks_log() { - if [[ -f "/root/ks-post.log" ]]; then - echo_message "Kickstart post log - Start" - cat /root/ks-post.log - rm /root/ks-post.log - echo_message "Kickstart post log - End" - fi -} - ####################################### # Kernel configuration # Assume that we already run the latest selected kernel # (Asserted in the kickstart file) # Globals: -# DRACUT_CMD, KERNEL, UPDATE_TO_LATEST, YUM_VERBOSE +# DRACUT_CMD, KERNEL, LINUX_FIRMWARE, UEK_RELEASE, UPDATE_TO_LATEST, YUM_VERBOSE # Arguments: # None # Returns: # None ####################################### distr::kernel_config() { - local current_kernel kernel kernels old_kernel + local target_kernel # shellcheck disable=SC2153 - echo_message "Configure kernel: ${KERNEL^^}" - echo_message "Running kernel: $(uname -r)" + common::echo_message "Configure kernel: ${KERNEL^^}" + + # Configure repos and remove old kernels + yum-config-manager --disable ol7_UEKR\* >/dev/null + if [[ "${KERNEL,,}" = "modrhck" ]]; then + yum-config-manager --enable ol7_MODRHCK >/dev/null + fi + + if [[ "${KERNEL,,}" = "uek" ]]; then + target_kernel=$(common::latest_kernel kernel-uek) + common::echo_message "Target kernel: ${target_kernel}" + yum-config-manager --enable "ol7_UEKR${UEK_RELEASE}" >/dev/null + yum install -y "${YUM_VERBOSE}" kernel-transition + common::remove_kernels kernel + common::remove_kernels kernel-uek "${target_kernel}" + else + target_kernel=$(common::latest_kernel kernel) + common::echo_message "Target kernel: ${target_kernel}" + common::remove_kernels kernel-uek + common::remove_kernels kernel "${target_kernel}" + fi + # Add virtual drivers for xen,virtualbox and hyperv into the initrd using # dracut configuration files so that they get installed into the initrd # during fresh kernel installs. # This makes it is easy to move VM images between these virtual environments - - # Available virtio modules depends on kernel build... local virtio modules - modules=$(find "/lib/modules/$(uname -r)" -name "virtio*.ko*" -printf '%f\n') + modules=$(find "/lib/modules/${target_kernel}" -name "virtio*.ko*" -printf '%f\n') while read -r module; do virtio="${virtio} ${module%.ko*}" done <<<"${modules}" @@ -83,38 +84,14 @@ distr::kernel_config() { add_drivers+=" ahci libahci " EOF - # Configure repos and remove old kernels - yum-config-manager --disable ol7_UEKR\* >/dev/null - if [[ "${KERNEL,,}" = "modrhck" ]]; then - yum-config-manager --enable ol7_MODRHCK >/dev/null - fi - - if [[ "${KERNEL,,}" = "uek" ]]; then - kernel="kernel-uek" - yum-config-manager --enable "ol7_UEKR${UEK_RELEASE}" >/dev/null - yum install -y "${YUM_VERBOSE}" kernel-transition - yum remove -y "${YUM_VERBOSE}" kernel - else - kernel="kernel" - yum remove -y "${YUM_VERBOSE}" kernel-uek - fi - - current_kernel=$(uname -r) - kernels=$(rpm -q ${kernel} --qf "%{VERSION}-%{RELEASE}.%{ARCH} ") - for old_kernel in $kernels; do - if [[ ${old_kernel} != "${current_kernel}" ]]; then - yum remove -y "${kernel}-${old_kernel}" - fi - done - # Regenerate initrd - ${DRACUT_CMD} -f "/boot/initramfs-${current_kernel}.img" "${current_kernel}" + ${DRACUT_CMD} -f "/boot/initramfs-${target_kernel}.img" "${target_kernel}" # Ensure grub is properly setup grub2-mkconfig -o /boot/grub2/grub.cfg - grubby --set-default="/boot/vmlinuz-${current_kernel}" + grubby --set-default="/boot/vmlinuz-${target_kernel}" - echo_message "Linux firmware: ${LINUX_FIRMWARE^^}" + common::echo_message "Linux firmware: ${LINUX_FIRMWARE^^}" if [[ "${LINUX_FIRMWARE,,}" = "no" ]]; then yum remove -y linux-firmware fi @@ -123,7 +100,7 @@ distr::kernel_config() { ####################################### # Common configuration # Globals: -# UPDATE_TO_LATEST, YUM_VERBOSE, BUILD_INFO +# BUILD_INFO, PERMIT_ROOT_LOGIN, SELINUX, UPDATE_TO_LATEST, YUM_VERBOSE # Arguments: # None # Returns: @@ -139,7 +116,7 @@ distr::common_cfg() { yum-config-manager --disable ol7_ociyum_config >/dev/null 2>&1 # Run yum update if flag is set to yes in image build page - echo_message "Update image: ${UPDATE_TO_LATEST^^}" + common::echo_message "Update image: ${UPDATE_TO_LATEST^^}" if [[ "${UPDATE_TO_LATEST,,}" = "yes" ]]; then yum update -y "${YUM_VERBOSE}" elif [[ "${UPDATE_TO_LATEST,,}" = "security" ]]; then @@ -147,20 +124,27 @@ distr::common_cfg() { yum update --security -y "${YUM_VERBOSE}" fi + common::echo_message "sshd root login policy: ${PERMIT_ROOT_LOGIN}" + ex -s /etc/ssh/sshd_config <<-EOF + :%substitute/^#\?\(PermitRootLogin\) .*$/\1 ${PERMIT_ROOT_LOGIN,,}/ + :update + :quit + EOF + # TODO: Do we really want to use RH servers? sed -i -e '/^server .*/d' /etc/chrony.conf sed -i -e '/joining the pool/a \server 0.rhel.pool.ntp.org iburst \n\server 1.rhel.pool.ntp.org iburst \n\server 2.rhel.pool.ntp.org iburst \n\server 3.rhel.pool.ntp.org iburst' /etc/chrony.conf # If you want to remove rsyslog and just use journald, remove this! - echo_message "Disabling persistent journal" + common::echo_message "Disabling persistent journal" rm -rf /var/log/journal/ # setup systemd to boot to the right runlevel - echo_message "Setting default runlevel to multiuser text mode" + common::echo_message "Setting default runlevel to multiuser text mode" rm -f /etc/systemd/system/default.target ln -s /lib/systemd/system/multi-user.target /etc/systemd/system/default.target - echo_message "Disable services" + common::echo_message "Disable services" for service in \ kdump.service \ ntpd.service \ @@ -174,41 +158,37 @@ distr::common_cfg() { NetworkManager.service do # Most of these aren't enabled, errors are expected... - echo_message " ${service}" + common::echo_message " ${service}" systemctl disable ${service} 2>&1 || true done - echo_message "Set rp_filter to loose mode" + common::echo_message "Set rp_filter to loose mode" echo "net.ipv4.conf.default.rp_filter = 2" >> /etc/sysctl.conf - echo_message "Set SELinux to ${SELINUX^^}" + common::echo_message "Set SELinux to ${SELINUX^^}" sed -i -e "s/^SELINUX[ ]*=.*/SELINUX=${SELINUX,,}/" /etc/selinux/config if [[ ${SELINUX,,} != "enforcing" ]]; then # Relax SELinux for the provisioning as well setenforce Permissive fi - echo_message "Clear network persistent data" + common::echo_message "Clear network persistent data" rm -f /etc/udev/rules.d/70-persistent-net.rules - echo_message "Configure yum" + common::echo_message "Configure yum" # bypass update kernel-uek-headers echo "exclude=kernel-uek-headers" >> /etc/yum.conf # fix "Metadata file does not match checksum" for public-yum # https://forums.oracle.com/thread/2550364 echo "http_caching=none" >> /etc/yum.conf - echo_message "Enable login on serial console ports" + common::echo_message "Enable login on serial console ports" for tty in "hvc0" "ttyS0" "ttyS0" do grep -q "${tty}" /etc/securetty || echo "${tty}" >>/etc/securetty done - # TODO: simplify -- this is done as well in cleanup()! - # 27601618 - set the machine-id file - sed -i -e 's@^ExecStart=.*@ExecStart=/usr/bin/systemd-firstboot --prompt-locale --prompt-timezone --prompt-root-password --setup-machine-id@g' /usr/lib/systemd/system/systemd-firstboot.service - - echo_message "Remove unneeded RPMs" + common::echo_message "Remove unneeded RPMs" distr::remove_rpms \ NetworkManager \ NetworkManager-team \ @@ -234,7 +214,7 @@ distr::common_cfg() { # None ####################################### distr::provision() { - distr::ks_log + common::ks_log distr::kernel_config distr::common_cfg } @@ -242,149 +222,20 @@ distr::provision() { ####################################### # Cleanup # Globals: -# BUILD_INFO +# STRIP_LOCALES # Arguments: # None # Returns: # None ####################################### distr::cleanup() { - echo_message "Stoppping services" - systemctl stop rsyslog || true - systemctl stop auditd || true - - echo_message "Remove orabackup files" - find /etc -name "*.orabackup*" -exec rm -rf {} \; - find /boot -name "*.orabackup*" -exec rm -rf {} \; - - echo_message "Remove leftover firewall rules" - if [[ -f /etc/sysconfig/iptables ]]; then - sed -i -e '/-p 50/d' /etc/sysconfig/iptables - sed -i -e '/-p 51/d' /etc/sysconfig/iptables - sed -i -e '/--dport 5353/d' /etc/sysconfig/iptables - sed -i -e '/--dport 631/d' /etc/sysconfig/iptables - fi - if [[ -f /etc/sysconfig/ip6tables ]]; then - sed -i -e '/-p 50/d' /etc/sysconfig/ip6tables - sed -i -e '/-p 51/d' /etc/sysconfig/ip6tables - sed -i -e '/--dport 5353/d' /etc/sysconfig/ip6tables - sed -i -e '/--dport 631/d' /etc/sysconfig/ip6tables - fi - - echo_message "Remove MAC addresses" - for f in /etc/sysconfig/network-scripts/ifcfg-eth*; do - [ -e "$f" ] && sed -i '/^HWADDR=/d' "$f" - done - - echo_message "Yum cleanup" - yum -q repolist > "${BUILD_INFO}/repolist.txt" - : > /etc/yum/vars/ociregion - echo "oracle.com" > /etc/yum/vars/ocidomain - rm -rf /var/cache/yum/* - rm -rf /var/lib/yum/* - find /etc/ -name "./*.uln-*" -exec rm -rf {} \; - - # Cleanup and regenerate /etc/machine-id - # Todo -- already done in provisioning! - echo_message "Reset machine id" - : > /etc/machine-id - if ! grep -q setup-machine-id /usr/lib/systemd/system/systemd-firstboot.service; then - sed -i.old -e "/^ExecStart=/s/$/ --setup-machine-id/" /usr/lib/systemd/system/systemd-firstboot.service - fi - echo_message "Cleanup all log files" - rm -f /var/log/anaconda.* /var/log/oraclevm-template.log - rm -f /tmp/ks* - rm -f /root/install.log /root/install.log.syslog /root/anaconda-ks.cfg - : > /etc/resolv.conf - /bin/rm -f /etc/resolv.conf.* - /bin/rm -f /var/lib/dhclient/* - [ -e /var/log/acpid ] && : > /var/log/acpid - [ -e /var/log/messages ] && : > /var/log/messages - [ -e /var/log/btmp ] && : > /var/log/btmp - [ -e /var/log/grubby ] && : > /var/log/grubby - [ -e /var/log/secure ] && : > /var/log/secure - [ -e /var/log/wtmp ] && : > /var/log/wtmp - [ -e /var/log/boot.log ] && : > /var/log/boot.log - [ -e /var/log/dracut.log ] && : > /var/log/dracut.log - [ -e /var/log/tuned/tuned.log ] && : > /var/log/tuned/tuned.log - [ -e /var/log/maillog ] && : > /var/log/maillog - [ -e /var/log/lastlog ] && : > /var/log/lastlog - [ -e /var/log/yum.log ] && : > /var/log/yum.log - [ -e /var/log/ovm-template-config.log ] && rm -f /var/log/ovm-template-config.log - /bin/rm -f /var/log/audit/audit.log* - [ -e /var/log/audit/audit.log ] && : > /var/log/audit/audit.log - - # Lock root user - if [[ "${LOCK_ROOT,,}" = "yes" ]]; then - passwd -d root - passwd -l root - fi - - # cleanup ssh config files - if [ -z "${SSH_KEY_FILE}" ]; then - [ -d /root/.ssh ] && /bin/rm -fr /root/.ssh - else - find /root/.ssh -type f -not -name authorized_keys -delete - fi - - # Rebuild rpmdb to save some space - rpm --rebuilddb - - # Remove man and info pages - echo_message "Exclude documentation: ${EXCLUDE_DOCS^^}" - if [[ "${EXCLUDE_DOCS,,}" = "minimal" ]]; then - rm -rf /usr/share/{man,info} - fi + common::distr_cleanup - echo_message "Strip locales: ${STRIP_LOCALES^^}" + common::echo_message "Strip locales: ${STRIP_LOCALES^^}" if [[ "${STRIP_LOCALES,,}" = "yes" ]]; then # Remove unused locale files find /usr/share/locale -mindepth 1 -maxdepth 1 -type d \ -not -name en_US -a -not -name C \ -exec rm -rf {} + fi - - # cleanup vnc cache files - if [ -d /root/.vnc ]; then - /bin/rm -f /root/.vnc/*.log - /bin/rm -f /root/.vnc/passwd - fi - - rm -rf /var/log/cups/error_log - rm -rf /var/log/setroubleshoot/setroubleshootd.log - rm -rf /var/log/spooler - # cleanup bash history - [ -e /root/.bash_history ] && : > /root/.bash_history - rm -f /root/.viminfo - rm -rf /.autorelabel - rm -rf /var/log/mail/statistics - rm -rf /var/log/sa/* - rm -rf /var/log/acpid /var/log/boot.log /var/log/cron /var/log/dmesg.* /var/log/ovm* - rm -rf /poweroff - rm -rf /tmp/* - rm -f /etc/ssh/ssh_host_* - rm -rf /var/lib/yum/* - rm -rf /root/* - rm -f /etc/udev/rules.d/70-persistent-net.rules - rm -f /etc/udev/rules.d/70-persistent-cd.rules - - find /var/log -type f | while read -r f; do echo -ne '' > "$f"; done; - find /etc/ -name "*.old" -exec rm -f {} \; - rm -f /etc/sysconfig/network-scripts/ifcfg-enp* - rm -rf /lost+found/* - rm -rf /root/.vbox_version - export HISTSIZE=0 - rm -f /var/log/ovm-template-config.log - - echo_message "Save list of installed packages" - rpm -qa --qf "%{name}.%{arch}\n" | sort -u > "${BUILD_INFO}/pkglist.txt" - rpm -qa --qf '"%{NAME}","%{EPOCHNUM}","%{VERSION}","%{RELEASE}","%{ARCH}"\n' | sort > "${BUILD_INFO}/pkglist.csv" - uname -r > "${BUILD_INFO}/kernel.txt" - - echo_message "Relabel SELinux" - genhomedircon - fixfiles -f -F relabel - restorecon -R / - history -c - swapoff -a } diff --git a/oracle-linux-image-tools/distr/ol8-aarch64/env.properties b/oracle-linux-image-tools/distr/ol8-aarch64/env.properties index 931bd19..88df571 100644 --- a/oracle-linux-image-tools/distr/ol8-aarch64/env.properties +++ b/oracle-linux-image-tools/distr/ol8-aarch64/env.properties @@ -14,18 +14,28 @@ SETUP_SWAP="yes" # Root filesystem: xfs, lvm or btrfs ROOT_FS="xfs" -# Label of the ISO image -ISO_LABEL="OL-8-9-0-BaseOS-aarch64" +# Location of the kernel and initrd on the distribution image. +BOOT_LOCATION="images/pxeboot" +# Boot mode +BOOT_MODE="uefi" # Boot command # Variables MUST be escaped as they are evaluated at build time. +# Following variables are available: +# KS_FILE: name of the kickstart file which will be in the root directory +# ISO_LABEL: label of the ISO image BOOT_COMMAND=( - 'c' - 'linux /images/pxeboot/vmlinuz inst.stage2=hd:LABEL=${ISO_LABEL} ro ' - 'inst.text inst.notmux inst.ks=${KS_CONFIG} setup_swap=${SETUP_SWAP} ' - 'biosdevname=0 net.ifnames=0' - 'initrd /images/pxeboot/initrd.img' - 'boot' + 'inst.text' + 'inst.ks=file:/${KS_FILE}' + 'inst.geoloc=0' + 'inst.stage2=hd:LABEL=${ISO_LABEL}' + 'ro', + 'biosdevname=0', + 'net.ifnames=0' +) +# Additional parameters to enable serial console +BOOT_COMMAND_SERIAL_CONSOLE=( + 'inst.notmux' ) # Kernel: must be UEK! diff --git a/oracle-linux-image-tools/distr/ol8-aarch64/image-scripts.sh b/oracle-linux-image-tools/distr/ol8-aarch64/image-scripts.sh index da40e93..d0f1d77 100755 --- a/oracle-linux-image-tools/distr/ol8-aarch64/image-scripts.sh +++ b/oracle-linux-image-tools/distr/ol8-aarch64/image-scripts.sh @@ -2,14 +2,13 @@ # # image scripts for OL8 - aarch64 # -# Copyright (c) 2021,2022 Oracle and/or its affiliates. +# Copyright (c) 2021, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # # Description: this module provides the following function: # distr::validate: basic parameter validation # distr::kickstart: hook for kickstart file updates -# distr::image_cleanup: distribution specific actions to cleanup the image # All functions are optional # # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. @@ -18,46 +17,29 @@ ####################################### # Validate distribution parameters # Globals: -# ROOT_FS TMP_IN_TMPFS UEK_RELEASE RESCUE_KERNEL ISO_LABEL LINUX_FIRMWARE KERNEL_MODULES EXCLUDE_DOCS +# KERNEL_MODULES, LINUX_FIRMWARE, ROOT_FS, RESCUE_KERNEL, TMP_IN_TMPFS, UEK_RELEASE, EXCLUDE_DOCS # Arguments: # None # Returns: # None ####################################### distr::validate() { - [[ "${ROOT_FS,,}" =~ ^(xfs)|(btrfs)|(lvm)$ ]] || error "ROOT_FS must be xfs, btrfs or lvm" - [[ "${ROOT_FS,,}" = "btrfs" ]] && echo_message "Note that for btrfs root filesystem you need to use an UEK boot ISO" - [[ "${TMP_IN_TMPFS,,}" =~ ^(yes)|(no)$ ]] || error "TMP_IN_TMPFS must be yes or no" - [[ "${UEK_RELEASE}" =~ ^[67]$ ]] || error "UEK_RELEASE must be 6 or 7" - [[ "${RESCUE_KERNEL,,}" =~ ^(yes)|(no)$ ]] || error "RESCUE_KERNEL must be yes or no" - [[ -n ${ISO_LABEL} ]] || error "ISO_LABEL must be provided" - [[ "${LINUX_FIRMWARE,,}" =~ ^(yes)|(no)$ ]] || error "LINUX_FIRMWARE must be yes or no" - [[ "${KERNEL_MODULES,,}" =~ ^(yes)|(no)$ ]] || error "KERNEL_MODULES must be yes or no" - [[ "${EXCLUDE_DOCS,,}" =~ ^(yes)|(no)|(minimal)$ ]] || error "EXCLUDE_DOCS must be yes, no or minimal" - readonly ROOT_FS TMP_IN_TMPFS UEK_RELEASE RESCUE_KERNEL ISO_LABEL LINUX_FIRMWARE KERNEL_MODULES EXCLUDE_DOCS -} - -####################################### -# Packer configuration -# Globals: -# None -# Arguments: -# Packer configuration file -# Returns: -# None -####################################### -distr::packer_conf() { - if [[ -c /dev/kvm && $(uname -m) == "aarch64" ]]; then - cat >>"$1" <<-EOF - accel = "kvm" - EOF - fi + [[ "${ROOT_FS,,}" =~ ^((xfs)|(btrfs)|(lvm))$ ]] || common::error "ROOT_FS must be xfs, btrfs or lvm" + [[ "${ROOT_FS,,}" = "btrfs" ]] && common::echo_message "Note that for btrfs root filesystem you need to use an UEK boot ISO" + [[ "${TMP_IN_TMPFS,,}" =~ ^((yes)|(no))$ ]] || common::error "TMP_IN_TMPFS must be yes or no" + [[ "${UEK_RELEASE}" =~ ^[67]$ ]] || common::error "UEK_RELEASE must be 6 or 7" + [[ "${RESCUE_KERNEL,,}" =~ ^((yes)|(no))$ ]] || common::error "RESCUE_KERNEL must be yes or no" + [[ "${LINUX_FIRMWARE,,}" =~ ^((yes)|(no))$ ]] || common::error "LINUX_FIRMWARE must be yes or no" + [[ "${KERNEL_MODULES,,}" =~ ^((yes)|(no))$ ]] || common::error "KERNEL_MODULES must be yes or no" + [[ "${EXCLUDE_DOCS,,}" =~ ^((yes)|(no)|(minimal))$ ]] || common::error "EXCLUDE_DOCS must be yes, no or minimal" + readonly ROOT_FS TMP_IN_TMPFS UEK_RELEASE RESCUE_KERNEL LINUX_FIRMWARE KERNEL_MODULES EXCLUDE_DOCS } ####################################### # Kickcstart fixup # Globals: -# RESCUE_KERNEL ROOT_FS +# AUTHSELECT, KERNEL, RESCUE_KERNEL, ROOT_FS, UEK_RELEASE +# EXCLUDE_DOCS, STRIP_LOCALES, TMP_IN_TMPFS # Arguments: # kickstart file name # Returns: @@ -104,40 +86,5 @@ logvol / --fstype=\"xfs\" --vgname=vg_main --size=4096 --name=lv_root --gr fi # /tmp in tmpfs - sed -i -e "s!^TMP_IN_TMPFS=no!TMP_IN_TMPFS=$TMP_IN_TMPFS!" "${ks_file}" -} - -####################################### -# Cleanup actions run directly on the image -# Globals: -# WORKSPACE VM_NAME BUILD_INFO -# Arguments: -# root filesystem directory -# boot filesystem directory -# Returns: -# None -####################################### -distr::image_cleanup() { - local root_fs="$1" - local boot_fs="$2" - - # Ensure we don't blindly cleanup local host! - [[ -z ${root_fs} ]] && error "Undefined root filesystem" - [[ -z ${boot_fs} ]] && error "Undefined boot filesystem" - - if [[ -n ${BUILD_INFO} && -d "${root_fs}${BUILD_INFO}" ]]; then - find "${root_fs}${BUILD_INFO}" -type f -exec cp {} "${WORKSPACE}/${VM_NAME}/" \; - fi - - sudo chroot "${root_fs}" /bin/bash <<-EOF - : > /var/log/wtmp - : > /var/log/lastlog - rm -f /var/log/audit/audit.log - rm -f /var/log/tuned/tuned.log - rm -rf /root/.gemrc /root/.gem - rm -rf /var/spool/root /var/spool/mail/root - rm -rf /var/lib/NetworkManager - rm -rf /var/tmp/* - [[ -n "${BUILD_INFO}" ]] && rm -rf "${BUILD_INFO}" - EOF + sed -i -e "s!^TMP_IN_TMPFS=no!TMP_IN_TMPFS=${TMP_IN_TMPFS}!" "${ks_file}" } diff --git a/oracle-linux-image-tools/distr/ol8-aarch64/ol8-aarch64-ks.cfg b/oracle-linux-image-tools/distr/ol8-aarch64/ol8-aarch64-ks.cfg index c69bad9..e842ad3 100644 --- a/oracle-linux-image-tools/distr/ol8-aarch64/ol8-aarch64-ks.cfg +++ b/oracle-linux-image-tools/distr/ol8-aarch64/ol8-aarch64-ks.cfg @@ -118,7 +118,7 @@ glibc-langpack-en -gnupg2-smime -hardlink -ipcalc --kpartx +# -kpartx -libmaxminddb -libxkbcommon -memstrack diff --git a/oracle-linux-image-tools/distr/ol8-aarch64/provision.sh b/oracle-linux-image-tools/distr/ol8-aarch64/provision.sh index df19667..6eadda0 100644 --- a/oracle-linux-image-tools/distr/ol8-aarch64/provision.sh +++ b/oracle-linux-image-tools/distr/ol8-aarch64/provision.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # -# Packer provisioning script for OL8 - aarch64 +# Provisioning script for OL8 - aarch64 # -# Copyright (c) 2021,2022 Oracle and/or its affiliates. +# Copyright (c) 2021, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # @@ -31,87 +31,57 @@ distr::remove_rpms() { dnf -C -y remove "$@" } -####################################### -# Print kickstart log -# Globals: -# Arguments: -# None -# Returns: -# None -####################################### -distr::ks_log() { - if [[ -f "/root/ks-post.log" ]]; then - echo_message "Kickstart post log - Start" - cat /root/ks-post.log - rm /root/ks-post.log - echo_message "Kickstart post log - End" - fi -} - ####################################### # Kernel configuration # Assume that we already run the latest selected kernel # (Asserted in the kickstart file) # Globals: -# DRACUT_CMD, KERNEL, UPDATE_TO_LATEST +# DRACUT_CMD, KERNEL, LINUX_FIRMWARE, SETUP_SWAP, UEK_RELEASE, UPDATE_TO_LATEST # Arguments: # None # Returns: # None ####################################### distr::kernel_config() { - local current_kernel kernel kernels old_kernel - - # Ensure swap is properly formatted (UEK6/7 change) - if [[ ${SETUP_SWAP,,} == "yes" ]]; then - swapon -a || swapon -a --fixpgsz - fi + local target_kernel # shellcheck disable=SC2153 - echo_message "Configure kernel: ${KERNEL^^}" - echo_message "Running kernel: $(uname -r)" + common::echo_message "Configure kernel: ${KERNEL^^}" # Note: there is no need to force drivers in intrd as dracut-config-generic # is installed - # Remove old kernels + # Configure repos and remove old kernels + target_kernel=$(common::latest_kernel kernel-uek) + common::echo_message "Target kernel: ${target_kernel}" dnf config-manager --disable ol8_UEKR\* || : if [[ ${UEK_RELEASE} != 6 ]]; then # UEK R6 doesn't have its own repo dnf config-manager --enable "ol8_UEKR${UEK_RELEASE}" fi + common::remove_kernels kernel-uek "${target_kernel}" - current_kernel=$(uname -r) - for kernel in "kernel-uek" "kernel-uek-core"; do - if kernels=$(rpm -q ${kernel} --qf "%{VERSION}-%{RELEASE}.%{ARCH} "); then - for old_kernel in $kernels; do - if [[ ${old_kernel} != "${current_kernel}" ]]; then - distr::remove_rpms "${kernel}-${old_kernel}" - fi - done - fi - done if [[ ${UEK_RELEASE} != 6 ]]; then if [[ ${KERNEL_MODULES,,} == "no" ]]; then - echo_message "Removing kernel modules" + common::echo_message "Removing kernel modules" distr::remove_rpms kernel-uek-modules else - echo_message "Ensure kernel modules are installed" + common::echo_message "Ensure kernel modules are installed" dnf install -y kernel-uek fi fi # Regenerate initrd - ${DRACUT_CMD} -f "/boot/initramfs-${current_kernel}.img" "${current_kernel}" + ${DRACUT_CMD} -f "/boot/initramfs-${target_kernel}.img" "${target_kernel}" # Ensure grub is properly setup - grub2-mkconfig -o /etc/grub2-efi.cfg - grubby --set-default="/boot/vmlinuz-${current_kernel}" + : grub2-mkconfig -o /etc/grub2-efi.cfg + : grubby --set-default="/boot/vmlinuz-${target_kernel}" - echo_message "Linux firmware: ${LINUX_FIRMWARE^^}" + common::echo_message "Linux firmware: ${LINUX_FIRMWARE^^}" if [[ "${LINUX_FIRMWARE,,}" = "no" ]]; then - echo_message "Removing linux firmware" + common::echo_message "Removing linux firmware" distr::remove_rpms linux-firmware fi } @@ -119,42 +89,51 @@ distr::kernel_config() { ####################################### # Common configuration # Globals: -# UPDATE_TO_LATEST, BUILD_INFO +# BUILD_INFO, PERMIT_ROOT_LOGIN, SELINUX, UPDATE_TO_LATEST # Arguments: # None # Returns: # None ####################################### -distr::common_cfg() { +distr::configure() { local service # Directory to save build information mkdir -p "${BUILD_INFO}" # Run dnf update if flag is set to yes in image build page - echo_message "Update image: ${UPDATE_TO_LATEST^^}" + common::echo_message "Update image: ${UPDATE_TO_LATEST^^}" if [[ "${UPDATE_TO_LATEST,,}" = "yes" ]]; then dnf update -y elif [[ "${UPDATE_TO_LATEST,,}" = "security" ]]; then dnf update --security -y fi + common::echo_message "sshd root login policy: ${PERMIT_ROOT_LOGIN}" + # Unused anaconda leftover + rm -f /etc/sysconfig/sshd-permitrootlogin + ex -s /etc/ssh/sshd_config <<-EOF + :%substitute/^#\?\(PermitRootLogin\) .*$/\1 ${PERMIT_ROOT_LOGIN,,}/ + :update + :quit + EOF + # SSSD profile needs clients if authselect current -r | grep -q '^sssd'; then - echo_message "Installing SSSD client" + common::echo_message "Installing SSSD client" dnf install -y sssd-client fi # If you want to remove rsyslog and just use journald, remove this! - echo_message "Disabling persistent journal" + common::echo_message "Disabling persistent journal" rm -rf /var/log/journal/ # setup systemd to boot to the right runlevel - echo_message "Setting default runlevel to multiuser text mode" + common::echo_message "Setting default runlevel to multiuser text mode" rm -f /etc/systemd/system/default.target ln -s /lib/systemd/system/multi-user.target /etc/systemd/system/default.target - echo_message "Disable services" + common::echo_message "Disable services" # NetworkManager.service for service in \ kdump.service \ @@ -168,35 +147,31 @@ distr::common_cfg() { syslog.target do # Most of these aren't enabled, errors are expected... - echo_message " ${service}" + common::echo_message " ${service}" systemctl disable ${service} 2>&1 || true done - echo_message "Set rp_filter to loose mode" + common::echo_message "Set rp_filter to loose mode" echo "net.ipv4.conf.default.rp_filter = 2" >> /etc/sysctl.conf - echo_message "Set SELinux to ${SELINUX^^}" + common::echo_message "Set SELinux to ${SELINUX^^}" sed -i -e "s/^SELINUX[ ]*=.*/SELINUX=${SELINUX,,}/" /etc/selinux/config if [[ ${SELINUX,,} != "enforcing" ]]; then # Relax SELinux for the provisioning as well setenforce Permissive fi - echo_message "Clear network persistent data" + common::echo_message "Clear network persistent data" rm -f /etc/udev/rules.d/70-persistent-net.rules - echo_message "Configure dnf" + common::echo_message "Configure dnf" # bypass update kernel-uek-headers echo "exclude=kernel-uek-headers" >> /etc/dnf/dnf.conf # fix "Metadata file does not match checksum" for public-yum # https://forums.oracle.com/thread/2550364 echo "http_caching=none" >> /etc/dnf/dnf.conf - # TODO: simplify -- this is done as well in cleanup()! - # 27601618 - set the machine-id file - sed -i -e 's@^ExecStart=.*@ExecStart=/usr/bin/systemd-firstboot --prompt-locale --prompt-timezone --prompt-root-password --setup-machine-id@g' /usr/lib/systemd/system/systemd-firstboot.service - - echo_message "Remove unneeded RPMs" + common::echo_message "Remove unneeded RPMs" distr::remove_rpms \ iwl7265-firmware \ mozjs17 \ @@ -214,151 +189,20 @@ distr::common_cfg() { # None ####################################### distr::provision() { - distr::ks_log + common::ks_log distr::kernel_config - distr::common_cfg + distr::configure } ####################################### # Cleanup # Globals: -# BUILD_INFO +# None # Arguments: # None # Returns: # None ####################################### distr::cleanup() { - echo_message "Stoppping services" - systemctl stop rsyslog || true - systemctl stop auditd || true - - echo_message "Remove orabackup files" - find /etc -name "*.orabackup*" -exec rm -rf {} \; - find /boot -name "*.orabackup*" -exec rm -rf {} \; - - echo_message "Remove leftover firewall rules" - if [[ -f /etc/sysconfig/iptables ]]; then - sed -i -e '/-p 50/d' /etc/sysconfig/iptables - sed -i -e '/-p 51/d' /etc/sysconfig/iptables - sed -i -e '/--dport 5353/d' /etc/sysconfig/iptables - sed -i -e '/--dport 631/d' /etc/sysconfig/iptables - fi - if [[ -f /etc/sysconfig/ip6tables ]]; then - sed -i -e '/-p 50/d' /etc/sysconfig/ip6tables - sed -i -e '/-p 51/d' /etc/sysconfig/ip6tables - sed -i -e '/--dport 5353/d' /etc/sysconfig/ip6tables - sed -i -e '/--dport 631/d' /etc/sysconfig/ip6tables - fi - - echo_message "Remove MAC addresses" - for f in /etc/sysconfig/network-scripts/ifcfg-eth*; do - [ -e "$f" ] && sed -i '/^HWADDR=/d' "$f" - done - - echo_message "Dnf cleanup" - dnf -q repolist > "${BUILD_INFO}/repolist.txt" - : > /etc/dnf/vars/ociregion - echo "oracle.com" > /etc/dnf/vars/ocidomain - rm -rf /var/cache/dnf/* - rm -rf /var/lib/dnf/* - find /etc/ -name "./*.uln-*" -exec rm -rf {} \; - - # Cleanup and regenerate /etc/machine-id - # Todo -- already done in provisioning! - echo_message "Reset machine id" - : > /etc/machine-id - if ! grep -q setup-machine-id /usr/lib/systemd/system/systemd-firstboot.service; then - sed -i.old -e "/^ExecStart=/s/$/ --setup-machine-id/" /usr/lib/systemd/system/systemd-firstboot.service - fi - - echo_message "Cleanup all log files" - rm -f /var/log/anaconda.* /var/log/oraclevm-template.log - rm -f /tmp/ks* - rm -f /root/install.log /root/install.log.syslog /root/anaconda-ks.cfg - : > /etc/resolv.conf - /bin/rm -f /etc/resolv.conf.* - /bin/rm -f /var/lib/dhclient/* - [ -e /var/log/acpid ] && : > /var/log/acpid - [ -e /var/log/messages ] && : > /var/log/messages - [ -e /var/log/btmp ] && : > /var/log/btmp - [ -e /var/log/grubby ] && : > /var/log/grubby - [ -e /var/log/secure ] && : > /var/log/secure - [ -e /var/log/wtmp ] && : > /var/log/wtmp - [ -e /var/log/boot.log ] && : > /var/log/boot.log - [ -e /var/log/dracut.log ] && : > /var/log/dracut.log - [ -e /var/log/tuned/tuned.log ] && : > /var/log/tuned/tuned.log - [ -e /var/log/maillog ] && : > /var/log/maillog - [ -e /var/log/lastlog ] && : > /var/log/lastlog - [ -e /var/log/dnf.log ] && : > /var/log/dnf.log - [ -e /var/log/dnf.librepo.log ] && : > /var/log/dnf.librepo.log - [ -e /var/log/dnf.rpm.log ] && : > /var/log/dnf.rpm.log - [ -e /var/log/ovm-template-config.log ] && rm -f /var/log/ovm-template-config.log - /bin/rm -f /var/log/audit/audit.log* - [ -e /var/log/audit/audit.log ] && : > /var/log/audit/audit.log - - # Lock root user - if [[ "${LOCK_ROOT,,}" = "yes" ]]; then - passwd -d root - passwd -l root - fi - - # cleanup ssh config files - if [ -z "${SSH_KEY_FILE}" ]; then - [ -d /root/.ssh ] && /bin/rm -fr /root/.ssh - else - find /root/.ssh -type f -not -name authorized_keys -delete - fi - - # Rebuild rpmdb to save some space - rpm --rebuilddb - - # Remove man and info pages - echo_message "Exclude documentation: ${EXCLUDE_DOCS^^}" - if [[ "${EXCLUDE_DOCS,,}" = "minimal" ]]; then - rm -rf /usr/share/{man,info} - fi - - # cleanup vnc cache files - if [ -d /root/.vnc ]; then - /bin/rm -f /root/.vnc/*.log - /bin/rm -f /root/.vnc/passwd - fi - - rm -rf /var/log/cups/error_log - rm -rf /var/log/setroubleshoot/setroubleshootd.log - rm -rf /var/log/spooler - # cleanup bash history - [ -e /root/.bash_history ] && : > /root/.bash_history - rm -f /root/.viminfo - rm -rf /.autorelabel - rm -rf /var/log/mail/statistics - rm -rf /var/log/sa/* - rm -rf /var/log/acpid /var/log/boot.log /var/log/cron /var/log/dmesg.* /var/log/ovm* - rm -rf /poweroff - rm -rf /tmp/* - rm -f /etc/ssh/ssh_host_* - rm -rf /root/* - rm -f /etc/udev/rules.d/70-persistent-net.rules - rm -f /etc/udev/rules.d/70-persistent-cd.rules - - find /var/log -type f | while read -r f; do echo -ne '' > "$f"; done; - find /etc/ -name "*.old" -exec rm -f {} \; - rm -f /etc/sysconfig/network-scripts/ifcfg-enp* - rm -rf /lost+found/* - rm -rf /root/.vbox_version - export HISTSIZE=0 - rm -f /var/log/ovm-template-config.log - - echo_message "Save list of installed packages" - rpm -qa --qf "%{name}.%{arch}\n" | sort -u > "${BUILD_INFO}/pkglist.txt" - rpm -qa --qf '"%{NAME}","%{EPOCHNUM}","%{VERSION}","%{RELEASE}","%{ARCH}"\n' | sort > "${BUILD_INFO}/pkglist.csv" - uname -r > "${BUILD_INFO}/kernel.txt" - - echo_message "Relabel SELinux" - genhomedircon - fixfiles -f -F relabel - restorecon -R / || true - history -c - swapoff -a + common::distr_cleanup } diff --git a/oracle-linux-image-tools/distr/ol8-slim/env.properties b/oracle-linux-image-tools/distr/ol8-slim/env.properties index 2e4f63f..205b9cc 100644 --- a/oracle-linux-image-tools/distr/ol8-slim/env.properties +++ b/oracle-linux-image-tools/distr/ol8-slim/env.properties @@ -14,9 +14,29 @@ SETUP_SWAP="yes" # Root filesystem: xfs, lvm or btrfs ROOT_FS="xfs" +# Location of the kernel and initrd on the distribution image. +BOOT_LOCATION="isolinux" + +# Boot mode +BOOT_MODE="bios" # Boot command # Variables MUST be escaped as they are evaluated at build time. -BOOT_COMMAND=( '${CONSOLE} inst.text inst.ks=${KS_CONFIG} setup_swap=${SETUP_SWAP} ' ) +# Following variables are available: +# KS_FILE: name of the kickstart file which will be in the root directory +# ISO_LABEL: label of the ISO image +BOOT_COMMAND=( + 'inst.text' + 'inst.ks=file:/${KS_FILE}' + 'inst.geoloc=0' + 'inst.stage2=hd:LABEL=${ISO_LABEL}' + 'net.ifnames=0' +) +# Additional parameters to enable serial console +BOOT_COMMAND_SERIAL_CONSOLE=( + 'console=tty0' + 'console=ttyS0' + 'inst.notmux' +) # Kernel: uek, rhck KERNEL="uek" diff --git a/oracle-linux-image-tools/distr/ol8-slim/image-scripts.sh b/oracle-linux-image-tools/distr/ol8-slim/image-scripts.sh index afbe1af..a7bf380 100755 --- a/oracle-linux-image-tools/distr/ol8-slim/image-scripts.sh +++ b/oracle-linux-image-tools/distr/ol8-slim/image-scripts.sh @@ -2,15 +2,13 @@ # # image scripts for OL8 # -# Copyright (c) 2020, 2022 Oracle and/or its affiliates. +# Copyright (c) 2020, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # # Description: this module provides the following function: # distr::validate: basic parameter validation # distr::kickstart: hook for kickstart file updates -# distr::packer_conf: hook for packer configuration file updates -# distr::image_cleanup: distribution specific actions to cleanup the image # All functions are optional # # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. @@ -19,28 +17,29 @@ ####################################### # Validate distribution parameters # Globals: -# ROOT_FS TMP_IN_TMPFS UEK_RELEASE RESCUE_KERNEL KERNEL_MODULES LINUX_FIRMWARE EXCLUDE_DOCS +# KERNEL_MODULES, LINUX_FIRMWARE, ROOT_FS, RESCUE_KERNEL, TMP_IN_TMPFS, UEK_RELEASE, EXCLUDE_DOCS # Arguments: # None # Returns: # None ####################################### distr::validate() { - [[ "${ROOT_FS,,}" =~ ^(xfs)|(btrfs)|(lvm)$ ]] || error "ROOT_FS must be xfs, btrfs or lvm" - [[ "${ROOT_FS,,}" = "btrfs" ]] && echo_message "Note that for btrfs root filesystem you need to use an UEK boot ISO" - [[ "${TMP_IN_TMPFS,,}" =~ ^(yes)|(no)$ ]] || error "TMP_IN_TMPFS must be yes or no" - [[ "${UEK_RELEASE}" =~ ^[67]$ ]] || error "UEK_RELEASE must be 6 or 7" - [[ "${RESCUE_KERNEL,,}" =~ ^(yes)|(no)$ ]] || error "RESCUE_KERNEL must be yes or no" - [[ "${KERNEL_MODULES,,}" =~ ^(yes)|(no)$ ]] || error "KERNEL_MODULES must be yes or no" - [[ "${LINUX_FIRMWARE,,}" =~ ^(yes)|(no)$ ]] || error "LINUX_FIRMWARE must be yes or no" - [[ "${EXCLUDE_DOCS,,}" =~ ^(yes)|(no)|(minimal)$ ]] || error "EXCLUDE_DOCS must be yes, no or minimal" + [[ "${ROOT_FS,,}" =~ ^((xfs)|(btrfs)|(lvm))$ ]] || common::error "ROOT_FS must be xfs, btrfs or lvm" + [[ "${ROOT_FS,,}" = "btrfs" ]] && common::echo_message "Note that for btrfs root filesystem you need to use an UEK boot ISO" + [[ "${TMP_IN_TMPFS,,}" =~ ^((yes)|(no))$ ]] || common::error "TMP_IN_TMPFS must be yes or no" + [[ "${UEK_RELEASE}" =~ ^[67]$ ]] || common::error "UEK_RELEASE must be 6 or 7" + [[ "${RESCUE_KERNEL,,}" =~ ^((yes)|(no))$ ]] || common::error "RESCUE_KERNEL must be yes or no" + [[ "${KERNEL_MODULES,,}" =~ ^((yes)|(no))$ ]] || common::error "KERNEL_MODULES must be yes or no" + [[ "${LINUX_FIRMWARE,,}" =~ ^((yes)|(no))$ ]] || common::error "LINUX_FIRMWARE must be yes or no" + [[ "${EXCLUDE_DOCS,,}" =~ ^((yes)|(no)|(minimal))$ ]] || common::error "EXCLUDE_DOCS must be yes, no or minimal" readonly ROOT_FS TMP_IN_TMPFS UEK_RELEASE RESCUE_KERNEL KERNEL_MODULES LINUX_FIRMWARE EXCLUDE_DOCS } ####################################### # Kickcstart fixup # Globals: -# RESCUE_KERNEL ROOT_FS +# AUTHSELECT, KERNEL, RESCUE_KERNEL, ROOT_FS, UEK_RELEASE +# EXCLUDE_DOCS, STRIP_LOCALES, TMP_IN_TMPFS # Arguments: # kickstart file name # Returns: @@ -86,36 +85,5 @@ logvol / --fstype=\"xfs\" --vgname=vg_main --size=4096 --name=lv_root --gr fi # /tmp in tmpfs - sed -i -e "s!^TMP_IN_TMPFS=no!TMP_IN_TMPFS=$TMP_IN_TMPFS!" "${ks_file}" + sed -i -e "s!^TMP_IN_TMPFS=no!TMP_IN_TMPFS=${TMP_IN_TMPFS}!" "${ks_file}" } - -####################################### -# Packer configuration -# Globals: -# BUILD_INFO -# Arguments: -# Packer configuration file -# Returns: -# None -####################################### -distr::packer_conf() { - if [[ -n "${BUILD_INFO}" ]]; then - cat >>"$1" <<-EOF - build_info = "${BUILD_INFO}" - EOF - fi -} - -####################################### -# Cleanup actions run directly on the image -# Globals: -# WORKSPACE VM_NAME BUILD_INFO -# Arguments: -# root filesystem directory -# boot filesystem directory -# Returns: -# None -####################################### -# distr::image_cleanup_no() { -# : -# } diff --git a/oracle-linux-image-tools/distr/ol8-slim/provision.sh b/oracle-linux-image-tools/distr/ol8-slim/provision.sh index 84bcf6e..234e192 100644 --- a/oracle-linux-image-tools/distr/ol8-slim/provision.sh +++ b/oracle-linux-image-tools/distr/ol8-slim/provision.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # -# Packer provisioning script for OL8 +# Provisioning script for OL8 # -# Copyright (c) 2019, 2022 Oracle and/or its affiliates. +# Copyright (c) 2019, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # @@ -10,7 +10,6 @@ # both are optional. # distr::provision: provision the instance # distr::cleanup: instance cleanup before shutdown -# distr::seal: final instance sealing # # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. # @@ -32,93 +31,68 @@ distr::remove_rpms() { dnf -C -y remove "$@" } -####################################### -# Print kickstart log -# Globals: -# Arguments: -# None -# Returns: -# None -####################################### -distr::ks_log() { - if [[ -f "/root/ks-post.log" ]]; then - echo_message "Kickstart post log - Start" - cat /root/ks-post.log - rm /root/ks-post.log - echo_message "Kickstart post log - End" - fi -} - ####################################### # Kernel configuration # Assume that we already run the latest selected kernel # (Asserted in the kickstart file) # Globals: -# DRACUT_CMD, KERNEL, UPDATE_TO_LATEST +# DRACUT_CMD, KERNEL, LINUX_FIRMWARE, SETUP_SWAP, UEK_RELEASE # Arguments: # None # Returns: # None ####################################### distr::kernel_config() { - local current_kernel kernel kernels old_kernel + local target_kernel # shellcheck disable=SC2153 - echo_message "Configure kernel: ${KERNEL^^}" - echo_message "Running kernel: $(uname -r)" + common::echo_message "Configure kernel: ${KERNEL^^}" # Note: there is no need to force drivers in intrd as dracut-config-generic # is installed # Configure repos and remove old kernels if [[ "${KERNEL,,}" = "uek" ]]; then + target_kernel=$(common::latest_kernel kernel-uek) + common::echo_message "Target kernel: ${target_kernel}" dnf config-manager --disable ol8_UEKR\* || : dnf config-manager --enable "ol8_UEKR${UEK_RELEASE}" - kernel_list=( "kernel-uek" "kernel-uek-core" ) - distr::remove_rpms kernel + common::remove_kernels kernel + common::remove_kernels kernel-uek "${target_kernel}" else - kernel_list=( "kernel" ) - distr::remove_rpms kernel-uek kernel-uek-core + target_kernel=$(common::latest_kernel kernel) + common::echo_message "Target kernel: ${target_kernel}" + common::remove_kernels kernel-uek + common::remove_kernels kernel "${target_kernel}" fi - current_kernel=$(uname -r) - for kernel in "${kernel_list[@]}"; do - if kernels=$(rpm -q "${kernel}" --qf "%{VERSION}-%{RELEASE}.%{ARCH} "); then - for old_kernel in $kernels; do - if [[ ${old_kernel} != "${current_kernel}" ]]; then - distr::remove_rpms "${kernel}-${old_kernel}" - fi - done - fi - done - if [[ ${KERNEL,,} = "uek" && ${UEK_RELEASE} != 6 ]]; then if [[ ${KERNEL_MODULES,,} == "no" ]]; then - echo_message "Removing kernel modules" + common::echo_message "Removing kernel modules" distr::remove_rpms kernel-uek-modules else - echo_message "Ensure kernel modules are installed" + common::echo_message "Ensure kernel modules are installed" dnf install -y kernel-uek fi fi # Workaround for orabug 32816428 - if [[ "${KERNEL,,}" = "uek" && -f "/etc/ld.so.conf.d/kernel-${current_kernel}.conf" ]]; then - cat > "/etc/ld.so.conf.d/kernel-${current_kernel}.conf" <<-EOF + if [[ "${KERNEL,,}" = "uek" && -f "/etc/ld.so.conf.d/kernel-${target_kernel}.conf" ]]; then + cat > "/etc/ld.so.conf.d/kernel-${target_kernel}.conf" <<-EOF # Placeholder file, no vDSO hwcap entries used in this kernel." EOF fi # Regenerate initrd - ${DRACUT_CMD} -f "/boot/initramfs-${current_kernel}.img" "${current_kernel}" + ${DRACUT_CMD} -f "/boot/initramfs-${target_kernel}.img" "${target_kernel}" # Ensure grub is properly setup grub2-mkconfig -o /boot/grub2/grub.cfg - grubby --set-default="/boot/vmlinuz-${current_kernel}" + grubby --set-default="/boot/vmlinuz-${target_kernel}" - echo_message "Linux firmware: ${LINUX_FIRMWARE^^}" + common::echo_message "Linux firmware: ${LINUX_FIRMWARE^^}" if [[ "${LINUX_FIRMWARE,,}" = "no" ]]; then - echo_message "Removing linux firmware" + common::echo_message "Removing linux firmware" distr::remove_rpms linux-firmware fi } @@ -126,42 +100,51 @@ distr::kernel_config() { ####################################### # Common configuration # Globals: -# UPDATE_TO_LATEST, BUILD_INFO +# BUILD_INFO, PERMIT_ROOT_LOGIN, SELINUX, UPDATE_TO_LATEST # Arguments: # None # Returns: # None ####################################### -distr::common_cfg() { +distr::configure() { local service tty # Directory to save build information mkdir -p "${BUILD_INFO}" # Run dnf update if flag is set to yes in image build page - echo_message "Update image: ${UPDATE_TO_LATEST^^}" + common::echo_message "Update image: ${UPDATE_TO_LATEST^^}" if [[ "${UPDATE_TO_LATEST,,}" = "yes" ]]; then dnf update -y elif [[ "${UPDATE_TO_LATEST,,}" = "security" ]]; then dnf update --security -y fi + common::echo_message "sshd root login policy: ${PERMIT_ROOT_LOGIN}" + # Unused anaconda leftover + rm -f /etc/sysconfig/sshd-permitrootlogin + ex -s /etc/ssh/sshd_config <<-EOF + :%substitute/^#\?\(PermitRootLogin\) .*$/\1 ${PERMIT_ROOT_LOGIN,,}/ + :update + :quit + EOF + # SSSD profile needs clients if authselect current -r | grep -q '^sssd'; then - echo_message "Installing SSSD client" + common::echo_message "Installing SSSD client" dnf install -y sssd-client fi # If you want to remove rsyslog and just use journald, remove this! - echo_message "Disabling persistent journal" + common::echo_message "Disabling persistent journal" rm -rf /var/log/journal/ # setup systemd to boot to the right runlevel - echo_message "Setting default runlevel to multiuser text mode" + common::echo_message "Setting default runlevel to multiuser text mode" rm -f /etc/systemd/system/default.target ln -s /lib/systemd/system/multi-user.target /etc/systemd/system/default.target - echo_message "Disable services" + common::echo_message "Disable services" # NetworkManager.service for service in \ kdump.service \ @@ -175,41 +158,37 @@ distr::common_cfg() { syslog.target do # Most of these aren't enabled, errors are expected... - echo_message " ${service}" + common::echo_message " ${service}" systemctl disable ${service} 2>&1 || true done - echo_message "Set rp_filter to loose mode" + common::echo_message "Set rp_filter to loose mode" echo "net.ipv4.conf.default.rp_filter = 2" >> /etc/sysctl.conf - echo_message "Set SELinux to ${SELINUX^^}" + common::echo_message "Set SELinux to ${SELINUX^^}" sed -i -e "s/^SELINUX[ ]*=.*/SELINUX=${SELINUX,,}/" /etc/selinux/config if [[ ${SELINUX,,} != "enforcing" ]]; then # Relax SELinux for the provisioning as well setenforce Permissive fi - echo_message "Clear network persistent data" + common::echo_message "Clear network persistent data" rm -f /etc/udev/rules.d/70-persistent-net.rules - echo_message "Configure dnf" + common::echo_message "Configure dnf" # bypass update kernel-uek-headers echo "exclude=kernel-uek-headers" >> /etc/dnf/dnf.conf # fix "Metadata file does not match checksum" for public-yum # https://forums.oracle.com/thread/2550364 echo "http_caching=none" >> /etc/dnf/dnf.conf - echo_message "Enable login on serial console ports" + common::echo_message "Enable login on serial console ports" for tty in "hvc0" "ttyS0" "ttyS0" do grep -q "${tty}" /etc/securetty || echo "${tty}" >>/etc/securetty done - # TODO: simplify -- this is done as well in cleanup()! - # 27601618 - set the machine-id file - sed -i -e 's@^ExecStart=.*@ExecStart=/usr/bin/systemd-firstboot --prompt-locale --prompt-timezone --prompt-root-password --setup-machine-id@g' /usr/lib/systemd/system/systemd-firstboot.service - - echo_message "Remove unneeded RPMs" + common::echo_message "Remove unneeded RPMs" distr::remove_rpms \ iwl7265-firmware \ mozjs17 \ @@ -227,176 +206,20 @@ distr::common_cfg() { # None ####################################### distr::provision() { - distr::ks_log + common::ks_log distr::kernel_config - distr::common_cfg + distr::configure } ####################################### # Cleanup # Globals: -# BUILD_INFO +# None # Arguments: # None # Returns: # None ####################################### distr::cleanup() { - echo_message "Stoppping services" - systemctl stop rsyslog || true - systemctl stop auditd || true - - echo_message "Remove orabackup files" - find /etc -name "*.orabackup*" -exec rm -rf {} \; - find /boot -name "*.orabackup*" -exec rm -rf {} \; - - echo_message "Remove leftover firewall rules" - if [[ -f /etc/sysconfig/iptables ]]; then - sed -i -e '/-p 50/d' /etc/sysconfig/iptables - sed -i -e '/-p 51/d' /etc/sysconfig/iptables - sed -i -e '/--dport 5353/d' /etc/sysconfig/iptables - sed -i -e '/--dport 631/d' /etc/sysconfig/iptables - fi - if [[ -f /etc/sysconfig/ip6tables ]]; then - sed -i -e '/-p 50/d' /etc/sysconfig/ip6tables - sed -i -e '/-p 51/d' /etc/sysconfig/ip6tables - sed -i -e '/--dport 5353/d' /etc/sysconfig/ip6tables - sed -i -e '/--dport 631/d' /etc/sysconfig/ip6tables - fi - - echo_message "Remove MAC addresses" - for f in /etc/sysconfig/network-scripts/ifcfg-eth*; do - [ -e "$f" ] && sed -i '/^HWADDR=/d' "$f" - done - - echo_message "Dnf cleanup" - dnf -q repolist > "${BUILD_INFO}/repolist.txt" - : > /etc/dnf/vars/ociregion - echo "oracle.com" > /etc/dnf/vars/ocidomain - rm -rf /var/cache/dnf/* - rm -rf /var/lib/dnf/* - find /etc/ -name "./*.uln-*" -exec rm -rf {} \; - - # Cleanup and regenerate /etc/machine-id - # Todo -- already done in provisioning! - echo_message "Reset machine id" - : > /etc/machine-id - if ! grep -q setup-machine-id /usr/lib/systemd/system/systemd-firstboot.service; then - sed -i.old -e "/^ExecStart=/s/$/ --setup-machine-id/" /usr/lib/systemd/system/systemd-firstboot.service - fi - - echo_message "Cleanup all log files" - rm -f /var/log/anaconda.* /var/log/oraclevm-template.log - rm -f /tmp/ks* - rm -f /root/install.log /root/install.log.syslog /root/anaconda-ks.cfg - : > /etc/resolv.conf - /bin/rm -f /etc/resolv.conf.* - /bin/rm -f /var/lib/dhclient/* - [ -e /var/log/acpid ] && : > /var/log/acpid - [ -e /var/log/messages ] && : > /var/log/messages - [ -e /var/log/btmp ] && : > /var/log/btmp - [ -e /var/log/grubby ] && : > /var/log/grubby - [ -e /var/log/secure ] && : > /var/log/secure - [ -e /var/log/wtmp ] && : > /var/log/wtmp - [ -e /var/log/boot.log ] && : > /var/log/boot.log - [ -e /var/log/dracut.log ] && : > /var/log/dracut.log - [ -e /var/log/tuned/tuned.log ] && : > /var/log/tuned/tuned.log - [ -e /var/log/maillog ] && : > /var/log/maillog - [ -e /var/log/lastlog ] && : > /var/log/lastlog - [ -e /var/log/dnf.log ] && : > /var/log/dnf.log - [ -e /var/log/dnf.librepo.log ] && : > /var/log/dnf.librepo.log - [ -e /var/log/dnf.rpm.log ] && : > /var/log/dnf.rpm.log - [ -e /var/log/ovm-template-config.log ] && rm -f /var/log/ovm-template-config.log - /bin/rm -f /var/log/audit/audit.log* - [ -e /var/log/audit/audit.log ] && : > /var/log/audit/audit.log - - # Lock root user - if [[ "${LOCK_ROOT,,}" = "yes" ]]; then - passwd -d root - passwd -l root - fi - - # cleanup ssh config files - if [ -z "${SSH_KEY_FILE}" ]; then - [ -d /root/.ssh ] && /bin/rm -fr /root/.ssh - else - find /root/.ssh -type f -not -name authorized_keys -delete - fi - - # Rebuild rpmdb to save some space - rpm --rebuilddb - - # Remove man and info pages - echo_message "Exclude documentation: ${EXCLUDE_DOCS^^}" - if [[ "${EXCLUDE_DOCS,,}" = "minimal" ]]; then - rm -rf /usr/share/{man,info} - fi - - # cleanup vnc cache files - if [ -d /root/.vnc ]; then - /bin/rm -f /root/.vnc/*.log - /bin/rm -f /root/.vnc/passwd - fi - - rm -rf /var/log/cups/error_log - rm -rf /var/log/setroubleshoot/setroubleshootd.log - rm -rf /var/log/spooler - # cleanup bash history - [ -e /root/.bash_history ] && : > /root/.bash_history - rm -f /root/.viminfo - rm -rf /.autorelabel - rm -rf /var/log/mail/statistics - rm -rf /var/log/sa/* - rm -rf /var/log/acpid /var/log/boot.log /var/log/cron /var/log/dmesg.* /var/log/ovm* - rm -rf /poweroff - rm -f /etc/ssh/ssh_host_* - rm -rf /root/* - rm -f /etc/udev/rules.d/70-persistent-net.rules - rm -f /etc/udev/rules.d/70-persistent-cd.rules - - find /var/log -type f | while read -r f; do echo -ne '' > "$f"; done; - find /etc/ -name "*.old" -exec rm -f {} \; - rm -f /etc/sysconfig/network-scripts/ifcfg-enp* - rm -rf /lost+found/* - rm -rf /root/.vbox_version - export HISTSIZE=0 - rm -f /var/log/ovm-template-config.log - - echo_message "Save list of installed packages" - rpm -qa --qf "%{name}.%{arch}\n" | sort -u > "${BUILD_INFO}/pkglist.txt" - rpm -qa --qf '"%{NAME}","%{EPOCHNUM}","%{VERSION}","%{RELEASE}","%{ARCH}"\n' | sort > "${BUILD_INFO}/pkglist.csv" - uname -r > "${BUILD_INFO}/kernel.txt" -} - -####################################### -# Final seal of the image -# Globals: -# BUILD_INFO -# Returns: -# None -####################################### -distr::seal() { - echo_message "File cleanup" - : > /var/log/wtmp - : > /var/log/lastlog - rm -f /var/log/audit/audit.log - rm -f /var/log/tuned/tuned.log - rm -rf /root/.gemrc /root/.gem - rm -rf /var/spool/root /var/spool/mail/root - rm -rf /var/lib/NetworkManager - [[ -n "${BUILD_INFO}" ]] && rm -rf "${BUILD_INFO}" - rm -rf /var/tmp/* /tmp/* - - echo_message "Relabel SELinux" - genhomedircon - fixfiles -f -F relabel - restorecon -R / || true - - echo_message "Trim filesystem" - sync; sync; sync - for fs in /boot /; do - echo_message " ${fs}" - dd if=/dev/zero of="${fs}"/EMPTY bs=1M >/dev/null 2>&1 || : - rm -f "${fs}"/EMPTY - done + common::distr_cleanup } diff --git a/oracle-linux-image-tools/distr/ol9-aarch64/env.properties b/oracle-linux-image-tools/distr/ol9-aarch64/env.properties index a842d74..6435ece 100644 --- a/oracle-linux-image-tools/distr/ol9-aarch64/env.properties +++ b/oracle-linux-image-tools/distr/ol9-aarch64/env.properties @@ -14,18 +14,28 @@ SETUP_SWAP="yes" # Root filesystem: xfs, lvm or btrfs ROOT_FS="xfs" -# Label of the ISO image -ISO_LABEL="OL-9-3-0-BaseOS-aarch64" +# Location of the kernel and initrd on the distribution image. +BOOT_LOCATION="images/pxeboot" +# Boot mode +BOOT_MODE="uefi" # Boot command # Variables MUST be escaped as they are evaluated at build time. +# Following variables are available: +# KS_FILE: name of the kickstart file which will be in the root directory +# ISO_LABEL: label of the ISO image BOOT_COMMAND=( - 'c' - 'linux /images/pxeboot/vmlinuz inst.stage2=hd:LABEL=${ISO_LABEL} ro ' - 'inst.text inst.notmux inst.ks=${KS_CONFIG} setup_swap=${SETUP_SWAP} ' - 'biosdevname=0 net.ifnames=0' - 'initrd /images/pxeboot/initrd.img' - 'boot' + 'inst.text' + 'inst.ks=file:/${KS_FILE}' + 'inst.geoloc=0' + 'inst.stage2=hd:LABEL=${ISO_LABEL}' + 'ro', + 'biosdevname=0', + 'net.ifnames=0' +) +# Additional parameters to enable serial console +BOOT_COMMAND_SERIAL_CONSOLE=( + 'inst.notmux' ) # Kernel: must be UEK! diff --git a/oracle-linux-image-tools/distr/ol9-aarch64/image-scripts.sh b/oracle-linux-image-tools/distr/ol9-aarch64/image-scripts.sh index 18dbf4f..7f80b64 100755 --- a/oracle-linux-image-tools/distr/ol9-aarch64/image-scripts.sh +++ b/oracle-linux-image-tools/distr/ol9-aarch64/image-scripts.sh @@ -2,15 +2,13 @@ # # image scripts for OL9 - aarch64 # -# Copyright (c) 2022 Oracle and/or its affiliates. +# Copyright (c) 2022, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # # Description: this module provides the following function: # distr::validate: basic parameter validation # distr::kickstart: hook for kickstart file updates -# distr::packer_conf: hook for packer configuration file updates -# distr::image_cleanup: distribution specific actions to cleanup the image # All functions are optional # # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. @@ -19,27 +17,27 @@ ####################################### # Validate distribution parameters # Globals: -# ROOT_FS TMP_IN_TMPFS RESCUE_KERNEL ISO_LABEL KERNEL_MODULES EXCLUDE_DOCS +# KERNEL_MODULES, ROOT_FS, RESCUE_KERNEL, TMP_IN_TMPFS, EXCLUDE_DOCS # Arguments: # None # Returns: # None ####################################### distr::validate() { - [[ "${ROOT_FS,,}" =~ ^(xfs)|(btrfs)|(lvm)$ ]] || error "ROOT_FS must be xfs, btrfs or lvm" - [[ "${ROOT_FS,,}" = "btrfs" ]] && echo_message "Note that for btrfs root filesystem you need to use an UEK boot ISO" - [[ "${TMP_IN_TMPFS,,}" =~ ^(yes)|(no)$ ]] || error "TMP_IN_TMPFS must be yes or no" - [[ "${RESCUE_KERNEL,,}" =~ ^(yes)|(no)$ ]] || error "RESCUE_KERNEL must be yes or no" - [[ -n ${ISO_LABEL} ]] || error "ISO_LABEL must be provided" - [[ "${KERNEL_MODULES,,}" =~ ^(yes)|(no)$ ]] || error "KERNEL_MODULES must be yes or no" - [[ "${EXCLUDE_DOCS,,}" =~ ^(yes)|(no)|(minimal)$ ]] || error "EXCLUDE_DOCS must be yes, no or minimal" - readonly ROOT_FS TMP_IN_TMPFS RESCUE_KERNEL ISO_LABEL KERNEL_MODULES EXCLUDE_DOCS + [[ "${ROOT_FS,,}" =~ ^((xfs)|(btrfs)|(lvm))$ ]] || common::error "ROOT_FS must be xfs, btrfs or lvm" + [[ "${ROOT_FS,,}" = "btrfs" ]] && common::echo_message "Note that for btrfs root filesystem you need to use an UEK boot ISO" + [[ "${TMP_IN_TMPFS,,}" =~ ^((yes)|(no))$ ]] || common::error "TMP_IN_TMPFS must be yes or no" + [[ "${RESCUE_KERNEL,,}" =~ ^((yes)|(no))$ ]] || common::error "RESCUE_KERNEL must be yes or no" + [[ "${KERNEL_MODULES,,}" =~ ^((yes)|(no))$ ]] || common::error "KERNEL_MODULES must be yes or no" + [[ "${EXCLUDE_DOCS,,}" =~ ^((yes)|(no)|(minimal))$ ]] || common::error "EXCLUDE_DOCS must be yes, no or minimal" + readonly ROOT_FS TMP_IN_TMPFS RESCUE_KERNEL KERNEL_MODULES EXCLUDE_DOCS } ####################################### # Kickcstart fixup # Globals: -# RESCUE_KERNEL ROOT_FS +# AUTHSELECT, KERNEL, RESCUE_KERNEL, ROOT_FS +# EXCLUDE_DOCS, TMP_IN_TMPFS # Arguments: # kickstart file name # Returns: @@ -85,41 +83,5 @@ logvol / --fstype=\"xfs\" --vgname=vg_main --size=4096 --name=lv_root --gr fi # /tmp in tmpfs - sed -i -e "s!^TMP_IN_TMPFS=no!TMP_IN_TMPFS=$TMP_IN_TMPFS!" "${ks_file}" + sed -i -e "s!^TMP_IN_TMPFS=no!TMP_IN_TMPFS=${TMP_IN_TMPFS}!" "${ks_file}" } - -####################################### -# Packer configuration -# Globals: -# BUILD_INFO -# Arguments: -# Packer configuration file -# Returns: -# None -####################################### -distr::packer_conf() { - if [[ -c /dev/kvm && $(uname -m) == "aarch64" ]]; then - cat >>"$1" <<-EOF - accel = "kvm" - EOF - fi - if [[ -n "${BUILD_INFO}" ]]; then - cat >>"$1" <<-EOF - build_info = "${BUILD_INFO}" - EOF - fi -} - -####################################### -# Cleanup actions run directly on the image -# Globals: -# WORKSPACE VM_NAME BUILD_INFO -# Arguments: -# root filesystem directory -# boot filesystem directory -# Returns: -# None -####################################### -# distr::image_cleanup() { -# : -# } diff --git a/oracle-linux-image-tools/distr/ol9-aarch64/ol9-aarch64-ks.cfg b/oracle-linux-image-tools/distr/ol9-aarch64/ol9-aarch64-ks.cfg index 4e2d1fc..d724973 100644 --- a/oracle-linux-image-tools/distr/ol9-aarch64/ol9-aarch64-ks.cfg +++ b/oracle-linux-image-tools/distr/ol9-aarch64/ol9-aarch64-ks.cfg @@ -163,12 +163,6 @@ if [[ "${RESCUE_KERNEL,,}" = "no" ]]; then rm -f /boot/loader/entries/$(cat /etc/machine-id)-0-rescue.conf fi -# Allow password login -cat > /etc/ssh/sshd_config.d/01-permitrootlogin.conf << EOF -# Allow root to log in using ssh. Remove this file to opt-out. -PermitRootLogin yes -EOF - EXCLUDE_DOCS="no" echo "Exclude documentation: ${EXCLUDE_DOCS^^}" if [[ "${EXCLUDE_DOCS,,}" = "yes" ]]; then diff --git a/oracle-linux-image-tools/distr/ol9-aarch64/provision.sh b/oracle-linux-image-tools/distr/ol9-aarch64/provision.sh index c3df848..bc7cc95 100644 --- a/oracle-linux-image-tools/distr/ol9-aarch64/provision.sh +++ b/oracle-linux-image-tools/distr/ol9-aarch64/provision.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # -# Packer provisioning script for OL9 - aarch64 +# Provisioning script for OL9 - aarch64 # -# Copyright (c) 2022 Oracle and/or its affiliates. +# Copyright (c) 2022, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # @@ -32,115 +32,98 @@ distr::remove_rpms() { dnf -C -y remove "$@" } -####################################### -# Print kickstart log -# Globals: -# Arguments: -# None -# Returns: -# None -####################################### -distr::ks_log() { - if [[ -f "/root/ks-post.log" ]]; then - echo_message "Kickstart post log - Start" - cat /root/ks-post.log - rm /root/ks-post.log - echo_message "Kickstart post log - End" - fi -} - ####################################### # Kernel configuration # Assume that we already run the latest selected kernel # (Asserted in the kickstart file) # Globals: -# DRACUT_CMD, KERNEL, UPDATE_TO_LATEST +# DRACUT_CMD, KERNEL # Arguments: # None # Returns: # None ####################################### distr::kernel_config() { - local current_kernel kernel kernels old_kernel + local target_kernel # shellcheck disable=SC2153 - echo_message "Configure kernel: ${KERNEL^^}" - echo_message "Running kernel: $(uname -r)" + common::echo_message "Configure kernel: ${KERNEL^^}" # Note: there is no need to force drivers in intrd as dracut-config-generic # is installed # Configure repos and remove old kernels - kernel="kernel-uek" + target_kernel=$(common::latest_kernel kernel-uek) + common::echo_message "Target kernel: ${target_kernel}" dnf config-manager --set-enabled ol9_UEKR7 - - current_kernel=$(uname -r) - kernels=$(rpm -q "${kernel}-core" --qf "%{VERSION}-%{RELEASE}.%{ARCH} ") - for old_kernel in $kernels; do - if [[ ${old_kernel} != "${current_kernel}" ]]; then - distr::remove_rpms "${kernel}-core-${old_kernel}" - fi - done + common::remove_kernels kernel-uek "${target_kernel}" # Clean dnf cache which contains odd dependencies and prevents removal # of kernel modules rm -rf /var/cache/dnf/* rm -rf /var/lib/dnf/* + if [[ ${KERNEL_MODULES,,} == "no" ]]; then - echo_message "Removing kernel modules and linux firmware" - distr::remove_rpms "${kernel}-modules" linux-firmware + common::echo_message "Removing kernel modules and linux firmware" + distr::remove_rpms kernel-uek-modules linux-firmware else - echo_message "Ensure kernel modules are installed" - dnf install -y ${kernel} linux-firmware + common::echo_message "Ensure kernel modules are installed" + dnf install -y kernel-uek linux-firmware fi # Regenerate initrd - ${DRACUT_CMD} -f "/boot/initramfs-${current_kernel}.img" "${current_kernel}" + ${DRACUT_CMD} -f "/boot/initramfs-${target_kernel}.img" "${target_kernel}" # Ensure grub is properly setup grub2-mkconfig -o /etc/grub2-efi.cfg - grubby --set-default="/boot/vmlinuz-${current_kernel}" + grubby --set-default="/boot/vmlinuz-${target_kernel}" } ####################################### # Common configuration # Globals: -# UPDATE_TO_LATEST, BUILD_INFO +# BUILD_INFO, PERMIT_ROOT_LOGIN, SELINUX, UPDATE_TO_LATEST # Arguments: # None # Returns: # None ####################################### -distr::common_cfg() { +distr::configure() { local service # Directory to save build information mkdir -p "${BUILD_INFO}" # Run dnf update if flag is set to yes in image build page - echo_message "Update image: ${UPDATE_TO_LATEST^^}" + common::echo_message "Update image: ${UPDATE_TO_LATEST^^}" if [[ "${UPDATE_TO_LATEST,,}" = "yes" ]]; then dnf update -y elif [[ "${UPDATE_TO_LATEST,,}" = "security" ]]; then dnf update --security -y fi + common::echo_message "sshd root login policy: ${PERMIT_ROOT_LOGIN}" + cat > /etc/ssh/sshd_config.d/01-permitrootlogin.conf <<-EOF + # root login policy when using ssh. Remove this file to revert to default. + PermitRootLogin ${PERMIT_ROOT_LOGIN,,} + EOF + # SSSD profile needs clients if authselect current -r | grep -q '^sssd'; then - echo_message "Installing SSSD client" + common::echo_message "Installing SSSD client" dnf install -y sssd-client fi # If you want to remove rsyslog and just use journald, remove this! - echo_message "Disabling persistent journal" + common::echo_message "Disabling persistent journal" rm -rf /var/log/journal/ # setup systemd to boot to the right runlevel - echo_message "Setting default runlevel to multiuser text mode" + common::echo_message "Setting default runlevel to multiuser text mode" rm -f /etc/systemd/system/default.target ln -s /lib/systemd/system/multi-user.target /etc/systemd/system/default.target - echo_message "Disable services" + common::echo_message "Disable services" # NetworkManager.service for service in \ kdump.service \ @@ -154,31 +137,31 @@ distr::common_cfg() { syslog.target do # Most of these aren't enabled, errors are expected... - echo_message " ${service}" + common::echo_message " ${service}" systemctl disable ${service} 2>&1 || true done - echo_message "Set rp_filter to loose mode" + common::echo_message "Set rp_filter to loose mode" echo "net.ipv4.conf.default.rp_filter = 2" >> /etc/sysctl.conf - echo_message "Set SELinux to ${SELINUX^^}" + common::echo_message "Set SELinux to ${SELINUX^^}" sed -i -e "s/^SELINUX[ ]*=.*/SELINUX=${SELINUX,,}/" /etc/selinux/config if [[ ${SELINUX,,} != "enforcing" ]]; then # Relax SELinux for the provisioning as well setenforce Permissive fi - echo_message "Clear network persistent data" + common::echo_message "Clear network persistent data" rm -f /etc/udev/rules.d/70-persistent-net.rules - echo_message "Configure dnf" + common::echo_message "Configure dnf" # bypass update kernel-uek-headers echo "exclude=kernel-uek-headers" >> /etc/dnf/dnf.conf # fix "Metadata file does not match checksum" for public-yum # https://forums.oracle.com/thread/2550364 echo "http_caching=none" >> /etc/dnf/dnf.conf - echo_message "Remove unneeded RPMs" + common::echo_message "Remove unneeded RPMs" distr::remove_rpms \ iwl7265-firmware \ mozjs17 \ @@ -196,157 +179,20 @@ distr::common_cfg() { # None ####################################### distr::provision() { - distr::ks_log + common::ks_log distr::kernel_config - distr::common_cfg + distr::configure } ####################################### # Cleanup # Globals: -# BUILD_INFO +# None # Arguments: # None # Returns: # None ####################################### distr::cleanup() { - echo_message "Stoppping services" - systemctl stop rsyslog || true - systemctl stop auditd || true - - echo_message "Dnf cleanup" - dnf -q repolist > "${BUILD_INFO}/repolist.txt" - : > /etc/dnf/vars/ociregion - echo "oracle.com" > /etc/dnf/vars/ocidomain - rm -rf /var/cache/dnf/* - rm -rf /var/lib/dnf/* - find /etc/ -name "./*.uln-*" -exec rm -rf {} \; - - # Cleanup and regenerate /etc/machine-id - echo_message "Reset machine id" - : > /etc/machine-id - if ! grep -q setup-machine-id /usr/lib/systemd/system/systemd-firstboot.service; then - sed -i -e "/^ExecStart=/s/$/ --setup-machine-id/" /usr/lib/systemd/system/systemd-firstboot.service - fi - rm -f /var/lib/systemd/random-seed - - echo_message "Cleanup all log files" - rm -f /var/log/anaconda.* /var/log/oraclevm-template.log - rm -f /tmp/ks* - rm -f /root/install.log /root/install.log.syslog /root/anaconda-ks.cfg - : > /etc/resolv.conf - /bin/rm -f /etc/resolv.conf.* - /bin/rm -f /var/lib/dhclient/* - [ -e /var/log/acpid ] && : > /var/log/acpid - [ -e /var/log/messages ] && : > /var/log/messages - [ -e /var/log/btmp ] && : > /var/log/btmp - [ -e /var/log/grubby ] && : > /var/log/grubby - [ -e /var/log/secure ] && : > /var/log/secure - [ -e /var/log/wtmp ] && : > /var/log/wtmp - [ -e /var/log/boot.log ] && : > /var/log/boot.log - [ -e /var/log/dracut.log ] && : > /var/log/dracut.log - [ -e /var/log/tuned/tuned.log ] && : > /var/log/tuned/tuned.log - [ -e /var/log/maillog ] && : > /var/log/maillog - [ -e /var/log/lastlog ] && : > /var/log/lastlog - [ -e /var/log/dnf.log ] && : > /var/log/dnf.log - [ -e /var/log/dnf.librepo.log ] && : > /var/log/dnf.librepo.log - [ -e /var/log/dnf.rpm.log ] && : > /var/log/dnf.rpm.log - [ -e /var/log/ovm-template-config.log ] && rm -f /var/log/ovm-template-config.log - /bin/rm -f /var/log/audit/audit.log* - [ -e /var/log/audit/audit.log ] && : > /var/log/audit/audit.log - - # Lock root user - if [[ "${LOCK_ROOT,,}" = "yes" ]]; then - passwd -d root - passwd -l root - fi - rm -f /etc/ssh/sshd_config.d/01-permitrootlogin.conf - - # cleanup ssh config files - if [ -z "${SSH_KEY_FILE}" ]; then - [ -d /root/.ssh ] && /bin/rm -fr /root/.ssh - else - find /root/.ssh -type f -not -name authorized_keys -delete - fi - - # Rebuild rpmdb to save some space - rpm --rebuilddb - - # Remove man and info pages - echo_message "Exclude documentation: ${EXCLUDE_DOCS^^}" - if [[ "${EXCLUDE_DOCS,,}" = "minimal" ]]; then - rm -rf /usr/share/{man,info} - fi - - # cleanup vnc cache files - if [ -d /root/.vnc ]; then - /bin/rm -f /root/.vnc/*.log - /bin/rm -f /root/.vnc/passwd - fi - - rm -rf /var/log/cups/error_log - rm -rf /var/log/setroubleshoot/setroubleshootd.log - rm -rf /var/log/spooler - # cleanup bash history - [ -e /root/.bash_history ] && : > /root/.bash_history - rm -f /root/.viminfo - rm -rf /.autorelabel - rm -rf /var/log/mail/statistics - rm -rf /var/log/sa/* - rm -rf /var/log/acpid /var/log/boot.log /var/log/cron /var/log/dmesg.* /var/log/ovm* - rm -rf /poweroff - rm -f /etc/ssh/ssh_host_* - rm -rf /root/* - rm -f /etc/udev/rules.d/70-persistent-net.rules - rm -f /etc/udev/rules.d/70-persistent-cd.rules - - find /var/log -type f | while read -r f; do echo -ne '' > "$f"; done; - find /etc/ -name "*.old" -exec rm -f {} \; - rm -f /etc/sysconfig/network-scripts/ifcfg-enp* - rm -rf /lost+found/* - rm -rf /root/.vbox_version - export HISTSIZE=0 - rm -f /var/log/ovm-template-config.log - - echo_message "Save list of installed packages" - rpm -qa --qf "%{name}.%{arch}\n" | sort -u > "${BUILD_INFO}/pkglist.txt" - rpm -qa --qf '"%{NAME}","%{EPOCHNUM}","%{VERSION}","%{RELEASE}","%{ARCH}"\n' | sort > "${BUILD_INFO}/pkglist.csv" - uname -r > "${BUILD_INFO}/kernel.txt" - - history -c - swapoff -a -} - -####################################### -# Final seal of the image -# Globals: -# BUILD_INFO -# Returns: -# None -####################################### -distr::seal() { - echo_message "File cleanup" - : > /var/log/wtmp - : > /var/log/lastlog - rm -f /var/log/audit/audit.log - rm -f /var/log/tuned/tuned.log - rm -rf /root/.gemrc /root/.gem - rm -rf /var/spool/root /var/spool/mail/root - rm -rf /var/lib/NetworkManager - [[ -n "${BUILD_INFO}" ]] && rm -rf "${BUILD_INFO}" - rm -rf /var/tmp/* /tmp/* - - echo_message "Relabel SELinux" - genhomedircon - fixfiles -f -F relabel - restorecon -R / || true - - echo_message "Trim filesystem" - sync; sync; sync - for fs in /boot /; do - echo_message " ${fs}" - dd if=/dev/zero of="${fs}"/EMPTY bs=1M >/dev/null 2>&1 || : - rm -f "${fs}"/EMPTY - done + common::distr_cleanup } diff --git a/oracle-linux-image-tools/distr/ol9-slim/env.properties b/oracle-linux-image-tools/distr/ol9-slim/env.properties index 79cb075..54165fe 100644 --- a/oracle-linux-image-tools/distr/ol9-slim/env.properties +++ b/oracle-linux-image-tools/distr/ol9-slim/env.properties @@ -14,9 +14,29 @@ SETUP_SWAP="yes" # Root filesystem: xfs, lvm or btrfs ROOT_FS="xfs" +# Location of the kernel and initrd on the distribution image. +BOOT_LOCATION="isolinux" + +# Boot mode +BOOT_MODE="bios" # Boot command # Variables MUST be escaped as they are evaluated at build time. -BOOT_COMMAND=( '${CONSOLE} inst.text inst.ks=${KS_CONFIG} setup_swap=${SETUP_SWAP} ' ) +# Following variables are available: +# KS_FILE: name of the kickstart file which will be in the root directory +# ISO_LABEL: label of the ISO image +BOOT_COMMAND=( + 'inst.text' + 'inst.ks=file:/${KS_FILE}' + 'inst.geoloc=0' + 'inst.stage2=hd:LABEL=${ISO_LABEL}' + 'net.ifnames=0' +) +# Additional parameters to enable serial console +BOOT_COMMAND_SERIAL_CONSOLE=( + 'console=tty0' + 'console=ttyS0' + 'inst.notmux' +) # Kernel: uek, rhck KERNEL="uek" diff --git a/oracle-linux-image-tools/distr/ol9-slim/image-scripts.sh b/oracle-linux-image-tools/distr/ol9-slim/image-scripts.sh index fc60828..339c2d0 100755 --- a/oracle-linux-image-tools/distr/ol9-slim/image-scripts.sh +++ b/oracle-linux-image-tools/distr/ol9-slim/image-scripts.sh @@ -2,15 +2,13 @@ # # image scripts for OL9 # -# Copyright (c) 2022 Oracle and/or its affiliates. +# Copyright (c) 2022, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # # Description: this module provides the following function: # distr::validate: basic parameter validation # distr::kickstart: hook for kickstart file updates -# distr::packer_conf: hook for packer configuration file updates -# distr::image_cleanup: distribution specific actions to cleanup the image # All functions are optional # # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. @@ -19,26 +17,27 @@ ####################################### # Validate distribution parameters # Globals: -# ROOT_FS TMP_IN_TMPFS RESCUE_KERNEL KERNEL_MODULES EXCLUDE_DOCS +# KERNEL_MODULES, ROOT_FS, RESCUE_KERNEL, TMP_IN_TMPFS, EXCLUDE_DOCS # Arguments: # None # Returns: # None ####################################### distr::validate() { - [[ "${ROOT_FS,,}" =~ ^(xfs)|(btrfs)|(lvm)$ ]] || error "ROOT_FS must be xfs, btrfs or lvm" - [[ "${ROOT_FS,,}" = "btrfs" ]] && echo_message "Note that for btrfs root filesystem you need to use an UEK boot ISO" - [[ "${TMP_IN_TMPFS,,}" =~ ^(yes)|(no)$ ]] || error "TMP_IN_TMPFS must be yes or no" - [[ "${RESCUE_KERNEL,,}" =~ ^(yes)|(no)$ ]] || error "RESCUE_KERNEL must be yes or no" - [[ "${KERNEL_MODULES,,}" =~ ^(yes)|(no)$ ]] || error "KERNEL_MODULES must be yes or no" - [[ "${EXCLUDE_DOCS,,}" =~ ^(yes)|(no)|(minimal)$ ]] || error "EXCLUDE_DOCS must be yes, no or minimal" + [[ "${ROOT_FS,,}" =~ ^((xfs)|(btrfs)|(lvm))$ ]] || common::error "ROOT_FS must be xfs, btrfs or lvm" + [[ "${ROOT_FS,,}" = "btrfs" ]] && common::echo_message "Note that for btrfs root filesystem you need to use an UEK boot ISO" + [[ "${TMP_IN_TMPFS,,}" =~ ^((yes)|(no))$ ]] || common::error "TMP_IN_TMPFS must be yes or no" + [[ "${RESCUE_KERNEL,,}" =~ ^((yes)|(no))$ ]] || common::error "RESCUE_KERNEL must be yes or no" + [[ "${KERNEL_MODULES,,}" =~ ^((yes)|(no))$ ]] || common::error "KERNEL_MODULES must be yes or no" + [[ "${EXCLUDE_DOCS,,}" =~ ^((yes)|(no)|(minimal))$ ]] || common::error "EXCLUDE_DOCS must be yes, no or minimal" readonly ROOT_FS TMP_IN_TMPFS RESCUE_KERNEL KERNEL_MODULES EXCLUDE_DOCS } ####################################### # Kickcstart fixup # Globals: -# RESCUE_KERNEL ROOT_FS +# AUTHSELECT, KERNEL, RESCUE_KERNEL, ROOT_FS +# EXCLUDE_DOCS, TMP_IN_TMPFS # Arguments: # kickstart file name # Returns: @@ -83,36 +82,5 @@ logvol / --fstype=\"xfs\" --vgname=vg_main --size=4096 --name=lv_root --gr fi # /tmp in tmpfs - sed -i -e "s!^TMP_IN_TMPFS=no!TMP_IN_TMPFS=$TMP_IN_TMPFS!" "${ks_file}" + sed -i -e "s!^TMP_IN_TMPFS=no!TMP_IN_TMPFS=${TMP_IN_TMPFS}!" "${ks_file}" } - -####################################### -# Packer configuration -# Globals: -# BUILD_INFO -# Arguments: -# Packer configuration file -# Returns: -# None -####################################### -distr::packer_conf() { - if [[ -n "${BUILD_INFO}" ]]; then - cat >>"$1" <<-EOF - build_info = "${BUILD_INFO}" - EOF - fi -} - -####################################### -# Cleanup actions run directly on the image -# Globals: -# WORKSPACE VM_NAME BUILD_INFO -# Arguments: -# root filesystem directory -# boot filesystem directory -# Returns: -# None -####################################### -# distr::image_cleanup() { -# : -# } diff --git a/oracle-linux-image-tools/distr/ol9-slim/ol9-ks.cfg b/oracle-linux-image-tools/distr/ol9-slim/ol9-ks.cfg index 4a3163f..b3e063c 100644 --- a/oracle-linux-image-tools/distr/ol9-slim/ol9-ks.cfg +++ b/oracle-linux-image-tools/distr/ol9-slim/ol9-ks.cfg @@ -156,12 +156,6 @@ sed -i \ /etc/default/grub grub2-mkconfig -o /boot/grub2/grub.cfg -# Allow password login -cat > /etc/ssh/sshd_config.d/01-permitrootlogin.conf << EOF -# Allow root to log in using ssh. Remove this file to opt-out. -PermitRootLogin yes -EOF - EXCLUDE_DOCS="no" echo "Exclude documentation: ${EXCLUDE_DOCS^^}" if [[ "${EXCLUDE_DOCS,,}" = "yes" ]]; then diff --git a/oracle-linux-image-tools/distr/ol9-slim/provision.sh b/oracle-linux-image-tools/distr/ol9-slim/provision.sh index cca9f95..515f451 100644 --- a/oracle-linux-image-tools/distr/ol9-slim/provision.sh +++ b/oracle-linux-image-tools/distr/ol9-slim/provision.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # -# Packer provisioning script for OL9 +# Provisioning script for OL9 # -# Copyright (c) 2022 Oracle and/or its affiliates. +# Copyright (c) 2022, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # https://oss.oracle.com/licenses/upl # @@ -10,7 +10,6 @@ # both are optional. # distr::provision: provision the instance # distr::cleanup: instance cleanup before shutdown -# distr::seal: final instance sealing # # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. # @@ -32,121 +31,108 @@ distr::remove_rpms() { dnf -C -y remove "$@" } -####################################### -# Print kickstart log -# Globals: -# Arguments: -# None -# Returns: -# None -####################################### -distr::ks_log() { - if [[ -f "/root/ks-post.log" ]]; then - echo_message "Kickstart post log - Start" - cat /root/ks-post.log - rm /root/ks-post.log - echo_message "Kickstart post log - End" - fi -} - ####################################### # Kernel configuration # Assume that we already run the latest selected kernel # (Asserted in the kickstart file) # Globals: -# DRACUT_CMD, KERNEL, UPDATE_TO_LATEST +# DRACUT_CMD, KERNEL # Arguments: # None # Returns: # None ####################################### distr::kernel_config() { - local current_kernel kernel kernels old_kernel + local target_kernel # shellcheck disable=SC2153 - echo_message "Configure kernel: ${KERNEL^^}" - echo_message "Running kernel: $(uname -r)" + common::echo_message "Configure kernel: ${KERNEL^^}" # Note: there is no need to force drivers in intrd as dracut-config-generic # is installed # Configure repos and remove old kernels + local kernel if [[ "${KERNEL,,}" = "uek" ]]; then kernel="kernel-uek" + target_kernel=$(common::latest_kernel kernel-uek) + common::echo_message "Target kernel: ${target_kernel}" dnf config-manager --set-enabled ol9_UEKR7 - distr::remove_rpms kernel kernel-core + common::remove_kernels kernel + common::remove_kernels kernel-uek "${target_kernel}" else kernel="kernel" - distr::remove_rpms kernel-uek kernel-uek-core + target_kernel=$(common::latest_kernel kernel) + common::echo_message "Target kernel: ${target_kernel}" + common::remove_kernels kernel-uek + common::remove_kernels kernel "${target_kernel}" fi - current_kernel=$(uname -r) - kernels=$(rpm -q "${kernel}-core" --qf "%{VERSION}-%{RELEASE}.%{ARCH} ") - for old_kernel in $kernels; do - if [[ ${old_kernel} != "${current_kernel}" ]]; then - distr::remove_rpms "${kernel}-core-${old_kernel}" - fi - done - # Clean dnf cache which contains odd dependencies and prevents removal # of kernel modules rm -rf /var/cache/dnf/* rm -rf /var/lib/dnf/* if [[ ${KERNEL_MODULES,,} == "no" ]]; then - echo_message "Removing kernel modules and linux firmware" + common::echo_message "Removing kernel modules and linux firmware" distr::remove_rpms "${kernel}-modules" linux-firmware else - echo_message "Ensure kernel modules are installed" + common::echo_message "Ensure kernel modules are installed" dnf install -y ${kernel} linux-firmware fi # Regenerate initrd - ${DRACUT_CMD} -f "/boot/initramfs-${current_kernel}.img" "${current_kernel}" + ${DRACUT_CMD} -f "/boot/initramfs-${target_kernel}.img" "${target_kernel}" # Ensure grub is properly setup grub2-mkconfig -o /boot/grub2/grub.cfg - grubby --set-default="/boot/vmlinuz-${current_kernel}" + grubby --set-default="/boot/vmlinuz-${target_kernel}" } ####################################### # Common configuration # Globals: -# UPDATE_TO_LATEST, BUILD_INFO +# BUILD_INFO, PERMIT_ROOT_LOGIN, SELINUX, UPDATE_TO_LATEST # Arguments: # None # Returns: # None ####################################### -distr::common_cfg() { +distr::configure() { local service tty # Directory to save build information mkdir -p "${BUILD_INFO}" # Run dnf update if flag is set to yes in image build page - echo_message "Update image: ${UPDATE_TO_LATEST^^}" + common::echo_message "Update image: ${UPDATE_TO_LATEST^^}" if [[ "${UPDATE_TO_LATEST,,}" = "yes" ]]; then dnf update -y elif [[ "${UPDATE_TO_LATEST,,}" = "security" ]]; then dnf update --security -y fi + common::echo_message "sshd root login policy: ${PERMIT_ROOT_LOGIN}" + cat > /etc/ssh/sshd_config.d/01-permitrootlogin.conf <<-EOF + # root login policy when using ssh. Remove this file to revert to default. + PermitRootLogin ${PERMIT_ROOT_LOGIN,,} + EOF + # SSSD profile needs clients if authselect current -r | grep -q '^sssd'; then - echo_message "Installing SSSD client" + common::echo_message "Installing SSSD client" dnf install -y sssd-client fi # If you want to remove rsyslog and just use journald, remove this! - echo_message "Disabling persistent journal" + common::echo_message "Disabling persistent journal" rm -rf /var/log/journal/ # setup systemd to boot to the right runlevel - echo_message "Setting default runlevel to multiuser text mode" + common::echo_message "Setting default runlevel to multiuser text mode" rm -f /etc/systemd/system/default.target ln -s /lib/systemd/system/multi-user.target /etc/systemd/system/default.target - echo_message "Disable services" + common::echo_message "Disable services" # NetworkManager.service for service in \ kdump.service \ @@ -160,37 +146,37 @@ distr::common_cfg() { syslog.target do # Most of these aren't enabled, errors are expected... - echo_message " ${service}" + common::echo_message " ${service}" systemctl disable ${service} 2>&1 || true done - echo_message "Set rp_filter to loose mode" + common::echo_message "Set rp_filter to loose mode" echo "net.ipv4.conf.default.rp_filter = 2" >> /etc/sysctl.conf - echo_message "Set SELinux to ${SELINUX^^}" + common::echo_message "Set SELinux to ${SELINUX^^}" sed -i -e "s/^SELINUX[ ]*=.*/SELINUX=${SELINUX,,}/" /etc/selinux/config if [[ ${SELINUX,,} != "enforcing" ]]; then # Relax SELinux for the provisioning as well setenforce Permissive fi - echo_message "Clear network persistent data" + common::echo_message "Clear network persistent data" rm -f /etc/udev/rules.d/70-persistent-net.rules - echo_message "Configure dnf" + common::echo_message "Configure dnf" # bypass update kernel-uek-headers echo "exclude=kernel-uek-headers" >> /etc/dnf/dnf.conf # fix "Metadata file does not match checksum" for public-yum # https://forums.oracle.com/thread/2550364 echo "http_caching=none" >> /etc/dnf/dnf.conf - echo_message "Enable login on serial console ports" + common::echo_message "Enable login on serial console ports" for tty in "hvc0" "ttyS0" "ttyS0" do grep -q "${tty}" /etc/securetty || echo "${tty}" >>/etc/securetty done - echo_message "Remove unneeded RPMs" + common::echo_message "Remove unneeded RPMs" distr::remove_rpms \ iwl7265-firmware \ mozjs17 \ @@ -208,157 +194,20 @@ distr::common_cfg() { # None ####################################### distr::provision() { - distr::ks_log + common::ks_log distr::kernel_config - distr::common_cfg + distr::configure } ####################################### # Cleanup # Globals: -# BUILD_INFO +# None # Arguments: # None # Returns: # None ####################################### distr::cleanup() { - echo_message "Stoppping services" - systemctl stop rsyslog || true - systemctl stop auditd || true - - echo_message "Dnf cleanup" - dnf -q repolist > "${BUILD_INFO}/repolist.txt" - : > /etc/dnf/vars/ociregion - echo "oracle.com" > /etc/dnf/vars/ocidomain - rm -rf /var/cache/dnf/* - rm -rf /var/lib/dnf/* - find /etc/ -name "./*.uln-*" -exec rm -rf {} \; - - # Cleanup and regenerate /etc/machine-id - echo_message "Reset machine id" - : > /etc/machine-id - if ! grep -q setup-machine-id /usr/lib/systemd/system/systemd-firstboot.service; then - sed -i -e "/^ExecStart=/s/$/ --setup-machine-id/" /usr/lib/systemd/system/systemd-firstboot.service - fi - rm -f /var/lib/systemd/random-seed - - echo_message "Cleanup all log files" - rm -f /var/log/anaconda.* /var/log/oraclevm-template.log - rm -f /tmp/ks* - rm -f /root/install.log /root/install.log.syslog /root/anaconda-ks.cfg - : > /etc/resolv.conf - /bin/rm -f /etc/resolv.conf.* - /bin/rm -f /var/lib/dhclient/* - [ -e /var/log/acpid ] && : > /var/log/acpid - [ -e /var/log/messages ] && : > /var/log/messages - [ -e /var/log/btmp ] && : > /var/log/btmp - [ -e /var/log/grubby ] && : > /var/log/grubby - [ -e /var/log/secure ] && : > /var/log/secure - [ -e /var/log/wtmp ] && : > /var/log/wtmp - [ -e /var/log/boot.log ] && : > /var/log/boot.log - [ -e /var/log/dracut.log ] && : > /var/log/dracut.log - [ -e /var/log/tuned/tuned.log ] && : > /var/log/tuned/tuned.log - [ -e /var/log/maillog ] && : > /var/log/maillog - [ -e /var/log/lastlog ] && : > /var/log/lastlog - [ -e /var/log/dnf.log ] && : > /var/log/dnf.log - [ -e /var/log/dnf.librepo.log ] && : > /var/log/dnf.librepo.log - [ -e /var/log/dnf.rpm.log ] && : > /var/log/dnf.rpm.log - [ -e /var/log/ovm-template-config.log ] && rm -f /var/log/ovm-template-config.log - /bin/rm -f /var/log/audit/audit.log* - [ -e /var/log/audit/audit.log ] && : > /var/log/audit/audit.log - - # Lock root user - if [[ "${LOCK_ROOT,,}" = "yes" ]]; then - passwd -d root - passwd -l root - fi - rm -f /etc/ssh/sshd_config.d/01-permitrootlogin.conf - - # cleanup ssh config files - if [ -z "${SSH_KEY_FILE}" ]; then - [ -d /root/.ssh ] && /bin/rm -fr /root/.ssh - else - find /root/.ssh -type f -not -name authorized_keys -delete - fi - - # Rebuild rpmdb to save some space - rpm --rebuilddb - - # Remove man and info pages - echo_message "Exclude documentation: ${EXCLUDE_DOCS^^}" - if [[ "${EXCLUDE_DOCS,,}" = "minimal" ]]; then - rm -rf /usr/share/{man,info} - fi - - # cleanup vnc cache files - if [ -d /root/.vnc ]; then - /bin/rm -f /root/.vnc/*.log - /bin/rm -f /root/.vnc/passwd - fi - - rm -rf /var/log/cups/error_log - rm -rf /var/log/setroubleshoot/setroubleshootd.log - rm -rf /var/log/spooler - # cleanup bash history - [ -e /root/.bash_history ] && : > /root/.bash_history - rm -f /root/.viminfo - rm -rf /.autorelabel - rm -rf /var/log/mail/statistics - rm -rf /var/log/sa/* - rm -rf /var/log/acpid /var/log/boot.log /var/log/cron /var/log/dmesg.* /var/log/ovm* - rm -rf /poweroff - rm -f /etc/ssh/ssh_host_* - rm -rf /root/* - rm -f /etc/udev/rules.d/70-persistent-net.rules - rm -f /etc/udev/rules.d/70-persistent-cd.rules - - find /var/log -type f | while read -r f; do echo -ne '' > "$f"; done; - find /etc/ -name "*.old" -exec rm -f {} \; - rm -f /etc/sysconfig/network-scripts/ifcfg-enp* - rm -rf /lost+found/* - rm -rf /root/.vbox_version - export HISTSIZE=0 - rm -f /var/log/ovm-template-config.log - - echo_message "Save list of installed packages" - rpm -qa --qf "%{name}.%{arch}\n" | sort -u > "${BUILD_INFO}/pkglist.txt" - rpm -qa --qf '"%{NAME}","%{EPOCHNUM}","%{VERSION}","%{RELEASE}","%{ARCH}"\n' | sort > "${BUILD_INFO}/pkglist.csv" - uname -r > "${BUILD_INFO}/kernel.txt" - - history -c - swapoff -a -} - -####################################### -# Final seal of the image -# Globals: -# BUILD_INFO -# Returns: -# None -####################################### -distr::seal() { - echo_message "File cleanup" - : > /var/log/wtmp - : > /var/log/lastlog - rm -f /var/log/audit/audit.log - rm -f /var/log/tuned/tuned.log - rm -rf /root/.gemrc /root/.gem - rm -rf /var/spool/root /var/spool/mail/root - rm -rf /var/lib/NetworkManager - [[ -n "${BUILD_INFO}" ]] && rm -rf "${BUILD_INFO}" - rm -rf /var/tmp/* /tmp/* - - echo_message "Relabel SELinux" - genhomedircon - fixfiles -f -F relabel - restorecon -R / || true - - echo_message "Trim filesystem" - sync; sync; sync - for fs in /boot /; do - echo_message " ${fs}" - dd if=/dev/zero of="${fs}"/EMPTY bs=1M >/dev/null 2>&1 || : - rm -f "${fs}"/EMPTY - done + common::distr_cleanup } diff --git a/oracle-linux-image-tools/env.properties b/oracle-linux-image-tools/env.properties index 9a0d1cb..06e766d 100644 --- a/oracle-linux-image-tools/env.properties +++ b/oracle-linux-image-tools/env.properties @@ -10,8 +10,12 @@ # Workspace for the builder and location of the artifacts WORKSPACE= -# ISO_URL -- location of the ISO file -# Can be local (file://) or remote (https://) but it must be an URL +# Distribution +DISTR= +# ISO_URL -- location of the ISO file for the distribution +# Can be local (file://) or remote (https://) but it must be an URL format +# If you use a "boot" ISO instead of a "full" ISO, you must also specify the +# location of the repositories (see REPO_URL and REPO[] below) ISO_URL= # ISO_CHECKSUM -- checksum for the ISO file ISO_CHECKSUM= @@ -20,22 +24,59 @@ ISO_CHECKSUM= # Optional parameters # -# Distribution (Default is ol7-slim) -# DISTR= # Cloud provider (Default is none) -CLOUD="none" +# CLOUD= + +# Waiting time in minutes for the OS install to complete ("virt-install" part) +# (default is 30 minutes) +# Increase for slow systems or with limitted internet connection bandwidth +# Note: this is only used when SERIAL_CONSOLE=no. There is no timeout when the +# serial console is enabled. +# INSTALL_WAIT_TIME= + +# ISO label of the distribution imge (Default: read from image) +# Specify only if label is not retrieved properly +# ISO_LABEL= + +# Location of the kernel and initrd on the distribution image. +# Typically set at distr level. When unset, let virt-install guess the Location +# (which might fail when building for a recent distr) +# BOOT_LOCATION= + +# OS Variant (Default: retrieved using `osinfo-query`) +# Should only be specified if it cannot be retrieved automatically +# (e.g. host not fully up-to-date or new distribution) +# OS_VARIANT= # Build BUILD_NUMBER (Default is 0) # BUILD_NUMBER= -# Root access to the VM at least one method must be provided: -# Root account password (Default is empty) -# SSH_PASSWORD= -# Path to an ssh private/public key pair (Default none). The public key will -# be installed for the root user -# SSH_KEY_FILE= -# Lock root account after provisioning? (Default: yes) -# LOCK_ROOT= +# Root access to the VM +# This is *not* needed for the build, use this only if you need root access +# to the generated image. +# (Parameters have been renamed to ensure safe defaults when using an older +# property file) +# +# Root password (Default: locked) +# The format of the ROOT_PASSWORD property is described in +# https://libguestfs.org/virt-builder.1.html#users-and-passwords +# E.g.: +# ROOT_PASSWORD="file:PASSWORD_FILE_NAME" +# ROOT_PASSWORD="password:PASSWORD_STRING" +# ROOT_PASSWORD= +# +# SSH Public key for the root user (default: no public key) +# The format of the ROOT_SSH_KEY property is described in +# https://libguestfs.org/virt-builder.1.html#ssh-keys +# E.g.: +# ROOT_SSH_KEY="file:PUBLIC_KEY_FILE_NAME" +# ROOT_SSH_KEY="string:PUBLIC_KEY_STRING" +# ROOT_SSH_KEY= +# +# Control root login over SSH (Default: prohibit-password) +# Possible values: yes, prohibit-password, forced-commands-only, or no. +# see sshd_config(5) for more details +# PERMIT_ROOT_LOGIN= # # Override examples @@ -89,27 +130,18 @@ CLOUD="none" # Allocated disk size for the image, default is distribution / cloud specific # DISK_SIZE_GB= -# If you experience issues when building in a nested virtualization environment, -# you can disable X2APIC in VirtualBox for the build: on/off (default: on) -# X2APIC= - # Capture serial console in serial-console.txt during Kickstart. (Yes, No, # default: no) -- Useful for debugging Kickstart issues. # SERIAL_CONSOLE= -# We use the full path for Hashicorp Packer from the Linux package generating tool -# PACKER="/usr/bin/packer" -# PACKER_BUILD_OPTIONS="-on-error=ask" -# Packer builder (virtualbox-iso.x86-64 or qemu.x86-64) -# Vagrant virtualbox images require virtualbox-iso.x86-64 builder -# PACKER_BUILDER="virtualbox-iso.x86-64" -# Location of the QEMU binary -# QEMU_BINARY="" - - # If defined, override generated VM_NAME # VM_NAME= +# Path to a cache directory for downloaded images (absolute or relative to WORKSPACE) +# Directory will be created it if does not exists, but the parent directory must exists. +# (Default: .cache) +# CACHE_DIR= + # If your ISO_URL points to a boot iso, you need to provide: # - an URL to an installation tree on a remote server # - optionally an associative array of additional yum repositories that may @@ -127,6 +159,9 @@ CLOUD="none" # CLOUD_INIT= # cloud-init user (Default is "opc" for OCI and "cloud-user" for OLVM) # CLOUD_USER= +# OCI: Install OCI repo mapper script to use the OCI yum miror in the region +# (Default: Yes; when set to No, the vm will use the public yum) +# OCI_REPO_MAPPER= # OLVM: Custom script passed to Cloud-init (user data) -- Example: # CUSTOM_SCRIPT="#cloud-config # disable_root: 1" @@ -163,3 +198,7 @@ CLOUD="none" # Vagrant all providers # Install developer release packages (Yes/No, default: no) # VAGRANT_DEVELOPER_REPOS= + +# UTM +# Password for the OPC user +# OPC_PASSWORD= diff --git a/oracle-linux-image-tools/env.properties.defaults b/oracle-linux-image-tools/env.properties.defaults index 509d57c..fc62b6e 100644 --- a/oracle-linux-image-tools/env.properties.defaults +++ b/oracle-linux-image-tools/env.properties.defaults @@ -2,11 +2,27 @@ # Do NOT change anything in this file, customisation must be done in separate # env file. -# Provide basic default for the image build: ol7 distribution, not cloud specific -DISTR="ol7-slim" +# Provide basic default for the image build: distribution not set, not cloud specific +DISTR="" CLOUD="none" CUSTOM="none" +# Label of the ISO distribution image. It is normally retrieved from the disk +# image and should only be specified if this is failing. +ISO_LABEL= + +# Waiting time in minutes for the OS install to complete ("virt-install" part) +INSTALL_WAIT_TIME=30 + +# Location of the kernel and initrd on the distribution media. +# Typically set at distr level. When unset, let virt-install guess the Location +# (which might fail when building for a recent distr) +BOOT_LOCATION= + +# OS Variant is required by `virt-install`. By default it is retrieved using +# `osinfo-query` and should not be specified. +OS_VARIANT= + # Build number BUILD_NUMBER=0 @@ -27,32 +43,20 @@ DISK_SIZE_GB=15 # Set /tmp to tmpfs TMP_IN_TMPFS="no" -# Root access to the VM. You must provide either a password or a key file. -SSH_PASSWORD= -SSH_KEY_FILE= - -# Lock root account after provisioning? -LOCK_ROOT="yes" - -# Use X2APIC in VirtualBox for the build? (on, off) -X2APIC="on" +# Root password and remote access +ROOT_PASSWORD="locked" +PERMIT_ROOT_LOGIN="prohibit-password" # Capture serial console in serial-console.txt during Kickstart? (yes, no) SERIAL_CONSOLE="no" -# We use the full path for Hashicorp Packer from the Linux package generating tool -PACKER="/usr/bin/packer" -PACKER_BUILD_OPTIONS="-on-error=ask" - -# Packer builder (virtualbox-iso.x86-64 or qemu.x86-64) -# Vagrant virtualbox images require virtualbox-iso.x86-64 builder -PACKER_BUILDER="virtualbox-iso.x86-64" -# Location of the QEMU binary -QEMU_BINARY="" - # If defined, override generated VM_NAME VM_NAME= +# Path to a cache directory for downloaded images (absolute or relative to WORKSPACE) +# Directory will be created it if does not exists, but the parent directory must exists. +CACHE_DIR=".cache" + # The following two parameters can be specified when using a boot install image # instead of a full DVD ISO image # URL to an installation tree on a remote server diff --git a/oracle-linux-image-tools/images/.gitattributes b/oracle-linux-image-tools/images/.gitattributes new file mode 100644 index 0000000..24a8e87 --- /dev/null +++ b/oracle-linux-image-tools/images/.gitattributes @@ -0,0 +1 @@ +*.png filter=lfs diff=lfs merge=lfs -text diff --git a/oracle-linux-image-tools/images/olit.png b/oracle-linux-image-tools/images/olit.png new file mode 100644 index 0000000..61f154a --- /dev/null +++ b/oracle-linux-image-tools/images/olit.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4e6d3aaaf30e56ce03ecd3d29f3a05cecfbc9f41c3a7b5514c65ff758f66f51 +size 222432 diff --git a/oracle-linux-image-tools/packer-template/build.pkr.hcl b/oracle-linux-image-tools/packer-template/build.pkr.hcl deleted file mode 100644 index 2f91720..0000000 --- a/oracle-linux-image-tools/packer-template/build.pkr.hcl +++ /dev/null @@ -1,30 +0,0 @@ -build { - sources = [ - "virtualbox-iso.x86-64", - "qemu.x86-64", - "qemu.aarch64", - ] - provisioner "file" { - source = var.packer_files - destination = "/tmp" - - } - provisioner "shell" { - script = var.provision_script - environment_vars = [ - "OLIT_ACTION=provision", - ] - } - provisioner "file" { - only = local.get_build_info - direction = "download" - source = "${var.build_info}/*" - destination = "${local.output_directory}/" - } - provisioner "shell" { - script = var.provision_script - environment_vars = [ - "OLIT_ACTION=seal", - ] - } -} diff --git a/oracle-linux-image-tools/packer-template/qemu-aarch64.pkr.hcl b/oracle-linux-image-tools/packer-template/qemu-aarch64.pkr.hcl deleted file mode 100644 index ca4da89..0000000 --- a/oracle-linux-image-tools/packer-template/qemu-aarch64.pkr.hcl +++ /dev/null @@ -1,88 +0,0 @@ -# aarch64 build with qemu-kvm - -# Hack to get cached file name -# Needed to workaround https://github.com/hashicorp/packer-plugin-qemu/issues/35 -# This will only work for remote files as local file are not cached! For local -# files, copy the ISO directly in the cache directory... -variable "cache_dir" { - description = "Packer cache directory" - type = string - default = env("PACKER_CACHE_DIR") == "" ? "./packer_cache" : env("PACKER_CACHE_DIR") -} - -locals { - iso_basename = regex("[^/]*$", var.iso_url) -} - -# KVM or TCG? -variable "accel" { - description = "Set accel to kvm if kvm is available" - type = string - default = "" -} - -locals { - accel = var.accel == "kvm" ? "kvm" : "tcg" - cpu = var.accel == "kvm" ? "host" : "cortex-a57" -} - -source "qemu" "aarch64" { - iso_url = var.iso_url - iso_checksum = var.iso_checksum - iso_target_path = "${var.cache_dir}/${local.iso_basename}" - output_directory = local.output_directory - vm_name = "System.img" - net_device = "virtio-net" - cdrom_interface = "virtio-scsi" - disk_interface = "virtio-scsi" - disk_size = var.disk_size - cpus = var.cpus - memory = var.memory - format = "raw" - headless = "true" - ssh_username = "root" - ssh_password = var.ssh_password - ssh_private_key_file = var.ssh_private_key_file - ssh_port = 22 - ssh_wait_timeout = "180m" - http_directory = local.http_directory - boot_wait = "20s" - boot_command = var.boot_command - shutdown_command = var.shutdown_command - qemu_binary = var.qemu_binary - qemuargs = concat( - var.qemu_args, - [ - ["-machine", "virt,accel=${local.accel},dump-guest-core=off,gic-version=2,pflash0=libvirt-pflash0-format"], - ["-cpu", local.cpu], - [ - "-blockdev", - <<-EOT - { - "driver": "raw", - "node-name": "libvirt-pflash0-format", - "file": - { - "driver": "file", - "node-name": "libvirt-pflash0-storage", - "filename": "/usr/share/edk2/aarch64/QEMU_EFI-silent-pflash.raw", - "auto-read-only": true, - "discard":"unmap" - }, - "read-only": true - } - EOT - ], - ["-boot", "strict=on"], - ["-device", "virtio-gpu-pci,id=video0,max_outputs=1"], - ["-device", "virtio-net,netdev=user.0"], - ["-device", "virtio-scsi-pci,id=scsi0"], - ["-device", "scsi-hd,bus=scsi0.0,scsi-id=0,drive=drive0"], - ["-device", "scsi-cd,drive=cdrom0"], - ["-device", "qemu-xhci,p2=15,p3=15,id=usb"], - ["-device", "usb-kbd,id=input1"], - ["-drive", "if=none,file=${var.cache_dir}/${local.iso_basename},index=1,id=cdrom0,media=cdrom"], - ["-drive", "if=none,file=${local.output_directory}/System.img,id=drive0,cache=writeback,discard=ignore,format=raw"] - ] - ) -} diff --git a/oracle-linux-image-tools/packer-template/qemu-x86-64.pkr.hcl b/oracle-linux-image-tools/packer-template/qemu-x86-64.pkr.hcl deleted file mode 100644 index eae63c6..0000000 --- a/oracle-linux-image-tools/packer-template/qemu-x86-64.pkr.hcl +++ /dev/null @@ -1,32 +0,0 @@ -# x86-64 build with qemu-kvm - -source "qemu" "x86-64" { - accelerator = "kvm" - iso_url = var.iso_url - iso_checksum = var.iso_checksum - output_directory = local.output_directory - vm_name = "System.img" - net_device = "virtio-net" - disk_interface = "virtio-scsi" - disk_size = var.disk_size - cpus = var.cpus - memory = var.memory - format = "raw" - headless = "true" - ssh_username = "root" - ssh_password = var.ssh_password - ssh_private_key_file = var.ssh_private_key_file - ssh_port = 22 - ssh_wait_timeout = "30m" - http_directory = local.http_directory - boot_wait = "20s" - boot_command = var.boot_command - shutdown_command = var.shutdown_command - qemu_binary = var.qemu_binary - qemuargs = concat( - var.qemu_args, - [ - ["-cpu", "host"] - ] - ) -} diff --git a/oracle-linux-image-tools/packer-template/variables.pkr.hcl b/oracle-linux-image-tools/packer-template/variables.pkr.hcl deleted file mode 100644 index 448961c..0000000 --- a/oracle-linux-image-tools/packer-template/variables.pkr.hcl +++ /dev/null @@ -1,122 +0,0 @@ -# Variables and locals declaration - -# Environment -variable "workspace" { - description = "Workspace directory" - type = string -} - -variable "packer_files" { - description = "Directory for the provisioning files" - type = string -} - -variable "provision_script" { - description = "Provisioning script" - type = string -} - -# ISO -variable "iso_url" { - description = "URL of the ISO" - type = string -} - -variable "iso_checksum" { - description = "Checksum of the ISO" - type = string -} - -# Generic VM properties -variable "vm_name" { - description = "Name of the VM" - type = string -} - -variable "disk_size" { - description = "Disk size for the VM in MB" - type = number -} - -variable "memory" { - description = "Memory for the VM in MB" - type = number -} - -variable "cpus" { - description = "Number of CPUs for the VM in MB" - type = number -} - -variable "ssh_password" { - description = "Password for the root user" - type = string - default = null -} - -variable "ssh_private_key_file" { - description = "SSH private key file for the root user" - type = string - default = null -} - -variable "boot_command" { - description = "Boot command" - type = list(string) -} - -variable "shutdown_command" { - description = "shutdown_command" - type = string -} - -# VirtualBox properties -variable "guest_additions_url" { - description = "URL of the VirtualBox Guest Additions" - type = string - default = null -} - -variable "guest_additions_sha256" { - description = "Checksum of the VirtualBox Guest Additions" - type = string - default = null -} - -variable "vbox_manage" { - description = "VirtualBox vboxmanage aditional stanzas (for the serial console)" - type = list(list(string)) - default = [] -} - -variable "x2apic" { - description = "X2APIC for VirtualBox" - type = string - default = "on" -} - -# QEMU properties -variable "qemu_binary" { - description = "QEMU binary" - type = string - default = null -} - -variable "qemu_args" { - description = "QEMU Arguments" - type = list(list(string)) - default = [] -} - -variable "build_info" { - description = "Guest directory with build information" - type = string - default = "" -} - -# Locals -locals { - output_directory = "${var.workspace}/${var.vm_name}" - http_directory = var.workspace - get_build_info = var.build_info == "" ? [ "none" ] : [] -} diff --git a/oracle-linux-image-tools/packer-template/virtualbox-x86-64.pkr.hcl b/oracle-linux-image-tools/packer-template/virtualbox-x86-64.pkr.hcl deleted file mode 100644 index 0538a27..0000000 --- a/oracle-linux-image-tools/packer-template/virtualbox-x86-64.pkr.hcl +++ /dev/null @@ -1,38 +0,0 @@ -# x86-64 build with VirtualBox - -source "virtualbox-iso" "x86-64" { - guest_os_type = "Oracle_64" - iso_url = var.iso_url - iso_checksum = var.iso_checksum - output_directory = local.output_directory - vm_name = var.vm_name - hard_drive_interface = "sata" - disk_size = var.disk_size - guest_additions_mode = "attach" - guest_additions_url = var.guest_additions_url - guest_additions_sha256 = var.guest_additions_sha256 - format = "ova" - headless = "true" - ssh_username = "root" - ssh_password = var.ssh_password - ssh_private_key_file = var.ssh_private_key_file - ssh_port = 22 - ssh_wait_timeout = "60m" - http_directory = local.http_directory - boot_wait = "20s" - boot_command = var.boot_command - shutdown_command = var.shutdown_command - vboxmanage = concat( - var.vbox_manage, - [ - ["modifyvm", "{{.Name}}", "--x2apic", var.x2apic], - ["modifyvm", "{{.Name}}", "--memory", var.memory], - ["modifyvm", "{{.Name}}", "--cpus", var.cpus], - ["modifyvm", "{{.Name}}", "--nictype1", "virtio"], - ] - ) - vboxmanage_post = [ - ["modifyvm", "{{.Name}}", "--uart1", "off", "--uartmode1", "disconnected"], - ["modifyvm", "{{.Name}}", "--x2apic", "on"], - ] -}