feat(azure): add KMS FIPS image (#142)

* feat(azure): add FIPS image * ci: bump KMS to 4.17.0 * docs: add example to gen cert with certbot * fix: image build * fix: reusing github cache * feat: add KMS FIPS playbook * chore: bump KMS version to 4.17.0
Cosmian · Jul 8, 2024 · 4e315cc · 4e315cc
1 parent a122f81
commit 4e315cc
Show file tree

Hide file tree

Showing 33 changed files with 296 additions and 152 deletions.
diff --git a/.github/scripts/azure-create-vm-image-definition.sh b/.github/scripts/azure-create-vm-image-definition.sh
@@ -5,7 +5,7 @@ set +e
 #!/bin/bash
 
 # Define the lists
-products=("base-image" "cosmian-vm" "kms" "ai-runner")
+products=("base-image" "cosmian-vm" "kms" "kms-fips" "ai-runner")
 distribs=("rhel" "ubuntu")
 technos=("sev" "tdx")
 

diff --git a/.github/scripts/azure-packer-build.sh b/.github/scripts/azure-packer-build.sh
@@ -27,7 +27,6 @@ fi
 PACKER_FILE="azure.pkr.hcl"
 
 if [ "$KEEP_OS_DISK" = "true" ]; then
-  OS_DISK_NAME="TEMPLATE_PRODUCT-TEMPLATE_DISTRIBUTION-TEMPLATE_TECHNO-TEMPLATE_AZURE_IMAGE_VERSION-vm-TEMPLATE_COSMIAN_VM_VERSION"
   sed -i "s#TEMPLATE_OS_DISK_NAME#$OS_DISK_NAME#g" "$PACKER_FILE"
 else
   sed -i "s#  temp_os_disk_name         = \"TEMPLATE_OS_DISK_NAME\"##g" "$PACKER_FILE"

diff --git a/.github/scripts/gcp-new-instance.sh b/.github/scripts/gcp-new-instance.sh
@@ -43,13 +43,19 @@ else
     # Cosmian Ubuntu SEV
     IMAGE="base-image-0-1-5-ubuntu-sev"
     IMAGE_PROJECT="cosmian-dev"
+    # Cosmian KMS Ubuntu SEV
+    IMAGE="cosmian-vm-1-2-4-kms-4-16-0-sev-ubuntu"
+    IMAGE_PROJECT="cosmian-dev"
   else
     # RHEL SEV
     IMAGE="rhel-9-v20240515"
     IMAGE_PROJECT="rhel-cloud"
     # Cosmian Ubuntu SEV
     IMAGE="base-image-0-1-5-rhel-sev"
     IMAGE_PROJECT="cosmian-dev"
+    # Cosmian Ubuntu SEV
+    IMAGE="cosmian-vm-1-2-4-kms-4-16-0-sev-rhel"
+    IMAGE_PROJECT="cosmian-dev"
   fi
   gcloud beta compute instances create "$NAME" \
     --machine-type n2d-standard-2 \

diff --git a/.github/scripts/push_packages.sh b/.github/scripts/push_packages.sh
@@ -2,7 +2,7 @@
 
 set -ex
 
-VERSION=1.2.4
+VERSION=1.2.5
 
 set -x
 if [[ ${GITHUB_REF} = *'refs/tags/'* ]]; then

diff --git a/.github/workflows/aws_ansible.yml b/.github/workflows/aws_ansible.yml
@@ -56,11 +56,7 @@ jobs:
           aws ec2 create-security-group --group-name ${CI_INSTANCE}-ansible-sg --description "Security group for ansible test"
           aws ec2 authorize-security-group-ingress --group-name ${CI_INSTANCE}-ansible-sg --protocol tcp --port 22 --cidr 0.0.0.0/0
           aws ec2 authorize-security-group-ingress --group-name ${CI_INSTANCE}-ansible-sg --protocol tcp --port 5555 --cidr 0.0.0.0/0
-          if [ "${{ inputs.product }}" = "kms" ]; then
-            aws ec2 authorize-security-group-ingress --group-name ${CI_INSTANCE}-ansible-sg --protocol tcp --port 443 --cidr 0.0.0.0/0
-          elif [ "${{ inputs.product }}" = "ai-runner" ]; then
-            aws ec2 authorize-security-group-ingress --group-name ${CI_INSTANCE}-ansible-sg --protocol tcp --port 443 --cidr 0.0.0.0/0
-          fi
+          aws ec2 authorize-security-group-ingress --group-name ${CI_INSTANCE}-ansible-sg --protocol tcp --port 443 --cidr 0.0.0.0/0
 
           if [ "${{ inputs.distrib }}" = "ubuntu" ]; then
             export AMI_BASE=$(aws ec2 describe-images --filters "Name=name,Values=ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-20240523.1" --query "Images[*].{ID:ImageId}" --output text)

diff --git a/.github/workflows/aws_image.yml b/.github/workflows/aws_image.yml
@@ -119,11 +119,7 @@ jobs:
           aws ec2 create-security-group --group-name ${CI_INSTANCE}-ci-sg --description "Security group for image test"
           aws ec2 authorize-security-group-ingress --group-name ${CI_INSTANCE}-ci-sg --protocol tcp --port 22 --cidr 0.0.0.0/0
           aws ec2 authorize-security-group-ingress --group-name ${CI_INSTANCE}-ci-sg --protocol tcp --port 5555 --cidr 0.0.0.0/0
-          if [ "${{ inputs.product }}" = "kms" ]; then
-            aws ec2 authorize-security-group-ingress --group-name ${CI_INSTANCE}-ci-sg --protocol tcp --port 443 --cidr 0.0.0.0/0
-          elif [ "${{ inputs.product }}" = "ai-runner" ]; then
-            aws ec2 authorize-security-group-ingress --group-name ${CI_INSTANCE}-ci-sg --protocol tcp --port 443 --cidr 0.0.0.0/0
-          fi
+          aws ec2 authorize-security-group-ingress --group-name ${CI_INSTANCE}-ci-sg --protocol tcp --port 443 --cidr 0.0.0.0/0
 
           AMI=$(aws ec2 run-instances \
             --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=$CI_INSTANCE}]" \
@@ -268,7 +264,7 @@ jobs:
           elif [ "${{ inputs.product }}" = "ai-runner" ]; then
             NEW_IMAGE_NAME=cosmian-vm-${VERSION}-ai-runner-${AI_RUNNER_VERSION}-${{ inputs.techno }}-${{ inputs.distrib }}
           else
-            NEW_IMAGE_NAME=cosmian-vm-${VERSION}-kms-${KMS_VERSION}-${{ inputs.techno }}-${{ inputs.distrib }}
+            NEW_IMAGE_NAME=cosmian-vm-${VERSION}-${{ inputs.product }}-${KMS_VERSION}-${{ inputs.techno }}-${{ inputs.distrib }}
           fi
 
           AMI=$(aws ec2 describe-images --filters "Name=name,Values=$IMAGE_NAME" --query 'Images[].ImageId' --output text)

diff --git a/.github/workflows/aws_main.yml b/.github/workflows/aws_main.yml
@@ -28,7 +28,7 @@ jobs:
       techno: sev
       distrib: ${{ matrix.distrib }}
       product: ${{ matrix.product }}
-      kms-version: 4.16.0
+      kms-version: 4.17.0
       ai-runner-version: 0.3.0
 
   post-clean-resources:

diff --git a/.github/workflows/azure_ansible.yml b/.github/workflows/azure_ansible.yml
@@ -84,8 +84,6 @@ jobs:
 
           if [ "${{ inputs.product }}" = "cosmian-vm" ]; then
             az vm open-port -g $RESOURCE_GROUP -n $CI_INSTANCE --priority 100 --port 5555
-          elif [ "${{ inputs.product }}" = "ai-runner" ]; then
-            az vm open-port -g $RESOURCE_GROUP -n $CI_INSTANCE --priority 100 --port 5555,443
           else
             az vm open-port -g $RESOURCE_GROUP -n $CI_INSTANCE --priority 100 --port 5555,443
           fi
@@ -106,11 +104,17 @@ jobs:
 
           COSMIAN_VM_VERSION="last_build/${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
 
+          PLAYBOOK="${{ inputs.product }}-playbook.yml"
+
           export ANSIBLE_HOST_KEY_CHECKING=False
           for i in {1..1}
           do
             echo "Iteration: $i"
-            ansible-playbook ${{ inputs.product }}-playbook.yml -i ${IP_ADDR}, -u cosmian -e cosmian_vm_version=$COSMIAN_VM_VERSION -e cosmian_kms_version=${{ inputs.kms-version }} -e cosmian_ai_runner_version=${{ inputs.ai-runner-version }} --skip-tags role-cleanup
+            ansible-playbook $PLAYBOOK -i ${IP_ADDR}, -u cosmian \
+              -e cosmian_vm_version=$COSMIAN_VM_VERSION \
+              -e cosmian_kms_version=${{ inputs.kms-version }} \
+              -e cosmian_ai_runner_version=${{ inputs.ai-runner-version }} \
+              --skip-tags role-cleanup
           done
 
       - name: Stop and delete Azure instance

diff --git a/.github/workflows/azure_image.yml b/.github/workflows/azure_image.yml
@@ -77,10 +77,10 @@ jobs:
           if [[ ${GITHUB_REF} = *'refs/tags/'* ]]; then
             if [ "${{ inputs.product }}" = "cosmian-vm" ]; then
               IMAGE_VERSION="${GITHUB_REF_NAME}"
-            elif [ "${{ inputs.product }}" = "kms" ]; then
-              IMAGE_VERSION="${{ inputs.kms-version }}"
-            else
+            elif [ "${{ inputs.product }}" = "ai-runner" ]; then
               IMAGE_VERSION="${{ inputs.ai-runner-version }}"
+            else
+              IMAGE_VERSION="${{ inputs.kms-version }}"
             fi
           fi
 
@@ -108,8 +108,6 @@ jobs:
 
           if [ "${{ inputs.product }}" = "cosmian-vm" ]; then
             az vm open-port -g $RESOURCE_GROUP -n $CI_INSTANCE --priority 100 --port 5555
-          elif [ "${{ inputs.product }}" = "ai-runner" ]; then
-            az vm open-port -g $RESOURCE_GROUP -n $CI_INSTANCE --priority 100 --port 5555,443
           else
             az vm open-port -g $RESOURCE_GROUP -n $CI_INSTANCE --priority 100 --port 5555,443
           fi
@@ -130,11 +128,17 @@ jobs:
             COSMIAN_VM_VERSION="last_build/${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
           fi
 
+          PLAYBOOK="${{ inputs.product }}-playbook.yml"
+
           cd ansible
           python3 -m pip install -r python_modules.txt
 
           export ANSIBLE_HOST_KEY_CHECKING=False
-          ansible-playbook ${{ inputs.product }}-playbook.yml -i ${IP_ADDR}, -u cosmian -e cosmian_vm_version=$COSMIAN_VM_VERSION -e cosmian_kms_version=${{ inputs.kms-version }} -e cosmian_ai_runner_version=${{ inputs.ai-runner-version }} --tags check-${{ inputs.product }}
+          ansible-playbook $PLAYBOOK -i ${IP_ADDR}, -u cosmian \
+            -e cosmian_vm_version=$COSMIAN_VM_VERSION \
+            -e cosmian_kms_version=${{ inputs.kms-version }} \
+            -e cosmian_ai_runner_version=${{ inputs.ai-runner-version }} \
+            --tags check-${{ inputs.product }}
 
       - name: Stop and delete Azure instance
         if: success() || cancelled()

diff --git a/.github/workflows/azure_main.yml b/.github/workflows/azure_main.yml
@@ -28,7 +28,7 @@ jobs:
       techno: sev
       distrib: ${{ matrix.distrib }}
       product: ${{ matrix.product }}
-      kms-version: 4.16.0
+      kms-version: 4.17.0
       ai-runner-version: 0.3.0
 
   post-clean-resources:

diff --git a/.github/workflows/azure_packer_build.yml b/.github/workflows/azure_packer_build.yml
@@ -85,7 +85,7 @@ jobs:
                 export AZURE_IMAGE_VERSION="$COSMIAN_VM_VERSION"
               elif [ "${{ inputs.product }}" = "ai-runner" ]; then
                 export AZURE_IMAGE_VERSION="$AI_RUNNER_VERSION"
-              elif [ "${{ inputs.product }}" = "kms" ]; then
+              elif [ "${{ inputs.product }}" = "kms" ] || [ "${{ inputs.product }}" = "kms-fips" ]; then
                 export AZURE_IMAGE_VERSION="$KMS_VERSION"
               else
                 echo "Unexpected product"
@@ -96,17 +96,17 @@ jobs:
 
           fi
 
-          OS_DISK="${{ inputs.product }}-${{ inputs.distrib }}-${{ inputs.techno }}-${AZURE_IMAGE_VERSION}-vm-${COSMIAN_VM_VERSION}"
-          az disk delete -g "$RESOURCE_GROUP" --name "$OS_DISK" --yes
+          OS_DISK_NAME="${{ inputs.product }}-${{ inputs.distrib }}-${{ inputs.techno }}-${AZURE_IMAGE_VERSION}-vm-${COSMIAN_VM_VERSION}"
+          az disk delete -g "$RESOURCE_GROUP" --name "$OS_DISK_NAME" --yes
 
           bash ../.github/scripts/azure-packer-build.sh "${{ inputs.product }}" "${{ inputs.distrib }}"
 
           # For Azure, create a final VHD from os disk
           if [[ ${GITHUB_REF} = *'refs/tags/'* ]]; then
-            sas=$(az disk grant-access --resource-group "$RESOURCE_GROUP" --name "$OS_DISK" --duration-in-seconds 3600 --access-level Read --query accessSas -o tsv)
+            sas=$(az disk grant-access --resource-group "$RESOURCE_GROUP" --name "$OS_DISK_NAME" --duration-in-seconds 3600 --access-level Read --query accessSas -o tsv)
 
-            az storage blob copy start --destination-blob $OS_DISK.vhd --destination-container packer --account-name "$AZURE_STORAGE_ACCOUNT_NAME" --account-key "$AZURE_STORAGE_ACCOUNT_KEY" --source-uri "$sas"
+            az storage blob copy start --destination-blob $OS_DISK_NAME.vhd --destination-container packer --account-name "$AZURE_STORAGE_ACCOUNT_NAME" --account-key "$AZURE_STORAGE_ACCOUNT_KEY" --source-uri "$sas"
 
-            # az disk revoke-access --resource-group "$RESOURCE_GROUP" --name "$OS_DISK"
-            # az disk delete -g "$RESOURCE_GROUP" --name "$OS_DISK" --yes
+            # az disk revoke-access --resource-group "$RESOURCE_GROUP" --name "$OS_DISK_NAME"
+            # az disk delete -g "$RESOURCE_GROUP" --name "$OS_DISK_NAME" --yes
           fi
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -12,15 +12,13 @@ jobs:
     runs-on: ${{ inputs.distrib }}
 
     steps:
-      - name: Install dependencies
-        run: |
-          sudo curl -fsSLo /usr/share/keyrings/intel-sgx-deb.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key
-          echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-sgx-deb.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu jammy main" | sudo tee /etc/apt/sources.list.d/intel-sgx.list
-          sudo apt-get update && sudo apt-get install -y tpm2-tools libtss2-dev libtdx-attest-dev
-
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Fix permissions
+        run: |
+          sudo chown -R $USER /lib/x86_64-linux-gnu/
+
       - name: Cache dependencies
         id: cargo-cache
         uses: Swatinem/rust-cache@v2
@@ -31,6 +29,13 @@ jobs:
             - /lib/x86_64-linux-gnu/libtdx_attest.so.1
             - /lib/x86_64-linux-gnu/libtdx_attest.so.1.21.100.3
 
+      - name: Install dependencies
+        if: steps.cargo-cache.outputs.cache-hit != 'true'
+        run: |
+          sudo curl -fsSLo /usr/share/keyrings/intel-sgx-deb.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key
+          echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-sgx-deb.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu jammy main" | sudo tee /etc/apt/sources.list.d/intel-sgx.list
+          sudo apt-get update && sudo apt-get install -y tpm2-tools libtss2-dev libtdx-attest-dev
+
       - name: Install Rust toolchain and components
         if: steps.cargo-cache.outputs.cache-hit != 'true'
         uses: dtolnay/rust-toolchain@stable
@@ -63,9 +68,11 @@ jobs:
         run: cargo test
 
       - name: Cargo build
+        if: steps.cargo-cache.outputs.cache-hit != 'true'
         run: cargo build --release
 
       - name: Build Debian and RPM packages
+        if: steps.cargo-cache.outputs.cache-hit != 'true'
         run: |
           rm -rf target/debian
           rm -rf target/generate-rpm
@@ -81,6 +88,7 @@ jobs:
           fi
 
       - name: Upload artifacts
+        if: steps.cargo-cache.outputs.cache-hit != 'true'
         uses: actions/upload-artifact@v4
         with:
           name: cosmian_vm_${{ inputs.distrib }}
@@ -99,8 +107,12 @@ jobs:
           retention-days: 1
           if-no-files-found: error
 
+    outputs:
+      cache-hit: ${{ steps.cargo-cache.outputs.cache-hit }}
+
   push:
     needs: check
+    if: needs.check.outputs.cache-hit != 'true'
     name: (${{ inputs.distrib }}) package.cosmian.com
     runs-on: [self-hosted, no-tee]
     container:

diff --git a/.github/workflows/gcp_ansible.yml b/.github/workflows/gcp_ansible.yml
@@ -120,11 +120,6 @@ jobs:
               --network=default \
               --allow=tcp:22,tcp:5555 \
               --target-tags=$CI_INSTANCE-fw
-          elif [ "${{ inputs.product }}" = "ai-runner" ]; then
-            gcloud compute firewall-rules create ${CI_INSTANCE}-allow-ports-cli \
-              --network=default \
-              --allow=tcp:22,tcp:443,tcp:5555 \
-              --target-tags=$CI_INSTANCE-fw
           else
             gcloud compute firewall-rules create ${CI_INSTANCE}-allow-ports-cli \
               --network=default \

diff --git a/.github/workflows/gcp_image.yml b/.github/workflows/gcp_image.yml
@@ -165,11 +165,6 @@ jobs:
               --network=default \
               --allow=tcp:22,tcp:5555 \
               --target-tags=$CI_INSTANCE-fw
-          elif [ "${{ inputs.product }}" = "ai-runner" ]; then
-            gcloud compute firewall-rules create ${CI_INSTANCE}-allow-ports-cli \
-              --network=default \
-              --allow=tcp:22,tcp:443,tcp:5555 \
-              --target-tags=$CI_INSTANCE-fw
           else
             gcloud compute firewall-rules create ${CI_INSTANCE}-allow-ports-cli \
               --network=default \
@@ -253,7 +248,7 @@ jobs:
               NEW_IMAGE_NAME=cosmian-vm-${VERSION}-ai-runner-${AI_RUNNER_VERSION}-${{ inputs.techno }}-${{ inputs.distrib }}
             else
               LICENSE=${{ secrets.GCP_KMS_UBUNTU_LICENSE }}
-              NEW_IMAGE_NAME=cosmian-vm-${VERSION}-kms-${KMS_VERSION}-${{ inputs.techno }}-${{ inputs.distrib }}
+              NEW_IMAGE_NAME=cosmian-vm-${VERSION}-${{ inputs.product }}-${KMS_VERSION}-${{ inputs.techno }}-${{ inputs.distrib }}
             fi
           else
             if [ "${{ inputs.product }}" = "cosmian-vm" ]; then
@@ -263,7 +258,7 @@ jobs:
               NEW_IMAGE_NAME=cosmian-vm-${VERSION}-ai-runner-${AI_RUNNER_VERSION}-${{ inputs.techno }}-${{ inputs.distrib }}
             else
               LICENSE=${{ secrets.GCP_KMS_RHEL_LICENSE }}
-              NEW_IMAGE_NAME=cosmian-vm-${VERSION}-kms-${KMS_VERSION}-${{ inputs.techno }}-${{ inputs.distrib }}
+              NEW_IMAGE_NAME=cosmian-vm-${VERSION}-${{ inputs.product }}-${KMS_VERSION}-${{ inputs.techno }}-${{ inputs.distrib }}
             fi
           fi
 

diff --git a/.github/workflows/gcp_main.yml b/.github/workflows/gcp_main.yml
@@ -33,7 +33,7 @@ jobs:
       min-cpu-platform: AMD Milan
       mode: beta
       product: ${{ matrix.product }}
-      kms-version: 4.16.0
+      kms-version: 4.17.0
       ai-runner-version: 0.3.0
 
   post-clean-gcp-resources:

diff --git a/.github/workflows/nightly_aws_main.yml b/.github/workflows/nightly_aws_main.yml
@@ -33,7 +33,7 @@ jobs:
       techno: ${{ matrix.techno }}
       distrib: ${{ matrix.distrib }}
       product: ${{ matrix.product }}
-      kms-version: 4.16.0
+      kms-version: 4.17.0
       ai-runner-version: 0.3.0
       base-version: 0.1.5
 
@@ -60,7 +60,7 @@ jobs:
   #     techno: sev
   #     distrib: ${{ matrix.distrib }}
   #     product: ${{ matrix.product }}
-  #     kms-version: 4.16.0
+  #     kms-version: 4.17.0
   #     ai-runner-version: 0.3.0
 
   post-clean-resources:

diff --git a/.github/workflows/nightly_azure_main.yml b/.github/workflows/nightly_azure_main.yml
@@ -19,13 +19,15 @@ jobs:
       max-parallel: 3
       matrix:
         distrib: [rhel, ubuntu]
-        product: [cosmian-vm, kms, ai-runner]
+        product: [cosmian-vm, kms, kms-fips, ai-runner]
         techno: [sev, tdx]
         exclude:
           - product: ai-runner
             distrib: rhel
           - techno: tdx
             distrib: rhel
+          - product: kms-fips
+            distrib: rhel
     name: (AZURE) ${{ matrix.product }} - ${{ matrix.distrib }} - ${{ matrix.techno }}
     needs:
       - clean-resources
@@ -35,7 +37,7 @@ jobs:
       techno: ${{ matrix.techno }}
       distrib: ${{ matrix.distrib }}
       product: ${{ matrix.product }}
-      kms-version: 4.16.0
+      kms-version: 4.17.0
       ai-runner-version: 0.3.0
       base-version: 0.1.5
 
@@ -62,7 +64,7 @@ jobs:
       techno: ${{ matrix.techno }}
       distrib: ${{ matrix.distrib }}
       product: ${{ matrix.product }}
-      kms-version: 4.16.0
+      kms-version: 4.17.0
       ai-runner-version: 0.3.0
 
   post-clean-resources:

diff --git a/.github/workflows/nightly_gcp_main.yml b/.github/workflows/nightly_gcp_main.yml
@@ -53,7 +53,7 @@ jobs:
       min-cpu-platform: ${{ matrix.min-cpu-platform }}
       mode: ${{ matrix.mode }}
       product: ${{ matrix.product }}
-      kms-version: 4.16.0
+      kms-version: 4.17.0
       ai-runner-version: 0.3.0
       base-version: 0.1.5
 
@@ -98,7 +98,7 @@ jobs:
       min-cpu-platform: ${{ matrix.min-cpu-platform }}
       mode: ${{ matrix.mode }}
       product: ${{ matrix.product }}
-      kms-version: 4.16.0
+      kms-version: 4.17.0
       ai-runner-version: 0.3.0
 
   post-clean-gcp-resources: