[FIPS / CI] Fix ES ML startup issues, UUID permissions, FTR tests status, OpenSSL target. Switch to Ubuntu

.buildkite/scripts/steps/fips/smoke_test.sh

@@ -36,9 +36,9 @@ for config in "${configs[@]}"; do
     echo "^^^ +++"
 
     if [[ "$failedConfigs" ]]; then
-      failedConfigs="${failedConfigs}"$'\n'"$config"
+      failedConfigs="${failedConfigs}"$'\n'"- ${config}"
     else
-      failedConfigs="$config"
+      failedConfigs="### Failed FTR Configs"$'\n'"- ${config}"
     fi
   fi
 done

.buildkite/scripts/steps/package_testing/test.sh

@@ -58,12 +58,16 @@ trap "echoKibanaLogs" EXIT
 if [[ "$TEST_PACKAGE" == "fips" ]]; then
   set +e
   vagrant ssh $TEST_PACKAGE -t -c "/home/vagrant/kibana/.buildkite/scripts/steps/fips/smoke_test.sh"
+  exitCode=$?
+
   vagrant ssh $TEST_PACKAGE -t -c "cat /home/vagrant/ftr_failed_configs 2>/dev/null" >ftr_failed_configs
   set -e
 
   if [ -s ftr_failed_configs ]; then
-    buildkite-agent meta-data set "ftr-failed-configs" <./ftr_failed_configs
+    cat ftr_failed_configs | buildkite-agent annotate --style "error"
   fi
+
+  exit $exitCode
 else
   vagrant provision "$TEST_PACKAGE"
 

packages/kbn-es/src/cluster.ts

@@ -389,7 +389,9 @@ export class Cluster {
     this.process = execa(ES_BIN, args, {
       cwd: installPath,
       env: {
-        ...(installPath ? { ES_TMPDIR: path.resolve(installPath, 'ES_TMPDIR') } : {}),
+        ...(installPath && process.env.FTR_DISABLE_ES_TMPDIR !== 'true'
+          ? { ES_TMPDIR: path.resolve(installPath, 'ES_TMPDIR') }
+          : {}),
         ...process.env,
         JAVA_HOME: '', // By default, we want to always unset JAVA_HOME so that the bundled JDK will be used
         ES_JAVA_OPTS: esJavaOpts,

packages/kbn-es/src/install/install_archive.ts

@@ -62,9 +62,11 @@ export async function installArchive(archive: string, options?: InstallArchiveOp
   });
   log.info('extracted to %s', chalk.bold(installPath));
 
-  const tmpdir = path.resolve(installPath, 'ES_TMPDIR');
-  fs.mkdirSync(tmpdir, { recursive: true });
-  log.info('created %s', chalk.bold(tmpdir));
+  if (process.env.FTR_DISABLE_ES_TMPDIR !== 'true') {
+    const tmpdir = path.resolve(installPath, 'ES_TMPDIR');
+    fs.mkdirSync(tmpdir, { recursive: true });
+    log.info('created %s', chalk.bold(tmpdir));
+  }
 
   // starting in 6.3, security is disabled by default. Since we bootstrap
   // the keystore, we can enable security ourselves.

packages/kbn-test/src/es/test_es_cluster.ts

@@ -182,7 +182,6 @@ export function createTestEsCluster<
   } = options;
 
   const clusterName = `${CI_PARALLEL_PROCESS_PREFIX}${customClusterName}`;
-  const isFIPSMode = process.env.FTR_FIPS_MODE === '1';
 
   const defaultEsArgs = [
     `cluster.name=${clusterName}`,
@@ -193,12 +192,7 @@ export function createTestEsCluster<
       : ['discovery.type=single-node']),
   ];
 
-  const esArgs = assignArgs(
-    defaultEsArgs,
-    // ML has issues running in FIPS mode due to custom OpenSSL
-    // Remove after https://github.com/elastic/kibana-operations/issues/96
-    isFIPSMode ? [...customEsArgs, 'xpack.ml.enabled=false'] : customEsArgs
-  );
+  const esArgs = assignArgs(defaultEsArgs, customEsArgs);
 
   const config = {
     version: esTestConfig.getVersion(),

test/package/Vagrantfile

@@ -46,14 +46,7 @@ Vagrant.configure("2") do |config|
       vb.memory = 4096
       vb.cpus = 2
     end
-    fips.vm.box = 'generic/rhel9'
-    fips.vm.provision "shell", inline: <<-SHELL
-      echo "export OPENSSL_MODULES=/usr/local/lib64/ossl-modules" >> /etc/profile.d/kibana-fips-env.sh
-      echo "export TEST_BROWSER_HEADLESS=1" >> /etc/profile.d/kibana-fips-env.sh
-      echo "export ES_TMPDIR=/home/vagrant/kibana/.es/tmp" >> /etc/profile.d/kibana-fips-env.sh
-      # Remove after https://github.com/elastic/kibana-operations/issues/96
-      echo "export FTR_FIPS_MODE=1" >> /etc/profile.d/kibana-fips-env.sh
-    SHELL
+    fips.vm.box = 'ubuntu/jammy64'
     fips.vm.provision "ansible" do |ansible|
       ansible.playbook = "fips.yml"
     end

test/package/fips.yml

@@ -6,7 +6,9 @@
     nvm_ver: "0.39.7"
     openssl_sha: "sha256:6c13d2bf38fdf31eac3ce2a347073673f5d63263398f1f69d0df4a41253e4b3e"
     openssl_ver: "3.0.8"
+    openssl_src_path: "{{ kibana_dist_path }}/openssl-{{ openssl_ver }}"
+    openssl_path: "{{ kibana_dist_path }}/openssl"
   roles:
-    - upgrade_yum_packages
+    - upgrade_apt_packages
     - install_kibana_fips
     - assert_fips_enabled

test/package/roles/assert_fips_enabled/tasks/main.yml

@@ -1,5 +1,7 @@
 - name: register kibana node getFips
-  command: "{{ kibana_dist_path }}/node/bin/node --enable-fips --openssl-config={{ kibana_dist_path }}/config/nodejs.cnf -p 'crypto.getFips()'"
+  shell: 
+    cmd: "source /home/vagrant/.profile && {{ kibana_dist_path }}/node/bin/node --enable-fips --openssl-config={{ kibana_dist_path }}/config/nodejs.cnf -p 'crypto.getFips()'"
+    executable: /bin/bash
   register: kibana_node_fips
 
 - debug:

test/package/roles/install_kibana_fips/tasks/main.yml

@@ -6,18 +6,37 @@
       - "processor_cores"
   when: ansible_processor_vcpus is not defined
 
-- name: fix /var/log permissions for kibana
+- name: setup env variables
+  blockinfile:
+    path: "/home/vagrant/.profile"
+    block: |
+      export OPENSSL_MODULES=/usr/share/kibana/openssl/lib/ossl-modules
+      export TEST_BROWSER_HEADLESS=1
+      export FTR_DISABLE_ES_TMPDIR=true
+    owner: vagrant
+    group: vagrant
+    mode: '0644'
+
+- name: add chrome apt signing key
   become: yes
-  file:
-    path: /var/log
-    state: directory
-    recurse: true
-    mode: "0777"
+  apt_key:
+    url: https://dl.google.com/linux/linux_signing_key.pub
+    state: present
 
-- name: create tmp dir for ES
-  file:
-    path: "{{ kibana_src_path }}/.es/tmp"
-    state: directory
+- name: add chrome apt repository
+  become: yes
+  apt_repository:
+    repo: deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main
+    state: present
+
+- name: install apt packages
+  become: yes
+  apt:
+    pkg:
+      - build-essential
+      - google-chrome-stable
+      - unzip
+    state: latest
 
 - name: slurp kibana node version
   slurp:
@@ -31,7 +50,7 @@
 - name: install nvm
   shell:
     chdir: "$HOME"
-    cmd: curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v{{ nvm_ver }}/install.sh | bash
+    cmd: curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v{{ nvm_ver }}/install.sh | PROFILE=/home/vagrant/.profile bash
 
 - name: install kibana node version
   shell:
@@ -40,12 +59,11 @@
   args:
     executable: /bin/bash
 
-- name: "ensure {{ kibana_dist_path }} dir exists"
+- name: "ensure {{ openssl_path }} dir exists"
   become: yes
   file:
-    path: "{{ kibana_dist_path }}"
+    path: "{{ openssl_path }}"
     state: directory
-    mode: "0777"
 
 - name: find kibana distribution
   find:
@@ -99,35 +117,54 @@
   delay: 10
   get_url:
     url: "https://www.openssl.org/source/openssl-{{ openssl_ver }}.tar.gz"
-    dest: "{{ kibana_dist_path }}/openssl-{{ openssl_ver }}.tar.gz"
+    dest: "{{ openssl_src_path }}.tar.gz"
     checksum: "{{ openssl_sha }}"
 
 - name: extract OpenSSL
   become: yes
   unarchive:
-    src: "{{ kibana_dist_path }}/openssl-{{ openssl_ver }}.tar.gz"
+    src: "{{ openssl_src_path }}.tar.gz"
     dest: "{{ kibana_dist_path }}"
     remote_src: yes
 
 - name: configure OpenSSL for FIPS
   become: yes
   shell:
-    chdir: "{{ kibana_dist_path }}/openssl-{{ openssl_ver }}"
-    cmd: ./Configure enable-fips
+    chdir: "{{ openssl_src_path }}"
+    cmd: "./Configure --prefix={{ openssl_path }} --openssldir={{ openssl_path }}/ssl --libdir={{ openssl_path }}/lib enable-fips"
 
 - name: compile OpenSSL with FIPS
   become: yes
   make:
-    chdir: "{{ kibana_dist_path }}/openssl-{{ openssl_ver }}"
+    chdir: "{{ openssl_src_path }}"
     jobs: "{{ ansible_facts['processor_vcpus'] }}"
 
 - name: install OpenSSL with FIPS
   become: yes
   make:
-    chdir: "{{ kibana_dist_path }}/openssl-{{ openssl_ver }}"
+    chdir: "{{ openssl_src_path }}"
     target: install
 
-- name: link OpenSSL package
+- name: "change owner of {{ kibana_dist_path }} to vagrant"
   become: yes
-  shell:
-    cmd: ldconfig /usr/local/lib64/
+  file:
+    path: "{{ kibana_dist_path }}"
+    owner: vagrant
+    group: vagrant
+    recurse: yes
+
+- name: fix /var/log permissions for kibana
+  become: yes
+  file:
+    path: /var/log
+    state: directory
+    recurse: true
+    mode: "0777"
+
+- name: increase vm.max_map_count for ES
+  become: yes
+  sysctl:
+    name: vm.max_map_count
+    value: '262144'
+    state: present
+    reload: yes

test/package/templates/fips/nodejs.cnf

@@ -9,7 +9,7 @@
 ##########################################################################
 
 nodejs_conf = nodejs_init
-.include /usr/local/ssl/fipsmodule.cnf
+.include /usr/share/kibana/openssl/ssl/fipsmodule.cnf
 
 [nodejs_init]
 providers = provider_sect