poc - adding basic ui sanity

x-pack/test/cloud_security_posture_functional/config.ts

@@ -13,36 +13,36 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const xpackFunctionalConfig = await readConfigFile(
     require.resolve('../functional/config.base.js')
   );
-
-  return {
-    ...xpackFunctionalConfig.getAll(),
-    pageObjects,
-    testFiles: [resolve(__dirname, './pages')],
-    junit: {
-      reportName: 'X-Pack Cloud Security Posture Functional Tests',
-    },
-    kbnTestServer: {
-      ...xpackFunctionalConfig.get('kbnTestServer'),
-      serverArgs: [
-        ...xpackFunctionalConfig.get('kbnTestServer.serverArgs'),
-        /**
-         * Package version is fixed (not latest) so FTR won't suddenly break when package is changed.
-         *
-         * test a new package:
-         * 1. build the package and start the registry with elastic-package and uncomment the 'registryUrl' flag below
-         * 2. locally checkout the kibana version that matches the new package
-         * 3. update the package version below to use the new package version
-         * 4. run tests with NODE_EXTRA_CA_CERTS pointing to the elastic-package certificate
-         * 5. when test pass:
-         *   1. release a new package to EPR
-         *   2. merge the updated version number change to kibana
-         */
-        `--xpack.fleet.packages.0.name=cloud_security_posture`,
-        `--xpack.fleet.packages.0.version=1.7.4`,
-        // `--xpack.fleet.registryUrl=https://localhost:8080`,
-        `--xpack.fleet.agents.fleet_server.hosts=["https://ftr.kibana:8220"]`,
-        `--xpack.fleet.internal.fleetServerStandalone=true`,
-      ],
-    },
-  };
+  // If TEST_CLOUD environment varible is defined, return the configuration for cloud testing
+  if (process.env.TEST_CLOUD !== '1') {
+    return {
+      ...xpackFunctionalConfig.getAll(),
+      pageObjects,
+      testFiles: [resolve(__dirname, './pages')],
+      junit: {
+        reportName: 'X-Pack Cloud Security Posture Functional Tests',
+      },
+      kbnTestServer: {
+        ...xpackFunctionalConfig.get('kbnTestServer'),
+        serverArgs: [
+          ...xpackFunctionalConfig.get('kbnTestServer.serverArgs'),
+          `--xpack.fleet.packages.0.name=cloud_security_posture`,
+          `--xpack.fleet.packages.0.version=1.7.4`,
+          // `--xpack.fleet.registryUrl=https://localhost:8080`,
+          `--xpack.fleet.agents.fleet_server.hosts=["https://ftr.kibana:8220"]`,
+          `--xpack.fleet.internal.fleetServerStandalone=true`,
+        ],
+      },
+    };
+  } else {
+    // FTR configuration for cloud testing
+    return {
+      ...xpackFunctionalConfig.getAll(),
+      pageObjects,
+      testFiles: [resolve(__dirname, './pages')],
+      junit: {
+        reportName: 'X-Pack Cloud Security Posture Sanity Tests',
+      },
+    };
+  }
 }

x-pack/test/cloud_security_posture_functional/page_objects/csp_dashboard_page.ts

@@ -96,11 +96,26 @@ export function CspDashboardPageProvider({ getService, getPageObjects }: FtrProv
       return await testSubjects.find('dashboard-summary-section');
     },
 
+    getAllCloudComplianceScores: async () => {
+      await dashboard.getCloudDashboard();
+      return await testSubjects.findAll('dashboard-summary-section-compliance-score');
+    },
+
     getCloudComplianceScore: async () => {
       await dashboard.getCloudSummarySection();
       return await testSubjects.find('dashboard-summary-section-compliance-score');
     },
 
+    getCloudResourcesEvaluatedCard: async () => {
+      await dashboard.getCloudDashboard();
+      return await testSubjects.find('dashboard-counter-card-resources-evaluated');
+    },
+
+    getCloudResourcesEvaluated: async () => {
+      const resourcesEvaluatedCard = await dashboard.getCloudResourcesEvaluatedCard();
+      return await resourcesEvaluatedCard.findByXpath('//div/p/span');
+    },
+
     // Kubernetes Dashboard
 
     getKubernetesDashboard: async () => {
@@ -121,6 +136,16 @@ export function CspDashboardPageProvider({ getService, getPageObjects }: FtrProv
 
       return await testSubjects.find('dashboard-summary-section-compliance-score');
     },
+
+    getKubernetesResourcesEvaluatedCard: async () => {
+      await dashboard.getKubernetesDashboard();
+      return await testSubjects.find('dashboard-counter-card-resources-evaluated');
+    },
+
+    getKubernetesResourcesEvaluated: async () => {
+      const resourcesEvaluatedCard = await dashboard.getKubernetesResourcesEvaluatedCard();
+      return await resourcesEvaluatedCard.findByXpath('//div/p/span');
+    },
   };
 
   const navigateToComplianceDashboardPage = async () => {

x-pack/test/cloud_security_posture_functional/pages/basic_ui_sanity.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../ftr_provider_context';
+
+// eslint-disable-next-line import/no-default-export
+export default ({ getPageObjects, getService }: FtrProviderContext) => {
+  const retry = getService('retry');
+  const pageObjects = getPageObjects(['common', 'cloudPostureDashboard', 'header']);
+
+  describe('Cloud Posture Dashboard Page', function () {
+    this.tags(['cloud_security_posture_ui_sanity']);
+    let cspDashboard: typeof pageObjects.cloudPostureDashboard;
+    let dashboard: typeof pageObjects.cloudPostureDashboard.dashboard;
+
+    before(async () => {
+      cspDashboard = pageObjects.cloudPostureDashboard;
+      dashboard = pageObjects.cloudPostureDashboard.dashboard;
+      await cspDashboard.waitForPluginInitialized();
+      await cspDashboard.navigateToComplianceDashboardPage();
+      await retry.waitFor(
+        'Cloud posture integration dashboard to be displayed',
+        async () => !!dashboard.getIntegrationDashboardContainer()
+      );
+    });
+
+    describe('Cloud Dashboard', () => {
+      it('displays accurate summary compliance score', async () => {
+        await pageObjects.header.waitUntilLoadingHasFinished();
+        const scoreElement = await dashboard.getCloudComplianceScore();
+        expect((await scoreElement.getVisibleText()) === '41%').to.be(true);
+      });
+
+      it('displays accurate all complience scores', async () => {
+        const scoresElements = await dashboard.getAllCloudComplianceScores();
+        const scores: string[] = [];
+        for (const scoreElement of scoresElements) {
+          scores.push(await scoreElement.getVisibleText());
+        }
+        // 3 scores for each cloud provider + 1 summary score
+        expect(scores.length).to.be(4);
+        const expectedScores = ['41%', '14%', '55%', '59%'];
+        scores.forEach((score) => {
+          expect(expectedScores).contain(score);
+        });
+      });
+
+      it('displays correct number of resources evaluated', async () => {
+        const resourcesEvaluated = await dashboard.getCloudResourcesEvaluated();
+        expect((await resourcesEvaluated.getVisibleText()) === '3,342').to.be(true);
+      });
+    });
+
+    describe('Kubernetes Dashboard', () => {
+      it('displays accurate summary compliance score', async () => {
+        await pageObjects.header.waitUntilLoadingHasFinished();
+        const scoreElement = await dashboard.getKubernetesComplianceScore();
+        expect((await scoreElement.getVisibleText()) === '83%').to.be(true);
+      });
+
+      it('displays correct number of resources evaluated', async () => {
+        const resourcesEvaluated = await dashboard.getKubernetesResourcesEvaluated();
+        expect((await resourcesEvaluated.getVisibleText()) === '199').to.be(true);
+      });
+    });
+  });
+};

x-pack/test/cloud_security_posture_functional/pages/index.ts

@@ -10,21 +10,26 @@ import { FtrProviderContext } from '../ftr_provider_context';
 // eslint-disable-next-line import/no-default-export
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('Cloud Security Posture', function () {
-    loadTestFile(require.resolve('./rules'));
-    loadTestFile(require.resolve('./findings_onboarding'));
-    loadTestFile(require.resolve('./findings'));
-    loadTestFile(require.resolve('./findings_grouping'));
-    loadTestFile(require.resolve('./findings_alerts'));
-    loadTestFile(require.resolve('./compliance_dashboard'));
-    loadTestFile(require.resolve('./vulnerability_dashboard'));
-    loadTestFile(require.resolve('./cis_integrations/cnvm/cis_integration_cnvm'));
-    loadTestFile(require.resolve('./cis_integrations/cspm/cis_integration_aws'));
-    loadTestFile(require.resolve('./cis_integrations/cspm/cis_integration_gcp'));
-    loadTestFile(require.resolve('./cis_integrations/cspm/cis_integration_azure'));
-    loadTestFile(require.resolve('./cis_integrations/kspm/cis_integration_k8s'));
-    loadTestFile(require.resolve('./cis_integrations/kspm/cis_integration_eks'));
-    loadTestFile(require.resolve('./findings_old_data'));
-    loadTestFile(require.resolve('./vulnerabilities'));
-    loadTestFile(require.resolve('./vulnerabilities_grouping'));
+    if (process.env.TEST_CLOUD === '1') {
+      // Run basic UI sanity tests only in cloud testing
+      loadTestFile(require.resolve('./basic_ui_sanity'));
+    } else {
+      loadTestFile(require.resolve('./rules'));
+      loadTestFile(require.resolve('./findings_onboarding'));
+      loadTestFile(require.resolve('./findings'));
+      loadTestFile(require.resolve('./findings_grouping'));
+      loadTestFile(require.resolve('./findings_alerts'));
+      loadTestFile(require.resolve('./compliance_dashboard'));
+      loadTestFile(require.resolve('./vulnerability_dashboard'));
+      loadTestFile(require.resolve('./cis_integrations/cnvm/cis_integration_cnvm'));
+      loadTestFile(require.resolve('./cis_integrations/cspm/cis_integration_aws'));
+      loadTestFile(require.resolve('./cis_integrations/cspm/cis_integration_gcp'));
+      loadTestFile(require.resolve('./cis_integrations/cspm/cis_integration_azure'));
+      loadTestFile(require.resolve('./cis_integrations/kspm/cis_integration_k8s'));
+      loadTestFile(require.resolve('./cis_integrations/kspm/cis_integration_eks'));
+      loadTestFile(require.resolve('./findings_old_data'));
+      loadTestFile(require.resolve('./vulnerabilities'));
+      loadTestFile(require.resolve('./vulnerabilities_grouping'));
+    }
   });
 }