Chore/create test 021 server upgrade with nifi (#273)

* fix: use latest Nifi workbench * fix: add changeset 3 * Added test 021 - server upgrade w nifi * fixed: updated for new DB properties * fix: minor fixes in manual and auto test * chore: move test after automation --------- Co-authored-by: Ruben Huybrighs <ruben.huybrighs@cegeka.com>
Informatievlaanderen · May 5, 2023 · ad89651 · ad89651
1 parent ba48d7f
commit ad89651
Show file tree

Hide file tree

Showing 10 changed files with 332 additions and 357 deletions.
diff --git a/cypress/e2e/upgrading.feature b/cypress/e2e/upgrading.feature
@@ -3,7 +3,7 @@ Feature: server upgrade use case
   @upgrading @test-022
   Scenario: 022: Upgrade LDES Server with LDIO
     Given the members are stored in collection 'ldesmember' in database 'iow_devices'
-    Given context 'tests/022.server-upgrade-with-ldio' is started
+    And context 'tests/022.server-upgrade-with-ldio' is started
     And the old LDES server is available
     And I start the JSON Data Generator
     And the LDES contains at least 1 members
@@ -20,17 +20,42 @@ Feature: server upgrade use case
   @upgrading @test-026
   Scenario: 026: Upgrade LDES workbench with LDI
     Given the members are stored in collection 'ldesmember' in database 'iow_devices'
-    Given context 'tests/026.ldio-workbench-upgrade' is started
+    And context 'tests/026.ldio-workbench-upgrade' is started
     And the LDES server is available
-    When I set the TARGETURL to the old LDIO
+    And I set the TARGETURL to the old LDIO
     And I start the JSON Data Generator
     And the LDES contains at least 1 members
-    And I start the new LDIO workflow
-    When I pause the new LDIO workflow output
+    When I start the new LDIO workflow
+    And I pause the new LDIO workflow output
     And I set the TARGETURL to the new LDIO
     And the old server is done processing
     And I remember the last fragment member count
     And I bring the old LDIO workbench down
     And I resume the new LDIO workflow output
     Then the LDES member count increases
     And the last fragment member count increases
+
+  @upgrading @test-021
+  Scenario: 021: Upgrade LDES workbench with NiFi
+    Given the members are stored in collection 'ldesmember' in database 'iow_devices'
+    Given context 'tests/021.server-upgrade-with-nifi' is started
+    And the NiFi workbench is available
+    And I have uploaded the workflow
+    And I started the workflow
+    And I start the JSON Data Generator
+    And the LDES contains at least 11 members
+    And the ldesfragment collection is structured as expected
+    And the ldesmember collection is structured as expected
+
+    When I stop the http sender in the workflow
+    And the old server is done processing
+    And I remember the last fragment member count
+    And I bring the old server down
+    And I start the new LDES Server
+    And the LDES server is available
+    Then the ldesfragment collection on the new server is structured as expected
+    And the ldesmember collection on the new server is structured as expected
+
+    When I start the http sender in the workflow
+    Then the LDES member count increases
+    And the last fragment member count increases
diff --git a/cypress/support/services/ldes-workbench-nifi.ts b/cypress/support/services/ldes-workbench-nifi.ts
@@ -3,6 +3,7 @@
 import { CanCheckAvailability } from "./interfaces";
 
 export class LdesWorkbenchNiFi implements CanCheckAvailability {
+
     constructor(private baseUrl: string) {
     }
 
@@ -73,4 +74,16 @@ export class LdesWorkbenchNiFi implements CanCheckAvailability {
     pushStart() {
         cy.get('#operate-start').click();
     }
+
+    pushStop() {
+        cy.get('#operate-stop').click();
+    }
+
+    openWorkflow() {
+        cy.get('#operation-context-id').then(div => cy.get(`#id-${div.text()}`).dblclick());
+    }
+
+    selectProcessor(processorName: string) {
+        cy.get(`g.processor > text > title:contains(${processorName})`).parent().parent().click();
+    }
 }
diff --git a/cypress/support/step_definitions/common_step_definitions.ts b/cypress/support/step_definitions/common_step_definitions.ts
@@ -196,7 +196,7 @@ When('the LDES contains at least {int} members', (count: number) => {
 })
 
 When('the old server is done processing', () => {
-    let previousCount;
+    let previousCount: number;
     currentMemberCount().then(count => previousCount = count).then(count => cy.log(`Previous count: ${count}`));
     cy.waitUntil(() =>
         currentMemberCount().then(count =>
@@ -225,6 +225,17 @@ When('I bring the old LDIO workbench down', () => {
     dockerCompose.removeVolumesAndImage('old-ldio');
 })
 
+When('I stop the http sender in the workflow', () => {
+    workbenchNifi.openWorkflow();
+    workbenchNifi.selectProcessor('InvokeHTTP');
+    workbenchNifi.pushStop();
+})
+
+When('I start the http sender in the workflow', () => {
+    workbenchNifi.selectProcessor('InvokeHTTP');
+    workbenchNifi.pushStart();
+})
+
 // Then stuff
 
 Then('the sink contains {int} members', (count: number) => {

diff --git a/cypress/support/step_definitions/upgrading.feature.ts b/cypress/support/step_definitions/upgrading.feature.ts
@@ -1,8 +1,39 @@
-import { Then, When } from "@badeball/cypress-cucumber-preprocessor";
+import { Given, Then, When } from "@badeball/cypress-cucumber-preprocessor";
 import { mongo, setTargetUrl } from "./common_step_definitions";
 import { Fragment } from "../ldes";
 
-let lastMemberCount;
+let lastMemberCount: number;
+
+const commonFragmentProperties = ['_class', 'fragmentPairs', 'immutable', 'relations', 'root', 'viewName'];
+const commonMemberProperties = ['_class'];
+const fragmentCollectionUrl = 'http://localhost:9019/iow_devices/ldesfragment';
+const memberCollectionUrl = 'http://localhost:9019/iow_devices/ldesmember';
+
+function checkDatabaseStructure(collectionUrl: string, expected: string[],) {
+    cy.exec(`curl '${collectionUrl}?includeDocuments=true' | jq '[.documents[] | keys] | flatten | unique | map(select(. != "_id"))'`, { failOnNonZeroExit: false })
+        .then(result => {
+            const actual = JSON.parse(result.stdout) as string[];
+            expect(actual).to.have.same.members(expected);
+        });
+}
+
+Given('the ldesfragment collection is structured as expected', () => {
+    checkDatabaseStructure(fragmentCollectionUrl, [...commonFragmentProperties, 'members']);
+})
+
+Given('the ldesmember collection is structured as expected', () => {
+    checkDatabaseStructure(memberCollectionUrl, [...commonMemberProperties, 'ldesMember']);
+})
+
+Then('the ldesfragment collection on the new server is structured as expected', () => {
+    checkDatabaseStructure(fragmentCollectionUrl, 
+        [...commonFragmentProperties, 'collectionName', 'immutableTimestamp', 'numberOfMembers', 'parentId', 'softDeleted']);
+})
+
+Then('the ldesmember collection on the new server is structured as expected', () => {
+    checkDatabaseStructure(memberCollectionUrl, 
+        [...commonMemberProperties, 'collectionName', 'model', 'sequenceNr', 'timestamp', 'treeNodeReferences', 'versionOf']);
+})
 
 When('I remember the last fragment member count', () => {
     mongo.fragments('iow_devices', 'ldesfragment')
@@ -16,15 +47,15 @@ When('I remember the last fragment member count', () => {
 })
 
 Then('the last fragment member count increases', () => {
-    cy.waitUntil(() => 
-    mongo.fragments('iow_devices', 'ldesfragment')
-        .then(fragments => fragments.pop())
-        .then(partialUrl => new Fragment(`http://localhost:8080${partialUrl}`)
-            .visit()
-            .then(fragment => cy.log(`New member count: ${fragment.memberCount}`)
-                .then(() => lastMemberCount < fragment.memberCount)
-            )
-        ),
+    cy.waitUntil(() =>
+        mongo.fragments('iow_devices', 'ldesfragment')
+            .then(fragments => fragments.pop())
+            .then(partialUrl => new Fragment(`http://localhost:8080${partialUrl}`)
+                .visit()
+                .then(fragment => cy.log(`New member count: ${fragment.memberCount}`)
+                    .then(() => lastMemberCount < fragment.memberCount)
+                )
+            ),
         { timeout: 5000, interval: 1000 });
 })
 
@@ -34,4 +65,4 @@ When('I set the TARGETURL to the old LDIO', () => {
 
 When('I set the TARGETURL to the new LDIO', () => {
     setTargetUrl("http://new-ldio:8080/pipeline");
-})
+})