Merges develop in with pagination handling & fix for test shield image

kendraio · May 31, 2023 · 0773d28 · 0773d28 · vercel · Jun 5, 2023
2 parents 0e244b0 + ccef351
commit 0773d28
Show file tree

Hide file tree

Showing 5 changed files with 381 additions and 36 deletions.
diff --git a/README.md b/README.md
@@ -9,7 +9,7 @@ Kendraio App is an open source dashboard application for rights owners, music ma
 - More information: <https://www.kendra.io/kendraio-app>
 - Privacy policy: <https://www.kendra.io/privacy>
 
-[![deployment_tests](https://img.shields.io/github/workflow/status/kendraio/kendraio-app/vercel_deployment_tests)](https://github.com/kendraio/kendraio-app/actions/workflows/vercel_deployment_tests.yml)
+![Tests](https://img.shields.io/github/checks-status/kendraio/kendraio-app/main)
 
 ![Uptime Robot](https://img.shields.io/uptimerobot/ratio/7/m783523815-565ba269d3dc13ded01aae34)
 

diff --git a/cypress/e2e/http_block_page_following.ts b/cypress/e2e/http_block_page_following.ts
@@ -0,0 +1,196 @@
+import { loadFlowCode } from '../support/helper';
+
+// tslint:disable: quotemark
+/// <reference types="Cypress" />
+
+describe('HTTP Block Integration Tests', () => {
+
+    beforeEach(() => {
+        // Prevent external network request for adapter config
+        cy.intercept('GET', 'https://kendraio.github.io/kendraio-adapter/config.json', {
+            fixture: 'adapterConfig.json'
+        });
+
+        // Prevent external network requests for Workflow cloud
+        cy.intercept('GET', 'https://app.kendra.io/api/workflowCloud/listWorkflows', {
+            fixture: 'workflow-cloud.json'
+        });
+
+        // Prevent external network requests for fonts with empty CSS rule 
+        cy.intercept('https://fonts.googleapis.com/\*\*', "\*{ }");
+    });
+
+
+    it('should follow pagination links and merge results without a proxy', () => {
+        // We emulate the CORS proxy presenting the first set of results,
+        // with a link header pointing to the next page.
+        cy.intercept({
+            url: 'https://example.com/paginated'
+        }, {
+            statusCode: 200,
+            body: '["cats","dogs"]',
+            headers: {
+                'link': '<https://example.com/paginated&page=2>; rel="next"',
+            }
+        });
+
+        // If the target URL is for the second page, we return the second set of results:
+        cy.intercept({
+            url: 'https://example.com/paginated&page=2'
+        }, {
+            statusCode: 200,
+            body: '["fish","birds"]',
+            headers: {
+                'link': '<https://example.com/paginated>; rel="prev"',
+            }
+        });
+
+        // We test page following by setting the "followPaginationLinksMerged" flag:
+        loadFlowCode([
+            { "type": "init" },
+            {
+                "type": "http",
+                "method": "GET",
+                "endpoint": "https://example.com/paginated",
+                "useProxy": false,
+                "followPaginationLinksMerged": true,
+            },
+            {
+                "type": "debug",
+                "open": 2,
+                "showData": true
+            }
+        ]);
+        cy.contains('cats');
+        cy.contains('dogs');
+        cy.contains('fish');
+        cy.contains('birds');
+    });
+
+    it('should follow pagination links and merge results using a proxy', () => {
+        // We emulate the CORS proxy presenting the first set of results,
+        // with a link header pointing to the next page.
+        cy.intercept({
+            url: 'https://proxy.kendra.io/',
+            headers: {
+                'Target-URL': 'https://example.com/paginated',
+            }
+        }, {
+            statusCode: 200,
+            body: '["cats","dogs"]',
+            headers: {
+                'link': '<https://example.com/paginated&page=2>; rel="next"',
+            }
+        });
+
+        // If the target URL is for the second page, we return the second set of results:
+        cy.intercept({
+            url: 'https://proxy.kendra.io/',
+            headers: {
+                'Target-URL': 'https://example.com/paginated&page=2',
+            },
+        }, {
+            statusCode: 200,
+            body: '["fish","birds"]',
+            headers: {
+                'link': '<https://example.com/paginated>; rel="prev"',
+            }
+        });
+
+        // We test page following by setting the "followPaginationLinksMerged" flag:
+        loadFlowCode([
+            { "type": "init" },
+            {
+                "type": "http",
+                "method": "GET",
+                "endpoint": "https://example.com/paginated",
+                "useProxy": true,
+                "followPaginationLinksMerged": true,
+            },
+            {
+                "type": "debug",
+                "open": 2,
+                "showData": true
+            }
+        ]);
+        cy.contains('cats');
+        cy.contains('dogs');
+        cy.contains('fish');
+        cy.contains('birds');
+    });
+
+    it('should return a single set of results if response is not paginated', () => {
+        cy.intercept({
+            url: 'https://example.com/data'
+        }, {
+            statusCode: 200,
+            body: '["hippo", "giraffe"]'
+        });
+
+        loadFlowCode([
+            { "type": "init" },
+            {
+                "type": "http",
+                "method": "GET",
+                "endpoint": "https://example.com/data"
+            },
+            {
+                "type": "debug",
+                "open": 2,
+                "showData": true
+            }
+        ]);
+        cy.contains('hippo');
+        cy.contains('giraffe');
+    });
+
+    it('should return first results only if not paginated, with proxy', () => {
+        cy.intercept({
+            url: 'https://proxy.kendra.io/',
+            headers: {
+                'Target-URL': 'https://example.com/paginated',
+            }
+        }, {
+            statusCode: 200,
+            body: '["cats","dogs"]',
+            headers: {
+                'link': '<https://example.com/paginated&page=2>; rel="next"',
+            }
+        });
+
+        // We do not expect this to be called:
+        cy.intercept({
+            url: 'https://proxy.kendra.io/',
+            headers: {
+                'Target-URL': 'https://example.com/paginated&page=2',
+            },
+        }, {
+            statusCode: 200,
+            body: '["fish","birds"]',
+            headers: {
+                'link': '<https://example.com/paginated>; rel="prev"',
+            }
+        }).as('secondPage');
+
+        loadFlowCode([
+            { "type": "init" },
+            {
+                "type": "http",
+                "method": "GET",
+                "endpoint": "https://example.com/paginated",
+                "useProxy": true
+            },
+            {
+                "type": "debug",
+                "open": 2,
+                "showData": true
+            }
+        ]);
+        cy.contains('cats');
+        cy.contains('dogs');
+        // we check it does not contain a second page result:
+        cy.get('body').should('not.contain', 'fish');
+    });
+
+
+});
diff --git a/docs/workflow/blocks/http.rst b/docs/workflow/blocks/http.rst
@@ -100,3 +100,18 @@ It is possible to query a GraphQL endpoint using the HTTP block.
   }
 
 
+Pagination
+----------
+
+If a HTTP API returns paginated results with a standard link header, to fetch paginated API results, set the followPaginationLinksMerged option to true. This will fetch all pages of results and return the combined set of results from all the pages.
+
+With a proxy:
+```json
+{
+  "type": "http",
+  "method": "GET",
+  "endpoint": "https://example.com/paginated",
+  "useProxy": true,
+  "followPaginationLinksMerged": true
+}
+```
diff --git a/src/app/blocks/http-block/http-block.component.spec.ts b/src/app/blocks/http-block/http-block.component.spec.ts
@@ -0,0 +1,37 @@
+import { HttpBlockComponent } from './http-block.component';
+import { ContextDataService } from '../../services/context-data.service';
+import { MatLegacySnackBar as MatSnackBar } from '@angular/material/legacy-snack-bar';
+import { HttpClient } from '@angular/common/http';
+
+describe('extractNextPageUrl', () => {
+    let component: HttpBlockComponent;
+    let contextDataServiceMock: ContextDataService;
+    let httpClientMock: HttpClient;
+    let matSnackBarMock: MatSnackBar;
+
+    beforeEach(() => {
+        contextDataServiceMock = jasmine.createSpyObj('ContextDataService', ['getGlobalContext']);
+        httpClientMock = jasmine.createSpyObj('HttpClient', ['get']);
+        matSnackBarMock = jasmine.createSpyObj('MatLegacySnackBar', ['open']);
+        component = new HttpBlockComponent(contextDataServiceMock, matSnackBarMock, httpClientMock);
+    });
+
+    it('should extract the next page URL from a link header', () => {
+        const linkHeader = '<https://example.com/page2>; rel="next"';
+        expect(component.extractNextPageUrl(linkHeader)).toEqual('https://example.com/page2');
+    });
+
+    it('should return null if no next link is present', () => {
+        const linkHeader = '<https://example.com/page2>; rel="last"';
+        expect(component.extractNextPageUrl(linkHeader)).toBeNull();
+    });
+
+    it('should return null for an invalid link header', () => {
+        const linkHeader = 'invalid link header';
+        expect(component.extractNextPageUrl(linkHeader)).toBeNull();
+    });
+
+    it('should return null for an empty string', () => {
+        expect(component.extractNextPageUrl('')).toBeNull();
+    });
+});