Added support for EMT-Search to include serviceID not longer active

Fixes #114

apibuilder4elastic/custom_flow_nodes/api-builder-plugin-axway-api-management/src/actions.js

@@ -106,6 +106,42 @@ async function lookupCurrentUser(params, options) {
 	return user;
 }
 
+async function lookupTopology(params, options) {
+	const { requestHeaders } = params;
+	const logger = options.logger;
+	pluginConfig = options.pluginConfig;
+
+	cache = options.pluginContext.cache;
+	if (!requestHeaders) {
+		throw new Error('You need to provide Request-Headers with Cookies or an Authorization header.');
+	}
+	if(!requestHeaders.cookie && !requestHeaders.authorization) {
+		throw new Error('You must provide either the VIDUSR cookie + csrf-token or an HTTP-Basic Authorization header.');
+	}
+	let cacheKey = requestHeaders.host;
+	if(!requestHeaders.host) {
+		logger.warn(`Host header not found, using static cache-key for the API-Gateway topology lookup.`);
+		cacheKey = "apigwTopology";
+	}
+	if(cache.has(cacheKey)) {
+		return cache.get(cacheKey);
+	}
+	var topology;
+	if(requestHeaders.authorization) {
+		logger.debug(`Trying to get API-Gateway topology based on Authorization header.`);
+		topology = await _getTopology(headers = {'Authorization': `${requestHeaders.authorization}`});
+	} else {
+		logger.trace(`Trying to get API-Gateway topology based on VIDUSR cookie.`);
+		topology = await _getTopology(headers = {'Cookie': requestHeaders.cookie});
+	}
+	topology.services = topology.services.filter(function(service) {
+		return service.type!="nodemanager"; // Filter node manager service
+	});
+	logger.info(`Successfully retrieved topology from Admin-Node-Manager. Will be cached for 5 minutes.`);
+	cache.set( cacheKey, topology, 300);
+	return topology;
+}
+
 async function lookupAPIDetails(params, options) {
 	var { apiName, apiPath, operationId, groupId, disableCustomProperties } = params;
 	const { logger } = options;
@@ -412,6 +448,22 @@ async function _getCurrentGWUser(requestHeaders) {
 	return loginName;
 }
 
+async function _getTopology(requestHeaders) {
+	var options = {
+		path: '/api/topology',
+		headers: requestHeaders,
+		agent: new https.Agent({ rejectUnauthorized: false })
+	};
+	var topology = await sendRequest(pluginConfig.apigateway.url, options)
+		.then(response => {
+			return response.body.result;
+		})
+		.catch(err => {
+			throw new Error(`Error getting API-Gateway topology user. Request sent to: '${pluginConfig.apigateway.url}'. Response-Code: ${err.statusCode}`);
+		});
+	return topology;
+}
+
 async function _addCustomProperties(apiProxy, groupId, region, options) {
 	var apiCustomProperties = await _getConfiguredCustomProperties(groupId, region, options);
 	apiProxy.customProperties = {};
@@ -653,6 +705,7 @@ async function _getConfiguredCustomProperties(groupId, region, options) {
 
 module.exports = {
 	lookupCurrentUser, 
+	lookupTopology,
 	lookupAPIDetails,
 	lookupApplication,
 	getCustomPropertiesConfig,

apibuilder4elastic/custom_flow_nodes/api-builder-plugin-axway-api-management/src/flow-nodes.yml

@@ -38,6 +38,30 @@ flow-nodes:
             schema:
               type: object
 
+      lookupTopology:
+        name: Lookup topology
+        description: "Looks up the API-Gateway topology, as this is required when running in EMT mode. See https://github.com/Axway-API-Management-Plus/apigateway-openlogging-elk/issues/114. Admin-Node-Manager services are ignored."
+        parameters:
+          requestHeaders:
+            name: Request Headers
+            description: The request headers are used to lookup the topology using the currently logged in user.
+            required: true
+            schema:
+              type: string
+        outputs:
+          next:
+            name: Next
+            description: The API-Gateway topology
+            context: $.gatewayTopology
+            schema:
+              type: object
+          error:
+            name: Error
+            description: An unexpected error happened
+            context: $.error
+            schema:
+              type: object
+
       lookupAPIDetails:
         name: Lookup API
         description: Looks up the details of an API based on the given API-Name and API-Path. Additionally, it returns the configured custom properties for that API.

apibuilder4elastic/custom_flow_nodes/api-builder-plugin-axway-api-management/src/index.js

@@ -1,6 +1,6 @@
 const path = require('path');
 const { SDK } = require('@axway/api-builder-sdk');
-const { lookupCurrentUser, lookupAPIDetails, getCustomPropertiesConfig, isIgnoreAPI, lookupApplication } = require('./actions');
+const { lookupCurrentUser, lookupTopology, lookupAPIDetails, getCustomPropertiesConfig, isIgnoreAPI, lookupApplication } = require('./actions');
 const { mergeCustomProperties } = require('./customProperties');
 const NodeCache = require( "node-cache" );
 const { checkAPIManagers, parseAPIManagerConfig } = require('./utils');
@@ -46,7 +46,7 @@ async function getPlugin(pluginConfig, options) {
 		}
 	}
 
-	sdk.load(path.resolve(__dirname, 'flow-nodes.yml'), {lookupCurrentUser, lookupAPIDetails, getCustomPropertiesConfig, mergeCustomProperties, isIgnoreAPI, lookupApplication }, { pluginContext: { cache: cache }, pluginConfig});
+	sdk.load(path.resolve(__dirname, 'flow-nodes.yml'), {lookupCurrentUser, lookupTopology, lookupAPIDetails, getCustomPropertiesConfig, mergeCustomProperties, isIgnoreAPI, lookupApplication }, { pluginContext: { cache: cache }, pluginConfig});
 	return sdk.getPlugin();
 }
 

apibuilder4elastic/custom_flow_nodes/api-builder-plugin-axway-api-management/test/test-topologyLookup.js

@@ -0,0 +1,83 @@
+const { expect } = require('chai');
+const { MockRuntime } = require('@axway/api-builder-test-utils');
+const getPlugin = require('../src');
+const path = require('path');
+const fs = require('fs');
+const nock = require('nock');
+const envLoader = require('dotenv');
+const decache = require('decache');
+
+describe('Tests Topology-Lookup', () => {
+	let plugin;
+	let flowNode;
+
+	// Loads environment variables from .env if the file exists
+	const envFilePath = path.join(__dirname, '.env');
+	if (fs.existsSync(envFilePath)) {
+		delete process.env.API_MANAGER; // Otherwise it is not overwritten
+		envLoader.config({ path: envFilePath });
+	}
+	// Delete the cached module 
+	decache('../config/axway-api-utils.default.js');
+	var pluginConfig = require('../config/axway-api-utils.default.js').pluginConfig['api-builder-plugin-axway-api-management'];
+
+	beforeEach(async () => {
+		plugin = await MockRuntime.loadPlugin(getPlugin,pluginConfig);
+		plugin.setOptions({ validateOutputs: true });
+		flowNode = plugin.getFlowNode('axway-api-management');
+	});
+
+	describe('#lookupTopology', () => {
+		it('should error when requestHeaders are not set', async () => {
+			const { value, output } = await flowNode.lookupTopology({
+				requestHeaders: null
+			});
+
+			expect(value).to.be.instanceOf(Error)
+				.and.to.have.property('message', 'You need to provide Request-Headers with Cookies or an Authorization header.');
+			expect(output).to.equal('error');
+		});
+
+		it('should error when requestHeaders are set, but contain no cookie header', async () => {
+			const { value, output } = await flowNode.lookupTopology({
+				requestHeaders: {"host":"api-gateway:8090","max-forwards":"20"}
+			});
+
+			expect(value).to.be.instanceOf(Error)
+				.and.to.have.property('message', 'You must provide either the VIDUSR cookie + csrf-token or an HTTP-Basic Authorization header.');
+			expect(output).to.equal('error');
+		});
+
+		it('should error when requestHeaders are set, but the VIDUSR cookie is missing', async () => {
+			const { value, output } = await flowNode.lookupTopology({
+				requestHeaders: {"host":"api-gateway:8090","max-forwards":"20", "cookie":"XXX-VIDUSR=1597381095-XTawGDtJhBA7Zw%3d%3d;"}
+			});
+
+			expect(value).to.be.instanceOf(Error)
+				.and.to.have.property('message', 'The requestHeaders do not contain the required cookie VIDUSR');
+			expect(output).to.equal('error');
+		});
+
+		it('should error when requestHeaders are set, but the CSRF-Token is missing', async () => {
+			const { value, output } = await flowNode.lookupTopology({
+				requestHeaders: {"host":"api-gateway:8090","max-forwards":"20", "cookie":"VIDUSR=1597381095-XTawGDtJhBA7Zw%3d%3d;"}
+			});
+
+			expect(value).to.be.instanceOf(Error)
+				.and.to.have.property('message', 'The requestHeaders do not contain the required header csrf-token');
+			expect(output).to.equal('error');
+		});
+
+		it('should result into the API-Gateway topology', async () => {
+			nock('https://mocked-api-gateway:8190').get('/api/topology').replyWithFile(200, './test/testReplies/gateway/gatewayEMTTopology.json');
+
+			const { value, output } = await flowNode.lookupTopology({ 
+				requestHeaders: {"host":"api-gateway:8090","max-forwards":"20", "cookie":"VIDUSR=1597381095-XTawGDtJhBA7Zw==;", "csrf-token": "CF2796B3BD18C1B0B5AB1C8E95B75662E92FBC04BD799DEB97838FC5B9C39348"}
+			});
+
+			expect(value.emtEnabled).to.equal(true);
+			expect(value.services).to.lengthOf(3); // We expect only 3 services, as the ANM is removed already
+			expect(output).to.equal('next');
+		});
+	});
+});

apibuilder4elastic/custom_flow_nodes/api-builder-plugin-axway-api-management/test/testReplies/gateway/gatewayEMTTopology.json

@@ -0,0 +1,154 @@
+{
+    "result": {
+        "id": "DefaultDomain",
+        "version": 44,
+        "timestamp": 1624264680386,
+        "productVersion": "7.7.0",
+        "hosts": [
+            {
+                "id": "host-24",
+                "name": "192.168.81.10"
+            },
+            {
+                "id": "host-6",
+                "name": "192.168.86.81"
+            },
+            {
+                "id": "host-23",
+                "name": "192.168.94.252"
+            },
+            {
+                "id": "host-1",
+                "name": "anm-d5574b69-qwnmj"
+            }
+        ],
+        "groups": [
+            {
+                "id": "DefaultGroup",
+                "name": "DefaultGroup",
+                "tags": {},
+                "services": [
+                    {
+                        "id": "apimgr-6c9876cb48-g4sdq",
+                        "name": "apimgr-6c9876cb48-g4sdq",
+                        "type": "gateway",
+                        "scheme": "https",
+                        "hostID": "host-6",
+                        "managementPort": 8085,
+                        "tags": {
+                            "image": "docker.pkg.github.com/cwiechmann/axway-api-management-automated/manager:77-20210330-v1-833aab9",
+                            "productVersion": "7.7.20210330"
+                        },
+                        "enabled": true
+                    },
+                    {
+                        "id": "traffic-7cb4f6989f-bjw8n",
+                        "name": "traffic-7cb4f6989f-bjw8n",
+                        "type": "gateway",
+                        "scheme": "https",
+                        "hostID": "host-24",
+                        "managementPort": 8085,
+                        "tags": {
+                            "image": "docker.pkg.github.com/cwiechmann/axway-api-management-automated/manager:77-20210330-v1-696cc6d",
+                            "productVersion": "7.7.20210330"
+                        },
+                        "enabled": true
+                    },
+                    {
+                        "id": "traffic-7cb4f6989f-jbmf7",
+                        "name": "traffic-7cb4f6989f-jbmf7",
+                        "type": "gateway",
+                        "scheme": "https",
+                        "hostID": "host-23",
+                        "managementPort": 8085,
+                        "tags": {
+                            "image": "docker.pkg.github.com/cwiechmann/axway-api-management-automated/manager:77-20210330-v1-696cc6d",
+                            "productVersion": "7.7.20210330"
+                        },
+                        "enabled": true
+                    }
+                ],
+                "lock": null
+            },
+            {
+                "id": "emt_anm_group",
+                "name": "Node Manager Group",
+                "tags": {},
+                "services": [
+                    {
+                        "id": "nodemanager-1",
+                        "name": "Admin Node Manager",
+                        "type": "nodemanager",
+                        "scheme": "https",
+                        "hostID": "host-1",
+                        "managementPort": 8090,
+                        "tags": {
+                            "internal_admin_nm": "true"
+                        },
+                        "enabled": true
+                    }
+                ],
+                "lock": null
+            }
+        ],
+        "uniqueIdCounters": {
+            "Host": 25,
+            "NodeManager": 2,
+            "Group": 2
+        },
+        "emtEnabled": true,
+        "services": [
+            {
+                "id": "nodemanager-1",
+                "name": "Admin Node Manager",
+                "type": "nodemanager",
+                "scheme": "https",
+                "hostID": "host-1",
+                "managementPort": 8090,
+                "tags": {
+                    "internal_admin_nm": "true"
+                },
+                "enabled": true
+            },
+            {
+                "id": "apimgr-6c9876cb48-g4sdq",
+                "name": "apimgr-6c9876cb48-g4sdq",
+                "type": "gateway",
+                "scheme": "https",
+                "hostID": "host-6",
+                "managementPort": 8085,
+                "tags": {
+                    "image": "docker.pkg.github.com/cwiechmann/axway-api-management-automated/manager:77-20210330-v1-833aab9",
+                    "productVersion": "7.7.20210330"
+                },
+                "enabled": true
+            },
+            {
+                "id": "traffic-7cb4f6989f-bjw8n",
+                "name": "traffic-7cb4f6989f-bjw8n",
+                "type": "gateway",
+                "scheme": "https",
+                "hostID": "host-24",
+                "managementPort": 8085,
+                "tags": {
+                    "image": "docker.pkg.github.com/cwiechmann/axway-api-management-automated/manager:77-20210330-v1-696cc6d",
+                    "productVersion": "7.7.20210330"
+                },
+                "enabled": true
+            },
+            {
+                "id": "traffic-7cb4f6989f-jbmf7",
+                "name": "traffic-7cb4f6989f-jbmf7",
+                "type": "gateway",
+                "scheme": "https",
+                "hostID": "host-23",
+                "managementPort": 8085,
+                "tags": {
+                    "image": "docker.pkg.github.com/cwiechmann/axway-api-management-automated/manager:77-20210330-v1-696cc6d",
+                    "productVersion": "7.7.20210330"
+                },
+                "enabled": true
+            }
+        ]
+    }
+}