[backend] Adapt http client to handle exclusions (#6185)

opencti-platform/opencti-graphql/src/database/rabbitmq.js

@@ -103,15 +103,16 @@ export const metrics = async (context, user) => {
   const metricApi = async () => {
     const ssl = USE_SSL_MGMT ? 's' : '';
     const baseURL = `http${ssl}://${HOSTNAME_MGMT}:${PORT_MGMT}`;
-    const httpClient = getHttpClient({
+    const httpClientOptions = {
       baseURL,
       responseType: 'json',
       rejectUnauthorized: RABBITMQ_MGMT_REJECT_UNAUTHORIZED,
       auth: {
         username: USERNAME,
         password: PASSWORD,
       },
-    });
+    };
+    const httpClient = getHttpClient(httpClientOptions);
     const overview = await httpClient.get('/api/overview').then((response) => response.data);
     const queues = await httpClient.get(`/api/queues${VHOST_PATH}`).then((response) => response.data);
     // Compute number of push queues

opencti-platform/opencti-graphql/src/domain/connector.js

@@ -207,7 +207,7 @@ export const fetchRemoteStreams = async (context, user, { uri, token, ssl_verify
     const httpClientOptions = { headers, rejectUnauthorized: ssl_verify ?? false, responseType: 'json' };
     const httpClient = getHttpClient(httpClientOptions);
     const remoteUri = `${uri.endsWith('/') ? uri.slice(0, -1) : uri}/graphql`;
-    const { data } = await httpClient.post(remoteUri, { query }, { withCredentials: true });
+    const { data } = await httpClient.post(remoteUri, { query });
     return data.data.streamCollections.edges.map((e) => e.node);
   } catch (e) {
     throw ValidationError('uri', { message: 'Error getting the streams from remote OpenCTI', cause: e });

opencti-platform/opencti-graphql/src/manager/ingestionManager.ts

@@ -108,10 +108,11 @@ const rssItemV2Convert = (turndownService: TurndownService, channel: RssElement,
 };
 
 const rssHttpGetter = (): Getter => {
-  const httpClient = getHttpClient({
+  const httpClientOptions: GetHttpClient = {
     responseType: 'text',
     headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0' }
-  });
+  };
+  const httpClient = getHttpClient(httpClientOptions);
   return async (uri: string) => {
     const { data } = await httpClient.get(uri);
     return data;

opencti-platform/opencti-graphql/src/manager/publisherManager.ts

@@ -28,7 +28,7 @@ import { executionContext, SYSTEM_USER } from '../utils/access';
 import { now } from '../utils/format';
 import type { NotificationData } from '../utils/publisher-mock';
 import { type ActivityNotificationEvent, type DigestEvent, getNotifications, type KnowledgeNotificationEvent, type NotificationUser } from './notificationManager';
-import { getHttpClient } from '../utils/http-client';
+import { type GetHttpClient, getHttpClient } from '../utils/http-client';
 
 const DOC_URI = 'https://docs.opencti.io';
 const PUBLISHER_ENGINE_KEY = conf.get('publisher_manager:lock_key');
@@ -121,8 +121,9 @@ export const internalProcessNotification = async (
       const dataJson = JSON.parse(generatedWebhook);
       const dataHeaders = R.fromPairs((headers ?? []).map((h) => [h.attribute, h.value]));
       const dataParameters = R.fromPairs((params ?? []).map((h) => [h.attribute, h.value]));
-      const httpClient = getHttpClient({ responseType: 'json', headers: dataHeaders });
-      await httpClient({ url, method: verb, params: dataParameters, data: dataJson }).catch((err) => {
+      const httpClientOptions: GetHttpClient = { responseType: 'json', headers: dataHeaders };
+      const httpClient = getHttpClient(httpClientOptions);
+      await httpClient.call({ url, method: verb, params: dataParameters, data: dataJson }).catch((err) => {
         logApp.error(err, { manager: 'PUBLISHER_MANAGER' });
         return { error: err };
       });

opencti-platform/opencti-graphql/src/utils/http-client.ts

@@ -1,6 +1,7 @@
 import https from 'node:https';
-import axios, { type AxiosHeaders, type HeadersDefaults, type RawAxiosRequestHeaders } from 'axios';
-import { getPlatformHttpProxies } from '../config/conf';
+import axios, { type AxiosHeaders, type AxiosRequestConfig, type HeadersDefaults, type RawAxiosRequestHeaders } from 'axios';
+import { Agent } from 'https';
+import { getPlatformHttpProxyAgent } from '../config/conf';
 import { fromBase64, isNotEmptyField } from '../database/utils';
 
 export interface Certificates {
@@ -19,20 +20,33 @@ export interface GetHttpClient {
     password: string
   }
 }
+
+const buildHttpAgentOpts = (uri: string, baseURL: string | undefined, defaultHttpsAgent: Agent) => {
+  const agentUri = baseURL ? `${baseURL}${uri}` : uri;
+  return {
+    httpAgent: getPlatformHttpProxyAgent(agentUri),
+    httpsAgent: getPlatformHttpProxyAgent(agentUri) ?? defaultHttpsAgent,
+    proxy: false // Disable direct proxy protocol in axios http adapter
+  };
+};
 export const getHttpClient = ({ baseURL, headers, rejectUnauthorized, responseType, certificates, auth }: GetHttpClient) => {
-  const proxies = getPlatformHttpProxies();
+  // Build a default https agent to force query options if no proxy is setup
   const cert = isNotEmptyField(certificates?.cert) ? fromBase64(certificates?.cert) : undefined;
   const key = isNotEmptyField(certificates?.key) ? fromBase64(certificates?.key) : undefined;
   const ca = isNotEmptyField(certificates?.ca) ? fromBase64(certificates?.ca) : undefined;
   const defaultHttpsAgent = new https.Agent({ rejectUnauthorized: rejectUnauthorized === true, cert, key, ca });
-  return axios.create({
+  // Create the default axios caller
+  const caller = axios.create({
     baseURL,
     responseType,
     headers,
     auth,
     withCredentials: true,
-    httpAgent: proxies['http:']?.build(),
-    httpsAgent: proxies['https:']?.build() ?? defaultHttpsAgent,
-    proxy: false // Disable direct proxy protocol in axios http adapter
   });
+  // Override methods to setup correct http agents
+  return {
+    call: (config: AxiosRequestConfig) => caller(config),
+    get: async (url: string, opts: any = {}) => caller.get(url, { ...opts, ...buildHttpAgentOpts(url, baseURL, defaultHttpsAgent) }),
+    post: async (url: string, data: object, opts: any = {}) => caller.post(url, data, { ...opts, ...buildHttpAgentOpts(url, baseURL, defaultHttpsAgent) })
+  };
 };