Skip to content
Permalink
Browse files Browse the repository at this point in the history
Cleanup request handler
Squashed commit of the following:

commit 90368698c81ed731c6d9be8b02bfa8ad1b5edd3f
Author: rijkvanzanten <rijkvanzanten@me.com>
Date:   Mon Feb 13 15:49:12 2023 -0500

    Cleanup

commit 61514f4
Author: rijkvanzanten <rijkvanzanten@me.com>
Date:   Mon Feb 13 15:44:15 2023 -0500

    Rename to index

commit 38fe6b8
Author: rijkvanzanten <rijkvanzanten@me.com>
Date:   Mon Feb 13 15:41:23 2023 -0500

    Test coverage 100%

commit f2e36db
Author: rijkvanzanten <rijkvanzanten@me.com>
Date:   Mon Feb 13 15:22:45 2023 -0500

    Split up handler from validator

commit 800ac19
Author: Pascal Jufer <pascal-jufer@bluewin.ch>
Date:   Mon Feb 13 20:44:48 2023 +0100

    Use shared axios instance with URL check for outgoing requests
  • Loading branch information
rijkvanzanten committed Feb 13, 2023
1 parent ea91c40 commit ff53d3e
Show file tree
Hide file tree
Showing 11 changed files with 223 additions and 73 deletions.
2 changes: 1 addition & 1 deletion api/src/env.ts
Expand Up @@ -257,7 +257,7 @@ const defaults: Record<string, any> = {
IP_TRUST_PROXY: true,
IP_CUSTOM_HEADER: false,

IMPORT_IP_DENY_LIST: '0.0.0.0',
IMPORT_IP_DENY_LIST: ['0.0.0.0', '169.254.169.254'],

SERVE_APP: true,

Expand Down
15 changes: 8 additions & 7 deletions api/src/operations/request/index.test.ts
Expand Up @@ -2,13 +2,14 @@ import { afterEach, expect, test, vi } from 'vitest';

const axiosDefault = vi.fn();

vi.mock('axios', () => ({
default: axiosDefault.mockResolvedValue({
status: 200,
statusText: 'OK',
headers: {},
data: {},
}),
vi.mock('../../request', () => ({
getAxios: () =>
axiosDefault.mockResolvedValue({
status: 200,
statusText: 'OK',
headers: {},
data: {},
}),
}));

const url = '/';
Expand Down
4 changes: 2 additions & 2 deletions api/src/operations/request/index.ts
@@ -1,5 +1,6 @@
import { defineOperationApi, parseJSON } from '@directus/shared/utils';
import encodeUrl from 'encodeurl';
import { getAxios } from '../../request/index';

type Options = {
url: string;
Expand All @@ -12,8 +13,6 @@ export default defineOperationApi<Options>({
id: 'request',

handler: async ({ url, method, body, headers }) => {
const axios = (await import('axios')).default;

const customHeaders =
headers?.reduce((acc, { header, value }) => {
acc[header] = value;
Expand All @@ -24,6 +23,7 @@ export default defineOperationApi<Options>({
customHeaders['Content-Type'] = 'application/json';
}

const axios = await getAxios();
const result = await axios({
url: encodeUrl(url),
method,
Expand Down
31 changes: 31 additions & 0 deletions api/src/request/index.test.ts
@@ -0,0 +1,31 @@
import { test, vi, afterEach, beforeEach, expect } from 'vitest';
import { getAxios, _cache } from './index';
import axios from 'axios';
import type { AxiosInstance } from 'axios';

vi.mock('axios');

let mockAxiosInstance: AxiosInstance;

beforeEach(() => {
mockAxiosInstance = {
interceptors: {
response: {
use: vi.fn(),
},
},
} as unknown as AxiosInstance;

vi.mocked(axios.create).mockReturnValue(mockAxiosInstance);
});

afterEach(() => {
vi.resetAllMocks();
_cache.axiosInstance = null;
});

test('Creates and returns new axios instance if cache is empty', async () => {
const instance = await getAxios();
expect(axios.create).toHaveBeenCalled();
expect(instance).toBe(mockAxiosInstance);
});
16 changes: 16 additions & 0 deletions api/src/request/index.ts
@@ -0,0 +1,16 @@
import type { AxiosInstance } from 'axios';
import { responseInterceptor } from './response-interceptor';

export const _cache: { axiosInstance: AxiosInstance | null } = {
axiosInstance: null,
};

export async function getAxios() {
if (!_cache.axiosInstance) {
const axios = (await import('axios')).default;
_cache.axiosInstance = axios.create();
_cache.axiosInstance.interceptors.response.use(responseInterceptor);
}

return _cache.axiosInstance;
}
44 changes: 44 additions & 0 deletions api/src/request/response-interceptor.test.ts
@@ -0,0 +1,44 @@
import { randIp, randUrl } from '@ngneat/falso';
import type { AxiosResponse } from 'axios';
import { afterEach, beforeEach, expect, test, vi } from 'vitest';
import { responseInterceptor } from './response-interceptor';
import { validateIP } from './validate-ip';

vi.mock('./validate-ip');

let sample: {
remoteAddress: string;
url: string;
};

let sampleResponseConfig: AxiosResponse<any, any>;

beforeEach(() => {
sample = {
remoteAddress: randIp(),
url: randUrl(),
};

sampleResponseConfig = {
request: {
socket: {
remoteAddress: sample.remoteAddress,
},
url: sample.url,
},
} as AxiosResponse<any, any>;
});

afterEach(() => {
vi.resetAllMocks();
});

test(`Calls validateIP with IP/url from axios request config`, async () => {
await responseInterceptor(sampleResponseConfig);
expect(validateIP).toHaveBeenCalledWith(sample.remoteAddress, sample.url);
});

test(`Returns passed in config as-is`, async () => {
const config = await responseInterceptor(sampleResponseConfig);
expect(config).toBe(sampleResponseConfig);
});
7 changes: 7 additions & 0 deletions api/src/request/response-interceptor.ts
@@ -0,0 +1,7 @@
import type { AxiosResponse } from 'axios';
import { validateIP } from './validate-ip';

export const responseInterceptor = async (config: AxiosResponse<any, any>) => {
await validateIP(config.request.socket.remoteAddress, config.request.url);
return config;
};
81 changes: 81 additions & 0 deletions api/src/request/validate-ip.test.ts
@@ -0,0 +1,81 @@
import { randIp, randUrl } from '@ngneat/falso';
import os from 'node:os';
import { afterEach, beforeEach, expect, test, vi } from 'vitest';
import { getEnv } from '../env';
import { validateIP } from './validate-ip';

vi.mock('../env');
vi.mock('node:os');

let sample: {
ip: string;
url: string;
};

beforeEach(() => {
sample = {
ip: randIp(),
url: randUrl(),
};
});

afterEach(() => {
vi.resetAllMocks();
});

test(`Does nothing if IP is valid`, async () => {
vi.mocked(getEnv).mockReturnValue({ IMPORT_IP_DENY_LIST: [] });
await validateIP(sample.ip, sample.url);
});

test(`Throws error if passed IP is denylisted`, async () => {
vi.mocked(getEnv).mockReturnValue({ IMPORT_IP_DENY_LIST: [sample.ip] });

try {
await validateIP(sample.ip, sample.url);
} catch (err: any) {
expect(err).toBeInstanceOf(Error);
expect(err.message).toBe(`Requested URL "${sample.url}" resolves to a denied IP address`);
}
});

test(`Checks against IPs of local networkInterfaces if IP deny list contains 0.0.0.0`, async () => {
vi.mocked(getEnv).mockReturnValue({ IMPORT_IP_DENY_LIST: ['0.0.0.0'] });
vi.mocked(os.networkInterfaces).mockReturnValue({});
await validateIP(sample.ip, sample.url);
expect(os.networkInterfaces).toHaveBeenCalledOnce();
});

test(`Throws error if IP address matches resolved localhost IP`, async () => {
vi.mocked(getEnv).mockReturnValue({ IMPORT_IP_DENY_LIST: ['0.0.0.0'] });
vi.mocked(os.networkInterfaces).mockReturnValue({
fa0: undefined,
lo0: [
{
address: '127.0.0.1',
netmask: '255.0.0.0',
family: 'IPv4',
mac: '00:00:00:00:00:00',
internal: true,
cidr: '127.0.0.1/8',
},
],
en0: [
{
address: sample.ip,
netmask: '255.0.0.0',
family: 'IPv4',
mac: '00:00:00:00:00:00',
internal: true,
cidr: '127.0.0.1/8',
},
],
});

try {
await validateIP(sample.ip, sample.url);
} catch (err: any) {
expect(err).toBeInstanceOf(Error);
expect(err.message).toBe(`Requested URL "${sample.url}" resolves to a denied IP address`);
}
});
24 changes: 24 additions & 0 deletions api/src/request/validate-ip.ts
@@ -0,0 +1,24 @@
import os from 'node:os';
import { getEnv } from '../env';

export const validateIP = async (ip: string, url: string) => {
const env = getEnv();

if (env.IMPORT_IP_DENY_LIST.includes(ip)) {
throw new Error(`Requested URL "${url}" resolves to a denied IP address`);
}

if (env.IMPORT_IP_DENY_LIST.includes('0.0.0.0')) {
const networkInterfaces = os.networkInterfaces();

for (const networkInfo of Object.values(networkInterfaces)) {
if (!networkInfo) continue;

for (const info of networkInfo) {
if (info.address === ip) {
throw new Error(`Requested URL "${url}" resolves to a denied IP address`);
}
}
}
}
};
62 changes: 4 additions & 58 deletions api/src/services/files.ts
@@ -1,22 +1,19 @@
import { toArray } from '@directus/shared/utils';
import { lookup } from 'dns';
import encodeURL from 'encodeurl';
import exif from 'exif-reader';
import { parse as parseIcc } from 'icc';
import { clone, pick } from 'lodash';
import { extension } from 'mime-types';
import net from 'net';
import type { Readable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import os from 'os';
import path from 'path';
import sharp from 'sharp';
import url, { URL } from 'url';
import { promisify } from 'util';
import url from 'url';
import emitter from '../emitter';
import env from '../env';
import { ForbiddenException, InvalidPayloadException, ServiceUnavailableException } from '../exceptions';
import logger from '../logger';
import { getAxios } from '../request/index';
import { getStorage } from '../storage';
import { AbstractServiceOptions, File, Metadata, MutationOptions, PrimaryKey } from '../types';
import { parseIptc, parseXmp } from '../utils/parse-image-metadata';
Expand All @@ -25,8 +22,6 @@ import { ItemsService } from './items';
// @ts-ignore
import formatTitle from '@directus/format-title';

const lookupDNS = promisify(lookup);

export class FilesService extends ItemsService {
constructor(options: AbstractServiceOptions) {
super('directus_files', options);
Expand Down Expand Up @@ -224,8 +219,6 @@ export class FilesService extends ItemsService {
* Import a single file from an external URL
*/
async importOne(importURL: string, body: Partial<File>): Promise<PrimaryKey> {
const axios = (await import('axios')).default;

const fileCreatePermissions = this.accountability?.permissions?.find(
(permission) => permission.collection === 'directus_files' && permission.action === 'create'
);
Expand All @@ -234,62 +227,15 @@ export class FilesService extends ItemsService {
throw new ForbiddenException();
}

let resolvedUrl;

try {
resolvedUrl = new URL(importURL);
} catch (err: any) {
logger.warn(err, `Requested URL ${importURL} isn't a valid URL`);
throw new ServiceUnavailableException(`Couldn't fetch file from url "${importURL}"`, {
service: 'external-file',
});
}

let ip = resolvedUrl.hostname;

if (net.isIP(ip) === 0) {
try {
ip = (await lookupDNS(ip)).address;
} catch (err: any) {
logger.warn(err, `Couldn't lookup the DNS for url ${importURL}`);
throw new ServiceUnavailableException(`Couldn't fetch file from url "${importURL}"`, {
service: 'external-file',
});
}
}

if (env.IMPORT_IP_DENY_LIST.includes('0.0.0.0')) {
const networkInterfaces = os.networkInterfaces();

for (const networkInfo of Object.values(networkInterfaces)) {
if (!networkInfo) continue;

for (const info of networkInfo) {
if (info.address === ip) {
logger.warn(`Requested URL ${importURL} resolves to localhost.`);
throw new ServiceUnavailableException(`Couldn't fetch file from url "${importURL}"`, {
service: 'external-file',
});
}
}
}
}

if (env.IMPORT_IP_DENY_LIST.includes(ip)) {
logger.warn(`Requested URL ${importURL} resolves to a denied IP address.`);
throw new ServiceUnavailableException(`Couldn't fetch file from url "${importURL}"`, {
service: 'external-file',
});
}

let fileResponse;

try {
const axios = await getAxios();
fileResponse = await axios.get<Readable>(encodeURL(importURL), {
responseType: 'stream',
});
} catch (err: any) {
logger.warn(err, `Couldn't fetch file from url "${importURL}"`);
logger.warn(err, `Couldn't fetch file from URL "${importURL}"`);
throw new ServiceUnavailableException(`Couldn't fetch file from url "${importURL}"`, {
service: 'external-file',
});
Expand Down
10 changes: 5 additions & 5 deletions api/src/webhooks.ts
@@ -1,11 +1,12 @@
import { ActionHandler } from '@directus/shared/types';
import getDatabase from './database';
import emitter from './emitter';
import logger from './logger';
import { Webhook, WebhookHeader } from './types';
import { getMessenger } from './messenger';
import { getAxios } from './request/index';
import { WebhooksService } from './services';
import { Webhook, WebhookHeader } from './types';
import { getSchema } from './utils/get-schema';
import { ActionHandler } from '@directus/shared/types';
import { getMessenger } from './messenger';
import { JobQueue } from './utils/job-queue';

let registered: { event: string; handler: ActionHandler }[] = [];
Expand Down Expand Up @@ -55,9 +56,8 @@ export function unregister(): void {

function createHandler(webhook: Webhook, event: string): ActionHandler {
return async (meta, context) => {
const axios = (await import('axios')).default;

if (webhook.collections.includes(meta.collection) === false) return;
const axios = await getAxios();

const webhookPayload = {
event,
Expand Down

0 comments on commit ff53d3e

Please sign in to comment.