diff --git a/__tests__/config/utils/credentials.test.ts b/__tests__/config/utils/credentials.test.ts index 4328e9e8..5b34fb5f 100644 --- a/__tests__/config/utils/credentials.test.ts +++ b/__tests__/config/utils/credentials.test.ts @@ -1,18 +1,15 @@ -jest.mock('../../../src/config/utils/env'); - import { getCredentialsFromFlags } from '../../../src/config/utils/credentials'; const baseFlags = { config: '.twilio-function', cwd: process.cwd(), logLevel: 'info' as 'info', + loadSystemEnv: false, }; describe('getCredentialsFromFlags', () => { test('should return empty if nothing is passed', async () => { - require('../../../src/config/utils/env').__setVariables({}, ''); - - const credentials = await getCredentialsFromFlags(baseFlags, undefined); + const credentials = await getCredentialsFromFlags(baseFlags, {}, undefined); expect(credentials).toEqual({ accountSid: '', authToken: '', @@ -20,10 +17,9 @@ describe('getCredentialsFromFlags', () => { }); test('should return flag values if passed', async () => { - require('../../../src/config/utils/env').__setVariables({}, ''); - const credentials = await getCredentialsFromFlags( { ...baseFlags, accountSid: 'ACxxxxx', authToken: 'some-token' }, + {}, undefined ); expect(credentials).toEqual({ @@ -33,16 +29,12 @@ describe('getCredentialsFromFlags', () => { }); test('should read from env file', async () => { - require('../../../src/config/utils/env').__setVariables( + const credentials = await getCredentialsFromFlags( + { ...baseFlags }, { ACCOUNT_SID: 'ACyyyyyyyyy', AUTH_TOKEN: 'example-token', }, - '' - ); - - const credentials = await getCredentialsFromFlags( - { ...baseFlags }, undefined ); expect(credentials).toEqual({ @@ -52,10 +44,9 @@ describe('getCredentialsFromFlags', () => { }); test('should take external default options if nothing is passed', async () => { - require('../../../src/config/utils/env').__setVariables({}, ''); - const credentials = await getCredentialsFromFlags( { ...baseFlags }, + {}, { username: 'ACzzzzzzz', password: 'api-secret', profile: undefined } ); expect(credentials).toEqual({ @@ -65,16 +56,12 @@ describe('getCredentialsFromFlags', () => { }); test('env variables should override external default options', async () => { - require('../../../src/config/utils/env').__setVariables( + const credentials = await getCredentialsFromFlags( + { ...baseFlags }, { ACCOUNT_SID: 'ACyyyyyyyyy', AUTH_TOKEN: 'example-token', }, - '' - ); - - const credentials = await getCredentialsFromFlags( - { ...baseFlags }, { username: 'ACzzzzzzz', password: 'api-secret', profile: undefined } ); expect(credentials).toEqual({ @@ -84,16 +71,12 @@ describe('getCredentialsFromFlags', () => { }); test('external options with profile should override env variables', async () => { - require('../../../src/config/utils/env').__setVariables( + const credentials = await getCredentialsFromFlags( + { ...baseFlags }, { ACCOUNT_SID: 'ACyyyyyyyyy', AUTH_TOKEN: 'example-token', }, - '' - ); - - const credentials = await getCredentialsFromFlags( - { ...baseFlags }, { username: 'ACzzzzzzz', password: 'api-secret', profile: 'demo' } ); expect(credentials).toEqual({ @@ -105,18 +88,15 @@ describe('getCredentialsFromFlags', () => { test('external options with project should override env variables', async () => { // project flag is deprecated and removed in v3 @twilio/cli-core but // included here just to make sure - require('../../../src/config/utils/env').__setVariables( - { - ACCOUNT_SID: 'ACyyyyyyyyy', - AUTH_TOKEN: 'example-token', - }, - '' - ); const credentials = await getCredentialsFromFlags( { ...baseFlags, }, + { + ACCOUNT_SID: 'ACyyyyyyyyy', + AUTH_TOKEN: 'example-token', + }, { username: 'ACzzzzzzz', password: 'api-secret', @@ -130,16 +110,12 @@ describe('getCredentialsFromFlags', () => { }); test('should prefer external CLI if profile is passed', async () => { - require('../../../src/config/utils/env').__setVariables( + const credentials = await getCredentialsFromFlags( + { ...baseFlags }, { ACCOUNT_SID: 'ACyyyyyyyyy', AUTH_TOKEN: 'example-token', }, - '' - ); - - const credentials = await getCredentialsFromFlags( - { ...baseFlags }, { username: 'ACzzzzzzz', password: 'api-secret', profile: 'demo' } ); expect(credentials).toEqual({ @@ -151,16 +127,12 @@ describe('getCredentialsFromFlags', () => { test('should prefer external CLI if project is passed', async () => { // project flag is deprecated and removed in v3 @twilio/cli-core but // included here just to make sure - require('../../../src/config/utils/env').__setVariables( + const credentials = await getCredentialsFromFlags( + { ...baseFlags }, { ACCOUNT_SID: 'ACyyyyyyyyy', AUTH_TOKEN: 'example-token', }, - '' - ); - - const credentials = await getCredentialsFromFlags( - { ...baseFlags }, { username: 'ACzzzzzzz', password: 'api-secret', project: 'demo' } ); expect(credentials).toEqual({ @@ -170,16 +142,12 @@ describe('getCredentialsFromFlags', () => { }); test('should prefer flag over everything', async () => { - require('../../../src/config/utils/env').__setVariables( + const credentials = await getCredentialsFromFlags( + { ...baseFlags, accountSid: 'ACxxxxx', authToken: 'some-token' }, { ACCOUNT_SID: 'ACyyyyyyyyy', AUTH_TOKEN: 'example-token', }, - '' - ); - - const credentials = await getCredentialsFromFlags( - { ...baseFlags, accountSid: 'ACxxxxx', authToken: 'some-token' }, { username: 'ACzzzzzzz', password: 'api-secret', diff --git a/__tests__/config/utils/env.test.ts b/__tests__/config/utils/env.test.ts index f0366dd1..bc05212b 100644 --- a/__tests__/config/utils/env.test.ts +++ b/__tests__/config/utils/env.test.ts @@ -1,6 +1,16 @@ -import { filterEnvVariablesForDeploy } from '../../../src/config/utils/env'; +import { stripIndent } from 'common-tags'; +import mockFs from 'mock-fs'; +import path from 'path'; +import { + filterEnvVariablesForDeploy, + readLocalEnvFile, +} from '../../../src/config/utils/env'; import { EnvironmentVariablesWithAuth } from '../../../src/types/generic'; +function normalize(unixPath: string) { + return path.resolve('/', ...unixPath.split('/')); +} + describe('filterEnvVariablesForDeploy', () => { const testVars: EnvironmentVariablesWithAuth = { ACCOUNT_SID: 'ACCOUNT_SID', @@ -25,3 +35,141 @@ describe('filterEnvVariablesForDeploy', () => { expect(deployVars['hello']).toEqual('world'); }); }); + +describe('readLocalEnvFile', () => { + let backupSystemEnv = {}; + + const baseFlags = { + cwd: '/tmp/project', + env: undefined, + loadSystemEnv: false, + }; + + beforeEach(() => { + mockFs({ + '/tmp/project': { + '.env': stripIndent` + ACCOUNT_SID=ACxxxxxxx + AUTH_TOKEN=123456789f + MY_PHONE_NUMBER=+12345 + SECRET_API_KEY=abc + `, + '.env.prod': stripIndent` + ACCOUNT_SID= + AUTH_TOKEN= + MY_PHONE_NUMBER=+3333333 + SECRET_API_KEY= + `, + }, + '/tmp/project-two': { + '.env': stripIndent` + ACCOUNT_SID=ACyyyyyyy + AUTH_TOKEN=123456789a + TWILIO=https://www.twilio.com + `, + '.env.prod': stripIndent` + ACCOUNT_SID= + AUTH_TOKEN= + TWILIO=https://www.twilio.com + `, + }, + }); + backupSystemEnv = { ...process.env }; + }); + + afterEach(() => { + mockFs.restore(); + process.env = { ...backupSystemEnv }; + }); + + it('should throw an error if you use --load-system-env without --env', async () => { + const errorMessage = stripIndent` + If you are using --load-system-env you'll also have to supply a --env flag. + + The .env file you are pointing at will be used to primarily load environment variables. + Any empty entries in the .env file will fall back to the system's environment variables. + `; + + expect( + readLocalEnvFile({ ...baseFlags, loadSystemEnv: true }) + ).rejects.toEqual(new Error(errorMessage)); + }); + + it('should load the default env variables', async () => { + expect(await readLocalEnvFile(baseFlags)).toEqual({ + localEnv: { + ACCOUNT_SID: 'ACxxxxxxx', + AUTH_TOKEN: '123456789f', + MY_PHONE_NUMBER: '+12345', + SECRET_API_KEY: 'abc', + }, + envPath: normalize('/tmp/project/.env'), + }); + }); + + it('should load env variables from a different filename', async () => { + expect(await readLocalEnvFile({ ...baseFlags, env: '.env.prod' })).toEqual({ + localEnv: { + ACCOUNT_SID: '', + AUTH_TOKEN: '', + MY_PHONE_NUMBER: '+3333333', + SECRET_API_KEY: '', + }, + envPath: normalize('/tmp/project/.env.prod'), + }); + }); + + it('should load the default env variables with different cwd', async () => { + expect( + await readLocalEnvFile({ ...baseFlags, cwd: '/tmp/project-two' }) + ).toEqual({ + localEnv: { + ACCOUNT_SID: 'ACyyyyyyy', + AUTH_TOKEN: '123456789a', + TWILIO: 'https://www.twilio.com', + }, + envPath: normalize('/tmp/project-two/.env'), + }); + }); + + it('should load env variables from a different filename & cwd', async () => { + expect( + await readLocalEnvFile({ + ...baseFlags, + cwd: '/tmp/project-two', + env: '.env.prod', + }) + ).toEqual({ + localEnv: { + ACCOUNT_SID: '', + AUTH_TOKEN: '', + TWILIO: 'https://www.twilio.com', + }, + envPath: normalize('/tmp/project-two/.env.prod'), + }); + }); + + it('should fallback to system env variables for empty variables with loadSystemEnv', async () => { + process.env = { + TWILIO: 'https://www.twilio.com/blog', + ACCOUNT_SID: 'ACzzzzzzz', + SECRET_API_KEY: 'psst', + }; + + expect( + await readLocalEnvFile({ + ...baseFlags, + env: '.env.prod', + loadSystemEnv: true, + }) + ).toEqual({ + localEnv: { + ACCOUNT_SID: 'ACzzzzzzz', + AUTH_TOKEN: '', + MY_PHONE_NUMBER: '+3333333', + SECRET_API_KEY: 'psst', + }, + envPath: normalize('/tmp/project/.env.prod'), + }); + }); +}); diff --git a/package.json b/package.json index 44b84756..a3c88808 100644 --- a/package.json +++ b/package.json @@ -94,6 +94,7 @@ "@types/lodash.flatten": "^4.4.6", "@types/lodash.kebabcase": "^4.1.6", "@types/lodash.startcase": "^4.4.6", + "@types/mock-fs": "^4.10.0", "@types/prompts": "^2.0.1", "@types/supertest": "^2.0.8", "@types/title": "^1.0.5", @@ -107,6 +108,7 @@ "jest-express": "^1.10.1", "lint-staged": "^8.2.1", "listr-silent-renderer": "^1.1.1", + "mock-fs": "^4.12.0", "nock": "^12.0.2", "npm-run-all": "^4.1.5", "prettier": "^1.18.2", diff --git a/src/commands/shared.ts b/src/commands/shared.ts index dd8cf7da..c612fe9f 100644 --- a/src/commands/shared.ts +++ b/src/commands/shared.ts @@ -16,6 +16,7 @@ export type SharedFlagsWithCredentials = SharedFlags & { env?: string; region?: string; edge?: string; + loadSystemEnv: boolean; }; export type ExternalCliOptions = { @@ -60,6 +61,12 @@ export const sharedApiRelatedCliOptions: { [key: string]: Options } = { describe: 'Use a specific auth token for deployment. Uses fields from .env otherwise', }, + 'load-system-env': { + default: false, + type: 'boolean', + describe: + 'Uses system environment variables as fallback for variables specified in your .env file. Needs to be used with --env explicitly specified.', + }, }; export const sharedCliOptions: { [key: string]: Options } = { diff --git a/src/config/activate.ts b/src/config/activate.ts index ee36c356..42dcb9e4 100644 --- a/src/config/activate.ts +++ b/src/config/activate.ts @@ -9,7 +9,11 @@ import { } from '../commands/shared'; import { getFullCommand } from '../commands/utils'; import { readSpecializedConfig } from './global'; -import { getCredentialsFromFlags, readLocalEnvFile, filterEnvVariablesForDeploy } from './utils'; +import { + filterEnvVariablesForDeploy, + getCredentialsFromFlags, + readLocalEnvFile, +} from './utils'; import { mergeFlagsAndConfig } from './utils/mergeFlagsAndConfig'; type ActivateConfig = ApiActivateConfig & { @@ -57,12 +61,13 @@ export async function getConfigFromFlags( flags.environment = ''; } + const { localEnv: envVariables } = await readLocalEnvFile(flags); const { accountSid, authToken } = await getCredentialsFromFlags( flags, + envVariables, externalCliOptions ); - const { localEnv } = await readLocalEnvFile(flags); - const env = filterEnvVariablesForDeploy(localEnv); + const env = filterEnvVariablesForDeploy(envVariables); const command = getFullCommand(flags); const serviceSid = checkForValidServiceSid(command, flags.serviceSid); @@ -81,6 +86,6 @@ export async function getConfigFromFlags( sourceEnvironment: flags.sourceEnvironment, region, edge, - env + env, }; } diff --git a/src/config/deploy.ts b/src/config/deploy.ts index 65631bc3..ff1c2797 100644 --- a/src/config/deploy.ts +++ b/src/config/deploy.ts @@ -10,11 +10,11 @@ import { deprecateFunctionsEnv } from '../commands/utils'; import { getFunctionServiceSid } from '../serverless-api/utils'; import { readSpecializedConfig } from './global'; import { + filterEnvVariablesForDeploy, getCredentialsFromFlags, getServiceNameFromFlags, readLocalEnvFile, readPackageJsonContent, - filterEnvVariablesForDeploy } from './utils'; import { mergeFlagsAndConfig } from './utils/mergeFlagsAndConfig'; @@ -65,12 +65,14 @@ export async function getConfigFromFlags( flags.environment = ''; } + const { localEnv: envFileVars, envPath } = await readLocalEnvFile(flags); const { accountSid, authToken } = await getCredentialsFromFlags( flags, + envFileVars, externalCliOptions ); - const { localEnv, envPath } = await readLocalEnvFile(flags); - const env = filterEnvVariablesForDeploy(localEnv); + + const env = filterEnvVariablesForDeploy(envFileVars); const serviceSid = flags.serviceSid || diff --git a/src/config/list.ts b/src/config/list.ts index 89988ddc..b1f3c437 100644 --- a/src/config/list.ts +++ b/src/config/list.ts @@ -11,7 +11,11 @@ import { } from '../commands/shared'; import { getFunctionServiceSid } from '../serverless-api/utils'; import { readSpecializedConfig } from './global'; -import { getCredentialsFromFlags, getServiceNameFromFlags } from './utils'; +import { + getCredentialsFromFlags, + getServiceNameFromFlags, + readLocalEnvFile, +} from './utils'; import { mergeFlagsAndConfig } from './utils/mergeFlagsAndConfig'; export type ListConfig = ApiListConfig & { @@ -55,8 +59,10 @@ export async function getConfigFromFlags( flags = mergeFlagsAndConfig(configFlags, flags, cliInfo); cwd = flags.cwd || cwd; + const { localEnv: envFileVars, envPath } = await readLocalEnvFile(flags); const { accountSid, authToken } = await getCredentialsFromFlags( flags, + envFileVars, externalCliOptions ); diff --git a/src/config/logs.ts b/src/config/logs.ts index 6c880fbd..2d0276c3 100644 --- a/src/config/logs.ts +++ b/src/config/logs.ts @@ -12,7 +12,7 @@ import { } from '../commands/shared'; import { getFullCommand } from '../commands/utils'; import { readSpecializedConfig } from './global'; -import { getCredentialsFromFlags } from './utils'; +import { getCredentialsFromFlags, readLocalEnvFile } from './utils'; import { mergeFlagsAndConfig } from './utils/mergeFlagsAndConfig'; export type LogsConfig = ClientConfig & @@ -57,8 +57,10 @@ export async function getConfigFromFlags( cwd = flags.cwd || cwd; environment = flags.environment || environment; + const { localEnv: envFileVars, envPath } = await readLocalEnvFile(flags); const { accountSid, authToken } = await getCredentialsFromFlags( flags, + envFileVars, externalCliOptions ); diff --git a/src/config/utils/credentials.ts b/src/config/utils/credentials.ts index e8603f5d..436ab7b7 100644 --- a/src/config/utils/credentials.ts +++ b/src/config/utils/credentials.ts @@ -2,8 +2,8 @@ import { ExternalCliOptions, SharedFlagsWithCredentials, } from '../../commands/shared'; +import { EnvironmentVariablesWithAuth } from '../../types/generic'; import { getDebugFunction } from '../../utils/logger'; -import { readLocalEnvFile } from './env'; const debug = getDebugFunction('twilio-run:config:credentials'); @@ -21,11 +21,16 @@ export type Credentials = { * 5. value passed in through externalCliOptions * 6. empty string * @param flags Flags passed into command + * @param envVariables Environment variables from (.env or system environment) * @param externalCliOptions Any external information for example passed by the Twilio CLI */ export async function getCredentialsFromFlags< T extends SharedFlagsWithCredentials ->(flags: T, externalCliOptions?: ExternalCliOptions): Promise { +>( + flags: T, + envVariables: EnvironmentVariablesWithAuth, + externalCliOptions?: ExternalCliOptions +): Promise { // default Twilio CLI credentials (4) or empty string (5) let accountSid = (externalCliOptions && @@ -38,17 +43,14 @@ export async function getCredentialsFromFlags< externalCliOptions.password) || ''; - if (flags.cwd) { - // env file content (3) - const { localEnv } = await readLocalEnvFile(flags); - if (localEnv.ACCOUNT_SID) { - debug('Override value with .env ACCOUNT_SID value'); - accountSid = localEnv.ACCOUNT_SID; - } - if (localEnv.AUTH_TOKEN) { - debug('Override value with .env AUTH_TOKEN value'); - authToken = localEnv.AUTH_TOKEN; - } + // env file content (3) + if (envVariables.ACCOUNT_SID) { + debug('Override value with .env ACCOUNT_SID value'); + accountSid = envVariables.ACCOUNT_SID; + } + if (envVariables.AUTH_TOKEN) { + debug('Override value with .env AUTH_TOKEN value'); + authToken = envVariables.AUTH_TOKEN; } // specific profile specified. override both credentials (2) diff --git a/src/config/utils/env.ts b/src/config/utils/env.ts index 46a75aba..4b74fb6d 100644 --- a/src/config/utils/env.ts +++ b/src/config/utils/env.ts @@ -1,13 +1,24 @@ +import { EnvironmentVariables } from '@twilio-labs/serverless-api'; +import { stripIndent } from 'common-tags'; import dotenv from 'dotenv'; import path from 'path'; import { EnvironmentVariablesWithAuth } from '../../types/generic'; -import { EnvironmentVariables } from '@twilio-labs/serverless-api'; import { fileExists, readFile } from '../../utils/fs'; export async function readLocalEnvFile(flags: { cwd?: string; env?: string; + loadSystemEnv?: boolean; }): Promise<{ localEnv: EnvironmentVariablesWithAuth; envPath: string }> { + if (flags.loadSystemEnv && typeof flags.env === 'undefined') { + throw new Error(stripIndent` + If you are using --load-system-env you'll also have to supply a --env flag. + + The .env file you are pointing at will be used to primarily load environment variables. + Any empty entries in the .env file will fall back to the system's environment variables. + `); + } + if (flags.cwd) { const envPath = path.resolve(flags.cwd, flags.env || '.env'); @@ -22,12 +33,23 @@ export async function readLocalEnvFile(flags: { const localEnv = dotenv.parse(contentEnvFile); + if (flags.loadSystemEnv && typeof flags.env !== 'undefined') { + for (const key of Object.keys(localEnv)) { + const systemValue = process.env[key]; + if (systemValue) { + localEnv[key] = localEnv[key] || systemValue; + } + } + } + return { localEnv, envPath }; } return { envPath: '', localEnv: {} }; } -export function filterEnvVariablesForDeploy(localEnv: EnvironmentVariablesWithAuth): EnvironmentVariables { +export function filterEnvVariablesForDeploy( + localEnv: EnvironmentVariablesWithAuth +): EnvironmentVariables { const env = { ...localEnv, }; @@ -43,4 +65,4 @@ export function filterEnvVariablesForDeploy(localEnv: EnvironmentVariablesWithAu delete env.AUTH_TOKEN; return env; -} \ No newline at end of file +}