-
Notifications
You must be signed in to change notification settings - Fork 18
/
run.js
126 lines (116 loc) · 6.05 KB
/
run.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
const { flags: flagsHelper } = require('@oclif/command');
const fs = require('fs');
const path = require('path');
const loadJson = require('load-json-file');
const { ENV_VARS } = require('apify-shared/consts');
const semver = require('semver');
const execWithLog = require('../lib/exec');
const { DEFAULT_LOCAL_STORAGE_DIR, SUPPORTED_NODEJS_VERSION } = require('../lib/consts');
const { ApifyCommand } = require('../lib/apify_command');
const {
getLocalUserInfo, purgeDefaultQueue, purgeDefaultKeyValueStore,
purgeDefaultDataset, getLocalConfigOrThrow, getNpmCmd, checkIfStorageIsEmpty,
} = require('../lib/utils');
const { info, warning } = require('../lib/outputs');
const { replaceSecretsValue } = require('../lib/secrets');
class RunCommand extends ApifyCommand {
async run() {
const { flags } = this.parse(RunCommand);
const { proxy, id: userId, token } = getLocalUserInfo();
const localConfig = await getLocalConfigOrThrow();
const cwd = process.cwd();
const packageJsonPath = path.join(cwd, 'package.json');
if (!fs.existsSync(packageJsonPath)) {
throw new Error('The "package.json" file not found in the current directory. Call "npm init" to create it.');
}
const serverJsFile = path.join(cwd, 'server.js');
const packageJson = await loadJson(packageJsonPath);
if ((!packageJson.scripts || !packageJson.scripts.start) && !fs.existsSync(serverJsFile)) {
throw new Error('The npm start script not found in package.json. Please set it up for your project. '
+ 'For more information about that call "apify help run".');
}
// Purge stores
if (flags.purge) {
await Promise.all([purgeDefaultQueue(), purgeDefaultKeyValueStore(), purgeDefaultDataset()]);
info('All default local stores were purged.');
}
if (flags.purgeQueue) {
await purgeDefaultQueue();
info('Default local request queue was purged.');
}
if (flags.purgeDataset) {
await purgeDefaultDataset();
info('Default local dataset was purged.');
}
if (flags.purgeKeyValueStore) {
await purgeDefaultKeyValueStore();
info('Default local key-value store was purged.');
}
// Check if apify storage were purge, if not print error
if (!flags.purge) {
const isStorageEmpty = await checkIfStorageIsEmpty();
if (!isStorageEmpty) {
warning('The apify_storage directory contains a previous state, the actor will continue where it left off. '
+ 'To start from the initial state, use --purge parameter to clean the apify_storage directory.');
}
}
// Attach env vars from local config files
const localEnvVars = {
[ENV_VARS.LOCAL_STORAGE_DIR]: DEFAULT_LOCAL_STORAGE_DIR,
};
if (proxy && proxy.password) localEnvVars[ENV_VARS.PROXY_PASSWORD] = proxy.password;
if (userId) localEnvVars[ENV_VARS.USER_ID] = userId;
if (token) localEnvVars[ENV_VARS.TOKEN] = token;
if (localConfig.env) {
const updatedEnv = replaceSecretsValue(localConfig.env);
Object.assign(localEnvVars, updatedEnv);
}
// NOTE: User can overwrite env vars
const env = Object.assign(localEnvVars, process.env);
if (!userId) {
warning('You are not logged in with your Apify Account. Some features like Apify Proxy will not work. Call "apify login" to fix that.');
}
// --max-http-header-size=80000
// Increases default size of headers. The original limit was 80kb, but from node 10+ they decided to lower it to 8kb.
// However they did not think about all the sites there with large headers,
// so we put back the old limit of 80kb, which seems to work just fine.
const currentNodeVersion = process.versions.node;
const lastSupportedVersion = semver.minVersion(SUPPORTED_NODEJS_VERSION);
if (semver.gte(currentNodeVersion, lastSupportedVersion)) {
env.NODE_OPTIONS = env.NODE_OPTIONS ? `${env.NODE_OPTIONS} --max-http-header-size=80000` : '--max-http-header-size=80000';
} else {
warning(`You are running Node.js version ${currentNodeVersion}, which is no longer supported. `
+ `Please upgrade to Node.js version ${lastSupportedVersion} or later.`);
}
await execWithLog(getNpmCmd(), ['start'], { env });
}
}
// TODO: we should describe which env vars are set here:
RunCommand.description = 'Runs the actor locally in the current directory by executing "npm start".\n'
+ 'It sets various APIFY_XYZ environment variables '
+ 'in order to provide a working execution environment for the actor. For example, this causes '
+ 'the actor input, as well as all other data in key-value stores, '
+ `datasets or request queues to be stored in the "${DEFAULT_LOCAL_STORAGE_DIR}" directory, `
+ 'rather than on the Apify platform.\n\n'
+ 'NOTE: You can override the default behaviour of command overriding npm start script value in a package.json file. '
+ 'You can set up your own main file or environment variables by changing it.';
RunCommand.flags = {
purge: flagsHelper.boolean({
char: 'p',
description: 'Shortcut that combines the --purge-queue, --purge-dataset and --purge-key-value-store options.',
required: false,
}),
'purge-queue': flagsHelper.boolean({
description: 'Deletes the local directory containing the default request queue before the run starts.',
required: false,
}),
'purge-dataset': flagsHelper.boolean({
description: 'Deletes the local directory containing the default dataset before the run starts.',
required: false,
}),
'purge-key-value-store': flagsHelper.boolean({
description: 'Deletes all records from the default key-value store in the local directory before the run starts, except for the "INPUT" key.',
required: false,
}),
};
module.exports = RunCommand;