Skip to content

Commit

Permalink
Merge c53d52a into c59da6e
Browse files Browse the repository at this point in the history
  • Loading branch information
@fboucquez
fboucquez committed Jan 12, 2022
2 parents c59da6e + c53d52a commit ee05ba9
Show file tree
Hide file tree
Showing 23 changed files with 397 additions and 275 deletions.
2 changes: 1 addition & 1 deletion presets/mainnet/network.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ minVotingKeyLifetime: 112
maxVotingKeyLifetime: 360
votingKeyDesiredLifetime: 360
votingKeyDesiredFutureLifetime: 60
lastKnownNetworkEpoch: 603
lastKnownNetworkEpoch: 604
stepDuration: 5m
maxBlockFutureTime: 300ms
maxAccountRestrictionValues: 100
Expand Down
2 changes: 1 addition & 1 deletion presets/testnet/network.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ importanceGrouping: 180
votingSetGrouping: 720
votingKeyDesiredLifetime: 720
votingKeyDesiredFutureLifetime: 120
lastKnownNetworkEpoch: 124
lastKnownNetworkEpoch: 125
minVotingKeyLifetime: 28
maxVotingKeyLifetime: 720
stepDuration: 4m
Expand Down
16 changes: 13 additions & 3 deletions src/commands/wizard.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,17 @@ import { join } from 'path';
import { Account, NetworkType, PublicAccount } from 'symbol-sdk';
import { Logger, LoggerFactory, LogType } from '../logger';
import { CustomPreset, PrivateKeySecurityMode } from '../model';
import { Assembly, BootstrapService, BootstrapUtils, CommandUtils, ConfigLoader, ConfigService, KeyName, Preset } from '../service';
import {
Assembly,
BootstrapService,
BootstrapUtils,
CommandUtils,
ConfigLoader,
ConfigService,
KeyName,
Preset,
RuntimeService,
} from '../service';

export const assembliesDescriptions: Record<Assembly, string> = {
[Assembly.dual]: 'Dual Node',
Expand Down Expand Up @@ -136,10 +146,10 @@ export class Wizard {

if (!flags.skipPull) {
const service = new BootstrapService(this.logger);
const runtimeService = new RuntimeService(this.logger);
this.logger.info('\nPulling catapult tools image before asking to go offline...\n');
ConfigLoader.presetInfoLogged = true;
await BootstrapUtils.pullImage(
this.logger,
await runtimeService.pullImage(
service.resolveConfigPreset({
...ConfigService.defaultParams,
preset: preset,
Expand Down
189 changes: 5 additions & 184 deletions src/service/BootstrapUtils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
* limitations under the License.
*/

import { spawn } from 'child_process';
import {
createWriteStream,
existsSync,
Expand All @@ -33,16 +32,13 @@ import * as _ from 'lodash';
import { totalmem } from 'os';
import { basename, dirname, isAbsolute, join, resolve } from 'path';
import { Convert, DtoMapping, NetworkType } from 'symbol-sdk';
import * as util from 'util';
import { Logger } from '../logger';
import { CryptoUtils } from './CryptoUtils';
import { Utils } from './Utils';
// eslint-disable-next-line @typescript-eslint/no-var-requires
const version = require('../../package.json').version;

// eslint-disable-next-line @typescript-eslint/no-var-requires
const yaml = require('js-yaml');
// eslint-disable-next-line @typescript-eslint/no-var-requires
const exec = util.promisify(require('child_process').exec);

export type Password = string | false | undefined;

Expand All @@ -69,7 +65,6 @@ export class BootstrapUtils {
public static readonly defaultWorkingDir = '.';

public static readonly CURRENT_USER = 'current';
private static readonly pulledImages: string[] = [];

public static readonly VERSION = version;
/**
Expand Down Expand Up @@ -131,7 +126,7 @@ export class BootstrapUtils {
function showDownloadingProgress(received: number, total: number) {
const percentage = ((received * 100) / total).toFixed(2);
const message = percentage + '% | ' + received + ' bytes downloaded out of ' + total + ' bytes.';
BootstrapUtils.logSameLineMessage(message);
Utils.logSameLineMessage(message);
}
const request = get(url, (response) => {
const total = parseInt(response.headers['content-length'] || '0', 10);
Expand Down Expand Up @@ -182,11 +177,6 @@ export class BootstrapUtils {
}
}

public static logSameLineMessage(message: string): void {
process.stdout.write(BootstrapUtils.isWindows() ? '\x1b[0G' : '\r');
process.stdout.write(message);
}

public static deleteFolder(logger: Logger, folder: string, excludeFiles: string[] = []): void {
if (existsSync(folder)) {
logger.info(`Deleting folder ${folder}`);
Expand Down Expand Up @@ -224,35 +214,6 @@ export class BootstrapUtils {
}
}

// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
public static validateIsDefined(value: any, message: string): void {
if (value === undefined || value === null) {
throw new Error(message);
}
}

public static validateIsTrue(value: boolean, message: string): void {
if (!value) {
throw new Error(message);
}
}

public static async pullImage(logger: Logger, image: string): Promise<void> {
this.validateIsDefined(image, 'Image must be provided');
if (BootstrapUtils.pulledImages.indexOf(image) > -1) {
return;
}
try {
logger.info(`Pulling image ${image}`);
const stdout = await this.spawn(logger, 'docker', ['pull', image], true, `${image} `);
const outputLines = stdout.toString().split('\n');
logger.info(`Image pulled: ${outputLines[outputLines.length - 2]}`);
BootstrapUtils.pulledImages.push(image);
} catch (e) {
logger.warn(`Image ${image} could not be pulled!`);
}
}

public static resolveRootFolder(): string {
const rootFolder = resolve(__dirname, '../..');
if (!existsSync(join(rootFolder, 'presets', 'shared.yml'))) {
Expand All @@ -261,45 +222,11 @@ export class BootstrapUtils {
return rootFolder;
}

public static runImageUsingExec(
logger: Logger,
{
catapultAppFolder,
image,
userId,
workdir,
cmds,
binds,
}: {
catapultAppFolder?: string;
image: string;
userId?: string;
workdir?: string;
cmds: string[];
binds: string[];
},
): Promise<{ stdout: string; stderr: string }> {
const volumes = binds.map((b) => `-v ${b}`).join(' ');
const userParam = userId ? `-u ${userId}` : '';
const workdirParam = workdir ? `--workdir=${workdir}` : '';
const environmentParam = catapultAppFolder ? `--env LD_LIBRARY_PATH=${catapultAppFolder}/lib:${catapultAppFolder}/deps` : '';
const runCommand = `docker run --rm ${userParam} ${workdirParam} ${environmentParam} ${volumes} ${image} ${cmds
.map((a) => `"${a}"`)
.join(' ')}`;
logger.info(BootstrapUtils.secureString(`Running image using Exec: ${image} ${cmds.join(' ')}`));
return this.exec(logger, runCommand);
}

public static toAns1(privateKey: string): string {
const prefix = '302e020100300506032b657004220420';
return `${prefix}${privateKey.toLowerCase()}`;
}

public static secureString(text: string): string {
const regex = new RegExp('[0-9a-fA-F]{64}', 'g');
return text.replace(regex, 'HIDDEN_KEY');
}

public static sleep(ms: number): Promise<any> {
// Create a promise that rejects in <ms> milliseconds
return new Promise<void>((resolve) => {
Expand Down Expand Up @@ -407,9 +334,9 @@ export class BootstrapUtils {
const compiledTemplate = Handlebars.compile(template);
return compiledTemplate(templateContext);
} catch (e) {
const securedTemplate = BootstrapUtils.secureString(template);
const securedContext = BootstrapUtils.secureString(BootstrapUtils.toYaml(templateContext));
const securedMessage = BootstrapUtils.secureString(e.message || 'Unknown');
const securedTemplate = Utils.secureString(template);
const securedContext = Utils.secureString(BootstrapUtils.toYaml(templateContext));
const securedMessage = Utils.secureString(e.message || 'Unknown');

const message = `Unknown error rendering template. Error: ${securedMessage}\nTemplate:\n${securedTemplate}.`;
throw new Error(`${message}\nContext: \n${securedContext}`);
Expand Down Expand Up @@ -490,112 +417,6 @@ export class BootstrapUtils {
return await fsPromises.readFile(path, 'utf8');
}

private static dockerUserId: string;

public static async resolveDockerUserFromParam(logger: Logger, paramUser: string | undefined): Promise<string | undefined> {
if (!paramUser || paramUser.trim() === '') {
return undefined;
}
if (paramUser === BootstrapUtils.CURRENT_USER) {
return BootstrapUtils.getDockerUserGroup(logger);
}
return paramUser;
}
public static async createImageUsingExec(logger: Logger, targetFolder: string, dockerFile: string, tag: string): Promise<string> {
const runCommand = `docker build -f ${dockerFile} ${targetFolder} -t ${tag}`;
logger.info(`Creating image image '${tag}' from ${dockerFile}`);
return (await this.exec(logger, runCommand)).stdout;
}

public static async exec(logger: Logger, runCommand: string): Promise<{ stdout: string; stderr: string }> {
logger.debug(`Exec command: ${runCommand}`);
const { stdout, stderr } = await exec(runCommand);
return { stdout, stderr };
}

public static async spawn(logger: Logger, command: string, args: string[], useLogger: boolean, logPrefix = ''): Promise<string> {
const cmd = spawn(command, args);
return new Promise<string>((resolve, reject) => {
logger.info(`Spawn command: ${command} ${args.join(' ')}`);
let logText = useLogger ? '' : 'Check console for output....';
const log = (data: string, isError: boolean) => {
if (useLogger) {
logText = logText + `${data}\n`;
if (isError) logger.warn(BootstrapUtils.secureString(logPrefix + data));
else logger.info(BootstrapUtils.secureString(logPrefix + data));
} else {
console.log(logPrefix + data);
}
};

cmd.stdout.on('data', (data) => {
log(`${data}`.trim(), false);
});

cmd.stderr.on('data', (data) => {
log(`${data}`.trim(), true);
});

cmd.on('error', (error) => {
log(`${error.message}`.trim(), true);
});

cmd.on('exit', (code, signal) => {
if (code) {
log(`Process exited with code ${code} and signal ${signal}`, true);
reject(logText);
} else {
resolve(logText);
}
});

cmd.on('close', (code) => {
if (code) {
log(`Process closed with code ${code}`, true);
reject(logText);
} else {
resolve(logText);
}
});

process.on('SIGINT', () => {
resolve(logText);
});
});
}

public static async getDockerUserGroup(logger: Logger): Promise<string> {
const isWin = this.isWindows();
if (isWin) {
return '';
}
if (BootstrapUtils.dockerUserId !== undefined) {
return BootstrapUtils.dockerUserId;
}
try {
const userId = process?.getuid();
const groupId = process?.getgid();
const user = `${userId}:${groupId}`;
logger.info(`User for docker resolved: ${user}`);
if (userId === 0) {
logger.error('YOU ARE RUNNING BOOTSTRAP AS ROOT!!!! THIS IS NOT RECOMMENDED!!!');
}
BootstrapUtils.dockerUserId = user;
return user;
} catch (e) {
logger.info(`User for docker could not be resolved: ${e}`);
return '';
}
}

public static isRoot(): boolean {
return !this.isWindows() && process?.getuid() === 0;
}

public static isWindows(): boolean {
return process.platform === 'win32';
}

public static validateFolder(workingDirFullPath: string): void {
if (!existsSync(workingDirFullPath)) {
throw new Error(`${workingDirFullPath} folder does not exist`);
Expand Down
26 changes: 16 additions & 10 deletions src/service/CertificateService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,12 @@ import { CertificatePair } from '../model';
import { BootstrapUtils } from './BootstrapUtils';
import { CommandUtils } from './CommandUtils';
import { KeyName } from './ConfigService';
import { RuntimeService } from './RuntimeService';
import { Utils } from './Utils';

export interface CertificateParams {
readonly target: string;
readonly user: string;
readonly user?: string;
}

export interface CertificateMetadata {
Expand All @@ -42,7 +44,11 @@ export interface NodeCertificates {
export class CertificateService {
private static readonly METADATA_VERSION = 1;

constructor(private readonly logger: Logger, protected readonly params: CertificateParams) {}
private readonly runtimeService: RuntimeService;

constructor(private readonly logger: Logger, protected readonly params: CertificateParams) {
this.runtimeService = new RuntimeService(this.logger);
}

public static getCertificates(stdout: string): CertificatePair[] {
const locations = (string: string, substring: string): number[] => {
Expand Down Expand Up @@ -129,17 +135,17 @@ export class CertificateService {
await BootstrapUtils.writeTextFile(join(certFolder, 'createNodeCertificates.sh'), command);
const cmd = ['bash', 'createNodeCertificates.sh'];
const binds = [`${resolve(certFolder)}:/data:rw`];
const userId = await BootstrapUtils.resolveDockerUserFromParam(this.logger, this.params.user);
const { stdout, stderr } = await BootstrapUtils.runImageUsingExec(this.logger, {
const userId = await this.runtimeService.resolveDockerUserFromParam(this.params.user);
const { stdout, stderr } = await this.runtimeService.runImageUsingExec({
image: symbolServerImage,
userId: userId,
workdir: '/data',
cmds: cmd,
binds: binds,
});
if (stdout.indexOf('Certificate Created') < 0) {
this.logger.info(BootstrapUtils.secureString(stdout));
this.logger.error(BootstrapUtils.secureString(stderr));
this.logger.info(Utils.secureString(stdout));
this.logger.error(Utils.secureString(stderr));
throw new Error('Certificate creation failed. Check the logs!');
}

Expand All @@ -151,10 +157,10 @@ export class CertificateService {
const caCertificate = certificates[0];
const nodeCertificate = certificates[1];

BootstrapUtils.validateIsTrue(caCertificate.privateKey === mainAccountPrivateKey, 'Invalid ca private key');
BootstrapUtils.validateIsTrue(caCertificate.publicKey === providedCertificates.main.publicKey, 'Invalid ca public key');
BootstrapUtils.validateIsTrue(nodeCertificate.privateKey === transportPrivateKey, 'Invalid Node private key');
BootstrapUtils.validateIsTrue(nodeCertificate.publicKey === providedCertificates.transport.publicKey, 'Invalid Node public key');
Utils.validateIsTrue(caCertificate.privateKey === mainAccountPrivateKey, 'Invalid ca private key');
Utils.validateIsTrue(caCertificate.publicKey === providedCertificates.main.publicKey, 'Invalid ca public key');
Utils.validateIsTrue(nodeCertificate.privateKey === transportPrivateKey, 'Invalid Node private key');
Utils.validateIsTrue(nodeCertificate.publicKey === providedCertificates.transport.publicKey, 'Invalid Node public key');

const metadata: CertificateMetadata = {
version: CertificateService.METADATA_VERSION,
Expand Down
3 changes: 2 additions & 1 deletion src/service/ComposeService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import { Logger } from '../logger';
import { Addresses, ConfigPreset, DockerCompose, DockerComposeService, DockerServicePreset } from '../model';
import { BootstrapUtils } from './BootstrapUtils';
import { ConfigLoader } from './ConfigLoader';
import { RuntimeService } from './RuntimeService';

export type ComposeParams = { target: string; user?: string; upgrade?: boolean; password?: string };

Expand Down Expand Up @@ -83,7 +84,7 @@ export class ComposeService {

await BootstrapUtils.chmodRecursive(join(targetDocker, 'mongo'), 0o666);

const user: string | undefined = await BootstrapUtils.resolveDockerUserFromParam(this.logger, this.params.user);
const user: string | undefined = await new RuntimeService(this.logger).resolveDockerUserFromParam(this.params.user);

const vol = (hostFolder: string, imageFolder: string, readOnly: boolean): string => {
return `${hostFolder}:${imageFolder}:${readOnly ? 'ro' : 'rw'}`;
Expand Down
Loading

0 comments on commit ee05ba9

Please sign in to comment.