Skip to content

Commit

Permalink
Tsify compiler finder (#4390)
Browse files Browse the repository at this point in the history 
* Convert compiler-finder to ts

* Updated CompilerProps getter typings to be way more awesome

* A couple small tweaks

* Address review comments
  • Loading branch information
@jeremy-rifkin @mattgodbolt
jeremy-rifkin authored and mattgodbolt committed Jan 24, 2023
1 parent bdb7400 commit 1270599
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 32 deletions.
76 changes: 55 additions & 21 deletions lib/compiler-finder.js → lib/compiler-finder.ts
View file
Expand Up @@ -27,27 +27,56 @@ import https from 'https';
import path from 'path';
import {promisify} from 'util';

import AWS from 'aws-sdk';
import fs from 'fs-extra';
import _ from 'underscore';
import urljoin from 'url-join';

import {Language} from '../types/languages.interfaces';

import {InstanceFetcher} from './aws';
import {CompileHandler} from './handlers/compile';
import {logger} from './logger';
import {ClientOptionsHandler} from './options-handler';
import {CompilerProps, RawPropertiesGetter} from './properties';
import {PropertyGetter, PropertyValue, Widen} from './properties.interfaces';

const sleep = promisify(setTimeout);

export type CompilerFinderArguments = {
rootDir: string;
env: string[];
hostname: string[];
port: number;
gitReleaseName: string;
releaseBuildNumber: string;
wantedLanguages: string | null;
doCache: boolean;
fetchCompilersFromRemote: boolean;
ensureNoCompilerClash: boolean;
suppressConsoleLog: boolean;
};

/***
* Finds and initializes the compilers stored on the properties files
*/
export class CompilerFinder {
/***
* @param {CompileHandler} compileHandler
* @param {CompilerProps} compilerProps
* @param {propsFor} awsProps
* @param {Object} args
* @param {Object} optionsHandler
*/
constructor(compileHandler, compilerProps, awsProps, args, optionsHandler) {
compilerProps: CompilerProps['get'];
ceProps: PropertyGetter;
awsProps: PropertyGetter;
args: CompilerFinderArguments;
compileHandler: CompileHandler;
languages: Record<string, Language>;
awsPoller: InstanceFetcher | null = null;
optionsHandler: ClientOptionsHandler;

constructor(
compileHandler: CompileHandler,
compilerProps: CompilerProps,
awsProps: PropertyGetter,
args: CompilerFinderArguments,
optionsHandler: ClientOptionsHandler,
) {
this.compilerProps = compilerProps.get.bind(compilerProps);
this.ceProps = compilerProps.ceProps;
this.awsProps = awsProps;
Expand Down Expand Up @@ -95,7 +124,7 @@ export class CompilerFinder {
`${uriSchema}://${host}:${port}${apiPath}\n` +
`Status Code: ${statusCode}`,
);
} else if (!/^application\/json/.test(contentType)) {
} else if (!contentType || !/^application\/json/.test(contentType)) {
error = new Error(
'Invalid content-type.\n' +
`Expected application/json but received ${contentType}`,
Expand Down Expand Up @@ -129,7 +158,7 @@ export class CompilerFinder {
return compiler;
});
resolve(compilers);
} catch (e) {
} catch (e: any) {
logger.error(`Error parsing response from ${uri} '${str}': ${e.message}`);
reject(e);
}
Expand All @@ -154,7 +183,7 @@ export class CompilerFinder {
logger.info('Fetching instances from AWS');
const instances = await this.awsInstances();
return Promise.all(
instances.map(instance => {
(instances.filter(instance => instance !== undefined) as AWS.EC2.Instance[]).map(instance => {
logger.info('Checking instance ' + instance.InstanceId);
const address = this.awsProps('externalTestMode', false)
? instance.PublicDnsName
Expand All @@ -164,13 +193,16 @@ export class CompilerFinder {
);
}

async compilerConfigFor(langId, compilerId, parentProps) {
async compilerConfigFor(langId: string, compilerId: string, parentProps: RawPropertiesGetter) {
const base = `compiler.${compilerId}.`;

function props(propName, def) {
function props(propName: string, defaultValue: undefined): PropertyValue;
function props<T extends PropertyValue>(propName: string, defaultValue: Widen<T>): typeof defaultValue;
function props<T extends PropertyValue>(propName: string, defaultValue?: unknown): T;
function props(propName: string, defaultValue?: unknown) {
const propsForCompiler = parentProps(langId, base + propName);
if (propsForCompiler !== undefined) return propsForCompiler;
return parentProps(langId, propName, def);
return parentProps(langId, propName, defaultValue);
}

const ceToolsPath = props('ceToolsPath', './');
Expand Down Expand Up @@ -206,7 +238,7 @@ export class CompilerFinder {
if (envVarsString === '') {
return [];
}
const arr = [];
const arr: [string, string][] = [];
for (const el of envVarsString.split(':')) {
const [env, setting] = el.split('=');
arr.push([env, setting]);
Expand Down Expand Up @@ -261,7 +293,7 @@ export class CompilerFinder {
notification: props('notification', ''),
isSemVer: isSemVer,
semver: semverVer,
libsArr: this.getSupportedLibrariesArr(props, langId),
libsArr: this.getSupportedLibrariesArr(props),
tools: _.omit(this.optionsHandler.get().tools[langId], tool => tool.isCompilerExcluded(compilerId, props)),
unwiseOptions: props('unwiseOptions', '').split('|'),
hidden: props('hidden', false),
Expand Down Expand Up @@ -328,15 +360,15 @@ export class CompilerFinder {
}

async getCompilers() {
const compilers = [];
const compilers: any[] = [];
_.each(this.getExes(), (exs, langId) => {
_.each(exs, exe => compilers.push(this.recurseGetCompilers(langId, exe, this.compilerProps)));
});
return Promise.all(compilers);
}

ensureDistinct(compilers) {
const ids = {};
ensureDistinct(compilers: any[]) {
const ids: Record<string, any> = {};
let foundClash = false;
_.each(compilers, compiler => {
if (!ids[compiler.id]) ids[compiler.id] = [];
Expand Down Expand Up @@ -372,14 +404,16 @@ export class CompilerFinder {
}

getExes() {
const langToCompilers = this.compilerProps(this.languages, 'compilers', '', exs => _.compact(exs.split(':')));
const langToCompilers = this.compilerProps(this.languages, 'compilers', '', exs =>
_.compact((exs as string).split(':')),
);
this.addNdkExes(langToCompilers);
logger.info('Exes found:', langToCompilers);
return langToCompilers;
}

addNdkExes(langToCompilers) {
const ndkPaths = this.compilerProps(this.languages, 'androidNdk');
const ndkPaths = this.compilerProps(this.languages, 'androidNdk') as unknown as Record<string, string>;
_.each(ndkPaths, (ndkPath, langId) => {
if (ndkPath) {
const toolchains = fs.readdirSync(`${ndkPath}/toolchains`);
Expand Down
59 changes: 48 additions & 11 deletions lib/properties.ts
View file
Expand Up @@ -67,6 +67,8 @@ export function get(base: string, property: string, defaultValue?: unknown): unk
return result;
}

export type RawPropertiesGetter = typeof get;

export function parseProperties(blob, name) {
const props = {};
for (const [index, lineOrig] of blob.split('\n').entries()) {
Expand Down Expand Up @@ -132,9 +134,9 @@ type LanguageDef = {
* Compiler property fetcher
*/
export class CompilerProps {
private languages: Record<string, any>;
private propsByLangId: Record<string, PropertyGetter>;
private ceProps: any;
public readonly languages: Record<string, any>;
public readonly propsByLangId: Record<string, PropertyGetter>;
public readonly ceProps: PropertyGetter;

/***
* Creates a CompilerProps lookup function
Expand All @@ -149,6 +151,9 @@ export class CompilerProps {
_.each(this.languages, lang => (this.propsByLangId[lang.id] = propsFor(lang.id)));
}

$getInternal(base: string, property: string, defaultValue: undefined): PropertyValue;
$getInternal<T extends PropertyValue>(base: string, property: string, defaultValue: Widen<T>): typeof defaultValue;
$getInternal<T extends PropertyValue>(base: string, property: string, defaultValue?: PropertyValue): T;
$getInternal(langId: string, key: string, defaultValue: PropertyValue): PropertyValue {
const languagePropertyValue = this.propsByLangId[langId](key);
if (languagePropertyValue !== undefined) {
Expand All @@ -171,26 +176,58 @@ export class CompilerProps {
* @returns {*} Transformed value(s) found or fn(defaultValue)
*/
get(
langs: string | LanguageDef[],
base: string | LanguageDef[] | Record<string, any>,
property: string,
defaultValue?: undefined,
fn?: undefined,
): PropertyValue;
get<T extends PropertyValue>(
base: string | LanguageDef[] | Record<string, any>,
property: string,
defaultValue: Widen<T>,
fn?: undefined,
): typeof defaultValue;
get<T extends PropertyValue>(
base: string | LanguageDef[] | Record<string, any>,
property: string,
defaultValue?: PropertyValue,
fn?: undefined,
): T;

get<R>(
base: string | LanguageDef[] | Record<string, any>,
property: string,
defaultValue?: undefined,
fn?: (item: PropertyValue, language?: any) => R,
): R;
get<T extends PropertyValue, R>(
base: string | LanguageDef[] | Record<string, any>,
property: string,
defaultValue: Widen<T>,
fn?: (item: PropertyValue, language?: any) => R,
): typeof defaultValue | R;

get(
langs: string | LanguageDef[] | Record<string, any>,
key: string,
defaultValue: PropertyValue,
fn: (item: PropertyValue, language?: any) => PropertyValue = _.identity,
defaultValue?: PropertyValue,
fn?: (item: PropertyValue, language?: any) => unknown,
) {
fn = fn || _.identity;
const map_fn = fn || _.identity;
if (_.isEmpty(langs)) {
return fn(this.ceProps(key, defaultValue));
return map_fn(this.ceProps(key, defaultValue));
}
if (!_.isString(langs)) {
return _.chain(langs)
.map(lang => [lang.id, fn(this.$getInternal(lang.id, key, defaultValue), lang)])
.map(lang => [lang.id, map_fn(this.$getInternal(lang.id, key, defaultValue), lang)])
.object()
.value();
} else {
if (this.propsByLangId[langs]) {
return fn(this.$getInternal(langs, key, defaultValue), this.languages[langs]);
return map_fn(this.$getInternal(langs, key, defaultValue), this.languages[langs]);
} else {
logger.error(`Tried to pass ${langs} as a language ID`);
return fn(defaultValue);
return map_fn(defaultValue);
}
}
}
Expand Down

0 comments on commit 1270599

Please sign in to comment.