diff --git a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/lambda.test.ts b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/lambda.test.ts index 241cee517a..70be85f86e 100644 --- a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/lambda.test.ts +++ b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/lambda.test.ts @@ -3,7 +3,7 @@ import { scaleDown as scaleDownL, scaleUp as scaleUpL } from './lambda'; import { Context, SQSEvent, ScheduledEvent } from 'aws-lambda'; import { mocked } from 'ts-jest/utils'; import nock from 'nock'; -import scaleDown from './scale-runners/scale-down'; +import { scaleDown } from './scale-runners/scale-down'; import { scaleUp } from './scale-runners/scale-up'; jest.mock('./scale-runners/scale-down'); diff --git a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/lambda.ts b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/lambda.ts index ec650fe907..2a60f7a8a4 100644 --- a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/lambda.ts +++ b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/lambda.ts @@ -1,6 +1,6 @@ import { Context, SQSEvent, ScheduledEvent } from 'aws-lambda'; -import scaleDownR from './scale-runners/scale-down'; +import { scaleDown as scaleDownR } from './scale-runners/scale-down'; import { scaleUp as scaleUpR } from './scale-runners/scale-up'; // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/gh-runners.ts b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/gh-runners.ts index e46a7a3a68..0730846987 100644 --- a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/gh-runners.ts +++ b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/gh-runners.ts @@ -348,7 +348,7 @@ export async function getRunnerTypes( const config = YAML.parse(configYml); const result: Map = new Map( - /* eslint-disable @typescript-eslint/no-explicit-any */ + /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ (Object.entries(config.runner_types) as [string, any][]).map(([prop, runner_type]) => [ prop, { diff --git a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/metrics.ts b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/metrics.ts index ec0551ecd6..0d4c0b7563 100644 --- a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/metrics.ts +++ b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/metrics.ts @@ -578,6 +578,11 @@ export class ScaleDownMetrics extends Metrics { this.countEntry('run.count'); } + /* istanbul ignore next */ + exception() { + this.countEntry('run.exceptions_count'); + } + /* istanbul ignore next */ runnerLessMinimumTime(ec2Runner: RunnerInfo) { this.countEntry(`run.ec2runners.notMinTime`); diff --git a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts index 5d647f3849..0f907b9be1 100644 --- a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts +++ b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts @@ -1,42 +1,58 @@ -import { listRunners, resetRunnersCaches, terminateRunner } from './runners'; +import moment from 'moment'; +import nock from 'nock'; +import { mocked } from 'ts-jest/utils'; +import { Config } from './config'; +import { resetSecretCache } from './gh-auth'; +import { RunnerInfo, Repo } from './utils'; import { + GhRunner, GhRunners, getRunnerOrg, getRunnerRepo, + getRunnerTypes, listGithubRunnersOrg, listGithubRunnersRepo, removeGithubRunnerOrg, removeGithubRunnerRepo, + resetGHRunnersCaches, } from './gh-runners'; - -import { Config } from './config'; -import { mocked } from 'ts-jest/utils'; -import moment from 'moment'; -import nock from 'nock'; -import scaleDown from './scale-down'; import * as MetricsModule from './metrics'; +import { listRunners, resetRunnersCaches, terminateRunner, RunnerType } from './runners'; +import { + getGHRunnerOrg, + getGHRunnerRepo, + isEphemeralRunner, + isRunnerRemovable, + runnerMinimumTimeExceeded, + scaleDown, + sortRunnersByLaunchTime, +} from './scale-down'; jest.mock('./gh-runners', () => ({ - /* eslint-disable @typescript-eslint/no-explicit-any */ - ...(jest.requireActual('./runners') as any), - getRunnerRepo: jest.fn(), + /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ + ...(jest.requireActual('./gh-runners') as any), getRunnerOrg: jest.fn(), - listGithubRunnersRepo: jest.fn(), + getRunnerRepo: jest.fn(), + getRunnerTypes: jest.fn(), listGithubRunnersOrg: jest.fn(), + listGithubRunnersRepo: jest.fn(), removeGithubRunnerOrg: jest.fn(), removeGithubRunnerRepo: jest.fn(), resetGHRunnersCaches: jest.fn(), - terminateRunner: jest.fn(), })); jest.mock('./runners', () => ({ - /* eslint-disable @typescript-eslint/no-explicit-any */ + /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ ...(jest.requireActual('./runners') as any), listRunners: jest.fn(), resetRunnersCaches: jest.fn(), terminateRunner: jest.fn(), })); +jest.mock('./gh-auth', () => ({ + resetSecretCache: jest.fn(), +})); + beforeEach(() => { jest.resetModules(); jest.clearAllMocks(); @@ -44,71 +60,685 @@ beforeEach(() => { nock.disableNetConnect(); }); +/* eslint-disable-next-line @typescript-eslint/no-explicit-any */ +const mockRunner = (runnerDef: any) => { + return runnerDef as GhRunner; +}; + const metrics = new MetricsModule.ScaleDownMetrics(); -describe('scaleDown', () => { - const minimumRunningTimeInMinutes = 10; - const environment = 'environment'; +const minimumRunningTimeInMinutes = 10; +const environment = 'environment'; +const baseConfig = { + minimumRunningTimeInMinutes: minimumRunningTimeInMinutes, + environment: environment, + minAvailableRunners: 0, +}; +describe('scale-down', () => { beforeEach(() => { - jest.spyOn(Config, 'Instance', 'get').mockImplementation( - () => - ({ - minimumRunningTimeInMinutes: minimumRunningTimeInMinutes, - environment: environment, - } as Config), - ); + jest.spyOn(Config, 'Instance', 'get').mockImplementation(() => baseConfig as Config); jest.spyOn(MetricsModule, 'ScaleDownMetrics').mockReturnValue(metrics); jest.spyOn(metrics, 'sendMetrics').mockImplementation(async () => { return; }); }); - it('no runners are found', async () => { - const ec2runner = { - instanceId: 'WG113', - repo: 'owner/repo', - launchTime: moment(new Date()) - .subtract(minimumRunningTimeInMinutes + 5, 'minutes') - .toDate(), - ghRunnerId: '33', - }; - const matchRunner: GhRunners = [ + describe('scaleDown', () => { + it('no runners are found', async () => { + const ec2runner = { + instanceId: 'WG113', + repo: 'owner/repo', + launchTime: moment(new Date()) + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + ghRunnerId: '33', + }; + const matchRunner: GhRunners = [ + { + id: 1, + name: ec2runner.instanceId, + os: 'linux', + status: 'busy', + busy: true, + labels: [], + }, + ]; + const mockedListRunners = mocked(listRunners).mockResolvedValue([]); + mocked(listGithubRunnersRepo).mockResolvedValue(matchRunner); + + await scaleDown(); + + expect(mockedListRunners).toBeCalledWith(metrics, { environment: environment }); + }); + + it('ec2runner with repo = undefined && org = undefined', async () => { + mocked(listRunners).mockResolvedValue([ + { + instanceId: 'WG113', + launchTime: moment(new Date()) + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }, + ]); + const mockedResetRunnersCaches = mocked(resetRunnersCaches); + const mockedResetGHRunnersCaches = mocked(resetGHRunnersCaches); + const mockedResetSecretCache = mocked(resetSecretCache); + const mockedListGithubRunners = mocked(listGithubRunnersRepo); + const mockedListGithubRunnersOrg = mocked(listGithubRunnersOrg); + const mockedRemoveGithubRunnerOrg = mocked(removeGithubRunnerOrg); + const mockedRemoveGithubRunnerRepo = mocked(removeGithubRunnerRepo); + const mockedTerminateRunner = mocked(terminateRunner); + + await scaleDown(); + + expect(mockedResetRunnersCaches).toBeCalledTimes(1); + expect(mockedResetGHRunnersCaches).toBeCalledTimes(1); + expect(mockedResetSecretCache).toBeCalledTimes(1); + expect(mockedListGithubRunners).not.toBeCalled(); + expect(mockedListGithubRunnersOrg).not.toBeCalled(); + expect(mockedRemoveGithubRunnerOrg).not.toBeCalled(); + expect(mockedRemoveGithubRunnerRepo).not.toBeCalled(); + expect(mockedTerminateRunner).not.toBeCalled(); + }); + }); + + describe('org', () => { + const environment = 'environment'; + const scaleConfigRepo = 'test-infra'; + const theOrg = ' a-owner'; + const dateRef = moment(new Date()); + const runnerTypes = new Map([ + ['ignore-no-org-no-repo', { is_ephemeral: false } as RunnerType], + ['ignore-no-org', { is_ephemeral: false } as RunnerType], + ['keep-all-4', { is_ephemeral: false } as RunnerType], + ['a-ephemeral-runner', { is_ephemeral: true } as RunnerType], + ['keep-min-runners-oldest', { is_ephemeral: false } as RunnerType], + ['keep-lt-min-no-ghrunner', { is_ephemeral: false } as RunnerType], + ]); + const ghRunners = [ + mockRunner({ id: '0001', name: 'keep-this-not-min-time-01', busy: false }), + mockRunner({ id: '0002', name: 'keep-this-not-min-time-02', busy: false }), + mockRunner({ id: '0003', name: 'keep-this-is-busy-01', busy: true }), + mockRunner({ id: '0004', name: 'keep-this-is-busy-02', busy: true }), + mockRunner({ id: '0005', name: 'keep-this-not-min-time-03', busy: false }), + mockRunner({ id: '0006', name: 'keep-this-is-busy-03', busy: true }), + mockRunner({ id: '0007', name: 'remove-ephemeral-01', busy: false }), + mockRunner({ id: '0008', name: 'keep-min-runners-not-oldest-01', busy: false }), + mockRunner({ id: '0009', name: 'keep-min-runners-oldest-01', busy: false }), + mockRunner({ id: '0010', name: 'keep-min-runners-not-oldest-02', busy: false }), + mockRunner({ id: '0011', name: 'keep-min-runners-oldest-02', busy: false }), + mockRunner({ id: '0012', name: 'keep-lt-min-no-ghrunner-01', busy: false }), + ] as GhRunners; + const listRunnersRet = [ + { + runnerType: 'ignore-no-org-no-repo', + instanceId: '001', + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }, + { + runnerType: 'ignore-no-org-no-repo', + instanceId: '002', + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 3, 'minutes') + .toDate(), + }, + + { + runnerType: 'ignore-no-org', + instanceId: '003', + repo: 'a-owner/a-repo', + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 3, 'minutes') + .toDate(), + }, + { + runnerType: 'ignore-no-org', + instanceId: '004', + repo: 'a-owner/a-repo', + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 3, 'minutes') + .toDate(), + }, + + { + runnerType: 'keep-all-4', + instanceId: 'keep-this-not-min-time-01', + org: theOrg, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes - 5, 'minutes') + .toDate(), + }, + { + runnerType: 'keep-all-4', + instanceId: 'keep-this-not-min-time-02', + org: theOrg, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes - 3, 'minutes') + .toDate(), + }, + { + runnerType: 'keep-all-4', + instanceId: 'keep-this-is-busy-01', + org: theOrg, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }, + { + runnerType: 'keep-all-4', + instanceId: 'keep-this-is-busy-02', + org: theOrg, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }, + + { + runnerType: 'a-ephemeral-runner', + instanceId: 'keep-this-not-min-time-03', + org: theOrg, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes - 5, 'minutes') + .toDate(), + }, + { + runnerType: 'a-ephemeral-runner', + instanceId: 'keep-this-is-busy-03', + org: theOrg, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }, + { + runnerType: 'a-ephemeral-runner', + instanceId: 'remove-ephemeral-01', // X + org: theOrg, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }, + { + runnerType: 'a-ephemeral-runner', + instanceId: 'remove-ephemeral-02', // X + org: theOrg, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }, + + { + runnerType: 'keep-min-runners-oldest', + instanceId: 'keep-min-runners-not-oldest-01', + org: theOrg, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }, + { + runnerType: 'keep-min-runners-oldest', + instanceId: 'keep-min-runners-oldest-01', // X + org: theOrg, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 7, 'minutes') + .toDate(), + }, + { + runnerType: 'keep-min-runners-oldest', + instanceId: 'keep-min-runners-not-oldest-02', + org: theOrg, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 6, 'minutes') + .toDate(), + }, + { + runnerType: 'keep-min-runners-oldest', + instanceId: 'keep-min-runners-oldest-02', // X + org: theOrg, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 8, 'minutes') + .toDate(), + }, + { - id: 1, - name: ec2runner.instanceId, - os: 'linux', - status: 'busy', - busy: true, - labels: [], + runnerType: 'keep-lt-min-no-ghrunner', + instanceId: 'keep-lt-min-no-ghrunner-no-ghr-01', // X + org: theOrg, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }, + { + runnerType: 'keep-lt-min-no-ghrunner', + instanceId: 'keep-lt-min-no-ghrunner-no-ghr-02', // X + org: theOrg, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 7, 'minutes') + .toDate(), + }, + { + runnerType: 'keep-lt-min-no-ghrunner', + instanceId: 'keep-lt-min-no-ghrunner-01', + org: theOrg, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 6, 'minutes') + .toDate(), }, ]; - const mockedListRunners = mocked(listRunners).mockResolvedValue([]); - mocked(listGithubRunnersRepo).mockResolvedValue(matchRunner); + const getRunnerPair = (instanceId: string) => { + return { + awsR: listRunnersRet.find((itm) => { + return itm.instanceId === instanceId; + }) as RunnerInfo, + ghR: ghRunners.find((itm) => { + return itm.name === instanceId; + }) as GhRunner, + }; + }; + + beforeEach(() => { + jest.spyOn(Config, 'Instance', 'get').mockImplementation( + () => + ({ + ...baseConfig, + enableOrganizationRunners: true, + scaleConfigRepo: scaleConfigRepo, + minAvailableRunners: 2, + environment: environment, + } as Config), + ); + }); + + it('do according each one', async () => { + const mockedListRunners = mocked(listRunners); + const mockedListGithubRunnersOrg = mocked(listGithubRunnersOrg); + const mockedGetRunnerTypes = mocked(getRunnerTypes); + const mockedRemoveGithubRunnerOrg = mocked(removeGithubRunnerOrg); + const mockedTerminateRunner = mocked(terminateRunner); + + mockedListRunners.mockResolvedValueOnce(listRunnersRet); + mockedListGithubRunnersOrg.mockResolvedValue(ghRunners); + mockedGetRunnerTypes.mockResolvedValue(runnerTypes); + + await scaleDown(); + + expect(mockedListRunners).toBeCalledTimes(1); + expect(mockedListRunners).toBeCalledWith(metrics, { environment: environment }); - await scaleDown(); + expect(mockedListGithubRunnersOrg).toBeCalledTimes(15); + expect(mockedListGithubRunnersOrg).toBeCalledWith(theOrg, metrics); - expect(mockedListRunners).toBeCalledWith(metrics, { environment: environment }); + expect(mockedGetRunnerTypes).toBeCalledTimes(3); + expect(mockedGetRunnerTypes).toBeCalledWith({ owner: theOrg, repo: scaleConfigRepo }, metrics); + + expect(mockedRemoveGithubRunnerOrg).toBeCalledTimes(3); + { + const { awsR, ghR } = getRunnerPair('keep-min-runners-oldest-02'); + expect(mockedRemoveGithubRunnerOrg).toBeCalledWith(awsR, ghR.id, awsR.org as string, metrics); + } + { + const { awsR, ghR } = getRunnerPair('keep-min-runners-oldest-01'); + expect(mockedRemoveGithubRunnerOrg).toBeCalledWith(awsR, ghR.id, awsR.org as string, metrics); + } + { + const { awsR, ghR } = getRunnerPair('remove-ephemeral-01'); + expect(mockedRemoveGithubRunnerOrg).toBeCalledWith(awsR, ghR.id, awsR.org as string, metrics); + } + + expect(mockedTerminateRunner).toBeCalledTimes(6); + { + /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ + const { awsR, ghR } = getRunnerPair('keep-lt-min-no-ghrunner-no-ghr-02'); + expect(mockedTerminateRunner).toBeCalledWith(awsR, metrics); + } + { + /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ + const { awsR, ghR } = getRunnerPair('keep-lt-min-no-ghrunner-no-ghr-01'); + expect(mockedTerminateRunner).toBeCalledWith(awsR, metrics); + } + { + /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ + const { awsR, ghR } = getRunnerPair('keep-min-runners-oldest-02'); + expect(mockedTerminateRunner).toBeCalledWith(awsR, metrics); + } + { + /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ + const { awsR, ghR } = getRunnerPair('keep-min-runners-oldest-01'); + expect(mockedTerminateRunner).toBeCalledWith(awsR, metrics); + } + { + /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ + const { awsR, ghR } = getRunnerPair('remove-ephemeral-02'); + expect(mockedTerminateRunner).toBeCalledWith(awsR, metrics); + } + { + /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ + const { awsR, ghR } = getRunnerPair('remove-ephemeral-01'); + expect(mockedTerminateRunner).toBeCalledWith(awsR, metrics); + } + }); }); - it('runners not live for minimum time', async () => { - mocked(listRunners).mockResolvedValue([ + describe('repo', () => { + const environment = 'environment'; + const theRepo = 'a-owner/a-repo'; + const repo = { owner: 'a-owner', repo: 'a-repo' }; + const dateRef = moment(new Date()); + const runnerTypes = new Map([ + ['ignore-no-org-no-repo', { is_ephemeral: false } as RunnerType], + ['ignore-no-repo', { is_ephemeral: false } as RunnerType], + ['keep-all-4', { is_ephemeral: false } as RunnerType], + ['a-ephemeral-runner', { is_ephemeral: true } as RunnerType], + ['keep-min-runners-oldest', { is_ephemeral: false } as RunnerType], + ['keep-lt-min-no-ghrunner', { is_ephemeral: false } as RunnerType], + ]); + const ghRunners = [ + mockRunner({ id: '0001', name: 'keep-this-not-min-time-01', busy: false }), + mockRunner({ id: '0002', name: 'keep-this-not-min-time-02', busy: false }), + mockRunner({ id: '0003', name: 'keep-this-is-busy-01', busy: true }), + mockRunner({ id: '0004', name: 'keep-this-is-busy-02', busy: true }), + mockRunner({ id: '0005', name: 'keep-this-not-min-time-03', busy: false }), + mockRunner({ id: '0006', name: 'keep-this-is-busy-03', busy: true }), + mockRunner({ id: '0007', name: 'remove-ephemeral-01', busy: false }), + mockRunner({ id: '0008', name: 'keep-min-runners-not-oldest-01', busy: false }), + mockRunner({ id: '0009', name: 'keep-min-runners-oldest-01', busy: false }), + mockRunner({ id: '0010', name: 'keep-min-runners-not-oldest-02', busy: false }), + mockRunner({ id: '0011', name: 'keep-min-runners-oldest-02', busy: false }), + mockRunner({ id: '0012', name: 'keep-lt-min-no-ghrunner-01', busy: false }), + ] as GhRunners; + const listRunnersRet = [ { - instanceId: 'WG113', - repo: 'owner/repo', - launchTime: moment(new Date()).toDate(), + runnerType: 'ignore-no-org-no-repo', + instanceId: '001', + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }, + { + runnerType: 'ignore-no-org-no-repo', + instanceId: '002', + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 3, 'minutes') + .toDate(), + }, + + { + runnerType: 'ignore-no-repo', + instanceId: '003', + org: 'a-owner', + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 3, 'minutes') + .toDate(), + }, + { + runnerType: 'ignore-no-repo', + instanceId: '004', + org: 'a-owner', + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 3, 'minutes') + .toDate(), + }, + + { + runnerType: 'keep-all-4', + instanceId: 'keep-this-not-min-time-01', + repo: theRepo, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes - 5, 'minutes') + .toDate(), + }, + { + runnerType: 'keep-all-4', + instanceId: 'keep-this-not-min-time-02', + repo: theRepo, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes - 3, 'minutes') + .toDate(), + }, + { + runnerType: 'keep-all-4', + instanceId: 'keep-this-is-busy-01', + repo: theRepo, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }, + { + runnerType: 'keep-all-4', + instanceId: 'keep-this-is-busy-02', + repo: theRepo, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }, + + { + runnerType: 'a-ephemeral-runner', + instanceId: 'keep-this-not-min-time-03', + repo: theRepo, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes - 5, 'minutes') + .toDate(), + }, + { + runnerType: 'a-ephemeral-runner', + instanceId: 'keep-this-is-busy-03', + repo: theRepo, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }, + { + runnerType: 'a-ephemeral-runner', + instanceId: 'remove-ephemeral-01', // X + repo: theRepo, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }, + { + runnerType: 'a-ephemeral-runner', + instanceId: 'remove-ephemeral-02', // X + repo: theRepo, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), }, - ]); - const mockedResetRunnersCaches = mocked(resetRunnersCaches); - await scaleDown(); + { + runnerType: 'keep-min-runners-oldest', + instanceId: 'keep-min-runners-not-oldest-01', + repo: theRepo, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }, + { + runnerType: 'keep-min-runners-oldest', + instanceId: 'keep-min-runners-oldest-01', // X + repo: theRepo, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 7, 'minutes') + .toDate(), + }, + { + runnerType: 'keep-min-runners-oldest', + instanceId: 'keep-min-runners-not-oldest-02', + repo: theRepo, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 6, 'minutes') + .toDate(), + }, + { + runnerType: 'keep-min-runners-oldest', + instanceId: 'keep-min-runners-oldest-02', // X + repo: theRepo, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 8, 'minutes') + .toDate(), + }, - expect(mockedResetRunnersCaches).toBeCalledTimes(1); + { + runnerType: 'keep-lt-min-no-ghrunner', + instanceId: 'keep-lt-min-no-ghrunner-no-ghr-01', // X + repo: theRepo, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }, + { + runnerType: 'keep-lt-min-no-ghrunner', + instanceId: 'keep-lt-min-no-ghrunner-no-ghr-02', // X + repo: theRepo, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 7, 'minutes') + .toDate(), + }, + { + runnerType: 'keep-lt-min-no-ghrunner', + instanceId: 'keep-lt-min-no-ghrunner-01', + repo: theRepo, + launchTime: dateRef + .clone() + .subtract(minimumRunningTimeInMinutes + 6, 'minutes') + .toDate(), + }, + ]; + const getRunnerPair = (instanceId: string) => { + return { + awsR: listRunnersRet.find((itm) => { + return itm.instanceId === instanceId; + }) as RunnerInfo, + ghR: ghRunners.find((itm) => { + return itm.name === instanceId; + }) as GhRunner, + }; + }; + + beforeEach(() => { + jest.spyOn(Config, 'Instance', 'get').mockImplementation( + () => + ({ + ...baseConfig, + enableOrganizationRunners: false, + minAvailableRunners: 2, + environment: environment, + } as Config), + ); + }); + + it('do according each one', async () => { + const mockedListRunners = mocked(listRunners); + const mockedListGithubRunnersRepo = mocked(listGithubRunnersRepo); + const mockedGetRunnerTypes = mocked(getRunnerTypes); + const mockedRemoveGithubRunnerRepo = mocked(removeGithubRunnerRepo); + const mockedTerminateRunner = mocked(terminateRunner); + + mockedListRunners.mockResolvedValueOnce(listRunnersRet); + mockedListGithubRunnersRepo.mockResolvedValue(ghRunners); + mockedGetRunnerTypes.mockResolvedValue(runnerTypes); + + await scaleDown(); + + expect(mockedListRunners).toBeCalledTimes(1); + expect(mockedListRunners).toBeCalledWith(metrics, { environment: environment }); + + expect(mockedListGithubRunnersRepo).toBeCalledTimes(15); + expect(mockedListGithubRunnersRepo).toBeCalledWith(repo, metrics); + + expect(mockedGetRunnerTypes).toBeCalledTimes(3); + expect(mockedGetRunnerTypes).toBeCalledWith(repo, metrics); + + expect(mockedRemoveGithubRunnerRepo).toBeCalledTimes(3); + { + const { awsR, ghR } = getRunnerPair('keep-min-runners-oldest-02'); + expect(mockedRemoveGithubRunnerRepo).toBeCalledWith(awsR, ghR.id, repo, metrics); + } + { + const { awsR, ghR } = getRunnerPair('keep-min-runners-oldest-01'); + expect(mockedRemoveGithubRunnerRepo).toBeCalledWith(awsR, ghR.id, repo, metrics); + } + { + const { awsR, ghR } = getRunnerPair('remove-ephemeral-01'); + expect(mockedRemoveGithubRunnerRepo).toBeCalledWith(awsR, ghR.id, repo, metrics); + } + + expect(mockedTerminateRunner).toBeCalledTimes(6); + { + /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ + const { awsR, ghR } = getRunnerPair('keep-lt-min-no-ghrunner-no-ghr-02'); + expect(mockedTerminateRunner).toBeCalledWith(awsR, metrics); + } + { + /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ + const { awsR, ghR } = getRunnerPair('keep-lt-min-no-ghrunner-no-ghr-01'); + expect(mockedTerminateRunner).toBeCalledWith(awsR, metrics); + } + { + /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ + const { awsR, ghR } = getRunnerPair('keep-min-runners-oldest-02'); + expect(mockedTerminateRunner).toBeCalledWith(awsR, metrics); + } + { + /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ + const { awsR, ghR } = getRunnerPair('keep-min-runners-oldest-01'); + expect(mockedTerminateRunner).toBeCalledWith(awsR, metrics); + } + { + /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ + const { awsR, ghR } = getRunnerPair('remove-ephemeral-02'); + expect(mockedTerminateRunner).toBeCalledWith(awsR, metrics); + } + { + /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ + const { awsR, ghR } = getRunnerPair('remove-ephemeral-01'); + expect(mockedTerminateRunner).toBeCalledWith(awsR, metrics); + } + }); }); - describe('tests sorting - no particular goal here', () => { + describe('sortRunnersByLaunchTime', () => { it('two undefined', async () => { - mocked(listRunners).mockResolvedValue([ + const ret = sortRunnersByLaunchTime([ { instanceId: 'WG113', repo: undefined, @@ -121,11 +751,13 @@ describe('scaleDown', () => { }, ]); - await scaleDown(); + expect(ret[0].launchTime).toBeUndefined(); + expect(ret[1].launchTime).toBeUndefined(); }); it('undefined valid', async () => { - mocked(listRunners).mockResolvedValue([ + const dt = moment(new Date()).toDate(); + const ret = sortRunnersByLaunchTime([ { instanceId: 'WG113', repo: undefined, @@ -134,19 +766,21 @@ describe('scaleDown', () => { { instanceId: 'WG113', repo: undefined, - launchTime: moment(new Date()).toDate(), + launchTime: dt, }, ]); - await scaleDown(); + expect(ret[0].launchTime).toEqual(dt); + expect(ret[1].launchTime).toBeUndefined(); }); it('valid undefined', async () => { - mocked(listRunners).mockResolvedValue([ + const dt = moment(new Date()).toDate(); + const ret = sortRunnersByLaunchTime([ { instanceId: 'WG113', repo: undefined, - launchTime: moment(new Date()).toDate(), + launchTime: dt, }, { instanceId: 'WG113', @@ -155,46 +789,53 @@ describe('scaleDown', () => { }, ]); - await scaleDown(); + expect(ret[0].launchTime).toEqual(dt); + expect(ret[1].launchTime).toBeUndefined(); }); it('bigger smaller', async () => { - mocked(listRunners).mockResolvedValue([ + const dt1 = moment(new Date()).add(50, 'seconds').toDate(); + const dt2 = moment(new Date()).subtract(50, 'seconds').toDate(); + const ret = sortRunnersByLaunchTime([ { instanceId: 'WG113', repo: undefined, - launchTime: moment(new Date()).add(50, 'seconds').toDate(), + launchTime: dt1, }, { instanceId: 'WG113', repo: undefined, - launchTime: moment(new Date()).subtract(50, 'seconds').toDate(), + launchTime: dt2, }, ]); - await scaleDown(); + expect(ret[0].launchTime).toEqual(dt2); + expect(ret[1].launchTime).toEqual(dt1); }); it('smaller bigger', async () => { - mocked(listRunners).mockResolvedValue([ + const dt1 = moment(new Date()).add(50, 'seconds').toDate(); + const dt2 = moment(new Date()).subtract(50, 'seconds').toDate(); + const ret = sortRunnersByLaunchTime([ { instanceId: 'WG113', repo: undefined, - launchTime: moment(new Date()).subtract(50, 'seconds').toDate(), + launchTime: dt2, }, { instanceId: 'WG113', repo: undefined, - launchTime: moment(new Date()).add(50, 'seconds').toDate(), + launchTime: dt1, }, ]); - await scaleDown(); + expect(ret[0].launchTime).toEqual(dt2); + expect(ret[1].launchTime).toEqual(dt1); }); it('equal', async () => { const launchTime = moment(new Date()).subtract(50, 'seconds').toDate(); - mocked(listRunners).mockResolvedValue([ + const ret = sortRunnersByLaunchTime([ { instanceId: 'WG113', repo: undefined, @@ -207,231 +848,391 @@ describe('scaleDown', () => { }, ]); - await scaleDown(); + expect(ret[0].launchTime).toEqual(launchTime); + expect(ret[1].launchTime).toEqual(launchTime); }); }); - it('ec2runner with repo = undefined && org = undefined', async () => { - mocked(listRunners).mockResolvedValue([ - { - instanceId: 'WG113', + describe('runnerMinimumTimeExceeded', () => { + it('launchTime === undefined', () => { + const response = runnerMinimumTimeExceeded({ + instanceId: 'AGDGADUWG113', + launchTime: undefined, + }); + expect(response).toEqual(false); + }); + + it('exceeded minimum time', () => { + const response = runnerMinimumTimeExceeded({ + instanceId: 'AGDGADUWG113', launchTime: moment(new Date()) + .utc() .subtract(minimumRunningTimeInMinutes + 5, 'minutes') .toDate(), - }, - ]); - const mockedResetRunnersCaches = mocked(resetRunnersCaches); - const mockedListGithubRunners = mocked(listGithubRunnersRepo); + }); + expect(response).toEqual(true); + }); + + it('dont exceeded minimum time', () => { + const response = runnerMinimumTimeExceeded({ + instanceId: 'AGDGADUWG113', + launchTime: moment(new Date()) + .utc() + .subtract(minimumRunningTimeInMinutes - 5, 'minutes') + .toDate(), + }); + expect(response).toEqual(false); + }); + }); + + describe('isRunnerRemovable', () => { + describe('ghRunner === undefined', () => { + it('launchTime === undefined', () => { + const response = isRunnerRemovable(undefined, { + instanceId: 'AGDGADUWG113', + launchTime: undefined, + }); + expect(response).toEqual(false); + }); + + it('exceeded minimum time', () => { + const response = isRunnerRemovable(undefined, { + instanceId: 'AGDGADUWG113', + launchTime: moment(new Date()) + .utc() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }); + expect(response).toEqual(true); + }); + + it('dont exceeded minimum time', () => { + const response = isRunnerRemovable(undefined, { + instanceId: 'AGDGADUWG113', + launchTime: moment(new Date()) + .utc() + .subtract(minimumRunningTimeInMinutes - 5, 'minutes') + .toDate(), + }); + expect(response).toEqual(false); + }); + }); - await scaleDown(); + describe('ghRunner !== undefined', () => { + it('ghRunner.busy == true', () => { + const response = isRunnerRemovable( + { + busy: true, + } as GhRunner, + { + instanceId: 'AGDGADUWG113', + launchTime: undefined, + }, + ); + expect(response).toEqual(false); + }); + + it('ghRunner.busy == false, launchTime === undefined', () => { + const response = isRunnerRemovable( + { + busy: false, + } as GhRunner, + { + instanceId: 'AGDGADUWG113', + launchTime: undefined, + }, + ); + expect(response).toEqual(false); + }); - expect(mockedResetRunnersCaches).toBeCalledTimes(1); - expect(mockedListGithubRunners).not.toBeCalled(); + it('ghRunner.busy == false, launchTime exceeds', () => { + const response = isRunnerRemovable( + { + busy: false, + } as GhRunner, + { + instanceId: 'AGDGADUWG113', + launchTime: moment(new Date()) + .utc() + .subtract(minimumRunningTimeInMinutes + 5, 'minutes') + .toDate(), + }, + ); + expect(response).toEqual(true); + }); + + it('ghRunner.busy == false, launchTime dont exceeds', () => { + const response = isRunnerRemovable( + { + busy: false, + } as GhRunner, + { + instanceId: 'AGDGADUWG113', + launchTime: moment(new Date()) + .utc() + .subtract(minimumRunningTimeInMinutes - 5, 'minutes') + .toDate(), + }, + ); + expect(response).toEqual(false); + }); + }); }); - describe('RunnerInfo.repo !== undefined', () => { - it('listGithubRunnersRepo returns [], getRunnerRepo ret undefined and terminateRunner with success', async () => { - const ec2runner = { - instanceId: 'WG113', - repo: 'owner/repo', - launchTime: moment(new Date()) - .subtract(minimumRunningTimeInMinutes + 5, 'minutes') - .toDate(), - ghRunnerId: '33', - }; - const repo = { owner: 'owner', repo: 'repo' }; - mocked(listRunners).mockResolvedValue([ec2runner]); - const mockedListGithubRunners = mocked(listGithubRunnersRepo).mockResolvedValue([]); - const mockedGetRunner = mocked(getRunnerRepo).mockResolvedValue(undefined); - const mockedTerminateRunner = mocked(terminateRunner); + describe('isEphemeralRunner', () => { + it('ec2runner.runnerType === undefined', async () => { + expect(isEphemeralRunner({ runnerType: undefined } as RunnerInfo, metrics)).resolves.toEqual(false); + }); - await scaleDown(); + describe('org runners', () => { + const scaleConfigRepo = 'test-infra'; + const runnerType = 'runnerTypeDef'; + + beforeEach(() => { + jest.spyOn(Config, 'Instance', 'get').mockImplementation( + () => + ({ + ...baseConfig, + enableOrganizationRunners: true, + scaleConfigRepo: scaleConfigRepo, + } as Config), + ); + }); - expect(mockedListGithubRunners).toBeCalledWith(repo, metrics); - expect(mockedGetRunner).toBeCalledWith(repo, ec2runner.ghRunnerId, metrics); - expect(mockedTerminateRunner).toBeCalledWith(ec2runner, metrics); + it('org in runner, is_ephemeral === undefined', async () => { + const owner = 'the-org'; + const mockedGetRunnerTypes = mocked(getRunnerTypes); + + mockedGetRunnerTypes.mockResolvedValueOnce(new Map([[runnerType, {} as RunnerType]])); + + expect(await isEphemeralRunner({ runnerType: runnerType, org: owner } as RunnerInfo, metrics)).toEqual(false); + + expect(mockedGetRunnerTypes).toBeCalledTimes(1); + expect(mockedGetRunnerTypes).toBeCalledWith({ owner: owner, repo: scaleConfigRepo }, metrics); + }); + + it('org in runner, is_ephemeral === false', async () => { + const owner = 'the-org'; + const mockedGetRunnerTypes = mocked(getRunnerTypes); + + mockedGetRunnerTypes.mockResolvedValueOnce(new Map([[runnerType, { is_ephemeral: false } as RunnerType]])); + + expect(await isEphemeralRunner({ runnerType: runnerType, org: owner } as RunnerInfo, metrics)).toEqual(false); + + expect(mockedGetRunnerTypes).toBeCalledTimes(1); + expect(mockedGetRunnerTypes).toBeCalledWith({ owner: owner, repo: scaleConfigRepo }, metrics); + }); + + it('org not in runner, is_ephemeral === true', async () => { + const owner = 'the-org'; + const mockedGetRunnerTypes = mocked(getRunnerTypes); + + mockedGetRunnerTypes.mockResolvedValueOnce(new Map([[runnerType, { is_ephemeral: true } as RunnerType]])); + + expect( + await isEphemeralRunner({ runnerType: runnerType, repo: `${owner}/a-repo` } as RunnerInfo, metrics), + ).toEqual(true); + + expect(mockedGetRunnerTypes).toBeCalledTimes(1); + expect(mockedGetRunnerTypes).toBeCalledWith({ owner: owner, repo: scaleConfigRepo }, metrics); + }); }); - it('listGithubRunnersRepo returns [], getRunnerRepo returns undefined and terminateRunner raises', async () => { - const ec2runner = { - instanceId: 'WG113', - repo: 'owner/repo', - launchTime: moment(new Date()) - .subtract(minimumRunningTimeInMinutes + 5, 'minutes') - .toDate(), - ghRunnerId: '33', + describe('repo runners', () => { + const runnerType = 'runnerTypeDef'; + const owner = 'the-org'; + const repo: Repo = { + owner: owner, + repo: 'a-repo', }; - mocked(listRunners).mockResolvedValue([ec2runner]); - mocked(listGithubRunnersRepo).mockResolvedValue([]); - mocked(getRunnerRepo).mockResolvedValue(undefined); - const mockedTerminateRunner = mocked(terminateRunner).mockImplementation(async () => { - throw Error('error on terminateRunner'); + const repoKey = `${owner}/a-repo`; + + beforeEach(() => { + jest.spyOn(Config, 'Instance', 'get').mockImplementation( + () => + ({ + ...baseConfig, + enableOrganizationRunners: false, + } as Config), + ); }); - await scaleDown(); + it('is_ephemeral === undefined', async () => { + const mockedGetRunnerTypes = mocked(getRunnerTypes); + + mockedGetRunnerTypes.mockResolvedValueOnce(new Map([[runnerType, {} as RunnerType]])); + + expect(await isEphemeralRunner({ runnerType: runnerType, repo: repoKey } as RunnerInfo, metrics)).toEqual( + false, + ); + + expect(mockedGetRunnerTypes).toBeCalledTimes(1); + expect(mockedGetRunnerTypes).toBeCalledWith(repo, metrics); + }); - expect(mockedTerminateRunner).toBeCalled(); + it('is_ephemeral === true', async () => { + const mockedGetRunnerTypes = mocked(getRunnerTypes); + + mockedGetRunnerTypes.mockResolvedValueOnce(new Map([[runnerType, { is_ephemeral: true } as RunnerType]])); + + expect(await isEphemeralRunner({ runnerType: runnerType, repo: repoKey } as RunnerInfo, metrics)).toEqual(true); + + expect(mockedGetRunnerTypes).toBeCalledTimes(1); + expect(mockedGetRunnerTypes).toBeCalledWith(repo, metrics); + }); + + it('is_ephemeral === false', async () => { + const mockedGetRunnerTypes = mocked(getRunnerTypes); + + mockedGetRunnerTypes.mockResolvedValueOnce(new Map([[runnerType, { is_ephemeral: false } as RunnerType]])); + + expect(await isEphemeralRunner({ runnerType: runnerType, repo: repoKey } as RunnerInfo, metrics)).toEqual( + false, + ); + + expect(mockedGetRunnerTypes).toBeCalledTimes(1); + expect(mockedGetRunnerTypes).toBeCalledWith(repo, metrics); + }); }); + }); - it('listGithubRunnersRepo returns [(matches, nonbusy)], removeGithubRunnerRepo is called', async () => { - const ec2runner = { - instanceId: 'WG113', - repo: 'owner/repo', - launchTime: moment(new Date()) - .subtract(minimumRunningTimeInMinutes + 5, 'minutes') - .toDate(), - ghRunnerId: '33', + describe('getGHRunnerRepo', () => { + const ghRunners = [ + { name: 'instance-id-01', busy: true }, + { name: 'instance-id-02', busy: false }, + ] as GhRunners; + const repo: Repo = { + owner: 'the-org', + repo: 'a-repo', + }; + const repoKey = `the-org/a-repo`; + + it('finds on listGithubRunnersRepo, busy === true', async () => { + const mockedListGithubRunnersRepo = mocked(listGithubRunnersRepo); + const ec2runner: RunnerInfo = { + repo: repoKey, + instanceId: 'instance-id-01', + runnerType: 'runnerType-01', + ghRunnerId: 'ghRunnerId-01', }; - const matchRunner: GhRunners = [ - { - id: 1, - name: ec2runner.instanceId, - os: 'linux', - status: 'busy', - busy: false, - labels: [], - }, - ]; - const repo = { owner: 'owner', repo: 'repo' }; - mocked(listRunners).mockResolvedValue([ec2runner]); - mocked(listGithubRunnersRepo).mockResolvedValue(matchRunner); - const mockedGetRunner = mocked(getRunnerRepo).mockResolvedValue(undefined); - const mockedRemoveGithubRunner = mocked(removeGithubRunnerRepo); - await scaleDown(); + mockedListGithubRunnersRepo.mockResolvedValueOnce(ghRunners); + + expect(await getGHRunnerRepo(ec2runner, metrics)).toEqual(ghRunners[0]); - expect(mockedGetRunner).not.toBeCalled(); - expect(mockedRemoveGithubRunner).toBeCalledWith(ec2runner, matchRunner[0].id, repo, metrics); + expect(mockedListGithubRunnersRepo).toBeCalledTimes(1); + expect(mockedListGithubRunnersRepo).toBeCalledWith(repo, metrics); }); - it('listGithubRunnersRepo returns [(matches, nonbusy)], removeGithubRunnerRepo is NOT called', async () => { - const ec2runner = { - instanceId: 'WG113', - repo: 'owner/repo', - launchTime: moment(new Date()) - .subtract(minimumRunningTimeInMinutes + 5, 'minutes') - .toDate(), - ghRunnerId: '33', + it('dont finds on listGithubRunnersRep, finds with getRunnerRepo, busy === false', async () => { + const mockedListGithubRunnersRepo = mocked(listGithubRunnersRepo); + const mockedGetRunnerRepo = mocked(getRunnerRepo); + const ec2runner: RunnerInfo = { + repo: repoKey, + instanceId: 'instance-id-03', + runnerType: 'runnerType-01', + ghRunnerId: 'ghRunnerId-01', }; - const matchRunner: GhRunners = [ - { - id: 1, - name: ec2runner.instanceId, - os: 'linux', - status: 'busy', - busy: true, - labels: [], - }, - ]; - mocked(listRunners).mockResolvedValue([ec2runner]); - mocked(listGithubRunnersRepo).mockResolvedValue(matchRunner); - const mockedRemoveGithubRunner = mocked(removeGithubRunnerRepo); + const theGhRunner = { name: 'instance-id-03', busy: false } as GhRunner; - await scaleDown(); + mockedListGithubRunnersRepo.mockResolvedValueOnce(ghRunners); + mockedGetRunnerRepo.mockResolvedValueOnce(theGhRunner); + + expect(await getGHRunnerRepo(ec2runner, metrics)).toEqual(theGhRunner); - expect(mockedRemoveGithubRunner).not.toBeCalled(); + expect(mockedListGithubRunnersRepo).toBeCalledTimes(1); + expect(mockedListGithubRunnersRepo).toBeCalledWith(repo, metrics); + expect(mockedGetRunnerRepo).toBeCalledTimes(1); + expect(mockedGetRunnerRepo).toBeCalledWith(repo, ec2runner.ghRunnerId, metrics); }); - }); - describe('RunnerInfo.org !== undefined', () => { - it('listGithubRunnersOrg returns [], getRunnerOrg ret undefined and terminateRunner with success', async () => { - const ec2runner = { - instanceId: 'WG113', - org: 'owner', - launchTime: moment(new Date()) - .subtract(minimumRunningTimeInMinutes + 5, 'minutes') - .toDate(), - ghRunnerId: '33', + it('listGithubRunnersRep and getRunnerRepo throws exception', async () => { + const mockedListGithubRunnersRepo = mocked(listGithubRunnersRepo); + const mockedGetRunnerRepo = mocked(getRunnerRepo); + const ec2runner: RunnerInfo = { + repo: repoKey, + instanceId: 'instance-id-03', + runnerType: 'runnerType-01', + ghRunnerId: 'ghRunnerId-01', }; - mocked(listRunners).mockResolvedValue([ec2runner]); - const mockedListGithubRunners = mocked(listGithubRunnersOrg).mockResolvedValue([]); - const mockedGetRunner = mocked(getRunnerOrg).mockResolvedValue(undefined); - const mockedTerminateRunner = mocked(terminateRunner); - await scaleDown(); + mockedListGithubRunnersRepo.mockRejectedValueOnce('Error'); + mockedGetRunnerRepo.mockRejectedValueOnce('Error'); + + expect(await getGHRunnerRepo(ec2runner, metrics)).toBeUndefined(); - expect(mockedListGithubRunners).toBeCalledWith('owner', metrics); - expect(mockedGetRunner).toBeCalledWith('owner', ec2runner.ghRunnerId, metrics); - expect(mockedTerminateRunner).toBeCalledWith(ec2runner, metrics); + expect(mockedListGithubRunnersRepo).toBeCalledTimes(1); + expect(mockedListGithubRunnersRepo).toBeCalledWith(repo, metrics); + expect(mockedGetRunnerRepo).toBeCalledTimes(1); + expect(mockedGetRunnerRepo).toBeCalledWith(repo, ec2runner.ghRunnerId, metrics); }); + }); - it('listGithubRunnersOrg returns [], getRunnerOrg returns undefined and terminateRunner raises', async () => { - const ec2runner = { - instanceId: 'WG113', - org: 'owner', - launchTime: moment(new Date()) - .subtract(minimumRunningTimeInMinutes + 5, 'minutes') - .toDate(), - ghRunnerId: '33', + describe('getGHRunnerOrg', () => { + const ghRunners = [ + { name: 'instance-id-01', busy: true }, + { name: 'instance-id-02', busy: false }, + ] as GhRunners; + const org = 'the-org'; + + it('finds on listGithubRunnersOrg, busy === true', async () => { + const mockedListGithubRunnersOrg = mocked(listGithubRunnersOrg); + const ec2runner: RunnerInfo = { + org: org, + instanceId: 'instance-id-01', + runnerType: 'runnerType-01', + ghRunnerId: 'ghRunnerId-01', }; - mocked(listRunners).mockResolvedValue([ec2runner]); - mocked(listGithubRunnersOrg).mockResolvedValue([]); - mocked(getRunnerOrg).mockResolvedValue(undefined); - const mockedTerminateRunner = mocked(terminateRunner).mockImplementation(async () => { - throw Error('error on terminateRunner'); - }); - await scaleDown(); + mockedListGithubRunnersOrg.mockResolvedValueOnce(ghRunners); + + expect(await getGHRunnerOrg(ec2runner, metrics)).toEqual(ghRunners[0]); - expect(mockedTerminateRunner).toBeCalled(); + expect(mockedListGithubRunnersOrg).toBeCalledTimes(1); + expect(mockedListGithubRunnersOrg).toBeCalledWith(org, metrics); }); - it('listGithubRunnersOrg returns [(matches, nonbusy)], removeGithubRunnerOrg is called', async () => { - const ec2runner = { - instanceId: 'WG113', - org: 'owner', - launchTime: moment(new Date()) - .subtract(minimumRunningTimeInMinutes + 5, 'minutes') - .toDate(), - ghRunnerId: '33', + it('dont finds on listGithubRunnersOrg, finds with getRunnerOrg, busy === false', async () => { + const mockedListGithubRunnersOrg = mocked(listGithubRunnersOrg); + const mockedGetRunnerOrg = mocked(getRunnerOrg); + const ec2runner: RunnerInfo = { + org: org, + instanceId: 'instance-id-03', + runnerType: 'runnerType-01', + ghRunnerId: 'ghRunnerId-01', }; - const matchRunner: GhRunners = [ - { - id: 1, - name: ec2runner.instanceId, - os: 'linux', - status: 'busy', - busy: false, - labels: [], - }, - ]; - mocked(listRunners).mockResolvedValue([ec2runner]); - mocked(listGithubRunnersOrg).mockResolvedValue(matchRunner); - const mockedGetRunner = mocked(getRunnerOrg).mockResolvedValue(undefined); - const mockedRemoveGithubRunner = mocked(removeGithubRunnerOrg); + const theGhRunner = { name: 'instance-id-03', busy: false } as GhRunner; - await scaleDown(); + mockedListGithubRunnersOrg.mockResolvedValueOnce(ghRunners); + mockedGetRunnerOrg.mockResolvedValueOnce(theGhRunner); - expect(mockedGetRunner).not.toBeCalled(); - expect(mockedRemoveGithubRunner).toBeCalledWith(ec2runner, matchRunner[0].id, 'owner', metrics); + expect(await getGHRunnerOrg(ec2runner, metrics)).toEqual(theGhRunner); + + expect(mockedListGithubRunnersOrg).toBeCalledTimes(1); + expect(mockedListGithubRunnersOrg).toBeCalledWith(org, metrics); + expect(mockedGetRunnerOrg).toBeCalledTimes(1); + expect(mockedGetRunnerOrg).toBeCalledWith(org, ec2runner.ghRunnerId, metrics); }); - it('listGithubRunnersOrg returns [(matches, nonbusy)], removeGithubRunnerOrg is NOT called', async () => { - const ec2runner = { - instanceId: 'WG113', - org: 'owner', - launchTime: moment(new Date()) - .subtract(minimumRunningTimeInMinutes + 5, 'minutes') - .toDate(), - ghRunnerId: '33', + it('listGithubRunnersRep and getRunnerRepo throws exception', async () => { + const mockedListGithubRunnersOrg = mocked(listGithubRunnersOrg); + const mockedGetRunnerOrg = mocked(getRunnerOrg); + const ec2runner: RunnerInfo = { + org: org, + instanceId: 'instance-id-03', + runnerType: 'runnerType-01', + ghRunnerId: 'ghRunnerId-01', }; - const matchRunner: GhRunners = [ - { - id: 1, - name: ec2runner.instanceId, - os: 'linux', - status: 'busy', - busy: true, - labels: [], - }, - ]; - mocked(listRunners).mockResolvedValue([ec2runner]); - mocked(listGithubRunnersOrg).mockResolvedValue(matchRunner); - const mockedRemoveGithubRunner = mocked(removeGithubRunnerOrg); - await scaleDown(); + mockedListGithubRunnersOrg.mockRejectedValueOnce('Error'); + mockedGetRunnerOrg.mockRejectedValueOnce('Error'); + + expect(await getGHRunnerOrg(ec2runner, metrics)).toBeUndefined(); - expect(mockedRemoveGithubRunner).not.toBeCalled(); + expect(mockedListGithubRunnersOrg).toBeCalledTimes(1); + expect(mockedListGithubRunnersOrg).toBeCalledWith(org, metrics); + expect(mockedGetRunnerOrg).toBeCalledTimes(1); + expect(mockedGetRunnerOrg).toBeCalledWith(org, ec2runner.ghRunnerId, metrics); }); }); }); diff --git a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts index 9c21e3b4f0..b7cd950b2c 100644 --- a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts +++ b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts @@ -1,31 +1,22 @@ -import { listRunners, resetRunnersCaches, terminateRunner } from './runners'; +import moment from 'moment'; +import { Config } from './config'; +import { resetSecretCache } from './gh-auth'; import { - GhRunner, getRunnerOrg, getRunnerRepo, + getRunnerTypes, + GhRunner, listGithubRunnersOrg, listGithubRunnersRepo, removeGithubRunnerOrg, removeGithubRunnerRepo, resetGHRunnersCaches, } from './gh-runners'; -import { RunnerInfo, getRepo } from './utils'; - -import { Config } from './config'; -import moment from 'moment'; -import { resetSecretCache } from './gh-auth'; -import { ScaleDownMetrics, sendMetricsTimeoutVars, sendMetricsAtTimeout } from './metrics'; - -function runnerMinimumTimeExceeded(runner: RunnerInfo): boolean { - const launchTimePlusMinimum = moment(runner.launchTime) - .utc() - .add(Config.Instance.minimumRunningTimeInMinutes, 'minutes'); - const now = moment(new Date()).utc(); - return launchTimePlusMinimum < now; -} +import { ScaleDownMetrics, sendMetricsAtTimeout, sendMetricsTimeoutVars } from './metrics'; +import { listRunners, resetRunnersCaches, terminateRunner } from './runners'; +import { getRepo, groupBy, Repo, RunnerInfo } from './utils'; -export default async function scaleDown(): Promise { - // list and sort runners, newest first. This ensure we keep the newest runners longer. +export async function scaleDown(): Promise { const metrics = new ScaleDownMetrics(); const sndMetricsTimout: sendMetricsTimeoutVars = { metrics: metrics, @@ -41,59 +32,86 @@ export default async function scaleDown(): Promise { resetGHRunnersCaches(); resetSecretCache(); - const runners = ( - await listRunners(metrics, { - environment: Config.Instance.environment, - }) - ).sort((a, b): number => { - if (a.launchTime === undefined && b.launchTime === undefined) return 0; - if (a.launchTime === undefined) return 1; - if (b.launchTime === undefined) return 1; - if (a.launchTime < b.launchTime) return 1; - if (a.launchTime > b.launchTime) return -1; - return 0; - }); - - if (runners.length === 0) { + metrics.run(); + + const runnersDict = groupBy( + sortRunnersByLaunchTime(await listRunners(metrics, { environment: Config.Instance.environment })), + (itm) => { + if (Config.Instance.enableOrganizationRunners) return itm.runnerType; + return `${itm.runnerType}#${itm.repo}`; + }, + ); + + if (runnersDict.size === 0) { console.debug(`No active runners found for environment: '${Config.Instance.environment}'`); return; } - metrics.run(); - - for await (const ec2runner of runners) { - metrics.runnerFound(ec2runner); + for (const [runnerType, runners] of runnersDict.entries()) { + if (runners.length < 1 || runners[0].runnerType === undefined || runnerType === undefined) continue; - let nonOrphan = false; - if (ec2runner.repo !== undefined) { - nonOrphan = nonOrphan || (await checkNeedRemoveRunnerRepo(ec2runner, metrics)); - } - if (ec2runner.org !== undefined) { - nonOrphan = nonOrphan || (await checkNeedRemoveRunnerOrg(ec2runner, metrics)); + const ghRunnersRemovable: Array<[RunnerInfo, GhRunner | undefined]> = []; + for (const ec2runner of runners) { + // REPO assigned runners + if (ec2runner.repo !== undefined) { + const ghRunner = await getGHRunnerRepo(ec2runner, metrics); + // if configured to repo, don't mess with organization runners + if (!Config.Instance.enableOrganizationRunners && isRunnerRemovable(ghRunner, ec2runner)) { + if (ghRunner === undefined) { + ghRunnersRemovable.unshift([ec2runner, ghRunner]); + } else { + ghRunnersRemovable.push([ec2runner, ghRunner]); + } + } + // ORG assigned runners + } else if (ec2runner.org !== undefined) { + const ghRunner = await getGHRunnerOrg(ec2runner, metrics); + // if configured to org, don't mess with repo runners + if (Config.Instance.enableOrganizationRunners && isRunnerRemovable(ghRunner, ec2runner)) { + if (ghRunner === undefined) { + ghRunnersRemovable.unshift([ec2runner, ghRunner]); + } else { + ghRunnersRemovable.push([ec2runner, ghRunner]); + } + } + } } - // we only check if minimum time exceeded after other stuff even if the checks are - // not relevant to generate metrics - if (!runnerMinimumTimeExceeded(ec2runner)) { - metrics.runnerLessMinimumTime(ec2runner); - console.debug( - `Runner '${ec2runner.instanceId}' [${ec2runner.runnerType}] has not been alive long enough, skipping`, - ); - continue; - } + let removedRunners = 0; + for (const [ec2runner, ghRunner] of ghRunnersRemovable) { + // We only limit the number of removed instances here for the reason: while sorting and getting info + // on getRunner[Org|Repo] we send statistics that are relevant for monitoring + if ( + ghRunnersRemovable.length - removedRunners <= Config.Instance.minAvailableRunners && + ghRunner !== undefined && + !(await isEphemeralRunner(ec2runner, metrics)) + ) { + break; + } + + removedRunners += 1; + + if (ghRunner !== undefined) { + if (Config.Instance.enableOrganizationRunners) { + await removeGithubRunnerOrg(ec2runner, ghRunner.id, ec2runner.org as string, metrics); + } else { + await removeGithubRunnerRepo(ec2runner, ghRunner.id, getRepo(ec2runner.repo as string), metrics); + } + } - if (!nonOrphan) { - // Remove orphan AWS runners. - console.info(`Runner '${ec2runner.instanceId}' [${ec2runner.runnerType}] is orphaned, and will be removed.`); + console.info(`Runner '${ec2runner.instanceId}' [${ec2runner.runnerType}] will be removed.`); try { await terminateRunner(ec2runner, metrics); metrics.runnerTerminateSuccess(ec2runner); } catch (e) { metrics.runnerTerminateFailure(ec2runner); - console.error(`Orphan runner '${ec2runner.instanceId}' [${ec2runner.runnerType}] cannot be removed: ${e}`); + console.error(`Runner '${ec2runner.instanceId}' [${ec2runner.runnerType}] cannot be removed: ${e}`); } } } + } catch (e) { + metrics.exception(); + throw e; } finally { clearTimeout(sndMetricsTimout.setTimeout); sndMetricsTimout.metrics = undefined; @@ -102,71 +120,123 @@ export default async function scaleDown(): Promise { } } -async function checkNeedRemoveRunnerRepo(ec2runner: RunnerInfo, metrics: ScaleDownMetrics): Promise { - const repo = getRepo(ec2runner.repo as string); - const ghRunners = await listGithubRunnersRepo(repo, metrics); - let ghRunner: GhRunner | undefined = ghRunners.find((runner) => runner.name === ec2runner.instanceId); +export async function getGHRunnerOrg(ec2runner: RunnerInfo, metrics: ScaleDownMetrics): Promise { + const org = ec2runner.org as string; + let ghRunner: GhRunner | undefined = undefined; + + try { + const ghRunners = await listGithubRunnersOrg(org as string, metrics); + ghRunner = ghRunners.find((runner) => runner.name === ec2runner.instanceId); + } catch (e) { + console.warn('Failed to list active gh runners', e); + } + if (ghRunner === undefined && ec2runner.ghRunnerId !== undefined) { console.warn( - `Runner '${ec2runner.instanceId}' [${ec2runner.runnerType}](${repo}) not found in ` + - `listGithubRunnersRepo call, attempting to grab directly`, + `Runner '${ec2runner.instanceId}' [${ec2runner.runnerType}](${org}) not found in ` + + `listGithubRunnersOrg call, attempting to grab directly`, ); try { - ghRunner = await getRunnerRepo(repo, ec2runner.ghRunnerId, metrics); + ghRunner = await getRunnerOrg(ec2runner.org as string, ec2runner.ghRunnerId, metrics); } catch (e) { console.warn( - `Runner '${ec2runner.instanceId}' [${ec2runner.runnerType}](${repo}) error when ` + `getRunnerRepo call: ${e}`, + `Runner '${ec2runner.instanceId}' [${ec2runner.runnerType}](${org}) error when ` + + `listGithubRunnersOrg call: ${e}`, ); - return false; } } - // ec2Runner matches a runner that's registered to github if (ghRunner) { if (ghRunner.busy) { - metrics.runnerGhFoundBusyRepo(repo, ec2runner); - console.debug(`Runner '${ec2runner.instanceId}' [${ec2runner.runnerType}](${repo}) is busy, skipping`); + metrics.runnerGhFoundBusyOrg(org, ec2runner); } else { - metrics.runnerGhFoundNonBusyRepo(repo, ec2runner); - await removeGithubRunnerRepo(ec2runner, ghRunner.id, repo, metrics); + metrics.runnerGhFoundNonBusyOrg(org, ec2runner); } - return true; } else { - metrics.runnerGhNotFoundRepo(repo, ec2runner); - return false; + metrics.runnerGhNotFoundOrg(org, ec2runner); } + return ghRunner; } -async function checkNeedRemoveRunnerOrg(ec2runner: RunnerInfo, metrics: ScaleDownMetrics): Promise { - const org = ec2runner.org as string; - const ghRunners = await listGithubRunnersOrg(org as string, metrics); - let ghRunner: GhRunner | undefined = ghRunners.find((runner) => runner.name === ec2runner.instanceId); +export async function getGHRunnerRepo(ec2runner: RunnerInfo, metrics: ScaleDownMetrics): Promise { + const repo = getRepo(ec2runner.repo as string); + let ghRunner: GhRunner | undefined = undefined; + + try { + const ghRunners = await listGithubRunnersRepo(repo, metrics); + ghRunner = ghRunners.find((runner) => runner.name === ec2runner.instanceId); + } catch (e) { + console.warn('Failed to list active gh runners', e); + } + if (ghRunner === undefined && ec2runner.ghRunnerId !== undefined) { console.warn( - `Runner '${ec2runner.instanceId}' [${ec2runner.runnerType}](${org}) not found in ` + - `listGithubRunnersOrg call, attempting to grab directly`, + `Runner '${ec2runner.instanceId}' [${ec2runner.runnerType}](${repo}) not found in ` + + `listGithubRunnersRepo call, attempting to grab directly`, ); try { - ghRunner = await getRunnerOrg(ec2runner.org as string, ec2runner.ghRunnerId, metrics); + ghRunner = await getRunnerRepo(repo, ec2runner.ghRunnerId, metrics); } catch (e) { console.warn( - `Runner '${ec2runner.instanceId}' [${ec2runner.runnerType}](${org}) error when ` + - `listGithubRunnersOrg call: ${e}`, + `Runner '${ec2runner.instanceId}' [${ec2runner.runnerType}](${repo}) error when getRunnerRepo call: ${e}`, ); - return false; } } - // ec2Runner matches a runner that's registered to github - if (ghRunner) { + if (ghRunner !== undefined) { if (ghRunner.busy) { - metrics.runnerGhFoundBusyOrg(org, ec2runner); - console.debug(`Runner '${ec2runner.instanceId}' [${ec2runner.runnerType}](${org}) is busy, skipping`); + metrics.runnerGhFoundBusyRepo(repo, ec2runner); } else { - metrics.runnerGhFoundNonBusyOrg(org, ec2runner); - await removeGithubRunnerOrg(ec2runner, ghRunner.id, org, metrics); + metrics.runnerGhFoundNonBusyRepo(repo, ec2runner); } - return true; } else { - metrics.runnerGhNotFoundOrg(org, ec2runner); + metrics.runnerGhNotFoundRepo(repo, ec2runner); + } + return ghRunner; +} + +export async function isEphemeralRunner(ec2runner: RunnerInfo, metrics: ScaleDownMetrics): Promise { + if (ec2runner.runnerType === undefined) { return false; } + + const repo: Repo = (() => { + if (Config.Instance.enableOrganizationRunners) { + return { + owner: ec2runner.org !== undefined ? (ec2runner.org as string) : getRepo(ec2runner.repo as string).owner, + repo: Config.Instance.scaleConfigRepo, + }; + } + return getRepo(ec2runner.repo as string); + })(); + + const runnerTypes = await getRunnerTypes(repo, metrics); + + return runnerTypes.get(ec2runner.runnerType)?.is_ephemeral ?? false; +} + +export function isRunnerRemovable(ghRunner: GhRunner | undefined, ec2runner: RunnerInfo): boolean { + if (ghRunner !== undefined && ghRunner.busy) { + return false; + } + return runnerMinimumTimeExceeded(ec2runner); +} + +export function runnerMinimumTimeExceeded(runner: RunnerInfo): boolean { + if (runner.launchTime === undefined) { + // runner did not start yet, so it does not timeout + return false; + } + const launchTime = moment(runner.launchTime).utc(); + const maxTime = moment(new Date()).subtract(Config.Instance.minimumRunningTimeInMinutes, 'minutes').utc(); + return launchTime < maxTime; +} + +export function sortRunnersByLaunchTime(runners: RunnerInfo[]): RunnerInfo[] { + return runners.sort((a, b): number => { + if (a.launchTime === undefined && b.launchTime === undefined) return 0; + if (a.launchTime === undefined) return 1; + if (b.launchTime === undefined) return -1; + if (a.launchTime < b.launchTime) return -1; + if (a.launchTime > b.launchTime) return 1; + return 0; + }); } diff --git a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/utils.test.ts b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/utils.test.ts index cbfb457a1f..fa4b57334e 100644 --- a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/utils.test.ts +++ b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/utils.test.ts @@ -1,4 +1,4 @@ -import { getBoolean, getRepoKey, expBackOff, getRepo } from './utils'; +import { getBoolean, getRepoKey, expBackOff, getRepo, groupBy } from './utils'; import nock from 'nock'; beforeEach(() => { @@ -25,6 +25,19 @@ describe('./utils', () => { }); }); + describe('groupBy', () => { + it('just check grouping', () => { + const grouped = groupBy(['asdf', 'qwer', 'as', 'zxcv', 'fg', '123'], (str) => { + return str.length; + }); + + expect(grouped.size).toEqual(3); + expect(grouped.get(4)).toEqual(['asdf', 'qwer', 'zxcv']); + expect(grouped.get(3)).toEqual(['123']); + expect(grouped.get(2)).toEqual(['as', 'fg']); + }); + }); + describe('getBoolean', () => { it('check true values', () => { expect(getBoolean(true)).toBeTruthy(); diff --git a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/utils.ts b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/utils.ts index 33236e2457..3db8e3bc09 100644 --- a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/utils.ts +++ b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/utils.ts @@ -89,3 +89,17 @@ export function getRepo(repoDef: string, repoName?: string): Repo { throw e; } } + +export function groupBy(lst: T[], keyGetter: (itm: T) => V): Map> { + const map = new Map>(); + for (const itm of lst) { + const key = keyGetter(itm); + const collection = map.get(key); + if (collection !== undefined) { + collection.push(itm); + } else { + map.set(key, [itm]); + } + } + return map; +}