Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
b7c235c
run smoketests on mac
lerouxb Dec 20, 2024
2f29075
promisified async spawn helper rather
lerouxb Dec 23, 2024
b0840a8
try and preserve permissions
lerouxb Dec 23, 2024
58fdaf5
print args
lerouxb Dec 23, 2024
1222cca
print logs
lerouxb Dec 23, 2024
258cdfa
nevermind
lerouxb Dec 23, 2024
71f9174
trying random things
lerouxb Dec 23, 2024
7b8acdb
quotes
lerouxb Dec 23, 2024
ef9b20a
recursuvely list contents after copying
lerouxb Dec 24, 2024
53d549b
more fixes and debugging
lerouxb Dec 24, 2024
a0934c6
don't fail if the dir exists
lerouxb Dec 24, 2024
1c77502
the app might already exist
lerouxb Dec 24, 2024
27dc5c6
screencapture is not working
lerouxb Dec 24, 2024
c85a44e
unused imports
lerouxb Dec 24, 2024
f7f8056
Merge branch 'main' into test-dmg
lerouxb Dec 24, 2024
4648c16
Merge branch 'test-dmg' of https://github.com/mongodb-js/compass into…
lerouxb Dec 24, 2024
03d6a76
info level logging for now
lerouxb Dec 24, 2024
60b0a29
even more debugging
lerouxb Dec 24, 2024
abb9674
really enable chromedriver's verbose logging
lerouxb Dec 24, 2024
1e24a08
MOAR log output
lerouxb Dec 24, 2024
81ad1dd
does running compass crash?
lerouxb Dec 24, 2024
6809cf9
remove settings dir before starting
lerouxb Dec 26, 2024
a1db900
list app support dir
lerouxb Dec 26, 2024
b19bbbd
comment for clarification
lerouxb Dec 27, 2024
b31073f
trace output to see where it gets
lerouxb Dec 27, 2024
9401c55
more logging
lerouxb Dec 27, 2024
ba4f8b3
more granular and don't do anything before the app is ready
lerouxb Dec 27, 2024
194fd6a
wait for ready before doing anything
lerouxb Dec 27, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions .evergreen/buildvariants-and-tasks.in.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,18 +80,18 @@ const SMOKETEST_BUILD_VARIANTS = [
// run_on: 'rhel80-large',
// depends_on: 'package-rhel',
// },
// {
// name: 'smoketest-macos-x64',
// display_name: 'Smoketest MacOS Intel',
// run_on: 'macos-14',
// depends_on: 'package-macos-x64',
// },
// {
// name: 'smoketest-macos-arm',
// display_name: 'Smoketest MacOS Arm64',
// run_on: 'macos-14-arm64',
// depends_on: 'package-macos-arm',
// }
{
name: 'smoketest-macos-x64',
display_name: 'Smoketest MacOS Intel',
run_on: 'macos-14',
depends_on: 'package-macos-x64',
},
{
name: 'smoketest-macos-arm',
display_name: 'Smoketest MacOS Arm64',
run_on: 'macos-14-arm64',
depends_on: 'package-macos-arm',
}
];

const TEST_PACKAGED_APP_BUILD_VARIANTS = [
Expand Down
16 changes: 16 additions & 0 deletions .evergreen/buildvariants-and-tasks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,22 @@ buildvariants:
variant: package-ubuntu
tasks:
- name: smoketest-compass
- name: smoketest-macos-x64-compass
display_name: Smoketest MacOS Intel (compass)
run_on: macos-14
depends_on:
- name: package-compass
variant: package-macos-x64
tasks:
- name: smoketest-compass
- name: smoketest-macos-arm-compass
display_name: Smoketest MacOS Arm64 (compass)
run_on: macos-14-arm64
depends_on:
- name: package-compass
variant: package-macos-arm
tasks:
- name: smoketest-compass
- name: test-eol-servers
display_name: Test EoL Servers
run_on: ubuntu1804-large
Expand Down
2 changes: 1 addition & 1 deletion .evergreen/functions.yml
Original file line number Diff line number Diff line change
Expand Up @@ -674,7 +674,7 @@ functions:
export COMPASS_E2E_DISABLE_CLIPBOARD_USAGE="true"
fi
npm run --workspace compass-e2e-tests smoketest
npm run --unsafe-perm --workspace compass-e2e-tests smoketest
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"Let's see if we really need this" and then forget about it for a week 🤦

test-web-sandbox:
- command: shell.exec
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// Copied from https://github.com/webdriverio/webdriverio/blob/1825c633aead82bc650dff1f403ac30cff7c7cb3/packages/devtools/src/constants.ts
// These are the default flags that webdriverio uses to start Chrome driver.
// NOTE: this has since been removed along with the devtools automation protocol https://github.com/webdriverio/webdriverio/commit/28e64e439ffc36a95f24aeda9f1d21111429dfa3#diff-6ea151d6c0687197931735239f397b7f5f0140a588c5b2b82ff584bbe73be069
const DEFAULT_WEBDRIVER_FLAGS = [
// suppresses Save Password prompt window
'--enable-automation',
Expand Down
18 changes: 12 additions & 6 deletions packages/compass-e2e-tests/helpers/compass.ts
Original file line number Diff line number Diff line change
Expand Up @@ -547,7 +547,7 @@ async function processCommonOpts({

// https://webdriver.io/docs/options/#webdriver-options
const webdriverOptions = {
logLevel: 'warn' as const, // info is super verbose right now
logLevel: 'trace' as const,
outputDir: webdriverLogPath,
};

Expand Down Expand Up @@ -601,7 +601,9 @@ async function startCompassElectron(
// See https://www.electronjs.org/docs/latest/api/command-line-switches#--enable-loggingfile
'--enable-logging=file',
// See https://www.electronjs.org/docs/latest/api/command-line-switches#--log-filepath
`--log-file=${electronLogFile}`
`--log-file=${electronLogFile}`,
// See // https://chromium.googlesource.com/chromium/src/+/master/docs/chrome_os_logging.md
'--log-level=0'
);

if (opts.extraSpawnArgs) {
Expand Down Expand Up @@ -643,9 +645,10 @@ async function startCompassElectron(
},
// from https://github.com/webdriverio-community/wdio-electron-service/blob/32457f60382cb4970c37c7f0a19f2907aaa32443/packages/wdio-electron-service/src/launcher.ts#L102
'wdio:enforceWebDriverClassic': true,
},
'wdio:chromedriverOptions': {
// TODO: enable logging so we don't have to debug things blindly
'wdio:chromedriverOptions': {
// enable logging so we don't have to debug things blindly
verbose: true,
},
},
...webdriverOptions,
...wdioOptions,
Expand All @@ -657,7 +660,10 @@ async function startCompassElectron(
let browser: CompassBrowser;

try {
browser = (await remote(options)) as CompassBrowser;
// webdriverio's type is wrong for
// options.capabilities['wdio:chromedriverOptions'] and it doesn't allow
// verbose even though it does work
browser = (await remote(options as any)) as CompassBrowser;
} catch (err) {
debug('Failed to start remote webdriver session', {
error: (err as Error).stack,
Expand Down
45 changes: 45 additions & 0 deletions packages/compass-e2e-tests/installers/helpers.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import { spawn } from 'child_process';
import type { SpawnOptions } from 'child_process';

export function execute(
command: string,
args: string[],
options?: SpawnOptions
): Promise<void> {
return new Promise((resolve, reject) => {
console.log(command, ...args);
const p = spawn(command, args, {
stdio: 'inherit',
...options,
});
p.on('error', (err: any) => {
reject(err);
});
p.on('close', (code: number | null, signal: NodeJS.Signals | null) => {
if (code !== null) {
if (code === 0) {
resolve();
} else {
reject(
new Error(`${command} ${args.join(' ')} exited with code ${code}`)
);
}
} else {
if (signal !== null) {
reject(
new Error(
`${command} ${args.join(' ')} exited with signal ${signal}`
)
);
} else {
// shouldn't happen
reject(
new Error(
`${command} ${args.join(' ')} exited with no code or signal`
)
);
}
}
});
});
}
57 changes: 57 additions & 0 deletions packages/compass-e2e-tests/installers/mac-dmg.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
import path from 'path';
import { existsSync } from 'fs';
import type { InstalledAppInfo, Package } from './types';
import { execute } from './helpers';

export async function installMacDMG(
appName: string,
{ filepath }: Package
): Promise<InstalledAppInfo> {
const fullDestinationPath = `/Applications/${appName}.app`;

if (existsSync(fullDestinationPath)) {
// Would ideally just throw here, but unfortunately in CI the mac
// environments aren't all clean so somewhere we have to remove it anyway.
console.log(`${fullDestinationPath} already exists. Removing.`);
await execute('rm', ['-rf', fullDestinationPath]);
}

await execute('hdiutil', ['attach', filepath]);
try {
await execute('cp', [
'-Rp',
`/Volumes/${appName}/${appName}.app`,
'/Applications',
]);
} finally {
await execute('hdiutil', ['detach', `/Volumes/${appName}`]);
}

// get debug output so we can see that it copied everything with the correct
// permissions
await execute('ls', ['-laR', `/Applications/${appName}.app`]);

// see if the executable will run without being quarantined or similar
await execute(`/Applications/${appName}.app/Contents/MacOS/${appName}`, [
'--version',
]);

if (process.env.HOME) {
const settingsDir = path.resolve(
process.env.HOME,
'Library',
'Application Support',
appName
);

if (existsSync(settingsDir)) {
console.log(`${settingsDir} already exists. Removing.`);
await execute('rm', ['-rf', settingsDir]);
}
}

return Promise.resolve({
appName,
appPath: `/Applications/${appName}.app`,
});
}
9 changes: 9 additions & 0 deletions packages/compass-e2e-tests/installers/types.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
export type Package = {
filename: string;
filepath: string;
};

export type InstalledAppInfo = {
appName: string;
appPath: string;
};
76 changes: 65 additions & 11 deletions packages/compass-e2e-tests/smoke-test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ import { hideBin } from 'yargs/helpers';
import https from 'https';
import { pick } from 'lodash';
import { handler as writeBuildInfo } from 'hadron-build/commands/info';
import type { InstalledAppInfo, Package } from './installers/types';
import { installMacDMG } from './installers/mac-dmg';
import { execute } from './installers/helpers';

const argv = yargs(hideBin(process.argv))
.scriptName('smoke-tests')
Expand Down Expand Up @@ -137,6 +140,10 @@ async function run() {
writeBuildInfo(infoArgs);
const buildInfo = JSON.parse(await fs.readFile(infoArgs.out, 'utf8'));

if (!buildInfoIsCommon(buildInfo)) {
throw new Error('buildInfo is missing');
}

// filter the extensions given the platform (isWindows, isOSX, isUbuntu, isRHEL) and extension
const { isWindows, isOSX, isRHEL, isUbuntu, extension } = context;

Expand All @@ -150,9 +157,9 @@ async function run() {

if (!context.skipDownload) {
await Promise.all(
packages.map(async ({ name, filepath }) => {
packages.map(async ({ filename, filepath }) => {
await fs.mkdir(path.dirname(filepath), { recursive: true });
const url = `https://${context.bucketName}.s3.amazonaws.com/${context.bucketKeyPrefix}/${name}`;
const url = `https://${context.bucketName}.s3.amazonaws.com/${context.bucketKeyPrefix}/${filename}`;
console.log(url);
return downloadFile(url, filepath);
})
Expand All @@ -162,6 +169,24 @@ async function run() {
verifyPackagesExist(packages);

// TODO(COMPASS-8533): extract or install each package and then test the Compass binary
for (const pkg of packages) {
let appInfo: InstalledAppInfo | undefined = undefined;

console.log('installing', pkg.filepath);

if (pkg.filename.endsWith('.dmg')) {
appInfo = await installMacDMG(buildInfo.productName, pkg);
}

// TODO: all the other installers go here

if (appInfo) {
console.log('testing', appInfo.appPath);
await testInstalledApp(appInfo);
} else {
console.log(`no app got installed for ${pkg.filename}`);
}
}
}

function platformFromContext(
Expand Down Expand Up @@ -189,6 +214,18 @@ type PackageFilterConfig = Pick<

// subsets of the hadron-build info result

const commonKeys = ['productName'];
type CommonBuildInfo = Record<typeof commonKeys[number], string>;

function buildInfoIsCommon(buildInfo: any): buildInfo is CommonBuildInfo {
for (const key of commonKeys) {
if (!buildInfo[key]) {
return false;
}
}
return true;
}

const windowsFilenameKeys = [
'windows_setup_filename',
'windows_msi_filename',
Expand Down Expand Up @@ -245,11 +282,6 @@ function buildInfoIsRHEL(buildInfo: any): buildInfo is RHELBuildInfo {
return true;
}

type Package = {
name: string;
filepath: string;
};

function getFilteredPackages(
compassDir: string,
buildInfo: any,
Expand Down Expand Up @@ -282,11 +314,11 @@ function getFilteredPackages(
const extension = config.extension;

return names
.filter((name) => !extension || name.endsWith(extension))
.map((name) => {
.filter((filename) => !extension || filename.endsWith(extension))
.map((filename) => {
return {
name,
filepath: path.join(compassDir, 'dist', name),
filename,
filepath: path.join(compassDir, 'dist', filename),
};
});
}
Expand Down Expand Up @@ -333,6 +365,28 @@ function verifyPackagesExist(packages: Package[]): void {
}
}

async function testInstalledApp(appInfo: InstalledAppInfo): Promise<void> {
await execute(
'npm',
[
'run',
'--unsafe-perm',
'test-packaged',
'--workspace',
'compass-e2e-tests',
'--',
'--test-filter=time-to-first-query',
Copy link
Contributor Author

@lerouxb lerouxb Dec 20, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The filter will probably change once we add tests that test auto-update since it is easiest to test that by running the app inside an e2e test so we can use webdriverio. But this is good enough for now because it tests that the app starts up and can execute a query.

],
{
env: {
...process.env,
COMPASS_APP_NAME: appInfo.appName,
COMPASS_APP_PATH: appInfo.appPath,
},
}
);
}

run()
.then(function () {
console.log('done');
Expand Down
Loading
Loading