Axon 201 implement experiment flow split (#174)

* AXON-201: refactored exp fetching from statsig  + implemented new auth exp flow

* AXON-201: fix tests

* AXON-201: updates

* AXON-201: fixes

* AXON-201: updates

Author: cabella-dot

src/container.ts

@@ -65,7 +65,7 @@ import { VSCWelcomeActionApi } from './webview/welcome/vscWelcomeActionApi';
 import { VSCWelcomeWebviewControllerFactory } from './webview/welcome/vscWelcomeWebviewControllerFactory';
 import { WelcomeAction } from './lib/ipc/fromUI/welcome';
 import { WelcomeInitMessage } from './lib/ipc/toUI/welcome';
-import { FeatureFlagClient, Features } from './util/featureFlags';
+import { Experiments, FeatureFlagClient, Features } from './util/featureFlags';
 import { EventBuilder } from './util/featureFlags/eventBuilder';
 import { AtlascodeUriHandler } from './uriHandler';
 import { CheckoutHelper } from './bitbucket/interfaces';
@@ -79,7 +79,7 @@ const isDebuggingRegex = /^--(debug|inspect)\b(-brk\b|(?!-))=?/;
 const ConfigTargetKey = 'configurationTarget';
 
 export class Container {
-    static initialize(context: ExtensionContext, config: IConfig, version: string) {
+    static async initialize(context: ExtensionContext, config: IConfig, version: string) {
         const analyticsEnv: string = this.isDebugging ? 'staging' : 'prod';
 
         this._analyticsClient = analyticsClient({
@@ -187,7 +187,7 @@ export class Container {
 
         this._loginManager = new LoginManager(this._credentialManager, this._siteManager, this._analyticsClient);
         this._bitbucketHelper = new BitbucketCheckoutHelper(context.globalState);
-        FeatureFlagClient.initialize({
+        await FeatureFlagClient.initialize({
             analyticsClient: this._analyticsClient,
             identifiers: {
                 analyticsAnonymousId: env.machineId,
@@ -200,6 +200,7 @@ export class Container {
             .finally(() => {
                 this.initializeUriHandler(context, this._analyticsApi, this._bitbucketHelper);
                 this.initializeNewSidebarView(context, config);
+                this.initializeAAExperiment();
             });
 
         context.subscriptions.push((this._helpExplorer = new HelpExplorer()));
@@ -209,7 +210,9 @@ export class Container {
         const telemetryConfig = workspace.getConfiguration('telemetry');
         return telemetryConfig.get<boolean>('enableTelemetry', true);
     }
-
+    static initializeAAExperiment() {
+        FeatureFlagClient.checkExperimentValue(Experiments.AtlascodeAA);
+    }
     static initializeUriHandler(
         context: ExtensionContext,
         analyticsApi: VSCAnalyticsApi,

src/extension.ts

@@ -23,7 +23,7 @@ import { registerResources } from './resources';
 import { GitExtension } from './typings/git';
 import { pid } from 'process';
 import { startListening } from './atlclients/negotiate';
-import { FeatureFlagClient } from './util/featureFlags';
+import { Experiments, FeatureFlagClient } from './util/featureFlags';
 
 const AnalyticDelay = 5000;
 
@@ -32,7 +32,6 @@ export async function activate(context: ExtensionContext) {
     const atlascode = extensions.getExtension('atlassian.atlascode')!;
     const atlascodeVersion = atlascode.packageJSON.version;
     const previousVersion = context.globalState.get<string>(GlobalStateVersionKey);
-
     registerResources(context);
 
     Configuration.configure(context);
@@ -42,7 +41,7 @@ export async function activate(context: ExtensionContext) {
     context.globalState.update('rulingPid', pid);
 
     try {
-        Container.initialize(context, configuration.get<IConfig>(), atlascodeVersion);
+        await Container.initialize(context, configuration.get<IConfig>(), atlascodeVersion);
 
         registerCommands(context);
         activateCodebucket(context);
@@ -63,11 +62,13 @@ export async function activate(context: ExtensionContext) {
         Container.clientManager.requestSite(site);
     });
 
-    if (previousVersion === undefined && window.state.focused) {
-        commands.executeCommand(Commands.ShowOnboardingPage); //This is shown to users who have never opened our extension before
+    // new user for auth exp
+    if (previousVersion === undefined) {
+        initializeNewAuthExperiment();
     } else {
         showWelcomePage(atlascodeVersion, previousVersion);
     }
+
     const delay = Math.floor(Math.random() * Math.floor(AnalyticDelay));
     setTimeout(() => {
         sendAnalytics(atlascodeVersion, context.globalState);
@@ -165,6 +166,35 @@ async function sendAnalytics(version: string, globalState: Memento) {
     });
 }
 
+function initializeNewAuthExperiment() {
+    let onboardingFlow = FeatureFlagClient.checkExperimentValue(Experiments.NewAuthUI);
+    Logger.debug(`Onboarding Experiment flow: ${onboardingFlow}`);
+    if (!onboardingFlow) {
+        onboardingFlow = 'control';
+    }
+
+    switch (onboardingFlow) {
+        case 'settings': {
+            commands.executeCommand(Commands.ShowConfigPage);
+            break;
+        }
+        case 'legacy': {
+            commands.executeCommand(Commands.ShowOnboardingPage);
+            break;
+        }
+        case 'new': {
+            commands.executeCommand(Commands.ShowOnboardingPage);
+            break;
+        }
+        default: {
+            // control
+            if (window.state.focused) {
+                commands.executeCommand(Commands.ShowOnboardingPage);
+            }
+            break;
+        }
+    }
+}
 // this method is called when your extension is deactivated
 export function deactivate() {
     FeatureFlagClient.dispose();

src/react/atlascode/onboarding/OnboardingPage.tsx

@@ -15,10 +15,10 @@ import { OnboardingControllerContext, useOnboardingController } from './onboardi
 import ProductSelector from './ProductSelector';
 import { SimpleSiteAuthenticator } from './SimpleSiteAuthenticator';
 import { AtlascodeErrorBoundary } from '../common/ErrorBoundary';
-import { AnalyticsView } from 'src/analyticsTypes';
-import { Features } from 'src/util/featureFlags';
-import { CommonMessageType } from 'src/lib/ipc/toUI/common';
-import { OnboardingActionType } from 'src/lib/ipc/fromUI/onboarding';
+import { AnalyticsView } from '../../../analyticsTypes';
+import { Experiments } from '../../../util/featureFlags';
+import { CommonMessageType } from '../../../lib/ipc/toUI/common';
+import { OnboardingActionType } from '../../../lib/ipc/fromUI/onboarding';
 import { JiraOnboarding } from './JiraOnboarding';
 import { BitbucketOnboarding } from './BitbucketOnboarding';
 
@@ -69,9 +69,11 @@ export const OnboardingPage: React.FunctionComponent = () => {
     React.useEffect(() => {
         window.addEventListener('message', (event) => {
             const message = event.data;
-            if (message.command === CommonMessageType.UpdateFeatureFlags) {
-                const featureValue = message.featureFlags[Features.EnableAuthUI];
-                setUseAuthUI(featureValue);
+            if (message.command === CommonMessageType.UpdateExperimentValues) {
+                const experimentValue = message.experimentValues[Experiments.NewAuthUI];
+                if (typeof experimentValue === 'string' && experimentValue !== undefined && experimentValue !== null) {
+                    setUseAuthUI(experimentValue === 'new');
+                }
             }
         });
     }, [controller]);

src/util/featureFlags/client.test.ts

@@ -102,19 +102,21 @@ describe('FeatureFlagClient', () => {
             expect(FeatureFlagClient.featureGates[featureName]).toBe(true);
         });
 
-        it('experiments default values are correctly assigned', async () => {
+        it('experiments overrides are correctly applied', async () => {
+            process.env.ATLASCODE_EXP_OVERRIDES_STRING = `${experimentName}=another value`;
             await FeatureFlagClient.initialize(options);
-            expect(FeatureGates.getExperimentValue).toHaveBeenCalled();
-            expect(Object.keys(FeatureFlagClient.experimentValues).length).toBe(1);
             expect(FeatureFlagClient.experimentValues[experimentName]).toBeDefined();
-            expect(FeatureFlagClient.experimentValues[experimentName]).toBe('a default value');
+            expect(FeatureFlagClient.experimentValues[experimentName]).toBe('another value');
         });
+    });
 
-        it('experiments overrides are correctly applied', async () => {
-            process.env.ATLASCODE_EXP_OVERRIDES_STRING = `${experimentName}=another value`;
+    describe('checkExperimentValue', () => {
+        it('should return the value of the experiment and save value in client', async () => {
             await FeatureFlagClient.initialize(options);
+            const value = FeatureFlagClient.checkExperimentValue(experimentName);
             expect(FeatureFlagClient.experimentValues[experimentName]).toBeDefined();
-            expect(FeatureFlagClient.experimentValues[experimentName]).toBe('another value');
+            expect(FeatureFlagClient.experimentValues[experimentName]).toBe('a default value');
+            expect(value).toBe('a default value');
         });
     });
 });

src/util/featureFlags/client.ts

@@ -72,7 +72,7 @@ export class FeatureFlagClient {
 
     private static finalizeInit(): void {
         this._featureGates = this.evaluateFeatures();
-        this._experimentValues = this.evaluateExperiments();
+        this._experimentValues = {} as ExperimentGateValues;
 
         const ffSplit = (process.env.ATLASCODE_FF_OVERRIDES || '')
             .split(',')
@@ -136,7 +136,7 @@ export class FeatureFlagClient {
         return gateValue;
     }
 
-    private static checkExperimentValue(experiment: Experiments): any {
+    static checkExperimentValue(experiment: Experiments): any {
         const experimentGate = ExperimentGates[experiment];
         if (!experimentGate) {
             return undefined;
@@ -153,6 +153,7 @@ export class FeatureFlagClient {
             );
         }
         console.log(`ExperimentGateValue: ${experiment} -> ${gateValue}`);
+        this._experimentValues[experiment] = gateValue;
         return gateValue;
     }
 
@@ -162,12 +163,6 @@ export class FeatureFlagClient {
         return featureFlags;
     }
 
-    private static evaluateExperiments(): ExperimentGateValues {
-        const experimentGates = {} as ExperimentGateValues;
-        Object.values(Experiments).forEach((exp) => (experimentGates[exp] = this.checkExperimentValue(exp)));
-        return experimentGates;
-    }
-
     static dispose() {
         FeatureGates.shutdownStatsig();
     }

src/util/featureFlags/features.ts

@@ -11,8 +11,8 @@ export enum Experiments {
 
 export const ExperimentGates: Record<Experiments, ExperimentPayload> = {
     [Experiments.NewAuthUI]: {
-        parameter: 'isEnabled',
-        defaultValue: false,
+        parameter: 'onboardingFlow',
+        defaultValue: 'control',
     },
     [Experiments.AtlascodeAA]: {
         parameter: 'isEnabled',