ARC-814 RepoSyncState table as source of truth (#928)

* initial commit for new repo table as source

* adding more tests for reposyncstate

* fixing API tests

* fixing more tests that are using RepoSyncState table

package.json

@@ -105,6 +105,7 @@
     "uuid": "^8.3.2"
   },
   "devDependencies": {
+    "@types/jest-when": "^2.7.3",
     "@typescript-eslint/eslint-plugin": "~4.23.0",
     "@typescript-eslint/parser": "~4.23.0",
     "dotenv-cli": "^4.0.0",

src/api/index.ts

@@ -6,7 +6,7 @@ import RedisStore from "rate-limit-redis";
 import Redis from "ioredis";
 import BodyParser from "body-parser";
 import GithubAPI from "../config/github-api";
-import { Installation, Subscription } from "../models";
+import { Installation, RepoSyncState, Subscription } from "../models";
 import verifyInstallation from "../jira/verify-installation";
 import logMiddleware from "../middleware/frontend-log-middleware";
 import JiraClient from "../models/jira-client";
@@ -18,6 +18,7 @@ import { getLogger } from "../config/logger";
 import { Job, Queue } from "bull";
 import { WhereOptions } from "sequelize";
 import getJiraClient from "../jira/client";
+import { booleanFlag, BooleanFlags } from "../config/feature-flags";
 
 const router = express.Router();
 const bodyParser = BodyParser.urlencoded({ extended: false });
@@ -168,7 +169,11 @@ router.get(
 				return;
 			}
 
-			res.json(subscription.repoSyncState);
+			if(await booleanFlag(BooleanFlags.REPO_SYNC_STATE_AS_SOURCE, false)) {
+				res.json(await RepoSyncState.toRepoJson(subscription));
+			} else {
+				res.json(subscription.repoSyncState);
+			}
 		} catch (err) {
 			res.status(500).json(err);
 		}
@@ -382,23 +387,18 @@ router.post(
 	bodyParser,
 	check("installationId").isInt(),
 	returnOnValidationError,
-	async (req: Request, response: Response): Promise<void> => {
+	async (req: Request, res: Response): Promise<void> => {
 		const { installationId } = req.params;
 		const installation = await Installation.findByPk(installationId);
-
 		const isValid = await verifyInstallation(installation, req.log)();
-		const respondWith = (message) =>
-			response.json({
-				message,
-				installation: {
-					enabled: isValid,
-					id: installation.id,
-					jiraHost: installation.jiraHost
-				}
-			});
-		respondWith(
-			isValid ? "Verification successful" : "Verification failed"
-		);
+		res.json({
+			message: isValid ? "Verification successful" : "Verification failed",
+			installation: {
+				enabled: isValid,
+				id: installation.id,
+				jiraHost: installation.jiraHost
+			}
+		})
 	}
 );
 

src/config/feature-flags.ts

@@ -25,6 +25,7 @@ export enum BooleanFlags {
 	SUPPORT_BRANCH_AND_MERGE_WORKFLOWS_FOR_BUILDS = "support-branch-and-merge-workflows-for-builds",
 	USE_NEW_GITHUB_CLIENT_FOR_PUSH = "use-new-github-client-for-push",
 	USE_NEW_GITHUB_CLIENT_TO_COUNT_REPOS = "use-new-github-client-to-count-repos",
+	REPO_SYNC_STATE_AS_SOURCE = "repo-sync-state-as-source",
 	CALL_IS_ADMIN_AS_APP = "call-is-admin-as-app"
 }
 

src/frontend/get-jira-configuration.ts

@@ -1,13 +1,15 @@
 import format from "date-fns/format";
 import moment from "moment";
-import { Subscription } from "../models";
-import SubscriptionClass from "../models/subscription";
+import SubscriptionClass, { SyncStatus } from "../models/subscription";
+import { RepoSyncState, Subscription } from "../models";
 import { NextFunction, Request, Response } from "express";
 import statsd from "../config/statsd";
 import { metricError } from "../config/metric-names";
 import { FailedInstallations } from "../config/interfaces";
+import { booleanFlag, BooleanFlags } from "../config/feature-flags";
+import Logger from "bunyan";
 
-function mapSyncStatus(syncStatus: string): string {
+function mapSyncStatus(syncStatus: SyncStatus = SyncStatus.PENDING): string {
 	switch (syncStatus) {
 		case "ACTIVE":
 			return "IN PROGRESS";
@@ -18,23 +20,26 @@ function mapSyncStatus(syncStatus: string): string {
 	}
 }
 
-export async function 	getInstallation(client, subscription, reqLog?) {
+export async function getInstallation(client, subscription: SubscriptionClass, reqLog?: Logger) {
 	const id = subscription.gitHubInstallationId;
 	try {
 		const response = await client.apps.getInstallation({ installation_id: id });
 		response.data.syncStatus = mapSyncStatus(subscription.syncStatus);
 		response.data.syncWarning = subscription.syncWarning;
 		response.data.subscriptionUpdatedAt = formatDate(subscription.updatedAt);
-		response.data.totalNumberOfRepos = Object.keys(
-			subscription.repoSyncState?.repos || {}
-		).length;
-		response.data.numberOfSyncedRepos =
-			subscription.repoSyncState?.numberOfSyncedRepos || 0;
+		if (await booleanFlag(BooleanFlags.REPO_SYNC_STATE_AS_SOURCE, false, subscription.jiraHost)) {
+			response.data.totalNumberOfRepos = await RepoSyncState.countFromSubscription(subscription);
+			response.data.numberOfSyncedRepos = await RepoSyncState.countSyncedReposFromSubscription(subscription);
+		} else {
+			response.data.totalNumberOfRepos = Object.keys(subscription.repoSyncState?.repos || {}).length;
+			response.data.numberOfSyncedRepos = subscription.repoSyncState?.numberOfSyncedRepos || 0;
+		}
+
 		response.data.jiraHost = subscription.jiraHost;
 
 		return response.data;
 	} catch (err) {
-		reqLog.error(
+		reqLog?.error(
 			{ installationId: id, error: err, uninstalled: err.code === 404 },
 			"Failed connection"
 		);
@@ -44,10 +49,10 @@ export async function 	getInstallation(client, subscription, reqLog?) {
 	}
 }
 
-const formatDate = function (date) {
+const formatDate = function(date) {
 	return {
 		relative: moment(date).fromNow(),
-		absolute: format(date, "MMMM D, YYYY h:mm a"),
+		absolute: format(date, "MMMM D, YYYY h:mm a")
 	};
 };
 
@@ -57,27 +62,30 @@ interface FailedConnections {
 	orgName: string | undefined;
 }
 
-export const getFailedConnections = (
+export const getFailedConnections = async (
 	installations: FailedInstallations[],
 	subscriptions: SubscriptionClass[]
-): FailedConnections[] => {
-	return installations
+): Promise<FailedConnections[]> => {
+	return Promise.all(installations
 		.filter((response) => !!response.error)
-		.map((failedConnection: FailedInstallations) => {
-			const sub = subscriptions.find(
-				(subscription: SubscriptionClass) =>
-					failedConnection.id === subscription.gitHubInstallationId
-			);
-			const repos = sub?.repoSyncState?.repos || {};
-			const repoId = Object.keys(repos);
-			const orgName = repos[repoId[0]]?.repository?.owner.login || undefined;
+		.map(async (failedConnection: FailedInstallations) => {
+			const sub = subscriptions.find(sub => failedConnection.id === sub.gitHubInstallationId);
+			let orgName;
+			if (sub && await booleanFlag(BooleanFlags.REPO_SYNC_STATE_AS_SOURCE, false, sub.jiraHost)) {
+				const repo = await RepoSyncState.findOneFromSubscription(sub);
+				orgName = repo.repoOwner;
+			} else {
+				const repos = sub?.repoSyncState?.repos || {};
+				const repoId = Object.keys(repos);
+				orgName = repos[repoId[0]]?.repository?.owner.login || undefined;
+			}
 
 			return {
 				id: failedConnection.id,
 				deleted: failedConnection.deleted,
-				orgName,
+				orgName
 			};
-		});
+		}));
 };
 
 export default async (
@@ -104,7 +112,7 @@ export default async (
 			)
 		);
 
-		const failedConnections = getFailedConnections(
+		const failedConnections = await getFailedConnections(
 			installations,
 			subscriptions
 		);
@@ -116,7 +124,7 @@ export default async (
 				isGlobalInstall: data.repository_selection === "all",
 				installedAt: formatDate(data.updated_at),
 				syncState: data.syncState,
-				repoSyncState: data.repoSyncState,
+				repoSyncState: data.repoSyncState
 			}));
 
 		const hasConnections =
@@ -129,7 +137,7 @@ export default async (
 			hasConnections,
 			APP_URL: process.env.APP_URL,
 			csrfToken: req.csrfToken(),
-			nonce: res.locals.nonce,
+			nonce: res.locals.nonce
 		});
 
 		req.log.info("Jira configuration rendered successfully.");

src/models/installation.ts

@@ -1,6 +1,7 @@
 import crypto from "crypto";
 import Sequelize from "sequelize";
-import Subscription from "./subscription";
+import { Subscription } from "./index";
+import SubscriptionClass from "./subscription";
 
 // TODO: this should not be there.  Should only check once a function is called
 if (!process.env.STORAGE_SECRET) {
@@ -35,7 +36,7 @@ export default class Installation extends Sequelize.Model {
 	static async getForHost(host: string): Promise<Installation | null> {
 		return Installation.findOne( {
 			where: {
-				jiraHost: host,
+				jiraHost: host
 			},
 			order: [["id", "DESC"]]
 		});
@@ -44,7 +45,7 @@ export default class Installation extends Sequelize.Model {
 	static async getAllForHost(host: string): Promise<Installation[]> {
 		return Installation.findAll({
 			where: {
-				jiraHost: host,
+				jiraHost: host
 			},
 			order: [["id", "DESC"]]
 		});
@@ -94,7 +95,7 @@ export default class Installation extends Sequelize.Model {
 		await this.destroy();
 	}
 
-	async subscriptions(): Promise<Subscription[]> {
+	async subscriptions(): Promise<SubscriptionClass[]> {
 		return Subscription.getAllForClientKey(this.clientKey);
 	}
 }