Skip to content

Commit

Permalink
fix: pull project with removed notifications (#11378)
Browse files Browse the repository at this point in the history 
* fix: pull project with removed notifications
  • Loading branch information
@lazpavel
lazpavel committed Nov 21, 2022
1 parent 36bc08e commit 091d1d6
Show file tree
Hide file tree
Showing 8 changed files with 234 additions and 46 deletions.
67 changes: 32 additions & 35 deletions packages/amplify-category-notifications/src/multi-env-manager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,44 +123,41 @@ const constructPinpointNotificationsMeta = async (context: $TSContext) : Promise
// from cloud meta and as no new resources are created during pull, we should not look for
// Pinpoint app in analytics category.
const isPulling = context.input.command === 'pull' || (context.input.command === 'env' && context.input.subCommands[0] === 'pull');
const currentAmplifyMeta = stateManager.getCurrentMeta(undefined, {
throwIfNotExist: false,
});

if (isPulling) {
const currentAmplifyMeta = stateManager.getCurrentMeta(undefined, {
throwIfNotExist: false,
});

if (currentAmplifyMeta) {
const currentNotificationsMeta = currentAmplifyMeta[AmplifyCategories.NOTIFICATIONS];

// We only support single Pinpoint across notifications and analytics categories
if (currentNotificationsMeta && Object.keys(currentNotificationsMeta).length > 0) {
const pinpointResource = _.get(currentNotificationsMeta, Object.keys(currentNotificationsMeta)[0], undefined);
// if pinpoint resource ID is not found in Notifications, we will ge it from the Analytics category
if (!(pinpointResource.output.Id)) {
const analyticsPinpointApp: Partial<ICategoryMeta>|undefined = getPinpointAppFromAnalyticsMeta(currentAmplifyMeta);
// eslint-disable-next-line max-depth
if (analyticsPinpointApp) {
pinpointResource.output.Id = analyticsPinpointApp.Id;
pinpointResource.output.Region = analyticsPinpointApp.Region;
pinpointResource.output.Name = analyticsPinpointApp.Name;
pinpointResource.ResourceName = analyticsPinpointApp.regulatedResourceName;
}
}

if (!pinpointResource.output.Id) {
throw new AmplifyError('ResourceNotReadyError', {
message: 'Pinpoint resource ID not found.',
resolution: 'Run "amplify add analytics" to create a new Pinpoint resource.',
});
if (isPulling && currentAmplifyMeta) {
const currentNotificationsMeta = currentAmplifyMeta[AmplifyCategories.NOTIFICATIONS];

// We only support single Pinpoint across notifications and analytics categories
if (currentNotificationsMeta && Object.keys(currentNotificationsMeta).length > 0) {
const pinpointResource = _.get(currentNotificationsMeta, Object.keys(currentNotificationsMeta)[0], undefined);
// if pinpoint resource ID is not found in Notifications, we will ge it from the Analytics category
if (!(pinpointResource.output.Id)) {
const analyticsPinpointApp: Partial<ICategoryMeta>|undefined = getPinpointAppFromAnalyticsMeta(currentAmplifyMeta);
// eslint-disable-next-line max-depth
if (analyticsPinpointApp) {
pinpointResource.output.Id = analyticsPinpointApp.Id;
pinpointResource.output.Region = analyticsPinpointApp.Region;
pinpointResource.output.Name = analyticsPinpointApp.Name;
pinpointResource.ResourceName = analyticsPinpointApp.regulatedResourceName;
}
}

pinpointApp = {
Id: pinpointResource.output.Id,
};
pinpointApp.Name = pinpointResource.output.Name || pinpointResource.output.appName;
pinpointApp.Region = pinpointResource.output.Region;
pinpointApp.lastPushTimeStamp = pinpointResource.lastPushTimeStamp;
if (!pinpointResource.output.Id) {
throw new AmplifyError('ResourceNotReadyError', {
message: 'Pinpoint resource ID not found.',
resolution: 'Run "amplify add analytics" to create a new Pinpoint resource.',
});
}

pinpointApp = {
Id: pinpointResource.output.Id,
};
pinpointApp.Name = pinpointResource.output.Name || pinpointResource.output.appName;
pinpointApp.Region = pinpointResource.output.Region;
pinpointApp.lastPushTimeStamp = pinpointResource.lastPushTimeStamp;
}
}

Expand Down Expand Up @@ -214,7 +211,7 @@ const constructPinpointNotificationsMeta = async (context: $TSContext) : Promise
}
}

if (pinpointApp) {
if (pinpointApp && (!isPulling || (isPulling && currentAmplifyMeta[AmplifyCategories.NOTIFICATIONS]))) {
await notificationManager.pullAllChannels(context, pinpointApp);
pinpointNotificationsMeta = {
Name: pinpointApp.Name,
Expand Down
15 changes: 8 additions & 7 deletions packages/amplify-category-notifications/src/notifications-amplify-meta-api.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,20 +247,21 @@ export const constructPartialNotificationsAppMeta = (
export const constructResourceMeta = (amplifyMeta : $TSMeta,
resourceName: string, pinpointOutput:
Partial<ICategoryMeta>): Partial<ICategoryMeta> => {
const tmpAmplifyMeta = amplifyMeta;
if (!tmpAmplifyMeta[AmplifyCategories.NOTIFICATIONS]) {
tmpAmplifyMeta[AmplifyCategories.NOTIFICATIONS] = { [resourceName]: { output: {} } };
if (!amplifyMeta[AmplifyCategories.NOTIFICATIONS] || Object.keys(amplifyMeta[AmplifyCategories.NOTIFICATIONS]).length === 0) {
// eslint-disable-next-line no-param-reassign
amplifyMeta[AmplifyCategories.NOTIFICATIONS] = { [resourceName]: { output: {} } };
}
tmpAmplifyMeta[AmplifyCategories.NOTIFICATIONS][resourceName] = {
...tmpAmplifyMeta[AmplifyCategories.NOTIFICATIONS][resourceName],
// eslint-disable-next-line no-param-reassign
amplifyMeta[AmplifyCategories.NOTIFICATIONS][resourceName] = {
...amplifyMeta[AmplifyCategories.NOTIFICATIONS][resourceName],
service: AmplifySupportedService.PINPOINT,
output: {
...tmpAmplifyMeta[AmplifyCategories.NOTIFICATIONS][resourceName].output,
...amplifyMeta[AmplifyCategories.NOTIFICATIONS][resourceName].output,
...pinpointOutput,
},
lastPushTimeStamp: new Date(),
};
return tmpAmplifyMeta;
return amplifyMeta;
};

/**
Expand Down
3 changes: 2 additions & 1 deletion packages/amplify-e2e-core/src/init/amplifyPull.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ export const amplifyPull = (
envName?: string;
yesFlag?: boolean;
},
testingWithLatestCodebase = false,
): Promise<void> => {
// Note:- Table checks have been removed since they are not necessary for push/pull flows and prone to breaking because
// of stylistic changes. A simpler content based check will be added in the future.
Expand All @@ -36,7 +37,7 @@ export const amplifyPull = (
args.push('--yes');
}

const chain = spawn(getCLIPath(), args, { cwd, stripColors: true });
const chain = spawn(getCLIPath(testingWithLatestCodebase), args, { cwd, stripColors: true });

if (settings.emptyDir) {
chain
Expand Down
53 changes: 53 additions & 0 deletions packages/amplify-e2e-tests/src/__tests__/notifications-sms-pull.test.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
import {
addNotificationChannel,
amplifyPull,
createNewProjectDir,
deleteProject,
deleteProjectDir,
getAppId,
initJSProjectWithProfile,
removeAllNotificationChannel,
} from '@aws-amplify/amplify-e2e-core';
import {
getShortId,
} from '../import-helpers';

describe('notification category pull test', () => {
const testChannel = 'SMS';
const testChannelSelection = 'SMS';
const projectPrefix = `notification${testChannel}`.substring(0, 19);
const projectSettings = {
name: projectPrefix,
disableAmplifyAppCreation: false,
};

let projectRoot: string;

beforeEach(async () => {
projectRoot = await createNewProjectDir(projectPrefix);
});

afterEach(async () => {
await deleteProject(projectRoot);
deleteProjectDir(projectRoot);
});

it(`should add and remove the ${testChannel} channel and pull in new directory`, async () => {
await initJSProjectWithProfile(projectRoot, projectSettings);

const settings = { resourceName: `${projectPrefix}${getShortId()}` };
await addNotificationChannel(projectRoot, settings, testChannelSelection);

const appId = getAppId(projectRoot);
expect(appId).toBeDefined();

await removeAllNotificationChannel(projectRoot);

const projectRootPull = await createNewProjectDir('removed-notifications-pull');
try {
await amplifyPull(projectRootPull, { override: false, emptyDir: true, appId });
} finally {
deleteProjectDir(projectRootPull);
}
});
});
76 changes: 73 additions & 3 deletions packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { deleteS3Bucket } from '@aws-amplify/amplify-e2e-core';

// Ensure to update scripts/split-e2e-tests.ts is also updated this gets updated
const AWS_REGIONS_TO_RUN_TESTS = [
'us-east-1',
'us-east-2',
'us-west-2',
'eu-west-2',
Expand Down Expand Up @@ -59,6 +60,14 @@ type S3BucketInfo = {
cciInfo?: CircleCIJobDetails;
};

type PinpointAppInfo = {
id: string;
name: string;
arn: string;
region: string;
cciInfo?: CircleCIJobDetails;
};

type IamRoleInfo = {
name: string;
cciInfo?: CircleCIJobDetails;
Expand All @@ -73,6 +82,7 @@ type ReportEntry = {
stacks: Record<string, StackInfo>;
buckets: Record<string, S3BucketInfo>;
roles: Record<string, IamRoleInfo>;
pinpointApps: Record<string, PinpointAppInfo>;
};

type JobFilterPredicate = (job: ReportEntry) => boolean;
Expand All @@ -91,6 +101,7 @@ type AWSAccountInfo = {
sessionToken: string;
};

const PINPOINT_TEST_REGEX = /integtest/;
const BUCKET_TEST_REGEX = /test/;
const IAM_TEST_REGEX = /!RotateE2eAwsToken-e2eTestContextRole|-integtest$|^amplify-|^eu-|^us-|^ap-/;
const STALE_DURATION_MS = 2 * 60 * 60 * 1000; // 2 hours in milliseconds
Expand Down Expand Up @@ -118,6 +129,12 @@ const testRoleStalenessFilter = (resource: aws.IAM.Role): boolean => {
return isTestResource && isStaleResource;
};

const testPinpointAppStalenessFilter = (resource: aws.Pinpoint.ApplicationResponse): boolean => {
const isTestResource = resource.Name.match(PINPOINT_TEST_REGEX);
const isStaleResource = (Date.now() - new Date(resource.CreationDate).getTime()) > STALE_DURATION_MS;
return isTestResource && isStaleResource;
};

/**
* Get all S3 buckets in the account, and filter down to the ones we consider stale.
*/
Expand All @@ -138,6 +155,25 @@ const getOrphanTestIamRoles = async (account: AWSAccountInfo): Promise<IamRoleIn
return staleRoles.map(it => ({ name: it.RoleName }));
};

const getOrphanPinpointApplications = async (account: AWSAccountInfo, region: string): Promise<PinpointAppInfo[]> => {
const pinpoint = new aws.Pinpoint(getAWSConfig(account, region));
const apps: PinpointAppInfo[] = [];
let nextToken = null;

do {
const result = await pinpoint.getApps({
Token: nextToken,
}).promise();
apps.push(...result.ApplicationsResponse.Item.filter(testPinpointAppStalenessFilter).map(it => ({
id: it.Id, name: it.Name, arn: it.Arn, region,
})));

nextToken = result.ApplicationsResponse.NextToken;
} while (nextToken);

return apps;
};

/**
* Get the relevant AWS config object for a given account and region.
*/
Expand Down Expand Up @@ -342,6 +378,7 @@ const mergeResourcesByCCIJob = (
s3Buckets: S3BucketInfo[],
orphanS3Buckets: S3BucketInfo[],
orphanIamRoles: IamRoleInfo[],
orphanPinpointApplications: PinpointAppInfo[],
): Record<string, ReportEntry> => {
const result: Record<string, ReportEntry> = {};

Expand Down Expand Up @@ -412,6 +449,16 @@ const mergeResourcesByCCIJob = (
roles: src,
}));

const orphanPinpointApps = {
[ORPHAN]: orphanPinpointApplications,
};

_.mergeWith(result, orphanPinpointApps, (val, src, key) => ({
...val,
jobId: key,
pinpointApps: src,
}));

return result;
};

Expand Down Expand Up @@ -527,6 +574,23 @@ const deleteBucket = async (account: AWSAccountInfo, accountIndex: number, bucke
}
};

const deletePinpointApps = async (account: AWSAccountInfo, accountIndex: number, apps: PinpointAppInfo[]): Promise<void> => {
await Promise.all(apps.map(app => deletePinpointApp(account, accountIndex, app)));
};

const deletePinpointApp = async (account: AWSAccountInfo, accountIndex: number, app: PinpointAppInfo): Promise<void> => {
const {
id, name, region,
} = app;
try {
console.log(`[ACCOUNT ${accountIndex}] Deleting Pinpoint App ${name}`);
const pinpoint = new aws.Pinpoint(getAWSConfig(account, region));
await pinpoint.deleteApp({ ApplicationId: id }).promise();
} catch (e) {
console.log(`[ACCOUNT ${accountIndex}] Deleting pinpoint app ${name} failed with error ${e.message}`);
}
};

const deleteCfnStacks = async (account: AWSAccountInfo, accountIndex: number, stacks: StackInfo[]): Promise<void> => {
await Promise.all(stacks.map(stack => deleteCfnStack(account, accountIndex, stack)));
};
Expand Down Expand Up @@ -578,6 +642,10 @@ const deleteResources = async (
if (resources.roles) {
await deleteIamRoles(account, accountIndex, Object.values(resources.roles));
}

if (resources.pinpointApps) {
await deletePinpointApps(account, accountIndex, Object.values(resources.pinpointApps));
}
}
};

Expand Down Expand Up @@ -626,8 +694,8 @@ const getAccountsToCleanup = async (): Promise<AWSAccountInfo[]> => {
const orgAccounts = await orgApi.listAccounts().promise();
const allAccounts = orgAccounts.Accounts;
let nextToken = orgAccounts.NextToken;
while(nextToken){
const nextPage = await orgApi.listAccounts({"NextToken": nextToken }).promise();
while (nextToken) {
const nextPage = await orgApi.listAccounts({ NextToken: nextToken }).promise();
allAccounts.push(...nextPage.Accounts);
nextToken = nextPage.NextToken;
}
Expand Down Expand Up @@ -672,6 +740,7 @@ const cleanupAccount = async (account: AWSAccountInfo, accountIndex: number, fil
const appPromises = AWS_REGIONS_TO_RUN_TESTS.map(region => getAmplifyApps(account, region));
const stackPromises = AWS_REGIONS_TO_RUN_TESTS.map(region => getStacks(account, region));
const bucketPromise = getS3Buckets(account);
const orphanPinpointApplicationsPromise = AWS_REGIONS_TO_RUN_TESTS.map(region => getOrphanPinpointApplications(account, region));
const orphanBucketPromise = getOrphanS3TestBuckets(account);
const orphanIamRolesPromise = getOrphanTestIamRoles(account);

Expand All @@ -680,8 +749,9 @@ const cleanupAccount = async (account: AWSAccountInfo, accountIndex: number, fil
const buckets = await bucketPromise;
const orphanBuckets = await orphanBucketPromise;
const orphanIamRoles = await orphanIamRolesPromise;
const orphanPinpointApplications = (await Promise.all(orphanPinpointApplicationsPromise)).flat();

const allResources = mergeResourcesByCCIJob(apps, stacks, buckets, orphanBuckets, orphanIamRoles);
const allResources = mergeResourcesByCCIJob(apps, stacks, buckets, orphanBuckets, orphanIamRoles, orphanPinpointApplications);
const staleResources = _.pickBy(allResources, filterPredicate);

generateReport(staleResources);
Expand Down

0 comments on commit 091d1d6

Please sign in to comment.