Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(enum-updater): add enum updater tool #33681

Open
wants to merge 17 commits into
base: main
Choose a base branch
from
Open

chore(enum-updater): add enum updater tool #33681

Changes from 1 commit
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
51189a5
chore(enum-updater): add enum updater tool
paulhcsun Feb 24, 2025
a2345c0
update parseCdk method to include enumLike flag
paulhcsun Feb 24, 2025
c490248
remove format matching for missing values, remove similar name check …
paulhcsun Feb 25, 2025
7c12d5b
Add simple enum-like update logic
Feb 25, 2025
967c457
Merge branch 'enum-updater-tool' of https://github.com/paulhcsun/aws-…
Feb 25, 2025
9b9f6dc
Ensure the replacement logic works properly
Feb 26, 2025
160f06c
Place values in the right place and comment them
Feb 26, 2025
8a27fbd
add logic for updating regular enums
paulhcsun Feb 26, 2025
e464a45
update spacing for code updates
paulhcsun Feb 27, 2025
fde6057
update to not add extra empty lines
paulhcsun Feb 27, 2025
6d600c7
extract code writing logic to enum-updater tool + add workflow file
paulhcsun Feb 28, 2025
0341da9
Cleanup indentation and whatnot
Feb 28, 2025
8226a3d
Merge branch 'aws:main' into enum-updater-tool
paulhcsun Feb 28, 2025
69716c9
refactor + add tests
paulhcsun Mar 4, 2025
eb169bb
add additional unit tests
paulhcsun Mar 4, 2025
48fdeb8
Merge branch 'main' into enum-updater-tool
xazhao Mar 6, 2025
e00c3f4
Change data source to cfn lint
xazhao Mar 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
update parseCdk method to include enumLike flag
  • Loading branch information
@paulhcsun
paulhcsun committed Feb 24, 2025
commit a2345c01f2ef8c1ed3d76d18a7d408d559288287
262 changes: 138 additions & 124 deletions tools/@aws-cdk/enum-updater/lib/enum-updater.ts
View file
Original file line number Diff line number Diff line change
@@ -10,7 +10,7 @@ const ENUM_LIKE_CLASSES_URL = "https://raw.githubusercontent.com/aws/aws-cdk/mai
const AWS_SDK_MODELS_URL = "https://github.com/awslabs/aws-sdk-rust/archive/refs/heads/main.zip";
const MODULE_MAPPING = path.join(LOCAL_DIR, "./lib/module_mapping.json");
const STATIC_MAPPING = path.join(LOCAL_DIR, "static-enum-mapping.json");
// const SDK_ENUMS = path.join(LOCAL_DIR, "sdk_enums.json");
const SDK_ENUMS = path.join(LOCAL_DIR, "sdk_enums.json");

// Set up cleanup handlers for process termination
tmp.setGracefulCleanup();
@@ -26,26 +26,27 @@ interface SdkEnums {
};
}

interface Shape {
type: string;
members?: {
[key: string]: {
traits?: {
'smithy.api#enumValue'?: string | number;
[key: string]: any;
};
[key: string]: any;
};
};
}

interface ParseResult {
path: string;
cleanup: () => Promise<void>;
}
// interface Shape {
// type: string;
// members?: {
// [key: string]: {
// traits?: {
// 'smithy.api#enumValue'?: string | number;
// [key: string]: any;
// };
// [key: string]: any;
// };
// };
// }

// interface ParseResult {
// path: string;
// cleanup: () => Promise<void>;
// }

interface EnumValue {
path: string;
enumLike: boolean;
values: (string | number)[];
}

@@ -219,82 +220,82 @@ async function downloadGithubRawFile(url: string): Promise<DownloadResult> {
}
}

async function parseAwsSdkEnums(sdkModelsPath: string): Promise<ParseResult> {
const sdk_enums: SdkEnums = {};
// async function parseAwsSdkEnums(sdkModelsPath: string): Promise<ParseResult> {
// const sdk_enums: SdkEnums = {};

try {
const files = fs.readdirSync(sdkModelsPath);
const jsonFiles = files.filter(file => file.endsWith('.json'));

for (const file of jsonFiles) {
try {
const fileContent = fs.readFileSync(path.join(sdkModelsPath, file), 'utf8');
const jsonData = JSON.parse(fileContent);

if ("shapes" in jsonData) {
for (const [shapeName, shape] of Object.entries(jsonData.shapes)) {
if (!shape || typeof shape !== 'object') continue;

const typedShape = shape as Shape;
if (typedShape.type !== 'enum' || !typedShape.members) continue;

try {
const serviceName = shapeName.split('.')[2].split('#')[0];
const enumName = shapeName.split('#')[1];

if (!serviceName || !enumName) {
console.warn(`Warning: Invalid shape name format: ${shapeName}`);
continue;
}

const enumValues = Object.entries(typedShape.members)
.filter(([_, memberData]) =>
memberData.traits &&
'smithy.api#enumValue' in memberData.traits &&
memberData.traits['smithy.api#enumValue'] !== undefined)
.map(([_, memberData]) => memberData.traits!['smithy.api#enumValue']!);

if (enumValues.length) {
if (!sdk_enums[serviceName]) {
sdk_enums[serviceName] = {};
}
if (!sdk_enums[serviceName][enumName]) {
sdk_enums[serviceName][enumName] = enumValues;
}
}
} catch (error) {
console.warn(`Warning: Error processing shape ${shapeName}:`, error);
continue;
}
}
}
} catch (error) {
console.warn(`Warning: Error processing file ${file}:`, error);
continue;
}
}

const tmpFile = tmp.fileSync({ postfix: '.json' });
fs.writeFileSync(tmpFile.name, JSON.stringify(sdk_enums, null, 2));

// ===== BEGIN LOCAL FILE WRITING =====
// TODO: Remove this section after testing
fs.writeFileSync('sdk_enums.json', JSON.stringify(sdk_enums, null, 2));
console.log('Wrote SDK enums to sdk_enums.json');
// ===== END LOCAL FILE WRITING =====

return {
path: tmpFile.name,
cleanup: async () => {
tmpFile.removeCallback();
}
};

} catch (error) {
console.error('Error parsing SDK enums:', error);
throw error;
}
}
// try {
// const files = fs.readdirSync(sdkModelsPath);
// const jsonFiles = files.filter(file => file.endsWith('.json'));

// for (const file of jsonFiles) {
// try {
// const fileContent = fs.readFileSync(path.join(sdkModelsPath, file), 'utf8');
// const jsonData = JSON.parse(fileContent);

// if ("shapes" in jsonData) {
// for (const [shapeName, shape] of Object.entries(jsonData.shapes)) {
// if (!shape || typeof shape !== 'object') continue;

// const typedShape = shape as Shape;
// if (typedShape.type !== 'enum' || !typedShape.members) continue;

// try {
// const serviceName = shapeName.split('.')[2].split('#')[0];
// const enumName = shapeName.split('#')[1];

// if (!serviceName || !enumName) {
// console.warn(`Warning: Invalid shape name format: ${shapeName}`);
// continue;
// }

// const enumValues = Object.entries(typedShape.members)
// .filter(([_, memberData]) =>
// memberData.traits &&
// 'smithy.api#enumValue' in memberData.traits &&
// memberData.traits['smithy.api#enumValue'] !== undefined)
// .map(([_, memberData]) => memberData.traits!['smithy.api#enumValue']!);

// if (enumValues.length) {
// if (!sdk_enums[serviceName]) {
// sdk_enums[serviceName] = {};
// }
// if (!sdk_enums[serviceName][enumName]) {
// sdk_enums[serviceName][enumName] = enumValues;
// }
// }
// } catch (error) {
// console.warn(`Warning: Error processing shape ${shapeName}:`, error);
// continue;
// }
// }
// }
// } catch (error) {
// console.warn(`Warning: Error processing file ${file}:`, error);
// continue;
// }
// }

// const tmpFile = tmp.fileSync({ postfix: '.json' });
// fs.writeFileSync(tmpFile.name, JSON.stringify(sdk_enums, null, 2));

// // ===== BEGIN LOCAL FILE WRITING =====
// // TODO: Remove this section after testing
// fs.writeFileSync('sdk_enums.json', JSON.stringify(sdk_enums, null, 2));
// console.log('Wrote SDK enums to sdk_enums.json');
// // ===== END LOCAL FILE WRITING =====

// return {
// path: tmpFile.name,
// cleanup: async () => {
// tmpFile.removeCallback();
// }
// };

// } catch (error) {
// console.error('Error parsing SDK enums:', error);
// throw error;
// }
// }


function extractModuleName(path: string): string | null {
@@ -317,31 +318,17 @@ function extractModuleName(path: string): string | null {
}
}

async function processCdkEnums(inputFilePaths: string[]): Promise<ProcessResult> {

async function processCdkEnums(enumsFilePath: string, enumsLikeFilePath: string): Promise<ProcessResult> {
const processedData: CdkEnums = {};
let totalEnums = 0;

try {
for (const filePath of inputFilePaths) {
const rawData = JSON.parse(fs.readFileSync(filePath, 'utf8'));

for (const [path, enums] of Object.entries(rawData)) {
const moduleName = extractModuleName(path);
if (!moduleName) continue;
// Process the first file (enums)
await processFile(enumsFilePath, false);

if (!processedData[moduleName]) {
processedData[moduleName] = {};
}

for (const [enumName, values] of Object.entries(enums as Record<string, (string | number)[]>)) {
processedData[moduleName][enumName] = {
path,
values: values
};
totalEnums++;
}
}
}
// Process the second file (enum-like)
await processFile(enumsLikeFilePath, true);

// Create temporary file
const tmpFile = tmp.fileSync({ postfix: '.json' });
@@ -363,8 +350,31 @@ async function processCdkEnums(inputFilePaths: string[]): Promise<ProcessResult>
console.error('Error processing CDK enums:', error);
throw error;
}

async function processFile(filePath: string, isEnumLike: boolean) {
const rawData = JSON.parse(fs.readFileSync(filePath, 'utf8'));

for (const [path, enums] of Object.entries(rawData)) {
const moduleName = extractModuleName(path);
if (!moduleName) continue;

if (!processedData[moduleName]) {
processedData[moduleName] = {};
}

for (const [enumName, values] of Object.entries(enums as Record<string, (string | number)[]>)) {
processedData[moduleName][enumName] = {
path,
enumLike: isEnumLike,
values: values
};
totalEnums++;
}
}
}
}


function normalizeValue(value: string | number): string {
const strValue = String(value);
if (strValue.match(/^\d+$/)) {
@@ -734,30 +744,34 @@ export async function analyzeMissingEnumValues(

export async function entryMethod(): Promise<void> {
try {
const sdkDownload = await downloadAwsSdkModels(AWS_SDK_MODELS_URL);
const notTrue = false;
if (notTrue) {
await downloadAwsSdkModels(AWS_SDK_MODELS_URL);
}
// const sdkDownload = await downloadAwsSdkModels(AWS_SDK_MODELS_URL);
const downloadedCdkEnumsPath = await downloadGithubRawFile(ENUMS_URL);
const downloadedCdkEnumsLikePath = await downloadGithubRawFile(ENUM_LIKE_CLASSES_URL);

if (downloadedCdkEnumsPath.path && downloadedCdkEnumsLikePath.path && sdkDownload.path) {
// if (downloadedCdkEnumsPath.path && downloadedCdkEnumsLikePath.path) {
// if (downloadedCdkEnumsPath.path && downloadedCdkEnumsLikePath.path && sdkDownload.path) {
if (downloadedCdkEnumsPath.path && downloadedCdkEnumsLikePath.path) {
console.log("CDK enums downloaded successfully.");

const cleanedCdkEnumsFilePath = await processCdkEnums([downloadedCdkEnumsPath.path, downloadedCdkEnumsLikePath.path]);
const cleanedCdkEnumsFilePath = await processCdkEnums(downloadedCdkEnumsPath.path, downloadedCdkEnumsLikePath.path);

console.log("SDK models downloaded successfully.");

const sdkEnumsParsedPath = await parseAwsSdkEnums(sdkDownload.path);
// const sdkEnumsParsedPath = await parseAwsSdkEnums(sdkDownload.path);

if (cleanedCdkEnumsFilePath.path && sdkEnumsParsedPath.path) {
// if (cleanedCdkEnumsFilePath.path) {
// if (cleanedCdkEnumsFilePath.path && sdkEnumsParsedPath.path) {
if (cleanedCdkEnumsFilePath.path) {
// Read the files
const cdkEnums: CdkEnums = JSON.parse(
fs.readFileSync(cleanedCdkEnumsFilePath.path, 'utf8')
);

const sdkEnums: SdkEnums = JSON.parse(
fs.readFileSync(sdkEnumsParsedPath.path, 'utf8')
// fs.readFileSync(SDK_ENUMS, 'utf8')
// fs.readFileSync(sdkEnumsParsedPath.path, 'utf8')
fs.readFileSync(SDK_ENUMS, 'utf8')
);

const manualMappings: Record<string, string[]> = JSON.parse(
@@ -768,8 +782,8 @@ export async function entryMethod(): Promise<void> {
await saveStaticMapping(cdkEnums, sdkEnums, manualMappings);

// Analyze missing values
await analyzeMissingEnumValues(STATIC_MAPPING, cleanedCdkEnumsFilePath.path, sdkEnumsParsedPath.path);
// await analyzeMissingEnumValues(STATIC_MAPPING, cleanedCdkEnumsFilePath.path, SDK_ENUMS);
// await analyzeMissingEnumValues(STATIC_MAPPING, cleanedCdkEnumsFilePath.path, sdkEnumsParsedPath.path);
await analyzeMissingEnumValues(STATIC_MAPPING, cleanedCdkEnumsFilePath.path, SDK_ENUMS);

// TODO TODO TODO TODO TODO FUCK FUCK FUCK
// try removing the format pattern matching cuz it sucks and there are like 0 missing values....
Loading
Oops, something went wrong.