Skip to content

Commit

Permalink
Generate all version results(Markdown) based on the data file on each…
Browse files Browse the repository at this point in the history 
… update run
  • Loading branch information
@acheroncrypto
acheroncrypto committed Apr 18, 2023
1 parent 376d492 commit 079a06b
Show file tree
Hide file tree
Showing 6 changed files with 167 additions and 137 deletions.
178 changes: 90 additions & 88 deletions bench/COMPUTE_UNITS.md
View file
Expand Up @@ -18,7 +18,7 @@ The results documented in this file are autogenerated. Running the tests will up
| accountInfo8 | 3856 | - |
| accountEmptyInit1 | 5958 | - |
| accountEmpty1 | 1090 | - |
| accountEmptyInit2 | 10574 | - |
| accountEmptyInit2 | 10583 | - |
| accountEmpty2 | 1852 | - |
| accountEmptyInit4 | 19557 | - |
| accountEmpty4 | 2646 | - |
Expand Down Expand Up @@ -108,90 +108,92 @@ The results documented in this file are autogenerated. Running the tests will up

| Instruction | Compute Units | +/- |
| --------------------------- | ------------- | --- |
| accountInfo1 | 954 | - |
| accountInfo2 | 1567 | - |
| accountInfo4 | 2059 | - |
| accountInfo8 | 3856 | - |
| accountEmptyInit1 | 5958 | - |
| accountEmpty1 | 1090 | - |
| accountEmptyInit2 | 10574 | - |
| accountEmpty2 | 1852 | - |
| accountEmptyInit4 | 19557 | - |
| accountEmpty4 | 2646 | - |
| accountEmptyInit8 | 37541 | - |
| accountEmpty8 | 5043 | - |
| accountSizedInit1 | 6063 | - |
| accountSized1 | 1135 | - |
| accountSizedInit2 | 10783 | - |
| accountSized2 | 1966 | - |
| accountSizedInit4 | 19975 | - |
| accountSized4 | 2787 | - |
| accountSizedInit8 | 38381 | - |
| accountSized8 | 5359 | - |
| accountUnsizedInit1 | 6193 | - |
| accountUnsized1 | 1243 | - |
| accountUnsizedInit2 | 11042 | - |
| accountUnsized2 | 1893 | - |
| accountUnsizedInit4 | 20495 | - |
| accountUnsized4 | 3104 | - |
| accountUnsizedInit8 | 39419 | - |
| accountUnsized8 | 6051 | - |
| boxedAccountEmptyInit1 | 6160 | - |
| boxedAccountEmpty1 | 976 | - |
| boxedAccountEmptyInit2 | 10784 | - |
| boxedAccountEmpty2 | 1499 | - |
| boxedAccountEmptyInit4 | 19500 | - |
| boxedAccountEmpty4 | 2530 | - |
| boxedAccountEmptyInit8 | 37415 | - |
| boxedAccountEmpty8 | 4780 | - |
| boxedAccountSizedInit1 | 6256 | - |
| boxedAccountSized1 | 1003 | - |
| boxedAccountSizedInit2 | 10975 | - |
| boxedAccountSized2 | 1554 | - |
| boxedAccountSizedInit4 | 19884 | - |
| boxedAccountSized4 | 2642 | - |
| boxedAccountSizedInit8 | 38182 | - |
| boxedAccountSized8 | 5003 | - |
| boxedAccountUnsizedInit1 | 6374 | - |
| boxedAccountUnsized1 | 1069 | - |
| boxedAccountUnsizedInit2 | 11211 | - |
| boxedAccountUnsized2 | 1679 | - |
| boxedAccountUnsizedInit4 | 20351 | - |
| boxedAccountUnsized4 | 2899 | - |
| boxedAccountUnsizedInit8 | 39118 | - |
| boxedAccountUnsized8 | 5517 | - |
| boxedInterfaceAccountMint1 | 2299 | - |
| boxedInterfaceAccountMint2 | 4053 | - |
| boxedInterfaceAccountMint4 | 7538 | - |
| boxedInterfaceAccountMint8 | 14699 | - |
| boxedInterfaceAccountToken1 | 1737 | - |
| boxedInterfaceAccountToken2 | 2928 | - |
| boxedInterfaceAccountToken4 | 5291 | - |
| boxedInterfaceAccountToken8 | 10205 | - |
| interfaceAccountMint1 | 2530 | - |
| interfaceAccountMint2 | 4726 | - |
| interfaceAccountMint4 | 9431 | - |
| interfaceAccountMint8 | 17709 | - |
| interfaceAccountToken1 | 1755 | - |
| interfaceAccountToken2 | 3211 | - |
| interfaceAccountToken4 | 6006 | - |
| interface1 | 999 | - |
| interface2 | 1574 | - |
| interface4 | 1996 | - |
| interface8 | 3651 | - |
| program1 | 999 | - |
| program2 | 1573 | - |
| program4 | 1998 | - |
| program8 | 3651 | - |
| signer1 | 958 | - |
| signer2 | 1576 | - |
| signer4 | 2079 | - |
| signer8 | 3895 | - |
| systemAccount1 | 1013 | - |
| systemAccount2 | 1686 | - |
| systemAccount4 | 2298 | - |
| systemAccount8 | 4336 | - |
| uncheckedAccount1 | 953 | - |
| uncheckedAccount2 | 1567 | - |
| uncheckedAccount4 | 2060 | - |
| uncheckedAccount8 | 3855 | - |
| accountInfo1 | 954 | N/A |
| accountInfo2 | 1567 | N/A |
| accountInfo4 | 2059 | N/A |
| accountInfo8 | 3856 | N/A |
| accountEmptyInit1 | 5958 | N/A |
| accountEmpty1 | 1090 | N/A |
| accountEmptyInit2 | 10574 | N/A |
| accountEmpty2 | 1852 | N/A |
| accountEmptyInit4 | 19557 | N/A |
| accountEmpty4 | 2646 | N/A |
| accountEmptyInit8 | 37541 | N/A |
| accountEmpty8 | 5043 | N/A |
| accountSizedInit1 | 6063 | N/A |
| accountSized1 | 1135 | N/A |
| accountSizedInit2 | 10783 | N/A |
| accountSized2 | 1966 | N/A |
| accountSizedInit4 | 19975 | N/A |
| accountSized4 | 2787 | N/A |
| accountSizedInit8 | 38381 | N/A |
| accountSized8 | 5359 | N/A |
| accountUnsizedInit1 | 6193 | N/A |
| accountUnsized1 | 1243 | N/A |
| accountUnsizedInit2 | 11042 | N/A |
| accountUnsized2 | 1893 | N/A |
| accountUnsizedInit4 | 20495 | N/A |
| accountUnsized4 | 3104 | N/A |
| accountUnsizedInit8 | 39419 | N/A |
| accountUnsized8 | 6051 | N/A |
| boxedAccountEmptyInit1 | 6160 | N/A |
| boxedAccountEmpty1 | 976 | N/A |
| boxedAccountEmptyInit2 | 10784 | N/A |
| boxedAccountEmpty2 | 1499 | N/A |
| boxedAccountEmptyInit4 | 19500 | N/A |
| boxedAccountEmpty4 | 2530 | N/A |
| boxedAccountEmptyInit8 | 37415 | N/A |
| boxedAccountEmpty8 | 4780 | N/A |
| boxedAccountSizedInit1 | 6256 | N/A |
| boxedAccountSized1 | 1003 | N/A |
| boxedAccountSizedInit2 | 10975 | N/A |
| boxedAccountSized2 | 1554 | N/A |
| boxedAccountSizedInit4 | 19884 | N/A |
| boxedAccountSized4 | 2642 | N/A |
| boxedAccountSizedInit8 | 38182 | N/A |
| boxedAccountSized8 | 5003 | N/A |
| boxedAccountUnsizedInit1 | 6374 | N/A |
| boxedAccountUnsized1 | 1069 | N/A |
| boxedAccountUnsizedInit2 | 11211 | N/A |
| boxedAccountUnsized2 | 1679 | N/A |
| boxedAccountUnsizedInit4 | 20351 | N/A |
| boxedAccountUnsized4 | 2899 | N/A |
| boxedAccountUnsizedInit8 | 39118 | N/A |
| boxedAccountUnsized8 | 5517 | N/A |
| boxedInterfaceAccountMint1 | 2299 | N/A |
| boxedInterfaceAccountMint2 | 4053 | N/A |
| boxedInterfaceAccountMint4 | 7538 | N/A |
| boxedInterfaceAccountMint8 | 14699 | N/A |
| boxedInterfaceAccountToken1 | 1737 | N/A |
| boxedInterfaceAccountToken2 | 2928 | N/A |
| boxedInterfaceAccountToken4 | 5291 | N/A |
| boxedInterfaceAccountToken8 | 10205 | N/A |
| interfaceAccountMint1 | 2530 | N/A |
| interfaceAccountMint2 | 4726 | N/A |
| interfaceAccountMint4 | 9431 | N/A |
| interfaceAccountMint8 | 17709 | N/A |
| interfaceAccountToken1 | 1755 | N/A |
| interfaceAccountToken2 | 3211 | N/A |
| interfaceAccountToken4 | 6006 | N/A |
| interface1 | 999 | N/A |
| interface2 | 1574 | N/A |
| interface4 | 1996 | N/A |
| interface8 | 3651 | N/A |
| program1 | 999 | N/A |
| program2 | 1573 | N/A |
| program4 | 1998 | N/A |
| program8 | 3651 | N/A |
| signer1 | 958 | N/A |
| signer2 | 1576 | N/A |
| signer4 | 2079 | N/A |
| signer8 | 3895 | N/A |
| systemAccount1 | 1013 | N/A |
| systemAccount2 | 1686 | N/A |
| systemAccount4 | 2298 | N/A |
| systemAccount8 | 4336 | N/A |
| uncheckedAccount1 | 953 | N/A |
| uncheckedAccount2 | 1567 | N/A |
| uncheckedAccount4 | 2060 | N/A |
| uncheckedAccount8 | 3855 | N/A |

---
2 changes: 1 addition & 1 deletion tests/bench/README.md
View file
Expand Up @@ -10,7 +10,7 @@ The script will check whether there is a difference between the current result a

## Scripts

`anchor test --skip-lint`: Run all tests. This is the only command that needs to be run for most use cases.
`anchor test --skip-lint`: Run all tests and update benchmark files when necessary. This is the only command that needs to be run for most use cases.

---

Expand Down
2 changes: 1 addition & 1 deletion tests/bench/bench.json
View file
Expand Up @@ -98,7 +98,7 @@
"accountInfo8": 3856,
"accountEmptyInit1": 5958,
"accountEmpty1": 1090,
"accountEmptyInit2": 10574,
"accountEmptyInit2": 10583,
"accountEmpty2": 1852,
"accountEmptyInit4": 19557,
"accountEmpty4": 2646,
Expand Down
85 changes: 55 additions & 30 deletions tests/bench/scripts/update-bench.ts
View file
Expand Up @@ -5,38 +5,63 @@ import { BenchData, Markdown } from "./utils";
(async () => {
const bench = await BenchData.open();

await BenchData.forEachMarkdown((markdown) => {
// Create table
const table = Markdown.createTable("Instruction", "Compute Units", "+/-");

const newComputeUnitsResult = bench.getUnreleased().computeUnits;
const oldComputeUnitsResult = bench.getLastVersionResult().computeUnits;
bench.compareComputeUnits(
newComputeUnitsResult,
oldComputeUnitsResult,
(ixName, newComputeUnits, oldComputeUnits) => {
const percentChange = (
(newComputeUnits / oldComputeUnits - 1) *
100
).toFixed(2);

let changeText;
if (isNaN(oldComputeUnits)) {
changeText = "N/A";
} else if (+percentChange > 0) {
changeText = `🔴 **+${percentChange}%**`;
} else {
changeText = `🟢 **${percentChange}%**`;
await BenchData.forEachMarkdown((markdown, fileName) => {
if (fileName === "COMPUTE_UNITS.md") {
const versions = bench.getVersions();

// On the first version, compare with itself to update it with no changes
versions.unshift(versions[0]);

for (const i in versions) {
const currentVersion = versions[i];
const nextVersion = versions[+i + 1];

if (currentVersion === "unreleased") {
return;
}

table.insert(ixName, newComputeUnits.toString(), changeText);
},
(ixName, newComputeUnits) => {
table.insert(ixName, newComputeUnits.toString(), "-");
}
);
const newComputeUnitsResult = bench.get(nextVersion).computeUnits;
const oldComputeUnitsResult = bench.get(currentVersion).computeUnits;

// Create table
const table = Markdown.createTable(
"Instruction",
"Compute Units",
"+/-"
);

// Update unreleased section
markdown.updateUnreleased(table);
bench.compareComputeUnits(
newComputeUnitsResult,
oldComputeUnitsResult,
(ixName, newComputeUnits, oldComputeUnits) => {
const percentChange = (
(newComputeUnits / oldComputeUnits - 1) *
100
).toFixed(2);

let changeText;
if (isNaN(oldComputeUnits)) {
changeText = "N/A";
} else if (+percentChange > 0) {
changeText = `🔴 **+${percentChange}%**`;
} else {
changeText = `🟢 **${percentChange}%**`;
}

table.insert(ixName, newComputeUnits.toString(), changeText);
},
(ixName, computeUnits) => {
table.insert(
ixName,
computeUnits.toString(),
+i === 0 ? "N/A" : "-"
);
}
);

// Update version's table
markdown.updateTable(nextVersion, table);
}
}
});
})();
31 changes: 17 additions & 14 deletions tests/bench/scripts/utils.ts
View file
Expand Up @@ -65,11 +65,9 @@ export class BenchData {
return this.get("unreleased");
}

/** Get the last released version result */
getLastVersionResult() {
const versions = Object.keys(this.#data);
const lastVersion = versions[versions.length - 2];
return this.get(lastVersion);
/** Get all versions */
getVersions() {
return Object.keys(this.#data);
}

/** Compare and update compute units changes */
Expand All @@ -81,7 +79,7 @@ export class BenchData {
newComputeUnits: number,
oldComputeUnits: number
) => void,
noChangeCb?: (ixName: string, newComputeUnits: number) => void
noChangeCb?: (ixName: string, computeUnits: number) => void
) {
let needsUpdate = false;

Expand Down Expand Up @@ -155,13 +153,15 @@ export class BenchData {
* Loop through all of the markdown files and run the given callback before
* saving the file.
*/
static async forEachMarkdown(cb: (markdown: Markdown) => void) {
static async forEachMarkdown(
cb: (markdown: Markdown, fileName: string) => void
) {
const fileNames = await fs.readdir(BENCH_DIR_PATH);
const markdownFileNames = fileNames.filter((n) => n.endsWith(".md"));

for (const fileName of markdownFileNames) {
const markdown = await Markdown.open(path.join(BENCH_DIR_PATH, fileName));
cb(markdown);
cb(markdown, fileName);
await markdown.save();
}

Expand Down Expand Up @@ -207,17 +207,20 @@ export class Markdown {
await fs.writeFile(this.#path, this.#text);
}

/** Change unreleased table with the given table */
updateUnreleased(table: MarkdownTable) {
/** Change version table with the given table */
updateTable(version: string, table: MarkdownTable) {
const md = this.#text;
const titleStartIndex = md.indexOf(Markdown.#UNRELEASED_VERSION);

let titleStartIndex = md.indexOf(`[${version}]`);
if (titleStartIndex === -1) {
titleStartIndex = md.indexOf(Markdown.#UNRELEASED_VERSION);
}

const startIndex = titleStartIndex + md.slice(titleStartIndex).indexOf("|");
const endIndex = startIndex + md.slice(startIndex).indexOf("\n\n");

this.#text =
this.#text.slice(0, startIndex) +
table.toString() +
this.#text.slice(endIndex + 1);
md.slice(0, startIndex) + table.toString() + md.slice(endIndex + 1);
}

/** Bump the version to the given version */
Expand Down
6 changes: 3 additions & 3 deletions tests/bench/tests/compute-units.ts
View file
Expand Up @@ -224,12 +224,12 @@ describe(IDL.name, () => {
const bench = await BenchData.open();

// Compare and update compute units changes
const unreleasedComputeUnits = bench.getUnreleased().computeUnits;
const oldComputeUnits = bench.getUnreleased().computeUnits;
const { needsUpdate } = bench.compareComputeUnits(
computeUnits,
unreleasedComputeUnits,
oldComputeUnits,
(ixName, newComputeUnits) => {
unreleasedComputeUnits[ixName] = newComputeUnits;
oldComputeUnits[ixName] = newComputeUnits;
}
);

Expand Down

0 comments on commit 079a06b

Please sign in to comment.