From 4c96f577dd65de2d1038f2a9037b958a91a729fb Mon Sep 17 00:00:00 2001 From: anyangml <137014849+anyangml@users.noreply.github.com> Date: Thu, 14 Aug 2025 00:30:13 +0000 Subject: [PATCH] [create-pull-request] automated change --- lambench/metrics/results/barplot.json | 6 +- lambench/metrics/results/final_rankings.json | 6 +- lambench/metrics/results/radar.json | 6 +- lambench/metrics/results/results.json | 282 +++++++++---------- lambench/metrics/results/scatter.json | 6 +- 5 files changed, 153 insertions(+), 153 deletions(-) diff --git a/lambench/metrics/results/barplot.json b/lambench/metrics/results/barplot.json index 28b9594c..1205907d 100644 --- a/lambench/metrics/results/barplot.json +++ b/lambench/metrics/results/barplot.json @@ -5,7 +5,7 @@ "MACE-MP-0": 0.36, "MACE-MPA-0": 0.34, "Orb-v2": 0.22, - "Orb-v3": 0.19, + "Orb-v3": 0.18, "SevenNet-l3i5": 0.25, "SevenNet-MF-ompa": 0.25, "MatterSim-v1-5M": 0.26, @@ -17,7 +17,7 @@ "MACE-MP-0": 0.28, "MACE-MPA-0": 0.21, "Orb-v2": 0.25, - "Orb-v3": 0.15, + "Orb-v3": 0.18, "SevenNet-l3i5": 0.24, "SevenNet-MF-ompa": 0.17, "MatterSim-v1-5M": 0.18, @@ -29,7 +29,7 @@ "MACE-MP-0": 0.41, "MACE-MPA-0": 0.38, "Orb-v2": 0.29, - "Orb-v3": 0.25, + "Orb-v3": 0.29, "SevenNet-l3i5": 0.49, "SevenNet-MF-ompa": 0.34, "MatterSim-v1-5M": 0.41, diff --git a/lambench/metrics/results/final_rankings.json b/lambench/metrics/results/final_rankings.json index 113f5b7f..1b38c45c 100644 --- a/lambench/metrics/results/final_rankings.json +++ b/lambench/metrics/results/final_rankings.json @@ -8,10 +8,10 @@ }, { "Model": "Orb-v3", - "Generalizability-FF Error \u2193": 0.197, - "Generalizability-PC Error \u2193": 0.406, + "Generalizability-FF Error \u2193": 0.215, + "Generalizability-PC Error \u2193": 0.414, "Applicability-Instability \u2193": 0.0, - "Applicability-Efficiency \u2191": 0.39 + "Applicability-Efficiency \u2191": 0.396 }, { "Model": "DPA-2.4-7M", diff --git a/lambench/metrics/results/radar.json b/lambench/metrics/results/radar.json index ee7c3a62..8e893918 100644 --- a/lambench/metrics/results/radar.json +++ b/lambench/metrics/results/radar.json @@ -87,9 +87,9 @@ { "name": "Orb-v3", "value": [ - 0.81, - 0.85, - 0.75 + 0.8200000000000001, + 0.8200000000000001, + 0.71 ] }, { diff --git a/lambench/metrics/results/results.json b/lambench/metrics/results/results.json index 6037ebca..bfbdbeb7 100644 --- a/lambench/metrics/results/results.json +++ b/lambench/metrics/results/results.json @@ -1194,222 +1194,222 @@ }, "Orb-v3": { "generalizability_force_field_results": { - "HEA25_S": { - "energy_rmse": 2498.1, - "energy_mae": 1746.8, - "energy_rmse_natoms": 58.1, - "energy_mae_natoms": 41.0, - "force_rmse": 285.7, - "force_mae": 174.3, - "virial_rmse": 8473.6, - "virial_mae": 3772.6, - "virial_rmse_natoms": 204.7, - "virial_mae_natoms": 89.6 - }, "HPt_NC_2022": { - "energy_rmse": 225.4, - "energy_mae": 162.4, - "energy_rmse_natoms": 3.0, - "energy_mae_natoms": 2.2, - "force_rmse": 89.4, + "energy_rmse": 488.1, + "energy_mae": 364.2, + "energy_rmse_natoms": 6.8, + "energy_mae_natoms": 5.0, + "force_rmse": 92.6, "force_mae": 67.7, - "virial_rmse": 2189.4, - "virial_mae": 1310.8, - "virial_rmse_natoms": 34.7, - "virial_mae_natoms": 19.1 - }, - "MoS2": { - "energy_rmse": 298.6, - "energy_mae": 226.9, - "energy_rmse_natoms": 14.4, - "energy_mae_natoms": 10.7, - "force_rmse": 63.9, - "force_mae": 39.6, - "virial_rmse": 1936.3, - "virial_mae": 1092.3, - "virial_rmse_natoms": 76.1, - "virial_mae_natoms": 46.5 + "virial_rmse": 3558.0, + "virial_mae": 2173.2, + "virial_rmse_natoms": 51.2, + "virial_mae_natoms": 30.8 }, - "ANI": { - "energy_rmse": 330.4, - "energy_mae": 252.9, - "energy_rmse_natoms": 23.2, - "energy_mae_natoms": 16.7, - "force_rmse": 253.0, - "force_mae": 166.0, + "REANN_CO2_Ni100": { + "energy_rmse": 18129.6, + "energy_mae": 14775.2, + "energy_rmse_natoms": 464.9, + "energy_mae_natoms": 378.8, + "force_rmse": 1169.4, + "force_mae": 479.7, "virial_rmse": null, "virial_mae": null, "virial_rmse_natoms": null, "virial_mae_natoms": null }, - "REANN_CO2_Ni100": { - "energy_rmse": 11250.8, - "energy_mae": 9301.7, - "energy_rmse_natoms": 288.5, - "energy_mae_natoms": 238.5, - "force_rmse": 968.7, - "force_mae": 479.5, + "AIMD-Chig": { + "energy_rmse": 390.8, + "energy_mae": 298.1, + "energy_rmse_natoms": 2.4, + "energy_mae_natoms": 1.8, + "force_rmse": 200.2, + "force_mae": 143.2, "virial_rmse": null, "virial_mae": null, "virial_rmse_natoms": null, "virial_mae_natoms": null }, + "HEA25_S": { + "energy_rmse": 2433.0, + "energy_mae": 1616.6, + "energy_rmse_natoms": 56.4, + "energy_mae_natoms": 37.9, + "force_rmse": 324.2, + "force_mae": 185.1, + "virial_rmse": 8342.7, + "virial_mae": 4083.8, + "virial_rmse_natoms": 198.2, + "virial_mae_natoms": 96.8 + }, "HEA25_bulk": { - "energy_rmse": 2563.8, - "energy_mae": 1759.0, - "energy_rmse_natoms": 61.2, - "energy_mae_natoms": 42.5, - "force_rmse": 272.6, - "force_mae": 172.9, - "virial_rmse": 11002.9, - "virial_mae": 5275.0, - "virial_rmse_natoms": 265.9, - "virial_mae_natoms": 128.2 + "energy_rmse": 2993.5, + "energy_mae": 2037.7, + "energy_rmse_natoms": 71.0, + "energy_mae_natoms": 49.0, + "force_rmse": 330.6, + "force_mae": 189.9, + "virial_rmse": 10211.6, + "virial_mae": 5060.3, + "virial_rmse_natoms": 242.4, + "virial_mae_natoms": 122.1 }, - "AIMD-Chig": { - "energy_rmse": 373.9, - "energy_mae": 288.4, - "energy_rmse_natoms": 2.3, - "energy_mae_natoms": 1.7, - "force_rmse": 196.9, - "force_mae": 143.4, + "ANI": { + "energy_rmse": 287.0, + "energy_mae": 214.0, + "energy_rmse_natoms": 20.0, + "energy_mae_natoms": 14.1, + "force_rmse": 247.9, + "force_mae": 160.7, "virial_rmse": null, "virial_mae": null, "virial_rmse_natoms": null, "virial_mae_natoms": null }, - "Ca_batteries_CM2021": { - "energy_rmse": 244.9, - "energy_mae": 152.3, + "MoS2": { + "energy_rmse": 457.1, + "energy_mae": 372.8, + "energy_rmse_natoms": 20.4, + "energy_mae_natoms": 16.9, + "force_rmse": 167.4, + "force_mae": 115.4, + "virial_rmse": 3594.9, + "virial_mae": 2343.9, + "virial_rmse_natoms": 141.4, + "virial_mae_natoms": 100.1 + }, + "MD22": { + "energy_rmse": 180.8, + "energy_mae": 122.7, "energy_rmse_natoms": 2.8, - "energy_mae_natoms": 1.6, - "force_rmse": 153.5, - "force_mae": 104.8, + "energy_mae_natoms": 2.0, + "force_rmse": 163.9, + "force_mae": 123.1, "virial_rmse": null, "virial_mae": null, "virial_rmse_natoms": null, "virial_mae_natoms": null }, - "NequIP_NC_2022": { - "energy_rmse": 81.1, - "energy_mae": 61.5, - "energy_rmse_natoms": 0.6, - "energy_mae_natoms": 0.4, - "force_rmse": 78.2, - "force_mae": 55.0, + "Ca_batteries_CM2021": { + "energy_rmse": 235.0, + "energy_mae": 119.1, + "energy_rmse_natoms": 2.8, + "energy_mae_natoms": 1.3, + "force_rmse": 141.9, + "force_mae": 99.8, "virial_rmse": null, "virial_mae": null, "virial_rmse_natoms": null, "virial_mae_natoms": null }, - "MD22": { - "energy_rmse": 219.2, - "energy_mae": 149.5, - "energy_rmse_natoms": 3.4, - "energy_mae_natoms": 2.5, - "force_rmse": 170.1, - "force_mae": 127.6, + "NequIP_NC_2022": { + "energy_rmse": 92.8, + "energy_mae": 68.3, + "energy_rmse_natoms": 0.6, + "energy_mae_natoms": 0.5, + "force_rmse": 103.7, + "force_mae": 72.4, "virial_rmse": null, "virial_mae": null, "virial_rmse_natoms": null, "virial_mae_natoms": null }, "Cu_MgO_catalysts": { - "energy_rmse": 153.8, - "energy_mae": 121.5, - "energy_rmse_natoms": 2.2, - "energy_mae_natoms": 1.7, - "force_rmse": 81.1, - "force_mae": 53.2, + "energy_rmse": 142.9, + "energy_mae": 114.6, + "energy_rmse_natoms": 2.0, + "energy_mae_natoms": 1.6, + "force_rmse": 86.4, + "force_mae": 56.3, "virial_rmse": null, "virial_mae": null, "virial_rmse_natoms": null, "virial_mae_natoms": null }, "Si_ZEO22": { - "energy_rmse": 656.7, - "energy_mae": 214.1, - "energy_rmse_natoms": 6.6, - "energy_mae_natoms": 1.9, - "force_rmse": 176.2, - "force_mae": 29.0, + "energy_rmse": 664.0, + "energy_mae": 219.0, + "energy_rmse_natoms": 6.5, + "energy_mae_natoms": 2.0, + "force_rmse": 182.7, + "force_mae": 29.4, "virial_rmse": null, "virial_mae": null, "virial_rmse_natoms": null, "virial_mae_natoms": null }, "Weighted": { - "energy_mae": 344.5, - "energy_mae_natoms": 5.8, - "energy_rmse": 503.2, - "energy_rmse_natoms": 8.7, - "force_mae": 100.4, - "force_rmse": 171.0, - "virial_mae": 2310.4, - "virial_mae_natoms": 56.5, - "virial_rmse": 4458.8, - "virial_rmse_natoms": 109.5 + "energy_mae": 384.9, + "energy_mae_natoms": 6.6, + "energy_rmse": 572.4, + "energy_rmse_natoms": 9.8, + "force_mae": 113.3, + "force_rmse": 198.2, + "virial_mae": 3203.1, + "virial_mae_natoms": 77.7, + "virial_rmse": 5745.4, + "virial_rmse_natoms": 136.6 } }, "generalizability_domain_specific_results": { - "phonon_mdr": { - "mae_entropy": 32.38591728194254, - "mae_max_freq": 93.89442414761066, - "success_rate": 0.9945772243422374, - "mae_free_energy": 5.695154547210946, - "mae_heat_capacity": 15.914308827078742 - }, - "elastic": { - "MAE_G_VRH": 9.123557283879205, - "MAE_K_VRH": 12.216947037339477, - "success_rate": 1.0 + "wiggle150": { + "MAE": 11.921534596232886, + "RMSE": 12.893628847700404 }, "neb": { - "MAE_Ea": 1.8298036252970449, - "MAE_dE": 0.97203609582927, - "\u00f8_Transfer": 68.0, + "MAE_Ea": 2.2984248017860205, + "MAE_dE": 1.4704194238032444, + "\u00f8_Transfer": 61.71, "success_rate": 100.0, - "\u00f8_Desorption": 81.89, - "\u00f8_Dissociation": 65.82 - }, - "wiggle150": { - "MAE": 8.402569029444615, - "RMSE": 9.707347145470738 + "\u00f8_Desorption": 87.4, + "\u00f8_Dissociation": 72.15 }, "torsionnet": { - "MAE": 1.0834818993842432, - "MAEB": 1.587115880918382, - "RMSE": 1.5290812677070975, - "NABH_h": 296 + "MAE": 0.9879792522348037, + "MAEB": 1.4207570019586342, + "RMSE": 1.4191254209497088, + "NABH_h": 268 + }, + "elastic": { + "MAE_G_VRH": 9.749303912653655, + "MAE_K_VRH": 7.58211347508265, + "success_rate": 1.0 + }, + "phonon_mdr": { + "mae_entropy": 31.45324606722395, + "mae_max_freq": 72.06635756187076, + "success_rate": 1, + "mae_free_energy": 7.258615471769131, + "mae_heat_capacity": 13.612062983832525 } }, "applicability_results": { + "inference_efficiency": { + "average_time": 252.241111, + "standard_deviation": 32.266986, + "success_rate": 100.0 + }, "nve_md": { - "slope": 9e-06, + "slope": 5e-06, "steps": 10000.0, "momenta_diff": 3e-06, - "simulation_time": 199.693032, + "simulation_time": 182.441901, "success_rate": 1.0 - }, - "inference_efficiency": { - "average_time": 256.721357, - "standard_deviation": 33.334154, - "success_rate": 100.0 } }, "model": { - "model_name": "orb_v3_conservative_inf_omat", + "model_name": "orb_v3_conservative_inf_mpa", "model_family": "ORB", "model_type": "ASE", - "virtualenv": "registry.dp.tech/dptech/dp/native/prod-375/lambench:v2.10", + "virtualenv": "registry.dp.tech/dptech/dp/native/prod-375/lambench:v2.15", "model_metadata": { "pretty_name": "Orb-v3", "num_parameters": 25510582, - "date_added": "2025-04-22 00:00:00", + "date_added": "2025-08-12 00:00:00", "packages": { "orb-models": "0.5.2" }, - "model_description": "https://orbitalmaterials-public-models.s3.us-west-1.amazonaws.com/forcefields/orb-v3/orb-v3-conservative-inf-omat-20250404.ckpt" + "model_description": "https://orbitalmaterials-public-models.s3.us-west-1.amazonaws.com/forcefields/orb-v3/orb-v3-conservative-inf-mpa-20250404.ckpt" }, "show_direct_task": true, "show_finetune_task": false, diff --git a/lambench/metrics/results/scatter.json b/lambench/metrics/results/scatter.json index 995d6812..2d248a0f 100644 --- a/lambench/metrics/results/scatter.json +++ b/lambench/metrics/results/scatter.json @@ -43,9 +43,9 @@ "name": "Orb-v3", "family": "ORB", "nparams": 25510582, - "efficiency": 256.72, - "std": 33.33, - "generalizability error": 0.2 + "efficiency": 252.24, + "std": 32.27, + "generalizability error": 0.21 }, { "name": "SevenNet-l3i5",