From b557a443a5931da81969a79ed76ec7918537dd1d Mon Sep 17 00:00:00 2001 From: Nicolas Vasilache Date: Mon, 12 Jun 2023 08:33:19 +0000 Subject: [PATCH 1/6] Add a new unaligned matmul test that will exercise failsafes to avoid bad configurations For future reference, here are the steps required to add a new matmul benchmark to IREE: Step 1. Add a new entry to model_groups.py Step 2. Generate uuid with python: ``` python import uuid uuid.uuid4() ``` Step 3. Add an entry and plug UUID in build_tools/python/e2e_test_framework/unique_ids.py Step 4. Run echo "$(date +'%Y%m%d')_$(date +'%s')" to get a date + timestamp. Step 5. Add an entry to build_tools/python/e2e_test_framework/models/matmul.py using the date + timestamp directory name. Step 6. Write the desired .mlir and generate an a .mlirbc with `iree-opt --emit-bytecode` Step 7. Upload the .mlirbc to the GCS directory `https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/timestamp/` with timestamp created in Step 5. Step 8. Run `build_tools/scripts/generate_cmake_files.sh` Commit everything. --- build_tools/python/e2e_test_framework/models/model_groups.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_tools/python/e2e_test_framework/models/model_groups.py b/build_tools/python/e2e_test_framework/models/model_groups.py index 0d08c21ac122..d3a0b2a82bb4 100644 --- a/build_tools/python/e2e_test_framework/models/model_groups.py +++ b/build_tools/python/e2e_test_framework/models/model_groups.py @@ -127,6 +127,7 @@ matmul.MATMUL_2564x2564x2564_FP32_MLIR, matmul.MATMUL_2562x2564x2562_FP32_MLIR, matmul.MATMUL_2562x2561x2561_FP32_MLIR, + matmul.MATMUL_123x2561x2561_FP32_MLIR, ] MICRO_MATMUL_SPLITK = [ From a3a1c838fadbbbfea9efc962eded14d4c6050de9 Mon Sep 17 00:00:00 2001 From: Nicolas Vasilache Date: Mon, 12 Jun 2023 09:13:06 +0000 Subject: [PATCH 2/6] Step 3. generate UUID and plug build_tools/python/e2e_test_framework/unique_ids.py --- build_tools/python/e2e_test_framework/unique_ids.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_tools/python/e2e_test_framework/unique_ids.py b/build_tools/python/e2e_test_framework/unique_ids.py index befc6bac4b34..7589f4fb8893 100644 --- a/build_tools/python/e2e_test_framework/unique_ids.py +++ b/build_tools/python/e2e_test_framework/unique_ids.py @@ -125,6 +125,7 @@ def hash_composite_id(keys: Sequence[str]) -> str: MICRO_MATMUL_2564x2564x2564_FP32_MLIR = "4e75ff72-f807-49f6-b740-febca1794334" MICRO_MATMUL_2562x2564x2562_FP32_MLIR = "8d6be288-9b88-41bd-bc5a-5644df0481bb" MICRO_MATMUL_2562x2561x2561_FP32_MLIR = "0a3d952b-41ca-43d2-9ec3-ccb8dde20ce3" +MICRO_MATMUL_123x2561x2561_FP32_MLIR = "9b81a2f5-bab2-4f6f-b8b5-13cc4b3e9413" # Model input data MODEL_INPUT_DATA_ZEROS = "8d4a034e-944d-4725-8402-d6f6e61be93c" From 59a21094fa0438e26b32b249a2bb761fea49283c Mon Sep 17 00:00:00 2001 From: Nicolas Vasilache Date: Mon, 12 Jun 2023 09:47:36 +0000 Subject: [PATCH 3/6] Step 5. Add entry to matmul.py --- build_tools/python/e2e_test_framework/models/matmul.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/build_tools/python/e2e_test_framework/models/matmul.py b/build_tools/python/e2e_test_framework/models/matmul.py index a0791017d9ea..16a635fe5905 100644 --- a/build_tools/python/e2e_test_framework/models/matmul.py +++ b/build_tools/python/e2e_test_framework/models/matmul.py @@ -97,3 +97,13 @@ "https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230525_1685058259/matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc", entry_function="matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default", input_types=["2562x2561xf32", "2561x2561xf32", "2562x2561xf32"]) + +MATMUL_123x2561x2561_FP32_MLIR = common_definitions.Model( + id=unique_ids.MICRO_MATMUL_123x2561x2561_FP32_MLIR, + name="matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default", + tags=["fp32", "ubench", "matmul"], + source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, + source_url= + "https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230612_1686563210/matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc", + entry_function="matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default", + input_types=["123x2561xf32", "2561x2561xf32", "123x2561xf32"]) From bd62eac96a66c2fa784d0616f87dc37394000711 Mon Sep 17 00:00:00 2001 From: Nicolas Vasilache Date: Mon, 12 Jun 2023 09:55:13 +0000 Subject: [PATCH 4/6] Step 8. Run build_tools/scripts/generate_cmake_files.sh --- .../generated_e2e_test_fetch_models.cmake | 7 ++++ .../generated_e2e_test_iree_artifacts.cmake | 33 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake b/tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake index 8293d814b102..a26f632ec14d 100644 --- a/tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake +++ b/tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake @@ -264,6 +264,13 @@ iree_fetch_artifact( UNPACK ) +iree_fetch_artifact( + NAME "model-9b81a2f5-bab2-4f6f-b8b5-13cc4b3e9413" + SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230612_1686563210/matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc" + OUTPUT "${ROOT_ARTIFACTS_DIR}/model_9b81a2f5-bab2-4f6f-b8b5-13cc4b3e9413_matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc" + UNPACK +) + iree_fetch_artifact( NAME "model-699fd533-02a9-49f0-bf26-1902d8dbb5af" SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230410_1681181224/matmul_128x256x8192_f16t_f16t_f16t_tile_config_default.mlirbc" diff --git a/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake b/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake index 6dcf3869e013..6d20c7bec995 100644 --- a/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake +++ b/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake @@ -815,6 +815,19 @@ iree_bytecode_module( PUBLIC ) +iree_bytecode_module( + NAME "iree-module-bd985950d72dbe45dfcaa1bf4e0a438621642c077226cb42242ec091f8d37b0f" + SRC "${ROOT_ARTIFACTS_DIR}/model_9b81a2f5-bab2-4f6f-b8b5-13cc4b3e9413_matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc" + MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default_module_bd985950d72dbe45dfcaa1bf4e0a438621642c077226cb42242ec091f8d37b0f/module.vmfb" + FLAGS + "--iree-hal-target-backends=cuda" + "--iree-input-type=none" + "--iree-hal-cuda-llvm-target-arch=sm_80" + "--iree-hal-benchmark-dispatch-repeat-count=100" + FRIENDLY_NAME "matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul]" + PUBLIC +) + iree_bytecode_module( NAME "iree-module-e6503d42c202eff85bcddb53e9869017943e756af09bfe2c05ffc27844ac77e9" SRC "${ROOT_ARTIFACTS_DIR}/model_699fd533-02a9-49f0-bf26-1902d8dbb5af_matmul_128x256x8192_f16t_tile_config_default.mlirbc" @@ -3745,6 +3758,23 @@ iree_bytecode_module( PUBLIC ) +iree_bytecode_module( + NAME "iree-module-4d3bfb94dfe03a269350ad28fb9b1fa8db8c88ff19e2535cef09aafdbdfd83f5" + SRC "${ROOT_ARTIFACTS_DIR}/model_9b81a2f5-bab2-4f6f-b8b5-13cc4b3e9413_matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc" + MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default_module_4d3bfb94dfe03a269350ad28fb9b1fa8db8c88ff19e2535cef09aafdbdfd83f5/module.vmfb" + FLAGS + "--iree-hal-target-backends=cuda" + "--iree-input-type=none" + "--iree-hal-cuda-llvm-target-arch=sm_80" + "--iree-hal-benchmark-dispatch-repeat-count=100" + "--iree-vm-emit-polyglot-zip=true" + "--iree-llvmcpu-debug-symbols=false" + "--iree-scheduling-dump-statistics-format=json" + "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default_module_4d3bfb94dfe03a269350ad28fb9b1fa8db8c88ff19e2535cef09aafdbdfd83f5/scheduling_stats.json" + FRIENDLY_NAME "matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul,compile-stats]" + PUBLIC +) + iree_bytecode_module( NAME "iree-module-371a68ee85e41abf2d2a2847fe6c9f85fdc51f76516d74e3576b9d72670c1a0b" SRC "${ROOT_ARTIFACTS_DIR}/model_699fd533-02a9-49f0-bf26-1902d8dbb5af_matmul_128x256x8192_f16t_tile_config_default.mlirbc" @@ -6336,6 +6366,7 @@ add_dependencies(iree-benchmark-import-models ${PACKAGE_NAME}_model-8871f602-571c-4eb8-b94d-554cc8ceec5a ${PACKAGE_NAME}_model-8d6be288-9b88-41bd-bc5a-5644df0481bb ${PACKAGE_NAME}_model-9a9515c7-cb68-4c34-b1d2-0e8c0a3620b8 + ${PACKAGE_NAME}_model-9b81a2f5-bab2-4f6f-b8b5-13cc4b3e9413 ${PACKAGE_NAME}_model-a55afe1c-9410-47a6-b417-04b0d75ee5f4 ${PACKAGE_NAME}_model-a6c2b812-0a71-45e7-9ea5-f3d8529213ef ${PACKAGE_NAME}_model-c1be9a9d-64c3-4d88-8551-89a8b4f305ba-batch-1 @@ -6439,6 +6470,7 @@ add_dependencies(iree-benchmark-suites ${PACKAGE_NAME}_iree-module-baec9d4086496a94853f349354f87acb8397bf36169134d3269d5803888dcf49 ${PACKAGE_NAME}_iree-module-bb1cb8e00fd4cee513b2481bd5faf39842ad9c57f80f2e50ddb48763fd030721 ${PACKAGE_NAME}_iree-module-bd0ea10065a27dea6875ceb70f769e7e0f67a08e857920ca0d0322593441e9ed + ${PACKAGE_NAME}_iree-module-bd985950d72dbe45dfcaa1bf4e0a438621642c077226cb42242ec091f8d37b0f ${PACKAGE_NAME}_iree-module-bdd904cc5614ebf77609c7802a2dfc09f139aee2a247a247d10d320de72b0e28 ${PACKAGE_NAME}_iree-module-c0ea83b865fd0c67a3013086ef2250464e2aecfc615a95170ef46128d2e57208 ${PACKAGE_NAME}_iree-module-c37bfe2bb995b2703170a890bd81e372b687bed57087b1d8a6d8bb16b91c5ad4 @@ -6673,6 +6705,7 @@ add_dependencies(iree-e2e-compile-stats-suites ${PACKAGE_NAME}_iree-module-480b59f233af720e16db8e5da1988a8d69bd61169bf5b5899f425ff98dc0dc19 ${PACKAGE_NAME}_iree-module-4a3b570ba18c3c9eee458455aaff4aa29293a5c936a19862c698b4b3ddaf06e7 ${PACKAGE_NAME}_iree-module-4c74339076df00d23baa17dcb3194043e0472da9d09db4e42a23841ff7bf67b0 + ${PACKAGE_NAME}_iree-module-4d3bfb94dfe03a269350ad28fb9b1fa8db8c88ff19e2535cef09aafdbdfd83f5 ${PACKAGE_NAME}_iree-module-4ec47dd2b4a43dd434d041d4d9db548076b70cfd63a2fec2971035394954f1d5 ${PACKAGE_NAME}_iree-module-4f5ab4bfb26a82d0f83133b9e85585f0c5b97cdb00143de31675158a5a71b457 ${PACKAGE_NAME}_iree-module-50567a33e0bd9aa5a32a6f61fca9ef8a70ac4d94313024f2c4ec92d9c543c599 From 5cfd16ceea79df3d72610a00d3d8dff642720b50 Mon Sep 17 00:00:00 2001 From: Nicolas Vasilache Date: Mon, 12 Jun 2023 09:57:04 +0000 Subject: [PATCH 5/6] [DoNotCommit] Put the mlir bytecode file somewhere --- ...2561_f32t_f32t_f32t_tile_config_default.mlirbc | Bin 0 -> 735 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc diff --git a/matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc b/matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc new file mode 100644 index 0000000000000000000000000000000000000000..f51d9779ccb54d03a835a9ff96b6f122aea10300 GIT binary patch literal 735 zcmb7C&1%~~5Z=*fvsD%$)J2G3f*ee7O%ngew7#S{^iU{Gz~s=9H{4q z5nzmzwJ1wEU53XV8;mFTYQ$l{nc!Rq5eGtyh2Zgey|$j>Ti`57g2efT^e~A5(f6qJ zlv#YT_+qi=2!I&aK@48hVKf9n$ObX+2r)ULN5DFdx_MbDEwegboINgvC+%Wnk@niB z=k47+?VXdqlG07lzernUOWJhe{O-h$VhDIAUn+CcFY}eDb8~a1=USc<-lE@ee+n(S zKeY}Vhccvy=>DKErNG>;2o$Frnfn=om@%IREF|oCL=k)^MhsuK7~1W6)EWvv0v9n5 zZ;lT=L Date: Mon, 12 Jun 2023 09:57:09 +0000 Subject: [PATCH 6/6] Revert "[DoNotCommit] Put the mlir bytecode file somewhere" This reverts commit 5cfd16ceea79df3d72610a00d3d8dff642720b50. --- ...2561_f32t_f32t_f32t_tile_config_default.mlirbc | Bin 735 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc diff --git a/matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc b/matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc deleted file mode 100644 index f51d9779ccb54d03a835a9ff96b6f122aea10300..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 735 zcmb7C&1%~~5Z=*fvsD%$)J2G3f*ee7O%ngew7#S{^iU{Gz~s=9H{4q z5nzmzwJ1wEU53XV8;mFTYQ$l{nc!Rq5eGtyh2Zgey|$j>Ti`57g2efT^e~A5(f6qJ zlv#YT_+qi=2!I&aK@48hVKf9n$ObX+2r)ULN5DFdx_MbDEwegboINgvC+%Wnk@niB z=k47+?VXdqlG07lzernUOWJhe{O-h$VhDIAUn+CcFY}eDb8~a1=USc<-lE@ee+n(S zKeY}Vhccvy=>DKErNG>;2o$Frnfn=om@%IREF|oCL=k)^MhsuK7~1W6)EWvv0v9n5 zZ;lT=L