From a5a82ef6a500a94bcff2f2df920d9c514a30d157 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Thu, 27 Apr 2023 08:05:09 -0700 Subject: [PATCH 01/56] apply mlagility to create ONNX model in CI Signed-off-by: jcwchen --- .github/workflows/create_mlagility.yml | 43 ++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/create_mlagility.yml diff --git a/.github/workflows/create_mlagility.yml b/.github/workflows/create_mlagility.yml new file mode 100644 index 000000000..42773fdf0 --- /dev/null +++ b/.github/workflows/create_mlagility.yml @@ -0,0 +1,43 @@ +# Create ONNX model from mlagility + +name: Linux CI + +on: + schedule: + - cron: '00 00 * * MON' + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.8'] + + steps: + - uses: actions/checkout@v3 + name: Checkout repo + - uses: conda-incubator/setup-miniconda@v2 + with: + miniconda-version: "latest" + activate-environment: mla + python-version: ${{ matrix.python-version }} + + - name: Install dependencies and mlagility + run: | + python -m pip install --upgrade pip + python -m pip install onnx onnxruntime requests py-cpuinfo + # Print CPU info for debugging ONNX Runtime inference difference + python -m cpuinfo + python -m pip install mlagility + python -m pip install transformers + git clone https://github.com/groq/mlagility.git + pip install -r groq/mlagility/models/requirements.txt + + - name: Run mlagility + run: | + sudo benchit models/torch_hub/alexnet.py --cache-dir .cache + ls .cache \ No newline at end of file From 9db5fb294424cf9f76da83f439a11da45b096309 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Thu, 27 Apr 2023 08:30:31 -0700 Subject: [PATCH 02/56] correct path Signed-off-by: jcwchen --- .github/workflows/create_mlagility.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create_mlagility.yml b/.github/workflows/create_mlagility.yml index 42773fdf0..00f2be55f 100644 --- a/.github/workflows/create_mlagility.yml +++ b/.github/workflows/create_mlagility.yml @@ -1,6 +1,6 @@ # Create ONNX model from mlagility -name: Linux CI +name: Create ONNX model from mlagility on: schedule: @@ -35,7 +35,7 @@ jobs: python -m pip install mlagility python -m pip install transformers git clone https://github.com/groq/mlagility.git - pip install -r groq/mlagility/models/requirements.txt + pip install -r mlagility/models/requirements.txt - name: Run mlagility run: | From 65c5d7ecfd1ae40d2d16ea19039943a1300c7964 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Thu, 27 Apr 2023 08:39:07 -0700 Subject: [PATCH 03/56] which benchit Signed-off-by: jcwchen --- .github/workflows/create_mlagility.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/create_mlagility.yml b/.github/workflows/create_mlagility.yml index 00f2be55f..8a31bd9d8 100644 --- a/.github/workflows/create_mlagility.yml +++ b/.github/workflows/create_mlagility.yml @@ -39,5 +39,6 @@ jobs: - name: Run mlagility run: | - sudo benchit models/torch_hub/alexnet.py --cache-dir .cache + which benchit + benchit models/torch_hub/alexnet.py --cache-dir .cache ls .cache \ No newline at end of file From d913609789af19d3e3ab6974d982280795ae9da7 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Thu, 27 Apr 2023 08:53:02 -0700 Subject: [PATCH 04/56] cd mlagility Signed-off-by: jcwchen --- .github/workflows/create_mlagility.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create_mlagility.yml b/.github/workflows/create_mlagility.yml index 8a31bd9d8..256b7d71a 100644 --- a/.github/workflows/create_mlagility.yml +++ b/.github/workflows/create_mlagility.yml @@ -39,6 +39,6 @@ jobs: - name: Run mlagility run: | - which benchit + cd mlagility benchit models/torch_hub/alexnet.py --cache-dir .cache ls .cache \ No newline at end of file From ccbaf7270e275f5e6271d70e70618aac8cd1b9e8 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Thu, 27 Apr 2023 09:36:37 -0700 Subject: [PATCH 05/56] pip install -e . Signed-off-by: jcwchen --- .github/workflows/create_mlagility.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create_mlagility.yml b/.github/workflows/create_mlagility.yml index 256b7d71a..e4c1b086f 100644 --- a/.github/workflows/create_mlagility.yml +++ b/.github/workflows/create_mlagility.yml @@ -32,10 +32,12 @@ jobs: python -m pip install onnx onnxruntime requests py-cpuinfo # Print CPU info for debugging ONNX Runtime inference difference python -m cpuinfo - python -m pip install mlagility + # or python -m pip install mlagility python -m pip install transformers git clone https://github.com/groq/mlagility.git - pip install -r mlagility/models/requirements.txt + cd mlagility + pip install -r models/requirements.txt + pip install -e . - name: Run mlagility run: | From b24081d87f0e769e30293e4535f8247bed0d545c Mon Sep 17 00:00:00 2001 From: Chun-Wei Chen Date: Thu, 27 Apr 2023 10:08:55 -0700 Subject: [PATCH 06/56] add models Signed-off-by: Chun-Wei Chen --- .github/workflows/create_mlagility.yml | 3 ++- .../onnx/alexnet_torch_hub_2891f54c-op14-base.onnx | 3 +++ .../onnx/alexnet_torch_hub_2891f54c-op14-opt-f16.onnx | 3 +++ .../onnx/alexnet_torch_hub_2891f54c-op14-opt.onnx | 3 +++ 4 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-base.onnx create mode 100644 mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-opt-f16.onnx create mode 100644 mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-opt.onnx diff --git a/.github/workflows/create_mlagility.yml b/.github/workflows/create_mlagility.yml index e4c1b086f..7217822c7 100644 --- a/.github/workflows/create_mlagility.yml +++ b/.github/workflows/create_mlagility.yml @@ -43,4 +43,5 @@ jobs: run: | cd mlagility benchit models/torch_hub/alexnet.py --cache-dir .cache - ls .cache \ No newline at end of file + ls .cache/alexnet_torch_hub_2891f54c/onnx/ + diff .cache/alexnet_torch_hub_2891f54c/onnx/ mlagility_models/alexnet_torch_hub_2891f54c/onnx/ diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-base.onnx b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-base.onnx new file mode 100644 index 000000000..592fb0a6a --- /dev/null +++ b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-base.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eca997cef9eab6553b5ccfdba2df2a0b8b964d4b3cb003e08232affbbc91434b +size 244407335 diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-opt-f16.onnx b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-opt-f16.onnx new file mode 100644 index 000000000..4ff79dbc5 --- /dev/null +++ b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-opt-f16.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14b36d9bd632cd11fa35ffc211c2fe4da648782b72ee677c6de462d4b6d98dfa +size 122206997 diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-opt.onnx b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-opt.onnx new file mode 100644 index 000000000..0283d24d9 --- /dev/null +++ b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-opt.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:597d578f833cf6e3f94d6b1d86e676ca32beb8606b39b5dcaf245f3069b931ba +size 244408687 From acf23269437fa17b477e5fdd08352fe0df24a276 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Thu, 27 Apr 2023 10:19:32 -0700 Subject: [PATCH 07/56] ../mlagility_models/alexnet_torch_hub_2891f54c/onnx/ Signed-off-by: jcwchen --- .github/workflows/create_mlagility.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create_mlagility.yml b/.github/workflows/create_mlagility.yml index 7217822c7..5a0dd16ab 100644 --- a/.github/workflows/create_mlagility.yml +++ b/.github/workflows/create_mlagility.yml @@ -44,4 +44,4 @@ jobs: cd mlagility benchit models/torch_hub/alexnet.py --cache-dir .cache ls .cache/alexnet_torch_hub_2891f54c/onnx/ - diff .cache/alexnet_torch_hub_2891f54c/onnx/ mlagility_models/alexnet_torch_hub_2891f54c/onnx/ + diff .cache/alexnet_torch_hub_2891f54c/onnx/ ../mlagility_models/alexnet_torch_hub_2891f54c/onnx/ From 2731ca417a45d07ac72c83a88bdf467b98758d63 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Thu, 27 Apr 2023 16:21:54 -0700 Subject: [PATCH 08/56] git diff --exit-code -- . Signed-off-by: jcwchen --- .github/workflows/create_mlagility.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create_mlagility.yml b/.github/workflows/create_mlagility.yml index 5a0dd16ab..509c99521 100644 --- a/.github/workflows/create_mlagility.yml +++ b/.github/workflows/create_mlagility.yml @@ -42,6 +42,7 @@ jobs: - name: Run mlagility run: | cd mlagility - benchit models/torch_hub/alexnet.py --cache-dir .cache - ls .cache/alexnet_torch_hub_2891f54c/onnx/ - diff .cache/alexnet_torch_hub_2891f54c/onnx/ ../mlagility_models/alexnet_torch_hub_2891f54c/onnx/ + benchit models/torch_hub/alexnet.py --cache-dir mlagility_models + ls mlagility_models/alexnet_torch_hub_2891f54c/onnx/ + cd mlagility_models/alexnet_torch_hub_2891f54c/onnx/ + git diff --exit-code -- . From 398b63dc732d60429a0fa3f0e6a7d2f3058b2e30 Mon Sep 17 00:00:00 2001 From: Chun-Wei Chen Date: Tue, 2 May 2023 09:58:46 -0700 Subject: [PATCH 09/56] add opset 17 models Signed-off-by: Chun-Wei Chen --- .../onnx/alexnet_torch_hub_2891f54c-op14-base.onnx | 3 --- .../onnx/alexnet_torch_hub_2891f54c-op14-opt-f16.onnx | 3 --- .../onnx/alexnet_torch_hub_2891f54c-op14-opt.onnx | 3 --- .../onnx/alexnet_torch_hub_2891f54c-op17-base.onnx | 3 +++ .../onnx/alexnet_torch_hub_2891f54c-op17-opt-f16.onnx | 3 +++ .../onnx/alexnet_torch_hub_2891f54c-op17-opt.onnx | 3 +++ ...terrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-base.onnx | 3 +++ ...rcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt-f16.onnx | 3 +++ ...sterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt.onnx | 3 +++ 9 files changed, 18 insertions(+), 9 deletions(-) delete mode 100644 mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-base.onnx delete mode 100644 mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-opt-f16.onnx delete mode 100644 mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-opt.onnx create mode 100644 mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-base.onnx create mode 100644 mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-opt-f16.onnx create mode 100644 mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-opt.onnx create mode 100644 mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-base.onnx create mode 100644 mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt-f16.onnx create mode 100644 mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt.onnx diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-base.onnx b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-base.onnx deleted file mode 100644 index 592fb0a6a..000000000 --- a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-base.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:eca997cef9eab6553b5ccfdba2df2a0b8b964d4b3cb003e08232affbbc91434b -size 244407335 diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-opt-f16.onnx b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-opt-f16.onnx deleted file mode 100644 index 4ff79dbc5..000000000 --- a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-opt-f16.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:14b36d9bd632cd11fa35ffc211c2fe4da648782b72ee677c6de462d4b6d98dfa -size 122206997 diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-opt.onnx b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-opt.onnx deleted file mode 100644 index 0283d24d9..000000000 --- a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op14-opt.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:597d578f833cf6e3f94d6b1d86e676ca32beb8606b39b5dcaf245f3069b931ba -size 244408687 diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-base.onnx b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-base.onnx new file mode 100644 index 000000000..baa5314d6 --- /dev/null +++ b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-base.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b0f1ccba5f3a1cdf982bbf80491c45728a7bc64c42efb81181aeaea18af3140 +size 244407335 diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-opt-f16.onnx b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-opt-f16.onnx new file mode 100644 index 000000000..3cf4efa7c --- /dev/null +++ b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-opt-f16.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b56551810a2719643f4c1927d6d82a528a111657d0f06eb81df6497691cc62e1 +size 122206997 diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-opt.onnx b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-opt.onnx new file mode 100644 index 000000000..425dc78a2 --- /dev/null +++ b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-opt.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74c1fcf865ee7d8298a98a8d34b69294a8b9f2656358508578cf9a55197b8d70 +size 244408687 diff --git a/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-base.onnx b/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-base.onnx new file mode 100644 index 000000000..250485802 --- /dev/null +++ b/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-base.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70518d488cd0762d973a7bd3875400b8c1a217b371e25a5a40a66be68bb0e827 +size 174895826 diff --git a/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt-f16.onnx b/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt-f16.onnx new file mode 100644 index 000000000..39465e4cb --- /dev/null +++ b/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt-f16.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2173eaac97929e544bf48b6a043aed08ca94330e5d18353010c10a52ceda290c +size 90029457 diff --git a/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt.onnx b/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt.onnx new file mode 100644 index 000000000..2ea679b2a --- /dev/null +++ b/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:619c82a1b1cd557300fdf164cebaafe0fb8c099d76010a3455ed7271c346286c +size 178614527 From f6c54a1e1f1051648b38610feec97f39800f6582 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Tue, 2 May 2023 10:01:06 -0700 Subject: [PATCH 10/56] add faster-rcnn in CIs Signed-off-by: jcwchen --- .github/workflows/create_mlagility.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/create_mlagility.yml b/.github/workflows/create_mlagility.yml index 509c99521..20cb76bf5 100644 --- a/.github/workflows/create_mlagility.yml +++ b/.github/workflows/create_mlagility.yml @@ -41,8 +41,10 @@ jobs: - name: Run mlagility run: | - cd mlagility - benchit models/torch_hub/alexnet.py --cache-dir mlagility_models - ls mlagility_models/alexnet_torch_hub_2891f54c/onnx/ + ZOO_OPSET_VERSION=17 + benchit mlagility/models/torch_hub/alexnet.py --cache-dir mlagility_models --onnx-opset $ZOO_OPSET_VERSION cd mlagility_models/alexnet_torch_hub_2891f54c/onnx/ - git diff --exit-code -- . + git diff --exit-code -- . + benchit mlagility/models/torchvision/fasterrcnn_resnet50_fpn_v2.py --cache-dir .cache --onnx-opset $ZOO_OPSET_VERSION + cd mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/ + git diff --exit-code -- . From 0c06ef3035e03eeb178ca0539df574b5cf8e0513 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Tue, 2 May 2023 10:20:29 -0700 Subject: [PATCH 11/56] cd Signed-off-by: jcwchen --- .github/workflows/create_mlagility.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/create_mlagility.yml b/.github/workflows/create_mlagility.yml index 20cb76bf5..10876d27f 100644 --- a/.github/workflows/create_mlagility.yml +++ b/.github/workflows/create_mlagility.yml @@ -45,6 +45,7 @@ jobs: benchit mlagility/models/torch_hub/alexnet.py --cache-dir mlagility_models --onnx-opset $ZOO_OPSET_VERSION cd mlagility_models/alexnet_torch_hub_2891f54c/onnx/ git diff --exit-code -- . + cd ~ benchit mlagility/models/torchvision/fasterrcnn_resnet50_fpn_v2.py --cache-dir .cache --onnx-opset $ZOO_OPSET_VERSION - cd mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/ + cd mlagility_models/torchvision/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/ git diff --exit-code -- . From ac5f5ec9b3e1ee948446cf13c3739f8980d22e69 Mon Sep 17 00:00:00 2001 From: Chun-Wei Chen Date: Tue, 2 May 2023 10:29:10 -0700 Subject: [PATCH 12/56] use 18 instead of 17 Signed-off-by: Chun-Wei Chen --- .../onnx/alexnet_torch_hub_2891f54c-op17-base.onnx | 3 --- .../onnx/alexnet_torch_hub_2891f54c-op17-opt-f16.onnx | 3 --- .../onnx/alexnet_torch_hub_2891f54c-op17-opt.onnx | 3 --- .../onnx/alexnet_torch_hub_2891f54c-op18-base.onnx | 3 +++ .../onnx/alexnet_torch_hub_2891f54c-op18-opt-f16.onnx | 3 +++ .../onnx/alexnet_torch_hub_2891f54c-op18-opt.onnx | 3 +++ ...terrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-base.onnx | 3 --- ...rcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt-f16.onnx | 3 --- ...sterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt.onnx | 3 --- ...terrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-base.onnx | 3 +++ ...rcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt-f16.onnx | 3 +++ ...sterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt.onnx | 3 +++ 12 files changed, 18 insertions(+), 18 deletions(-) delete mode 100644 mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-base.onnx delete mode 100644 mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-opt-f16.onnx delete mode 100644 mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-opt.onnx create mode 100644 mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-base.onnx create mode 100644 mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-opt-f16.onnx create mode 100644 mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-opt.onnx delete mode 100644 mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-base.onnx delete mode 100644 mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt-f16.onnx delete mode 100644 mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt.onnx create mode 100644 mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-base.onnx create mode 100644 mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt-f16.onnx create mode 100644 mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt.onnx diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-base.onnx b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-base.onnx deleted file mode 100644 index baa5314d6..000000000 --- a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-base.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2b0f1ccba5f3a1cdf982bbf80491c45728a7bc64c42efb81181aeaea18af3140 -size 244407335 diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-opt-f16.onnx b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-opt-f16.onnx deleted file mode 100644 index 3cf4efa7c..000000000 --- a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-opt-f16.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b56551810a2719643f4c1927d6d82a528a111657d0f06eb81df6497691cc62e1 -size 122206997 diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-opt.onnx b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-opt.onnx deleted file mode 100644 index 425dc78a2..000000000 --- a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op17-opt.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:74c1fcf865ee7d8298a98a8d34b69294a8b9f2656358508578cf9a55197b8d70 -size 244408687 diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-base.onnx b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-base.onnx new file mode 100644 index 000000000..354f9be58 --- /dev/null +++ b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-base.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bd1eb1e34c9c3e0501ccb1a9858cf2cc20e2360cf0b2e361a31c8cd0148bab6 +size 244407334 diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-opt-f16.onnx b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-opt-f16.onnx new file mode 100644 index 000000000..23fe189a2 --- /dev/null +++ b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-opt-f16.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:391424ab1935a6227de2f5229c64c1f41c3f203cab4fc29f7d953dd740bd1c98 +size 122206996 diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-opt.onnx b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-opt.onnx new file mode 100644 index 000000000..cf3101d6b --- /dev/null +++ b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-opt.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ee28997a1fb47a4af054bb1c9f3704afc88c7063ba833f527efd29ed3d8a686 +size 244408686 diff --git a/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-base.onnx b/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-base.onnx deleted file mode 100644 index 250485802..000000000 --- a/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-base.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:70518d488cd0762d973a7bd3875400b8c1a217b371e25a5a40a66be68bb0e827 -size 174895826 diff --git a/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt-f16.onnx b/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt-f16.onnx deleted file mode 100644 index 39465e4cb..000000000 --- a/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt-f16.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2173eaac97929e544bf48b6a043aed08ca94330e5d18353010c10a52ceda290c -size 90029457 diff --git a/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt.onnx b/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt.onnx deleted file mode 100644 index 2ea679b2a..000000000 --- a/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b-op17-opt.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:619c82a1b1cd557300fdf164cebaafe0fb8c099d76010a3455ed7271c346286c -size 178614527 diff --git a/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-base.onnx b/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-base.onnx new file mode 100644 index 000000000..a16ac0195 --- /dev/null +++ b/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-base.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90f0837b92091e1a4193d293c900ee6e7f8797a480311340f42201aebe28db51 +size 174896554 diff --git a/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt-f16.onnx b/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt-f16.onnx new file mode 100644 index 000000000..bcb94feb5 --- /dev/null +++ b/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt-f16.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2660379385e7d9f4f8400b17bc5040fee39574864619618acddc19662cb629af +size 90029794 diff --git a/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt.onnx b/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt.onnx new file mode 100644 index 000000000..cb614246a --- /dev/null +++ b/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ecad362e98e0b12bf6f9a360f01d581d40921b83e3cfe0863d9ce6437713e25 +size 178614864 From 1a583ec01efe6133e47169c8cc044f82d5fb9171 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Tue, 2 May 2023 16:10:04 -0700 Subject: [PATCH 13/56] test opset_version 18 Signed-off-by: jcwchen --- .github/workflows/create_mlagility.yml | 2 +- .../onnx/alexnet_torch_hub_2891f54c-op18-opt-f16.onnx | 3 --- .../onnx/alexnet_torch_hub_2891f54c-op18-opt.onnx | 3 --- ...rcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt-f16.onnx | 3 --- ...sterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt.onnx | 3 --- 5 files changed, 1 insertion(+), 13 deletions(-) delete mode 100644 mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-opt-f16.onnx delete mode 100644 mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-opt.onnx delete mode 100644 mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt-f16.onnx delete mode 100644 mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt.onnx diff --git a/.github/workflows/create_mlagility.yml b/.github/workflows/create_mlagility.yml index 10876d27f..ba7e1d374 100644 --- a/.github/workflows/create_mlagility.yml +++ b/.github/workflows/create_mlagility.yml @@ -41,7 +41,7 @@ jobs: - name: Run mlagility run: | - ZOO_OPSET_VERSION=17 + ZOO_OPSET_VERSION=18 benchit mlagility/models/torch_hub/alexnet.py --cache-dir mlagility_models --onnx-opset $ZOO_OPSET_VERSION cd mlagility_models/alexnet_torch_hub_2891f54c/onnx/ git diff --exit-code -- . diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-opt-f16.onnx b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-opt-f16.onnx deleted file mode 100644 index 23fe189a2..000000000 --- a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-opt-f16.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:391424ab1935a6227de2f5229c64c1f41c3f203cab4fc29f7d953dd740bd1c98 -size 122206996 diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-opt.onnx b/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-opt.onnx deleted file mode 100644 index cf3101d6b..000000000 --- a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-opt.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3ee28997a1fb47a4af054bb1c9f3704afc88c7063ba833f527efd29ed3d8a686 -size 244408686 diff --git a/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt-f16.onnx b/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt-f16.onnx deleted file mode 100644 index bcb94feb5..000000000 --- a/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt-f16.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2660379385e7d9f4f8400b17bc5040fee39574864619618acddc19662cb629af -size 90029794 diff --git a/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt.onnx b/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt.onnx deleted file mode 100644 index cb614246a..000000000 --- a/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-opt.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9ecad362e98e0b12bf6f9a360f01d581d40921b83e3cfe0863d9ce6437713e25 -size 178614864 From 1f1db90d9931444b01721fc25a6f415b6bf04738 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 3 May 2023 09:02:22 -0700 Subject: [PATCH 14/56] git diff subdirectory Signed-off-by: jcwchen --- .github/workflows/create_mlagility.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/create_mlagility.yml b/.github/workflows/create_mlagility.yml index ba7e1d374..ca5fd0f14 100644 --- a/.github/workflows/create_mlagility.yml +++ b/.github/workflows/create_mlagility.yml @@ -43,9 +43,6 @@ jobs: run: | ZOO_OPSET_VERSION=18 benchit mlagility/models/torch_hub/alexnet.py --cache-dir mlagility_models --onnx-opset $ZOO_OPSET_VERSION - cd mlagility_models/alexnet_torch_hub_2891f54c/onnx/ - git diff --exit-code -- . - cd ~ + git diff --exit-code -- mlagility_models/alexnet_torch_hub_2891f54c/onnx/ benchit mlagility/models/torchvision/fasterrcnn_resnet50_fpn_v2.py --cache-dir .cache --onnx-opset $ZOO_OPSET_VERSION - cd mlagility_models/torchvision/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/ - git diff --exit-code -- . + git diff --exit-code -- mlagility_models/torchvision/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/ From 500b54295f66a3d7e37e30903eb21e1e8a0a6c84 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 3 May 2023 09:08:54 -0700 Subject: [PATCH 15/56] torch==2.0.0 torchvision==0.15.1 Signed-off-by: jcwchen --- .github/workflows/create_mlagility.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/create_mlagility.yml b/.github/workflows/create_mlagility.yml index ca5fd0f14..3d20dad34 100644 --- a/.github/workflows/create_mlagility.yml +++ b/.github/workflows/create_mlagility.yml @@ -41,6 +41,7 @@ jobs: - name: Run mlagility run: | + pip install torch==2.0.0 torchvision==0.15.1 ZOO_OPSET_VERSION=18 benchit mlagility/models/torch_hub/alexnet.py --cache-dir mlagility_models --onnx-opset $ZOO_OPSET_VERSION git diff --exit-code -- mlagility_models/alexnet_torch_hub_2891f54c/onnx/ From ab889aed4f0ad666a45f6bbc485548d01a4279bf Mon Sep 17 00:00:00 2001 From: jcwchen Date: Thu, 4 May 2023 16:13:22 -0700 Subject: [PATCH 16/56] run_mlagility.py Signed-off-by: jcwchen --- .github/workflows/create_mlagility.yml | 6 +---- workflow_scripts/run_mlagility.py | 37 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 workflow_scripts/run_mlagility.py diff --git a/.github/workflows/create_mlagility.yml b/.github/workflows/create_mlagility.yml index 3d20dad34..03558ec58 100644 --- a/.github/workflows/create_mlagility.yml +++ b/.github/workflows/create_mlagility.yml @@ -42,8 +42,4 @@ jobs: - name: Run mlagility run: | pip install torch==2.0.0 torchvision==0.15.1 - ZOO_OPSET_VERSION=18 - benchit mlagility/models/torch_hub/alexnet.py --cache-dir mlagility_models --onnx-opset $ZOO_OPSET_VERSION - git diff --exit-code -- mlagility_models/alexnet_torch_hub_2891f54c/onnx/ - benchit mlagility/models/torchvision/fasterrcnn_resnet50_fpn_v2.py --cache-dir .cache --onnx-opset $ZOO_OPSET_VERSION - git diff --exit-code -- mlagility_models/torchvision/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/ + python workflow_scripts/run_mlagility.py \ No newline at end of file diff --git a/workflow_scripts/run_mlagility.py b/workflow_scripts/run_mlagility.py new file mode 100644 index 000000000..1a1b9f8a6 --- /dev/null +++ b/workflow_scripts/run_mlagility.py @@ -0,0 +1,37 @@ +import os.path as osp +from pathlib import Path +import subprocess +import sys + + +models_info = [ + # (script_path, model_name) + ("torch_hub/alexnet.py", "alexnet_torch_hub_2891f54c"), + ("torchvision/fasterrcnn_resnet50_fpn_v2.py", "fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48"), +] + +cwd_path = Path.cwd() +mlagility_root = "mlagility/models" +mlagility_models_dir = "mlagility_models" +ZOO_OPSET_VERSION = "18" + +errors = 0 + +for model_info in models_info: + try: + subprocess.run(["benchit", osp.join(mlagility_root, model_info[0]), "--cache-dir", mlagility_models_dir, "--onnx-opset", ZOO_OPSET_VERSION], + cwd=cwd_path, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + subprocess.run(["git", "diff", "--exit-code", "--", osp.join(mlagility_models_dir, model_info[1])], + cwd=cwd_path, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + print(f"Successfully checked {model_info[1]}.") + except: + errors += 1 + print(f"Failed to check {model_info[1]}.") + +if errors > 0: + print(f"All {len(models_info)} models have been checked, but {errors} model(s) failed") + sys.exit(1) +else: + print(f"All {len(models_info)} models have been checked. ") From 0380b2f9b1bf0709eb34f71b4de260146f81c330 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Fri, 5 May 2023 16:55:07 -0700 Subject: [PATCH 17/56] use better path to store .onnx Signed-off-by: jcwchen --- .../classification/alexnet/alexnet-18.onnx} | 0 .../fasterrcnn_resnet50_fpn_v2-18.onnx} | 0 workflow_scripts/run_mlagility.py | 27 ++++++++++++------- 3 files changed, 17 insertions(+), 10 deletions(-) rename mlagility_models/{alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-base.onnx => vision/classification/alexnet/alexnet-18.onnx} (100%) rename mlagility_models/{fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-base.onnx => vision/object_detection_segmentation/faster-rcnn/fasterrcnn_resnet50_fpn_v2-18.onnx} (100%) diff --git a/mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-base.onnx b/mlagility_models/vision/classification/alexnet/alexnet-18.onnx similarity index 100% rename from mlagility_models/alexnet_torch_hub_2891f54c/onnx/alexnet_torch_hub_2891f54c-op18-base.onnx rename to mlagility_models/vision/classification/alexnet/alexnet-18.onnx diff --git a/mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-base.onnx b/mlagility_models/vision/object_detection_segmentation/faster-rcnn/fasterrcnn_resnet50_fpn_v2-18.onnx similarity index 100% rename from mlagility_models/fasterrcnn_resnet50_fpn_v2_torchvision_7147702b/onnx/fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48-op18-base.onnx rename to mlagility_models/vision/object_detection_segmentation/faster-rcnn/fasterrcnn_resnet50_fpn_v2-18.onnx diff --git a/workflow_scripts/run_mlagility.py b/workflow_scripts/run_mlagility.py index 1a1b9f8a6..94ffd87fe 100644 --- a/workflow_scripts/run_mlagility.py +++ b/workflow_scripts/run_mlagility.py @@ -1,13 +1,18 @@ import os.path as osp from pathlib import Path +import shutil import subprocess import sys +vision_dir = "vision" +classification_dir = osp.join(vision_dir, "classification") +object_detection_dir = osp.join(vision_dir, "object_detection_segmentation") + models_info = [ - # (script_path, model_name) - ("torch_hub/alexnet.py", "alexnet_torch_hub_2891f54c"), - ("torchvision/fasterrcnn_resnet50_fpn_v2.py", "fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48"), + # (script_path, model_name, model_zoo_path) + ("torch_hub/alexnet.py", "alexnet_torch_hub_2891f54c", osp .join(classification_dir, "alexnet/alexnet.onnx")), + ("torchvision/fasterrcnn_resnet50_fpn_v2.py", "fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48", osp.join(object_detection_dir, "faster-rcnn/fasterrcnn_resnet50_fpn_v2.onnx")), ] cwd_path = Path.cwd() @@ -17,21 +22,23 @@ errors = 0 -for model_info in models_info: +for script_path, model_name, model_zoo_path in models_info: try: - subprocess.run(["benchit", osp.join(mlagility_root, model_info[0]), "--cache-dir", mlagility_models_dir, "--onnx-opset", ZOO_OPSET_VERSION], + final_model_path = osp.join(mlagility_models_dir, model_zoo_path.replace(".onnx", "-" + ZOO_OPSET_VERSION + ".onnx")) + subprocess.run(["benchit", osp.join(mlagility_root, script_path), "--cache-dir", mlagility_models_dir, "--onnx-opset", ZOO_OPSET_VERSION], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - subprocess.run(["git", "diff", "--exit-code", "--", osp.join(mlagility_models_dir, model_info[1])], + shutil.move(osp.join(mlagility_models_dir, model_name), final_model_path) + subprocess.run(["git", "diff", "--exit-code", "--", model_zoo_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - print(f"Successfully checked {model_info[1]}.") + print(f"Successfully checked {model_zoo_path}.") except: errors += 1 - print(f"Failed to check {model_info[1]}.") + print(f"Failed to check {model_zoo_path}.") if errors > 0: - print(f"All {len(models_info)} models have been checked, but {errors} model(s) failed") + print(f"All {len(models_info)} models have been checked, but {errors} model(s) failed.") sys.exit(1) else: - print(f"All {len(models_info)} models have been checked. ") + print(f"All {len(models_info)} models have been checked.") From 6ff10143d68fb1f940ef9312ae7c9bfa0c1cee81 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Fri, 5 May 2023 17:06:32 -0700 Subject: [PATCH 18/56] print exception Signed-off-by: jcwchen --- workflow_scripts/run_mlagility.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow_scripts/run_mlagility.py b/workflow_scripts/run_mlagility.py index 94ffd87fe..2717c237e 100644 --- a/workflow_scripts/run_mlagility.py +++ b/workflow_scripts/run_mlagility.py @@ -29,13 +29,13 @@ cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) shutil.move(osp.join(mlagility_models_dir, model_name), final_model_path) - subprocess.run(["git", "diff", "--exit-code", "--", model_zoo_path], + subprocess.run(["git", "diff", "--exit-code", "--", final_model_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) print(f"Successfully checked {model_zoo_path}.") - except: + except Exception as e: errors += 1 - print(f"Failed to check {model_zoo_path}.") + print(f"Failed to check {model_zoo_path} because of {e}.") if errors > 0: print(f"All {len(models_info)} models have been checked, but {errors} model(s) failed.") From db69afb49c570db6b430043b7fae6b1e9d1833af Mon Sep 17 00:00:00 2001 From: jcwchen Date: Fri, 5 May 2023 17:15:12 -0700 Subject: [PATCH 19/56] copy Signed-off-by: jcwchen --- workflow_scripts/run_mlagility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow_scripts/run_mlagility.py b/workflow_scripts/run_mlagility.py index 2717c237e..d03636a8f 100644 --- a/workflow_scripts/run_mlagility.py +++ b/workflow_scripts/run_mlagility.py @@ -28,7 +28,7 @@ subprocess.run(["benchit", osp.join(mlagility_root, script_path), "--cache-dir", mlagility_models_dir, "--onnx-opset", ZOO_OPSET_VERSION], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - shutil.move(osp.join(mlagility_models_dir, model_name), final_model_path) + shutil.copy(osp.join(mlagility_models_dir, model_name), final_model_path) subprocess.run(["git", "diff", "--exit-code", "--", final_model_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) From a0f0339b6de20a761c9cf814460ad06146029530 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Fri, 5 May 2023 17:28:35 -0700 Subject: [PATCH 20/56] +base_name Signed-off-by: jcwchen --- workflow_scripts/run_mlagility.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflow_scripts/run_mlagility.py b/workflow_scripts/run_mlagility.py index d03636a8f..a8dc597d2 100644 --- a/workflow_scripts/run_mlagility.py +++ b/workflow_scripts/run_mlagility.py @@ -15,6 +15,7 @@ ("torchvision/fasterrcnn_resnet50_fpn_v2.py", "fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48", osp.join(object_detection_dir, "faster-rcnn/fasterrcnn_resnet50_fpn_v2.onnx")), ] +base_name = "-op18-base.onnx" cwd_path = Path.cwd() mlagility_root = "mlagility/models" mlagility_models_dir = "mlagility_models" @@ -28,7 +29,7 @@ subprocess.run(["benchit", osp.join(mlagility_root, script_path), "--cache-dir", mlagility_models_dir, "--onnx-opset", ZOO_OPSET_VERSION], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - shutil.copy(osp.join(mlagility_models_dir, model_name), final_model_path) + shutil.copy(osp.join(mlagility_models_dir, model_name, model_name + base_name), final_model_path) subprocess.run(["git", "diff", "--exit-code", "--", final_model_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) From dd41d77aec39cda00b51364687761773bf75fcc1 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Sat, 6 May 2023 05:11:32 -0700 Subject: [PATCH 21/56] correct path from mlagility Signed-off-by: jcwchen --- workflow_scripts/run_mlagility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow_scripts/run_mlagility.py b/workflow_scripts/run_mlagility.py index a8dc597d2..a9f592ad8 100644 --- a/workflow_scripts/run_mlagility.py +++ b/workflow_scripts/run_mlagility.py @@ -29,7 +29,7 @@ subprocess.run(["benchit", osp.join(mlagility_root, script_path), "--cache-dir", mlagility_models_dir, "--onnx-opset", ZOO_OPSET_VERSION], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - shutil.copy(osp.join(mlagility_models_dir, model_name, model_name + base_name), final_model_path) + shutil.copy(osp.join(mlagility_models_dir, model_name, "onnx", model_name + base_name), final_model_path) subprocess.run(["git", "diff", "--exit-code", "--", final_model_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) From 2795d202afa026d7fc82eee4a10f1114e3f96414 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Mon, 5 Jun 2023 20:35:47 -0700 Subject: [PATCH 22/56] add another way to upload new model Signed-off-by: jcwchen --- .github/PULL_REQUEST_TEMPLATE.md | 13 +++++++++++++ .../vision/classification/alexnet/alexnet-18.onnx | 0 .../faster-rcnn/fasterrcnn_resnet50_fpn_v2-18.onnx | 0 models/python/distilbert-18/distilbert-18.onnx | 3 +++ models/python/distilbert-18/model.py | 4 ++++ models/python/distilbert-18/requirements.txt | 0 workflow_scripts/run_mlagility.py | 14 +++++--------- 7 files changed, 25 insertions(+), 9 deletions(-) rename {mlagility_models => models/mlagility}/vision/classification/alexnet/alexnet-18.onnx (100%) rename {mlagility_models => models/mlagility}/vision/object_detection_segmentation/faster-rcnn/fasterrcnn_resnet50_fpn_v2-18.onnx (100%) create mode 100644 models/python/distilbert-18/distilbert-18.onnx create mode 100644 models/python/distilbert-18/model.py create mode 100644 models/python/distilbert-18/requirements.txt diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index cfa02c061..0c13b4fb8 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -21,6 +21,19 @@ checker.check_model(model_proto) * Once approved transfer model from your username to ONNX organization * Add a badge for model in model table, see examples in [Models list](https://github.com/onnx/models#models) + +## Two ways to contribute a new model to ONNX Model Zoo + +### Contribute a new ONNX model through mlagility +* Create a Python script under https://github.com/groq/mlagility for creating Python models from PyTorch or Huggingface; pass their CIs +* Create a PR under https://github.com/onnx/models with specified (script_path, model_name, model_zoo_path) with converted ONNX models from mlagility. The CI will run the script with the provided config and verify the converted ONNX models. +ONNX_HUB_MANIFEST.json will need to be updated accordingly. + +### Contribute a new ONNX model directly with Python script for reproducing the model +* Create a Python script under models for converting ONNX models from PyTorch, Tensorflow, Huggingface or etc. Also requirements.txt for required dependencies is NEEDED. +* Create a PR with provided ONNX models, Python script, requirements.txt. The CI will run the script with the provided requirements.txt. and verify the converted ONNX models. ONNX_HUB_MANIFEST.json will need to be updated accordingly. + + ## Model |Model |Download | Download (with sample test data)|ONNX version|Opset version|Accuracy | diff --git a/mlagility_models/vision/classification/alexnet/alexnet-18.onnx b/models/mlagility/vision/classification/alexnet/alexnet-18.onnx similarity index 100% rename from mlagility_models/vision/classification/alexnet/alexnet-18.onnx rename to models/mlagility/vision/classification/alexnet/alexnet-18.onnx diff --git a/mlagility_models/vision/object_detection_segmentation/faster-rcnn/fasterrcnn_resnet50_fpn_v2-18.onnx b/models/mlagility/vision/object_detection_segmentation/faster-rcnn/fasterrcnn_resnet50_fpn_v2-18.onnx similarity index 100% rename from mlagility_models/vision/object_detection_segmentation/faster-rcnn/fasterrcnn_resnet50_fpn_v2-18.onnx rename to models/mlagility/vision/object_detection_segmentation/faster-rcnn/fasterrcnn_resnet50_fpn_v2-18.onnx diff --git a/models/python/distilbert-18/distilbert-18.onnx b/models/python/distilbert-18/distilbert-18.onnx new file mode 100644 index 000000000..cc9d4cb12 --- /dev/null +++ b/models/python/distilbert-18/distilbert-18.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ab6e1a21a1bbe5a33251156d757a447b7ee4360390b965c7a8138538705df97 +size 262173211 diff --git a/models/python/distilbert-18/model.py b/models/python/distilbert-18/model.py new file mode 100644 index 000000000..635d84361 --- /dev/null +++ b/models/python/distilbert-18/model.py @@ -0,0 +1,4 @@ +import torch +from transformers import AutoModelForSequenceClassification +model = AutoModelForSequenceClassification.from_pretrained("textattack/distilbert-base-cased-CoLA") +torch.onnx.export(model.base_model, model.dummy_inputs, "distilbert-18.onnx", verbose=True, input_names=["input_ids", "attention_mask"], output_names=["logits"], opset_version=11, dynamic_axes={"input_ids": [0, 1], "attention_mask": [0, 1], "logits": [0, 1]}) \ No newline at end of file diff --git a/models/python/distilbert-18/requirements.txt b/models/python/distilbert-18/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/workflow_scripts/run_mlagility.py b/workflow_scripts/run_mlagility.py index a9f592ad8..27d56a589 100644 --- a/workflow_scripts/run_mlagility.py +++ b/workflow_scripts/run_mlagility.py @@ -1,3 +1,4 @@ +import config import os.path as osp from pathlib import Path import shutil @@ -9,21 +10,16 @@ classification_dir = osp.join(vision_dir, "classification") object_detection_dir = osp.join(vision_dir, "object_detection_segmentation") -models_info = [ - # (script_path, model_name, model_zoo_path) - ("torch_hub/alexnet.py", "alexnet_torch_hub_2891f54c", osp .join(classification_dir, "alexnet/alexnet.onnx")), - ("torchvision/fasterrcnn_resnet50_fpn_v2.py", "fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48", osp.join(object_detection_dir, "faster-rcnn/fasterrcnn_resnet50_fpn_v2.onnx")), -] base_name = "-op18-base.onnx" cwd_path = Path.cwd() mlagility_root = "mlagility/models" -mlagility_models_dir = "mlagility_models" +mlagility_models_dir = "models/mlagility" ZOO_OPSET_VERSION = "18" errors = 0 -for script_path, model_name, model_zoo_path in models_info: +for script_path, model_name, model_zoo_path in config.models_info: try: final_model_path = osp.join(mlagility_models_dir, model_zoo_path.replace(".onnx", "-" + ZOO_OPSET_VERSION + ".onnx")) subprocess.run(["benchit", osp.join(mlagility_root, script_path), "--cache-dir", mlagility_models_dir, "--onnx-opset", ZOO_OPSET_VERSION], @@ -39,7 +35,7 @@ print(f"Failed to check {model_zoo_path} because of {e}.") if errors > 0: - print(f"All {len(models_info)} models have been checked, but {errors} model(s) failed.") + print(f"All {len(config.models_info)} models have been checked, but {errors} model(s) failed.") sys.exit(1) else: - print(f"All {len(models_info)} models have been checked.") + print(f"All {len(config.models_info)} models have been checked.") From 37cf55ebb5a2af46c3c1f9709b402a5b6640090e Mon Sep 17 00:00:00 2001 From: jcwchen Date: Mon, 5 Jun 2023 20:54:18 -0700 Subject: [PATCH 23/56] validate_model Signed-off-by: jcwchen --- ...reate_mlagility.yml => validate_model.yml} | 12 +++-- workflow_scripts/config.py | 5 ++ workflow_scripts/run_python.py | 51 +++++++++++++++++++ 3 files changed, 63 insertions(+), 5 deletions(-) rename .github/workflows/{create_mlagility.yml => validate_model.yml} (79%) create mode 100644 workflow_scripts/config.py create mode 100644 workflow_scripts/run_python.py diff --git a/.github/workflows/create_mlagility.yml b/.github/workflows/validate_model.yml similarity index 79% rename from .github/workflows/create_mlagility.yml rename to .github/workflows/validate_model.yml index 03558ec58..54e8d3510 100644 --- a/.github/workflows/create_mlagility.yml +++ b/.github/workflows/validate_model.yml @@ -1,6 +1,4 @@ -# Create ONNX model from mlagility - -name: Create ONNX model from mlagility +name: Validate created ONNX model from mlagility or Python on: schedule: @@ -39,7 +37,11 @@ jobs: pip install -r models/requirements.txt pip install -e . - - name: Run mlagility + - name: Validate created ONNX model from mlagility run: | pip install torch==2.0.0 torchvision==0.15.1 - python workflow_scripts/run_mlagility.py \ No newline at end of file + python workflow_scripts/run_mlagility.py + + - name: Validate created ONNX model from Python + run: | + python workflow_scripts/run_python.py \ No newline at end of file diff --git a/workflow_scripts/config.py b/workflow_scripts/config.py new file mode 100644 index 000000000..22c2c1e6a --- /dev/null +++ b/workflow_scripts/config.py @@ -0,0 +1,5 @@ +models_info = [ + # (script_path, model_name, model_zoo_path) + ("torch_hub/alexnet.py", "alexnet_torch_hub_2891f54c", osp .join(classification_dir, "alexnet/alexnet.onnx")), + ("torchvision/fasterrcnn_resnet50_fpn_v2.py", "fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48", osp.join(object_detection_dir, "faster-rcnn/fasterrcnn_resnet50_fpn_v2.onnx")), +] \ No newline at end of file diff --git a/workflow_scripts/run_python.py b/workflow_scripts/run_python.py new file mode 100644 index 000000000..227ba5b6c --- /dev/null +++ b/workflow_scripts/run_python.py @@ -0,0 +1,51 @@ +import os +import subprocess +import sys +import Path + +python_root = "models/python" +cwd_path = Path.cwd() +errors = 0 +total_models = 0 + +for directory in ["text", "vision"]: + for root, dirs, files in os.walk(python_root): + for dir in dirs: + dir_path = os.path.join(root, dir) + model_path = os.path.join(dir_path, dir + ".onnx") + requirements_path = os.path.join(dir_path, "requirements.txt") + model_python_path = os.path.join(dir_path, "model.py") + if not os.path.exists(model_python_path) and not os.path.exists(model_path): + continue + total_models += 1 + if not os.path.exists(model_python_path): + print(f"Model {model_python_path} does not exist.") + errors += 1 + continue + if not os.path.exists(model_path): + print(f"Model {model_path} does not exist.") + errors += 1 + continue + if not os.path.exists(requirements_path): + errors += 1 + continue + os.remove(model_path) + subprocess.run("pip install -r", requirements_path, cwd=cwd_path, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + subprocess.run("python", model_python_path, cwd=cwd_path, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if not os.path.exists(model_path): + print(f"Model {model_path} was not created by {model_python_path}.") + errors += 1 + continue + subprocess.run(["git", "diff", "--exit-code", "--", model_path], + cwd=cwd_path, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + print(f"Successfully checked {model_path}.") + + +if errors > 0: + print(f"All {total_models} models have been checked, but {errors} model(s) failed.") + sys.exit(1) +else: + print(f"All {total_models} models have been checked.") \ No newline at end of file From cea6c52a293744238735f4620f8f396f97cc5f70 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Mon, 5 Jun 2023 21:00:42 -0700 Subject: [PATCH 24/56] import os.path as osp Signed-off-by: jcwchen --- workflow_scripts/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workflow_scripts/config.py b/workflow_scripts/config.py index 22c2c1e6a..a5efb7eb1 100644 --- a/workflow_scripts/config.py +++ b/workflow_scripts/config.py @@ -1,3 +1,5 @@ +import os.path as osp + models_info = [ # (script_path, model_name, model_zoo_path) ("torch_hub/alexnet.py", "alexnet_torch_hub_2891f54c", osp .join(classification_dir, "alexnet/alexnet.onnx")), From 19f6bc3f414db41ae917daf7b2fca9368bf12515 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Mon, 5 Jun 2023 21:09:07 -0700 Subject: [PATCH 25/56] correct path Signed-off-by: jcwchen --- workflow_scripts/config.py | 6 ++- workflow_scripts/run_mlagility.py | 5 --- workflow_scripts/run_python.py | 67 +++++++++++++++---------------- 3 files changed, 38 insertions(+), 40 deletions(-) diff --git a/workflow_scripts/config.py b/workflow_scripts/config.py index a5efb7eb1..5a3afa4af 100644 --- a/workflow_scripts/config.py +++ b/workflow_scripts/config.py @@ -1,7 +1,11 @@ import os.path as osp +vision_dir = "vision" +classification_dir = osp.join(vision_dir, "classification") +object_detection_dir = osp.join(vision_dir, "object_detection_segmentation") + models_info = [ # (script_path, model_name, model_zoo_path) ("torch_hub/alexnet.py", "alexnet_torch_hub_2891f54c", osp .join(classification_dir, "alexnet/alexnet.onnx")), ("torchvision/fasterrcnn_resnet50_fpn_v2.py", "fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48", osp.join(object_detection_dir, "faster-rcnn/fasterrcnn_resnet50_fpn_v2.onnx")), -] \ No newline at end of file +] diff --git a/workflow_scripts/run_mlagility.py b/workflow_scripts/run_mlagility.py index 27d56a589..6294f37e2 100644 --- a/workflow_scripts/run_mlagility.py +++ b/workflow_scripts/run_mlagility.py @@ -6,11 +6,6 @@ import sys -vision_dir = "vision" -classification_dir = osp.join(vision_dir, "classification") -object_detection_dir = osp.join(vision_dir, "object_detection_segmentation") - - base_name = "-op18-base.onnx" cwd_path = Path.cwd() mlagility_root = "mlagility/models" diff --git a/workflow_scripts/run_python.py b/workflow_scripts/run_python.py index 227ba5b6c..a3f4d2d3d 100644 --- a/workflow_scripts/run_python.py +++ b/workflow_scripts/run_python.py @@ -8,40 +8,39 @@ errors = 0 total_models = 0 -for directory in ["text", "vision"]: - for root, dirs, files in os.walk(python_root): - for dir in dirs: - dir_path = os.path.join(root, dir) - model_path = os.path.join(dir_path, dir + ".onnx") - requirements_path = os.path.join(dir_path, "requirements.txt") - model_python_path = os.path.join(dir_path, "model.py") - if not os.path.exists(model_python_path) and not os.path.exists(model_path): - continue - total_models += 1 - if not os.path.exists(model_python_path): - print(f"Model {model_python_path} does not exist.") - errors += 1 - continue - if not os.path.exists(model_path): - print(f"Model {model_path} does not exist.") - errors += 1 - continue - if not os.path.exists(requirements_path): - errors += 1 - continue - os.remove(model_path) - subprocess.run("pip install -r", requirements_path, cwd=cwd_path, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - subprocess.run("python", model_python_path, cwd=cwd_path, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - if not os.path.exists(model_path): - print(f"Model {model_path} was not created by {model_python_path}.") - errors += 1 - continue - subprocess.run(["git", "diff", "--exit-code", "--", model_path], - cwd=cwd_path, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - print(f"Successfully checked {model_path}.") +for root, dirs, files in os.walk(python_root): + for dir in dirs: + dir_path = os.path.join(root, dir) + model_path = os.path.join(dir_path, dir + ".onnx") + requirements_path = os.path.join(dir_path, "requirements.txt") + model_python_path = os.path.join(dir_path, "model.py") + if not os.path.exists(model_python_path) and not os.path.exists(model_path): + continue + total_models += 1 + if not os.path.exists(model_python_path): + print(f"Model {model_python_path} does not exist.") + errors += 1 + continue + if not os.path.exists(model_path): + print(f"Model {model_path} does not exist.") + errors += 1 + continue + if not os.path.exists(requirements_path): + errors += 1 + continue + os.remove(model_path) + subprocess.run("pip install -r", requirements_path, cwd=cwd_path, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + subprocess.run("python", model_python_path, cwd=cwd_path, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if not os.path.exists(model_path): + print(f"Model {model_path} was not created by {model_python_path}.") + errors += 1 + continue + subprocess.run(["git", "diff", "--exit-code", "--", model_path], + cwd=cwd_path, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + print(f"Successfully checked {model_path}.") if errors > 0: From 934765a227538d769ef48d9ff9ca2d5b042e63c4 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Mon, 5 Jun 2023 21:33:02 -0700 Subject: [PATCH 26/56] from pathlib import Path Signed-off-by: jcwchen --- workflow_scripts/run_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow_scripts/run_python.py b/workflow_scripts/run_python.py index a3f4d2d3d..c9dd7e5b4 100644 --- a/workflow_scripts/run_python.py +++ b/workflow_scripts/run_python.py @@ -1,7 +1,7 @@ import os import subprocess import sys -import Path +from pathlib import Path python_root = "models/python" cwd_path = Path.cwd() From a6f9618374cf4fda35c0a646d5153b83a0da27c6 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Mon, 5 Jun 2023 21:55:59 -0700 Subject: [PATCH 27/56] pip install -r Signed-off-by: jcwchen --- workflow_scripts/run_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow_scripts/run_python.py b/workflow_scripts/run_python.py index c9dd7e5b4..30a9a0192 100644 --- a/workflow_scripts/run_python.py +++ b/workflow_scripts/run_python.py @@ -29,7 +29,7 @@ errors += 1 continue os.remove(model_path) - subprocess.run("pip install -r", requirements_path, cwd=cwd_path, stdout=subprocess.PIPE, + subprocess.run("pip", "install", "-r", requirements_path, cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) subprocess.run("python", model_python_path, cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) From fa5d2904e71b69c2fb99b2cf27e800db039f3abd Mon Sep 17 00:00:00 2001 From: jcwchen Date: Mon, 5 Jun 2023 22:07:20 -0700 Subject: [PATCH 28/56] [] Signed-off-by: jcwchen --- workflow_scripts/run_python.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow_scripts/run_python.py b/workflow_scripts/run_python.py index 30a9a0192..2f51917a6 100644 --- a/workflow_scripts/run_python.py +++ b/workflow_scripts/run_python.py @@ -29,9 +29,9 @@ errors += 1 continue os.remove(model_path) - subprocess.run("pip", "install", "-r", requirements_path, cwd=cwd_path, stdout=subprocess.PIPE, + subprocess.run(["pip", "install", "-r", requirements_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - subprocess.run("python", model_python_path, cwd=cwd_path, stdout=subprocess.PIPE, + subprocess.run(["python", model_python_path] cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if not os.path.exists(model_path): print(f"Model {model_path} was not created by {model_python_path}.") From 00e4e015b389d6c1e904f3c76461b9f659ae850f Mon Sep 17 00:00:00 2001 From: jcwchen Date: Tue, 6 Jun 2023 06:51:58 -0700 Subject: [PATCH 29/56] fix Signed-off-by: jcwchen --- workflow_scripts/run_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow_scripts/run_python.py b/workflow_scripts/run_python.py index 2f51917a6..5b204bfda 100644 --- a/workflow_scripts/run_python.py +++ b/workflow_scripts/run_python.py @@ -31,7 +31,7 @@ os.remove(model_path) subprocess.run(["pip", "install", "-r", requirements_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - subprocess.run(["python", model_python_path] cwd=cwd_path, stdout=subprocess.PIPE, + subprocess.run(["python", model_python_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if not os.path.exists(model_path): print(f"Model {model_path} was not created by {model_python_path}.") From 5edb28f2bbaf84a97cf3fd4698ff1b095b9ce2f2 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Tue, 6 Jun 2023 07:02:37 -0700 Subject: [PATCH 30/56] dir_path Signed-off-by: jcwchen --- workflow_scripts/run_python.py | 1 + 1 file changed, 1 insertion(+) diff --git a/workflow_scripts/run_python.py b/workflow_scripts/run_python.py index 5b204bfda..aaac357df 100644 --- a/workflow_scripts/run_python.py +++ b/workflow_scripts/run_python.py @@ -29,6 +29,7 @@ errors += 1 continue os.remove(model_path) + cwd_path = dir_path subprocess.run(["pip", "install", "-r", requirements_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) subprocess.run(["python", model_python_path], cwd=cwd_path, stdout=subprocess.PIPE, From 388806f8381e696c3a48751a62f52c9400d8330b Mon Sep 17 00:00:00 2001 From: jcwchen Date: Tue, 6 Jun 2023 08:48:05 -0700 Subject: [PATCH 31/56] correct path Signed-off-by: jcwchen --- models/python/distilbert-18/distilbert-18.onnx | 2 +- workflow_scripts/run_mlagility.py | 4 ++-- workflow_scripts/run_python.py | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/models/python/distilbert-18/distilbert-18.onnx b/models/python/distilbert-18/distilbert-18.onnx index cc9d4cb12..676b6eb35 100644 --- a/models/python/distilbert-18/distilbert-18.onnx +++ b/models/python/distilbert-18/distilbert-18.onnx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7ab6e1a21a1bbe5a33251156d757a447b7ee4360390b965c7a8138538705df97 +oid sha256:8e7ddbdae80656842d5ad73c57a6345b83e3e01077621a96148f7859afebcdbf size 262173211 diff --git a/workflow_scripts/run_mlagility.py b/workflow_scripts/run_mlagility.py index 6294f37e2..4ff6deab8 100644 --- a/workflow_scripts/run_mlagility.py +++ b/workflow_scripts/run_mlagility.py @@ -30,7 +30,7 @@ print(f"Failed to check {model_zoo_path} because of {e}.") if errors > 0: - print(f"All {len(config.models_info)} models have been checked, but {errors} model(s) failed.") + print(f"All {len(config.models_info)} model(s) have been checked, but {errors} model(s) failed.") sys.exit(1) else: - print(f"All {len(config.models_info)} models have been checked.") + print(f"All {len(config.models_info)} model(s) have been checked.") diff --git a/workflow_scripts/run_python.py b/workflow_scripts/run_python.py index aaac357df..f821263be 100644 --- a/workflow_scripts/run_python.py +++ b/workflow_scripts/run_python.py @@ -11,7 +11,8 @@ for root, dirs, files in os.walk(python_root): for dir in dirs: dir_path = os.path.join(root, dir) - model_path = os.path.join(dir_path, dir + ".onnx") + model_name = dir + ".onnx" + model_path = os.path.join(dir_path, model_name) requirements_path = os.path.join(dir_path, "requirements.txt") model_python_path = os.path.join(dir_path, "model.py") if not os.path.exists(model_python_path) and not os.path.exists(model_path): @@ -28,12 +29,11 @@ if not os.path.exists(requirements_path): errors += 1 continue - os.remove(model_path) - cwd_path = dir_path subprocess.run(["pip", "install", "-r", requirements_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) subprocess.run(["python", model_python_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + os.replace(model_name, model_path) if not os.path.exists(model_path): print(f"Model {model_path} was not created by {model_python_path}.") errors += 1 @@ -45,7 +45,7 @@ if errors > 0: - print(f"All {total_models} models have been checked, but {errors} model(s) failed.") + print(f"All {total_models} model(s) have been checked, but {errors} model(s) failed.") sys.exit(1) else: - print(f"All {total_models} models have been checked.") \ No newline at end of file + print(f"All {total_models} model(s) have been checked.") \ No newline at end of file From 943fc343b3629a28e34c6d6d8244bf6055257912 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 7 Jun 2023 15:59:40 -0700 Subject: [PATCH 32/56] add bart.py Signed-off-by: jcwchen --- models/python/bart-18/bert-18.onnx | 3 ++ models/python/bart-18/model.py | 27 ++++++++++++++++++ .../requirements.txt | 0 .../python/distilbert-18/distilbert-18.onnx | 3 -- models/python/distilbert-18/model.py | 4 --- workflow_scripts/run_python.py | 28 +++++++++++++++++-- 6 files changed, 56 insertions(+), 9 deletions(-) create mode 100644 models/python/bart-18/bert-18.onnx create mode 100644 models/python/bart-18/model.py rename models/python/{distilbert-18 => bart-18}/requirements.txt (100%) delete mode 100644 models/python/distilbert-18/distilbert-18.onnx delete mode 100644 models/python/distilbert-18/model.py diff --git a/models/python/bart-18/bert-18.onnx b/models/python/bart-18/bert-18.onnx new file mode 100644 index 000000000..47f1433b4 --- /dev/null +++ b/models/python/bart-18/bert-18.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3a32b95468cde7d9b52cab4879d6bf5b296ef7403c5b7d7a1b03a2c85f34788 +size 437579710 diff --git a/models/python/bart-18/model.py b/models/python/bart-18/model.py new file mode 100644 index 000000000..dc0c5fcbb --- /dev/null +++ b/models/python/bart-18/model.py @@ -0,0 +1,27 @@ +# labels: test_group::mlagility name::bert author::transformers task::Natural_Language_Processing +""" +https://huggingface.co/docs/transformers/v4.26.1/en/model_doc/bert#overview +""" +from mlagility.parser import parse +import transformers +import torch + +torch.manual_seed(0) + +# Parsing command-line arguments +batch_size, max_seq_length = parse(["batch_size", "max_seq_length"]) + + +# Model and input configurations +config = transformers.BertConfig() +model = transformers.BertModel(config) +inputs = { + "input_ids": torch.ones(batch_size, max_seq_length, dtype=torch.long), + "attention_mask": torch.ones(batch_size, max_seq_length, dtype=torch.float), +} + + +# Call model +model(**inputs) + +torch.onnx.export(model.base_model, inputs, "bert-18.onnx", opset_version=18) \ No newline at end of file diff --git a/models/python/distilbert-18/requirements.txt b/models/python/bart-18/requirements.txt similarity index 100% rename from models/python/distilbert-18/requirements.txt rename to models/python/bart-18/requirements.txt diff --git a/models/python/distilbert-18/distilbert-18.onnx b/models/python/distilbert-18/distilbert-18.onnx deleted file mode 100644 index 676b6eb35..000000000 --- a/models/python/distilbert-18/distilbert-18.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8e7ddbdae80656842d5ad73c57a6345b83e3e01077621a96148f7859afebcdbf -size 262173211 diff --git a/models/python/distilbert-18/model.py b/models/python/distilbert-18/model.py deleted file mode 100644 index 635d84361..000000000 --- a/models/python/distilbert-18/model.py +++ /dev/null @@ -1,4 +0,0 @@ -import torch -from transformers import AutoModelForSequenceClassification -model = AutoModelForSequenceClassification.from_pretrained("textattack/distilbert-base-cased-CoLA") -torch.onnx.export(model.base_model, model.dummy_inputs, "distilbert-18.onnx", verbose=True, input_names=["input_ids", "attention_mask"], output_names=["logits"], opset_version=11, dynamic_axes={"input_ids": [0, 1], "attention_mask": [0, 1], "logits": [0, 1]}) \ No newline at end of file diff --git a/workflow_scripts/run_python.py b/workflow_scripts/run_python.py index f821263be..0d17e5c82 100644 --- a/workflow_scripts/run_python.py +++ b/workflow_scripts/run_python.py @@ -4,9 +4,18 @@ from pathlib import Path python_root = "models/python" +cache_dir = ".cache" cwd_path = Path.cwd() errors = 0 total_models = 0 +ZOO_OPSET_VERSION = 18 + +def find_base_onnx(root_dir): + for root, _, files in os.walk(root_dir): + for file in files: + if "-base.onnx" in file: + return os.path.join(root, file) + return None for root, dirs, files in os.walk(python_root): for dir in dirs: @@ -31,16 +40,31 @@ continue subprocess.run(["pip", "install", "-r", requirements_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + os.remove(model_path) subprocess.run(["python", model_python_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - os.replace(model_name, model_path) - if not os.path.exists(model_path): + if not os.path.exists(model_name): print(f"Model {model_path} was not created by {model_python_path}.") errors += 1 continue + os.replace(model_name, model_path) + subprocess.run(["git", "diff", "--exit-code", "--", model_path], + cwd=cwd_path, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + os.remove(model_path) + subprocess.run(["benchit", model_python_path, "--cache-dir", cache_dir, "--onnx-opset", ZOO_OPSET_VERSION], + cwd=cwd_path, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + cache_model = find_base_onnx(cache_dir, dir) + if cache_dir is None: + print(f"Model {model_path} was not created by benchit from mlagility.") + errors += 1 + continue + os.replace(cache_model, model_path) subprocess.run(["git", "diff", "--exit-code", "--", model_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(f"Successfully checked {model_path}.") From d789edd9d761b81a0d7f8b84edc29d2506c88a42 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 7 Jun 2023 16:26:40 -0700 Subject: [PATCH 33/56] change name Signed-off-by: jcwchen --- models/python/bart-18/{bert-18.onnx => bart-18.onnx} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/python/bart-18/{bert-18.onnx => bart-18.onnx} (100%) diff --git a/models/python/bart-18/bert-18.onnx b/models/python/bart-18/bart-18.onnx similarity index 100% rename from models/python/bart-18/bert-18.onnx rename to models/python/bart-18/bart-18.onnx From 3b34a1925b8eb8bf5e6b9f693d9abcbc9dfa79aa Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 7 Jun 2023 16:49:12 -0700 Subject: [PATCH 34/56] update path Signed-off-by: jcwchen --- models/python/bart-18/model.py | 2 +- workflow_scripts/run_python.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/python/bart-18/model.py b/models/python/bart-18/model.py index dc0c5fcbb..8308eb3b4 100644 --- a/models/python/bart-18/model.py +++ b/models/python/bart-18/model.py @@ -24,4 +24,4 @@ # Call model model(**inputs) -torch.onnx.export(model.base_model, inputs, "bert-18.onnx", opset_version=18) \ No newline at end of file +torch.onnx.export(model.base_model, inputs, "bert-18.onnx", opset_version=18) diff --git a/workflow_scripts/run_python.py b/workflow_scripts/run_python.py index 0d17e5c82..004d50da0 100644 --- a/workflow_scripts/run_python.py +++ b/workflow_scripts/run_python.py @@ -43,11 +43,11 @@ def find_base_onnx(root_dir): os.remove(model_path) subprocess.run(["python", model_python_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + os.replace(model_name, model_path) if not os.path.exists(model_name): print(f"Model {model_path} was not created by {model_python_path}.") errors += 1 continue - os.replace(model_name, model_path) subprocess.run(["git", "diff", "--exit-code", "--", model_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) From ff87370d6ac13f606da62981ef004feedeee0e56 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 7 Jun 2023 16:57:03 -0700 Subject: [PATCH 35/56] bart Signed-off-by: jcwchen --- models/python/bart-18/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/python/bart-18/model.py b/models/python/bart-18/model.py index 8308eb3b4..0dd1a71db 100644 --- a/models/python/bart-18/model.py +++ b/models/python/bart-18/model.py @@ -24,4 +24,4 @@ # Call model model(**inputs) -torch.onnx.export(model.base_model, inputs, "bert-18.onnx", opset_version=18) +torch.onnx.export(model.base_model, inputs, "bart-18.onnx", opset_version=18) From 28c7da1585700b0e9a5ac48cf690ce54dc6cce95 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 7 Jun 2023 17:09:25 -0700 Subject: [PATCH 36/56] not os.path.exists(model_name) Signed-off-by: jcwchen --- workflow_scripts/run_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow_scripts/run_python.py b/workflow_scripts/run_python.py index 004d50da0..0d17e5c82 100644 --- a/workflow_scripts/run_python.py +++ b/workflow_scripts/run_python.py @@ -43,11 +43,11 @@ def find_base_onnx(root_dir): os.remove(model_path) subprocess.run(["python", model_python_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - os.replace(model_name, model_path) if not os.path.exists(model_name): print(f"Model {model_path} was not created by {model_python_path}.") errors += 1 continue + os.replace(model_name, model_path) subprocess.run(["git", "diff", "--exit-code", "--", model_path], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) From d7550041abaa5d8817fecc301c5cf613874a52bd Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 7 Jun 2023 17:43:09 -0700 Subject: [PATCH 37/56] "18" Signed-off-by: jcwchen --- workflow_scripts/run_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow_scripts/run_python.py b/workflow_scripts/run_python.py index 0d17e5c82..8a3f7869d 100644 --- a/workflow_scripts/run_python.py +++ b/workflow_scripts/run_python.py @@ -8,7 +8,7 @@ cwd_path = Path.cwd() errors = 0 total_models = 0 -ZOO_OPSET_VERSION = 18 +ZOO_OPSET_VERSION = "18" def find_base_onnx(root_dir): for root, _, files in os.walk(root_dir): From 9eff439089abc20f7f281c2567cf90e86886f802 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 7 Jun 2023 18:05:39 -0700 Subject: [PATCH 38/56] correct Signed-off-by: jcwchen --- workflow_scripts/run_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow_scripts/run_python.py b/workflow_scripts/run_python.py index 8a3f7869d..1d14517a7 100644 --- a/workflow_scripts/run_python.py +++ b/workflow_scripts/run_python.py @@ -55,7 +55,7 @@ def find_base_onnx(root_dir): subprocess.run(["benchit", model_python_path, "--cache-dir", cache_dir, "--onnx-opset", ZOO_OPSET_VERSION], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - cache_model = find_base_onnx(cache_dir, dir) + cache_model = find_base_onnx(cache_dir) if cache_dir is None: print(f"Model {model_path} was not created by benchit from mlagility.") errors += 1 From e0f0ff08e30fe6c70b1be189aba3afa95bb09ad1 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 14 Jun 2023 09:43:39 -0700 Subject: [PATCH 39/56] change path Signed-off-by: jcwchen --- .github/PULL_REQUEST_TEMPLATE.md | 13 ---- .../alexnet => }/alexnet-18.onnx | 0 .../fasterrcnn_resnet50_fpn_v2-18.onnx | 0 models/python/bart-18/bart-18.onnx | 3 - models/python/bart-18/model.py | 27 ------- models/python/bart-18/requirements.txt | 0 workflow_scripts/config.py | 10 +-- workflow_scripts/run_python.py | 75 ------------------- 8 files changed, 2 insertions(+), 126 deletions(-) rename models/mlagility/{vision/classification/alexnet => }/alexnet-18.onnx (100%) rename models/mlagility/{vision/object_detection_segmentation/faster-rcnn => }/fasterrcnn_resnet50_fpn_v2-18.onnx (100%) delete mode 100644 models/python/bart-18/bart-18.onnx delete mode 100644 models/python/bart-18/model.py delete mode 100644 models/python/bart-18/requirements.txt delete mode 100644 workflow_scripts/run_python.py diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 0c13b4fb8..cfa02c061 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -21,19 +21,6 @@ checker.check_model(model_proto) * Once approved transfer model from your username to ONNX organization * Add a badge for model in model table, see examples in [Models list](https://github.com/onnx/models#models) - -## Two ways to contribute a new model to ONNX Model Zoo - -### Contribute a new ONNX model through mlagility -* Create a Python script under https://github.com/groq/mlagility for creating Python models from PyTorch or Huggingface; pass their CIs -* Create a PR under https://github.com/onnx/models with specified (script_path, model_name, model_zoo_path) with converted ONNX models from mlagility. The CI will run the script with the provided config and verify the converted ONNX models. -ONNX_HUB_MANIFEST.json will need to be updated accordingly. - -### Contribute a new ONNX model directly with Python script for reproducing the model -* Create a Python script under models for converting ONNX models from PyTorch, Tensorflow, Huggingface or etc. Also requirements.txt for required dependencies is NEEDED. -* Create a PR with provided ONNX models, Python script, requirements.txt. The CI will run the script with the provided requirements.txt. and verify the converted ONNX models. ONNX_HUB_MANIFEST.json will need to be updated accordingly. - - ## Model |Model |Download | Download (with sample test data)|ONNX version|Opset version|Accuracy | diff --git a/models/mlagility/vision/classification/alexnet/alexnet-18.onnx b/models/mlagility/alexnet-18.onnx similarity index 100% rename from models/mlagility/vision/classification/alexnet/alexnet-18.onnx rename to models/mlagility/alexnet-18.onnx diff --git a/models/mlagility/vision/object_detection_segmentation/faster-rcnn/fasterrcnn_resnet50_fpn_v2-18.onnx b/models/mlagility/fasterrcnn_resnet50_fpn_v2-18.onnx similarity index 100% rename from models/mlagility/vision/object_detection_segmentation/faster-rcnn/fasterrcnn_resnet50_fpn_v2-18.onnx rename to models/mlagility/fasterrcnn_resnet50_fpn_v2-18.onnx diff --git a/models/python/bart-18/bart-18.onnx b/models/python/bart-18/bart-18.onnx deleted file mode 100644 index 47f1433b4..000000000 --- a/models/python/bart-18/bart-18.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b3a32b95468cde7d9b52cab4879d6bf5b296ef7403c5b7d7a1b03a2c85f34788 -size 437579710 diff --git a/models/python/bart-18/model.py b/models/python/bart-18/model.py deleted file mode 100644 index 0dd1a71db..000000000 --- a/models/python/bart-18/model.py +++ /dev/null @@ -1,27 +0,0 @@ -# labels: test_group::mlagility name::bert author::transformers task::Natural_Language_Processing -""" -https://huggingface.co/docs/transformers/v4.26.1/en/model_doc/bert#overview -""" -from mlagility.parser import parse -import transformers -import torch - -torch.manual_seed(0) - -# Parsing command-line arguments -batch_size, max_seq_length = parse(["batch_size", "max_seq_length"]) - - -# Model and input configurations -config = transformers.BertConfig() -model = transformers.BertModel(config) -inputs = { - "input_ids": torch.ones(batch_size, max_seq_length, dtype=torch.long), - "attention_mask": torch.ones(batch_size, max_seq_length, dtype=torch.float), -} - - -# Call model -model(**inputs) - -torch.onnx.export(model.base_model, inputs, "bart-18.onnx", opset_version=18) diff --git a/models/python/bart-18/requirements.txt b/models/python/bart-18/requirements.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/workflow_scripts/config.py b/workflow_scripts/config.py index 5a3afa4af..fb67c1e30 100644 --- a/workflow_scripts/config.py +++ b/workflow_scripts/config.py @@ -1,11 +1,5 @@ -import os.path as osp - -vision_dir = "vision" -classification_dir = osp.join(vision_dir, "classification") -object_detection_dir = osp.join(vision_dir, "object_detection_segmentation") - models_info = [ # (script_path, model_name, model_zoo_path) - ("torch_hub/alexnet.py", "alexnet_torch_hub_2891f54c", osp .join(classification_dir, "alexnet/alexnet.onnx")), - ("torchvision/fasterrcnn_resnet50_fpn_v2.py", "fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48", osp.join(object_detection_dir, "faster-rcnn/fasterrcnn_resnet50_fpn_v2.onnx")), + ("torch_hub/alexnet.py", "alexnet_torch_hub_2891f54c", "alexnet-18/alexnet-18.onnx"), + ("torchvision/fasterrcnn_resnet50_fpn_v2.py", "fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48", "faster-rcnn/fasterrcnn_resnet50_fpn_v2.onnx"), ] diff --git a/workflow_scripts/run_python.py b/workflow_scripts/run_python.py deleted file mode 100644 index 1d14517a7..000000000 --- a/workflow_scripts/run_python.py +++ /dev/null @@ -1,75 +0,0 @@ -import os -import subprocess -import sys -from pathlib import Path - -python_root = "models/python" -cache_dir = ".cache" -cwd_path = Path.cwd() -errors = 0 -total_models = 0 -ZOO_OPSET_VERSION = "18" - -def find_base_onnx(root_dir): - for root, _, files in os.walk(root_dir): - for file in files: - if "-base.onnx" in file: - return os.path.join(root, file) - return None - -for root, dirs, files in os.walk(python_root): - for dir in dirs: - dir_path = os.path.join(root, dir) - model_name = dir + ".onnx" - model_path = os.path.join(dir_path, model_name) - requirements_path = os.path.join(dir_path, "requirements.txt") - model_python_path = os.path.join(dir_path, "model.py") - if not os.path.exists(model_python_path) and not os.path.exists(model_path): - continue - total_models += 1 - if not os.path.exists(model_python_path): - print(f"Model {model_python_path} does not exist.") - errors += 1 - continue - if not os.path.exists(model_path): - print(f"Model {model_path} does not exist.") - errors += 1 - continue - if not os.path.exists(requirements_path): - errors += 1 - continue - subprocess.run(["pip", "install", "-r", requirements_path], cwd=cwd_path, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - os.remove(model_path) - subprocess.run(["python", model_python_path], cwd=cwd_path, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - if not os.path.exists(model_name): - print(f"Model {model_path} was not created by {model_python_path}.") - errors += 1 - continue - os.replace(model_name, model_path) - subprocess.run(["git", "diff", "--exit-code", "--", model_path], - cwd=cwd_path, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - os.remove(model_path) - subprocess.run(["benchit", model_python_path, "--cache-dir", cache_dir, "--onnx-opset", ZOO_OPSET_VERSION], - cwd=cwd_path, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - cache_model = find_base_onnx(cache_dir) - if cache_dir is None: - print(f"Model {model_path} was not created by benchit from mlagility.") - errors += 1 - continue - os.replace(cache_model, model_path) - subprocess.run(["git", "diff", "--exit-code", "--", model_path], - cwd=cwd_path, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - print(f"Successfully checked {model_path}.") - - -if errors > 0: - print(f"All {total_models} model(s) have been checked, but {errors} model(s) failed.") - sys.exit(1) -else: - print(f"All {total_models} model(s) have been checked.") \ No newline at end of file From 358887add4b436622c98ed2f29b68c9912b773a2 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 14 Jun 2023 09:48:45 -0700 Subject: [PATCH 40/56] add new-models to CI trigger Signed-off-by: jcwchen --- .github/workflows/codeql.yml | 5 ++--- .github/workflows/linux_ci.yml | 4 ++-- .github/workflows/validate_model.yml | 6 ++---- .github/workflows/windows_ci.yml | 5 ++--- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 53cc3c923..3e89b2d06 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -13,10 +13,9 @@ name: "CodeQL" on: push: - branches: [ "main" ] + branches: [ main, new-models] pull_request: - # The branches below must be a subset of the branches above - branches: [ "main" ] + branches: [ main, new-models] schedule: - cron: '31 11 * * 4' diff --git a/.github/workflows/linux_ci.yml b/.github/workflows/linux_ci.yml index 3d386f860..35b74cdd6 100644 --- a/.github/workflows/linux_ci.yml +++ b/.github/workflows/linux_ci.yml @@ -6,9 +6,9 @@ name: Linux CI # Triggers the workflow on push or pull request events but only for the main branch on: push: - branches: [ main ] + branches: [ main, new-models] pull_request: - branches: [ main ] + branches: [ main, new-models] jobs: # This workflow contains a single job called "build" diff --git a/.github/workflows/validate_model.yml b/.github/workflows/validate_model.yml index 54e8d3510..d5c2ac515 100644 --- a/.github/workflows/validate_model.yml +++ b/.github/workflows/validate_model.yml @@ -1,12 +1,10 @@ name: Validate created ONNX model from mlagility or Python on: - schedule: - - cron: '00 00 * * MON' push: - branches: [ main ] + branches: [ main, new-models] pull_request: - branches: [ main ] + branches: [ main, new-models] jobs: build: diff --git a/.github/workflows/windows_ci.yml b/.github/workflows/windows_ci.yml index aab61ba3c..bf52c25f9 100644 --- a/.github/workflows/windows_ci.yml +++ b/.github/workflows/windows_ci.yml @@ -3,12 +3,11 @@ name: Windows CI -# Triggers the workflow on push or pull request events but only for the main branch on: push: - branches: [ main ] + branches: [ main, new-models] pull_request: - branches: [ main ] + branches: [ main, new-models] jobs: # This workflow contains a single job called "build" From bec94e58a6834b521acaa89caf189252749c45a2 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 14 Jun 2023 10:11:29 -0700 Subject: [PATCH 41/56] add test_data_set Signed-off-by: jcwchen --- .github/workflows/linux_ci.yml | 9 +--- .github/workflows/validate_model.yml | 45 ------------------- .github/workflows/windows_ci.yml | 8 +--- .../{ => alexnet-18}/alexnet-18.onnx | 0 .../alexnet-18/test_data_set_0/input_0.pb | 3 ++ .../alexnet-18/test_data_set_0/output_0.pb | 3 ++ .../fasterrcnn_resnet50_fpn_v2-18.onnx | 0 .../test_data_set_0/input_0.pb | 3 ++ .../test_data_set_0/output_0.pb | 3 ++ .../test_data_set_0/output_1.pb | 3 ++ .../test_data_set_0/output_2.pb | 3 ++ models/mlagility/requirements.txt | 0 workflow_scripts/check_model.py | 9 +++- workflow_scripts/test_models.py | 29 +++++++++++- 14 files changed, 57 insertions(+), 61 deletions(-) delete mode 100644 .github/workflows/validate_model.yml rename models/mlagility/{ => alexnet-18}/alexnet-18.onnx (100%) create mode 100644 models/mlagility/alexnet-18/test_data_set_0/input_0.pb create mode 100644 models/mlagility/alexnet-18/test_data_set_0/output_0.pb rename models/mlagility/{ => fasterrcnn_resnet50_fpn_v2-18}/fasterrcnn_resnet50_fpn_v2-18.onnx (100%) create mode 100644 models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/input_0.pb create mode 100644 models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_0.pb create mode 100644 models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_1.pb create mode 100644 models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_2.pb create mode 100644 models/mlagility/requirements.txt diff --git a/.github/workflows/linux_ci.yml b/.github/workflows/linux_ci.yml index 35b74cdd6..75867e255 100644 --- a/.github/workflows/linux_ci.yml +++ b/.github/workflows/linux_ci.yml @@ -3,7 +3,6 @@ name: Linux CI -# Triggers the workflow on push or pull request events but only for the main branch on: push: branches: [ main, new-models] @@ -43,10 +42,6 @@ jobs: python workflow_scripts/generate_onnx_hub_manifest.py --target diff --drop git diff --exit-code -- ONNX_HUB_MANIFEST.json || { echo 'Please use "python workflow_scripts/generate_onnx_hub_manifest.py --target diff" to update ONNX_HUB_MANIFEST.json.' ; exit 1; } - - name: Test new models by onnx + - name: Test new models by onnx and onnxruntime run: | - python workflow_scripts/test_models.py --target onnx --drop - - - name: Test new models by onnxruntime - run: | - python workflow_scripts/test_models.py --target onnxruntime --drop + python workflow_scripts/test_models.py --target all --drop \ No newline at end of file diff --git a/.github/workflows/validate_model.yml b/.github/workflows/validate_model.yml deleted file mode 100644 index d5c2ac515..000000000 --- a/.github/workflows/validate_model.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Validate created ONNX model from mlagility or Python - -on: - push: - branches: [ main, new-models] - pull_request: - branches: [ main, new-models] - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.8'] - - steps: - - uses: actions/checkout@v3 - name: Checkout repo - - uses: conda-incubator/setup-miniconda@v2 - with: - miniconda-version: "latest" - activate-environment: mla - python-version: ${{ matrix.python-version }} - - - name: Install dependencies and mlagility - run: | - python -m pip install --upgrade pip - python -m pip install onnx onnxruntime requests py-cpuinfo - # Print CPU info for debugging ONNX Runtime inference difference - python -m cpuinfo - # or python -m pip install mlagility - python -m pip install transformers - git clone https://github.com/groq/mlagility.git - cd mlagility - pip install -r models/requirements.txt - pip install -e . - - - name: Validate created ONNX model from mlagility - run: | - pip install torch==2.0.0 torchvision==0.15.1 - python workflow_scripts/run_mlagility.py - - - name: Validate created ONNX model from Python - run: | - python workflow_scripts/run_python.py \ No newline at end of file diff --git a/.github/workflows/windows_ci.yml b/.github/workflows/windows_ci.yml index bf52c25f9..34e1daaa6 100644 --- a/.github/workflows/windows_ci.yml +++ b/.github/workflows/windows_ci.yml @@ -36,10 +36,6 @@ jobs: # Print CPU info for debugging ONNX Runtime inference difference python -m cpuinfo - - name: Test new models by onnx + - name: Test new models by onnx and onnxruntime run: | - python workflow_scripts/test_models.py --target onnx --drop - - - name: Test new models by onnxruntime - run: | - python workflow_scripts/test_models.py --target onnxruntime --drop + python workflow_scripts/test_models.py --target all --drop diff --git a/models/mlagility/alexnet-18.onnx b/models/mlagility/alexnet-18/alexnet-18.onnx similarity index 100% rename from models/mlagility/alexnet-18.onnx rename to models/mlagility/alexnet-18/alexnet-18.onnx diff --git a/models/mlagility/alexnet-18/test_data_set_0/input_0.pb b/models/mlagility/alexnet-18/test_data_set_0/input_0.pb new file mode 100644 index 000000000..e8f1b84d9 --- /dev/null +++ b/models/mlagility/alexnet-18/test_data_set_0/input_0.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb8b6cf13a2f3ef60d0673a844698c51e338c2819e83cd1426946d2197107c1f +size 602137 diff --git a/models/mlagility/alexnet-18/test_data_set_0/output_0.pb b/models/mlagility/alexnet-18/test_data_set_0/output_0.pb new file mode 100644 index 000000000..7be7372da --- /dev/null +++ b/models/mlagility/alexnet-18/test_data_set_0/output_0.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24fce9a5a7a7e4f44d7093502c91bc5ee217bd39badbfeb337ea4d33cd541675 +size 4014 diff --git a/models/mlagility/fasterrcnn_resnet50_fpn_v2-18.onnx b/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/fasterrcnn_resnet50_fpn_v2-18.onnx similarity index 100% rename from models/mlagility/fasterrcnn_resnet50_fpn_v2-18.onnx rename to models/mlagility/fasterrcnn_resnet50_fpn_v2-18/fasterrcnn_resnet50_fpn_v2-18.onnx diff --git a/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/input_0.pb b/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/input_0.pb new file mode 100644 index 000000000..02d2065fc --- /dev/null +++ b/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/input_0.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64df214768ec3d41b6f61f01239666f933e766417c2adfbc6b117c386c184944 +size 602136 diff --git a/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_0.pb b/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_0.pb new file mode 100644 index 000000000..3fb76d8c8 --- /dev/null +++ b/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_0.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:088937d53abc08344fbb81854e2e50750e025d4d86744dc5a4272fd67e77e417 +size 367 diff --git a/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_1.pb b/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_1.pb new file mode 100644 index 000000000..6767b0bca --- /dev/null +++ b/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_1.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71b884a9bae94fec0a92c533ab4b0171838f0b6000e70c6d2247f3295d48ff5f +size 189 diff --git a/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_2.pb b/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_2.pb new file mode 100644 index 000000000..571c79559 --- /dev/null +++ b/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_2.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f34283c5f0fc1476af46b622be22675385091e3a08d07c00c8e56363d96530d9 +size 100 diff --git a/models/mlagility/requirements.txt b/models/mlagility/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/workflow_scripts/check_model.py b/workflow_scripts/check_model.py index 6174bcc76..2119e4e1a 100644 --- a/workflow_scripts/check_model.py +++ b/workflow_scripts/check_model.py @@ -16,7 +16,7 @@ def has_vnni_support(): def run_onnx_checker(model_path): model = onnx.load(model_path) - onnx.checker.check_model(model) + onnx.checker.check_model(model, full_check=True) def ort_skip_reason(model_path): @@ -66,3 +66,10 @@ def run_backend_ort(model_path, test_data_set=None, tar_gz_path=None): ort_test_dir_utils.run_test_dir(test_dir_from_tar) # remove the produced test_dir from ORT test_utils.remove_onnxruntime_test_dir() + +def run_backend_ort_with_data(model_path): + skip_reason = ort_skip_reason(model_path) + if skip_reason: + print(skip_reason) + return + ort_test_dir_utils.run_test_dir(model_path) diff --git a/workflow_scripts/test_models.py b/workflow_scripts/test_models.py index 2a2351ecb..9a99f6674 100644 --- a/workflow_scripts/test_models.py +++ b/workflow_scripts/test_models.py @@ -15,7 +15,7 @@ def get_all_models(): model_list = [] - for directory in ["text", "vision"]: + for directory in ["text", "vision", "models"]: for root, _, files in os.walk(directory): for file in files: if file.endswith(tar_ext_name) or file.endswith(onnx_ext_name): @@ -100,10 +100,35 @@ def main(): print("[PASS] {} is checked by onnx. ".format(model_name)) # check uploaded standalone ONNX model by ONNX elif onnx_ext_name in model_name: + test_utils.pull_lfs_file(model_path) if args.target == "onnx" or args.target == "all": - test_utils.pull_lfs_file(model_path) check_model.run_onnx_checker(model_path) print("[PASS] {} is checked by onnx. ".format(model_name)) + if args.target == "onnxruntime" or args.target == "all": + try: + # git lfs pull those test_data_set_* folders + root_dir = Path(model_path).parent + for _, dirs, _ in os.walk(root_dir): + for dir in dirs: + if "test_data_set_" in dir: + test_data_set_dir = os.path.join(root_dir, dir) + for _, _, files in os.walk(test_data_set_dir): + for file in files: + if file.endswith(".pb"): + test_utils.pull_lfs_file(os.path.join(test_data_set_dir, file)) + check_model.run_backend_ort_with_data(model_path) + print("[PASS] {} is checked by onnxruntime. ".format(model_name)) + except Exception as e: + if not args.create: + raise + else: + print("Warning: original test data for {} is broken: {}".format(model_path, e)) + test_utils.remove_onnxruntime_test_dir() + if (not model_name.endswith("-int8.onnx") and not model_name.endswith("-qdq.onnx")) or check_model.has_vnni_support(): + check_model.run_backend_ort(model_path, None, model_path) + else: + print("Skip quantized models because their test_data_set was created in avx512vnni machines. ") + print("[PASS] {} is checked by onnxruntime. ".format(model_name)) except Exception as e: print("[FAIL] {}: {}".format(model_name, e)) From 2e99292328254a6e3503c4628dce3b2d73ca3aa0 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 14 Jun 2023 10:33:10 -0700 Subject: [PATCH 42/56] add .yml Signed-off-by: jcwchen --- .github/workflows/mlagility_validation.yml | 41 +++++++++++++++++++++ models/mlagility/alexnet-18/alexnet-18.onnx | 4 +- 2 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/mlagility_validation.yml diff --git a/.github/workflows/mlagility_validation.yml b/.github/workflows/mlagility_validation.yml new file mode 100644 index 000000000..7a62aa6a3 --- /dev/null +++ b/.github/workflows/mlagility_validation.yml @@ -0,0 +1,41 @@ +name: Validate created ONNX model from mlagility + +on: + push: + branches: [ main, new-models] + pull_request: + branches: [ main, new-models] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.8'] + + steps: + - uses: actions/checkout@v3 + name: Checkout repo + - uses: conda-incubator/setup-miniconda@v2 + with: + miniconda-version: "latest" + activate-environment: mla + python-version: ${{ matrix.python-version }} + + - name: Install dependencies and mlagility + run: | + python -m pip install --upgrade pip + python -m pip install onnx onnxruntime requests py-cpuinfo + # Print CPU info for debugging ONNX Runtime inference difference + python -m cpuinfo + # or python -m pip install mlagility + python -m pip install transformers + git clone https://github.com/groq/mlagility.git + cd mlagility + pip install -r models/requirements.txt + pip install -e . + + - name: Validate created ONNX model from mlagility + run: | + pip install -r models/mlagility/requirements.txt + python workflow_scripts/run_mlagility.py diff --git a/models/mlagility/alexnet-18/alexnet-18.onnx b/models/mlagility/alexnet-18/alexnet-18.onnx index 354f9be58..e601daa98 100644 --- a/models/mlagility/alexnet-18/alexnet-18.onnx +++ b/models/mlagility/alexnet-18/alexnet-18.onnx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4bd1eb1e34c9c3e0501ccb1a9858cf2cc20e2360cf0b2e361a31c8cd0148bab6 -size 244407334 +oid sha256:3f954fd5efd73ebaf2095770fa69c0c7cd254042f1c3d819c33fc5c8fe5199fa +size 244407346 From 9a1c97af2bd051ac3e72821773a1c30ff342a62a Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 14 Jun 2023 13:18:17 -0700 Subject: [PATCH 43/56] ls Signed-off-by: jcwchen --- workflow_scripts/run_mlagility.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflow_scripts/run_mlagility.py b/workflow_scripts/run_mlagility.py index 4ff6deab8..d7123320d 100644 --- a/workflow_scripts/run_mlagility.py +++ b/workflow_scripts/run_mlagility.py @@ -9,7 +9,7 @@ base_name = "-op18-base.onnx" cwd_path = Path.cwd() mlagility_root = "mlagility/models" -mlagility_models_dir = "models/mlagility" +mlagility_models_dir = ".cache" ZOO_OPSET_VERSION = "18" errors = 0 @@ -20,6 +20,7 @@ subprocess.run(["benchit", osp.join(mlagility_root, script_path), "--cache-dir", mlagility_models_dir, "--onnx-opset", ZOO_OPSET_VERSION], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + subprocess.run(["ls", ".cache"], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) shutil.copy(osp.join(mlagility_models_dir, model_name, "onnx", model_name + base_name), final_model_path) subprocess.run(["git", "diff", "--exit-code", "--", final_model_path], cwd=cwd_path, stdout=subprocess.PIPE, From 6192ee7ff1c8afbece6e9aa859c5dcca9764997c Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 14 Jun 2023 13:26:57 -0700 Subject: [PATCH 44/56] stdout=sys.stdout Signed-off-by: jcwchen --- workflow_scripts/run_mlagility.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow_scripts/run_mlagility.py b/workflow_scripts/run_mlagility.py index d7123320d..a1ab69d8c 100644 --- a/workflow_scripts/run_mlagility.py +++ b/workflow_scripts/run_mlagility.py @@ -18,9 +18,9 @@ try: final_model_path = osp.join(mlagility_models_dir, model_zoo_path.replace(".onnx", "-" + ZOO_OPSET_VERSION + ".onnx")) subprocess.run(["benchit", osp.join(mlagility_root, script_path), "--cache-dir", mlagility_models_dir, "--onnx-opset", ZOO_OPSET_VERSION], - cwd=cwd_path, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - subprocess.run(["ls", ".cache"], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + cwd=cwd_path, stdout=sys.stdout, + stderr=stdout=sys.stdout) + subprocess.run(["ls", ".cache"], cwd=cwd_path, stdout=stdout=sys.stdout, stderr=stdout=sys.stdout) shutil.copy(osp.join(mlagility_models_dir, model_name, "onnx", model_name + base_name), final_model_path) subprocess.run(["git", "diff", "--exit-code", "--", final_model_path], cwd=cwd_path, stdout=subprocess.PIPE, From 53b3c98bfb420dd97f2280f0a96f84bade68dacd Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 14 Jun 2023 13:50:26 -0700 Subject: [PATCH 45/56] change commit Signed-off-by: jcwchen --- workflow_scripts/config.py | 4 ++-- workflow_scripts/run_mlagility.py | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/workflow_scripts/config.py b/workflow_scripts/config.py index fb67c1e30..4f5f2949e 100644 --- a/workflow_scripts/config.py +++ b/workflow_scripts/config.py @@ -1,5 +1,5 @@ models_info = [ # (script_path, model_name, model_zoo_path) - ("torch_hub/alexnet.py", "alexnet_torch_hub_2891f54c", "alexnet-18/alexnet-18.onnx"), - ("torchvision/fasterrcnn_resnet50_fpn_v2.py", "fasterrcnn_resnet50_fpn_v2_torchvision_ae446d48", "faster-rcnn/fasterrcnn_resnet50_fpn_v2.onnx"), + ("torch_hub/alexnet.py", "alexnet_torch_hub_7df2a577", "alexnet-18/alexnet-18.onnx"), + ("torchvision/fasterrcnn_resnet50_fpn_v2.py", "fasterrcnn_resnet50_fpn_v2_torchvision_ec445cac", "faster-rcnn/fasterrcnn_resnet50_fpn_v2.onnx"), ] diff --git a/workflow_scripts/run_mlagility.py b/workflow_scripts/run_mlagility.py index a1ab69d8c..95a331e86 100644 --- a/workflow_scripts/run_mlagility.py +++ b/workflow_scripts/run_mlagility.py @@ -18,13 +18,12 @@ try: final_model_path = osp.join(mlagility_models_dir, model_zoo_path.replace(".onnx", "-" + ZOO_OPSET_VERSION + ".onnx")) subprocess.run(["benchit", osp.join(mlagility_root, script_path), "--cache-dir", mlagility_models_dir, "--onnx-opset", ZOO_OPSET_VERSION], - cwd=cwd_path, stdout=sys.stdout, - stderr=stdout=sys.stdout) - subprocess.run(["ls", ".cache"], cwd=cwd_path, stdout=stdout=sys.stdout, stderr=stdout=sys.stdout) + cwd=cwd_path, stdout=subprocess.PIPE, + stderr=sys.stderr) shutil.copy(osp.join(mlagility_models_dir, model_name, "onnx", model_name + base_name), final_model_path) subprocess.run(["git", "diff", "--exit-code", "--", final_model_path], - cwd=cwd_path, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + cwd=cwd_path, stdout=sys.stdout, + stderr=sys.stderr) print(f"Successfully checked {model_zoo_path}.") except Exception as e: errors += 1 From 0158b32d9e75e1ffd7e20bbef4fab1ab2d3be358 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 14 Jun 2023 13:56:02 -0700 Subject: [PATCH 46/56] ls .cache Signed-off-by: jcwchen --- .github/workflows/mlagility_validation.yml | 1 + workflow_scripts/run_mlagility.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mlagility_validation.yml b/.github/workflows/mlagility_validation.yml index 7a62aa6a3..f7c09ec85 100644 --- a/.github/workflows/mlagility_validation.yml +++ b/.github/workflows/mlagility_validation.yml @@ -39,3 +39,4 @@ jobs: run: | pip install -r models/mlagility/requirements.txt python workflow_scripts/run_mlagility.py + ls .cache diff --git a/workflow_scripts/run_mlagility.py b/workflow_scripts/run_mlagility.py index 95a331e86..263a39606 100644 --- a/workflow_scripts/run_mlagility.py +++ b/workflow_scripts/run_mlagility.py @@ -26,7 +26,7 @@ stderr=sys.stderr) print(f"Successfully checked {model_zoo_path}.") except Exception as e: - errors += 1 + #errors += 1 print(f"Failed to check {model_zoo_path} because of {e}.") if errors > 0: From 6196319e475b04b318fca0f5165867e8a3a99aa9 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 14 Jun 2023 14:04:50 -0700 Subject: [PATCH 47/56] ls .cache/alexnet_torch_hub_7df2a577/onnx Signed-off-by: jcwchen --- .github/workflows/mlagility_validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mlagility_validation.yml b/.github/workflows/mlagility_validation.yml index f7c09ec85..b0ba76ab1 100644 --- a/.github/workflows/mlagility_validation.yml +++ b/.github/workflows/mlagility_validation.yml @@ -39,4 +39,4 @@ jobs: run: | pip install -r models/mlagility/requirements.txt python workflow_scripts/run_mlagility.py - ls .cache + ls .cache/alexnet_torch_hub_7df2a577/onnx From 2f6d3ee39500e663f43dc415f9bdfb86de71ab6b Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 14 Jun 2023 14:11:39 -0700 Subject: [PATCH 48/56] ls Signed-off-by: jcwchen --- .github/workflows/mlagility_validation.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/mlagility_validation.yml b/.github/workflows/mlagility_validation.yml index b0ba76ab1..7644c0ce6 100644 --- a/.github/workflows/mlagility_validation.yml +++ b/.github/workflows/mlagility_validation.yml @@ -39,4 +39,8 @@ jobs: run: | pip install -r models/mlagility/requirements.txt python workflow_scripts/run_mlagility.py + ls .cache/ + ls .cache/alexnet_torch_hub_7df2a577/ ls .cache/alexnet_torch_hub_7df2a577/onnx + ls .cache/fasterrcnn_resnet50_fpn_v2_torchvision_ec445cac/ + ls .cache/fasterrcnn_resnet50_fpn_v2_torchvision_ec445cac/onnx From cfeacf1f8100606ce035c9231c7137061f1ce011 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 14 Jun 2023 14:17:42 -0700 Subject: [PATCH 49/56] stdout=sys.stdout Signed-off-by: jcwchen --- workflow_scripts/run_mlagility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow_scripts/run_mlagility.py b/workflow_scripts/run_mlagility.py index 263a39606..36277fad8 100644 --- a/workflow_scripts/run_mlagility.py +++ b/workflow_scripts/run_mlagility.py @@ -18,7 +18,7 @@ try: final_model_path = osp.join(mlagility_models_dir, model_zoo_path.replace(".onnx", "-" + ZOO_OPSET_VERSION + ".onnx")) subprocess.run(["benchit", osp.join(mlagility_root, script_path), "--cache-dir", mlagility_models_dir, "--onnx-opset", ZOO_OPSET_VERSION], - cwd=cwd_path, stdout=subprocess.PIPE, + cwd=cwd_path, stdout=sys.stdout, stderr=sys.stderr) shutil.copy(osp.join(mlagility_models_dir, model_name, "onnx", model_name + base_name), final_model_path) subprocess.run(["git", "diff", "--exit-code", "--", final_model_path], From 38d44afcd2088925ae928310aaf16a2b6150e4fb Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 14 Jun 2023 14:26:57 -0700 Subject: [PATCH 50/56] torch==2.0.0 torchvision==0.15.1 Signed-off-by: jcwchen --- .github/workflows/mlagility_validation.yml | 5 ----- models/mlagility/requirements.txt | 2 ++ workflow_scripts/run_mlagility.py | 3 ++- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/mlagility_validation.yml b/.github/workflows/mlagility_validation.yml index 7644c0ce6..7a62aa6a3 100644 --- a/.github/workflows/mlagility_validation.yml +++ b/.github/workflows/mlagility_validation.yml @@ -39,8 +39,3 @@ jobs: run: | pip install -r models/mlagility/requirements.txt python workflow_scripts/run_mlagility.py - ls .cache/ - ls .cache/alexnet_torch_hub_7df2a577/ - ls .cache/alexnet_torch_hub_7df2a577/onnx - ls .cache/fasterrcnn_resnet50_fpn_v2_torchvision_ec445cac/ - ls .cache/fasterrcnn_resnet50_fpn_v2_torchvision_ec445cac/onnx diff --git a/models/mlagility/requirements.txt b/models/mlagility/requirements.txt index e69de29bb..66ba79ae3 100644 --- a/models/mlagility/requirements.txt +++ b/models/mlagility/requirements.txt @@ -0,0 +1,2 @@ +torch==2.0.0 +torchvision==0.15.1 diff --git a/workflow_scripts/run_mlagility.py b/workflow_scripts/run_mlagility.py index 36277fad8..0abf620e6 100644 --- a/workflow_scripts/run_mlagility.py +++ b/workflow_scripts/run_mlagility.py @@ -16,6 +16,7 @@ for script_path, model_name, model_zoo_path in config.models_info: try: + print(f"----------------Checking {model_zoo_path}----------------") final_model_path = osp.join(mlagility_models_dir, model_zoo_path.replace(".onnx", "-" + ZOO_OPSET_VERSION + ".onnx")) subprocess.run(["benchit", osp.join(mlagility_root, script_path), "--cache-dir", mlagility_models_dir, "--onnx-opset", ZOO_OPSET_VERSION], cwd=cwd_path, stdout=sys.stdout, @@ -26,7 +27,7 @@ stderr=sys.stderr) print(f"Successfully checked {model_zoo_path}.") except Exception as e: - #errors += 1 + errors += 1 print(f"Failed to check {model_zoo_path} because of {e}.") if errors > 0: From d7d3d2758de7b9e53367e8389139f21cecab3d3c Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 14 Jun 2023 14:38:25 -0700 Subject: [PATCH 51/56] mlagility_models_dir Signed-off-by: jcwchen --- workflow_scripts/run_mlagility.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/workflow_scripts/run_mlagility.py b/workflow_scripts/run_mlagility.py index 0abf620e6..abbcad75d 100644 --- a/workflow_scripts/run_mlagility.py +++ b/workflow_scripts/run_mlagility.py @@ -9,7 +9,8 @@ base_name = "-op18-base.onnx" cwd_path = Path.cwd() mlagility_root = "mlagility/models" -mlagility_models_dir = ".cache" +mlagility_models_dir = "models/mlagility" +cache_converted_dir = ".cache" ZOO_OPSET_VERSION = "18" errors = 0 @@ -18,10 +19,10 @@ try: print(f"----------------Checking {model_zoo_path}----------------") final_model_path = osp.join(mlagility_models_dir, model_zoo_path.replace(".onnx", "-" + ZOO_OPSET_VERSION + ".onnx")) - subprocess.run(["benchit", osp.join(mlagility_root, script_path), "--cache-dir", mlagility_models_dir, "--onnx-opset", ZOO_OPSET_VERSION], + subprocess.run(["benchit", osp.join(mlagility_root, script_path), "--cache-dir", cache_converted_dir, "--onnx-opset", ZOO_OPSET_VERSION], cwd=cwd_path, stdout=sys.stdout, stderr=sys.stderr) - shutil.copy(osp.join(mlagility_models_dir, model_name, "onnx", model_name + base_name), final_model_path) + shutil.copy(osp.join(cache_converted_dir, model_name, "onnx", model_name + base_name), final_model_path) subprocess.run(["git", "diff", "--exit-code", "--", final_model_path], cwd=cwd_path, stdout=sys.stdout, stderr=sys.stderr) From 7af0369af150b846406bc827b894b8589349fbf0 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Wed, 14 Jun 2023 14:49:11 -0700 Subject: [PATCH 52/56] new config Signed-off-by: jcwchen --- .../{alexnet-18/alexnet-18.onnx => alexnet/alexnet.onnx} | 0 .../{alexnet-18 => alexnet}/test_data_set_0/input_0.pb | 0 .../{alexnet-18 => alexnet}/test_data_set_0/output_0.pb | 0 .../fasterrcnn_resnet50_fpn_v2.onnx} | 0 .../test_data_set_0/input_0.pb | 0 .../test_data_set_0/output_0.pb | 0 .../test_data_set_0/output_1.pb | 0 .../test_data_set_0/output_2.pb | 0 workflow_scripts/config.py | 4 ++-- 9 files changed, 2 insertions(+), 2 deletions(-) rename models/mlagility/{alexnet-18/alexnet-18.onnx => alexnet/alexnet.onnx} (100%) rename models/mlagility/{alexnet-18 => alexnet}/test_data_set_0/input_0.pb (100%) rename models/mlagility/{alexnet-18 => alexnet}/test_data_set_0/output_0.pb (100%) rename models/mlagility/{fasterrcnn_resnet50_fpn_v2-18/fasterrcnn_resnet50_fpn_v2-18.onnx => fasterrcnn_resnet50_fpn_v2/fasterrcnn_resnet50_fpn_v2.onnx} (100%) rename models/mlagility/{fasterrcnn_resnet50_fpn_v2-18 => fasterrcnn_resnet50_fpn_v2}/test_data_set_0/input_0.pb (100%) rename models/mlagility/{fasterrcnn_resnet50_fpn_v2-18 => fasterrcnn_resnet50_fpn_v2}/test_data_set_0/output_0.pb (100%) rename models/mlagility/{fasterrcnn_resnet50_fpn_v2-18 => fasterrcnn_resnet50_fpn_v2}/test_data_set_0/output_1.pb (100%) rename models/mlagility/{fasterrcnn_resnet50_fpn_v2-18 => fasterrcnn_resnet50_fpn_v2}/test_data_set_0/output_2.pb (100%) diff --git a/models/mlagility/alexnet-18/alexnet-18.onnx b/models/mlagility/alexnet/alexnet.onnx similarity index 100% rename from models/mlagility/alexnet-18/alexnet-18.onnx rename to models/mlagility/alexnet/alexnet.onnx diff --git a/models/mlagility/alexnet-18/test_data_set_0/input_0.pb b/models/mlagility/alexnet/test_data_set_0/input_0.pb similarity index 100% rename from models/mlagility/alexnet-18/test_data_set_0/input_0.pb rename to models/mlagility/alexnet/test_data_set_0/input_0.pb diff --git a/models/mlagility/alexnet-18/test_data_set_0/output_0.pb b/models/mlagility/alexnet/test_data_set_0/output_0.pb similarity index 100% rename from models/mlagility/alexnet-18/test_data_set_0/output_0.pb rename to models/mlagility/alexnet/test_data_set_0/output_0.pb diff --git a/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/fasterrcnn_resnet50_fpn_v2-18.onnx b/models/mlagility/fasterrcnn_resnet50_fpn_v2/fasterrcnn_resnet50_fpn_v2.onnx similarity index 100% rename from models/mlagility/fasterrcnn_resnet50_fpn_v2-18/fasterrcnn_resnet50_fpn_v2-18.onnx rename to models/mlagility/fasterrcnn_resnet50_fpn_v2/fasterrcnn_resnet50_fpn_v2.onnx diff --git a/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/input_0.pb b/models/mlagility/fasterrcnn_resnet50_fpn_v2/test_data_set_0/input_0.pb similarity index 100% rename from models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/input_0.pb rename to models/mlagility/fasterrcnn_resnet50_fpn_v2/test_data_set_0/input_0.pb diff --git a/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_0.pb b/models/mlagility/fasterrcnn_resnet50_fpn_v2/test_data_set_0/output_0.pb similarity index 100% rename from models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_0.pb rename to models/mlagility/fasterrcnn_resnet50_fpn_v2/test_data_set_0/output_0.pb diff --git a/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_1.pb b/models/mlagility/fasterrcnn_resnet50_fpn_v2/test_data_set_0/output_1.pb similarity index 100% rename from models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_1.pb rename to models/mlagility/fasterrcnn_resnet50_fpn_v2/test_data_set_0/output_1.pb diff --git a/models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_2.pb b/models/mlagility/fasterrcnn_resnet50_fpn_v2/test_data_set_0/output_2.pb similarity index 100% rename from models/mlagility/fasterrcnn_resnet50_fpn_v2-18/test_data_set_0/output_2.pb rename to models/mlagility/fasterrcnn_resnet50_fpn_v2/test_data_set_0/output_2.pb diff --git a/workflow_scripts/config.py b/workflow_scripts/config.py index 4f5f2949e..1a36bb7c4 100644 --- a/workflow_scripts/config.py +++ b/workflow_scripts/config.py @@ -1,5 +1,5 @@ models_info = [ # (script_path, model_name, model_zoo_path) - ("torch_hub/alexnet.py", "alexnet_torch_hub_7df2a577", "alexnet-18/alexnet-18.onnx"), - ("torchvision/fasterrcnn_resnet50_fpn_v2.py", "fasterrcnn_resnet50_fpn_v2_torchvision_ec445cac", "faster-rcnn/fasterrcnn_resnet50_fpn_v2.onnx"), + ("torch_hub/alexnet.py", "alexnet_torch_hub_7df2a577", "alexnet/alexnet.onnx"), + ("torchvision/fasterrcnn_resnet50_fpn_v2.py", "fasterrcnn_resnet50_fpn_v2_torchvision_ec445cac", "fasterrcnn_resnet50_fpn_v2/fasterrcnn_resnet50_fpn_v2.onnx"), ] From 355c6c31b4617fea573b5349d5737e77487ded55 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Thu, 15 Jun 2023 12:05:31 -0700 Subject: [PATCH 53/56] add 3 more models Signed-off-by: jcwchen --- models/mlagility/bert/bert.onnx | 3 +++ models/mlagility/gpt2/gpt2.onnx | 3 +++ models/mlagility/resnet50/resnet50.onnx | 3 +++ 3 files changed, 9 insertions(+) create mode 100644 models/mlagility/bert/bert.onnx create mode 100644 models/mlagility/gpt2/gpt2.onnx create mode 100644 models/mlagility/resnet50/resnet50.onnx diff --git a/models/mlagility/bert/bert.onnx b/models/mlagility/bert/bert.onnx new file mode 100644 index 000000000..0edc4d781 --- /dev/null +++ b/models/mlagility/bert/bert.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88a61279e3e81faa0ccd001f16684eb14f6cdd0ba2b0d84b732e7eda9334f8f2 +size 437579714 diff --git a/models/mlagility/gpt2/gpt2.onnx b/models/mlagility/gpt2/gpt2.onnx new file mode 100644 index 000000000..2cb41b81c --- /dev/null +++ b/models/mlagility/gpt2/gpt2.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1cdc65d364d30bfe2f05bf5ef5737ecbdc7514ee40fc3bcbec1137a9e761c7c +size 497427212 diff --git a/models/mlagility/resnet50/resnet50.onnx b/models/mlagility/resnet50/resnet50.onnx new file mode 100644 index 000000000..e682f9114 --- /dev/null +++ b/models/mlagility/resnet50/resnet50.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:837e0e4e6fe7fcd48c3b9e222b033ee580739618f3ba064ce90bfbbaa8e427cf +size 102057631 From 4dae47aeb9df30dc78cec281637a37b745f2bf85 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Thu, 15 Jun 2023 14:10:50 -0700 Subject: [PATCH 54/56] add config Signed-off-by: jcwchen --- models/mlagility/bert/test_data_set_1/input_0.pb | 3 +++ models/mlagility/bert/test_data_set_1/input_1.pb | 3 +++ models/mlagility/bert/test_data_set_1/output_0.pb | 3 +++ models/mlagility/bert/test_data_set_1/output_1.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/input_0.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/input_1.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_0.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_1.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_10.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_11.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_12.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_13.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_14.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_15.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_16.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_17.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_18.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_19.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_2.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_20.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_21.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_22.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_23.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_24.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_3.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_4.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_5.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_6.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_7.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_8.pb | 3 +++ models/mlagility/gpt2/test_data_set_0/output_9.pb | 3 +++ models/mlagility/resnet50/test_data_set_0/input_0.pb | 3 +++ models/mlagility/resnet50/test_data_set_0/output_0.pb | 3 +++ workflow_scripts/config.py | 3 +++ 34 files changed, 102 insertions(+) create mode 100644 models/mlagility/bert/test_data_set_1/input_0.pb create mode 100644 models/mlagility/bert/test_data_set_1/input_1.pb create mode 100644 models/mlagility/bert/test_data_set_1/output_0.pb create mode 100644 models/mlagility/bert/test_data_set_1/output_1.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/input_0.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/input_1.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_0.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_1.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_10.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_11.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_12.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_13.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_14.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_15.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_16.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_17.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_18.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_19.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_2.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_20.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_21.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_22.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_23.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_24.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_3.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_4.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_5.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_6.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_7.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_8.pb create mode 100644 models/mlagility/gpt2/test_data_set_0/output_9.pb create mode 100644 models/mlagility/resnet50/test_data_set_0/input_0.pb create mode 100644 models/mlagility/resnet50/test_data_set_0/output_0.pb diff --git a/models/mlagility/bert/test_data_set_1/input_0.pb b/models/mlagility/bert/test_data_set_1/input_0.pb new file mode 100644 index 000000000..f122ae8fc --- /dev/null +++ b/models/mlagility/bert/test_data_set_1/input_0.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f2d3d51941e7e5f1ea51b6ca2e63468439e70770dc4331c516ee9d8c5b56e52 +size 1045 diff --git a/models/mlagility/bert/test_data_set_1/input_1.pb b/models/mlagility/bert/test_data_set_1/input_1.pb new file mode 100644 index 000000000..4286e0260 --- /dev/null +++ b/models/mlagility/bert/test_data_set_1/input_1.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f72f15842b13b6818247300edfc2a8defebe32c118a9ed5fe40836138c35cabb +size 538 diff --git a/models/mlagility/bert/test_data_set_1/output_0.pb b/models/mlagility/bert/test_data_set_1/output_0.pb new file mode 100644 index 000000000..84bfc4617 --- /dev/null +++ b/models/mlagility/bert/test_data_set_1/output_0.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:056aae95255646f0c9c3266ce2483758b2df8886f90979eed0a1c224127c03fd +size 393249 diff --git a/models/mlagility/bert/test_data_set_1/output_1.pb b/models/mlagility/bert/test_data_set_1/output_1.pb new file mode 100644 index 000000000..2aa1e5ef8 --- /dev/null +++ b/models/mlagility/bert/test_data_set_1/output_1.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea832845573e6986a5d592c567c40cfcb59b9104ad7e37f6c8385e708b6d699d +size 3088 diff --git a/models/mlagility/gpt2/test_data_set_0/input_0.pb b/models/mlagility/gpt2/test_data_set_0/input_0.pb new file mode 100644 index 000000000..adfa3d306 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/input_0.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be09fb02827652c36c26f915f6086fc452dcf51f05878b31eed715d0ab4dc420 +size 1045 diff --git a/models/mlagility/gpt2/test_data_set_0/input_1.pb b/models/mlagility/gpt2/test_data_set_0/input_1.pb new file mode 100644 index 000000000..e0718e23e --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/input_1.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76479b1a6fe72e89dab58d77cc13fd7ce4cd23b39109415d0347de2a8cd1965a +size 538 diff --git a/models/mlagility/gpt2/test_data_set_0/output_0.pb b/models/mlagility/gpt2/test_data_set_0/output_0.pb new file mode 100644 index 000000000..fdbf97886 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_0.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a22f093912d38b75422507b64c8e560b3d07b13299d67f662a8e451b09fa83f2 +size 393236 diff --git a/models/mlagility/gpt2/test_data_set_0/output_1.pb b/models/mlagility/gpt2/test_data_set_0/output_1.pb new file mode 100644 index 000000000..1da24f5b9 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_1.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6b72222325be3bfe166131fe237e85e8fabcba54a655863401d3cf365538a6b +size 393238 diff --git a/models/mlagility/gpt2/test_data_set_0/output_10.pb b/models/mlagility/gpt2/test_data_set_0/output_10.pb new file mode 100644 index 000000000..3f7607b89 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_10.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74fe653d69bb750c8b30a404e77b27d7606a95082972d4ece0a0fd6d21e04ef3 +size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_11.pb b/models/mlagility/gpt2/test_data_set_0/output_11.pb new file mode 100644 index 000000000..ae682567f --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_11.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c0435c6c55f8cd958adaf488d0bf0641904d5026daa840b53f7c8f971d448eb +size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_12.pb b/models/mlagility/gpt2/test_data_set_0/output_12.pb new file mode 100644 index 000000000..1e08ebb51 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_12.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d519f090e420c39e6048a1f85a9025de4ef6d340404a933ed417bd4389dea957 +size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_13.pb b/models/mlagility/gpt2/test_data_set_0/output_13.pb new file mode 100644 index 000000000..999e2e108 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_13.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a6ba8cd889ddf3a4d15c955f337a908811b236d2c5819965d2dab77811d677d +size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_14.pb b/models/mlagility/gpt2/test_data_set_0/output_14.pb new file mode 100644 index 000000000..36ca9be19 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_14.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ec444e09803bd0e8fde860a03fb6bbcb77bc80d3fd0531e407bb6eac3b8c588 +size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_15.pb b/models/mlagility/gpt2/test_data_set_0/output_15.pb new file mode 100644 index 000000000..40f739312 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_15.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6935c1b97fda5b136b0db796a19c0849190b442d19aaa5d0d229b9658252b33e +size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_16.pb b/models/mlagility/gpt2/test_data_set_0/output_16.pb new file mode 100644 index 000000000..0459cc222 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_16.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f258f37e7aa637d6517079202dc92f569daa8968944b002bf16a6bec145be86d +size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_17.pb b/models/mlagility/gpt2/test_data_set_0/output_17.pb new file mode 100644 index 000000000..b2d5aef57 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_17.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44f85c9fc1bf90d5359f0c0aa226f5324fc5e8cc1c06a56346f15987694a8eaf +size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_18.pb b/models/mlagility/gpt2/test_data_set_0/output_18.pb new file mode 100644 index 000000000..a1c7914b2 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_18.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:517605eaeb9b03c1b9175963936f59c6a04feb2f05155a02ea855b75edffadc6 +size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_19.pb b/models/mlagility/gpt2/test_data_set_0/output_19.pb new file mode 100644 index 000000000..7f675f57b --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_19.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4047ac9af4038f96429e077fbd48d3210abe6aa4c3bfdc399faeecbb09a74b8 +size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_2.pb b/models/mlagility/gpt2/test_data_set_0/output_2.pb new file mode 100644 index 000000000..c7345d335 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_2.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9f72c93010870878d78772947f0338daa88875af1b61ef15f80f16cd188a371 +size 393240 diff --git a/models/mlagility/gpt2/test_data_set_0/output_20.pb b/models/mlagility/gpt2/test_data_set_0/output_20.pb new file mode 100644 index 000000000..37af48397 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_20.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc0cc48e8785778ae5afb4e7e9e6279544dab96f080ffd8aa778134c48d97a46 +size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_21.pb b/models/mlagility/gpt2/test_data_set_0/output_21.pb new file mode 100644 index 000000000..d9b9171d3 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_21.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9a5240bb25d24ea0ad2fc75f5e1484f6741f8d9afac69cbc82ef21245400f60 +size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_22.pb b/models/mlagility/gpt2/test_data_set_0/output_22.pb new file mode 100644 index 000000000..8307322df --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_22.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b2a041737c2b9d78e0567fd592e8f8d08dd63a9efc37f6c027f45601d034aad +size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_23.pb b/models/mlagility/gpt2/test_data_set_0/output_23.pb new file mode 100644 index 000000000..018c95fc9 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_23.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb7ea4801b8b726251a4ccc59d25a70315bfecdae7c2544585f3e7784859570b +size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_24.pb b/models/mlagility/gpt2/test_data_set_0/output_24.pb new file mode 100644 index 000000000..c69803c1c --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_24.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abc91f2ab295173e52d76f8bdbdb696f96d4379f42b478d4b0e26491a704c497 +size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_3.pb b/models/mlagility/gpt2/test_data_set_0/output_3.pb new file mode 100644 index 000000000..5c0ac6383 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_3.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e5b21f2cc47c7443fa47218ed48bea90668446fd93154225a4e5e2342a318a0 +size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_4.pb b/models/mlagility/gpt2/test_data_set_0/output_4.pb new file mode 100644 index 000000000..269c510fc --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_4.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c472ce06d57acfe3ca455225dc1c1d3169cd9acd20269cc981b1a6bb5b5ce90 +size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_5.pb b/models/mlagility/gpt2/test_data_set_0/output_5.pb new file mode 100644 index 000000000..81f449adc --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_5.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6379c99ff91a45c3ad909d98c99876a4cc011b4bc87d8add50ebc836865e4a6a +size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_6.pb b/models/mlagility/gpt2/test_data_set_0/output_6.pb new file mode 100644 index 000000000..e9bc62fee --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_6.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83650e18bfc38932b134bae1cd0edd3f28c4e19c792da164fb57dab0ec999b0e +size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_7.pb b/models/mlagility/gpt2/test_data_set_0/output_7.pb new file mode 100644 index 000000000..be59157f2 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_7.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ba99caaf4290b8e236affb94bf211c3207bdc522a4bbaefc16050ef6c21fe1b +size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_8.pb b/models/mlagility/gpt2/test_data_set_0/output_8.pb new file mode 100644 index 000000000..13aa9b155 --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_8.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bec2818b37d1dff0c0c03591b943fd6c0d3c62a11fbe27a42f8ae94e011b1ca +size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_9.pb b/models/mlagility/gpt2/test_data_set_0/output_9.pb new file mode 100644 index 000000000..d7330777e --- /dev/null +++ b/models/mlagility/gpt2/test_data_set_0/output_9.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c83918ed52d529670eedbbb3aa6f4a5ba631c28e8638037f468994a6efba1eb1 +size 393239 diff --git a/models/mlagility/resnet50/test_data_set_0/input_0.pb b/models/mlagility/resnet50/test_data_set_0/input_0.pb new file mode 100644 index 000000000..7721ce265 --- /dev/null +++ b/models/mlagility/resnet50/test_data_set_0/input_0.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82753266743285dd73bcfcf936ac5e9dd356135230deff82f1b7d2b877876205 +size 602131 diff --git a/models/mlagility/resnet50/test_data_set_0/output_0.pb b/models/mlagility/resnet50/test_data_set_0/output_0.pb new file mode 100644 index 000000000..82135a819 --- /dev/null +++ b/models/mlagility/resnet50/test_data_set_0/output_0.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f4b52fce5e3009c79693c0de44c9900cd1d8fa52d17f1a4d03b907fbd7b2aee +size 4015 diff --git a/workflow_scripts/config.py b/workflow_scripts/config.py index 1a36bb7c4..c27407053 100644 --- a/workflow_scripts/config.py +++ b/workflow_scripts/config.py @@ -1,5 +1,8 @@ models_info = [ # (script_path, model_name, model_zoo_path) ("torch_hub/alexnet.py", "alexnet_torch_hub_7df2a577", "alexnet/alexnet.onnx"), + ("torch_hub/resnet50.py", "resnet50_torch_hub_31acb52e", "resnet/resent50.onnx"), ("torchvision/fasterrcnn_resnet50_fpn_v2.py", "fasterrcnn_resnet50_fpn_v2_torchvision_ec445cac", "fasterrcnn_resnet50_fpn_v2/fasterrcnn_resnet50_fpn_v2.onnx"), + ("transformers/bert.py", "bert_transformers_c764983a", "bert/bert.onnx"), + ("transformers/gpt2.py", "gpt2_transformers_73ad0e00", "gpt2/gpt2.onnx"), ] From 0e1a570ba603d7b0725aac218c1de809056633a4 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Thu, 15 Jun 2023 15:30:15 -0700 Subject: [PATCH 55/56] remove bert and gpt2 for now Signed-off-by: jcwchen --- models/mlagility/bert/bert.onnx | 3 --- models/mlagility/bert/test_data_set_1/input_0.pb | 3 --- models/mlagility/bert/test_data_set_1/input_1.pb | 3 --- models/mlagility/bert/test_data_set_1/output_0.pb | 3 --- models/mlagility/bert/test_data_set_1/output_1.pb | 3 --- models/mlagility/gpt2/gpt2.onnx | 3 --- models/mlagility/gpt2/test_data_set_0/input_0.pb | 3 --- models/mlagility/gpt2/test_data_set_0/input_1.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_0.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_1.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_10.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_11.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_12.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_13.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_14.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_15.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_16.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_17.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_18.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_19.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_2.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_20.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_21.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_22.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_23.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_24.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_3.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_4.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_5.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_6.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_7.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_8.pb | 3 --- models/mlagility/gpt2/test_data_set_0/output_9.pb | 3 --- workflow_scripts/config.py | 2 -- 34 files changed, 101 deletions(-) delete mode 100644 models/mlagility/bert/bert.onnx delete mode 100644 models/mlagility/bert/test_data_set_1/input_0.pb delete mode 100644 models/mlagility/bert/test_data_set_1/input_1.pb delete mode 100644 models/mlagility/bert/test_data_set_1/output_0.pb delete mode 100644 models/mlagility/bert/test_data_set_1/output_1.pb delete mode 100644 models/mlagility/gpt2/gpt2.onnx delete mode 100644 models/mlagility/gpt2/test_data_set_0/input_0.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/input_1.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_0.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_1.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_10.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_11.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_12.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_13.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_14.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_15.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_16.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_17.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_18.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_19.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_2.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_20.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_21.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_22.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_23.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_24.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_3.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_4.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_5.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_6.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_7.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_8.pb delete mode 100644 models/mlagility/gpt2/test_data_set_0/output_9.pb diff --git a/models/mlagility/bert/bert.onnx b/models/mlagility/bert/bert.onnx deleted file mode 100644 index 0edc4d781..000000000 --- a/models/mlagility/bert/bert.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:88a61279e3e81faa0ccd001f16684eb14f6cdd0ba2b0d84b732e7eda9334f8f2 -size 437579714 diff --git a/models/mlagility/bert/test_data_set_1/input_0.pb b/models/mlagility/bert/test_data_set_1/input_0.pb deleted file mode 100644 index f122ae8fc..000000000 --- a/models/mlagility/bert/test_data_set_1/input_0.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0f2d3d51941e7e5f1ea51b6ca2e63468439e70770dc4331c516ee9d8c5b56e52 -size 1045 diff --git a/models/mlagility/bert/test_data_set_1/input_1.pb b/models/mlagility/bert/test_data_set_1/input_1.pb deleted file mode 100644 index 4286e0260..000000000 --- a/models/mlagility/bert/test_data_set_1/input_1.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f72f15842b13b6818247300edfc2a8defebe32c118a9ed5fe40836138c35cabb -size 538 diff --git a/models/mlagility/bert/test_data_set_1/output_0.pb b/models/mlagility/bert/test_data_set_1/output_0.pb deleted file mode 100644 index 84bfc4617..000000000 --- a/models/mlagility/bert/test_data_set_1/output_0.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:056aae95255646f0c9c3266ce2483758b2df8886f90979eed0a1c224127c03fd -size 393249 diff --git a/models/mlagility/bert/test_data_set_1/output_1.pb b/models/mlagility/bert/test_data_set_1/output_1.pb deleted file mode 100644 index 2aa1e5ef8..000000000 --- a/models/mlagility/bert/test_data_set_1/output_1.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ea832845573e6986a5d592c567c40cfcb59b9104ad7e37f6c8385e708b6d699d -size 3088 diff --git a/models/mlagility/gpt2/gpt2.onnx b/models/mlagility/gpt2/gpt2.onnx deleted file mode 100644 index 2cb41b81c..000000000 --- a/models/mlagility/gpt2/gpt2.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a1cdc65d364d30bfe2f05bf5ef5737ecbdc7514ee40fc3bcbec1137a9e761c7c -size 497427212 diff --git a/models/mlagility/gpt2/test_data_set_0/input_0.pb b/models/mlagility/gpt2/test_data_set_0/input_0.pb deleted file mode 100644 index adfa3d306..000000000 --- a/models/mlagility/gpt2/test_data_set_0/input_0.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:be09fb02827652c36c26f915f6086fc452dcf51f05878b31eed715d0ab4dc420 -size 1045 diff --git a/models/mlagility/gpt2/test_data_set_0/input_1.pb b/models/mlagility/gpt2/test_data_set_0/input_1.pb deleted file mode 100644 index e0718e23e..000000000 --- a/models/mlagility/gpt2/test_data_set_0/input_1.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:76479b1a6fe72e89dab58d77cc13fd7ce4cd23b39109415d0347de2a8cd1965a -size 538 diff --git a/models/mlagility/gpt2/test_data_set_0/output_0.pb b/models/mlagility/gpt2/test_data_set_0/output_0.pb deleted file mode 100644 index fdbf97886..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_0.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a22f093912d38b75422507b64c8e560b3d07b13299d67f662a8e451b09fa83f2 -size 393236 diff --git a/models/mlagility/gpt2/test_data_set_0/output_1.pb b/models/mlagility/gpt2/test_data_set_0/output_1.pb deleted file mode 100644 index 1da24f5b9..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_1.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d6b72222325be3bfe166131fe237e85e8fabcba54a655863401d3cf365538a6b -size 393238 diff --git a/models/mlagility/gpt2/test_data_set_0/output_10.pb b/models/mlagility/gpt2/test_data_set_0/output_10.pb deleted file mode 100644 index 3f7607b89..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_10.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:74fe653d69bb750c8b30a404e77b27d7606a95082972d4ece0a0fd6d21e04ef3 -size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_11.pb b/models/mlagility/gpt2/test_data_set_0/output_11.pb deleted file mode 100644 index ae682567f..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_11.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4c0435c6c55f8cd958adaf488d0bf0641904d5026daa840b53f7c8f971d448eb -size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_12.pb b/models/mlagility/gpt2/test_data_set_0/output_12.pb deleted file mode 100644 index 1e08ebb51..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_12.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d519f090e420c39e6048a1f85a9025de4ef6d340404a933ed417bd4389dea957 -size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_13.pb b/models/mlagility/gpt2/test_data_set_0/output_13.pb deleted file mode 100644 index 999e2e108..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_13.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1a6ba8cd889ddf3a4d15c955f337a908811b236d2c5819965d2dab77811d677d -size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_14.pb b/models/mlagility/gpt2/test_data_set_0/output_14.pb deleted file mode 100644 index 36ca9be19..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_14.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3ec444e09803bd0e8fde860a03fb6bbcb77bc80d3fd0531e407bb6eac3b8c588 -size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_15.pb b/models/mlagility/gpt2/test_data_set_0/output_15.pb deleted file mode 100644 index 40f739312..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_15.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6935c1b97fda5b136b0db796a19c0849190b442d19aaa5d0d229b9658252b33e -size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_16.pb b/models/mlagility/gpt2/test_data_set_0/output_16.pb deleted file mode 100644 index 0459cc222..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_16.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f258f37e7aa637d6517079202dc92f569daa8968944b002bf16a6bec145be86d -size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_17.pb b/models/mlagility/gpt2/test_data_set_0/output_17.pb deleted file mode 100644 index b2d5aef57..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_17.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:44f85c9fc1bf90d5359f0c0aa226f5324fc5e8cc1c06a56346f15987694a8eaf -size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_18.pb b/models/mlagility/gpt2/test_data_set_0/output_18.pb deleted file mode 100644 index a1c7914b2..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_18.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:517605eaeb9b03c1b9175963936f59c6a04feb2f05155a02ea855b75edffadc6 -size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_19.pb b/models/mlagility/gpt2/test_data_set_0/output_19.pb deleted file mode 100644 index 7f675f57b..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_19.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a4047ac9af4038f96429e077fbd48d3210abe6aa4c3bfdc399faeecbb09a74b8 -size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_2.pb b/models/mlagility/gpt2/test_data_set_0/output_2.pb deleted file mode 100644 index c7345d335..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_2.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f9f72c93010870878d78772947f0338daa88875af1b61ef15f80f16cd188a371 -size 393240 diff --git a/models/mlagility/gpt2/test_data_set_0/output_20.pb b/models/mlagility/gpt2/test_data_set_0/output_20.pb deleted file mode 100644 index 37af48397..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_20.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:dc0cc48e8785778ae5afb4e7e9e6279544dab96f080ffd8aa778134c48d97a46 -size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_21.pb b/models/mlagility/gpt2/test_data_set_0/output_21.pb deleted file mode 100644 index d9b9171d3..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_21.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a9a5240bb25d24ea0ad2fc75f5e1484f6741f8d9afac69cbc82ef21245400f60 -size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_22.pb b/models/mlagility/gpt2/test_data_set_0/output_22.pb deleted file mode 100644 index 8307322df..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_22.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2b2a041737c2b9d78e0567fd592e8f8d08dd63a9efc37f6c027f45601d034aad -size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_23.pb b/models/mlagility/gpt2/test_data_set_0/output_23.pb deleted file mode 100644 index 018c95fc9..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_23.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:fb7ea4801b8b726251a4ccc59d25a70315bfecdae7c2544585f3e7784859570b -size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_24.pb b/models/mlagility/gpt2/test_data_set_0/output_24.pb deleted file mode 100644 index c69803c1c..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_24.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:abc91f2ab295173e52d76f8bdbdb696f96d4379f42b478d4b0e26491a704c497 -size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_3.pb b/models/mlagility/gpt2/test_data_set_0/output_3.pb deleted file mode 100644 index 5c0ac6383..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_3.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3e5b21f2cc47c7443fa47218ed48bea90668446fd93154225a4e5e2342a318a0 -size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_4.pb b/models/mlagility/gpt2/test_data_set_0/output_4.pb deleted file mode 100644 index 269c510fc..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_4.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6c472ce06d57acfe3ca455225dc1c1d3169cd9acd20269cc981b1a6bb5b5ce90 -size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_5.pb b/models/mlagility/gpt2/test_data_set_0/output_5.pb deleted file mode 100644 index 81f449adc..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_5.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6379c99ff91a45c3ad909d98c99876a4cc011b4bc87d8add50ebc836865e4a6a -size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_6.pb b/models/mlagility/gpt2/test_data_set_0/output_6.pb deleted file mode 100644 index e9bc62fee..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_6.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:83650e18bfc38932b134bae1cd0edd3f28c4e19c792da164fb57dab0ec999b0e -size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_7.pb b/models/mlagility/gpt2/test_data_set_0/output_7.pb deleted file mode 100644 index be59157f2..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_7.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2ba99caaf4290b8e236affb94bf211c3207bdc522a4bbaefc16050ef6c21fe1b -size 393239 diff --git a/models/mlagility/gpt2/test_data_set_0/output_8.pb b/models/mlagility/gpt2/test_data_set_0/output_8.pb deleted file mode 100644 index 13aa9b155..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_8.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7bec2818b37d1dff0c0c03591b943fd6c0d3c62a11fbe27a42f8ae94e011b1ca -size 393241 diff --git a/models/mlagility/gpt2/test_data_set_0/output_9.pb b/models/mlagility/gpt2/test_data_set_0/output_9.pb deleted file mode 100644 index d7330777e..000000000 --- a/models/mlagility/gpt2/test_data_set_0/output_9.pb +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c83918ed52d529670eedbbb3aa6f4a5ba631c28e8638037f468994a6efba1eb1 -size 393239 diff --git a/workflow_scripts/config.py b/workflow_scripts/config.py index c27407053..79c70b8fa 100644 --- a/workflow_scripts/config.py +++ b/workflow_scripts/config.py @@ -3,6 +3,4 @@ ("torch_hub/alexnet.py", "alexnet_torch_hub_7df2a577", "alexnet/alexnet.onnx"), ("torch_hub/resnet50.py", "resnet50_torch_hub_31acb52e", "resnet/resent50.onnx"), ("torchvision/fasterrcnn_resnet50_fpn_v2.py", "fasterrcnn_resnet50_fpn_v2_torchvision_ec445cac", "fasterrcnn_resnet50_fpn_v2/fasterrcnn_resnet50_fpn_v2.onnx"), - ("transformers/bert.py", "bert_transformers_c764983a", "bert/bert.onnx"), - ("transformers/gpt2.py", "gpt2_transformers_73ad0e00", "gpt2/gpt2.onnx"), ] From 0ef4ef4e3abb0cf8459639893c65ab808e261079 Mon Sep 17 00:00:00 2001 From: jcwchen Date: Thu, 15 Jun 2023 15:56:30 -0700 Subject: [PATCH 56/56] clean path Signed-off-by: jcwchen --- .github/workflows/mlagility_validation.yml | 2 -- .../alexnet/{alexnet.onnx => alexnet-18.onnx} | 0 ....onnx => fasterrcnn_resnet50_fpn_v2-18.onnx} | 0 .../{resnet50.onnx => resnet50-18.onnx} | 0 workflow_scripts/config.py | 8 ++++---- workflow_scripts/run_mlagility.py | 17 +++++++++-------- 6 files changed, 13 insertions(+), 14 deletions(-) rename models/mlagility/alexnet/{alexnet.onnx => alexnet-18.onnx} (100%) rename models/mlagility/fasterrcnn_resnet50_fpn_v2/{fasterrcnn_resnet50_fpn_v2.onnx => fasterrcnn_resnet50_fpn_v2-18.onnx} (100%) rename models/mlagility/resnet50/{resnet50.onnx => resnet50-18.onnx} (100%) diff --git a/.github/workflows/mlagility_validation.yml b/.github/workflows/mlagility_validation.yml index 7a62aa6a3..a7611b5b1 100644 --- a/.github/workflows/mlagility_validation.yml +++ b/.github/workflows/mlagility_validation.yml @@ -28,8 +28,6 @@ jobs: python -m pip install onnx onnxruntime requests py-cpuinfo # Print CPU info for debugging ONNX Runtime inference difference python -m cpuinfo - # or python -m pip install mlagility - python -m pip install transformers git clone https://github.com/groq/mlagility.git cd mlagility pip install -r models/requirements.txt diff --git a/models/mlagility/alexnet/alexnet.onnx b/models/mlagility/alexnet/alexnet-18.onnx similarity index 100% rename from models/mlagility/alexnet/alexnet.onnx rename to models/mlagility/alexnet/alexnet-18.onnx diff --git a/models/mlagility/fasterrcnn_resnet50_fpn_v2/fasterrcnn_resnet50_fpn_v2.onnx b/models/mlagility/fasterrcnn_resnet50_fpn_v2/fasterrcnn_resnet50_fpn_v2-18.onnx similarity index 100% rename from models/mlagility/fasterrcnn_resnet50_fpn_v2/fasterrcnn_resnet50_fpn_v2.onnx rename to models/mlagility/fasterrcnn_resnet50_fpn_v2/fasterrcnn_resnet50_fpn_v2-18.onnx diff --git a/models/mlagility/resnet50/resnet50.onnx b/models/mlagility/resnet50/resnet50-18.onnx similarity index 100% rename from models/mlagility/resnet50/resnet50.onnx rename to models/mlagility/resnet50/resnet50-18.onnx diff --git a/workflow_scripts/config.py b/workflow_scripts/config.py index 79c70b8fa..a757924a7 100644 --- a/workflow_scripts/config.py +++ b/workflow_scripts/config.py @@ -1,6 +1,6 @@ models_info = [ - # (script_path, model_name, model_zoo_path) - ("torch_hub/alexnet.py", "alexnet_torch_hub_7df2a577", "alexnet/alexnet.onnx"), - ("torch_hub/resnet50.py", "resnet50_torch_hub_31acb52e", "resnet/resent50.onnx"), - ("torchvision/fasterrcnn_resnet50_fpn_v2.py", "fasterrcnn_resnet50_fpn_v2_torchvision_ec445cac", "fasterrcnn_resnet50_fpn_v2/fasterrcnn_resnet50_fpn_v2.onnx"), + # (script_path, model_name, model_zoo_directory) + ("torch_hub/alexnet.py", "alexnet_torch_hub_7df2a577", "alexnet"), + ("torch_hub/resnet50.py", "resnet50_torch_hub_31acb52e", "resnet50"), + ("torchvision/fasterrcnn_resnet50_fpn_v2.py", "fasterrcnn_resnet50_fpn_v2_torchvision_ec445cac", "fasterrcnn_resnet50_fpn_v2"), ] diff --git a/workflow_scripts/run_mlagility.py b/workflow_scripts/run_mlagility.py index abbcad75d..92ea3972a 100644 --- a/workflow_scripts/run_mlagility.py +++ b/workflow_scripts/run_mlagility.py @@ -6,30 +6,31 @@ import sys -base_name = "-op18-base.onnx" +ZOO_OPSET_VERSION = "18" +base_name = f"-op{ZOO_OPSET_VERSION}-base.onnx" cwd_path = Path.cwd() mlagility_root = "mlagility/models" mlagility_models_dir = "models/mlagility" cache_converted_dir = ".cache" -ZOO_OPSET_VERSION = "18" errors = 0 -for script_path, model_name, model_zoo_path in config.models_info: +for script_path, model_name, model_zoo_dir in config.models_info: try: - print(f"----------------Checking {model_zoo_path}----------------") - final_model_path = osp.join(mlagility_models_dir, model_zoo_path.replace(".onnx", "-" + ZOO_OPSET_VERSION + ".onnx")) - subprocess.run(["benchit", osp.join(mlagility_root, script_path), "--cache-dir", cache_converted_dir, "--onnx-opset", ZOO_OPSET_VERSION], + print(f"----------------Checking {model_zoo_dir}----------------") + final_model_path = osp.join(mlagility_models_dir, model_zoo_dir, f"{model_zoo_dir}-{ZOO_OPSET_VERSION}.onnx") + subprocess.run(["benchit", osp.join(mlagility_root, script_path), "--cache-dir", cache_converted_dir, + "--onnx-opset", ZOO_OPSET_VERSION, "--export-only"], cwd=cwd_path, stdout=sys.stdout, stderr=sys.stderr) shutil.copy(osp.join(cache_converted_dir, model_name, "onnx", model_name + base_name), final_model_path) subprocess.run(["git", "diff", "--exit-code", "--", final_model_path], cwd=cwd_path, stdout=sys.stdout, stderr=sys.stderr) - print(f"Successfully checked {model_zoo_path}.") + print(f"Successfully checked {model_zoo_dir}.") except Exception as e: errors += 1 - print(f"Failed to check {model_zoo_path} because of {e}.") + print(f"Failed to check {model_zoo_dir} because of {e}.") if errors > 0: print(f"All {len(config.models_info)} model(s) have been checked, but {errors} model(s) failed.")