From 8b5ae3f345ba9ea22756e535636f8235d5d11572 Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Mon, 16 Dec 2024 11:34:19 -0500 Subject: [PATCH 1/6] Add TF MNIST classification cost benchmark --- ...ython_TF_MNIST_Classification_Dataflow.yml | 91 +++++++++++++++++++ .../python_tf_mnist_classification.txt | 29 ++++++ ...low_mnist_classification_cost_benchmark.py | 40 ++++++++ 3 files changed, 160 insertions(+) create mode 100644 .github/workflows/beam_CostBenchmark_Python_TF_MNIST_Classification_Dataflow.yml create mode 100644 .github/workflows/cost-benchmarks-pipeline-options/python_tf_mnist_classification.txt create mode 100644 sdks/python/apache_beam/testing/benchmarks/inference/tensorflow_mnist_classification_cost_benchmark.py diff --git a/.github/workflows/beam_CostBenchmark_Python_TF_MNIST_Classification_Dataflow.yml b/.github/workflows/beam_CostBenchmark_Python_TF_MNIST_Classification_Dataflow.yml new file mode 100644 index 000000000000..7e53e977a645 --- /dev/null +++ b/.github/workflows/beam_CostBenchmark_Python_TF_MNIST_Classification_Dataflow.yml @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Cost Benchmark Python TF MNIST Classification Dataflow + +on: + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} + INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} + +jobs: + beam_Inference_Python_Benchmarks_Dataflow: + if: | + github.event_name == 'workflow_dispatch' + runs-on: [self-hosted, ubuntu-20.04, main] + timeout-minutes: 900 + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_CostBenchmark_Python_TF_MNIST_Classification_Dataflow"] + job_phrase: ["Run TF MNIST Classification Cost Benchmark"] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup Python environment + uses: ./.github/actions/setup-environment-action + with: + python-version: '3.10' + - name: Prepare test arguments + uses: ./.github/actions/test-arguments-action + with: + test-type: load + test-language: python + argument-file-paths: | + ${{ github.workspace }}/.github/workflows/cost-benchmarks-pipeline-options/python_tf_mnist_classification.txt + # The env variables are created and populated in the test-arguments-action as "_test_arguments_" + - name: get current time + run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV + - name: run wordcount on Dataflow Python + uses: ./.github/actions/gradle-command-self-hosted-action + timeout-minutes: 30 + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.tensorflow_mnist_classification_cost_benchmark \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.10 \ + '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_1 }} --job_name=benchmark-tests-tf-mnist-classification-python-${{env.NOW_UTC}} --input_file=gs://apache-beam-ml/testing/inputs/it_mnist_data.csv --output_file=gs://temp-storage-for-end-to-end-tests/wordcount/result_tf_mnist-${{env.NOW_UTC}}.txt --model=gs://apache-beam-ml/models/tensorflow/mnist/' \ \ No newline at end of file diff --git a/.github/workflows/cost-benchmarks-pipeline-options/python_tf_mnist_classification.txt b/.github/workflows/cost-benchmarks-pipeline-options/python_tf_mnist_classification.txt new file mode 100644 index 000000000000..01f4460b8c7e --- /dev/null +++ b/.github/workflows/cost-benchmarks-pipeline-options/python_tf_mnist_classification.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--region=us-central1 +--machine_type=n1-standard-2 +--num_workers=1 +--disk_size_gb=50 +--autoscaling_algorithm=NONE +--input_options={} +--staging_location=gs://temp-storage-for-perf-tests/loadtests +--temp_location=gs://temp-storage-for-perf-tests/loadtests +--requirements_file=apache_beam/ml/inference/tensorflow_tests_requirements.txt +--publish_to_big_query=true +--metrics_dataset=beam_run_inference +--metrics_table=tf_mnist_classification +--runner=DataflowRunner \ No newline at end of file diff --git a/sdks/python/apache_beam/testing/benchmarks/inference/tensorflow_mnist_classification_cost_benchmark.py b/sdks/python/apache_beam/testing/benchmarks/inference/tensorflow_mnist_classification_cost_benchmark.py new file mode 100644 index 000000000000..f98d5e5d6229 --- /dev/null +++ b/sdks/python/apache_beam/testing/benchmarks/inference/tensorflow_mnist_classification_cost_benchmark.py @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pytype: skip-file + +import logging + +from apache_beam.examples.inference import tensorflow_mnist_classification +from apache_beam.testing.load_tests.dataflow_cost_benchmark import DataflowCostBenchmark + +class TensorflowMNISTClassificationCostBenchmark(DataflowCostBenchmark): + def __init__(self): + super().__init__() + + def test(self): + extra_opts = {} + extra_opts['input'] = self.pipeline.get_option('input_file') + extra_opts['output'] = self.pipeline.get_option('output_file') + extra_opts['model_path'] = self.pipeline.get_option('model') + tensorflow_mnist_classification.run( + self.pipeline.get_full_options_as_args(**extra_opts), + save_main_session=False) + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + TensorflowMNISTClassificationCostBenchmark().run() \ No newline at end of file From 80b01ad896f3e0f8af48cc7eff6c1528073c51c7 Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Mon, 16 Dec 2024 11:58:51 -0500 Subject: [PATCH 2/6] linting --- .../tensorflow_mnist_classification_cost_benchmark.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/testing/benchmarks/inference/tensorflow_mnist_classification_cost_benchmark.py b/sdks/python/apache_beam/testing/benchmarks/inference/tensorflow_mnist_classification_cost_benchmark.py index f98d5e5d6229..f7e12dcead03 100644 --- a/sdks/python/apache_beam/testing/benchmarks/inference/tensorflow_mnist_classification_cost_benchmark.py +++ b/sdks/python/apache_beam/testing/benchmarks/inference/tensorflow_mnist_classification_cost_benchmark.py @@ -21,6 +21,7 @@ from apache_beam.examples.inference import tensorflow_mnist_classification from apache_beam.testing.load_tests.dataflow_cost_benchmark import DataflowCostBenchmark + class TensorflowMNISTClassificationCostBenchmark(DataflowCostBenchmark): def __init__(self): super().__init__() @@ -37,4 +38,4 @@ def test(self): if __name__ == '__main__': logging.basicConfig(level=logging.INFO) - TensorflowMNISTClassificationCostBenchmark().run() \ No newline at end of file + TensorflowMNISTClassificationCostBenchmark().run() From e8b189dbe8b7a6b3a95f468dc286e465097eea2d Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Tue, 17 Dec 2024 10:14:40 -0500 Subject: [PATCH 3/6] Generalize to single workflow file for cost benchmarks --- ....yml => beam_Python_CostBenchmarks_Dataflow.yml} | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) rename .github/workflows/{beam_CostBenchmark_Python_TF_MNIST_Classification_Dataflow.yml => beam_Python_CostBenchmarks_Dataflow.yml} (90%) diff --git a/.github/workflows/beam_CostBenchmark_Python_TF_MNIST_Classification_Dataflow.yml b/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml similarity index 90% rename from .github/workflows/beam_CostBenchmark_Python_TF_MNIST_Classification_Dataflow.yml rename to .github/workflows/beam_Python_CostBenchmarks_Dataflow.yml index 7e53e977a645..2a3df41e3a2e 100644 --- a/.github/workflows/beam_CostBenchmark_Python_TF_MNIST_Classification_Dataflow.yml +++ b/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml @@ -13,9 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Cost Benchmark Python TF MNIST Classification Dataflow +name: Python Cost Benchmarks Dataflow on: + schedule: + - cron: '30 18 * * 6' # Run at 11:30 pm UTC on Saturdays workflow_dispatch: #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event @@ -49,14 +51,15 @@ env: jobs: beam_Inference_Python_Benchmarks_Dataflow: if: | - github.event_name == 'workflow_dispatch' + github.event_name == 'workflow_dispatch' || + (github.event_name == 'schedule' && github.repository == 'apache/beam') runs-on: [self-hosted, ubuntu-20.04, main] timeout-minutes: 900 name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) strategy: matrix: - job_name: ["beam_CostBenchmark_Python_TF_MNIST_Classification_Dataflow"] - job_phrase: ["Run TF MNIST Classification Cost Benchmark"] + job_name: ["beam_Python_CostBenchmark_Dataflow"] + job_phrase: ["Run Python Dataflow Cost Benchmarks"] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -79,7 +82,7 @@ jobs: # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: get current time run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV - - name: run wordcount on Dataflow Python + - name: Run Tensorflow MNIST Image Classification on Dataflow uses: ./.github/actions/gradle-command-self-hosted-action timeout-minutes: 30 with: From d9fd86791501c3e67f6f2f20f18fa518d9bb5579 Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Tue, 17 Dec 2024 10:16:13 -0500 Subject: [PATCH 4/6] fix incorrect UTC time in comment --- .github/workflows/beam_Python_CostBenchmarks_Dataflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml b/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml index 2a3df41e3a2e..0209993d09b7 100644 --- a/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml +++ b/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml @@ -17,7 +17,7 @@ name: Python Cost Benchmarks Dataflow on: schedule: - - cron: '30 18 * * 6' # Run at 11:30 pm UTC on Saturdays + - cron: '30 18 * * 6' # Run at 6:30 pm UTC on Saturdays workflow_dispatch: #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event From 6976360a6e6bd70c834c3fdcebcbe5f3ac21de7b Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Tue, 17 Dec 2024 10:30:55 -0500 Subject: [PATCH 5/6] move wordcount to same workflow --- .../beam_Python_CostBenchmarks_Dataflow.yml | 13 ++- ...rdcount_Python_Cost_Benchmark_Dataflow.yml | 91 ------------------- 2 files changed, 12 insertions(+), 92 deletions(-) delete mode 100644 .github/workflows/beam_Wordcount_Python_Cost_Benchmark_Dataflow.yml diff --git a/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml b/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml index 0209993d09b7..960dba3a58b7 100644 --- a/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml +++ b/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml @@ -78,10 +78,21 @@ jobs: test-type: load test-language: python argument-file-paths: | + ${{ github.workspace }}/.github/workflows/cost-benchmarks-pipeline-options/python_wordcount.txt ${{ github.workspace }}/.github/workflows/cost-benchmarks-pipeline-options/python_tf_mnist_classification.txt # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: get current time run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV + - name: Run wordcount on Dataflow + uses: ./.github/actions/gradle-command-self-hosted-action + timeout-minutes: 30 + with: + gradle-command: :sdks:python:apache_beam:testing:load_tests:run + arguments: | + -PloadTest.mainClass=apache_beam.testing.benchmarks.wordcount.wordcount \ + -Prunner=DataflowRunner \ + -PpythonVersion=3.10 \ + '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_1 }} --job_name=benchmark-tests-wordcount-python-${{env.NOW_UTC}} --output_file=gs://temp-storage-for-end-to-end-tests/wordcount/result_wordcount-${{env.NOW_UTC}}.txt' \ - name: Run Tensorflow MNIST Image Classification on Dataflow uses: ./.github/actions/gradle-command-self-hosted-action timeout-minutes: 30 @@ -91,4 +102,4 @@ jobs: -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.tensorflow_mnist_classification_cost_benchmark \ -Prunner=DataflowRunner \ -PpythonVersion=3.10 \ - '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_1 }} --job_name=benchmark-tests-tf-mnist-classification-python-${{env.NOW_UTC}} --input_file=gs://apache-beam-ml/testing/inputs/it_mnist_data.csv --output_file=gs://temp-storage-for-end-to-end-tests/wordcount/result_tf_mnist-${{env.NOW_UTC}}.txt --model=gs://apache-beam-ml/models/tensorflow/mnist/' \ \ No newline at end of file + '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_2 }} --job_name=benchmark-tests-tf-mnist-classification-python-${{env.NOW_UTC}} --input_file=gs://apache-beam-ml/testing/inputs/it_mnist_data.csv --output_file=gs://temp-storage-for-end-to-end-tests/wordcount/result_tf_mnist-${{env.NOW_UTC}}.txt --model=gs://apache-beam-ml/models/tensorflow/mnist/' \ \ No newline at end of file diff --git a/.github/workflows/beam_Wordcount_Python_Cost_Benchmark_Dataflow.yml b/.github/workflows/beam_Wordcount_Python_Cost_Benchmark_Dataflow.yml deleted file mode 100644 index 51d1005affbc..000000000000 --- a/.github/workflows/beam_Wordcount_Python_Cost_Benchmark_Dataflow.yml +++ /dev/null @@ -1,91 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Wordcount Python Cost Benchmarks Dataflow - -on: - workflow_dispatch: - -#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event -permissions: - actions: write - pull-requests: read - checks: read - contents: read - deployments: read - id-token: none - issues: read - discussions: read - packages: read - pages: read - repository-projects: read - security-events: read - statuses: read - -# This allows a subsequently queued workflow run to interrupt previous runs -concurrency: - group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' - cancel-in-progress: true - -env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} - GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} - INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} - INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} - -jobs: - beam_Inference_Python_Benchmarks_Dataflow: - if: | - github.event_name == 'workflow_dispatch' - runs-on: [self-hosted, ubuntu-20.04, main] - timeout-minutes: 900 - name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - strategy: - matrix: - job_name: ["beam_Wordcount_Python_Cost_Benchmarks_Dataflow"] - job_phrase: ["Run Wordcount Cost Benchmark"] - steps: - - uses: actions/checkout@v4 - - name: Setup repository - uses: ./.github/actions/setup-action - with: - comment_phrase: ${{ matrix.job_phrase }} - github_token: ${{ secrets.GITHUB_TOKEN }} - github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - - name: Setup Python environment - uses: ./.github/actions/setup-environment-action - with: - python-version: '3.10' - - name: Prepare test arguments - uses: ./.github/actions/test-arguments-action - with: - test-type: load - test-language: python - argument-file-paths: | - ${{ github.workspace }}/.github/workflows/cost-benchmarks-pipeline-options/python_wordcount.txt - # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - - name: get current time - run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV - - name: run wordcount on Dataflow Python - uses: ./.github/actions/gradle-command-self-hosted-action - timeout-minutes: 30 - with: - gradle-command: :sdks:python:apache_beam:testing:load_tests:run - arguments: | - -PloadTest.mainClass=apache_beam.testing.benchmarks.wordcount.wordcount \ - -Prunner=DataflowRunner \ - -PpythonVersion=3.10 \ - '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_1 }} --job_name=benchmark-tests-wordcount-python-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/wordcount/result_wordcount-${{env.NOW_UTC}}.txt' \ \ No newline at end of file From a2a60fb4f1a48f429732ca535dd25e94b8d282d8 Mon Sep 17 00:00:00 2001 From: Jack McCluskey Date: Tue, 17 Dec 2024 10:31:58 -0500 Subject: [PATCH 6/6] update workflow job name --- .github/workflows/beam_Python_CostBenchmarks_Dataflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml b/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml index 960dba3a58b7..18fe37e142ac 100644 --- a/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml +++ b/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml @@ -49,7 +49,7 @@ env: INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} jobs: - beam_Inference_Python_Benchmarks_Dataflow: + beam_Python_Cost_Benchmarks_Dataflow: if: | github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && github.repository == 'apache/beam')