From 0834f48da1dd0a953776758fd1cd92c26828d9bc Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 17 Jan 2023 13:42:17 -0600 Subject: [PATCH 1/9] Include new CI and template for issues --- .github/ISSUE_TEMPLATE/bug_report.yml | 50 +++++++++++ .github/ISSUE_TEMPLATE/feature_request.yaml | 22 +++++ .github/workflows/[flake8] | 3 + .github/workflows/explorer.yml | 93 +++++++++++++++++++++ .github/workflows/lint.yml | 19 +++++ 5 files changed, 187 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yaml create mode 100644 .github/workflows/[flake8] create mode 100644 .github/workflows/explorer.yml create mode 100644 .github/workflows/lint.yml diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..9056e42 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,50 @@ +name: Bug report +description: File a bug report +title: "[Bug]: " +labels: ["bug", "triage"] + +assignees: + - octocat +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + + - type: textarea + id: what-happened + attributes: + label: What happened? + description: Please, describe the issue you are facing. + placeholder: + value: + validations: + required: true + + - type: textarea + id: what-is-expected + attributes: + label: What did you expect to happen? + description: Please, describe what you expected to happen. + placeholder: + value: + validations: + required: true + + - type: dropdown + id: version + attributes: + label: Version + description: What version of Drishti are you running? + options: + - 0.4 (Default) + - 0.3 + validations: + required: true + + - type: textarea + id: logs + attributes: + label: Relevant log output + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + render: shell \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml new file mode 100644 index 0000000..8f6ab2a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -0,0 +1,22 @@ +name: Feature request +description: Request a new feature +title: "[Feature]: " +labels: ["enhancement", "triage"] + +assignees: + - octocat +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this feature request! + + - type: textarea + id: what-feature + attributes: + label: What feature do you want? + description: Please, describe the feature you are requesting. + placeholder: + value: + validations: + required: true diff --git a/.github/workflows/[flake8] b/.github/workflows/[flake8] new file mode 100644 index 0000000..76be966 --- /dev/null +++ b/.github/workflows/[flake8] @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 160 +exclude = .git,__pycache__,docs/source/conf.py,build,dist \ No newline at end of file diff --git a/.github/workflows/explorer.yml b/.github/workflows/explorer.yml new file mode 100644 index 0000000..916e2b8 --- /dev/null +++ b/.github/workflows/explorer.yml @@ -0,0 +1,93 @@ +name: Drishti + +on: + pull_request: + push: + workflow_dispatch: + +jobs: + explorer: + runs-on: ubuntu-latest + container: jlbez/dxt-explorer-ubuntu-20.04 + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + - name: Dependencies + run: | + sudo apt-get update + sudo apt-get install cmake gcc git libtool python3 python3-pip libcurl4-openssl-dev software-properties-common -y + + # Darshan + git clone https://github.com/darshan-hpc/darshan.git + + - name: Build Darshan + run: | + export DARSHAN_DIR=/opt/darshan + + mkdir $DARSHAN_DIR + cd darshan + bash prepare.sh + cd darshan-util + ./configure --prefix=$DARSHAN_DIR + make + make install + + - name: Install Drishti + run: | + pip install --upgrade pip + pip install -r requirements.txt + pip install . + + - name: Run Drishti (--help) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti -h + + - name: Run Drishti (--issues) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --issues sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Run Drishti (--code) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --code sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Run DXT Explorer (--verbose) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --verbose sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Run DXT Explorer (--path) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --path sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Run DXT Explorer (--html) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --html sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Run DXT Explorer (--svg) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --svg sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Upload Artifact + if: always() + uses: actions/upload-artifact@v2 + with: + name: tests + path: sample/** + retention-days: 1 \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..57d937b --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,19 @@ +name: lint + +on: + workflow_dispatch: + push: + +jobs: + lint: + name: Python Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: "3.9" + - name: Run flake8 + uses: julianwachholz/flake8-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From 7ca6b5f4af79ec1704977711eb895c78734f7f44 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 17 Jan 2023 13:58:45 -0600 Subject: [PATCH 2/9] Update CI configuration --- ...explorer.yml => drishti-darshan-3.4.0.yml} | 2 +- .github/workflows/lint.yml | 19 ------------------- 2 files changed, 1 insertion(+), 20 deletions(-) rename .github/workflows/{explorer.yml => drishti-darshan-3.4.0.yml} (98%) delete mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/explorer.yml b/.github/workflows/drishti-darshan-3.4.0.yml similarity index 98% rename from .github/workflows/explorer.yml rename to .github/workflows/drishti-darshan-3.4.0.yml index 916e2b8..18d2028 100644 --- a/.github/workflows/explorer.yml +++ b/.github/workflows/drishti-darshan-3.4.0.yml @@ -1,4 +1,4 @@ -name: Drishti +name: Drishti (Darshan 3.4.0) on: pull_request: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 57d937b..0000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: lint - -on: - workflow_dispatch: - push: - -jobs: - lint: - name: Python Lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: "3.9" - - name: Run flake8 - uses: julianwachholz/flake8-action@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From 256b7886681943bc94afda5b16c184a74c2a69d5 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 17 Jan 2023 15:11:50 -0600 Subject: [PATCH 3/9] Pin Darshan version for testing --- .github/workflows/drishti-darshan-3.4.0.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/drishti-darshan-3.4.0.yml b/.github/workflows/drishti-darshan-3.4.0.yml index 18d2028..7a86ee8 100644 --- a/.github/workflows/drishti-darshan-3.4.0.yml +++ b/.github/workflows/drishti-darshan-3.4.0.yml @@ -23,6 +23,7 @@ jobs: # Darshan git clone https://github.com/darshan-hpc/darshan.git + git checkout darshan-3.4.0 - name: Build Darshan run: | From 69b8f9fd49a5731fe3eae277ffd4d6e4565000c8 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 17 Jan 2023 15:14:47 -0600 Subject: [PATCH 4/9] Pin Darshan version for testing --- .github/workflows/drishti-darshan-3.4.0.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/drishti-darshan-3.4.0.yml b/.github/workflows/drishti-darshan-3.4.0.yml index 7a86ee8..915cfef 100644 --- a/.github/workflows/drishti-darshan-3.4.0.yml +++ b/.github/workflows/drishti-darshan-3.4.0.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: jobs: - explorer: + drishti: runs-on: ubuntu-latest container: jlbez/dxt-explorer-ubuntu-20.04 timeout-minutes: 60 @@ -22,7 +22,8 @@ jobs: sudo apt-get install cmake gcc git libtool python3 python3-pip libcurl4-openssl-dev software-properties-common -y # Darshan - git clone https://github.com/darshan-hpc/darshan.git + git clone https://github.com/darshan-hpc/darshan.git darshan-3.4.0 + cd darshan-3.4.0 git checkout darshan-3.4.0 - name: Build Darshan From 64750f765b37c847694fedf0b01ba7fdbb40810b Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 17 Jan 2023 15:16:26 -0600 Subject: [PATCH 5/9] Fix path --- .github/workflows/drishti-darshan-3.4.0.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/drishti-darshan-3.4.0.yml b/.github/workflows/drishti-darshan-3.4.0.yml index 915cfef..229fe43 100644 --- a/.github/workflows/drishti-darshan-3.4.0.yml +++ b/.github/workflows/drishti-darshan-3.4.0.yml @@ -31,7 +31,7 @@ jobs: export DARSHAN_DIR=/opt/darshan mkdir $DARSHAN_DIR - cd darshan + cd darshan-3.4.0 bash prepare.sh cd darshan-util ./configure --prefix=$DARSHAN_DIR From bb3c964a79c545826b8b3e00d6f5a4e3cfc689b5 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 17 Jan 2023 15:21:20 -0600 Subject: [PATCH 6/9] Pin py-darshan version in CI --- .github/workflows/drishti-darshan-3.4.0.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/drishti-darshan-3.4.0.yml b/.github/workflows/drishti-darshan-3.4.0.yml index 229fe43..9d019a7 100644 --- a/.github/workflows/drishti-darshan-3.4.0.yml +++ b/.github/workflows/drishti-darshan-3.4.0.yml @@ -26,6 +26,8 @@ jobs: cd darshan-3.4.0 git checkout darshan-3.4.0 + pip install --upgrade pip + - name: Build Darshan run: | export DARSHAN_DIR=/opt/darshan @@ -38,9 +40,12 @@ jobs: make make install + - name: Install py-darshan + run: | + pip install darshan==3.4.0 + - name: Install Drishti run: | - pip install --upgrade pip pip install -r requirements.txt pip install . From 239f3ed86d6db0325007cee099d051b41da24ce7 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 17 Jan 2023 15:24:10 -0600 Subject: [PATCH 7/9] Pin py-darshan version in CI --- .github/workflows/drishti-darshan-3.4.0.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/drishti-darshan-3.4.0.yml b/.github/workflows/drishti-darshan-3.4.0.yml index 9d019a7..6535241 100644 --- a/.github/workflows/drishti-darshan-3.4.0.yml +++ b/.github/workflows/drishti-darshan-3.4.0.yml @@ -42,7 +42,7 @@ jobs: - name: Install py-darshan run: | - pip install darshan==3.4.0 + pip install darshan==3.4.0.1 - name: Install Drishti run: | From d6731ce2f4bf3013d594fd16341609b90be47701 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 17 Jan 2023 15:35:17 -0600 Subject: [PATCH 8/9] Include new CI tests with novel Darshan releases --- .github/workflows/drishti-darshan-3.4.1.yml | 100 ++++++++++++++++++++ .github/workflows/drishti-darshan-3.4.2.yml | 100 ++++++++++++++++++++ 2 files changed, 200 insertions(+) create mode 100644 .github/workflows/drishti-darshan-3.4.1.yml create mode 100644 .github/workflows/drishti-darshan-3.4.2.yml diff --git a/.github/workflows/drishti-darshan-3.4.1.yml b/.github/workflows/drishti-darshan-3.4.1.yml new file mode 100644 index 0000000..29ed011 --- /dev/null +++ b/.github/workflows/drishti-darshan-3.4.1.yml @@ -0,0 +1,100 @@ +name: Drishti (Darshan 3.4.1) + +on: + pull_request: + push: + workflow_dispatch: + +jobs: + drishti: + runs-on: ubuntu-latest + container: jlbez/dxt-explorer-ubuntu-20.04 + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + - name: Dependencies + run: | + sudo apt-get update + sudo apt-get install cmake gcc git libtool python3 python3-pip libcurl4-openssl-dev software-properties-common -y + + # Darshan + git clone https://github.com/darshan-hpc/darshan.git darshan-3.4.1 + cd darshan-3.4.1 + git checkout darshan-3.4.1 + + pip install --upgrade pip + + - name: Build Darshan + run: | + export DARSHAN_DIR=/opt/darshan + + mkdir $DARSHAN_DIR + cd darshan-3.4.1 + bash prepare.sh + cd darshan-util + ./configure --prefix=$DARSHAN_DIR + make + make install + + - name: Install py-darshan + run: | + pip install darshan==3.4.1 + + - name: Install Drishti + run: | + pip install -r requirements.txt + pip install . + + - name: Run Drishti (--help) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti -h + + - name: Run Drishti (--issues) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --issues sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Run Drishti (--code) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --code sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Run DXT Explorer (--verbose) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --verbose sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Run DXT Explorer (--path) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --path sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Run DXT Explorer (--html) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --html sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Run DXT Explorer (--svg) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --svg sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Upload Artifact + if: always() + uses: actions/upload-artifact@v2 + with: + name: tests + path: sample/** + retention-days: 1 \ No newline at end of file diff --git a/.github/workflows/drishti-darshan-3.4.2.yml b/.github/workflows/drishti-darshan-3.4.2.yml new file mode 100644 index 0000000..0398f37 --- /dev/null +++ b/.github/workflows/drishti-darshan-3.4.2.yml @@ -0,0 +1,100 @@ +name: Drishti (Darshan 3.4.2) + +on: + pull_request: + push: + workflow_dispatch: + +jobs: + drishti: + runs-on: ubuntu-latest + container: jlbez/dxt-explorer-ubuntu-20.04 + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + - name: Dependencies + run: | + sudo apt-get update + sudo apt-get install cmake gcc git libtool python3 python3-pip libcurl4-openssl-dev software-properties-common -y + + # Darshan + git clone https://github.com/darshan-hpc/darshan.git darshan-3.4.2 + cd darshan-3.4.2 + git checkout darshan-3.4.2 + + pip install --upgrade pip + + - name: Build Darshan + run: | + export DARSHAN_DIR=/opt/darshan + + mkdir $DARSHAN_DIR + cd darshan-3.4.2 + bash prepare.sh + cd darshan-util + ./configure --prefix=$DARSHAN_DIR + make + make install + + - name: Install py-darshan + run: | + pip install darshan==3.4.2 + + - name: Install Drishti + run: | + pip install -r requirements.txt + pip install . + + - name: Run Drishti (--help) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti -h + + - name: Run Drishti (--issues) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --issues sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Run Drishti (--code) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --code sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Run DXT Explorer (--verbose) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --verbose sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Run DXT Explorer (--path) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --path sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Run DXT Explorer (--html) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --html sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Run DXT Explorer (--svg) + run: | + export PATH=/opt/darshan/bin:$PATH + + drishti --svg sample/jlbez_8a_benchmark_write_parallel_id1321662_8-21-5892-15802854900629188750_106.darshan + + - name: Upload Artifact + if: always() + uses: actions/upload-artifact@v2 + with: + name: tests + path: sample/** + retention-days: 1 \ No newline at end of file From c0a7020921901b267a19f3b021309e49ca02c20a Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 17 Jan 2023 15:41:28 -0600 Subject: [PATCH 9/9] Fix support with py-darshan 3.4.1 --- drishti/main.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drishti/main.py b/drishti/main.py index 8dad467..8605a48 100644 --- a/drishti/main.py +++ b/drishti/main.py @@ -1423,8 +1423,13 @@ def main(): insights_end_time = time.time() - job_start = datetime.datetime.fromtimestamp(job['job']['start_time'], datetime.timezone.utc) - job_end = datetime.datetime.fromtimestamp(job['job']['end_time'], datetime.timezone.utc) + # Version 3.4.1 of py-darshan changed the contents on what is reported in 'job' + if 'start_time' in job['job']: + job_start = datetime.datetime.fromtimestamp(job['job']['start_time'], datetime.timezone.utc) + job_end = datetime.datetime.fromtimestamp(job['job']['end_time'], datetime.timezone.utc) + else: + job_start = datetime.datetime.fromtimestamp(job['job']['start_time_sec'], datetime.timezone.utc) + job_end = datetime.datetime.fromtimestamp(job['job']['end_time_sec'], datetime.timezone.utc) console.print()