Merge branch 'main' into main

.azure/ci-testig-parameterized.yml

@@ -23,3 +23,4 @@ jobs:
     - "Lightning-AI/flash_pl-release.yaml"
     - "microsoft/deepspeed-release.yaml"
     - "neptune-ai/lightning_integration.yaml"
+    - "manujosephv/pytorch-tabular_lit-release.yaml"

.azure/testing-template.yml

@@ -12,7 +12,7 @@ jobs:
 
     - script: |
         echo $PR_NUMBER
-        CONFIGS=$(python actions/assistant.py changed_configs $PR_NUMBER --as_list=False 2>&1)
+        CONFIGS=$(python _actions/assistant.py changed_configs $PR_NUMBER --as_list=False 2>&1)
         printf "Changed configs: $CONFIGS\n"
         echo "##vso[task.setvariable variable=diff;isOutput=true]$CONFIGS"
       name: files
@@ -29,7 +29,7 @@ jobs:
       # map the output variable from A into this job
       configs: $[ dependencies.check_diff.outputs['files.diff'] ]
       config: "${{ config }}"
-      DEVICES: $( python -c 'name = "$(Agent.Name)" ; gpus = name.split("_")[-1] if "_" in name else "0,1"; print(gpus)' )
+      DEVICES: $( python -c 'name = "$(Agent.Name)" ; gpus = name.split("_")[-1] if "_" in name else "0"; print(gpus)' )
 
     condition: or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), contains(variables['configs'], variables['config']))
     # how long to run the job before automatically cancelling
@@ -49,27 +49,23 @@ jobs:
       options: "--gpus=all --shm-size=8g -v /usr/bin/docker:/tmp/docker:ro"
     steps:
 
-    - bash: |
-        echo "##vso[task.setvariable variable=CONTAINER_ID]$(head -1 /proc/self/cgroup|cut -d/ -f3)"
-        echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
-      displayName: 'Set environment variables'
-
     - bash: |
         whoami && id
         lspci | egrep 'VGA|3D'
         whereis nvidia
         nvidia-smi
-        echo $CUDA_VISIBLE_DEVICES
-        echo $CONTAINER_ID
         python --version
         pip --version
         pip list
         python -c "import torch ; print(torch.cuda.get_arch_list())"
+        echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
       displayName: 'Image info & NVIDIA'
 
     - script: |
-        /tmp/docker exec -t -u 0 $CONTAINER_ID \
-        sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
+        container_id=$(head -1 /proc/self/cgroup|cut -d/ -f3)
+        /tmp/docker exec -t -u 0 $container_id \
+          sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
+        echo "##vso[task.setvariable variable=CONTAINER_ID]$container_id"
       displayName: 'Install Sudo in container (thanks Microsoft!)'
 
     - bash: |
@@ -82,11 +78,13 @@ jobs:
       displayName: 'Install dependencies'
 
     - bash: |
+        echo $CUDA_VISIBLE_DEVICES
+        echo $CONTAINER_ID
         python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu > 0, f'GPU: {mgpu}'"
       displayName: 'Sanity check'
 
     - bash: |
-        python actions/assistant.py prepare_env --config_file=${{config}} > prepare_env.sh
+        python _actions/assistant.py prepare_env --config_file=${{config}} > prepare_env.sh
         cat prepare_env.sh
       displayName: 'Create scripts'
 
@@ -97,10 +95,10 @@ jobs:
       displayName: 'Prepare env.'
 
     - script: |
-        ENVS=$(python actions/assistant.py list_env --config_file=${{config}} --export 2>&1)
+        ENVS=$(python _actions/assistant.py list_env --config_file=${{config}} --export 2>&1)
         printf "PyTest env. variables: $ENVS\n"
         echo "##vso[task.setvariable variable=envs;isOutput=true]$ENVS"
-        ARGS=$(python actions/assistant.py specify_tests --config_file=${{config}} 2>&1)
+        ARGS=$(python _actions/assistant.py specify_tests --config_file=${{config}} 2>&1)
         printf "PyTest arguments: $ARGS\n"
         echo "##vso[task.setvariable variable=args;isOutput=true]$ARGS"
       name: testing

.codecov.yml

@@ -20,6 +20,7 @@ coverage:
     # https://codecov.readme.io/v1.0/docs/commit-status
     project:
       default:
+        informational: true
         target: 95% # specify the target coverage for each commit status
         threshold: 30% # allow this little decrease on project
         # https://github.com/codecov/support/wiki/Filtering-Branches
@@ -28,6 +29,7 @@ coverage:
     # https://github.com/codecov/support/wiki/Patch-Status
     patch:
       default:
+        informational: true
         threshold: 50% # allow this much decrease on patch
     changes: false
 

.github/CODEOWNERS

@@ -1,23 +1,14 @@
 # These owners will be the default owners for everything in the repo. Unless a later match takes precedence,
 # @global-owner1 and @global-owner2 will be requested for review when someone opens a pull request.
-* @borda
+* @borda @lantiga @akihironitta
 
 # owners
 /.github/CODEOWNERS   @Borda
 # main
-/README.md            @borda @tchaton
+/README.md            @borda @lantiga
 
 # CI/CD
-/.github/workflows/   @borda
-/.azure/              @borda
+/.github/workflows/   @borda @akihironitta
+/.azure/              @borda @akihironitta
 # configs in root
 /*.yml                @borda
-
-# Lightning-AI
-/configs/Lightning-AI/metrics*  @borda
-/configs/Lightning-AI/flash*    @ethanwharris
-/configs/Lightning-AI/lightning_transformers*    @seannaren
-/configs/SeanNaren/*    @seannaren
-
-# neptune-ai
-/configs/neptune-ai/*  @AleksanderWWW @aniezurawski @shnela @Raalsky @pitercl @twolodzko @PatrykGala

.github/ISSUE_TEMPLATE/bug_report.md

@@ -18,14 +18,17 @@ assignees: ''
 
 <!-- A clear and concise description of what you expected to happen. -->
 
-### Environment
+### Additional context
+
+<details>
+  <summary>Environment</summary>
 
 - OS (e.g., Linux):
 - Python version:
 - PyTorch Version (e.g., 1.0):
 - How you installed PyTorch (`conda`, `pip`, source):
 - Any other relevant information:
 
-### Additional context
+</details>
 
 <!-- Add any other context about the problem here. -->

.github/ISSUE_TEMPLATE/feature_request.md

@@ -14,10 +14,6 @@ assignees: ''
 
 <!-- Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too -->
 
-### Pitch
-
-<!-- A clear and concise description of what you want to happen. -->
-
 ### Alternatives
 
 <!-- A clear and concise description of any alternative solutions or features you've considered, if any. -->

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,16 +1,20 @@
-## Before submitting
+## What does this PR do?
 
-- [ ] Was this discussed/approved via a GitHub issue? (no need for typos and docs improvements)
-- [ ] Did you create/update your **configuration file**?
-- [ ] Did you **set `runtimes` in config** for GitHub action integration?
-- [ ] Did you **add your config to CI** in Azure pipeline (only projects with 100+ GitHub stars)?
-- [ ] Are all integration **tests passing**?
+Fixes # <issue> (add a link to the created issue in your repository if any)
+OR link your project for clarity.
 
-## What does this PR do? \[optional\]
+<details>
+  <summary>Before submitting</summary>
 
-Fixes # (issue) \[add a link to the created issue in your repository if any\]
-OR link your project for clarity.
+- [ ] Was this discussed/agreed via a GitHub issue? (no need for typos and docs improvements)
+- [ ] Did you create/update your **configuration file**? (in case you are adding new integration)
+- Did you **set `runtimes` in config** for GitHub action integration?
+- [ ] Are all integration **tests passing**?
+
+</details>
 
-## Did you have fun?
+<!--
+Did you have fun?
 
 Make sure you had fun coding 🙃
+-->

.github/actions/job-dump/action.yml

@@ -0,0 +1,38 @@
+name: Dump job details
+description: save and upload job meta and status
+
+inputs:
+  artifact-name:
+    description: unique hash or index
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+
+  # TODO: this seems to mix jobs from multiple jobs not only this strategy
+  - uses: ReeganExE/github-action-job-id@v1.0
+
+  - name: Dump details
+    run: |
+      import os, json
+      data = json.loads('''${{ toJSON(matrix) }}''')
+      data.update(json.loads('''${{ toJSON(job) }}'''))
+      gh_job_idx = ${{ strategy.job-index }} + 1  # offset is the generator job
+      gh_job_name = f"GH_JOB_{gh_job_idx}"
+      gh_job_vars = [(n, v) for n, v in os.environ.items() if n.startswith(gh_job_name)]
+      assert gh_job_vars, f"missing env for: {gh_job_name}"
+      data.update({n.replace(f"{gh_job_name}_", "").lower(): v for n, v in gh_job_vars})
+      with open("meta.json", "w") as fp:
+        json.dump(data, fp)
+    shell: python
+
+  - name: Show job's meta
+    run: cat meta.json
+    shell: bash
+
+  - name: Upload pytest test results
+    uses: actions/upload-artifact@v2
+    with:
+      name: ${{ inputs.artifact-name }}
+      path: meta.json

.github/actions/pytester/action.yml

@@ -0,0 +1,66 @@
+name: Project testing
+description: running the pytest on given project
+
+inputs:
+  python:
+    description: python version to run project on...
+    required: true
+  config:
+    description: path to the particular config
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+  - name: Set up Python ${{ inputs.python }}
+    uses: actions/setup-python@v2
+    with:
+      python-version: ${{ inputs.python }}
+
+  - name: Get pip cache dir
+    id: pip-cache
+    run: echo "::set-output name=dir::$(pip cache dir)"
+    shell: bash
+
+  - name: Cache pip
+    uses: actions/cache@v3
+    with:
+      path: ${{ steps.pip-cache.outputs.dir }}
+      key: $COMMON-pip-$HASH_FILES
+      restore-keys: $COMMON-pip-
+    env:
+      COMMON: ${{ runner.os }}-py${{ inputs.python }}
+      HASH_FILES: ${{ hashFiles('requirements.txt') }}
+
+  - name: Create script
+    run: |
+      pip install -q -r requirements.txt
+      python _actions/assistant.py prepare_env --config_file=${{ inputs.config }} > prepare_env.sh
+      cat prepare_env.sh
+    shell: bash
+
+  - name: Setting outputs
+    id: meta
+    run: |
+      echo "args=$(python _actions/assistant.py specify_tests --config_file=${{ inputs.config }})" >> $GITHUB_OUTPUT
+      echo "env=$(python _actions/assistant.py dict_env --config_file=${{ inputs.config }})" >> $GITHUB_OUTPUT
+    shell: bash
+
+  - name: Prepare environment
+    run: bash prepare_env.sh
+    shell: bash
+
+  - name: Show dir tree
+    if: runner.os == 'Linux'
+    run: |
+      sudo apt install -q -y tree
+      tree .
+    shell: bash
+
+  - name: Testing
+    env: ${{ fromJSON(steps.meta.outputs.env) }}
+    working-directory: _integrations
+    run: |
+      coverage run -m pytest ${{ steps.meta.outputs.args }} -v \
+        --junitxml="junit/test-project.xml"
+    shell: bash

.github/dependabot.yml

@@ -0,0 +1,39 @@
+# Basic dependabot.yml file with
+# minimum configuration for two package managers
+
+version: 2
+updates:
+  # Enable version updates for python
+  - package-ecosystem: "pip"
+    # Look for a `requirements` in the `root` directory
+    directory: "/"
+    # Check for updates once a week
+    schedule:
+      interval: "monthly"
+    # Labels on pull requests for version updates only
+    labels: ["CI/CD"]
+    pull-request-branch-name:
+      # Separate sections of the branch name with a hyphen
+      # for example, `dependabot-npm_and_yarn-next_js-acorn-6.4.1`
+      separator: "-"
+    # Allow up to 5 open pull requests for pip dependencies
+    open-pull-requests-limit: 5
+    reviewers:
+      - "Lightning-AI/teams/core-lightning"
+
+  # Enable version updates for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    # Check for updates once a week
+    schedule:
+      interval: "monthly"
+    # Labels on pull requests for version updates only
+    labels: ["CI/CD"]
+    pull-request-branch-name:
+      # Separate sections of the branch name with a hyphen
+      # for example, `dependabot-npm_and_yarn-next_js-acorn-6.4.1`
+      separator: "-"
+    # Allow up to 5 open pull requests for GitHub Actions
+    open-pull-requests-limit: 5
+    reviewers:
+      - "Lightning-AI/teams/core-lightning"