Crowdsourcing tests (#5017)

* new config file

* only crowdsourcing tests

* torch cpu for crowdsourcing tests

* the prometheus exception catch

* event name

* format

* string format

* github.event_name

* eval

* event globing

* wth env variables

* debugged

* same fix for the mypy

* force npm install

* Mephisto cache

* parameter

* single cachename

* 3.9.0 -> 3.9.16

* --editable

* invalidate cache

* no apt update

* -x flag for crowdsourcing tests

* qa fixed

* Revert "only crowdsourcing tests"

This reverts commit fe48e9d.

* pytho 3.9 install

* checking if python3.9 exists

* adding repos to install 3.9

* wrong condition fixed

.circleci/config.yml

@@ -6,14 +6,14 @@ version: 2.1
 executors:
   standard_cpu39:
     docker:
-      - image: cimg/python:3.9.0
+      - image: cimg/python:3.9.16
     environment:
       PYTHONUNBUFFERED: 1
     resource_class: xlarge
 
   small_cpu39:
     docker:
-      - image: cimg/python:3.9.0
+      - image: cimg/python:3.9.16
     environment:
       PYTHONUNBUFFERED: 1
     resource_class: medium
@@ -23,7 +23,7 @@ executors:
       # https://circleci.com/docs/2.0/testing-ios/
       xcode: "12.5.1"
     environment:
-      PYTHON: 3.9.0
+      PYTHON: 3.9.16
       PYTHONUNBUFFERED: 1
       HOMEBREW_NO_AUTO_UPDATE: 1
     resource_class: macos.x86.medium.gen2
@@ -59,7 +59,20 @@ commands:
       - run:
           name: Sets up the virtual environment
           command: |
-            if [[ "$OSTYPE" == "linux-gnu"* ]]; then sudo apt update && sudo apt install python3.9 python3.9-dev; fi
+            PYTHON39_VERSION=-1
+            if [[ ! -z $(which python3.9) ]]; then
+              PYTHON39_VERSION=$(python3.9 --version | grep -o "[[:digit:]]\+\$")
+            fi
+
+            if [[ "$OSTYPE" == "linux-gnu"* ]] && [[ $PYTHON39_VERSION -lt 12 ]]; then 
+              sudo apt update && \
+              sudo apt install software-properties-common && \
+              sudo add-apt-repository --yes ppa:deadsnakes/ppa && \
+              sudo apt install python3.9 python3.9-dev
+              PYTHON39_VERSION=$(python3.9 --version)
+              echo "Python version updated to ${PYTHON39_VERSION}"
+            fi
+
             mkdir -p ~/venv
             virtualenv --python=python3.9 ~/venv
             echo ". ~/venv/bin/activate" >> $BASH_ENV
@@ -93,7 +106,7 @@ commands:
             python -m pip install --progress-bar off torch==2.0.0 torchvision==0.15.1+cu118 torchaudio==2.0.0 --index-url https://download.pytorch.org/whl/cu118
             python -m pip install --progress-bar off 'fairscale~=0.4.0'
             python -m pip install --progress-bar off pytorch-pretrained-bert
-            python -m pip install --progress-bar off 'transformers==4.20'
+            python -m pip install --progress-bar off 'transformers==4.22.1'
             python -m pip install --progress-bar off 'fairseq==0.10.0'
             python -m pip install --progress-bar off 'faiss-gpu==1.7.0'
             python -m pip uninstall dataclasses -y
@@ -108,7 +121,7 @@ commands:
       - run:
           name: Install torch CPU and dependencies
           command: |
-            python -m pip install --progress-bar off 'transformers==4.20'
+            python -m pip install --progress-bar off 'transformers==4.22.1'
             python -m pip install --progress-bar off torch==2.0.0 torchvision==0.15.1 torchaudio==2.0.0
             python -c 'import torch; print("Torch version:", torch.__version__)'
             python -m torch.utils.collect_env
@@ -128,15 +141,28 @@ commands:
   installcrowdsourcingdeps:
     description: Install Mephisto
     steps:
+      - restore_cache:
+          key: mephisto-20230419-{{ checksum "requirements.txt" }}
       - run:
             name: Install Mephisto
             command: |
+              curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash -
+              sudo apt update
+              sudo apt upgrade
+              sudo apt install -y nodejs
+              node -v
+              npm -v
               cd ..
               git clone git@github.com:facebookresearch/Mephisto.git Mephisto
-              cd Mephisto; git checkout v1.0.3 -b stable
+              cd Mephisto
               pip install -e .
               # `echo` so that ENTER will be pressed at the prompt
               echo | mephisto check
+      - save_cache:
+          key: mephisto-20230419-{{ checksum "requirements.txt" }}
+          paths:
+            - "~/venv/bin"
+            - "~/venv/lib"
 
   installchecklist:
     description: Install checklist and dependencies
@@ -355,8 +381,9 @@ jobs:
           cachename: crowdsourcing
           marker: crowdsourcing
           more_installs:
-            - installtorchgpu
+            - installtorchcpu
             - installcrowdsourcingdeps
+          pytest_flags: -x
 
   teacher_tests:
     executor: standard_cpu39

.github/workflows/lint.yml

@@ -78,6 +78,16 @@ jobs:
           if [[ "${{ github.event_name }}" == "pull_request" ]]; then
             git branch -f main origin/main
           fi
+      - name: Get head SHA
+        run: |
+          HEAD_SHA="NONE"
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            HEAD_SHA="${{ github.event.pull_request.head.sha }}"
+          else
+            HEAD_SHA="${{ github.event.push.head.sha }}"
+          fi
+          echo $HEAD_SHA
+          echo "HEAD_SHA=${HEAD_SHA}" >> $GITHUB_ENV
       - name: caching dependencies
         uses: actions/cache@v1
         with:
@@ -104,7 +114,7 @@ jobs:
         with:
           check_name: 'mypy'
           linter_output_path: 'output-annotations.txt'
-          commit_sha: ${{ github.event.pull_request.head.sha }}
+          commit_sha: ${{ env.HEAD_SHA }}
           regex: '^(?<filename>.*?):(?<lineNumber>\d+):(?<columnNumber>\d+): (?<errorCode>\w+): (?<errorDesc>.*)'
           annotation_level: warning
         env:
@@ -119,6 +129,16 @@ jobs:
         uses: actions/checkout@v1
         with:
           ref: ${{ github.event.pull_request.head.sha }}
+      - name: Get head SHA
+        run: |
+          HEAD_SHA="NONE"
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            HEAD_SHA="${{ github.event.pull_request.head.sha }}"
+          else
+            HEAD_SHA="${{ github.event.push.head.sha }}"
+          fi
+          echo $HEAD_SHA
+          echo "HEAD_SHA=${HEAD_SHA}" >> $GITHUB_ENV
       - name: Setup
         working-directory: parlai/crowdsourcing/
         run: |
@@ -137,7 +157,7 @@ jobs:
         with:
           check_name: 'jslint'
           linter_output_path: 'output-annotations.txt'
-          commit_sha: ${{ github.event.pull_request.head.sha }}
+          commit_sha: ${{ env.HEAD_SHA }}
           regex: '^(?<filename>.*?):(?<lineNumber>\d+):(?<columnNumber>\d+): (?<errorDesc>.*) \[(?<errorCode>.*)\]'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

parlai/crowdsourcing/utils/frontend.py

@@ -26,7 +26,7 @@ def build_task(task_directory: str):
     os.chdir(frontend_source_dir)
     if os.path.exists(frontend_build_dir):
         shutil.rmtree(frontend_build_dir)
-    packages_installed = subprocess.call(["npm", "install"])
+    packages_installed = subprocess.call(["npm", "install", "--force"])
     if packages_installed != 0:
         raise Exception(
             "please make sure npm is installed, otherwise view "

parlai/crowdsourcing/utils/tests.py

@@ -16,11 +16,14 @@
 
 import numpy as np
 import torch
+from parlai.utils import logging
 from hydra.experimental import compose, initialize
 from mephisto.abstractions.blueprint import SharedTaskState
 from mephisto.abstractions.databases.local_database import LocalMephistoDB
+from mephisto.data_model.task_run import TaskRunArgs
 from mephisto.operations.operator import Operator
 from mephisto.tools.scripts import augment_config_from_db
+from mephisto.utils.metrics import InaccessiblePrometheusServer
 from pytest_regressions.data_regression import DataRegressionFixture
 
 
@@ -53,8 +56,11 @@ def _teardown(self):
         Should be called in a pytest setup/teardown fixture.
         """
 
-        if self.operator is not None:
-            self.operator.force_shutdown()
+        try:
+            if self.operator is not None:
+                self.operator.force_shutdown()
+        except InaccessiblePrometheusServer as e:
+            logging.warning(f'InaccessiblePrometheusServer: {e}')
 
         if self.server is not None:
             self.server.shutdown_mock()
@@ -102,6 +108,9 @@ def _set_up_config(
         self.db = LocalMephistoDB(self.database_path)
         self.config = augment_config_from_db(self.config, self.db)
         self.config.mephisto.architect.should_run_server = True
+        self.config.mephisto.task = TaskRunArgs(
+            **self.config.mephisto.task, no_submission_patience=1
+        )
 
     def _set_up_server(self, shared_state: Optional[SharedTaskState] = None):
         """

requirements.txt

@@ -1,4 +1,4 @@
-# comment to bump caches
+# Comment to bump caches
 coloredlogs==14.0
 datasets<2.2.2,>=1.4.1
 docutils<0.16,>=0.14

tests/crowdsourcing/tasks/qa_data_collection/expected_states/state.json

@@ -1,7 +1,7 @@
 {
   "outputs": {
     "messages": [
-      {"episode_done": false, "id": "QA Collector", "passage": "Pope Paul VI became the first reigning pontiff ever to visit the Americas when he flew to New York in October 1965 to address the United Nations. As a gesture of goodwill, the pope gave to the UN two pieces of papal jewelry, a diamond cross and ring, with the hopes that the proceeds from their sale at auction would contribute to the UN's efforts to end human suffering. During the pope's visit, as the U.S. involvement in the Vietnam War escalated under President Johnson, Paul VI pleaded for peace before the UN:", "text": "Please provide a question given the passage."},
+      {"episode_done": false, "id": "QA Collector", "passage": "Architecturally, the school has a Catholic character. Atop the Main Building's gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary."},
       {"text": "Who was the first reigning pope to ever visit the Americas?", "task_data": {}, "id": "QA Agent", "episode_done": false},
       {"episode_done": false, "id": "QA Collector", "text": "Thanks. And what is the answer to your question?"},
       {"text": "Pope Paul VI", "task_data": {}, "id": "QA Agent", "episode_done": false},

tests/crowdsourcing/tasks/qa_data_collection/test_qa_data_collection.py

@@ -13,6 +13,8 @@
 
 
 # Inputs
+# The conversation is not exactly making sense if put in context.
+# It is just to have the interaction between the agents simulated.
 AGENT_DISPLAY_IDS = ('QA Agent',)
 AGENT_MESSAGES = [("Who was the first reigning pope to ever visit the Americas?",)]
 FORM_MESSAGES = ("Pope Paul VI",)
@@ -64,6 +66,8 @@ def test_base_task(self):
 
             self._set_up_config(task_directory=TASK_DIRECTORY, overrides=overrides)
 
+            # 'train:ordered' in order to avoid randomness that might break the test.
+            self.config.teacher.datatype = 'train:ordered'
             # Set up the operator and server
             teacher = get_teacher(self.config)
             world_opt = {"turn_timeout": self.config.turn_timeout, "teacher": teacher}