diff --git a/.circleci/config.yml b/.circleci/config.yml index 23b31e309e3..f17e256c209 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,14 +6,14 @@ version: 2.1 executors: standard_cpu39: docker: - - image: cimg/python:3.9.0 + - image: cimg/python:3.9.16 environment: PYTHONUNBUFFERED: 1 resource_class: xlarge small_cpu39: docker: - - image: cimg/python:3.9.0 + - image: cimg/python:3.9.16 environment: PYTHONUNBUFFERED: 1 resource_class: medium @@ -23,7 +23,7 @@ executors: # https://circleci.com/docs/2.0/testing-ios/ xcode: "12.5.1" environment: - PYTHON: 3.9.0 + PYTHON: 3.9.16 PYTHONUNBUFFERED: 1 HOMEBREW_NO_AUTO_UPDATE: 1 resource_class: macos.x86.medium.gen2 @@ -59,7 +59,20 @@ commands: - run: name: Sets up the virtual environment command: | - if [[ "$OSTYPE" == "linux-gnu"* ]]; then sudo apt update && sudo apt install python3.9 python3.9-dev; fi + PYTHON39_VERSION=-1 + if [[ ! -z $(which python3.9) ]]; then + PYTHON39_VERSION=$(python3.9 --version | grep -o "[[:digit:]]\+\$") + fi + + if [[ "$OSTYPE" == "linux-gnu"* ]] && [[ $PYTHON39_VERSION -lt 12 ]]; then + sudo apt update && \ + sudo apt install software-properties-common && \ + sudo add-apt-repository --yes ppa:deadsnakes/ppa && \ + sudo apt install python3.9 python3.9-dev + PYTHON39_VERSION=$(python3.9 --version) + echo "Python version updated to ${PYTHON39_VERSION}" + fi + mkdir -p ~/venv virtualenv --python=python3.9 ~/venv echo ". ~/venv/bin/activate" >> $BASH_ENV @@ -93,7 +106,7 @@ commands: python -m pip install --progress-bar off torch==2.0.0 torchvision==0.15.1+cu118 torchaudio==2.0.0 --index-url https://download.pytorch.org/whl/cu118 python -m pip install --progress-bar off 'fairscale~=0.4.0' python -m pip install --progress-bar off pytorch-pretrained-bert - python -m pip install --progress-bar off 'transformers==4.20' + python -m pip install --progress-bar off 'transformers==4.22.1' python -m pip install --progress-bar off 'fairseq==0.10.0' python -m pip install --progress-bar off 'faiss-gpu==1.7.0' python -m pip uninstall dataclasses -y @@ -108,7 +121,7 @@ commands: - run: name: Install torch CPU and dependencies command: | - python -m pip install --progress-bar off 'transformers==4.20' + python -m pip install --progress-bar off 'transformers==4.22.1' python -m pip install --progress-bar off torch==2.0.0 torchvision==0.15.1 torchaudio==2.0.0 python -c 'import torch; print("Torch version:", torch.__version__)' python -m torch.utils.collect_env @@ -128,15 +141,28 @@ commands: installcrowdsourcingdeps: description: Install Mephisto steps: + - restore_cache: + key: mephisto-20230419-{{ checksum "requirements.txt" }} - run: name: Install Mephisto command: | + curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash - + sudo apt update + sudo apt upgrade + sudo apt install -y nodejs + node -v + npm -v cd .. git clone git@github.com:facebookresearch/Mephisto.git Mephisto - cd Mephisto; git checkout v1.0.3 -b stable + cd Mephisto pip install -e . # `echo` so that ENTER will be pressed at the prompt echo | mephisto check + - save_cache: + key: mephisto-20230419-{{ checksum "requirements.txt" }} + paths: + - "~/venv/bin" + - "~/venv/lib" installchecklist: description: Install checklist and dependencies @@ -355,8 +381,9 @@ jobs: cachename: crowdsourcing marker: crowdsourcing more_installs: - - installtorchgpu + - installtorchcpu - installcrowdsourcingdeps + pytest_flags: -x teacher_tests: executor: standard_cpu39 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1686cf48ab3..53aa030b049 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -78,6 +78,16 @@ jobs: if [[ "${{ github.event_name }}" == "pull_request" ]]; then git branch -f main origin/main fi + - name: Get head SHA + run: | + HEAD_SHA="NONE" + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + HEAD_SHA="${{ github.event.pull_request.head.sha }}" + else + HEAD_SHA="${{ github.event.push.head.sha }}" + fi + echo $HEAD_SHA + echo "HEAD_SHA=${HEAD_SHA}" >> $GITHUB_ENV - name: caching dependencies uses: actions/cache@v1 with: @@ -104,7 +114,7 @@ jobs: with: check_name: 'mypy' linter_output_path: 'output-annotations.txt' - commit_sha: ${{ github.event.pull_request.head.sha }} + commit_sha: ${{ env.HEAD_SHA }} regex: '^(?.*?):(?\d+):(?\d+): (?\w+): (?.*)' annotation_level: warning env: @@ -119,6 +129,16 @@ jobs: uses: actions/checkout@v1 with: ref: ${{ github.event.pull_request.head.sha }} + - name: Get head SHA + run: | + HEAD_SHA="NONE" + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + HEAD_SHA="${{ github.event.pull_request.head.sha }}" + else + HEAD_SHA="${{ github.event.push.head.sha }}" + fi + echo $HEAD_SHA + echo "HEAD_SHA=${HEAD_SHA}" >> $GITHUB_ENV - name: Setup working-directory: parlai/crowdsourcing/ run: | @@ -137,7 +157,7 @@ jobs: with: check_name: 'jslint' linter_output_path: 'output-annotations.txt' - commit_sha: ${{ github.event.pull_request.head.sha }} + commit_sha: ${{ env.HEAD_SHA }} regex: '^(?.*?):(?\d+):(?\d+): (?.*) \[(?.*)\]' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/parlai/crowdsourcing/utils/frontend.py b/parlai/crowdsourcing/utils/frontend.py index 5b7bd0a5a16..8a15440935e 100644 --- a/parlai/crowdsourcing/utils/frontend.py +++ b/parlai/crowdsourcing/utils/frontend.py @@ -26,7 +26,7 @@ def build_task(task_directory: str): os.chdir(frontend_source_dir) if os.path.exists(frontend_build_dir): shutil.rmtree(frontend_build_dir) - packages_installed = subprocess.call(["npm", "install"]) + packages_installed = subprocess.call(["npm", "install", "--force"]) if packages_installed != 0: raise Exception( "please make sure npm is installed, otherwise view " diff --git a/parlai/crowdsourcing/utils/tests.py b/parlai/crowdsourcing/utils/tests.py index 86c72a6ea58..072d6a0a163 100644 --- a/parlai/crowdsourcing/utils/tests.py +++ b/parlai/crowdsourcing/utils/tests.py @@ -16,11 +16,14 @@ import numpy as np import torch +from parlai.utils import logging from hydra.experimental import compose, initialize from mephisto.abstractions.blueprint import SharedTaskState from mephisto.abstractions.databases.local_database import LocalMephistoDB +from mephisto.data_model.task_run import TaskRunArgs from mephisto.operations.operator import Operator from mephisto.tools.scripts import augment_config_from_db +from mephisto.utils.metrics import InaccessiblePrometheusServer from pytest_regressions.data_regression import DataRegressionFixture @@ -53,8 +56,11 @@ def _teardown(self): Should be called in a pytest setup/teardown fixture. """ - if self.operator is not None: - self.operator.force_shutdown() + try: + if self.operator is not None: + self.operator.force_shutdown() + except InaccessiblePrometheusServer as e: + logging.warning(f'InaccessiblePrometheusServer: {e}') if self.server is not None: self.server.shutdown_mock() @@ -102,6 +108,9 @@ def _set_up_config( self.db = LocalMephistoDB(self.database_path) self.config = augment_config_from_db(self.config, self.db) self.config.mephisto.architect.should_run_server = True + self.config.mephisto.task = TaskRunArgs( + **self.config.mephisto.task, no_submission_patience=1 + ) def _set_up_server(self, shared_state: Optional[SharedTaskState] = None): """ diff --git a/requirements.txt b/requirements.txt index edfffcdebcc..c0f0f83fa2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -# comment to bump caches +# Comment to bump caches coloredlogs==14.0 datasets<2.2.2,>=1.4.1 docutils<0.16,>=0.14 diff --git a/tests/crowdsourcing/tasks/qa_data_collection/expected_states/state.json b/tests/crowdsourcing/tasks/qa_data_collection/expected_states/state.json index 68c3daa104b..b90059157b9 100644 --- a/tests/crowdsourcing/tasks/qa_data_collection/expected_states/state.json +++ b/tests/crowdsourcing/tasks/qa_data_collection/expected_states/state.json @@ -1,7 +1,7 @@ { "outputs": { "messages": [ - {"episode_done": false, "id": "QA Collector", "passage": "Pope Paul VI became the first reigning pontiff ever to visit the Americas when he flew to New York in October 1965 to address the United Nations. As a gesture of goodwill, the pope gave to the UN two pieces of papal jewelry, a diamond cross and ring, with the hopes that the proceeds from their sale at auction would contribute to the UN's efforts to end human suffering. During the pope's visit, as the U.S. involvement in the Vietnam War escalated under President Johnson, Paul VI pleaded for peace before the UN:", "text": "Please provide a question given the passage."}, + {"episode_done": false, "id": "QA Collector", "passage": "Architecturally, the school has a Catholic character. Atop the Main Building's gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary."}, {"text": "Who was the first reigning pope to ever visit the Americas?", "task_data": {}, "id": "QA Agent", "episode_done": false}, {"episode_done": false, "id": "QA Collector", "text": "Thanks. And what is the answer to your question?"}, {"text": "Pope Paul VI", "task_data": {}, "id": "QA Agent", "episode_done": false}, diff --git a/tests/crowdsourcing/tasks/qa_data_collection/test_qa_data_collection.py b/tests/crowdsourcing/tasks/qa_data_collection/test_qa_data_collection.py index 98f8891119e..d96b67eb5a1 100644 --- a/tests/crowdsourcing/tasks/qa_data_collection/test_qa_data_collection.py +++ b/tests/crowdsourcing/tasks/qa_data_collection/test_qa_data_collection.py @@ -13,6 +13,8 @@ # Inputs +# The conversation is not exactly making sense if put in context. +# It is just to have the interaction between the agents simulated. AGENT_DISPLAY_IDS = ('QA Agent',) AGENT_MESSAGES = [("Who was the first reigning pope to ever visit the Americas?",)] FORM_MESSAGES = ("Pope Paul VI",) @@ -64,6 +66,8 @@ def test_base_task(self): self._set_up_config(task_directory=TASK_DIRECTORY, overrides=overrides) + # 'train:ordered' in order to avoid randomness that might break the test. + self.config.teacher.datatype = 'train:ordered' # Set up the operator and server teacher = get_teacher(self.config) world_opt = {"turn_timeout": self.config.turn_timeout, "teacher": teacher}