Merge pull request #2035 from recommenders-team/staging

Staging to main: Fix bug in MAP and added new notebook programmatic execution
recommenders-team · Dec 23, 2023 · 0d9d7c7 · 0d9d7c7
2 parents b000b78 + 5bf18a0
commit 0d9d7c7
Show file tree

Hide file tree

Showing 75 changed files with 3,319 additions and 4,017 deletions.
diff --git a/.github/workflows/azureml-unit-tests.yml b/.github/workflows/azureml-unit-tests.yml
@@ -19,6 +19,7 @@ on:
       - tests/**
       - '!tests/**/*.md'
       - setup.py
+    types: [opened, synchronize, reopened, ready_for_review]
 
   # Enable manual trigger
   workflow_dispatch:
@@ -46,6 +47,7 @@ jobs:
         shell: bash
     outputs:
       test_groups: ${{ steps.get_test_groups.outputs.test_groups }}
+    if: github.event.pull_request.draft == false
 
   execute-tests:
     needs: get-test-groups
@@ -69,3 +71,4 @@ jobs:
           AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }}
           PYTHON_VERSION: ${{ matrix.python-version }}
           TEST_GROUP: ${{ matrix.test-group }}
+    if: github.event.pull_request.draft == false
diff --git a/examples/00_quick_start/als_movielens.ipynb b/examples/00_quick_start/als_movielens.ipynb
diff --git a/examples/00_quick_start/dkn_MIND.ipynb b/examples/00_quick_start/dkn_MIND.ipynb
@@ -86,14 +86,14 @@
                 "import os\n",
                 "import sys\n",
                 "from tempfile import TemporaryDirectory\n",
-                "import scrapbook as sb\n",
                 "import tensorflow as tf\n",
                 "tf.get_logger().setLevel(\"ERROR\") # only show error messages\n",
                 "tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)\n",
                 "\n",
                 "from recommenders.models.deeprec.deeprec_utils import download_deeprec_resources, prepare_hparams\n",
                 "from recommenders.models.deeprec.models.dkn import DKN\n",
                 "from recommenders.models.deeprec.io.dkn_iterator import DKNTextIterator\n",
+                "from recommenders.utils.notebook_utils import store_metadata\n",
                 "\n",
                 "print(f\"System version: {sys.version}\")\n",
                 "print(f\"Tensorflow version: {tf.__version__}\")"
@@ -345,7 +345,12 @@
             "metadata": {},
             "outputs": [],
             "source": [
-                "sb.glue(\"res\", res)"
+                "# Record results for tests - ignore this cell\n",
+                "store_metadata(\"auc\", res[\"auc\"])\n",
+                "store_metadata(\"group_auc\", res[\"group_auc\"])\n",
+                "store_metadata(\"ndcg@5\", res[\"ndcg@5\"])\n",
+                "store_metadata(\"ndcg@10\", res[\"ndcg@10\"])\n",
+                "store_metadata(\"mean_mrr\", res[\"mean_mrr\"])\n"
             ]
         },
         {
@@ -395,4 +400,4 @@
     },
     "nbformat": 4,
     "nbformat_minor": 2
-}
+}
diff --git a/examples/00_quick_start/fastai_movielens.ipynb b/examples/00_quick_start/fastai_movielens.ipynb
@@ -38,14 +38,14 @@
                 }
             ],
             "source": [
-                "# set the environment path to find Recommenders\n",
-                "from tempfile import TemporaryDirectory\n",
-                "import sys\n",
                 "import os\n",
-                "import pandas as pd\n",
+                "import sys\n",
                 "import numpy as np\n",
-                "import scrapbook as sb\n",
-                "import torch, fastai\n",
+                "import pandas as pd\n",
+                "import torch\n",
+                "import fastai\n",
+                "from tempfile import TemporaryDirectory\n",
+                "\n",
                 "from fastai.collab import collab_learner, CollabDataBunch, load_learner\n",
                 "\n",
                 "from recommenders.utils.constants import (\n",
@@ -59,8 +59,9 @@
                 "from recommenders.datasets import movielens\n",
                 "from recommenders.datasets.python_splitters import python_stratified_split\n",
                 "from recommenders.models.fastai.fastai_utils import cartesian_product, score\n",
-                "from recommenders.evaluation.python_evaluation import map_at_k, ndcg_at_k, precision_at_k, recall_at_k\n",
+                "from recommenders.evaluation.python_evaluation import map, ndcg_at_k, precision_at_k, recall_at_k\n",
                 "from recommenders.evaluation.python_evaluation import rmse, mae, rsquared, exp_var\n",
+                "from recommenders.utils.notebook_utils import store_metadata\n",
                 "\n",
                 "print(\"System version: {}\".format(sys.version))\n",
                 "print(\"Pandas version: {}\".format(pd.__version__))\n",
@@ -598,9 +599,9 @@
             "metadata": {},
             "outputs": [],
             "source": [
-                "eval_map = map_at_k(test_df, top_k_scores, col_user=USER, col_item=ITEM, \n",
-                "                    col_rating=RATING, col_prediction=PREDICTION, \n",
-                "                    relevancy_method=\"top_k\", k=TOP_K)"
+                "eval_map = map(test_df, top_k_scores, col_user=USER, col_item=ITEM, \n",
+                "               col_rating=RATING, col_prediction=PREDICTION, \n",
+                "               relevancy_method=\"top_k\", k=TOP_K)"
             ]
         },
         {
@@ -914,17 +915,17 @@
                 }
             ],
             "source": [
-                "# Record results with papermill for tests\n",
-                "sb.glue(\"map\", eval_map)\n",
-                "sb.glue(\"ndcg\", eval_ndcg)\n",
-                "sb.glue(\"precision\", eval_precision)\n",
-                "sb.glue(\"recall\", eval_recall)\n",
-                "sb.glue(\"rmse\", eval_rmse)\n",
-                "sb.glue(\"mae\", eval_mae)\n",
-                "sb.glue(\"exp_var\", eval_exp_var)\n",
-                "sb.glue(\"rsquared\", eval_r2)\n",
-                "sb.glue(\"train_time\", train_time.interval)\n",
-                "sb.glue(\"test_time\", test_time.interval)"
+                "# Record results for tests - ignore this cell\n",
+                "store_metadata(\"map\", eval_map)\n",
+                "store_metadata(\"ndcg\", eval_ndcg)\n",
+                "store_metadata(\"precision\", eval_precision)\n",
+                "store_metadata(\"recall\", eval_recall)\n",
+                "store_metadata(\"rmse\", eval_rmse)\n",
+                "store_metadata(\"mae\", eval_mae)\n",
+                "store_metadata(\"exp_var\", eval_exp_var)\n",
+                "store_metadata(\"rsquared\", eval_r2)\n",
+                "store_metadata(\"train_time\", train_time.interval)\n",
+                "store_metadata(\"test_time\", test_time.interval)"
             ]
         },
         {
@@ -964,4 +965,4 @@
     },
     "nbformat": 4,
     "nbformat_minor": 2
-}
+}