make builds more reliable by adding retry logic to urlretrieve calls …

…in notebooks (#1218)
microsoft · Mar 21, 2022 · 70895df · 70895df
1 parent 8700697
commit 70895df
Show file tree

Hide file tree

Showing 9 changed files with 32 additions and 44 deletions.
diff --git a/notebooks/cognitive-services-examples/face-verification/analyze_face_fairness.ipynb b/notebooks/cognitive-services-examples/face-verification/analyze_face_fairness.ipynb
@@ -80,13 +80,12 @@
    "outputs": [],
    "source": [
     "import zipfile\n",
+    "from raiutils.dataset import fetch_dataset\n",
     "outdirname = 'responsibleai.12.28.21'\n",
-    "try:\n",
-    "    from urllib import urlretrieve\n",
-    "except ImportError:\n",
-    "    from urllib.request import urlretrieve\n",
     "zipfilename = outdirname + '.zip'\n",
-    "urlretrieve('https://publictestdatasets.blob.core.windows.net/data/' + zipfilename, zipfilename)\n",
+    "\n",
+    "fetch_dataset('https://publictestdatasets.blob.core.windows.net/data/' + zipfilename, zipfilename)\n",
+    "\n",
     "with zipfile.ZipFile(zipfilename, 'r') as unzip:\n",
     "    unzip.extractall('.')\n",
     "results_csv = \"face_verify_sample_rand_data.csv\""

diff --git a/notebooks/cognitive-services-examples/speech-to-text/analyze_stt_fairness.ipynb b/notebooks/cognitive-services-examples/speech-to-text/analyze_stt_fairness.ipynb
@@ -99,13 +99,12 @@
    "outputs": [],
    "source": [
     "import zipfile\n",
+    "from raiutils.dataset import fetch_dataset\n",
     "outdirname = 'responsibleai.12.28.21'\n",
-    "try:\n",
-    "    from urllib import urlretrieve\n",
-    "except ImportError:\n",
-    "    from urllib.request import urlretrieve\n",
     "zipfilename = outdirname + '.zip'\n",
-    "urlretrieve('https://publictestdatasets.blob.core.windows.net/data/' + zipfilename, zipfilename)\n",
+    "\n",
+    "fetch_dataset('https://publictestdatasets.blob.core.windows.net/data/' + zipfilename, zipfilename)\n",
+    "\n",
     "with zipfile.ZipFile(zipfilename, 'r') as unzip:\n",
     "    unzip.extractall('.')\n",
     "stt_results_csv = \"stt_testing_data.csv\""

diff --git a/...-dashboards/erroranalysis-dashboard/erroranalysis-interpretability-dashboard-census.ipynb b/...-dashboards/erroranalysis-dashboard/erroranalysis-interpretability-dashboard-census.ipynb
@@ -81,13 +81,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "from raiutils.dataset import fetch_dataset\n",
     "outdirname = 'erroranalysis.12.3.20'\n",
-    "try:\n",
-    "    from urllib import urlretrieve\n",
-    "except ImportError:\n",
-    "    from urllib.request import urlretrieve\n",
     "zipfilename = outdirname + '.zip'\n",
-    "urlretrieve('https://publictestdatasets.blob.core.windows.net/data/' + zipfilename, zipfilename)\n",
+    "\n",
+    "fetch_dataset('https://publictestdatasets.blob.core.windows.net/data/' + zipfilename, zipfilename)\n",
+    "\n",
     "with zipfile.ZipFile(zipfilename, 'r') as unzip:\n",
     "    unzip.extractall('.')"
    ]

diff --git a/...ndividual-dashboards/explanation-dashboard/explanation-dashboard-employee-attrition.ipynb b/...ndividual-dashboards/explanation-dashboard/explanation-dashboard-employee-attrition.ipynb
@@ -59,8 +59,6 @@
     "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
     "from lightgbm import LGBMClassifier\n",
     "import pandas as pd\n",
-    "import numpy as np\n",
-    "from urllib.request import urlretrieve\n",
     "import zipfile"
    ]
   },
@@ -77,9 +75,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "from raiutils.dataset import fetch_dataset\n",
     "outdirname = 'dataset.6.21.19'\n",
     "zipfilename = outdirname + '.zip'\n",
-    "urlretrieve('https://publictestdatasets.blob.core.windows.net/data/' + zipfilename, zipfilename)\n",
+    "\n",
+    "fetch_dataset('https://publictestdatasets.blob.core.windows.net/data/' + zipfilename, zipfilename)\n",
+    "\n",
     "with zipfile.ZipFile(zipfilename, 'r') as unzip:\n",
     "    unzip.extractall('.')\n",
     "attritionData = pd.read_csv('./WA_Fn-UseC_-HR-Employee-Attrition.csv')\n",

diff --git a/...responsibleaidashboard/responsibleaidashboard-census-classification-model-debugging.ipynb b/...responsibleaidashboard/responsibleaidashboard-census-classification-model-debugging.ipynb
@@ -61,14 +61,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import sklearn\n",
     "import zipfile\n",
     "\n",
     "from sklearn.pipeline import Pipeline\n",
     "from sklearn.impute import SimpleImputer\n",
     "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
     "from sklearn.compose import ColumnTransformer\n",
-    "from sklearn.model_selection import train_test_split\n",
     "\n",
     "import pandas as pd\n",
     "from lightgbm import LGBMClassifier"
@@ -89,6 +87,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "from raiutils.dataset import fetch_dataset\n",
     "from sklearn.pipeline import Pipeline\n",
     "from sklearn.impute import SimpleImputer\n",
     "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
@@ -100,8 +99,6 @@
     "    return X, y\n",
     "\n",
     "def create_classification_pipeline(X, y, target_feature):\n",
-    "    features = X.columns.values.tolist()\n",
-    "    classes = y[target_feature].unique().tolist()\n",
     "    pipe_cfg = {\n",
     "        'num_cols': X.dtypes[X.dtypes == 'int64'].index.values.tolist(),\n",
     "        'cat_cols': X.dtypes[X.dtypes == 'object'].index.values.tolist(),\n",
@@ -127,12 +124,10 @@
     "    return pipeline\n",
     "\n",
     "outdirname = 'responsibleai.12.28.21'\n",
-    "try:\n",
-    "    from urllib import urlretrieve\n",
-    "except ImportError:\n",
-    "    from urllib.request import urlretrieve\n",
     "zipfilename = outdirname + '.zip'\n",
-    "urlretrieve('https://publictestdatasets.blob.core.windows.net/data/' + zipfilename, zipfilename)\n",
+    "\n",
+    "fetch_dataset('https://publictestdatasets.blob.core.windows.net/data/' + zipfilename, zipfilename)\n",
+    "\n",
     "with zipfile.ZipFile(zipfilename, 'r') as unzip:\n",
     "    unzip.extractall('.')\n",
     "\n",

diff --git a/...esponsibleaidashboard/responsibleaidashboard-housing-classification-model-debugging.ipynb b/...esponsibleaidashboard/responsibleaidashboard-housing-classification-model-debugging.ipynb
@@ -48,8 +48,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import shap\n",
-    "import sklearn\n",
     "import pandas as pd\n",
     "\n",
     "from sklearn.model_selection import train_test_split\n",
@@ -72,9 +70,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "from raiutils.dataset import fetch_dataset\n",
     "from sklearn.pipeline import Pipeline\n",
     "from sklearn.impute import SimpleImputer\n",
-    "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
+    "from sklearn.preprocessing import OneHotEncoder\n",
     "from sklearn.compose import ColumnTransformer\n",
     "\n",
     "def split_label(dataset, target_feature):\n",
@@ -90,8 +89,7 @@
     "        'cat_cols': X.dtypes[X.dtypes == 'object'].index.values.tolist(),\n",
     "    }\n",
     "    num_pipe = Pipeline([\n",
-    "        ('num_imputer', SimpleImputer(strategy='median'))#,\n",
-    "        #('num_scaler', StandardScaler())\n",
+    "        ('num_imputer', SimpleImputer(strategy='median'))\n",
     "    ])\n",
     "    cat_pipe = Pipeline([\n",
     "        ('cat_imputer', SimpleImputer(strategy='constant', fill_value='?')),\n",
@@ -109,12 +107,10 @@
     "categorical_features = []\n",
     "\n",
     "outdirname = 'responsibleai.12.28.21'\n",
-    "try:\n",
-    "    from urllib import urlretrieve\n",
-    "except ImportError:\n",
-    "    from urllib.request import urlretrieve\n",
     "zipfilename = outdirname + '.zip'\n",
-    "urlretrieve('https://publictestdatasets.blob.core.windows.net/data/' + zipfilename, zipfilename)\n",
+    "\n",
+    "fetch_dataset('https://publictestdatasets.blob.core.windows.net/data/' + zipfilename, zipfilename)\n",
+    "\n",
     "with zipfile.ZipFile(zipfilename, 'r') as unzip:\n",
     "    unzip.extractall('.')\n",
     "\n",

diff --git a/notebooks/responsibleaidashboard/responsibleaidashboard-housing-decision-making.ipynb b/notebooks/responsibleaidashboard/responsibleaidashboard-housing-decision-making.ipynb
@@ -39,7 +39,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import sklearn\n",
     "import pandas as pd\n",
     "from sklearn.model_selection import train_test_split\n",
     "import zipfile"
@@ -60,6 +59,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "from raiutils.dataset import fetch_dataset\n",
     "from sklearn.pipeline import Pipeline\n",
     "from sklearn.impute import SimpleImputer\n",
     "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
@@ -97,12 +97,10 @@
     "categorical_features = []\n",
     "\n",
     "outdirname = 'responsibleai.12.28.21'\n",
-    "try:\n",
-    "    from urllib import urlretrieve\n",
-    "except ImportError:\n",
-    "    from urllib.request import urlretrieve\n",
     "zipfilename = outdirname + '.zip'\n",
-    "urlretrieve('https://publictestdatasets.blob.core.windows.net/data/' + zipfilename, zipfilename)\n",
+    "\n",
+    "fetch_dataset('https://publictestdatasets.blob.core.windows.net/data/' + zipfilename, zipfilename)\n",
+    "\n",
     "with zipfile.ZipFile(zipfilename, 'r') as unzip:\n",
     "    unzip.extractall('.')\n",
     "\n",

diff --git a/raiwidgets/requirements-dev.txt b/raiwidgets/requirements-dev.txt
@@ -4,7 +4,7 @@ pytest==5.0.1
 mock==4.0.1
 pytest-cov
 pytest-mock==3.1.1
-requests==2.24.0
+requests==2.25.1
 
 requirements-parser==0.2.0
 

diff --git a/responsibleai/requirements.txt b/responsibleai/requirements.txt
@@ -15,3 +15,4 @@ semver~=2.13.0
 networkx<=2.5
 ipykernel<=6.6.0
 markupsafe<2.1.0
+raiutils>=0.0.1