Merge pull request #8 from FZJ-IEK3-VSA/develop

Develop
FZJ-IEK3-VSA · Apr 15, 2024 · 15dac3d · 15dac3d
2 parents e477439 + 72bab2f
commit 15dac3d
Show file tree

Hide file tree

Showing 5 changed files with 117 additions and 58 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -10,16 +10,17 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.10
+    - name: Set up Python 3.11
       uses: actions/setup-python@v3
       with:
-        python-version: 3.10.*
+        python-version: 3.11.*
     - name: Add conda to system path
       run: |
         # $CONDA is an environment variable pointing to the root of the miniconda directory
         echo $CONDA/bin >> $GITHUB_PATH
     - name: Install dependencies
       run: |
+        conda install -y python=3.11
         conda env update --file requirements.yml --name base
     - name: Test with Pytest
       env:

diff --git a/README.md b/README.md
@@ -13,11 +13,11 @@ Step-by-step instructions to access data, published on LOCALISED datasharing pla
 2. Installing dependencies in a clean conda environment:
     ```bash
     cd LOCALISED-Datasharing-API-Client
-    conda env update -n zoomin-client -f requirements.yml 
-    conda activate zoomin-client
+    conda env update -n dsp_client -f requirements.yml 
+    conda activate dsp_client
     ```
 
-3. Run the example Jupyter notebooks -  `examples/all_regions_single_variable.ipynb` and `examples/single_region_all_variables.ipynb`
+3. Run the example Jupyter notebooks -  `examples/all_regions_data_.ipynb` and `examples/single_region_data.ipynb`
 
     **Depending on type of query, the list of required parameters change. Please refer to the doc strings of each funtion to get the entire list of relevant parameters. They would be a subset of the ones below:**
 

diff --git a/examples/all_regions_single_variable.ipynb → examples/all_regions_data.ipynb b/examples/all_regions_single_variable.ipynb → examples/all_regions_data.ipynb
@@ -18,7 +18,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "30e6e6e9",
    "metadata": {},
    "outputs": [],
@@ -42,7 +42,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "446199c1",
    "metadata": {},
    "outputs": [],
@@ -65,18 +65,27 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "id": "82c0e062",
    "metadata": {},
    "outputs": [],
    "source": [
     "var_metadata = client.get_variable_metadata(api_key=api_key, \n",
     "                                variable_name=\"population\",\n",
-    "                                country_code=\"DE\",\n",
-    "                                spatial_resolution=\"LAU\",\n",
+    "                                country_code=\"ES\"\n",
     "                                )"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7f7221f0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "var_metadata"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "3b4ebce9",
@@ -87,22 +96,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "73caf789",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:utils:Elapsed time for <function get_variable_data at 0x0000013F5757F1C0>: 0.87 minutes\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "result_data = client.get_variable_data(api_key=api_key, \n",
     "                                variable_name=\"population\",\n",
-    "                                country_code=\"DE\",\n",
+    "                                country_code=\"ES\",\n",
     "                                spatial_resolution=\"LAU\",\n",
     "                                result_format=\"df\",\n",
     "                                      )"
@@ -114,6 +115,16 @@
    "id": "c44bd695",
    "metadata": {},
    "outputs": [],
+   "source": [
+    "result_data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "64ad1a22",
+   "metadata": {},
+   "outputs": [],
    "source": []
   }
  ],
@@ -133,7 +144,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.4"
+   "version": "3.10.13"
   }
  },
  "nbformat": 4,

diff --git a/examples/single_region_all_variables.ipynb → examples/single_region_data.ipynb b/examples/single_region_all_variables.ipynb → examples/single_region_data.ipynb
@@ -18,7 +18,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "4cb92d26",
    "metadata": {},
    "outputs": [],
@@ -42,7 +42,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "d0c2beef",
    "metadata": {},
    "outputs": [],
@@ -65,7 +65,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "id": "bf15db85",
    "metadata": {},
    "outputs": [],
@@ -79,7 +79,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
+   "id": "ec83e661",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "region_list"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
    "id": "da9a5a25",
    "metadata": {},
    "outputs": [],
@@ -88,10 +98,20 @@
     "single_region = client.get_region_metadata(api_key=api_key, \n",
     "                                spatial_resolution=\"LAU\",\n",
     "                                country_code = \"DE\",\n",
-    "                                region_code = \"11000000\"\n",
+    "                                region_code = \"DE600_02000000\"\n",
     "                                )"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "01229f87",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "single_region"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "e7774b86",
@@ -102,23 +122,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "0bdec9ab",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:utils:Elapsed time for <function get_region_data at 0x0000015407D6F010>: 0.89 minutes\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
+    "# All data\n",
     "region_data = client.get_region_data(api_key=api_key, \n",
-    "                                spatial_resolution=\"LAU\",\n",
-    "                                country_code = \"DE\",\n",
-    "                                region_code = \"11000000\",\n",
+    "                                region_code = \"DE600_02000000\",\n",
     "                                mini_version=True\n",
     "                                )\n",
     "#NOTE: Set mini_version to False to get all fields "
@@ -127,7 +138,42 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "19975267",
+   "id": "6c32b11a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "region_data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# single variable\n",
+    "region_data_single_var = client.get_region_data(api_key=api_key, \n",
+    "                                region_code = \"DE711_06411000\",\n",
+    "                                variable_name= \"population\",\n",
+    "                                mini_version=False\n",
+    "                                )\n",
+    "#NOTE: single variable is only possible for full version at the moment. This should be fixed in the next release"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0c8d1816",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "region_data_single_var"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "02d39431",
    "metadata": {},
    "outputs": [],
    "source": []
@@ -149,7 +195,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.4"
+   "version": "3.10.13"
   }
  },
  "nbformat": 4,

diff --git a/zoomin_client/client.py b/zoomin_client/client.py
@@ -89,7 +89,7 @@ def get_region_metadata(
     :rtype: list/dict
     """
     # request
-    next_request_url = f"http://data.localised-project.eu/api/v1/region_metadata/?api_key={api_key}&resolution={spatial_resolution}&country={country_code}"
+    next_request_url = f"http://data.localised-project.eu/dsp/v1/region_metadata/?api_key={api_key}&resolution={spatial_resolution}&country={country_code}"
 
     if region_code is not None:
         next_request_url = f"{next_request_url}&region={region_code}"
@@ -121,9 +121,8 @@ def get_region_metadata(
 @measure_time
 def get_region_data(
     api_key: str,
-    spatial_resolution: str,
     region_code: str,
-    country_code: str,
+    variable_name: Optional[str] = None,
     mini_version: Optional[bool] = True,
     result_format: Optional[str] = "json",
     save_result: Optional[bool] = False,
@@ -136,17 +135,15 @@ def get_region_data(
     :param api_key: the secret api key
     :type api_key: str
 
-    :param spatial_resolution: the required spatial resolution
-    :type spatial_resolution: str, one of {'NUTS0', 'NUTS1', 'NUTS2', 'NUTS3', 'LAU'}
-
     :param region_code: the code of the region to filter on
     :type region_code: str
 
-    :param country_code: the code of the country to which `region_code` belongs.
-    :type region_code: str
-
     **Default arguments:**
 
+    :param variable_name: the variable to filter on
+        |br| * the default value is None
+    :type variable_name: str
+
     :param mini_version: indicates if a reduced number of fields on data should be returned
         |br| * the default value is True
     :type mini_version: bool
@@ -174,14 +171,18 @@ def get_region_data(
     :rtype: list/pd.DataFrame
     """
     # request
-    base_url = "http://data.localised-project.eu/api/v1/region_data/"
+    base_url = "http://data.localised-project.eu/dsp/v1/region_data/"
     if mini_version:
         base_url = f"{base_url}mini_version/"
 
-    next_request_url = f"{base_url}?api_key={api_key}&resolution={spatial_resolution}&region={region_code}&country={country_code}"
+    if variable_name is None:
+        next_request_url = f"{base_url}?api_key={api_key}&region={region_code}"
+    else:
+        next_request_url = f"{base_url}?api_key={api_key}&region={region_code}&variable={variable_name}"
 
     result_collection = []
     while next_request_url is not None:
+        print(next_request_url)
         response = requests.get(next_request_url, stream=True, timeout=240).json()
 
         next_request_url = response["next"]
@@ -224,7 +225,6 @@ def get_variable_metadata(
     api_key: str,
     variable_name: str,
     country_code: str,
-    spatial_resolution: str,
     save_result: Optional[bool] = False,
     save_path: Optional[str] = None,
     save_name: Optional[str] = "variable_metadata",
@@ -239,7 +239,7 @@ def get_variable_metadata(
     :type variable_name: str
 
     :param country_code: the code of the country for which data should be returned
-    :type region_code: str
+    :type country_code: str
 
     :param spatial_resolution: the required spatial resolution
     :type spatial_resolution: str, one of {'NUTS0', 'NUTS1', 'NUTS2', 'NUTS3', 'LAU'}
@@ -264,7 +264,7 @@ def get_variable_metadata(
     :rtype: dict
     """
     # request
-    request_url = f"http://data.localised-project.eu/api/v1/variable_metadata/?api_key={api_key}&country={country_code}&resolution={spatial_resolution}&variable={variable_name}"
+    request_url = f"http://data.localised-project.eu/dsp/v1/variable_metadata/?api_key={api_key}&country={country_code}&variable={variable_name}"
 
     response = requests.get(request_url, stream=True, timeout=240).json()
 
@@ -305,7 +305,7 @@ def get_variable_data(
     :type variable_name: str
 
     :param country_code: the code of the country for which data should be returned
-    :type region_code: str
+    :type country_code: str
 
     :param spatial_resolution: the required spatial resolution
     :type spatial_resolution: str, one of {'NUTS0', 'NUTS1', 'NUTS2', 'NUTS3', 'LAU'}
@@ -335,10 +335,11 @@ def get_variable_data(
     :rtype: list/pd.DataFrame
     """
     # request
-    next_request_url = f"http://data.localised-project.eu/api/v1/variable_data/?api_key={api_key}&country={country_code}&resolution={spatial_resolution}&variable={variable_name}"
+    next_request_url = f"http://data.localised-project.eu/dsp/v1/variable_data/?api_key={api_key}&country={country_code}&resolution={spatial_resolution}&variable={variable_name}"
 
     result_collection = []
     while next_request_url is not None:
+        print(next_request_url)
         response = requests.get(next_request_url, stream=True, timeout=240).json()
 
         next_request_url = response["next"]