diff --git a/.github/workflows/run-python-tests.yaml b/.github/workflows/run-python-tests.yaml index 7e3ec06e..59b1bb95 100644 --- a/.github/workflows/run-python-tests.yaml +++ b/.github/workflows/run-python-tests.yaml @@ -28,6 +28,9 @@ jobs: python -m pip install --upgrade pip pip install --upgrade hatch - name: Test with hatch/pytest + env: + HDX_KEY_TEST: ${{ secrets.HDX_BOT_SCRAPERS_API_TOKEN }} + GSHEET_AUTH: ${{ secrets.GSHEET_AUTH }} run: | hatch run test:test - name: Check styling diff --git a/pyproject.toml b/pyproject.toml index 6f9f535c..2c9b4906 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ dependencies = [ "defopt>=6.4.0", "email_validator", "hdx-python-country>=3.6.4", - "hdx-python-utilities>=3.6.4", + "hdx-python-utilities>=3.6.5", "libhxl>=5.2", "makefun", "ndg-httpsclient", @@ -57,7 +57,7 @@ content-type = "text/markdown" Homepage = "https://github.com/OCHA-DAP/hdx-python-api" [project.optional-dependencies] -test = ["pytest", "pytest-cov"] +test = ["pytest", "pytest-cov", "gspread"] dev = ["pre-commit"] diff --git a/requirements.txt b/requirements.txt index 259fa26b..d126c352 100755 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,8 @@ attrs==23.2.0 # frictionless # jsonlines # jsonschema +cachetools==5.3.3 + # via google-auth certifi==2024.2.2 # via requests cffi==1.16.0 @@ -27,17 +29,17 @@ click==8.1.7 # via typer colorama==0.4.6 # via typer -coverage[toml]==7.4.1 +coverage[toml]==7.4.3 # via # coverage # pytest-cov -cryptography==42.0.2 +cryptography==42.0.5 # via pyopenssl defopt==6.4.0 # via hdx-python-api (pyproject.toml) distlib==0.3.8 # via virtualenv -dnspython==2.5.0 +dnspython==2.6.1 # via email-validator docopt==0.6.2 # via @@ -45,7 +47,7 @@ docopt==0.6.2 # num2words docutils==0.20.1 # via defopt -email-validator==2.1.0.post1 +email-validator==2.1.1 # via hdx-python-api (pyproject.toml) et-xmlfile==1.1.0 # via openpyxl @@ -53,6 +55,14 @@ filelock==3.13.1 # via virtualenv frictionless==5.16.1 # via hdx-python-utilities +google-auth==2.28.1 + # via + # google-auth-oauthlib + # gspread +google-auth-oauthlib==1.2.0 + # via gspread +gspread==6.0.2 + # via hdx-python-api (pyproject.toml) hdx-python-country==3.6.4 # via hdx-python-api (pyproject.toml) hdx-python-utilities==3.6.5 @@ -61,7 +71,7 @@ hdx-python-utilities==3.6.5 # hdx-python-country humanize==4.9.0 # via frictionless -identify==2.5.34 +identify==2.5.35 # via pre-commit idna==3.6 # via @@ -95,7 +105,7 @@ makefun==1.15.2 # via hdx-python-api (pyproject.toml) markdown-it-py==3.0.0 # via rich -marko==2.0.2 +marko==2.0.3 # via frictionless markupsafe==2.1.5 # via jinja2 @@ -107,6 +117,8 @@ nodeenv==1.8.0 # via pre-commit num2words==0.5.13 # via quantulum3 +oauthlib==3.2.2 + # via requests-oauthlib openpyxl==3.1.2 # via hdx-python-utilities packaging==23.2 @@ -123,19 +135,23 @@ ply==3.11 # libhxl pockets==0.9.1 # via sphinxcontrib-napoleon -pre-commit==3.6.1 +pre-commit==3.6.2 # via hdx-python-api (pyproject.toml) pyasn1==0.5.1 # via # hdx-python-api (pyproject.toml) # ndg-httpsclient + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth pycparser==2.21 # via cffi -pydantic==2.6.1 +pydantic==2.6.3 # via # frictionless # inflect -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pygments==2.17.2 # via rich @@ -147,7 +163,7 @@ pyphonetics==0.5.3 # via hdx-python-country pyrsistent==0.20.0 # via jsonschema -pytest==8.0.0 +pytest==8.0.2 # via # hdx-python-api (pyproject.toml) # pytest-cov @@ -180,12 +196,17 @@ requests==2.31.0 # hdx-python-api (pyproject.toml) # libhxl # requests-file + # requests-oauthlib requests-file==2.0.0 # via hdx-python-utilities +requests-oauthlib==1.3.1 + # via google-auth-oauthlib rfc3986==2.0.0 # via frictionless -rich==13.7.0 +rich==13.7.1 # via typer +rsa==4.9 + # via google-auth ruamel-yaml==0.18.6 # via hdx-python-utilities ruamel-yaml-clib==0.2.8 @@ -203,6 +224,8 @@ six==1.16.0 # sphinxcontrib-napoleon sphinxcontrib-napoleon==0.7 # via defopt +strenum==0.4.15 + # via gspread stringcase==1.2.0 # via frictionless structlog==24.1.0 @@ -217,7 +240,7 @@ typer[all]==0.9.0 # via # frictionless # typer -typing-extensions==4.9.0 +typing-extensions==4.10.0 # via # frictionless # inflect @@ -228,13 +251,13 @@ unidecode==1.3.8 # via # libhxl # pyphonetics -urllib3==2.2.0 +urllib3==2.2.1 # via # libhxl # requests validators==0.22.0 # via frictionless -virtualenv==20.25.0 +virtualenv==20.25.1 # via pre-commit wheel==0.42.0 # via libhxl @@ -242,7 +265,7 @@ xlrd==2.0.1 # via hdx-python-utilities xlrd3==1.1.0 # via libhxl -xlsxwriter==3.1.9 +xlsxwriter==3.2.0 # via tableschema-to-template xlwt==1.3.0 # via hdx-python-utilities diff --git a/src/hdx/api/configuration.py b/src/hdx/api/configuration.py index cf205d71..d2d2c1fb 100755 --- a/src/hdx/api/configuration.py +++ b/src/hdx/api/configuration.py @@ -136,7 +136,7 @@ def __init__(self, **kwargs: Any) -> None: f"No HDX configuration parameter and no configuration file at default path: {Configuration.default_hdx_config_yaml}." ) hdx_config_yaml = None - hdx_config_dict = dict() + hdx_config_dict = {} if hdx_config_yaml: logger.info( f"Loading HDX configuration from: {hdx_config_yaml}" @@ -178,7 +178,7 @@ def __init__(self, **kwargs: Any) -> None: ) project_config_dict = load_yaml(project_config_yaml) else: - project_config_dict = dict() + project_config_dict = {} self.data = merge_two_dictionaries( hdx_base_config_dict, project_config_dict diff --git a/src/hdx/data/dataset.py b/src/hdx/data/dataset.py index 0e1ac901..06ffd60a 100755 --- a/src/hdx/data/dataset.py +++ b/src/hdx/data/dataset.py @@ -116,7 +116,7 @@ def __init__( configuration: Optional[Configuration] = None, ) -> None: if not initial_data: - initial_data = dict() + initial_data = {} self.init_resources() super().__init__(initial_data, configuration=configuration) self.preview_resourceview = None @@ -228,7 +228,7 @@ def init_resources(self) -> None: Returns: None """ - self.resources: List[res_module.Resource] = list() + self.resources: List[res_module.Resource] = [] def _get_resource_from_obj( self, resource: Union["Resource", Dict, str] @@ -308,7 +308,7 @@ def add_update_resources( Returns: None """ - resource_objects = list() + resource_objects = [] for resource in resources: resource = self._get_resource_from_obj(resource) if "package_id" in resource: @@ -412,7 +412,7 @@ def reorder_resources( data = {"id": dataset_id, "order": resource_ids} results = self._write_to_hdx("reorder", data) ordered_ids = results["order"] - reordered_resources = list() + reordered_resources = [] for resource_id in ordered_ids: resource = next( x for x in self.resources if x["id"] == resource_id @@ -514,7 +514,7 @@ def check_required_fields( Returns: None """ - dataset_ignore_fields = list() + dataset_ignore_fields = [] for ignore_field in ignore_fields: if not ignore_field.startswith("resource:"): dataset_ignore_fields.append(ignore_field) @@ -537,8 +537,8 @@ def check_required_fields( def revise( match: Dict[str, Any], filter: ListTuple[str] = tuple(), - update: Dict[str, Any] = dict(), - files_to_upload: Dict[str, str] = dict(), + update: Dict[str, Any] = {}, + files_to_upload: Dict[str, str] = {}, configuration: Optional[Configuration] = None, **kwargs: Any, ) -> "Dataset": @@ -547,8 +547,8 @@ def revise( Args: match (Dict[str,Any]): Metadata on which to match dataset filter (ListTuple[str]): Filters to apply. Defaults to tuple(). - update (Dict[str,Any]): Metadata updates to apply. Defaults to dict(). - files_to_upload (Dict[str,str]): Files to upload to HDX. Defaults to dict(). + update (Dict[str,Any]): Metadata updates to apply. Defaults to {}. + files_to_upload (Dict[str,str]): Files to upload to HDX. Defaults to {}. configuration (Optional[Configuration]): HDX configuration. Defaults to global configuration. **kwargs: Additional arguments to pass to package_revise @@ -575,36 +575,23 @@ def revise( dataset.separate_resources() return dataset - def _save_dataset_add_filestore_resources( - self, - default_operation: str, - id_field_name: str, - keys_to_delete: ListTuple[str], - resources_to_delete: ListTuple[int], - new_resource_order: Optional[ListTuple[str]], - filestore_resources: Dict[int, str], - hxl_update: bool, - create_default_views: bool = False, - **kwargs: Any, - ) -> Dict: - """Helper method to save the modified dataset and add any filestore resources + def _prepare_hdx_call(self, data: Dict, kwargs: Any) -> None: + """Common method used in create and update calls. Cleans tags and + processes keyword arguments populating updated_by_script (details + about what script is doing the update and when), batch + and setting batch_mode in kwargs if needed (whether datasets on the + CKAN /datasets page are grouped). Args: - default_operation (str): Operation to perform eg. patch. Defaults to update. - id_field_name (str): Name of field containing HDX object identifier - keys_to_delete (ListTuple[str]): List of top level metadata keys to delete - resources_to_delete (ListTuple[int]): List of indexes of resources to delete - new_resource_order (Optional[ListTuple[str]]): New resource order to use or None - filestore_resources (Dict[int, str]): List of (index of resources, file to upload) - hxl_update (bool): Whether to call package_hxl_update. - create_default_views (bool): Whether to create default views. Defaults to False. + data (Dict): Dataset data to update if needed **kwargs: See below - ignore_field (str): Any field to ignore when checking dataset metadata. Defaults to None. + updated_by_script (str): Script info. Defaults to user agent. + batch: Batch UUID for where multiple datasets are grouped into one batch + batch_mode: Whether to group by batch. Defaults to not grouping. Returns: - Dict: Dictionary of what gets passed to the revise call (for testing) + None """ - self.unseparate_resources() self.clean_tags() scriptinfo = kwargs.get("updated_by_script") if scriptinfo: @@ -612,83 +599,201 @@ def _save_dataset_add_filestore_resources( else: scriptinfo = self.configuration.get_user_agent() # No need to output timezone info here - self.data[ + data[ "updated_by_script" ] = f"{scriptinfo} ({datetime.now(timezone.utc).replace(tzinfo=None).isoformat(timespec='microseconds')})" batch = kwargs.get("batch") if batch: if not is_valid_uuid(batch): raise HDXError(f"{batch} is not a valid UUID!") - self.data["batch"] = batch + data["batch"] = batch del kwargs["batch"] if "batch_mode" not in kwargs: kwargs["batch_mode"] = "DONT_GROUP" - operation = kwargs.get("operation") - if not operation: - operation = default_operation - kwargs["operation"] = operation - existing_ignore_check = kwargs.get("ignore_check") - revise = True - if "test" in kwargs: - test = True - del kwargs["test"] - else: - test = False - if operation == "create": - if not existing_ignore_check: - kwargs["ignore_check"] = True - self._hdx_update( - "dataset", id_field_name, force_active=True, **kwargs - ) - if not filestore_resources and not keys_to_delete: - revise = False - self.init_resources() - self.separate_resources() - results = dict() - if revise: - self.old_data = self.data - self._check_kwargs_fields("dataset", **kwargs) - self.data["state"] = "active" - filter = list() - for key in keys_to_delete: - filter.append(f"-{key}") - self.data.pop(key, None) - files_to_upload = dict() - if not self.is_requestable(): - resources = self.data["resources"] - for resource_index in resources_to_delete: - del resources[resource_index] - if resource_index == len(resources): - filter.append(f"-resources__{resource_index}") - new_fsresources = dict() - for index in filestore_resources: - if index > resource_index: - new_fsresources[index - 1] = filestore_resources[ - index - ] - else: - new_fsresources[index] = filestore_resources[index] - filestore_resources = new_fsresources - for ( - resource_index, - file_to_upload, - ) in filestore_resources.items(): - files_to_upload[ - f"update__resources__{resource_index}__upload" - ] = file_to_upload - results["filter"] = filter - results["update"] = self.data - results["files_to_upload"] = files_to_upload - if test: - return results - new_dataset = self.revise( - {"id": self.data["id"]}, - filter=filter, - update=self.data, - files_to_upload=files_to_upload, + + def _revise_filter( + self, dataset_data_to_update: Dict, keys_to_delete: ListTuple[str] + ): + """Returns the revise filter parameter adding to it any keys in the + dataset metadata that are specified to be deleted. Also compare lists + in original and updated metadata to see if any have had elements + removed in which case these should be added to the filter + + Args: + dataset_data_to_update (Dict): Dataset data to be updated + keys_to_delete (ListTuple[str]): List of top level metadata keys to delete + """ + revise_filter = [] + for key in keys_to_delete: + revise_filter.append(f"-{key}") + for key, value in dataset_data_to_update.items(): + if not isinstance(value, list): + continue + if key not in self.old_data: + continue + orig_list = self.old_data[key] + elements_to_remove = [] + for i, orig_value in enumerate(orig_list): + if isinstance(orig_value, dict) and any( + x in orig_value for x in ("id", "name") + ): + # Where lists have an id or name, we use one of those to + # match elements and work out what needs to be deleted + # ie. any elements in the original list that don't exist + # in the updated list + el_id = orig_value.get("id") + el_name = orig_value.get("name") + present = False + for value_dict in value: + if ( + el_id + and "id" in value_dict + and value_dict["id"] == el_id + ): + present = True + break + if ( + el_name + and "name" in value_dict + and value_dict["name"] == el_name + ): + present = True + break + if not present: + elements_to_remove.append(i) + elif orig_value not in value: + # Otherwise we do a simple exact match of list elements + # and delete any elements that are in the original list but + # not in the updated list + elements_to_remove.append(i) + for element_index in reversed(elements_to_remove): + del orig_list[element_index] + if element_index == len(orig_list): + revise_filter.append(f"-{key}__{element_index}") + return revise_filter + + def _revise_files_to_upload_resource_deletions( + self, + revise_filter: List[str], + resources_to_update: ListTuple["Resource"], + resources_to_delete: ListTuple[int], + filestore_resources: Dict[int, str], + ): + """Returns the files to be uploaded and updates the revise filter with + any resources to be deleted also updating filestore_resources to + reflect any deletions. + + Args: + revise_filter (List[str]): Keys and list elements to delete + resources_to_update (ListTuple[Resource]): Resources to update + resources_to_delete (ListTuple[int]): List of indexes of resources to delete + filestore_resources (Dict[int, str]): List of (index of resources, file to upload) + """ + files_to_upload = {} + if not self.is_requestable(): + for resource_index in resources_to_delete: + del resources_to_update[resource_index] + if resource_index == len(resources_to_update): + revise_filter.append(f"-resources__{resource_index}") + new_fsresources = {} + for index in filestore_resources: + if index > resource_index: + new_fsresources[index - 1] = filestore_resources[index] + else: + new_fsresources[index] = filestore_resources[index] + filestore_resources = new_fsresources + for ( + resource_index, + file_to_upload, + ) in filestore_resources.items(): + files_to_upload[ + f"update__resources__{resource_index}__upload" + ] = file_to_upload + return files_to_upload + + def _revise_dataset( + self, + keys_to_delete: ListTuple[str], + resources_to_update: ListTuple["Resource"], + resources_to_delete: ListTuple[int], + filestore_resources: Dict[int, str], + new_resource_order: Optional[ListTuple[str]], + hxl_update: bool, + create_default_views: bool = False, + test: bool = False, + **kwargs: Any, + ) -> Dict: + """Helper method to save the modified dataset and add any filestore resources + + Args: + keys_to_delete (ListTuple[str]): List of top level metadata keys to delete + resources_to_update (ListTuple[Resource]): Resources to update + resources_to_delete (ListTuple[int]): List of indexes of resources to delete + filestore_resources (Dict[int, str]): List of (index of resources, file to upload) + new_resource_order (Optional[ListTuple[str]]): New resource order to use or None + hxl_update (bool): Whether to call package_hxl_update. + create_default_views (bool): Whether to create default views. Defaults to False. + test (bool): Whether running in a test. Defaults to False. + **kwargs: See below + ignore_field (str): Any field to ignore when checking dataset metadata. Defaults to None. + + Returns: + Dict: Dictionary of what gets passed to the revise call (for testing) + """ + results = {} + dataset_data_to_update = self.old_data + self.old_data = self.data + if ( + "batch_mode" in kwargs + ): # Whether or not CKAN should change groupings of datasets on /datasets page + dataset_data_to_update["batch_mode"] = kwargs["batch_mode"] + if ( + "skip_validation" in kwargs + ): # Whether or not CKAN should perform validation steps (checking fields present) + dataset_data_to_update["skip_validation"] = kwargs[ + "skip_validation" + ] + dataset_data_to_update["state"] = "active" + revise_filter = self._revise_filter( + dataset_data_to_update, keys_to_delete + ) + files_to_upload = self._revise_files_to_upload_resource_deletions( + revise_filter, + resources_to_update, + resources_to_delete, + filestore_resources, + ) + dataset_data_to_update["resources"] = self._convert_hdxobjects( + resources_to_update + ) + results["filter"] = revise_filter + results["update"] = dataset_data_to_update + results["files_to_upload"] = files_to_upload + if test: + return results + new_dataset = self.revise( + {"id": self.data["id"]}, + filter=revise_filter, + update=dataset_data_to_update, + files_to_upload=files_to_upload, + ) + self.data = new_dataset.data + self.resources = new_dataset.resources + + # We do field check after call so that we have the changed data + if "ignore_check" not in kwargs or not kwargs.get( + "ignore_check" + ): # allow ignoring of field checks + ignore_fields = kwargs.get("ignore_fields", list()) + ignore_field = self.configuration["dataset"].get( + "ignore_on_update" ) - self.data = new_dataset.data - self.resources = new_dataset.resources + if ignore_field and ignore_field not in ignore_fields: + ignore_fields.append(ignore_field) + ignore_field = kwargs.get("ignore_field") + if ignore_field and ignore_field not in ignore_fields: + ignore_fields.append(ignore_field) + self.check_required_fields(ignore_fields=ignore_fields) if new_resource_order: existing_order = [ (x["name"], x["format"].lower()) for x in self.resources @@ -710,17 +815,18 @@ def _save_dataset_add_filestore_resources( self.hxl_update() return results - def _dataset_merge_update_resources( + def _dataset_update_resources( self, update_resources: bool, match_resources_by_metadata: bool, remove_additional_resources: bool, match_resource_order: bool, **kwargs: Any, - ) -> Tuple[List, List, Dict]: - """Helper method to merge new and existing dataset data, update - resources including those with files in the filestore and delete extra - resources if needed. + ) -> Tuple[List, List, Dict, List]: + """Helper method to compare new and existing dataset data returning + resources to be updated, resources to be deleted, resources where files + need to be uploaded to the filestore and if match_resource_order is + True, then the new resource order. Args: update_resources (bool): Whether to update resources @@ -729,20 +835,18 @@ def _dataset_merge_update_resources( match_resource_order (bool): Match order of given resources by name Returns: - Tuple[List, List, Dict]: (resources_to_delete, new_resource_order, filestore_resources) + Tuple[List, List, Dict, List]: (resources_to_update, resources_to_delete, filestore_resources, new_resource_order) """ # When the user sets up a dataset, "data" contains the metadata. The # HDX dataset update process involves copying "data" to "old_data" and # then reading the existing dataset on HDX into "data". Hence, # "old_data" below contains the user-supplied data we want to use for # updating while "data" contains the data read from HDX - merge_two_dictionaries(self.data, self.old_data) - if "resources" in self.data: - del self.data["resources"] - updated_resources = self.old_data.get("resources", None) - resources_to_delete = list() - filestore_resources = dict() - if update_resources and updated_resources: + resources_metadata_to_update = self.old_data.get("resources", None) + resources_to_update = [] + resources_to_delete = [] + filestore_resources = {} + if update_resources and resources_metadata_to_update: if match_resources_by_metadata: ( resource_matches, @@ -750,35 +854,40 @@ def _dataset_merge_update_resources( resource_no_matches, updated_resource_no_matches, ) = ResourceMatcher.match_resource_lists( - self.resources, updated_resources + self.resources, resources_metadata_to_update ) - for i, resource_index in enumerate(resource_matches): - resource = self.resources[resource_index] - updated_resource = updated_resources[ - updated_resource_matches[i] + for i, resource in enumerate(self.resources): + try: + match_index = resource_matches.index(i) + except ValueError: + resources_to_update.append(res_module.Resource({})) + continue + resource_data_to_update = resources_metadata_to_update[ + updated_resource_matches[match_index] ] logger.warning( f"Resource exists. Updating {resource['name']}" ) - filestore_helper.FilestoreHelper.dataset_merge_filestore_resource( - resource, - updated_resource, + filestore_helper.FilestoreHelper.dataset_update_filestore_resource( + resource_data_to_update, filestore_resources, - resource_index, - **kwargs, + i, ) + resources_to_update.append(resource_data_to_update) + + resource_index = len(self.resources) for updated_resource_index in updated_resource_no_matches: - updated_resource = updated_resources[ + resource_data_to_update = resources_metadata_to_update[ updated_resource_index ] - resource_index = len(self.resources) filestore_helper.FilestoreHelper.check_filestore_resource( - updated_resource, + resource_data_to_update, filestore_resources, resource_index, **kwargs, ) - self.resources.append(updated_resource) + resource_index = resource_index + 1 + resources_to_update.append(resource_data_to_update) if remove_additional_resources: for resource_index in resource_no_matches: resource = self.resources[resource_index] @@ -787,11 +896,12 @@ def _dataset_merge_update_resources( ) resources_to_delete.append(resource_index) else: # update resources by position - for i, updated_resource in enumerate(updated_resources): + for i, resource_data_to_update in enumerate( + resources_metadata_to_update + ): if len(self.resources) > i: - updated_resource_name = updated_resource["name"] - resource = self.resources[i] - resource_name = resource["name"] + updated_resource_name = resource_data_to_update["name"] + resource_name = self.resources[i]["name"] logger.warning( f"Resource exists. Updating {resource_name}" ) @@ -799,36 +909,43 @@ def _dataset_merge_update_resources( logger.warning( f"Changing resource name to: {updated_resource_name}" ) - filestore_helper.FilestoreHelper.dataset_merge_filestore_resource( - resource, - updated_resource, + filestore_helper.FilestoreHelper.dataset_update_filestore_resource( + resource_data_to_update, filestore_resources, i, - **kwargs, ) else: filestore_helper.FilestoreHelper.check_filestore_resource( - updated_resource, filestore_resources, i, **kwargs + resource_data_to_update, + filestore_resources, + i, + **kwargs, ) - self.resources.append(updated_resource) + resources_to_update.append(resource_data_to_update) - if remove_additional_resources: - for i, resource in enumerate(self.resources): - if len(updated_resources) <= i: + for i, resource in enumerate(self.resources): + if len(resources_metadata_to_update) <= i: + resources_to_update.append(resource) + if remove_additional_resources: logger.warning( f"Removing additional resource {resource['name']}!" ) resources_to_delete.append(i) - resources_to_delete = sorted(resources_to_delete, reverse=True) + resources_to_delete = list(reversed(resources_to_delete)) if match_resource_order: new_resource_order = [ - (x["name"], x["format"]) for x in updated_resources + (x["name"], x["format"]) for x in resources_metadata_to_update ] else: new_resource_order = None - return resources_to_delete, new_resource_order, filestore_resources + return ( + resources_to_update, + resources_to_delete, + filestore_resources, + new_resource_order, + ) - def _dataset_merge_hdx_update( + def _dataset_hdx_update( self, update_resources: bool, match_resources_by_metadata: bool, @@ -839,7 +956,7 @@ def _dataset_merge_hdx_update( hxl_update: bool, **kwargs: Any, ) -> Dict: - """Helper method to merge new and existing dataset data, update + """Helper method to compare new and existing dataset data, update resources including those with files in the filestore, delete extra resources if needed and update dataset data and save the dataset and resources to HDX. @@ -857,23 +974,24 @@ def _dataset_merge_hdx_update( Dict: Dictionary of what gets passed to the revise call (for testing) """ ( + resources_to_update, resources_to_delete, - new_resource_order, filestore_resources, - ) = self._dataset_merge_update_resources( + new_resource_order, + ) = self._dataset_update_resources( update_resources, match_resources_by_metadata, remove_additional_resources, match_resource_order, **kwargs, ) - return self._save_dataset_add_filestore_resources( - "update", - "id", + self._prepare_hdx_call(self.old_data, kwargs) + return self._revise_dataset( keys_to_delete, + resources_to_update, resources_to_delete, - new_resource_order, filestore_resources, + new_resource_order, hxl_update, create_default_views=create_default_views, **kwargs, @@ -921,7 +1039,7 @@ def update_in_hdx( self._check_existing_object("dataset", "name") if not self._dataset_load_from_hdx(self.data["name"]): raise HDXError("No existing dataset to update!") - self._dataset_merge_hdx_update( + self._dataset_hdx_update( update_resources=update_resources, match_resources_by_metadata=match_resources_by_metadata, keys_to_delete=keys_to_delete, @@ -968,6 +1086,8 @@ def create_in_hdx( self.check_required_fields( allow_no_resources=allow_no_resources, **kwargs ) + # No need to check again after revising dataset + kwargs["ignore_check"] = True loadedid = None if "id" in self.data: if self._dataset_load_from_hdx(self.data["id"]): @@ -980,7 +1100,7 @@ def create_in_hdx( if self._dataset_load_from_hdx(self.data["name"]): loadedid = self.data["name"] if loadedid: - self._dataset_merge_hdx_update( + self._dataset_hdx_update( update_resources=update_resources, match_resources_by_metadata=match_resources_by_metadata, keys_to_delete=keys_to_delete, @@ -993,22 +1113,31 @@ def create_in_hdx( logger.info(f"Updated {self.get_hdx_url()}") return - filestore_resources = dict() + filestore_resources = {} if self.resources: for i, resource in enumerate(self.resources): filestore_helper.FilestoreHelper.check_filestore_resource( resource, filestore_resources, i, **kwargs ) - self._save_dataset_add_filestore_resources( - "create", - "name", - keys_to_delete, - list(), - None, - filestore_resources, - hxl_update, - **kwargs, - ) + self.unseparate_resources() + self._prepare_hdx_call(self.data, kwargs) + kwargs["operation"] = "create" + if not kwargs.get("ignore_check"): + kwargs["ignore_check"] = True + self._hdx_update("dataset", "name", force_active=True, **kwargs) + if not filestore_resources and not keys_to_delete: + self.init_resources() + self.separate_resources() + else: + self._revise_dataset( + keys_to_delete, + self.resources, + [], + filestore_resources, + None, + hxl_update, + **kwargs, + ) logger.info(f"Created {self.get_hdx_url()}") def delete_from_hdx(self) -> None: @@ -1088,7 +1217,7 @@ def search_in_hdx( while ( attempts < cls.max_attempts and all_datasets is None ): # if the count values vary for multiple calls, then must redo query - all_datasets = list() + all_datasets = [] counts = set() for page in range(total_rows // page_size + 1): pagetimespagesize = page * page_size @@ -1103,7 +1232,7 @@ def search_in_hdx( Dataset.actions()["search"], **kwargs, ) - datasets = list() + datasets = [] if result: count = result.get("count", None) if count: @@ -1111,7 +1240,7 @@ def search_in_hdx( no_results = len(result["results"]) for datasetdict in result["results"]: dataset = Dataset(configuration=configuration) - dataset.old_data = dict() + dataset.old_data = {} dataset.data = datasetdict dataset._dataset_create_resources() datasets.append(dataset) @@ -1506,7 +1635,7 @@ def get_location_iso3s( List[str]: list of location iso3s """ countries = self.data.get("groups") - countryisos = list() + countryisos = [] if not countries: return countryisos for country in countries: @@ -1531,7 +1660,7 @@ def get_location_names( List[str]: list of location names """ countries = self.data.get("groups") - countrynames = list() + countrynames = [] if not countries: return countrynames for country in countries: @@ -1658,7 +1787,7 @@ def add_other_location( if hdx_code in [x["name"] for x in groups]: return False else: - groups = list() + groups = [] groups.append({"name": hdx_code}) self.data["groups"] = groups return True @@ -1768,7 +1897,7 @@ def get_showcases(self) -> List["Showcase"]: fieldname="package_id", action=sc_module.Showcase.actions()["list_showcases"], ) - showcases = list() + showcases = [] if assoc_result: for showcase_dict in showcases_dicts: showcase = sc_module.Showcase( @@ -2452,7 +2581,7 @@ def generate_qc_resource_from_rows( Returns: Optional[Resource]: The created resource or None """ - qc_rows = list() + qc_rows = [] for row in rows: if row[columnname] in qc_identifiers: if headers: @@ -2539,7 +2668,7 @@ def generate_resource_from_iterator( """ if [datecol, yearcol, date_function].count(None) < 2: raise HDXError("Supply one of datecol, yearcol or date_function!") - retdict = dict() + retdict = {} if headers is None: return False, retdict rows = [Download.hxl_row(headers, hxltags, dict_form=True)] @@ -2585,7 +2714,7 @@ def generate_resource_from_iterator( if yearcol is not None: def yearcol_function(row): - result = dict() + result = {} year = row[yearcol] if year: result["startdate"], result["enddate"] = parse_date_range( @@ -2599,7 +2728,7 @@ def yearcol_function(row): elif datecol is not None: def datecol_function(row): - result = dict() + result = {} date = row[datecol] if date: date = parse_date(date) diff --git a/src/hdx/data/dataset_title_helper.py b/src/hdx/data/dataset_title_helper.py index 4ee7ae64..1aee0e73 100755 --- a/src/hdx/data/dataset_title_helper.py +++ b/src/hdx/data/dataset_title_helper.py @@ -62,7 +62,7 @@ def fuzzy_match_dates_in_title( start = match.start() end = match.end() stringlr = title[max(start - 13, 0) : end] - fuzzylr = dict() + fuzzylr = {} startdatelr = None enddatelr = None deltalr = timedelta(days=1000) @@ -77,7 +77,7 @@ def fuzzy_match_dates_in_title( deltalr = enddatelr - startdatelr except ParserError: pass - fuzzyrl = dict() + fuzzyrl = {} stringrl = title[start : min(end + 13, len(title))] startdaterl = None enddaterl = None @@ -112,7 +112,7 @@ def fuzzy_match_dates_in_title( logger.info(f"Removing date from title: {title} -> {newtitle}") title = newtitle try: - fuzzy = dict() + fuzzy = {} startdate, enddate = parse_date_range( title, fuzzy=fuzzy, @@ -187,8 +187,8 @@ def get_dates_from_title( Tuple[str,List[Tuple[datetime,datetime]]]: Cleaned title, list of start and end dates """ - ranges = list() - ignore_wrong_years = list() + ranges = [] + ignore_wrong_years = [] for match in cls.YEAR_RANGE_PATTERN.finditer(title): ( first_year, diff --git a/src/hdx/data/date_helper.py b/src/hdx/data/date_helper.py index ddb3200d..09e3fc9a 100755 --- a/src/hdx/data/date_helper.py +++ b/src/hdx/data/date_helper.py @@ -81,7 +81,7 @@ def get_time_period_info( Returns: Dict: Dictionary of date information """ - result = dict() + result = {} if hdx_time_period: if hdx_time_period[0] == "[" and hdx_time_period[-1] == "]": hdx_time_period = hdx_time_period[1:-1] @@ -167,7 +167,7 @@ def get_hdx_time_period_from_years( Returns: Tuple[str,List[int]]: (HDX time period, the start and end year if supplied or sorted list of years) """ - retval = list() + retval = [] if isinstance(startyear, str): startyear = int(startyear) if not isinstance(startyear, Iterable): diff --git a/src/hdx/data/filestore_helper.py b/src/hdx/data/filestore_helper.py index 10eb430f..0e014bea 100755 --- a/src/hdx/data/filestore_helper.py +++ b/src/hdx/data/filestore_helper.py @@ -3,8 +3,6 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, Dict -from hdx.utilities.dictandlist import merge_two_dictionaries - if TYPE_CHECKING: from hdx.data.resource import Resource @@ -35,7 +33,7 @@ def resource_check_required_fields( ): del resource.data["url"] ignore_fields = kwargs.get("ignore_fields", list()) - resource_ignore_fields = list() + resource_ignore_fields = [] for ignore_field in ignore_fields: if ignore_field.startswith("resource:"): resource_ignore_field = ignore_field[9:].strip() @@ -47,7 +45,7 @@ def resource_check_required_fields( @classmethod def check_filestore_resource( cls, - resource: "Resource", + resource_data_to_update: "Resource", filestore_resources: Dict[int, str], resource_index: int, **kwargs: Any, @@ -55,52 +53,44 @@ def check_filestore_resource( """Helper method to add new resource from dataset including filestore. Args: - resource (Resource): Resource to check + resource_data_to_update (Resource): Updated resource from dataset filestore_resources (Dict[int, str]): List of (index of resource, file to upload) resource_index (int): Index of resource Returns: None """ - cls.resource_check_required_fields(resource, **kwargs) - file_to_upload = resource.get_file_to_upload() + cls.resource_check_required_fields(resource_data_to_update, **kwargs) + file_to_upload = resource_data_to_update.get_file_to_upload() if file_to_upload: filestore_resources[resource_index] = file_to_upload - resource["url"] = cls.temporary_url + resource_data_to_update["url"] = cls.temporary_url @classmethod - def dataset_merge_filestore_resource( + def dataset_update_filestore_resource( cls, - resource: "Resource", - updated_resource: "Resource", + resource_data_to_update: "Resource", filestore_resources: Dict[int, str], resource_index: int, - **kwargs: Any, ) -> None: """Helper method to merge updated resource from dataset into HDX resource read from HDX including filestore. Args: - resource (Resource): Resource read from HDX - updated_resource (Resource): Updated resource from dataset + resource_data_to_update (Resource): Updated resource from dataset filestore_resources (Dict[int, str]): List of (index of resources, file to upload) resource_index (int): Index of resource Returns: None """ - file_to_upload = updated_resource.get_file_to_upload() + file_to_upload = resource_data_to_update.get_file_to_upload() if file_to_upload: - resource.set_file_to_upload(file_to_upload) filestore_resources[resource_index] = file_to_upload - data_updated = updated_resource.is_marked_data_updated() - merge_two_dictionaries(resource, updated_resource) - cls.resource_check_required_fields( - resource, check_upload=True, **kwargs - ) - if resource.get_file_to_upload(): - resource["url"] = cls.temporary_url + resource_data_to_update["url"] = cls.temporary_url + + data_updated = resource_data_to_update.is_marked_data_updated() if data_updated: - resource["last_modified"] = datetime.now(timezone.utc).isoformat( - timespec="microseconds" - ) - resource.data_updated = False + resource_data_to_update["last_modified"] = datetime.now( + timezone.utc + ).isoformat(timespec="microseconds") + resource_data_to_update.data_updated = False diff --git a/src/hdx/data/hdxobject.py b/src/hdx/data/hdxobject.py index 62ca1b78..39c3e869 100755 --- a/src/hdx/data/hdxobject.py +++ b/src/hdx/data/hdxobject.py @@ -53,11 +53,11 @@ def __init__( self.configuration: Configuration = configuration super().__init__(initial_data) - def get_old_data_dict(self) -> None: + def get_old_data_dict(self) -> Dict: """Get previous internal dictionary Returns: - dict: Previous internal dictionary + Dict: Previous internal dictionary """ return self.old_data @@ -206,7 +206,7 @@ def _check_load_existing_object( @abstractmethod def check_required_fields( - self, ignore_fields: ListTuple[str] = list() + self, ignore_fields: ListTuple[str] = [] ) -> None: """Abstract method to check that metadata for HDX object is complete. The parameter ignore_fields should be set if required to any fields that should be ignored for the particular operation. @@ -278,7 +278,7 @@ def _hdx_update( self, object_type: str, id_field_name: str, - files_to_upload: Dict = dict(), + files_to_upload: Dict = {}, force_active: bool = False, **kwargs: Any, ) -> None: @@ -306,7 +306,7 @@ def _merge_hdx_update( self, object_type: str, id_field_name: str, - files_to_upload: Dict = dict(), + files_to_upload: Dict = {}, force_active: bool = False, **kwargs: Any, ) -> None: @@ -345,7 +345,7 @@ def _update_in_hdx( self, object_type: str, id_field_name: str, - files_to_upload: Dict = dict(), + files_to_upload: Dict = {}, force_active: bool = True, **kwargs: Any, ) -> None: @@ -381,7 +381,7 @@ def _write_to_hdx( action: str, data: Dict, id_field_name: str = None, - files_to_upload: Dict = dict(), + files_to_upload: Dict = {}, ) -> Union[Dict, List]: """Creates or updates an HDX object in HDX and return HDX object metadata dict @@ -418,7 +418,7 @@ def _save_to_hdx( self, action: str, id_field_name: str, - files_to_upload: Dict = dict(), + files_to_upload: Dict = {}, force_active: bool = False, ) -> None: """Creates or updates an HDX object in HDX, saving current data and replacing with returned HDX object data @@ -454,7 +454,7 @@ def _create_in_hdx( object_type: str, id_field_name: str, name_field_name: str, - files_to_upload: Dict = dict(), + files_to_upload: Dict = {}, force_active: bool = True, **kwargs: Any, ) -> None: @@ -612,7 +612,7 @@ def _convert_hdxobjects( Returns: List[Dict]: List of HDX objects converted to simple dictionaries """ - newhdxobjects = list() + newhdxobjects = [] for hdxobject in hdxobjects: newhdxobjects.append(hdxobject.data) return newhdxobjects @@ -633,7 +633,7 @@ def _copy_hdxobjects( Returns: List[HDXObject]: Deep copy of list of HDX objects """ - newhdxobjects = list() + newhdxobjects = [] for hdxobject in hdxobjects: newhdxobjectdata = copy.deepcopy(hdxobject.data) newhdxobject = hdxobjectclass( @@ -693,7 +693,7 @@ def _get_tags(self) -> List[str]: """ tags = self.data.get("tags", None) if not tags: - return list() + return [] return [x["name"] for x in tags] def _add_tag(self, tag: str, vocabulary_id: Optional[str] = None) -> bool: @@ -712,7 +712,7 @@ def _add_tag(self, tag: str, vocabulary_id: Optional[str] = None) -> bool: if tag in [x["name"] for x in tags]: return False else: - tags = list() + tags = [] tagdict = {"name": tag} if vocabulary_id is not None: tagdict["vocabulary_id"] = vocabulary_id @@ -732,7 +732,7 @@ def _add_tags( Returns: List[str]: Tags that were successfully added """ - added_tags = list() + added_tags = [] for tag in tags: if self._add_tag(tag, vocabulary_id=vocabulary_id): added_tags.append(tag) @@ -751,7 +751,7 @@ def _get_stringlist_from_commastring(self, field: str) -> List[str]: if strings: return strings.split(",") else: - return list() + return [] def _add_string_to_commastring(self, field: str, string: str) -> bool: """Add a string to a comma separated list of strings diff --git a/src/hdx/data/organization.py b/src/hdx/data/organization.py index 1b3db0c7..44ebdd9e 100755 --- a/src/hdx/data/organization.py +++ b/src/hdx/data/organization.py @@ -29,7 +29,7 @@ def __init__( configuration: Optional[Configuration] = None, ) -> None: if not initial_data: - initial_data = dict() + initial_data = {} super().__init__(initial_data, configuration=configuration) @staticmethod @@ -137,7 +137,7 @@ def get_users(self, capacity: Optional[str] = None) -> List["User"]: Returns: List[User]: Organization's users. """ - users = list() + users = [] usersdicts = self.data.get("users") if usersdicts is not None: for userdata in usersdicts: @@ -179,7 +179,7 @@ def add_update_user( if isinstance(user, user_module.User): users = self.data.get("users") if users is None: - users = list() + users = [] self.data["users"] = users if capacity is not None: user["capacity"] = capacity diff --git a/src/hdx/data/resource.py b/src/hdx/data/resource.py index 85902575..fbdc5f38 100755 --- a/src/hdx/data/resource.py +++ b/src/hdx/data/resource.py @@ -37,7 +37,7 @@ def __init__( configuration: Optional[Configuration] = None, ) -> None: if not initial_data: - initial_data = dict() + initial_data = {} super().__init__(initial_data, configuration=configuration) self.file_to_upload = None self.data_updated = False @@ -176,7 +176,7 @@ def read_formats_mappings( if url is None: url = configuration["formats_mapping_url"] downloader.download(url) - cls._formats_dict = dict() + cls._formats_dict = {} for format_data in downloader.get_json(): hdx_format = format_data[0].lower() if hdx_format == "_comment": @@ -372,7 +372,7 @@ def _get_files(self) -> Dict: Dict: files parameter for CKANAPI """ if self.file_to_upload is None: - return dict() + return {} return {"upload": self.file_to_upload} def _resource_merge_hdx_update( @@ -559,7 +559,7 @@ def get_all_resource_ids_in_datastore( Resource.actions()["datastore_search"], limit=10000, ) - resource_ids = list() + resource_ids = [] if not success: logger.debug(result) else: @@ -675,7 +675,7 @@ def reorder_resource_views( """ if not isinstance(resource_views, list): raise HDXError("ResourceViews should be a list!") - ids = list() + ids = [] for resource_view in resource_views: if isinstance(resource_view, str): resource_view_id = resource_view diff --git a/src/hdx/data/resource_matcher.py b/src/hdx/data/resource_matcher.py index ca9e75e3..6e4e66c6 100755 --- a/src/hdx/data/resource_matcher.py +++ b/src/hdx/data/resource_matcher.py @@ -78,8 +78,8 @@ def match_resource_lists( groups2 = [x.get("grouping") for x in resources2] names2 = [x["name"] for x in resources2] formats2 = [x["format"].lower() for x in resources2] - index1_matches = list() - index2_matches = list() + index1_matches = [] + index2_matches = [] for i, id1 in enumerate(ids1): if id1 is None: continue diff --git a/src/hdx/data/resource_view.py b/src/hdx/data/resource_view.py index 421b2968..cb8cacc5 100755 --- a/src/hdx/data/resource_view.py +++ b/src/hdx/data/resource_view.py @@ -25,7 +25,7 @@ def __init__( configuration: Optional[Configuration] = None, ) -> None: if not initial_data: - initial_data = dict() + initial_data = {} super().__init__(initial_data, configuration=configuration) @staticmethod @@ -105,7 +105,7 @@ def get_all_for_resource( success, result = resourceview._read_from_hdx( "resource view", identifier, "id", ResourceView.actions()["list"] ) - resourceviews = list() + resourceviews = [] if success: for resourceviewdict in result: resourceview = ResourceView( diff --git a/src/hdx/data/showcase.py b/src/hdx/data/showcase.py index 7ef50a9d..a5ec69ca 100755 --- a/src/hdx/data/showcase.py +++ b/src/hdx/data/showcase.py @@ -32,7 +32,7 @@ def __init__( configuration: Optional[Configuration] = None, ) -> None: if not initial_data: - initial_data = dict() + initial_data = {} super().__init__(initial_data, configuration=configuration) @staticmethod @@ -235,7 +235,7 @@ def get_datasets(self) -> List["Dataset"]: # noqa: F821 fieldname="showcase_id", action=self.actions()["list_datasets"], ) - datasets = list() + datasets = [] if assoc_result: for dataset_dict in datasets_dicts: dataset = hdx.data.dataset.Dataset( @@ -377,7 +377,7 @@ def search_in_hdx( page_size=page_size, **kwargs, ) - showcases = list() + showcases = [] for dataset in datasets: showcase = Showcase(configuration=configuration) showcase.data = dataset.data diff --git a/src/hdx/data/user.py b/src/hdx/data/user.py index d18fa7dc..63ca5d66 100755 --- a/src/hdx/data/user.py +++ b/src/hdx/data/user.py @@ -25,7 +25,7 @@ def __init__( configuration: Optional[Configuration] = None, ) -> None: if not initial_data: - initial_data = dict() + initial_data = {} super().__init__(initial_data, configuration=configuration) @staticmethod @@ -184,7 +184,7 @@ def get_all_users( """ user = User(configuration=configuration) result = user._write_to_hdx("list", kwargs) - users = list() + users = [] if result: for userdict in result: user = User(userdict, configuration=configuration) @@ -225,13 +225,13 @@ def email_users( """ if not users: raise ValueError("No users supplied") - recipients = list() + recipients = [] for user in users: recipients.append(user.data["email"]) - ccemails = list() + ccemails = [] for user in cc: ccemails.append(user.data["email"]) - bccemails = list() + bccemails = [] for user in bcc: bccemails.append(user.data["email"]) if configuration is None: @@ -265,7 +265,7 @@ def get_organizations( self.actions()["listorgs"], permission=permission, ) - organizations = list() + organizations = [] if success: for organizationdict in result: org = hdx.data.organization.Organization.read_from_hdx( diff --git a/src/hdx/data/vocabulary.py b/src/hdx/data/vocabulary.py index a293793f..34c77fbe 100755 --- a/src/hdx/data/vocabulary.py +++ b/src/hdx/data/vocabulary.py @@ -37,7 +37,7 @@ def __init__( configuration: Optional[Configuration] = None, ) -> None: if not initial_data: - initial_data = dict() + initial_data = {} if name: initial_data["name"] = name super().__init__(initial_data, configuration=configuration) @@ -121,7 +121,7 @@ def get_all_vocabularies( vocabulary[ "id" ] = "all vocabulary names" # only for error message if produced - vocabularies = list() + vocabularies = [] for vocabularydict in vocabulary._write_to_hdx("list", {}): vocabularies.append( Vocabulary(vocabularydict, configuration=configuration) @@ -170,7 +170,7 @@ def delete_from_hdx(self, empty: bool = True) -> None: None """ if empty and len(self.data["tags"]) != 0: - self.data["tags"] = list() + self.data["tags"] = [] self._update_in_hdx( "vocabulary", "id", force_active=False, ignore_field="tags" ) @@ -314,7 +314,7 @@ def update_approved_vocabulary( configuration = Configuration.read() vocabulary = cls.get_approved_vocabulary(configuration=configuration) if replace: - vocabulary["tags"] = list() + vocabulary["tags"] = [] vocabulary.add_tags( cls._read_approved_tags(url=url, configuration=configuration) ) @@ -477,8 +477,8 @@ def get_mapped_tag( configuration = Configuration.read() tag = tag.lower() tags_dict = cls.read_tags_mappings(configuration=configuration) - tags = list() - deleted_tags = list() + tags = [] + deleted_tags = [] whattodo = tags_dict.get(tag) if whattodo is None: if cls.is_approved(tag): @@ -533,8 +533,8 @@ def get_mapped_tags( Returns: Tuple[List[str], List[str]]: Tuple containing list of mapped tags and list of deleted tags """ - new_tags = list() - deleted_tags = list() + new_tags = [] + deleted_tags = [] for tag in tags: mapped_tags, del_tags = cls.get_mapped_tag( tag, log_deleted=log_deleted, configuration=configuration @@ -607,7 +607,7 @@ def clean_tags( Tuple[List[str], List[str]]: Tuple containing list of mapped tags and list of deleted tags and tags not added """ tags = hdxobject._get_tags() - hdxobject["tags"] = list() + hdxobject["tags"] = [] return cls.add_mapped_tags(hdxobject, tags, log_deleted=log_deleted) @classmethod diff --git a/src/hdx/facades/infer_arguments.py b/src/hdx/facades/infer_arguments.py index 85c81f5b..3dd69958 100755 --- a/src/hdx/facades/infer_arguments.py +++ b/src/hdx/facades/infer_arguments.py @@ -50,7 +50,7 @@ def facade(projectmainfn: Callable[[Any], None], **kwargs: Any): main_doc = "".join(main_doc) main_sig = defopt.signature(projectmainfn) - param_names = list() + param_names = [] for param in main_sig.parameters.values(): param_names.append(str(param)) diff --git a/tests/fixtures/CKAN/hdx_dataset_static.yaml b/tests/fixtures/CKAN/hdx_dataset_static.yaml new file mode 100755 index 00000000..36a5b554 --- /dev/null +++ b/tests/fixtures/CKAN/hdx_dataset_static.yaml @@ -0,0 +1,9 @@ +license_id: "cc-by-sa" +notes: "Some notes." +caveats: "None" +methodology: "Registry" +dataset_source: "Someone" +package_creator: "mcarans" +private: False # has to be True or False +url: ~ +state: "active" # always "active". diff --git a/tests/fixtures/update_dataset_resources/expected_resources_to_update.json b/tests/fixtures/update_dataset_resources/expected_resources_to_update.json new file mode 100644 index 00000000..ba74a01b --- /dev/null +++ b/tests/fixtures/update_dataset_resources/expected_resources_to_update.json @@ -0,0 +1,87 @@ +[ + {}, + {}, + {}, + { + "dataset_preview_enabled": "False", + "description": "SDG 4 Global and Thematic data with HXL tags.\n\nIndicators: Adjusted attendance rate, Adjusted net attendance rate, Adjusted net enrolment rate, Administration of a nationally representative learning assessment in Grade 2 or 3 in mathematics, Administration of a nationally representative learning assessment in Grade 2 or 3 in reading, Administration of a nationally-representative learning assessment at the end of lower secondary education in mathematics, Administration of a nationally-representative learning assessment at the end of lower secondary education in reading, Administration of a nationally-representative learning assessment at the end of primary in mathematics, Administration of a nationally-representative learning assessment at the end of primary in reading, Adult literacy rate, Average teacher salary in lower secondary education relative to other professions requiring a comparable level of qualification, Average teacher salary in pre-primary education relative to other professions requiring a comparable level of qualification, Average teacher salary in primary education relative to other professions requiring a comparable level of qualification, Average teacher salary in upper secondary education relative to other professions requiring a comparable level of qualification, Completion rate, Educational attainment, Educational attainment rate, Elderly literacy rate, Expenditure on education as a percentage of total government expenditure, Government expenditure on education as a percentage of GDP, Gross attendance ratio for tertiary education, Gross enrolment ratio, Gross enrolment ratio for tertiary education, Gross intake ratio to the last grade of lower secondary general education, Gross intake ratio to the last grade of primary education, Initial government funding per lower secondary student, Initial government funding per lower secondary student as a percentage of GDP per capita, Initial government funding per pre-primary student, Initial government funding per pre-primary student as a percentage of GDP per capita, Initial government funding per primary student, Initial government funding per primary student as a percentage of GDP per capita, Initial government funding per secondary student, Initial government funding per secondary student as a percentage of GDP per capita, Initial government funding per tertiary student, Initial government funding per tertiary student as a percentage of GDP per capita, Initial government funding per upper secondary student, Initial government funding per upper secondary student as a percentage of GDP per capita, Initial household funding per primary student, Initial household funding per primary student as a percentage of GDP per capita, Initial household funding per secondary student, Initial household funding per secondary student as a percentage of GDP per capita, Initial household funding per tertiary student, Initial household funding per tertiary student as a percentage of GDP per capita, Literacy rate, Number of attacks on students, Number of years of compulsory pre-primary education guaranteed in legal frameworks, Number of years of compulsory primary and secondary education guaranteed in legal frameworks, Number of years of free pre-primary education guaranteed in legal frameworks, Number of years of free primary and secondary education guaranteed in legal frameworks, Out-of-school rate, Out-of-school rate for adolescents and youth of lower and upper secondary school age, Out-of-school rate for adolescents of lower secondary school age, Out-of-school rate for children, Out-of-school rate for children and adolescents of primary and lower secondary school age, Out-of-school rate for children of primary school age, Out-of-school rate for children one year younger than official age, Out-of-school rate for children one year younger than official primary entry age, Out-of-school rate for youth of upper secondary school age, Participants in literacy programmes as a % of the illiterate population, Participation rate of youth and adults in formal and non-formal education and training in the previous 12 months, Percentage of children under 5 years experiencing positive and stimulating home learning environments, Percentage of lower secondary schools providing life skills-based HIV and sexuality education, Percentage of primary schools providing life skills-based HIV and sexuality education, Percentage of pupils enrolled in lower secondary general education who are at least 2 years over-age for their current grade, Percentage of pupils enrolled in primary education who are at least 2 years over-age for their current grade, Percentage of qualified teachers in lower secondary education, Percentage of qualified teachers in pre-primary education, Percentage of qualified teachers in primary education, Percentage of qualified teachers in secondary education, Percentage of qualified teachers in upper secondary education, Percentage of students at the end of lower secondary education who have their first or home language as language of instruction, Percentage of students at the end of primary education who have their first or home language as language of instruction, Percentage of students experiencing bullying in the last 12 months in lower secondary education, Percentage of students experiencing bullying in the last 12 months in primary education, Percentage of students in early grades who have their first or home language as language of instruction, Percentage of students in lower secondary education who have their first or home language as language of instruction, Percentage of students in lower secondary showing adequate understanding of issues relating to global citizenship and sustainability, Percentage of students in lower secondary showing adequate understanding of issues relating to global citizenship and sustainability - Cognitive Dimension, Percentage of students in lower secondary showing adequate understanding of issues relating to global citizenship and sustainability \u00e2\u20ac\u201c Non-cognitive Dimension \u00e2\u20ac\u201c Freedom, Percentage of students in lower secondary showing adequate understanding of issues relating to global citizenship and sustainability \u00e2\u20ac\u201c Non-cognitive Dimension \u00e2\u20ac\u201c Gender equality, Percentage of students in lower secondary showing adequate understanding of issues relating to global citizenship and sustainability \u00e2\u20ac\u201c Non-cognitive Dimension \u00e2\u20ac\u201c Global-local thinking, Percentage of students in lower secondary showing adequate understanding of issues relating to global citizenship and sustainability \u00e2\u20ac\u201c Non-cognitive Dimension \u00e2\u20ac\u201c Multiculturalism, Percentage of students in lower secondary showing adequate understanding of issues relating to global citizenship and sustainability \u00e2\u20ac\u201c Non-cognitive Dimension \u00e2\u20ac\u201c Peace, Percentage of students in lower secondary showing adequate understanding of issues relating to global citizenship and sustainability \u00e2\u20ac\u201c Non-cognitive Dimension \u00e2\u20ac\u201c Social Justice, Percentage of students in lower secondary showing adequate understanding of issues relating to global citizenship and sustainability \u00e2\u20ac\u201c Non-cognitive Dimension \u00e2\u20ac\u201c Sustainable development, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience \u00e2\u20ac\u201c Cognitive dimension, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience \u00e2\u20ac\u201c Cognitive dimension \u00e2\u20ac\u201c adjusted gender parity index, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience \u00e2\u20ac\u201c Cognitive dimension \u00e2\u20ac\u201c adjusted location parity index, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience \u00e2\u20ac\u201c Cognitive dimension \u00e2\u20ac\u201c adjusted wealth parity index, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience \u00e2\u20ac\u201c Non-cognitive dimension - Confidence, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience \u00e2\u20ac\u201c Non-cognitive dimension - Enjoyment, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience \u00e2\u20ac\u201c Non-cognitive dimension \u00e2\u20ac\u201c Confidence, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience \u00e2\u20ac\u201c Non-cognitive dimension \u00e2\u20ac\u201c Confidence \u00e2\u20ac\u201c adjusted gender parity index, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience \u00e2\u20ac\u201c Non-cognitive dimension \u00e2\u20ac\u201c Confidence \u00e2\u20ac\u201c adjusted location parity index, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience \u00e2\u20ac\u201c Non-cognitive dimension \u00e2\u20ac\u201c Confidence \u00e2\u20ac\u201c adjusted wealth parity index, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience \u00e2\u20ac\u201c Non-cognitive dimension \u00e2\u20ac\u201c Enjoyment, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience \u00e2\u20ac\u201c Non-cognitive dimension \u00e2\u20ac\u201c Enjoyment \u00e2\u20ac\u201c adjusted gender parity index, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience \u00e2\u20ac\u201c Non-cognitive dimension \u00e2\u20ac\u201c Enjoyment \u00e2\u20ac\u201c adjusted location parity index, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience \u00e2\u20ac\u201c Non-cognitive dimension \u00e2\u20ac\u201c Enjoyment \u00e2\u20ac\u201c adjusted wealth parity index, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience \u00e2\u20ac\u201c adjusted gender parity index, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience \u00e2\u20ac\u201c adjusted location parity index, Percentage of students in lower secondary showing proficiency in knowledge of environmental science and geoscience \u00e2\u20ac\u201c adjusted wealth parity index, Percentage of teachers in lower secondary education who received in-service training in the last 12 months by type of trained, Percentage of teachers in lower secondary education who received in-service training in the last 12 months by type of training, Percentage of teachers in primary education who received in-service training in the last 12 months by type of trained, Percentage of teachers in primary education who received in-service training in the last 12 months by type of training, Percentage of total aid to education allocated to least developed countries, Percentage of upper secondary schools providing life skills-based HIV and sexuality education, Percentage of youth/adults who have achieved at least a minimum level of proficiency in digital literacy skills, Proportion of 15- to 24-year-olds enrolled in vocational education, Proportion of 15-24 year-olds enrolled in vocational education, Proportion of children aged 24-59 months who are developmentally on track in health, Proportion of lower secondary schools with access to Internet for pedagogical purposes, Proportion of lower secondary schools with access to adapted infrastructure and materials for students with disabilities, Proportion of lower secondary schools with access to basic drinking water, Proportion of lower secondary schools with access to computers for pedagogical purposes, Proportion of lower secondary schools with access to electricity, Proportion of lower secondary schools with basic handwashing facilities, Proportion of lower secondary schools with single-sex basic sanitation facilities, Proportion of population achieving at least a fixed level of proficiency in functional literacy skills, Proportion of population achieving at least a fixed level of proficiency in functional numeracy skills, Proportion of primary schools with access to Internet for pedagogical purposes, Proportion of primary schools with access to adapted infrastructure and materials for students with disabilities, Proportion of primary schools with access to basic drinking water, Proportion of primary schools with access to computers for pedagogical purposes, Proportion of primary schools with access to electricity, Proportion of primary schools with basic handwashing facilities, Proportion of primary schools with single-sex basic sanitation facilities, Proportion of pupils enrolled in lower secondary general education who are at least 2 years over-age for their current grade, Proportion of pupils enrolled in primary education who are at least 2 years over-age for their current grade, Proportion of qualified teachers in lower secondary education, Proportion of qualified teachers in pre-primary education, Proportion of qualified teachers in primary education, Proportion of qualified teachers in secondary education, Proportion of qualified teachers in upper secondary education, Proportion of secondary schools with access to Internet for pedagogical purposes, Proportion of secondary schools with access to computers for pedagogical purposes, Proportion of students at the end of lower secondary achieving at least a minimum proficiency level in mathematics, Proportion of students at the end of lower secondary achieving at least a minimum proficiency level in reading, Proportion of students at the end of lower secondary education achieving at least a minimum proficiency level in mathematics, Proportion of students at the end of lower secondary education achieving at least a minimum proficiency level in reading, Proportion of students at the end of primary achieving at least a minimum proficiency level in mathematics, Proportion of students at the end of primary achieving at least a minimum proficiency level in reading, Proportion of students at the end of primary education achieving at least a minimum proficiency level in mathematics, Proportion of students at the end of primary education achieving at least a minimum proficiency level in reading, Proportion of students in Grade 2 or 3 achieving at least a minimum proficiency level in mathematics, Proportion of students in Grade 2 or 3 achieving at least a minimum proficiency level in reading, Proportion of students\u00c2 at the end of primary education achieving at least a minimum proficiency level in reading, Proportion of teachers with the minimum required qualifications in lower secondary education, Proportion of teachers with the minimum required qualifications in pre-primary education, Proportion of teachers with the minimum required qualifications in primary education, Proportion of teachers with the minimum required qualifications in secondary education, Proportion of teachers with the minimum required qualifications in upper secondary education, Proportion of upper secondary schools with access to Internet for pedagogical purposes, Proportion of upper secondary schools with access to adapted infrastructure and materials for students with disabilities, Proportion of upper secondary schools with access to basic drinking water, Proportion of upper secondary schools with access to computers for pedagogical purposes, Proportion of upper secondary schools with access to electricity, Proportion of upper secondary schools with basic handwashing facilities, Proportion of upper secondary schools with single-sex basic sanitation facilities, Proportion of youth and adults who have Transferred files between a computer and other devices, Proportion of youth and adults who have connected and installed new devices, Proportion of youth and adults who have copied or moved a file or folder, Proportion of youth and adults who have created electronic presentations with presentation software, Proportion of youth and adults who have found, Proportion of youth and adults who have sent e-mails with attached files, Proportion of youth and adults who have transferred files between a computer and other devices, Proportion of youth and adults who have used basic arithmetic formulae in a spreadsheet, Proportion of youth and adults who have used copy and paste tools to duplicate or move information within a document, Proportion of youth and adults who have wrote a computer program using a specialised programming language, Pupil-qualified teacher ratio in lower secondary, Pupil-qualified teacher ratio in pre-primary education, Pupil-qualified teacher ratio in primary education, Pupil-qualified teacher ratio in secondary, Pupil-qualified teacher ratio in upper secondary, Pupil-trained teacher ratio in lower secondary education, Pupil-trained teacher ratio in pre-primary education, Pupil-trained teacher ratio in primary education, Pupil-trained teacher ratio in secondary education, Pupil-trained teacher ratio in upper secondary education, Teacher attrition rate from general secondary education, Teacher attrition rate from lower secondary education, Teacher attrition rate from pre-primary education, Teacher attrition rate from primary education, Teacher attrition rate from secondary education, Teacher attrition rate from upper secondary education, Teacher attrition rate from vocational secondary education, Volume of official development assistance flows for scholarships by sector and type of study, Youth literacy rate", + "format": "csv", + "name": "SDG 4 Global and Thematic data", + "resource_type": "file.upload", + "url": "updated_by_file_upload_step", + "url_type": "upload" + }, + { + "dataset_preview_enabled": "False", + "description": "SDG 4 Global and Thematic indicator list with HXL tags", + "format": "csv", + "name": "SDG 4 Global and Thematic indicator list", + "resource_type": "file.upload", + "url": "updated_by_file_upload_step", + "url_type": "upload" + }, + { + "dataset_preview_enabled": "False", + "description": "SDG 4 Global and Thematic metadata with HXL tags", + "format": "csv", + "name": "SDG 4 Global and Thematic metadata", + "resource_type": "file.upload", + "url": "updated_by_file_upload_step", + "url_type": "upload" + }, + { + "dataset_preview_enabled": "False", + "description": "Demographic and Socio-economic data with HXL tags.\n\nIndicators: DEC alternative conversion factor, Fertility rate, GDP, GDP at market prices, GDP deflator, GDP growth, GDP per capita, GNI, GNI per capita, General government total expenditure, Life expectancy at birth, Mortality rate, Official exchange rate, PPP conversion factor, Population aged 14 years or younger, Population aged 15-24 years, Population aged 25-64 years, Population aged 65 years or older, Population growth, Poverty headcount ratio at $3.20 a day, Prevalence of HIV, Price level ratio of PPP conversion factor, Rural population, Total debt service, Total population", + "format": "csv", + "name": "Demographic and Socio-economic data", + "resource_type": "file.upload", + "url": "updated_by_file_upload_step", + "url_type": "upload" + }, + { + "dataset_preview_enabled": "False", + "description": "Demographic and Socio-economic indicator list with HXL tags", + "format": "csv", + "name": "Demographic and Socio-economic indicator list", + "resource_type": "file.upload", + "url": "updated_by_file_upload_step", + "url_type": "upload" + }, + {}, + { + "dataset_preview_enabled": "False", + "description": "Other Policy Relevant Indicators data with HXL tags.\n\nIndicators: Adult illiterate population, Africa, All staff compensation as a percentage of total expenditure in lower secondary public institutions, All staff compensation as a percentage of total expenditure in post-secondary non-tertiary public institutions, All staff compensation as a percentage of total expenditure in pre-primary public institutions, All staff compensation as a percentage of total expenditure in primary public institutions, All staff compensation as a percentage of total expenditure in public institutions, All staff compensation as a percentage of total expenditure in secondary public institutions, All staff compensation as a percentage of total expenditure in tertiary public institutions, All staff compensation as a percentage of total expenditure in upper secondary public institutions, Asia, Capital expenditure as a percentage of total expenditure in lower secondary public institutions, Capital expenditure as a percentage of total expenditure in post-secondary non-tertiary public institutions, Capital expenditure as a percentage of total expenditure in pre-primary public institutions, Capital expenditure as a percentage of total expenditure in primary public institutions, Capital expenditure as a percentage of total expenditure in public institutions, Capital expenditure as a percentage of total expenditure in secondary public institutions, Capital expenditure as a percentage of total expenditure in tertiary public institutions, Capital expenditure as a percentage of total expenditure in upper-secondary public institutions, Caribbean and Central America, Current expenditure as a percentage of total expenditure in lower secondary public institutions, Current expenditure as a percentage of total expenditure in post-secondary non-tertiary public institutions, Current expenditure as a percentage of total expenditure in pre-primary public institutions, Current expenditure as a percentage of total expenditure in primary public institutions, Current expenditure as a percentage of total expenditure in public institutions, Current expenditure as a percentage of total expenditure in secondary public institutions, Current expenditure as a percentage of total expenditure in tertiary public institutions, Current expenditure as a percentage of total expenditure in upper-secondary public institutions, Current expenditure other than staff compensation as a percentage of total expenditure in lower secondary public institutions, Current expenditure other than staff compensation as a percentage of total expenditure in post-secondary non-tertiary public institutions, Current expenditure other than staff compensation as a percentage of total expenditure in pre-primary public institutions, Current expenditure other than staff compensation as a percentage of total expenditure in primary public institutions, Current expenditure other than staff compensation as a percentage of total expenditure in public institutions, Current expenditure other than staff compensation as a percentage of total expenditure in secondary public institutions, Current expenditure other than staff compensation as a percentage of total expenditure in tertiary public institutions, Current expenditure other than staff compensation as a percentage of total expenditure in upper secondary public institutions, Duration of compulsory education, End month of the academic school year, End of the academic school year, Enrolment in early childhood education, Enrolment in early childhood educational development programmes, Enrolment in lower secondary education, Enrolment in post-secondary non-tertiary education, Enrolment in pre-primary education, Enrolment in primary education, Enrolment in secondary education, Enrolment in tertiary education, Enrolment in upper secondary education, Europe, Expenditure on school books and teaching material as % of total expenditure in primary public institutions, Expenditure on school books and teaching material as % of total expenditure in secondary public institutions, Government expenditure on education, Government expenditure on education not specified by level, Government expenditure on lower secondary education, Government expenditure on lower secondary education as a percentage of GDP, Government expenditure on post-secondary non-tertiary education, Government expenditure on post-secondary non-tertiary education as a percentage of GDP, Government expenditure on pre-primary education, Government expenditure on pre-primary education as a percentage of GDP, Government expenditure on primary education, Government expenditure on primary education as a percentage of GDP, Government expenditure on secondary and post-secondary non-tertiary vocational education as a percentage of GDP, Government expenditure on secondary and post-secondary non-tertiary vocational education only, Government expenditure on secondary education, Government expenditure on secondary education as a percentage of GDP, Government expenditure on tertiary education, Government expenditure on tertiary education as a percentage of GDP, Government expenditure on upper secondary education, Government expenditure on upper secondary education as a percentage of GDP, Gross enrolment ratio, Gross graduation ratio from first degree programmes, Illiterate population, Illiterate youth population, Inbound internationally mobile students from Africa, Inbound internationally mobile students from Asia, Inbound internationally mobile students from Central Asia, Inbound internationally mobile students from Central and Eastern Europe, Inbound internationally mobile students from East Asia and the Pacific, Inbound internationally mobile students from Europe, Inbound internationally mobile students from Latin America and the Caribbean, Inbound internationally mobile students from North America, Inbound internationally mobile students from North America and Western Europe, Inbound internationally mobile students from Oceania, Inbound internationally mobile students from South America, Inbound internationally mobile students from South and West Asia, Inbound internationally mobile students from sub-Saharan Africa, Inbound internationally mobile students from the Arab States, Inbound internationally mobile students from the Caribbean and Central America, Inbound internationally mobile students from unknown continents, Inbound internationally mobile students from unknown regions, Inbound mobility rate, Mean years of schooling, Net flow of internationally mobile students, Net flow ratio of internationally mobile students, Non-teaching staff compensation as a percentage of total expenditure in lower secondary public institutions, Non-teaching staff compensation as a percentage of total expenditure in post-secondary non-tertiary public institutions, Non-teaching staff compensation as a percentage of total expenditure in pre-primary public institutions, Non-teaching staff compensation as a percentage of total expenditure in primary public institutions, Non-teaching staff compensation as a percentage of total expenditure in public institutions, Non-teaching staff compensation as a percentage of total expenditure in secondary public institutions, Non-teaching staff compensation as a percentage of total expenditure in tertiary public institutions, Non-teaching staff compensation as a percentage of total expenditure in upper secondary public institutions, North America, Oceania, Official entrance age to compulsory education, Official entrance age to early childhood education, Official entrance age to early childhood educational development, Official entrance age to lower secondary education, Official entrance age to post-secondary non-tertiary education, Official entrance age to pre-primary education, Official entrance age to primary education, Official entrance age to upper secondary education, Out-of-school adolescents and youth of secondary school age, Out-of-school adolescents of lower secondary school age, Out-of-school children, Out-of-school children and adolescents of primary and lower secondary school age, Out-of-school children of primary school age, Out-of-school youth of upper secondary school age, Outbound internationally mobile tertiary students studying in Central Asia, Outbound internationally mobile tertiary students studying in Central and Eastern Europe, Outbound internationally mobile tertiary students studying in East Asia and the Pacific, Outbound internationally mobile tertiary students studying in Latin America and the Caribbean, Outbound internationally mobile tertiary students studying in North America and Western Europe, Outbound internationally mobile tertiary students studying in South and West Asia, Outbound internationally mobile tertiary students studying in sub-Saharan Africa, Outbound internationally mobile tertiary students studying in the Arab States, Outbound mobility ratio, Outbound mobility ratio to Central Asia, Outbound mobility ratio to Central and Eastern Europe, Outbound mobility ratio to East Asia and the Pacific, Outbound mobility ratio to Latin America and the Caribbean, Outbound mobility ratio to North America and Western Europe, Outbound mobility ratio to South and West Asia, Outbound mobility ratio to sub-Saharan Africa, Outbound mobility ratio to the Arab States, Percentage of enrolment in early childhood education programmes in private institutions, Percentage of enrolment in early childhood educational development programmes in private institutions, Percentage of enrolment in lower secondary education in private institutions, Percentage of enrolment in post-secondary non-tertiary education in private institutions, Percentage of enrolment in pre-primary education in private institutions, Percentage of enrolment in primary education in private institutions, Percentage of enrolment in secondary education in private institutions, Percentage of enrolment in tertiary education in private institutions, Percentage of enrolment in upper secondary education in private institutions, Percentage of graduates from Science, Percentage of graduates from programmes other than Science, Percentage of graduates from tertiary education graduating from Agriculture, Percentage of graduates from tertiary education graduating from Arts and Humanities programmes, Percentage of graduates from tertiary education graduating from Business, Percentage of graduates from tertiary education graduating from Education programmes, Percentage of graduates from tertiary education graduating from Engineering, Percentage of graduates from tertiary education graduating from Health and Welfare programmes, Percentage of graduates from tertiary education graduating from Information and Communication Technologies programmes, Percentage of graduates from tertiary education graduating from Natural Sciences, Percentage of graduates from tertiary education graduating from Services programmes, Percentage of graduates from tertiary education graduating from Social Sciences, Percentage of graduates from tertiary education graduating from programmes in unspecified fields, Percentage of teachers in lower secondary education who are female, Percentage of teachers in post-secondary non-tertiary education who are female, Percentage of teachers in pre-primary education who are female, Percentage of teachers in primary education who are female, Percentage of teachers in secondary education who are female, Percentage of teachers in tertiary education who are female, Percentage of teachers in upper secondary education who are female, Population of compulsory school age, Population of the official entrance age to primary education, Population of the official entrance age to secondary general education, Repeaters in Grade 1 of lower secondary general education, Repeaters in Grade 1 of primary education, Repeaters in Grade 2 of lower secondary general education, Repeaters in Grade 2 of primary education, Repeaters in Grade 3 of lower secondary general education, Repeaters in Grade 3 of primary education, Repeaters in Grade 4 of lower secondary general education, Repeaters in Grade 4 of primary education, Repeaters in Grade 5 of lower secondary general education, Repeaters in Grade 5 of primary education, Repeaters in Grade 6 of lower secondary general education, Repeaters in Grade 6 of primary education, Repeaters in Grade 7 of primary education, Repeaters in grade unknown of lower secondary general education, Repeaters in grade unknown of primary education, Repeaters in lower secondary general education, Repeaters in primary education, Repetition rate in Grade 1 of lower secondary general education, Repetition rate in Grade 1 of primary education, Repetition rate in Grade 2 of lower secondary general education, Repetition rate in Grade 2 of primary education, Repetition rate in Grade 3 of lower secondary general education, Repetition rate in Grade 3 of primary education, Repetition rate in Grade 4 of lower secondary general education, Repetition rate in Grade 4 of primary education, Repetition rate in Grade 5 of lower secondary general education, Repetition rate in Grade 5 of primary education, Repetition rate in Grade 6 of primary education, Repetition rate in Grade 7 of primary education, Repetition rate in lower secondary general education, Repetition rate in primary education, School age population, School life expectancy, Share of all students in lower secondary education enrolled in general programmes, Share of all students in lower secondary education enrolled in vocational programmes, Share of all students in post-secondary non-tertiary education enrolled in general programmes, Share of all students in post-secondary non-tertiary education enrolled in vocational programmes, Share of all students in secondary education enrolled in general programmes, Share of all students in secondary education enrolled in vocational programmes, Share of all students in upper secondary education enrolled in general programmes, Share of all students in upper secondary education enrolled in vocational programmes, South America, Start month of the academic school year, Start of the academic school year, Survival rate to Grade 4 of primary education, Survival rate to Grade 5 of primary education, Survival rate to the last grade of primary education, Teachers in early childhood educational development programmes, Teachers in lower secondary education, Teachers in post-secondary non-tertiary education, Teachers in pre-primary education, Teachers in primary education, Teachers in secondary education, Teachers in tertiary education ISCED 5 programmes, Teachers in tertiary education ISCED 6, Teachers in tertiary education programmes, Teachers in upper secondary education, Teaching staff compensation as a percentage of total expenditure in lower secondary public institutions, Teaching staff compensation as a percentage of total expenditure in post-secondary non-tertiary public institutions, Teaching staff compensation as a percentage of total expenditure in pre-primary public institutions, Teaching staff compensation as a percentage of total expenditure in primary public institutions, Teaching staff compensation as a percentage of total expenditure in public institutions, Teaching staff compensation as a percentage of total expenditure in secondary public institutions, Teaching staff compensation as a percentage of total expenditure in tertiary public institutions, Teaching staff compensation as a percentage of total expenditure in upper secondary public institutions, Theoretical duration of early childhood education, Theoretical duration of early childhood educational development, Theoretical duration of lower secondary education, Theoretical duration of post-secondary non-tertiary education, Theoretical duration of pre-primary education, Theoretical duration of primary education, Theoretical duration of secondary education, Theoretical duration of upper secondary education, Total inbound internationally mobile students, Total net attendance rate, Total net enrolment rate, Total outbound internationally mobile tertiary students studying abroad, Youth illiterate population", + "format": "csv", + "name": "Other Policy Relevant Indicators data", + "resource_type": "file.upload", + "url": "updated_by_file_upload_step", + "url_type": "upload" + }, + { + "dataset_preview_enabled": "False", + "description": "Other Policy Relevant Indicators indicator list with HXL tags", + "format": "csv", + "name": "Other Policy Relevant Indicators indicator list", + "resource_type": "file.upload", + "url": "updated_by_file_upload_step", + "url_type": "upload" + }, + { + "dataset_preview_enabled": "False", + "description": "Other Policy Relevant Indicators metadata with HXL tags", + "format": "csv", + "name": "Other Policy Relevant Indicators metadata", + "resource_type": "file.upload", + "url": "updated_by_file_upload_step", + "url_type": "upload" + }, + { + "dataset_preview_enabled": "True", + "description": "Cut down data for QuickCharts", + "format": "csv", + "name": "QuickCharts-SDG 4 Global and Thematic data", + "resource_type": "file.upload", + "url": "updated_by_file_upload_step", + "url_type": "upload" + } +] diff --git a/tests/hdx/api/test_ckan.py b/tests/hdx/api/test_ckan.py new file mode 100644 index 00000000..3693d162 --- /dev/null +++ b/tests/hdx/api/test_ckan.py @@ -0,0 +1,280 @@ +""" +Unit tests for the freshness class. + +""" +import json +import logging +import random +from os import getenv +from os.path import join + +import gspread +import pytest +from gspread.urls import DRIVE_FILES_API_V3_URL + +from hdx.api.configuration import Configuration +from hdx.data.dataset import Dataset +from hdx.data.resource import Resource +from hdx.data.vocabulary import Vocabulary +from hdx.location.country import Country +from hdx.utilities.dateparse import now_utc + +logger = logging.getLogger(__name__) + + +class TestCKAN: + @pytest.fixture(scope="class") + def configuration(self): + hdx_key = getenv("HDX_KEY_TEST") + Configuration._create( + hdx_site="demo", + user_agent="test", + hdx_key=hdx_key, + ) + Country.countriesdata(use_live=False) + + @pytest.fixture(scope="function") + def datasetmetadata(self): + return join("tests", "fixtures", "CKAN", "hdx_dataset_static.yaml") + + @pytest.fixture(scope="function") + def testdata(self): + return join("tests", "fixtures", "test_data.csv") + + @pytest.fixture(scope="class") + def params(self): + return { + "corpora": "teamDrive", + "teamDriveId": "0AKCBfHI3H-hcUk9PVA", + "supportsAllDrives": True, + "includeItemsFromAllDrives": True, + } + + @pytest.fixture(scope="function") + def gclient(self): + gsheet_auth = getenv("GSHEET_AUTH") + if not gsheet_auth: + raise ValueError("No gsheet authorisation supplied!") + info = json.loads(gsheet_auth) + scopes = [ + "https://www.googleapis.com/auth/drive", + "https://www.googleapis.com/auth/spreadsheets", + ] + gclient = gspread.service_account_from_dict(info, scopes=scopes) + return gclient + + @pytest.fixture(scope="function") + def setup_teardown_folder(self, configuration, gclient, params): + payload = { + "name": "hdx_python_api_test_tmp", + "mimeType": "application/vnd.google-apps.folder", + "parents": ["1dvx0H0RG5ZfM9QL148uymWpbuxAqmOzD"], + } + r = gclient.http_client.request( + "post", DRIVE_FILES_API_V3_URL, json=payload, params=params + ) + folderid = r.json()["id"] + yield gclient, folderid + + payload = {"trashed": True} + url = f"{DRIVE_FILES_API_V3_URL}/{folderid}" + gclient.http_client.request("patch", url, json=payload, params=params) + Vocabulary._approved_vocabulary = None + Vocabulary._tags_dict = None + Configuration.delete() + + def test_create_dataset( + self, + datasetmetadata, + testdata, + setup_teardown_folder, + params, + ): + today = now_utc() + gclient, folderid = setup_teardown_folder + + def create_gsheet(name, update): + payload = { + "name": name, + "mimeType": "application/vnd.google-apps.spreadsheet", + "parents": [folderid], + } + r = gclient.http_client.request( + "post", DRIVE_FILES_API_V3_URL, json=payload, params=params + ) + spreadsheetid = r.json()["id"] + gsheet = gclient.open_by_key(spreadsheetid) + wks = gsheet.sheet1 + wks.update(update, "A1") + gsheet.share("", role="reader", perm_type="anyone") + return wks, f"{gsheet.url}/export?format=xlsx" + + name = "hdx_python_api_test" + dataset = Dataset.read_from_hdx(name) + if dataset: + dataset.delete_from_hdx() + title = "HDX Python API test" + dataset = Dataset({"name": name, "title": title}) + dataset.update_from_yaml(datasetmetadata) + maintainer_id = "196196be-6037-4488-8b71-d786adf4c081" + dataset.set_maintainer(maintainer_id) + dataset.set_organization("5a63012e-6c41-420c-8c33-e84b277fdc90") + dataset.set_time_period(today) + dataset.set_expected_update_frequency("Every week") + dataset.set_subnational(True) + countryiso3s = ["AFG", "PSE", "SYR", "YEM"] + dataset.add_country_locations(countryiso3s) + tags = ["conflict-violence", "displacement", "hxl"] + dataset.add_tags(tags) + resource_no = 0 + + def create_resource(): + nonlocal resource_no + + resource = Resource( + { + "name": f"test_resource_{resource_no}", + "description": f"Test Resource {resource_no}", + "last_modified": today.replace(tzinfo=None).isoformat(), + } + ) + filestore = resource_no % 2 == 0 + if filestore: + resource.set_format("csv") + resource.set_file_to_upload(testdata) + else: + wks, url = create_gsheet( + "resource1", + [[random.random() for i in range(10)] for j in range(10)], + ) + resource.set_format("xlsx") + resource["url"] = url + + resource_no += 1 + dataset.add_update_resource(resource) + + # add resources + for i in range(10): + create_resource() + + dataset.create_in_hdx( + hxl_update=False, updated_by_script="hdx_python_api_ignore" + ) + + # check created dataset + dataset = Dataset.read_from_hdx(name) + assert dataset["name"] == name + assert dataset["title"] == title + assert dataset.get_tags() == tags + assert dataset.get_maintainer()["id"] == maintainer_id + assert dataset.get_organization()["display_name"] == "INNAGO" + resources = dataset.get_resources() + for i, resource in enumerate(resources): + assert resource["name"] == f"test_resource_{i}" + if i % 2 == 0: + assert resource.get_format() == "csv" + assert resource["url_type"] == "upload" + assert "humdata" in resource["url"] + else: + assert resource.get_format() == "xlsx" + assert resource["url_type"] == "api" + assert "humdata" not in resource["url"] + + # modify dataset + dataset_id = dataset["id"] + title = "HDX Python API test changed" + notes = "added some notes" + caveats = "added some caveats" + # starting from a newly created Dataset object + dataset = Dataset( + { + "id": dataset_id, + "title": title, + "notes": notes, + "caveats": caveats, + } + ) + tags.remove("displacement") + dataset.add_tags(tags) + countryiso3s.remove("YEM") + dataset.add_country_locations(countryiso3s) + resources.pop(3) + resources.pop() + gsheet_resource = resources[5] + gsheet_resource.set_format("csv") + gsheet_resource.set_file_to_upload(testdata) + for resource in resources: + del resource["package_id"] + dataset.add_update_resources(resources) + dataset.update_in_hdx( + hxl_update=False, remove_additional_resources=True + ) + + # check updated dataset + dataset = Dataset.read_from_hdx(name) + assert dataset["name"] == name + assert dataset["title"] == title + assert dataset["notes"] == notes + assert dataset["caveats"] == caveats + assert dataset.get_tags() == tags + assert dataset.get_maintainer()["id"] == maintainer_id + assert dataset.get_organization()["display_name"] == "INNAGO" + updated_resources = dataset.get_resources() + for i, updated_resource in enumerate(updated_resources): + resource = resources[i] + assert updated_resource["name"] == resource["name"] + assert updated_resource.get_format() == resource.get_format() + assert updated_resource["url_type"].lower() == resource["url_type"] + url = resource.get("url") + if url: + if "humdata" in url: + assert "humdata" in updated_resource["url"] + else: + assert "humdata" not in updated_resource["url"] + else: + assert "humdata" in updated_resource["url"] + + # modify dataset again starting with existing dataset + title = "HDX Python API test changed again" + dataset["title"] = title + del dataset["caveats"] + tags = ["agriculture-livestock", "climate-weather", "hxl"] + dataset["tags"] = [] + dataset.add_tags(tags) + countryiso3s.append("YEM") + dataset.add_country_location("YEM") + dataset.delete_resource(updated_resources[5]) + updated_resources[0].set_file_to_upload(testdata) + create_resource() + resources = dataset.get_resources() + dataset.create_in_hdx( + hxl_update=False, + remove_additional_resources=True, + keys_to_delete=("caveats",), + ) + + # check dataset updated for second time + dataset = Dataset.read_from_hdx(name) + assert dataset["name"] == name + assert dataset["title"] == title + assert "caveats" not in dataset + assert dataset.get_tags() == tags + assert dataset.get_maintainer()["id"] == maintainer_id + assert dataset.get_organization()["display_name"] == "INNAGO" + updated_resources = dataset.get_resources() + for i, updated_resource in enumerate(updated_resources): + resource = resources[i] + assert updated_resource["name"] == resource["name"] + assert updated_resource.get_format() == resource.get_format() + assert updated_resource["url_type"].lower() == resource["url_type"] + url = resource.get("url") + if url: + if "humdata" in url: + assert "humdata" in updated_resource["url"] + else: + assert "humdata" not in updated_resource["url"] + else: + assert "humdata" in updated_resource["url"] + + # tear down + dataset.delete_from_hdx() diff --git a/tests/hdx/data/test_dataset_core.py b/tests/hdx/data/test_dataset_core.py index 94e78e10..a3659976 100755 --- a/tests/hdx/data/test_dataset_core.py +++ b/tests/hdx/data/test_dataset_core.py @@ -88,7 +88,7 @@ def mocksearch(url, datadict): newsearchdict["results"] = newsearchdict["results"][:5] else: newsearchdict["count"] = 0 - newsearchdict["results"] = list() + newsearchdict["results"] = [] result = json.dumps(newsearchdict) return MockResponse( 200, @@ -367,17 +367,16 @@ def post(url, data, headers, files, allow_redirects, auth=None): datadict = json.loads(datadict["update"]) if datadict["name"] in ["MyDataset1", "DatasetExist"]: resultdictcopy = copy.deepcopy(dataset_resultdict) + for i, resource in enumerate(datadict["resources"]): + if not resource: + datadict["resources"][i] = resultdictcopy[ + "resources" + ][i] + merge_two_dictionaries(resultdictcopy, datadict) for i, resource in enumerate( resultdictcopy["resources"] ): - for j, resource2 in enumerate( - resultdictcopy["resources"] - ): - if i != j: - if resource == resource2: - del resultdictcopy["resources"][j] - break resource["package_id"] = resultdictcopy["id"] resultdictcopy = {"package": resultdictcopy} result = json.dumps(resultdictcopy) @@ -541,7 +540,7 @@ def test_revise(self, configuration, test_file, post_revise): with pytest.raises(HDXError): dataset._write_to_hdx( "revise", - dict(), + {}, id_field_name="", files_to_upload={"update__resources__0__upload": "NOTEXIST"}, ) @@ -646,11 +645,13 @@ def test_update_in_hdx(self, configuration, post_update, date_pattern): assert dataset["dataset_date"] == "06/04/2016" dataset["dataset_date"] = "02/26/2016" + dataset.remove_tag("conflict") dataset["id"] = "TEST1" dataset["name"] = "MyDataset1" dataset.update_in_hdx() assert dataset["id"] == "TEST1" assert dataset["dataset_date"] == "02/26/2016" + assert dataset.get_tags() == ["political violence"] assert dataset["state"] == "active" pattern = ( r"HDXPythonLibrary/%s-test \([12]\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d.\d\d\d\d\d\d\)" @@ -690,7 +691,7 @@ def test_update_in_hdx(self, configuration, post_update, date_pattern): dataset.update_in_hdx() dataset["id"] = "TEST1" - dataset["groups"] = list() + dataset["groups"] = [] with pytest.raises(HDXError): dataset.update_in_hdx() dataset.update_in_hdx(ignore_check=True) @@ -1087,7 +1088,7 @@ def test_hdxconnect(self, configuration, post_create): assert dataset["private"] is True dataset.set_requestable() assert dataset.get("field_name") is None - assert dataset.get_fieldnames() == list() + assert dataset.get_fieldnames() == [] assert dataset.add_fieldname("myfield1") is True assert dataset.add_fieldnames(["myfield1", "myfield2"]) is False assert dataset.remove_fieldname("myfield1") is True @@ -1095,7 +1096,7 @@ def test_hdxconnect(self, configuration, post_create): assert dataset.add_fieldnames(["myfield3", "myfield4"]) is True assert dataset.get_fieldnames() == ["myfield2", "myfield3", "myfield4"] assert dataset.get("fiele_types") is None - assert dataset.get_filetypes() == list() + assert dataset.get_filetypes() == [] assert dataset.add_filetype("mytype1") is True assert dataset.add_filetypes(["mytype1", "mytype2"]) is False assert dataset.remove_filetype("mytype1") is True diff --git a/tests/hdx/data/test_dataset_noncore.py b/tests/hdx/data/test_dataset_noncore.py index 82d19b66..e8c06fa9 100755 --- a/tests/hdx/data/test_dataset_noncore.py +++ b/tests/hdx/data/test_dataset_noncore.py @@ -353,8 +353,8 @@ def test_get_add_location(self, locations): assert len(dataset["groups"]) == 60 assert len(dataset.get_location_names()) == 60 del dataset["groups"] - assert dataset.get_location_names() == list() - assert dataset.get_location_iso3s() == list() + assert dataset.get_location_names() == [] + assert dataset.get_location_iso3s() == [] with pytest.raises(HDXError): dataset.add_country_location("abc") with pytest.raises(HDXError): @@ -953,10 +953,10 @@ def test_remove_dates_from_title(self): assert "title" not in dataset title = "Title with no dates" dataset["title"] = title - assert dataset.remove_dates_from_title() == list() + assert dataset.remove_dates_from_title() == [] assert dataset["title"] == title assert "dataset_date" not in dataset - assert dataset.remove_dates_from_title(set_time_period=True) == list() + assert dataset.remove_dates_from_title(set_time_period=True) == [] title = "ICA Armenia, 2017 - Drought Risk, 1981-2015" dataset["title"] = title expected = [ @@ -985,7 +985,7 @@ def test_remove_dates_from_title(self): dataset["dataset_date"] == "[1981-01-01T00:00:00 TO 2015-12-31T23:59:59]" ) - assert dataset.remove_dates_from_title() == list() + assert dataset.remove_dates_from_title() == [] dataset["title"] = "Mon_State_Village_Tract_Boundaries 9999 2001" expected = [ ( @@ -1070,7 +1070,7 @@ def test_generate_qc_resource_from_rows(self, configuration): join("tests", "fixtures", "qc_from_rows", qc_filename), join(folder, qc_filename), ) - rows = list() + rows = [] resource = dataset.generate_qc_resource_from_rows( folder, qc_filename, diff --git a/tests/hdx/data/test_dataset_title_helper.py b/tests/hdx/data/test_dataset_title_helper.py index 7fb5a3aa..eb9d2c33 100755 --- a/tests/hdx/data/test_dataset_title_helper.py +++ b/tests/hdx/data/test_dataset_title_helper.py @@ -17,8 +17,8 @@ def expected_ranges_2019(self): ] def test_fuzzy_match_dates_in_title(self, expected_ranges_2019): - ignore_wrong_years = list() - ranges = list() + ignore_wrong_years = [] + ranges = [] assert ( DatasetTitleHelper.fuzzy_match_dates_in_title( "Myanmar Town July 2019", ranges, ignore_wrong_years @@ -26,7 +26,7 @@ def test_fuzzy_match_dates_in_title(self, expected_ranges_2019): == "Myanmar Town" ) assert ranges == expected_ranges_2019 - ranges = list() + ranges = [] assert ( DatasetTitleHelper.fuzzy_match_dates_in_title( "Myanmar Town 2019 July", ranges, ignore_wrong_years @@ -40,7 +40,7 @@ def test_get_date_from_title(self, expected_ranges_2019): "Myanmar Self Administered Regions Boundaries MIMU v9.2.1" ) == ( "Myanmar Self Administered Regions Boundaries MIMU v9.2.1", - list(), + [], ) assert DatasetTitleHelper.get_dates_from_title( "Myanmar Town 2019 July" @@ -178,7 +178,7 @@ def test_get_date_from_title(self, expected_ranges_2019): "ALCS 2014/13" ) == ( # not a month and range going down "ALCS 2014/13", - list(), + [], ) assert DatasetTitleHelper.get_dates_from_title( "Mon_State_Village_Tract_Boundaries 9999" diff --git a/tests/hdx/data/test_resource.py b/tests/hdx/data/test_resource.py index 64801c41..44e0a6a9 100755 --- a/tests/hdx/data/test_resource.py +++ b/tests/hdx/data/test_resource.py @@ -836,7 +836,7 @@ def test_update_in_hdx(self, configuration, date_pattern, post_update): with pytest.raises(HDXError): resource.update_in_hdx() - resource.data = dict() + resource.data = {} with pytest.raises(HDXError): resource.update_in_hdx() diff --git a/tests/hdx/data/test_showcase.py b/tests/hdx/data/test_showcase.py index 82350fdc..ada8c128 100755 --- a/tests/hdx/data/test_showcase.py +++ b/tests/hdx/data/test_showcase.py @@ -132,7 +132,7 @@ def mockallsearch(url, datadict): newsearchdict["results"] = newsearchdict["results"][:5] else: newsearchdict["count"] = 0 - newsearchdict["results"] = list() + newsearchdict["results"] = [] result = json.dumps(newsearchdict) return MockResponse( 200, diff --git a/tests/hdx/data/test_update_dataset_resources.py b/tests/hdx/data/test_update_dataset_resources.py index 46c7c572..4bbe70af 100644 --- a/tests/hdx/data/test_update_dataset_resources.py +++ b/tests/hdx/data/test_update_dataset_resources.py @@ -1,4 +1,3 @@ -import json from os.path import join import pytest @@ -9,6 +8,7 @@ from hdx.data.resource import Resource from hdx.data.vocabulary import Vocabulary from hdx.location.country import Country +from hdx.utilities.loader import load_json class TestUpdateDatasetResourcesLogic: @@ -35,7 +35,7 @@ def configuration(self): ) Locations.set_validlocations([{"name": "zmb", "title": "Zambia"}]) Country.countriesdata(use_live=False) - Vocabulary._tags_dict = dict() + Vocabulary._tags_dict = {} Vocabulary._approved_vocabulary = { "tags": [ {"name": "hxl"}, @@ -62,48 +62,46 @@ def new_dataset_json(self, fixture_path): def dataset_json(self, fixture_path): return join(fixture_path, "unesco_dataset.json") + @pytest.fixture(scope="class") + def expected_resources_to_update_json(self, fixture_path): + return join(fixture_path, "expected_resources_to_update.json") + @pytest.fixture(scope="function") def dataset(self, dataset_json): return Dataset.load_from_json(dataset_json) @pytest.fixture(scope="function") def new_dataset(self, fixture_path, new_dataset_json): - with open(new_dataset_json) as f: - jsonobj = json.loads(f.read()) - resourceobjs = jsonobj["resources"] - del jsonobj["resources"] - dataset = Dataset(jsonobj) - for resourceobj in resourceobjs: - resource = Resource(resourceobj) - filename = self.file_mapping[resourceobj["name"]] - resource.set_file_to_upload(join(fixture_path, filename)) - dataset.add_update_resource(resource) - return dataset + jsonobj = load_json(new_dataset_json) + resourceobjs = jsonobj["resources"] + del jsonobj["resources"] + dataset = Dataset(jsonobj) + for resourceobj in resourceobjs: + resource = Resource(resourceobj) + filename = self.file_mapping[resourceobj["name"]] + resource.set_file_to_upload(join(fixture_path, filename)) + dataset.add_update_resource(resource) + return dataset + + @pytest.fixture(scope="class") + def expected_resources_to_update(self, expected_resources_to_update_json): + return load_json(expected_resources_to_update_json) def test_dataset_update_resources( - self, configuration, dataset, new_dataset + self, configuration, dataset, new_dataset, expected_resources_to_update ): dataset.old_data = new_dataset.data dataset.old_data["resources"] = new_dataset._copy_hdxobjects( new_dataset.resources, Resource, ("file_to_upload", "data_updated") ) ( + resources_to_update, resources_to_delete, - new_resource_order, filestore_resources, - ) = dataset._dataset_merge_update_resources(True, True, True, True) + new_resource_order, + ) = dataset._dataset_update_resources(True, True, True, True) + assert resources_to_update == expected_resources_to_update assert resources_to_delete == [8, 2, 1, 0] - assert new_resource_order == [ - ("SDG 4 Global and Thematic data", "csv"), - ("SDG 4 Global and Thematic indicator list", "csv"), - ("SDG 4 Global and Thematic metadata", "csv"), - ("Other Policy Relevant Indicators data", "csv"), - ("Other Policy Relevant Indicators indicator list", "csv"), - ("Other Policy Relevant Indicators metadata", "csv"), - ("Demographic and Socio-economic data", "csv"), - ("Demographic and Socio-economic indicator list", "csv"), - ("QuickCharts-SDG 4 Global and Thematic data", "csv"), - ] assert filestore_resources == { 3: "tests/fixtures/update_dataset_resources/sdg_data_zwe.csv", 4: "tests/fixtures/update_dataset_resources/sdg_indicatorlist_zwe.csv", @@ -115,13 +113,24 @@ def test_dataset_update_resources( 11: "tests/fixtures/update_dataset_resources/opri_metadata_zwe.csv", 12: "tests/fixtures/update_dataset_resources/qc_sdg_data_zwe.csv", } - results = dataset._save_dataset_add_filestore_resources( - "update", - "id", + assert new_resource_order == [ + ("SDG 4 Global and Thematic data", "csv"), + ("SDG 4 Global and Thematic indicator list", "csv"), + ("SDG 4 Global and Thematic metadata", "csv"), + ("Other Policy Relevant Indicators data", "csv"), + ("Other Policy Relevant Indicators indicator list", "csv"), + ("Other Policy Relevant Indicators metadata", "csv"), + ("Demographic and Socio-economic data", "csv"), + ("Demographic and Socio-economic indicator list", "csv"), + ("QuickCharts-SDG 4 Global and Thematic data", "csv"), + ] + dataset._prepare_hdx_call(dataset.old_data, {}) + results = dataset._revise_dataset( tuple(), + resources_to_update, resources_to_delete, - new_resource_order, filestore_resources, + new_resource_order, hxl_update=False, create_default_views=False, test=True, @@ -138,15 +147,14 @@ def test_dataset_update_resources( "update__resources__8__upload": "tests/fixtures/update_dataset_resources/qc_sdg_data_zwe.csv", } resources = results["update"]["resources"] - cutdown_resources = list() + cutdown_resources = [] for resource in resources: - cutdown_resource = dict() + cutdown_resource = {} for key, value in resource.items(): if key in ( "dataset_preview_enabled", "format", "name", - "position", "resource_type", "url", "url_type", @@ -158,7 +166,6 @@ def test_dataset_update_resources( "dataset_preview_enabled": "False", "format": "csv", "name": "SDG 4 Global and Thematic data", - "position": 3, "resource_type": "file.upload", "url": "updated_by_file_upload_step", "url_type": "upload", @@ -167,7 +174,6 @@ def test_dataset_update_resources( "dataset_preview_enabled": "False", "format": "csv", "name": "SDG 4 Global and Thematic indicator list", - "position": 4, "resource_type": "file.upload", "url": "updated_by_file_upload_step", "url_type": "upload", @@ -176,7 +182,6 @@ def test_dataset_update_resources( "dataset_preview_enabled": "False", "format": "csv", "name": "SDG 4 Global and Thematic metadata", - "position": 5, "resource_type": "file.upload", "url": "updated_by_file_upload_step", "url_type": "upload", @@ -185,7 +190,6 @@ def test_dataset_update_resources( "dataset_preview_enabled": "False", "format": "csv", "name": "Demographic and Socio-economic data", - "position": 6, "resource_type": "file.upload", "url": "updated_by_file_upload_step", "url_type": "upload", @@ -194,7 +198,6 @@ def test_dataset_update_resources( "dataset_preview_enabled": "False", "format": "csv", "name": "Demographic and Socio-economic indicator list", - "position": 7, "resource_type": "file.upload", "url": "updated_by_file_upload_step", "url_type": "upload", diff --git a/tests/hdx/data/test_update_logic.py b/tests/hdx/data/test_update_logic.py index 43629131..5812623c 100644 --- a/tests/hdx/data/test_update_logic.py +++ b/tests/hdx/data/test_update_logic.py @@ -25,7 +25,7 @@ def configuration(self): ) Locations.set_validlocations([{"name": "zmb", "title": "Zambia"}]) Country.countriesdata(use_live=False) - Vocabulary._tags_dict = dict() + Vocabulary._tags_dict = {} Vocabulary._approved_vocabulary = { "tags": [ {"name": "hxl"}, @@ -94,7 +94,7 @@ def add_dataset_resources(dataset, yaml_path, include=None, exclude=None): if include is None: include = range(len(resources)) if exclude is None: - exclude = list() + exclude = [] for i, resource in enumerate(resources): if i not in include: continue @@ -110,7 +110,7 @@ def add_new_dataset_resources( if include is None: include = range(len(resources_uploads)) if exclude is None: - exclude = list() + exclude = [] for i, resource_upload in enumerate(resources_uploads): if i not in include: continue @@ -160,17 +160,17 @@ def test_update_logic_1( self.add_dataset_resources(dataset, resources_yaml) dataset.old_data = new_dataset.data dataset.old_data["resources"] = new_dataset.resources - results = dataset._dataset_merge_hdx_update( + results = dataset._dataset_hdx_update( update_resources=True, match_resources_by_metadata=True, - keys_to_delete=list(), + keys_to_delete=[], remove_additional_resources=True, match_resource_order=False, create_default_views=False, hxl_update=False, test=True, ) - assert results["filter"] == list() + assert results["filter"] == [] update = results["update"] del update["updated_by_script"] assert update == { @@ -194,10 +194,8 @@ def test_update_logic_1( "vocabulary_id": "4e61d464-4943-4e97-973a-84673c1aaa87", } ], - "id": "3adc4bb0-faef-42ae-bd67-0ea08918a629", "resources": [ { - "id": "1e7a68da-501a-444e-94f1-3606263e10c8", "name": "All Health Indicators for Zambia", "description": "See resource descriptions below for links to indicator metadata", "format": "csv", @@ -206,7 +204,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "40fbcfcc-e180-455d-9d65-a9f8d1f85643", "name": "Mortality and global health estimates Indicators for Zambia", "description": "*Mortality and global health estimates:*\n[Infant mortality rate (proba", "format": "csv", @@ -215,7 +212,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "018577c8-a397-4a4f-b13c-889c4990d5e2", "name": "Sustainable development goals Indicators for Zambia", "description": "*Sustainable development goals:*\n[Adolescent birth rate (per 1000 wome", "format": "csv", @@ -224,7 +220,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "b42f3a3e-82a9-4fe6-9f0c-81d520ce7666", "name": "Millennium Development Goals (MDGs) Indicators for Zambia", "description": "*Millennium Development Goals (MDGs):*\n[Contraceptive prevalence (%)](", "format": "csv", @@ -233,7 +228,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "b67738ca-812f-41b2-80e3-7c3340648398", "name": "Health systems Indicators for Zambia", "description": "*Health systems:*\n[Median availability of selected generic medicines (", "format": "csv", @@ -242,7 +236,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "a610cbb6-3bd2-4400-9aee-899dbe6b4a98", "name": "Malaria Indicators for Zambia", "description": "*Malaria:*\n[Children aged <5 years sleeping under insecticide-treated ", "format": "csv", @@ -251,7 +244,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "de38982e-ef0d-4e5c-b0a3-16f933eaa808", "name": "Tuberculosis Indicators for Zambia", "description": "*Tuberculosis:*\n[Deaths due to tuberculosis among HIV-negative people ", "format": "csv", @@ -260,7 +252,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "c330fc6e-624d-4d7f-bb46-f5ea67cf5d0a", "name": "Child health Indicators for Zambia", "description": "*Child health:*\n[Children aged <5 years stunted (%)](https://www.who.i", "format": "csv", @@ -269,7 +260,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "9903aea5-d193-4d9d-9018-e98213aef0f5", "name": "Infectious diseases Indicators for Zambia", "description": "*Infectious diseases:*\n[Cholera - number of reported cases](https://ww", "format": "csv", @@ -278,7 +268,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "fb24e4d1-9447-496d-9049-4712e4cd90c6", "name": "World Health Statistics Indicators for Zambia", "description": "*World Health Statistics:*\n[Literacy rate among adults aged >= 15 year", "format": "csv", @@ -287,7 +276,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "98e022a6-e054-41fd-af9a-e4c08ddcc5cd", "name": "Health financing Indicators for Zambia", "description": "*Health financing:*\n[Private prepaid plans as a percentage of private ", "format": "csv", @@ -296,7 +284,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "9a72db07-1bcc-4316-a427-1c208ebb5708", "name": "Public health and environment Indicators for Zambia", "description": "*Public health and environment:*\n[Population using solid fuels (%)](ht", "format": "csv", @@ -305,7 +292,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "873ead28-fff0-49d6-83c4-1ccda19cb0f7", "name": "Substance use and mental health Indicators for Zambia", "description": "*Substance use and mental health:*\n[Fines for violations](https://www.", "format": "csv", @@ -314,7 +300,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "74e93f31-494c-4950-b891-6db42537e8b4", "name": "Injuries and violence Indicators for Zambia", "description": "*Injuries and violence:*\n[Income level](https://www.who.int/data/gho/i", "format": "csv", @@ -323,7 +308,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "ef319875-76fb-4ff4-bddb-02715a7638df", "name": "HIV/AIDS and other STIs Indicators for Zambia", "description": "*HIV/AIDS and other STIs:*\n[Prevalence of HIV among adults aged 15 to ", "format": "csv", @@ -332,7 +316,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "1d4b9ac1-71ed-4ed3-a120-c4c6fea00a3d", "name": "Nutrition Indicators for Zambia", "description": "*Nutrition:*\n[Early initiation of breastfeeding (%)](https://www.who.i", "format": "csv", @@ -341,7 +324,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "1815f97a-d93c-4e64-909d-39f288880c50", "name": "Urban health Indicators for Zambia", "description": "*Urban health:*\n[Percentage of the total population living in cities >", "format": "csv", @@ -350,7 +332,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "23af66cb-cffb-491f-a51b-c41584b83f3b", "name": "Noncommunicable diseases Indicators for Zambia", "description": "*Noncommunicable diseases:*\n[Prevalence of overweight among adults, BM", "format": "csv", @@ -359,7 +340,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "2beb31ac-873d-47f4-9cfc-75cebd2303b5", "name": "Noncommunicable diseases CCS Indicators for Zambia", "description": "*Noncommunicable diseases CCS:*\n[Existence of operational policy/strat", "format": "csv", @@ -368,7 +348,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "9a14881d-196a-44dc-bcf6-bda199b1137b", "name": "Negelected tropical diseases Indicators for Zambia", "description": "*Negelected tropical diseases:*\n[Number of new reported cases of Burul", "format": "csv", @@ -377,7 +356,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "79b0d05f-9180-492c-b614-687a9203bc1c", "name": "Health Equity Monitor Indicators for Zambia", "description": "*Health Equity Monitor:*\n[Antenatal care coverage - at least one visit", "format": "csv", @@ -386,7 +364,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "4b4defd7-ce45-46e3-86ce-15154be90d0c", "name": "Infrastructure Indicators for Zambia", "description": "*Infrastructure:*\n[Total density per 100 000 population: Health posts]", "format": "csv", @@ -395,7 +372,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "29ac1338-b7b5-429f-bb38-455a66e8f6c1", "name": "Essential health technologies Indicators for Zambia", "description": "*Essential health technologies:*\n[Availability of national standards o", "format": "csv", @@ -404,7 +380,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "63ba2693-fe73-4555-9c64-93710f7c7404", "name": "Medical equipment Indicators for Zambia", "description": "*Medical equipment:*\n[Total density per million population: Magnetic R", "format": "csv", @@ -413,7 +388,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "2cdc3198-4ec7-4a25-86ee-8cdc80b670fc", "name": "Demographic and socioeconomic statistics Indicators for Zambia", "description": "*Demographic and socioeconomic statistics:*\n[Cellular subscribers (per", "format": "csv", @@ -422,7 +396,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "1028087c-af76-4ab2-ac30-0babb4ef00fc", "name": "Neglected tropical diseases Indicators for Zambia", "description": "*Neglected tropical diseases:*\n[Status of yaws endemicity](https://www", "format": "csv", @@ -431,7 +404,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "0ec8b0b2-1e0d-4338-b22a-e984f8d07f0a", "name": "International Health Regulations (2005) monitoring framework Indicators for Zambia", "description": "*International Health Regulations (2005) monitoring framework:*\n[Legis", "format": "csv", @@ -440,7 +412,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "2fa6877b-5b6d-446f-9ccf-52e2478d86da", "name": "Insecticide resistance Indicators for Zambia", "description": "*Insecticide resistance:*\n[Number of insecticide classes to which resi", "format": "csv", @@ -449,7 +420,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "262fddcb-62eb-42f1-94c6-ed20e6b18650", "name": "Universal Health Coverage Indicators for Zambia", "description": "*Universal Health Coverage:*\n[Cataract surgical coverage of adults age", "format": "csv", @@ -458,7 +428,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "5562627b-b7af-4d4d-a9ca-e4fcd996a180", "name": "Global Observatory for eHealth (GOe) Indicators for Zambia", "description": "*Global Observatory for eHealth (GOe):*\n[National universal health cov", "format": "csv", @@ -467,7 +436,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "53a1eeda-7a53-4b8c-87c8-c2ea276a52a3", "name": "RSUD: GOVERNANCE, POLICY AND FINANCING : PREVENTION Indicators for Zambia", "description": "*RSUD: GOVERNANCE, POLICY AND FINANCING : PREVENTION:*\n[Government uni", "format": "csv", @@ -476,7 +444,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "64c1aa0d-fa9a-4387-812e-fa7d00f382c2", "name": "RSUD: GOVERNANCE, POLICY AND FINANCING: TREATMENT Indicators for Zambia", "description": "*RSUD: GOVERNANCE, POLICY AND FINANCING: TREATMENT:*\n[Government unit/", "format": "csv", @@ -485,7 +452,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "5b82a27c-a461-475b-867b-0afdc93132fd", "name": "RSUD: GOVERNANCE, POLICY AND FINANCING: FINANCING Indicators for Zambia", "description": "*RSUD: GOVERNANCE, POLICY AND FINANCING: FINANCING:*\n[Five-year change", "format": "csv", @@ -494,7 +460,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "f0f95f26-0701-49ee-a9ac-d1525572f6a5", "name": "RSUD: SERVICE ORGANIZATION AND DELIVERY: TREATMENT SECTORS AND PROVIDERS Indicators for Zambia", "description": "*RSUD: SERVICE ORGANIZATION AND DELIVERY: TREATMENT SECTORS AND PROVID", "format": "csv", @@ -503,7 +468,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "9b805d00-3de4-48f2-a93f-d859961e9c08", "name": "RSUD: SERVICE ORGANIZATION AND DELIVERY: TREATMENT CAPACITY AND TREATMENT COVERAGE Indicators for Zambia", "description": "*RSUD: SERVICE ORGANIZATION AND DELIVERY: TREATMENT CAPACITY AND TREAT", "format": "csv", @@ -512,7 +476,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "9667ee27-97e5-4122-9dc6-e39df70eeb08", "name": "RSUD: SERVICE ORGANIZATION AND DELIVERY: PHARMACOLOGICAL TREATMENT Indicators for Zambia", "description": "*RSUD: SERVICE ORGANIZATION AND DELIVERY: PHARMACOLOGICAL TREATMENT:*\n", "format": "csv", @@ -521,7 +484,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "5e16b87e-5a06-4f7f-badb-3a2e44d64948", "name": "RSUD: SERVICE ORGANIZATION AND DELIVERY: SCREENING AND BRIEF INTERVENTIONS Indicators for Zambia", "description": "*RSUD: SERVICE ORGANIZATION AND DELIVERY: SCREENING AND BRIEF INTERVEN", "format": "csv", @@ -530,7 +492,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "d79c3183-471f-4331-94d6-13474171be91", "name": "RSUD: SERVICE ORGANIZATION AND DELIVERY: PREVENTION PROGRAMS AND PROVIDERS Indicators for Zambia", "description": "*RSUD: SERVICE ORGANIZATION AND DELIVERY: PREVENTION PROGRAMS AND PROV", "format": "csv", @@ -539,7 +500,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "daa99f76-d84b-4c05-b5dc-c9e82fbcb859", "name": "RSUD: SERVICE ORGANIZATION AND DELIVERY: SPECIAL PROGRAMMES AND SERVICES Indicators for Zambia", "description": "*RSUD: SERVICE ORGANIZATION AND DELIVERY: SPECIAL PROGRAMMES AND SERVI", "format": "csv", @@ -548,7 +508,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "ca1774e9-49c0-42a2-93c0-88fa32bb5adc", "name": "RSUD: HUMAN RESOURCES Indicators for Zambia", "description": "*RSUD: HUMAN RESOURCES:*\n[Health professionals providing treatment for", "format": "csv", @@ -557,7 +516,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "d244efc2-aa38-4197-b594-9bbd2b44bd9a", "name": "RSUD: INFORMATION SYSTEMS Indicators for Zambia", "description": "*RSUD: INFORMATION SYSTEMS:*\n[Epidemiological data collection for subs", "format": "csv", @@ -566,7 +524,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "ee06d52b-5c15-4a0c-8d00-830215aeff03", "name": "RSUD: YOUTH Indicators for Zambia", "description": "*RSUD: YOUTH:*\n[Epidemiological data collection system for substance u", "format": "csv", @@ -575,7 +532,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "1ef02dd7-07cc-44f9-83ee-fc945f6d22ea", "name": "FINANCIAL PROTECTION Indicators for Zambia", "description": "*FINANCIAL PROTECTION:*\n[Population with household expenditures on hea", "format": "csv", @@ -584,7 +540,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "913c96a9-48f9-4e4b-b11b-8ca019fdfd90", "name": "Noncommunicable diseases and mental health Indicators for Zambia", "description": "*Noncommunicable diseases and mental health:*\n[Number of deaths attrib", "format": "csv", @@ -593,7 +548,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "ccb28805-c35a-4393-a7eb-6653c73cbde7", "name": "Health workforce Indicators for Zambia", "description": "*Health workforce:*\n[Medical doctors (per 10 000 population)](https://", "format": "csv", @@ -602,7 +556,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "12fdb6cb-5e00-45b5-a6b5-611847866440", "name": "Neglected Tropical Diseases Indicators for Zambia", "description": "*Neglected Tropical Diseases:*\n[Number of new leprosy cases](https://w", "format": "csv", @@ -611,7 +564,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "ad2d521e-3a0e-451e-977c-bcbf125cec55", "name": "QuickCharts Indicators for Zambia", "description": "Cut down data for QuickCharts", "format": "csv", @@ -620,7 +572,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "19caff60-fd38-42b4-a17a-63abfaf0911e", "name": "TOBACCO Indicators for Zambia", "description": "*TOBACCO:*\n[Monitor](https://www.who.int/data/gho/indicator-metadata-r", "format": "csv", @@ -629,7 +580,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "b078ed42-079b-4a7c-b699-9c7894b87d4f", "name": "UHC Indicators for Zambia", "description": "*UHC:*\n[Population in malaria-endemic areas who slept under an insecti", "format": "csv", @@ -638,7 +588,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "61a61d4a-a50c-4248-bf3d-15c726ced591", "name": "ICD Indicators for Zambia", "description": "*ICD:*\n[ICD-11 implementation progress level](https://web-prod.who.int", "format": "csv", @@ -647,7 +596,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "beea81a0-dea1-45a3-891c-36d48a2c8bde", "name": "SEXUAL AND REPRODUCTIVE HEALTH Indicators for Zambia", "description": "*SEXUAL AND REPRODUCTIVE HEALTH:*\n[Institutional Births (birth taken p", "format": "csv", @@ -656,7 +604,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "9ed8a43b-d396-4a6f-948f-5bfe4c2fbba5", "name": "Immunization Indicators for Zambia", "description": "*Immunization:*\n[Proportion of vaccination cards seen (%)](https://www", "format": "csv", @@ -665,7 +612,6 @@ def test_update_logic_1( "url": "updated_by_file_upload_step", }, { - "id": "3a70eae8-5f44-497c-92c4-dd173e702429", "name": "NLIS Indicators for Zambia", "description": "*NLIS:*\n[Subclinical vitamin A deficiency in preschool-age children (s", "format": "csv", @@ -692,10 +638,10 @@ def test_update_logic_2( self.add_dataset_resources(dataset, resources_yaml) dataset.old_data = new_dataset.data dataset.old_data["resources"] = new_dataset.resources - results = dataset._dataset_merge_hdx_update( + results = dataset._dataset_hdx_update( update_resources=True, match_resources_by_metadata=True, - keys_to_delete=list(), + keys_to_delete=[], remove_additional_resources=True, match_resource_order=False, create_default_views=False, @@ -726,10 +672,8 @@ def test_update_logic_2( "vocabulary_id": "4e61d464-4943-4e97-973a-84673c1aaa87", } ], - "id": "3adc4bb0-faef-42ae-bd67-0ea08918a629", "resources": [ { - "id": "1e7a68da-501a-444e-94f1-3606263e10c8", "name": "All Health Indicators for Zambia", "description": "See resource descriptions below for links to indicator metadata", "format": "csv", @@ -738,7 +682,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "40fbcfcc-e180-455d-9d65-a9f8d1f85643", "name": "Mortality and global health estimates Indicators for Zambia", "description": "*Mortality and global health estimates:*\n[Infant mortality rate (proba", "format": "csv", @@ -747,7 +690,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "018577c8-a397-4a4f-b13c-889c4990d5e2", "name": "Sustainable development goals Indicators for Zambia", "description": "*Sustainable development goals:*\n[Adolescent birth rate (per 1000 wome", "format": "csv", @@ -756,7 +698,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "b42f3a3e-82a9-4fe6-9f0c-81d520ce7666", "name": "Millennium Development Goals (MDGs) Indicators for Zambia", "description": "*Millennium Development Goals (MDGs):*\n[Contraceptive prevalence (%)](", "format": "csv", @@ -765,7 +706,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "b67738ca-812f-41b2-80e3-7c3340648398", "name": "Health systems Indicators for Zambia", "description": "*Health systems:*\n[Median availability of selected generic medicines (", "format": "csv", @@ -774,7 +714,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "a610cbb6-3bd2-4400-9aee-899dbe6b4a98", "name": "Malaria Indicators for Zambia", "description": "*Malaria:*\n[Children aged <5 years sleeping under insecticide-treated ", "format": "csv", @@ -783,7 +722,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "de38982e-ef0d-4e5c-b0a3-16f933eaa808", "name": "Tuberculosis Indicators for Zambia", "description": "*Tuberculosis:*\n[Deaths due to tuberculosis among HIV-negative people ", "format": "csv", @@ -792,7 +730,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "c330fc6e-624d-4d7f-bb46-f5ea67cf5d0a", "name": "Child health Indicators for Zambia", "description": "*Child health:*\n[Children aged <5 years stunted (%)](https://www.who.i", "format": "csv", @@ -801,7 +738,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "9903aea5-d193-4d9d-9018-e98213aef0f5", "name": "Infectious diseases Indicators for Zambia", "description": "*Infectious diseases:*\n[Cholera - number of reported cases](https://ww", "format": "csv", @@ -810,7 +746,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "fb24e4d1-9447-496d-9049-4712e4cd90c6", "name": "World Health Statistics Indicators for Zambia", "description": "*World Health Statistics:*\n[Literacy rate among adults aged >= 15 year", "format": "csv", @@ -819,7 +754,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "98e022a6-e054-41fd-af9a-e4c08ddcc5cd", "name": "Health financing Indicators for Zambia", "description": "*Health financing:*\n[Private prepaid plans as a percentage of private ", "format": "csv", @@ -828,7 +762,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "9a72db07-1bcc-4316-a427-1c208ebb5708", "name": "Public health and environment Indicators for Zambia", "description": "*Public health and environment:*\n[Population using solid fuels (%)](ht", "format": "csv", @@ -837,7 +770,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "873ead28-fff0-49d6-83c4-1ccda19cb0f7", "name": "Substance use and mental health Indicators for Zambia", "description": "*Substance use and mental health:*\n[Fines for violations](https://www.", "format": "csv", @@ -846,7 +778,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "c07999f6-6ab3-4f43-abb5-e3bb245edceb", "name": "Tobacco Indicators for Zambia", "description": "*Tobacco:*\n[Prevalence of smoking any tobacco product among persons ag", "format": "csv", @@ -855,7 +786,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "74e93f31-494c-4950-b891-6db42537e8b4", "name": "Injuries and violence Indicators for Zambia", "description": "*Injuries and violence:*\n[Income level](https://www.who.int/data/gho/i", "format": "csv", @@ -864,7 +794,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "ef319875-76fb-4ff4-bddb-02715a7638df", "name": "HIV/AIDS and other STIs Indicators for Zambia", "description": "*HIV/AIDS and other STIs:*\n[Prevalence of HIV among adults aged 15 to ", "format": "csv", @@ -873,7 +802,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "1d4b9ac1-71ed-4ed3-a120-c4c6fea00a3d", "name": "Nutrition Indicators for Zambia", "description": "*Nutrition:*\n[Early initiation of breastfeeding (%)](https://www.who.i", "format": "csv", @@ -882,7 +810,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "1815f97a-d93c-4e64-909d-39f288880c50", "name": "Urban health Indicators for Zambia", "description": "*Urban health:*\n[Percentage of the total population living in cities >", "format": "csv", @@ -891,7 +818,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "23af66cb-cffb-491f-a51b-c41584b83f3b", "name": "Noncommunicable diseases Indicators for Zambia", "description": "*Noncommunicable diseases:*\n[Prevalence of overweight among adults, BM", "format": "csv", @@ -900,7 +826,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "2beb31ac-873d-47f4-9cfc-75cebd2303b5", "name": "Noncommunicable diseases CCS Indicators for Zambia", "description": "*Noncommunicable diseases CCS:*\n[Existence of operational policy/strat", "format": "csv", @@ -909,7 +834,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "9a14881d-196a-44dc-bcf6-bda199b1137b", "name": "Negelected tropical diseases Indicators for Zambia", "description": "*Negelected tropical diseases:*\n[Number of new reported cases of Burul", "format": "csv", @@ -918,7 +842,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "79b0d05f-9180-492c-b614-687a9203bc1c", "name": "Health Equity Monitor Indicators for Zambia", "description": "*Health Equity Monitor:*\n[Antenatal care coverage - at least one visit", "format": "csv", @@ -927,7 +850,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "4b4defd7-ce45-46e3-86ce-15154be90d0c", "name": "Infrastructure Indicators for Zambia", "description": "*Infrastructure:*\n[Total density per 100 000 population: Health posts]", "format": "csv", @@ -936,7 +858,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "29ac1338-b7b5-429f-bb38-455a66e8f6c1", "name": "Essential health technologies Indicators for Zambia", "description": "*Essential health technologies:*\n[Availability of national standards o", "format": "csv", @@ -945,7 +866,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "63ba2693-fe73-4555-9c64-93710f7c7404", "name": "Medical equipment Indicators for Zambia", "description": "*Medical equipment:*\n[Total density per million population: Magnetic R", "format": "csv", @@ -954,7 +874,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "2cdc3198-4ec7-4a25-86ee-8cdc80b670fc", "name": "Demographic and socioeconomic statistics Indicators for Zambia", "description": "*Demographic and socioeconomic statistics:*\n[Cellular subscribers (per", "format": "csv", @@ -963,7 +882,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "1028087c-af76-4ab2-ac30-0babb4ef00fc", "name": "Neglected tropical diseases Indicators for Zambia", "description": "*Neglected tropical diseases:*\n[Status of yaws endemicity](https://www", "format": "csv", @@ -972,7 +890,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "0ec8b0b2-1e0d-4338-b22a-e984f8d07f0a", "name": "International Health Regulations (2005) monitoring framework Indicators for Zambia", "description": "*International Health Regulations (2005) monitoring framework:*\n[Legis", "format": "csv", @@ -981,7 +898,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "2fa6877b-5b6d-446f-9ccf-52e2478d86da", "name": "Insecticide resistance Indicators for Zambia", "description": "*Insecticide resistance:*\n[Number of insecticide classes to which resi", "format": "csv", @@ -990,7 +906,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "262fddcb-62eb-42f1-94c6-ed20e6b18650", "name": "Universal Health Coverage Indicators for Zambia", "description": "*Universal Health Coverage:*\n[Cataract surgical coverage of adults age", "format": "csv", @@ -999,7 +914,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "5562627b-b7af-4d4d-a9ca-e4fcd996a180", "name": "Global Observatory for eHealth (GOe) Indicators for Zambia", "description": "*Global Observatory for eHealth (GOe):*\n[National universal health cov", "format": "csv", @@ -1008,7 +922,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "53a1eeda-7a53-4b8c-87c8-c2ea276a52a3", "name": "RSUD: GOVERNANCE, POLICY AND FINANCING : PREVENTION Indicators for Zambia", "description": "*RSUD: GOVERNANCE, POLICY AND FINANCING : PREVENTION:*\n[Government uni", "format": "csv", @@ -1017,7 +930,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "64c1aa0d-fa9a-4387-812e-fa7d00f382c2", "name": "RSUD: GOVERNANCE, POLICY AND FINANCING: TREATMENT Indicators for Zambia", "description": "*RSUD: GOVERNANCE, POLICY AND FINANCING: TREATMENT:*\n[Government unit/", "format": "csv", @@ -1026,7 +938,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "5b82a27c-a461-475b-867b-0afdc93132fd", "name": "RSUD: GOVERNANCE, POLICY AND FINANCING: FINANCING Indicators for Zambia", "description": "*RSUD: GOVERNANCE, POLICY AND FINANCING: FINANCING:*\n[Five-year change", "format": "csv", @@ -1035,7 +946,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "f0f95f26-0701-49ee-a9ac-d1525572f6a5", "name": "RSUD: SERVICE ORGANIZATION AND DELIVERY: TREATMENT SECTORS AND PROVIDERS Indicators for Zambia", "description": "*RSUD: SERVICE ORGANIZATION AND DELIVERY: TREATMENT SECTORS AND PROVID", "format": "csv", @@ -1044,7 +954,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "9b805d00-3de4-48f2-a93f-d859961e9c08", "name": "RSUD: SERVICE ORGANIZATION AND DELIVERY: TREATMENT CAPACITY AND TREATMENT COVERAGE Indicators for Zambia", "description": "*RSUD: SERVICE ORGANIZATION AND DELIVERY: TREATMENT CAPACITY AND TREAT", "format": "csv", @@ -1053,7 +962,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "9667ee27-97e5-4122-9dc6-e39df70eeb08", "name": "RSUD: SERVICE ORGANIZATION AND DELIVERY: PHARMACOLOGICAL TREATMENT Indicators for Zambia", "description": "*RSUD: SERVICE ORGANIZATION AND DELIVERY: PHARMACOLOGICAL TREATMENT:*\n", "format": "csv", @@ -1062,7 +970,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "5e16b87e-5a06-4f7f-badb-3a2e44d64948", "name": "RSUD: SERVICE ORGANIZATION AND DELIVERY: SCREENING AND BRIEF INTERVENTIONS Indicators for Zambia", "description": "*RSUD: SERVICE ORGANIZATION AND DELIVERY: SCREENING AND BRIEF INTERVEN", "format": "csv", @@ -1071,7 +978,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "d79c3183-471f-4331-94d6-13474171be91", "name": "RSUD: SERVICE ORGANIZATION AND DELIVERY: PREVENTION PROGRAMS AND PROVIDERS Indicators for Zambia", "description": "*RSUD: SERVICE ORGANIZATION AND DELIVERY: PREVENTION PROGRAMS AND PROV", "format": "csv", @@ -1080,7 +986,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "daa99f76-d84b-4c05-b5dc-c9e82fbcb859", "name": "RSUD: SERVICE ORGANIZATION AND DELIVERY: SPECIAL PROGRAMMES AND SERVICES Indicators for Zambia", "description": "*RSUD: SERVICE ORGANIZATION AND DELIVERY: SPECIAL PROGRAMMES AND SERVI", "format": "csv", @@ -1089,7 +994,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "ca1774e9-49c0-42a2-93c0-88fa32bb5adc", "name": "RSUD: HUMAN RESOURCES Indicators for Zambia", "description": "*RSUD: HUMAN RESOURCES:*\n[Health professionals providing treatment for", "format": "csv", @@ -1098,7 +1002,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "d244efc2-aa38-4197-b594-9bbd2b44bd9a", "name": "RSUD: INFORMATION SYSTEMS Indicators for Zambia", "description": "*RSUD: INFORMATION SYSTEMS:*\n[Epidemiological data collection for subs", "format": "csv", @@ -1107,7 +1010,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "ee06d52b-5c15-4a0c-8d00-830215aeff03", "name": "RSUD: YOUTH Indicators for Zambia", "description": "*RSUD: YOUTH:*\n[Epidemiological data collection system for substance u", "format": "csv", @@ -1116,7 +1018,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "1ef02dd7-07cc-44f9-83ee-fc945f6d22ea", "name": "FINANCIAL PROTECTION Indicators for Zambia", "description": "*FINANCIAL PROTECTION:*\n[Population with household expenditures on hea", "format": "csv", @@ -1125,7 +1026,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "913c96a9-48f9-4e4b-b11b-8ca019fdfd90", "name": "Noncommunicable diseases and mental health Indicators for Zambia", "description": "*Noncommunicable diseases and mental health:*\n[Number of deaths attrib", "format": "csv", @@ -1134,7 +1034,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "ccb28805-c35a-4393-a7eb-6653c73cbde7", "name": "Health workforce Indicators for Zambia", "description": "*Health workforce:*\n[Medical doctors (per 10 000 population)](https://", "format": "csv", @@ -1143,7 +1042,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "12fdb6cb-5e00-45b5-a6b5-611847866440", "name": "Neglected Tropical Diseases Indicators for Zambia", "description": "*Neglected Tropical Diseases:*\n[Number of new leprosy cases](https://w", "format": "csv", @@ -1152,7 +1050,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "ad2d521e-3a0e-451e-977c-bcbf125cec55", "name": "QuickCharts Indicators for Zambia", "description": "Cut down data for QuickCharts", "format": "csv", @@ -1161,7 +1058,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "19caff60-fd38-42b4-a17a-63abfaf0911e", "name": "TOBACCO Indicators for Zambia", "description": "*TOBACCO:*\n[Monitor](https://www.who.int/data/gho/indicator-metadata-r", "format": "csv", @@ -1170,7 +1066,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "b078ed42-079b-4a7c-b699-9c7894b87d4f", "name": "UHC Indicators for Zambia", "description": "*UHC:*\n[Population in malaria-endemic areas who slept under an insecti", "format": "csv", @@ -1179,7 +1074,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "61a61d4a-a50c-4248-bf3d-15c726ced591", "name": "ICD Indicators for Zambia", "description": "*ICD:*\n[ICD-11 implementation progress level](https://web-prod.who.int", "format": "csv", @@ -1188,7 +1082,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "beea81a0-dea1-45a3-891c-36d48a2c8bde", "name": "SEXUAL AND REPRODUCTIVE HEALTH Indicators for Zambia", "description": "*SEXUAL AND REPRODUCTIVE HEALTH:*\n[Institutional Births (birth taken p", "format": "csv", @@ -1197,7 +1090,6 @@ def test_update_logic_2( "url": "updated_by_file_upload_step", }, { - "id": "9ed8a43b-d396-4a6f-948f-5bfe4c2fbba5", "name": "Immunization Indicators for Zambia", "description": "*Immunization:*\n[Proportion of vaccination cards seen (%)](https://www", "format": "csv", @@ -1224,17 +1116,17 @@ def test_update_logic_3( self.add_dataset_resources(dataset, resources_yaml, include=[0, 1, 2]) dataset.old_data = new_dataset.data dataset.old_data["resources"] = new_dataset.resources - results = dataset._dataset_merge_hdx_update( + results = dataset._dataset_hdx_update( update_resources=True, match_resources_by_metadata=True, - keys_to_delete=list(), + keys_to_delete=[], remove_additional_resources=True, match_resource_order=False, create_default_views=False, hxl_update=False, test=True, ) - assert results["filter"] == list() + assert results["filter"] == [] update = results["update"] del update["updated_by_script"] assert update == { @@ -1258,10 +1150,8 @@ def test_update_logic_3( "vocabulary_id": "4e61d464-4943-4e97-973a-84673c1aaa87", } ], - "id": "3adc4bb0-faef-42ae-bd67-0ea08918a629", "resources": [ { - "id": "1e7a68da-501a-444e-94f1-3606263e10c8", "name": "All Health Indicators for Zambia", "description": "See resource descriptions below for links to indicator metadata", "format": "csv", diff --git a/tests/hdx/data/test_user.py b/tests/hdx/data/test_user.py index 9566f2a1..2c3e8de9 100755 --- a/tests/hdx/data/test_user.py +++ b/tests/hdx/data/test_user.py @@ -543,7 +543,7 @@ def test_get_all_users(self, configuration, post_list, mocksmtp): } with pytest.raises(ValueError): User.email_users( - list(), + [], TestUser.subject, TestUser.text_body, sender=TestUser.sender, diff --git a/tests/hdx/data/test_vocabulary.py b/tests/hdx/data/test_vocabulary.py index e3be33b6..b8778fa7 100755 --- a/tests/hdx/data/test_vocabulary.py +++ b/tests/hdx/data/test_vocabulary.py @@ -2250,7 +2250,7 @@ def vocabulary_delete(url, datadict): return vocabulary_mockshow(url, datadict) if "update" in url: resultdictcopy = copy.deepcopy(resultdict) - resultdictcopy["tags"] = list() + resultdictcopy["tags"] = [] result = json.dumps(resultdictcopy) return MockResponse( 200, @@ -2662,22 +2662,22 @@ def test_tag_mappings(self, configuration, read): } assert Vocabulary.get_mapped_tag("refugee") == (["refugees"], list()) assert Vocabulary.get_mapped_tag("monitoring") == ( - list(), + [], ["monitoring"], ) tags_dict["refugee"]["Action to Take"] = "ERROR" - assert Vocabulary.get_mapped_tag("refugee") == (list(), list()) + assert Vocabulary.get_mapped_tag("refugee") == ([], list()) refugeesdict = copy.deepcopy(tags_dict["refugees"]) del tags_dict["refugees"] assert Vocabulary.get_mapped_tag("refugees") == ( ["refugees"], - list(), + [], ) # tag is in CKAN approved list but not tag cleanup spreadsheet tags_dict["refugees"] = refugeesdict Vocabulary.get_approved_vocabulary().remove_tag("refugees") assert Vocabulary.get_mapped_tag("refugees") == ( - list(), - list(), + [], + [], ) # tag is not in CKAN approved list but is in tag cleanup spreadsheet Vocabulary._approved_vocabulary = None Vocabulary.get_approved_vocabulary()