From d7538c78fd6562d7f266dc6dc5f0c75b17add8c2 Mon Sep 17 00:00:00 2001 From: XnpioChV Date: Mon, 19 Jun 2023 14:37:36 -0500 Subject: [PATCH 1/3] feat: import/export Taxonomy API functions --- openedx_tagging/core/tagging/api.py | 193 +++++++++++++++++- .../core/fixtures/tagging.yaml | 44 ++-- .../openedx_tagging/core/tagging/test_api.py | 178 +++++++++++++++- 3 files changed, 390 insertions(+), 25 deletions(-) diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index 6b9bf2a0..bbe18361 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -10,13 +10,28 @@ Please look at the models.py file for more information about the kinds of data are stored in this app. """ +import csv +import json +from enum import Enum +from io import StringIO, BytesIO, TextIOWrapper from typing import List, Type +from django.db import transaction from django.db.models import QuerySet +from django.core.exceptions import ObjectDoesNotExist from django.utils.translation import gettext_lazy as _ from .models import ObjectTag, Tag, Taxonomy +csv_fields = ['id', 'name', 'parent_id', 'parent_name'] + +class TaxonomyDataFormat(Enum): + """ + Formats used to export and import Taxonomies + """ + CSV = 'CSV' + JSON = 'JSON' + def create_taxonomy( name, @@ -29,6 +44,7 @@ def create_taxonomy( """ Creates, saves, and returns a new Taxonomy with the given attributes. """ + return Taxonomy.objects.create( name=name, description=description, @@ -105,5 +121,180 @@ def tag_object( Raised ValueError if the proposed tags are invalid for this taxonomy. Preserves existing (valid) tags, adds new (valid) tags, and removes omitted (or invalid) tags. """ - return taxonomy.tag_object(tags, object_id, object_type) + + +def import_tags(taxonomy: Taxonomy, tags: BytesIO, format: TaxonomyDataFormat, replace=False): + """ + Imports the hierarchical tags from the given blob into the Taxonomy. + The blob can be CSV or JSON format. + + If replace, then removes any existing child Tags linked to this taxonomy before performing the import. + """ + + # Validations + if taxonomy.allow_free_text: + raise ValueError( + _( + f"Invalid taxonomy ({taxonomy.id}): You can't import free-from tags taxonomies" + ) + ) + if format not in TaxonomyDataFormat.__members__.values(): + raise ValueError( + _( + f"Invalid format: {format}" + ) + ) + + # Read file and build the tags data to be uploaded + try: + tags_data = {} + tags.seek(0) + if format == TaxonomyDataFormat.CSV: + text_tags = TextIOWrapper(tags, encoding='utf-8') + csv_reader = csv.DictReader(text_tags) + header_fields = csv_reader.fieldnames + if csv_fields != header_fields: + raise ValueError( + _( + f"Invalid CSV header: {header_fields}. Must be: {csv_fields}." + ) + ) + tags_data = list(csv_reader) + else: + # TaxonomyDataFormat.JSON + tags_data = json.load(tags) + if 'tags' not in tags_data: + raise ValueError( + _( + f"Invalid JSON format: Missing 'tags' list." + ) + ) + tags_data = tags_data.get('tags') + except ValueError as e: + raise e + finally: + tags.close() + + + new_tags = [] + updated_tags = [] + + def create_update_tag(tag): + """ + Function to create a new Tag or update an existing one. + + This function keeps a creation/update history with `new_tags` and `updated_tags`, + a same tag can't be created/updated in a same taxonomy import. + Also, recursively, creates the parents of the `tag`. + + Returns the created/updated Tag. + Raise KeyError if 'id' or 'name' don't exist on `tag` + """ + + tag_id = tag['id'] + tag_name = tag['name'] + tag_parent_id = tag.get('parent_id') + tag_parent_name = tag.get('parent_name') + + # Check if the tag has not already been created or updated + if tag_id not in new_tags and tag_id not in updated_tags: + try: + # Update tag + tag_instance = Tag.objects.get(external_id=tag_id) + tag_instance.value = tag_name + + if tag_instance.parent and (not tag_parent_id or not tag_parent_name): + # if there is no parent in the data import + tag_instance.parent = None + updated_tags.append(tag_id) + except ObjectDoesNotExist: + # Create tag + tag_instance = Tag( + taxonomy=taxonomy, + value=tag_name, + external_id=tag_id, + ) + new_tags.append(tag_id) + + if tag_parent_id and tag_parent_name: + # Parent creation/update + parent = create_update_tag({'id': tag_parent_id, 'name': tag_parent_name}) + tag_instance.parent = parent + + tag_instance.save() + return tag_instance + else: + # Returns the created/updated tag from history + return Tag.objects.get(external_id=tag_id) + + # Create and update tags + with transaction.atomic(): + # Delete all old Tags linked to the taxonomy + if replace: + Tag.objects.filter(taxonomy=taxonomy).delete() + + for tag in tags_data: + try: + create_update_tag(tag) + except KeyError as e: + key = e.args[0] + raise ValueError( + _( + f"Invalid JSON format: Missing '{key}' on a tag ({tag})" + ) + ) + resync_tags() + +def export_tags(taxonomy: Taxonomy, format: TaxonomyDataFormat) -> str: + """ + Creates a blob string describing all the tags in the given Taxonomy. + The output format can be CSV or JSON. + """ + + # Validations + if taxonomy.allow_free_text: + raise ValueError( + _( + f"Invalid taxonomy ({taxonomy.id}): You can't export free-from tags taxonomies" + ) + ) + if format not in TaxonomyDataFormat.__members__.values(): + raise ValueError( + _( + f"Invalid format: {format}" + ) + ) + + # Build list of tags in a dictionary format + tags = get_tags(taxonomy) + result = [] + for tag in tags: + result_tag = { + 'id': tag.external_id or tag.id, + 'name': tag.value, + } + if tag.parent: + result_tag['parent_id'] = tag.parent.external_id or tag.parent.id + result_tag['parent_name'] = tag.parent.value + result.append(result_tag) + + # Convert dictonary into the output format + if format == TaxonomyDataFormat.CSV: + with StringIO() as csv_buffer: + csv_writer = csv.DictWriter(csv_buffer, fieldnames=csv_fields) + csv_writer.writeheader() + + for tag in result: + csv_writer.writerow(tag) + + csv_string = csv_buffer.getvalue() + return csv_string + else: + # TaxonomyDataFormat.JSON + json_result = { + 'name': taxonomy.name, + 'description': taxonomy.description, + 'tags': result + } + return json.dumps(json_result) diff --git a/tests/openedx_tagging/core/fixtures/tagging.yaml b/tests/openedx_tagging/core/fixtures/tagging.yaml index 50b9459b..7db85aef 100644 --- a/tests/openedx_tagging/core/fixtures/tagging.yaml +++ b/tests/openedx_tagging/core/fixtures/tagging.yaml @@ -4,152 +4,152 @@ taxonomy: 1 parent: null value: Bacteria - external_id: null + external_id: tag_1 - model: oel_tagging.tag pk: 2 fields: taxonomy: 1 parent: null value: Archaea - external_id: null + external_id: tag_2 - model: oel_tagging.tag pk: 3 fields: taxonomy: 1 parent: null value: Eukaryota - external_id: null + external_id: tag_3 - model: oel_tagging.tag pk: 4 fields: taxonomy: 1 parent: 1 value: Eubacteria - external_id: null + external_id: tag_4 - model: oel_tagging.tag pk: 5 fields: taxonomy: 1 parent: 1 value: Archaebacteria - external_id: null + external_id: tag_5 - model: oel_tagging.tag pk: 6 fields: taxonomy: 1 parent: 2 value: DPANN - external_id: null + external_id: tag_6 - model: oel_tagging.tag pk: 7 fields: taxonomy: 1 parent: 2 value: Euryarchaeida - external_id: null + external_id: tag_7 - model: oel_tagging.tag pk: 8 fields: taxonomy: 1 parent: 2 value: Proteoarchaeota - external_id: null + external_id: tag_8 - model: oel_tagging.tag pk: 9 fields: taxonomy: 1 parent: 3 value: Animalia - external_id: null + external_id: tag_9 - model: oel_tagging.tag pk: 10 fields: taxonomy: 1 parent: 3 value: Plantae - external_id: null + external_id: tag_10 - model: oel_tagging.tag pk: 11 fields: taxonomy: 1 parent: 3 value: Fungi - external_id: null + external_id: tag_11 - model: oel_tagging.tag pk: 12 fields: taxonomy: 1 parent: 3 value: Protista - external_id: null + external_id: tag_12 - model: oel_tagging.tag pk: 13 fields: taxonomy: 1 parent: 3 value: Monera - external_id: null + external_id: tag_13 - model: oel_tagging.tag pk: 14 fields: taxonomy: 1 parent: 9 value: Arthropoda - external_id: null + external_id: tag_14 - model: oel_tagging.tag pk: 15 fields: taxonomy: 1 parent: 9 value: Chordata - external_id: null + external_id: tag_15 - model: oel_tagging.tag pk: 16 fields: taxonomy: 1 parent: 9 value: Gastrotrich - external_id: null + external_id: tag_16 - model: oel_tagging.tag pk: 17 fields: taxonomy: 1 parent: 9 value: Cnidaria - external_id: null + external_id: tag_17 - model: oel_tagging.tag pk: 18 fields: taxonomy: 1 parent: 9 value: Ctenophora - external_id: null + external_id: tag_18 - model: oel_tagging.tag pk: 19 fields: taxonomy: 1 parent: 9 value: Placozoa - external_id: null + external_id: tag_19 - model: oel_tagging.tag pk: 20 fields: taxonomy: 1 parent: 9 value: Porifera - external_id: null + external_id: tag_20 - model: oel_tagging.tag pk: 21 fields: taxonomy: 1 parent: 15 value: Mammalia - external_id: null + external_id: tag_21 - model: oel_tagging.taxonomy pk: 1 fields: name: Life on Earth - description: null + description: This taxonomy contains the Kingdoms of the Earth enabled: true required: false allow_multiple: false diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index 7a418e68..a68d487f 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -1,7 +1,10 @@ """ Test the tagging APIs """ -from unittest.mock import patch +from io import BytesIO +import json +import ddt +from unittest.mock import patch from django.test.testcases import TestCase import openedx_tagging.core.tagging.api as tagging_api @@ -9,12 +12,15 @@ from .test_models import TestTagTaxonomyMixin - +@ddt.ddt class TestApiTagging(TestTagTaxonomyMixin, TestCase): """ Test the Tagging API methods. """ + def get_tag(self, tags, tag_value): + return next((item for item in tags if item.value == tag_value), None) + def test_create_taxonomy(self): params = { "name": "Difficulty", @@ -188,3 +194,171 @@ def test_tag_object(self): assert object_tag.name == self.taxonomy.name assert object_tag.object_id == "biology101" assert object_tag.object_type == "course" + + def test_import_tags_csv(self): + csv_data = "id,name,parent_id,parent_name\n1,Tag 1,,\n2,Tag 2,1,Tag 1\n" + csv_file = BytesIO(csv_data.encode()) + + taxonomy = tagging_api.create_taxonomy("CSV_Taxonomy") + tagging_api.import_tags(taxonomy, csv_file, tagging_api.TaxonomyDataFormat.CSV) + tags = tagging_api.get_tags(taxonomy) + + # Assert that the tags are imported correctly + self.assertEqual(len(tags), 2) + item = self.get_tag(tags, 'Tag 1') + self.assertIsNotNone(item) + item = self.get_tag(tags, 'Tag 2') + self.assertIsNotNone(item) + self.assertEqual(item.parent.value, 'Tag 1') + + def test_import_tags_json(self): + json_data = {"tags": [ + {"id": "1", "name": "Tag 1"}, + {"id": "2", "name": "Tag 2", "parent_id": "1", "parent_name": "Tag 1"} + ]} + json_file = BytesIO(json.dumps(json_data).encode()) + + taxonomy = tagging_api.create_taxonomy("JSON_Taxonomy") + tagging_api.import_tags(taxonomy, json_file, tagging_api.TaxonomyDataFormat.JSON) + tags = tagging_api.get_tags(taxonomy) + + # Assert that the tags are imported correctly + self.assertEqual(len(tags), 2) + item = self.get_tag(tags, 'Tag 1') + self.assertIsNotNone(item) + item = self.get_tag(tags, 'Tag 2') + self.assertIsNotNone(item) + self.assertEqual(item.parent.value, 'Tag 1') + + def test_import_tags_replace(self): + self.assertEqual(len(tagging_api.get_tags(self.taxonomy)), 20) + + json_data = {"tags": [{'id': "tag_1", "name": "Bacteria"},{'id': "tag_2", "name": "Archaea"}]} + json_file = BytesIO(json.dumps(json_data).encode()) + + tagging_api.import_tags(self.taxonomy, json_file, tagging_api.TaxonomyDataFormat.JSON, replace=True) + tags = tagging_api.get_tags(self.taxonomy) + self.assertEqual(len(tags), 2) + item = self.get_tag(tags, 'Bacteria') + self.assertIsNotNone(item) + item = self.get_tag(tags, 'Archaea') + self.assertIsNotNone(item) + + def test_import_tags_update(self): + self.assertEqual(len(tagging_api.get_tags(self.taxonomy)), 20) + + json_data = {"tags": [ + # Name update + {"id": "tag_1", "name": "Bacteria 2.0"}, + # Parent update + {"id": "tag_4", "name": "Eubacteria 2.0", "parent_id": "tag_2", "parent_name": "Archaea"}, + # Parent deletion + {"id": "tag_5", "name": "Archaebacteria 2.0"}, + # New Tag + {"id": "tag_22", "name": "My new tag 2.0", "parent_id": "tag_1", "parent_name": "Bacteria 2.0"}, + ]} + + json_file = BytesIO(json.dumps(json_data).encode()) + tagging_api.import_tags(self.taxonomy, json_file, tagging_api.TaxonomyDataFormat.JSON) + tags = tagging_api.get_tags(self.taxonomy) + self.assertEqual(len(tags), 21) + + # Check name update + item = self.get_tag(tags, 'Bacteria 2.0') + self.assertIsNotNone(item) + + # Check parent update + item = self.get_tag(tags, 'Eubacteria 2.0') + self.assertIsNotNone(item) + + # Check parent deletion + self.assertEqual(item.parent.value, 'Archaea') + item = self.get_tag(tags, 'Archaebacteria 2.0') + self.assertIsNotNone(item) + self.assertIsNone(item.parent) + + # Check new tag + item = self.get_tag(tags, 'My new tag 2.0') + self.assertIsNotNone(item) + self.assertEqual(item.parent.value, 'Bacteria 2.0') + + # Check existing tag + item = self.get_tag(tags, 'Porifera') + self.assertIsNotNone(item) + + def test_import_tags_validations(self): + invalid_csv_data = "id,name,tag_parent_id,tag_parent_name\n1,Tag 1,,\n2,Tag 2,1,Tag 1\n" + invalid_csv_file = BytesIO(invalid_csv_data.encode()) + taxonomy = tagging_api.create_taxonomy("Validations_Taxonomy") + taxonomy_free_text = tagging_api.create_taxonomy("Free_Taxonomy", allow_free_text=True) + + with self.assertRaises(ValueError): + tagging_api.import_tags(taxonomy_free_text, invalid_csv_file, tagging_api.TaxonomyDataFormat.CSV) + + with self.assertRaises(ValueError): + tagging_api.import_tags(taxonomy, "", "XML") + + with self.assertRaises(ValueError): + tagging_api.import_tags(taxonomy, invalid_csv_file, tagging_api.TaxonomyDataFormat.CSV) + + @ddt.data( + # Invalid json format + {"taxonomy_tags": [ + {"id": "1", "name": "Tag 1"}, + {"id": "2", "name": "Tag 2", "parent_id": "1", "parent_name": "Tag 1"} + ]}, + # Invalid 'id' key + {"tags": [ + {"tag_id": "1", "name": "Tag 1"}, + ]}, + # Invalid 'name' key + {"tags": [ + {"id": "1", "tag_name": "Tag 1"}, + ]} + ) + def test_import_tags_json_validations(self, json_data): + json_file = BytesIO(json.dumps(json_data).encode()) + taxonomy = tagging_api.create_taxonomy("Validations_Taxonomy") + + with self.assertRaises(ValueError): + tagging_api.import_tags(taxonomy, json_file, tagging_api.TaxonomyDataFormat.JSON) + + def test_export_tags_json(self): + json_data = { + "name": "Life on Earth", + "description": "This taxonomy contains the Kingdoms of the Earth", + "tags": [ + {'id': "tag_2", "name": "Archaea"}, + {'id': "tag_1", "name": "Bacteria"}, + {'id': "tag_4", "name": "Euryarchaeida", "parent_id": "tag_2", "parent_name": "Archaea"}, + {'id': "tag_3", "name": "Eubacteria", "parent_id": "tag_1", "parent_name": "Bacteria"}, + ] + } + json_file = BytesIO(json.dumps(json_data).encode()) + + # Import and replace + tagging_api.import_tags(self.taxonomy, json_file, tagging_api.TaxonomyDataFormat.JSON, replace=True) + + result = tagging_api.export_tags(self.taxonomy, tagging_api.TaxonomyDataFormat.JSON) + self.assertEqual(json.loads(result), json_data) + + def test_export_tags_csv(self): + csv_data = "id,name,parent_id,parent_name\r\ntag_2,Archaea,,\r\ntag_1,Bacteria,,\r\n" \ + "tag_4,Euryarchaeida,tag_2,Archaea\r\ntag_3,Eubacteria,tag_1,Bacteria\r\n" + + csv_file = BytesIO(csv_data.encode()) + + # Import and replace + tagging_api.import_tags(self.taxonomy, csv_file, tagging_api.TaxonomyDataFormat.CSV, replace=True) + result = tagging_api.export_tags(self.taxonomy, tagging_api.TaxonomyDataFormat.CSV) + self.assertEqual(result, csv_data) + + def test_export_tags_validation(self): + taxonomy_free_text = tagging_api.create_taxonomy("Free_Taxonomy", allow_free_text=True) + taxonomy_xml = tagging_api.create_taxonomy("XML_Taxonomy") + + with self.assertRaises(ValueError): + tagging_api.export_tags(taxonomy_free_text, tagging_api.TaxonomyDataFormat.JSON) + + with self.assertRaises(ValueError): + tagging_api.export_tags(taxonomy_xml, "XML") From 577f44bcf13a337c1853656b3067a0dfec4c8dd1 Mon Sep 17 00:00:00 2001 From: XnpioChV Date: Wed, 21 Jun 2023 11:55:10 -0500 Subject: [PATCH 2/3] fix: Nits on import/export functions and on tests --- openedx_tagging/core/tagging/api.py | 28 +++++++++---------- .../openedx_tagging/core/tagging/test_api.py | 11 +++++--- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index bbe18361..590efb7a 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -136,13 +136,7 @@ def import_tags(taxonomy: Taxonomy, tags: BytesIO, format: TaxonomyDataFormat, r if taxonomy.allow_free_text: raise ValueError( _( - f"Invalid taxonomy ({taxonomy.id}): You can't import free-from tags taxonomies" - ) - ) - if format not in TaxonomyDataFormat.__members__.values(): - raise ValueError( - _( - f"Invalid format: {format}" + f"Invalid taxonomy ({taxonomy.id}): You cannot import into a free-form taxonomy." ) ) @@ -161,8 +155,7 @@ def import_tags(taxonomy: Taxonomy, tags: BytesIO, format: TaxonomyDataFormat, r ) ) tags_data = list(csv_reader) - else: - # TaxonomyDataFormat.JSON + elif format == TaxonomyDataFormat.JSON: tags_data = json.load(tags) if 'tags' not in tags_data: raise ValueError( @@ -171,6 +164,12 @@ def import_tags(taxonomy: Taxonomy, tags: BytesIO, format: TaxonomyDataFormat, r ) ) tags_data = tags_data.get('tags') + else: + raise ValueError( + _( + f"Invalid format: {format}" + ) + ) except ValueError as e: raise e finally: @@ -201,14 +200,14 @@ def create_update_tag(tag): if tag_id not in new_tags and tag_id not in updated_tags: try: # Update tag - tag_instance = Tag.objects.get(external_id=tag_id) + tag_instance = taxonomy.tag_set.get(external_id=tag_id) tag_instance.value = tag_name if tag_instance.parent and (not tag_parent_id or not tag_parent_name): # if there is no parent in the data import tag_instance.parent = None updated_tags.append(tag_id) - except ObjectDoesNotExist: + except Tag.DoesNotExist: # Create tag tag_instance = Tag( taxonomy=taxonomy, @@ -226,7 +225,7 @@ def create_update_tag(tag): return tag_instance else: # Returns the created/updated tag from history - return Tag.objects.get(external_id=tag_id) + return taxonomy.tag_set.get(external_id=tag_id) # Create and update tags with transaction.atomic(): @@ -256,7 +255,7 @@ def export_tags(taxonomy: Taxonomy, format: TaxonomyDataFormat) -> str: if taxonomy.allow_free_text: raise ValueError( _( - f"Invalid taxonomy ({taxonomy.id}): You can't export free-from tags taxonomies" + f"Invalid taxonomy ({taxonomy.id}): You cannot import into a free-form taxonomy." ) ) if format not in TaxonomyDataFormat.__members__.values(): @@ -266,7 +265,7 @@ def export_tags(taxonomy: Taxonomy, format: TaxonomyDataFormat) -> str: ) ) - # Build list of tags in a dictionary format + # Build tags in a dictionary format tags = get_tags(taxonomy) result = [] for tag in tags: @@ -292,6 +291,7 @@ def export_tags(taxonomy: Taxonomy, format: TaxonomyDataFormat) -> str: return csv_string else: # TaxonomyDataFormat.JSON + # Verification is made at the beginning before bringing and assembling tags data. json_result = { 'name': taxonomy.name, 'description': taxonomy.description, diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index a68d487f..5701e0f1 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -213,8 +213,8 @@ def test_import_tags_csv(self): def test_import_tags_json(self): json_data = {"tags": [ - {"id": "1", "name": "Tag 1"}, - {"id": "2", "name": "Tag 2", "parent_id": "1", "parent_name": "Tag 1"} + {"id": "tag_1", "name": "Tag 1"}, + {"id": "tag_2", "name": "Tag 2", "parent_id": "tag_1", "parent_name": "Tag 1"} ]} json_file = BytesIO(json.dumps(json_data).encode()) @@ -288,17 +288,20 @@ def test_import_tags_update(self): def test_import_tags_validations(self): invalid_csv_data = "id,name,tag_parent_id,tag_parent_name\n1,Tag 1,,\n2,Tag 2,1,Tag 1\n" - invalid_csv_file = BytesIO(invalid_csv_data.encode()) taxonomy = tagging_api.create_taxonomy("Validations_Taxonomy") taxonomy_free_text = tagging_api.create_taxonomy("Free_Taxonomy", allow_free_text=True) + # Open the file in each test since it always closes even if there is an error with self.assertRaises(ValueError): + invalid_csv_file = BytesIO(invalid_csv_data.encode()) tagging_api.import_tags(taxonomy_free_text, invalid_csv_file, tagging_api.TaxonomyDataFormat.CSV) with self.assertRaises(ValueError): - tagging_api.import_tags(taxonomy, "", "XML") + invalid_csv_file = BytesIO(invalid_csv_data.encode()) + tagging_api.import_tags(taxonomy, invalid_csv_file, "XML") with self.assertRaises(ValueError): + invalid_csv_file = BytesIO(invalid_csv_data.encode()) tagging_api.import_tags(taxonomy, invalid_csv_file, tagging_api.TaxonomyDataFormat.CSV) @ddt.data( From e5b1698fe93208ab88fc07d4635b007797cb7644 Mon Sep 17 00:00:00 2001 From: XnpioChV Date: Thu, 22 Jun 2023 15:54:33 -0500 Subject: [PATCH 3/3] chore: Improving 'reaplce' functionality of 'import_tags' --- openedx_tagging/core/tagging/api.py | 18 +++++----- .../openedx_tagging/core/tagging/test_api.py | 33 ++++++++++++++++++- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index 590efb7a..46d5e623 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -176,14 +176,13 @@ def import_tags(taxonomy: Taxonomy, tags: BytesIO, format: TaxonomyDataFormat, r tags.close() - new_tags = [] updated_tags = [] def create_update_tag(tag): """ Function to create a new Tag or update an existing one. - This function keeps a creation/update history with `new_tags` and `updated_tags`, + This function keeps a creation/update history with `updated_tags`, a same tag can't be created/updated in a same taxonomy import. Also, recursively, creates the parents of the `tag`. @@ -197,7 +196,7 @@ def create_update_tag(tag): tag_parent_name = tag.get('parent_name') # Check if the tag has not already been created or updated - if tag_id not in new_tags and tag_id not in updated_tags: + if tag_id not in updated_tags: try: # Update tag tag_instance = taxonomy.tag_set.get(external_id=tag_id) @@ -214,7 +213,7 @@ def create_update_tag(tag): value=tag_name, external_id=tag_id, ) - new_tags.append(tag_id) + updated_tags.append(tag_id) if tag_parent_id and tag_parent_name: # Parent creation/update @@ -229,10 +228,6 @@ def create_update_tag(tag): # Create and update tags with transaction.atomic(): - # Delete all old Tags linked to the taxonomy - if replace: - Tag.objects.filter(taxonomy=taxonomy).delete() - for tag in tags_data: try: create_update_tag(tag) @@ -243,7 +238,12 @@ def create_update_tag(tag): f"Invalid JSON format: Missing '{key}' on a tag ({tag})" ) ) - resync_tags() + + # If replace, delete all not updated tags (Not present in the file) + if replace: + taxonomy.tag_set.exclude(external_id__in=updated_tags).delete() + + resync_object_tags(ObjectTag.objects.filter(taxonomy=taxonomy)) def export_tags(taxonomy: Taxonomy, format: TaxonomyDataFormat) -> str: """ diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index 5701e0f1..eb32d198 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -2,9 +2,9 @@ from io import BytesIO import json +from unittest.mock import patch import ddt -from unittest.mock import patch from django.test.testcases import TestCase import openedx_tagging.core.tagging.api as tagging_api @@ -326,6 +326,37 @@ def test_import_tags_json_validations(self, json_data): with self.assertRaises(ValueError): tagging_api.import_tags(taxonomy, json_file, tagging_api.TaxonomyDataFormat.JSON) + def test_import_tags_resync(self): + object_id = 'course_1' + object_tag = ObjectTag( + object_id=object_id, + object_type='course', + taxonomy=self.taxonomy, + tag=self.bacteria, + ) + tagging_api.resync_object_tags([object_tag]) + object_tag = ObjectTag.objects.get(object_id=object_id) + self.assertEqual(object_tag.tag.value, 'Bacteria') + self.assertEqual(object_tag._value, 'Bacteria') + + json_data = {"tags": [{"id": "tag_1", "name": "Bacteria 2.0"}]} + json_file = BytesIO(json.dumps(json_data).encode()) + + # Import + tagging_api.import_tags(self.taxonomy, json_file, tagging_api.TaxonomyDataFormat.JSON) + object_tag = ObjectTag.objects.get(object_id=object_id) + self.assertEqual(object_tag.tag.value, 'Bacteria 2.0') + self.assertEqual(object_tag._value, 'Bacteria 2.0') + + json_data = {"tags": [{"id": "tag_1", "name": "Bacteria 3.0"}]} + json_file = BytesIO(json.dumps(json_data).encode()) + + # Import and replace + tagging_api.import_tags(self.taxonomy, json_file, tagging_api.TaxonomyDataFormat.JSON, replace=True) + object_tag = ObjectTag.objects.get(object_id=object_id) + self.assertEqual(object_tag.tag.value, 'Bacteria 3.0') + self.assertEqual(object_tag._value, 'Bacteria 3.0') + def test_export_tags_json(self): json_data = { "name": "Life on Earth",