Skip to content

Commit cccb5e8

Browse files
jnussbaumjnussbaum
authored
fix(xmlupload): prevent crash with incremental option (DEV-811) (#197)
* actual bugfix * write e2e test for incremental xmlupload * delete unused files. tmp folder should be empty in the VCS * create folder testdata/tmp during setUp()
1 parent d3efde8 commit cccb5e8

10 files changed

Lines changed: 65 additions & 158 deletions

‎.gitignore‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,7 @@ lists.json
6666
out.json
6767
id2iri_*
6868
**/~$*.*
69-
testdata/tmp/_*.json
70-
testdata/tmp/_*.xml
69+
testdata/tmp/
7170
testdata/test-list.json
7271

7372
# for testing in development

‎knora/dsplib/utils/xml_upload.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -880,6 +880,9 @@ def xml_upload(input_file: str, server: str, user: str, password: str, imgdir: s
880880
# temporarily remove circular references, but only if not an incremental upload
881881
if not incremental:
882882
resources, stashed_xml_texts, stashed_resptr_props = remove_circular_references(resources, verbose)
883+
else:
884+
stashed_xml_texts = dict()
885+
stashed_resptr_props = dict()
883886

884887
sipi_server = Sipi(sipi, con.get_token())
885888

‎test/e2e/test_tools.py‎

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
"""This test class tests the basic functionalities of dsp-tools"""
22
import json
33
import unittest
4+
import os
5+
import datetime
46

57
from knora.dsplib.utils import excel_to_json_lists
68
from knora.dsplib.utils.excel_to_json_lists import list_excel2json
@@ -16,6 +18,9 @@
1618
class TestTools(unittest.TestCase):
1719
server = 'http://0.0.0.0:3333'
1820
user = 'root@example.com'
21+
password = 'test'
22+
imgdir = '.'
23+
sipi = 'http://0.0.0.0:1024'
1924
test_onto_file = 'testdata/test-onto.json'
2025
test_list_file = 'testdata/test-list.json'
2126
test_data_file = 'testdata/test-data.xml'
@@ -24,6 +29,7 @@ def setUp(self) -> None:
2429
"""Is executed before each test"""
2530
excel_to_json_lists.list_of_lists = []
2631
excel_to_json_lists.cell_names = []
32+
os.makedirs('testdata/tmp', exist_ok=True)
2733

2834
def tearDown(self) -> None:
2935
"""Is executed after each test"""
@@ -173,19 +179,36 @@ def test_xml_upload(self) -> None:
173179
xml_upload(input_file=self.test_data_file,
174180
server=self.server,
175181
user=self.user,
176-
password='test',
177-
imgdir='.',
178-
sipi='http://0.0.0.0:1024',
182+
password=self.password,
183+
imgdir=self.imgdir,
184+
sipi=self.sipi,
179185
verbose=False,
180186
validate_only=False,
181187
incremental=False)
182188

183-
def test_id_to_iri(self) -> None:
189+
mapping_file = ''
190+
for mapping in [x for x in os.scandir('.') if x.name.startswith('id2iri_test-data_mapping_')]:
191+
delta = datetime.datetime.now() - datetime.datetime.fromtimestamp(mapping.stat().st_mtime_ns / 1000000000)
192+
if delta.seconds < 15:
193+
mapping_file = mapping.name
194+
184195
id_to_iri(xml_file='testdata/test-id2iri-data.xml',
185-
json_file='testdata/test-id2iri-mapping.json',
196+
json_file=mapping_file,
186197
out_file='testdata/tmp/_test-id2iri-replaced.xml',
187198
verbose=False)
188199

200+
xml_upload(
201+
input_file='testdata/tmp/_test-id2iri-replaced.xml',
202+
server=self.server,
203+
user=self.user,
204+
password=self.password,
205+
imgdir=self.imgdir,
206+
sipi=self.sipi,
207+
verbose=False,
208+
validate_only=False,
209+
incremental=True
210+
)
211+
189212

190213
if __name__ == '__main__':
191214
unittest.main()

‎test/unittests/test_excel_to_json_lists.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@
1111

1212
class TestExcelToJSONList(unittest.TestCase):
1313

14+
def setUp(self) -> None:
15+
"""Is executed before each test"""
16+
os.makedirs('testdata/tmp', exist_ok=True)
17+
1418
def test_excel2jsonlist(self) -> None:
1519
# check that the output file was created
1620
excelfolder = "testdata/lists"

‎test/unittests/test_excel_to_resource.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@
99

1010
class TestExcelToResource(unittest.TestCase):
1111

12+
def setUp(self) -> None:
13+
"""Is executed before each test"""
14+
os.makedirs('testdata/tmp', exist_ok=True)
15+
1216
def test_excel2json(self) -> None:
1317
in_file = "testdata/Resources.xlsx"
1418
out_file = "testdata/tmp/_out_res.json"

‎test/unittests/test_id_to_iri.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
"""Unit tests for id to iri mapping"""
22

33
import unittest
4+
import os
45

56
from lxml import etree
67

@@ -10,6 +11,10 @@
1011
class TestIdToIri(unittest.TestCase):
1112
out_file = 'testdata/tmp/_test-id2iri-replaced.xml'
1213

14+
def setUp(self) -> None:
15+
"""Is executed before each test"""
16+
os.makedirs('testdata/tmp', exist_ok=True)
17+
1318
def test_invalid_xml_file_name(self):
1419
with self.assertRaises(SystemExit) as cm:
1520
id_to_iri(xml_file='test.xml',

‎testdata/test-id2iri-data.xml‎

Lines changed: 18 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,19 @@
11
<?xml version='1.0' encoding='utf-8'?>
22

3-
<knora xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4-
xmlns="https://dasch.swiss/schema"
5-
xsi:schemaLocation="https://dasch.swiss/schema https://raw.githubusercontent.com/dasch-swiss/dsp-tools/main/knora/dsplib/schemas/data.xsd"
6-
shortcode="082E"
7-
default-ontology="rosetta">
3+
<knora xmlns="https://dasch.swiss/schema"
4+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
5+
xsi:schemaLocation="https://dasch.swiss/schema ../knora/dsplib/schemas/data.xsd"
6+
shortcode="4123"
7+
default-ontology="testonto">
88

99
<!-- :permissions see https://docs.dasch.swiss/DSP-API/05-internals/design/api-admin/administration/#permissions -->
1010
<permissions id="res-default">
11-
<allow group="UnknownUser">V</allow>
11+
<allow group="UnknownUser">RV</allow>
1212
<allow group="KnownUser">V</allow>
1313
<allow group="Creator">CR</allow>
1414
<allow group="ProjectAdmin">CR</allow>
1515
</permissions>
1616
<permissions id="res-restricted">
17-
<allow group="UnknownUser">V</allow>
1817
<allow group="KnownUser">V</allow>
1918
<allow group="Creator">CR</allow>
2019
<allow group="ProjectAdmin">CR</allow>
@@ -26,39 +25,25 @@
2625
<allow group="ProjectAdmin">CR</allow>
2726
</permissions>
2827
<permissions id="prop-restricted">
29-
<allow group="UnknownUser">V</allow>
3028
<allow group="KnownUser">V</allow>
3129
<allow group="Creator">CR</allow>
3230
<allow group="ProjectAdmin">CR</allow>
3331
</permissions>
3432

35-
<!-- :Image2D -->
36-
<resource label="Deluge2" restype=":Image2D" id="img_obj_1" permissions="res-default">
37-
<bitstream>Images/Danby-deluge.jpg</bitstream>
38-
<text-prop name=":hasTitle">
39-
<text permissions="prop-default" encoding="utf8">The Deluge</text>
33+
<resource label="TestThing2Incremental"
34+
restype=":TestThing2"
35+
id="TestThing2Incremental">
36+
<text-prop name=":hasText">
37+
<text encoding="xml">
38+
Text with XML standoff link to resource <a class="salsah-link" href="IRI:obj_0000:IRI">obj_0000</a>
39+
and <a class="salsah-link" href="IRI:obj_0011:IRI">obj_0011</a>
40+
</text>
4041
</text-prop>
41-
<resptr-prop name=":hasCreator">
42-
<resptr permissions="prop-default">person_2</resptr>
42+
<resptr-prop name=":hasTestThing">
43+
<resptr permissions="prop-default">obj_0001</resptr>
44+
<resptr permissions="prop-default">obj_0011</resptr>
4345
</resptr-prop>
44-
<date-prop name=":hasDate">
45-
<date permissions="prop-default">GREGORIAN:CE:1877:CE:1879</date>
46-
</date-prop>
47-
<text-prop name=":hasDescription">
48-
<text permissions="prop-default" encoding="utf8">Oil paint on canvas.</text>
49-
</text-prop>
50-
<text-prop name=":hasCopyright">
51-
<text permissions="prop-default" encoding="utf8">The Deluge</text>
52-
</text-prop>
53-
<resptr-prop name=":inInstitution">
54-
<resptr permissions="prop-default">institution_2</resptr>
55-
</resptr-prop>
56-
<geoname-prop name=":hasLocation">
57-
<geoname permissions="prop-default" comment="London">6695209</geoname>
58-
</geoname-prop>
59-
<uri-prop name=":hasExternalLink">
60-
<uri permissions="prop-default">https://www.tate.org.uk/art/artworks/danby-the-deluge-t01337</uri>
61-
</uri-prop>
6246
</resource>
6347

48+
6449
</knora>

‎testdata/test-id2iri-mapping.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
2-
"person_2": "http://rdfh.ch/082E/ylRvrg7tQI6aVpcTJbVrwg",
3-
"institution_2": "http://rdfh.ch/082E/JK63OpYWTDWNYVOYFN7FdQ"
2+
"obj_0001": "http://rdfh.ch/082E/ylRvrg7tQI6aVpcTJbVrwg",
3+
"obj_0011": "http://rdfh.ch/082E/JK63OpYWTDWNYVOYFN7FdQ"
44
}

‎testdata/test-id2iri-replaced.xml‎

Lines changed: 0 additions & 58 deletions
This file was deleted.

‎testdata/tmp/_test-id2iri-replaced.xml‎

Lines changed: 0 additions & 58 deletions
This file was deleted.

0 commit comments

Comments
 (0)