Merge 9e5a2e9 into dd8166d

onegov/election_day/formats/election/wabstic_proporz.py

@@ -69,34 +69,99 @@ def line_is_relevant(line, number, district=None):
         return line.sortgeschaeft == number
 
 
-def get_entity_id(line, entities):
-    entity_id = int(line.bfsnrgemeinde or 0)
-    return 0 if entity_id in EXPATS else entity_id
-
-
-def get_candidate_id(line):
-    return str(int(line.knr))
+def get_number_of_mandates(line):
+    col = 'sitze'
+    if not hasattr(line, col):
+        raise ValueError(_('Missing column: ${col}', mapping={'col': col}))
+    try:
+        return int(line.sitze or 0)
+    except ValueError:
+        raise ValueError(
+            _('Invalid integer: ${col}', mapping={'col': col}))
+
+
+def get_ausmittlungsstand(line):
+    col = 'ausmittlungsstand'
+    try:
+        t = int(line.ausmittlungsstand or 0)
+        assert 0 <= t <= 3
+    except ValueError:
+        raise ValueError(
+            _('Invalid integer: ${col}', mapping={'col': col}))
+    except AssertionError:
+        raise AssertionError(
+            _('Value ${col} is not between 0 and 3', mapping={'col': col}))
+
+
+def get_stimmberechtigte(line):
+    col = 'stimmberechtigte'
+    if not hasattr(line, col):
+        raise ValueError(_('Missing column: ${col}', mapping={'col': col}))
+    try:
+        return int(line.stimmberechtigte or 0)
+    except ValueError:
+        raise ValueError(
+            _('Invalid integer: ${col}', mapping={'col': col}))
+
+
+def get_stimmentotal(line):
+    col = 'stimmentotal'
+    if not hasattr(line, col):
+        raise ValueError(_('Missing column: ${col}', mapping={'col': col}))
+    try:
+        return int(line.stimmentotal)
+    except ValueError:
+        raise ValueError(_('Invalid integer: ${col}', mapping={'col': col}))
+
+
+def get_entity_id(line, expats):
+    if not hasattr(line, 'bfsnrgemeinde'):
+        raise ValueError(
+            _('Missing column: bfsnrgemeinde'))
+    try:
+        entity_id = int(line.bfsnrgemeinde or 0)
+    except ValueError:
+        raise ValueError(
+            _('Invalid integer: bfsnrgemeinde'
+              ' Can not extract entity_id.'))
+    return 0 if entity_id in expats else entity_id
+
+
+def get_list_id_from_knr(line):
+    """
+    Takes a line from csv file with a candidate number (knr) in it and
+    returns the derived listnr for this candidate. Will also handle the new
+    WabstiC Standard 2018.
+    """
+    col = 'knr'
+    if not hasattr(line, col):
+        raise ValueError(_('Missing column: ${col}', mapping={'col', col}))
+    if '.' in line.knr:
+        return line.knr.split('.')[0]
+    # replaces int(int(line.knr) / 100))
+    return line.knr[0:-2]
 
 
 def get_list_id(line):
-    if hasattr(line, 'listnr'):
-        number = int(line.listnr or 0)
-    else:
-        number = int(int(get_candidate_id(line)) / 100)
-    number = 999 if number == 99 else number  # blank list
-    return str(number)
+    col = 'listnr'
+    if not hasattr(line, col):
+        raise ValueError(_('Missing column: ${col}', mapping={'col': col}))
+    number = line.listnr or '0'
+    number = '999' if number == '99' else number  # blank list
+    return number
 
 
 def import_election_wabstic_proporz(
-    election, principal, number, district,
-    file_wp_wahl, mimetype_wp_wahl,
-    file_wpstatic_gemeinden, mimetype_wpstatic_gemeinden,
-    file_wp_gemeinden, mimetype_wp_gemeinden,
-    file_wp_listen, mimetype_wp_listen,
-    file_wp_listengde, mimetype_wp_listengde,
-    file_wpstatic_kandidaten, mimetype_wpstatic_kandidaten,
-    file_wp_kandidaten, mimetype_wp_kandidaten,
-    file_wp_kandidatengde, mimetype_wp_kandidatengde
+    election=None, principal=None, number=None, district=None,
+    file_wp_wahl=None, mimetype_wp_wahl=None,
+    file_wpstatic_gemeinden=None,
+    mimetype_wpstatic_gemeinden=None,
+    file_wp_gemeinden=None, mimetype_wp_gemeinden=None,
+    file_wp_listen=None, mimetype_wp_listen=None,
+    file_wp_listengde=None, mimetype_wp_listengde=None,
+    file_wpstatic_kandidaten=None, mimetype_wpstatic_kandidaten=None,
+    file_wp_kandidaten=None, mimetype_wp_kandidaten=None,
+    file_wp_kandidatengde=None, mimetype_wp_kandidatengde=None
 ):
     """ Tries to import the given CSV files from a WabstiCExport.
 
@@ -189,10 +254,9 @@ def import_election_wabstic_proporz(
             continue
 
         try:
-            complete = int(line.ausmittlungsstand or 0)
-            assert 0 <= complete <= 3
-        except (ValueError, AssertionError):
-            line_errors.append(_("Invalid values"))
+            complete = get_ausmittlungsstand(line)
+        except (ValueError, AssertionError) as e:
+            line_errors.append(e.args[0])
 
         # Pass the errors and continue to next line
         if line_errors:
@@ -214,11 +278,11 @@ def import_election_wabstic_proporz(
 
         # Parse the id of the entity
         try:
-            entity_id = get_entity_id(line, entities)
+            entity_id = get_entity_id(line, EXPATS)
             if entity_id == 3251:
                 pass
-        except ValueError:
-            line_errors.append(_("Invalid id"))
+        except ValueError as e:
+            line_errors.append(e.args[0])
         else:
             if entity_id and entity_id not in entities:
                 line_errors.append(
@@ -230,9 +294,9 @@ def import_election_wabstic_proporz(
 
         # Parse the eligible voters
         try:
-            eligible_voters = int(line.stimmberechtigte or 0)
-        except ValueError:
-            line_errors.append(_("Could not read the eligible voters"))
+            eligible_voters = get_stimmberechtigte(line)
+        except ValueError as e:
+            line_errors.append(e.args[0])
 
         # Skip expats if not enabled
         if entity_id == 0 and not election.expats:
@@ -261,9 +325,9 @@ def import_election_wabstic_proporz(
 
         # Parse the id of the entity
         try:
-            entity_id = get_entity_id(line, entities)
-        except ValueError:
-            line_errors.append(_("Invalid id"))
+            entity_id = get_entity_id(line, EXPATS)
+        except ValueError as e:
+            line_errors.append(e.args[0])
         else:
             if entity_id and entity_id not in entities:
                 line_errors.append(
@@ -284,9 +348,9 @@ def import_election_wabstic_proporz(
 
         # Parse the eligible voters
         try:
-            eligible_voters = int(line.stimmberechtigte or 0)
-        except ValueError:
-            line_errors.append(_("Invalid entity values"))
+            eligible_voters = get_stimmberechtigte(line)
+        except ValueError as e:
+            line_errors.append(e.args[0])
         else:
             eligible_voters = (
                 eligible_voters
@@ -329,13 +393,14 @@ def import_election_wabstic_proporz(
         try:
             list_id = get_list_id(line)
             name = line.listcode
-            number_of_mandates = int(line.sitze or 0)
+            number_of_mandates = get_number_of_mandates(line)
             connection = line.listverb or None
             subconnection = line.listuntverb or None
             if subconnection:
-                assert connection
-        except (ValueError, AssertionError):
-            line_errors.append(_("Invalid list values"))
+                assert connection, _('${var} is missing.',
+                                     mapping={'var': 'connection'})
+        except (ValueError, AssertionError) as e:
+            line_errors.append(e.args[0])
         else:
             if list_id in added_lists:
                 line_errors.append(
@@ -388,11 +453,11 @@ def import_election_wabstic_proporz(
         line_errors = []
 
         try:
-            entity_id = get_entity_id(line, entities)
+            entity_id = get_entity_id(line, EXPATS)
             list_id = get_list_id(line)
-            votes = int(line.stimmentotal)
-        except ValueError:
-            line_errors.append(_("Invalid list results"))
+            votes = get_stimmentotal(line)
+        except ValueError as e:
+            line_errors.append(e.args[0])
         else:
             if entity_id not in added_entities:
                 # Only add the list result if the entity is present (there is
@@ -438,8 +503,8 @@ def import_election_wabstic_proporz(
             continue
 
         try:
-            candidate_id = get_candidate_id(line)
-            list_id = get_list_id(line)
+            candidate_id = line.knr
+            list_id = get_list_id_from_knr(line)
             family_name = line.nachname
             first_name = line.vorname
         except ValueError:
@@ -451,7 +516,7 @@ def import_election_wabstic_proporz(
                       mapping={'name': candidate_id}))
 
             if list_id not in added_lists:
-                line_errors.append(_("Invalid list values"))
+                line_errors.append(_("Unknown derived list id"))
 
         # Pass the errors and continue to next line
         if line_errors:
@@ -480,7 +545,7 @@ def import_election_wabstic_proporz(
             continue
 
         try:
-            candidate_id = get_candidate_id(line)
+            candidate_id = line.knr
             assert candidate_id in added_candidates
             elected = True if line.gewahlt == '1' else False
         except (ValueError, AssertionError):
@@ -504,8 +569,8 @@ def import_election_wabstic_proporz(
         line_errors = []
 
         try:
-            entity_id = get_entity_id(line, entities)
-            candidate_id = get_candidate_id(line)
+            entity_id = get_entity_id(line, EXPATS)
+            candidate_id = line.knr
             votes = int(line.stimmen)
         except ValueError:
             line_errors.append(_("Invalid candidate results"))
@@ -549,6 +614,7 @@ def import_election_wabstic_proporz(
     if election.domain == 'region' and districts and election.distinct:
         if principal.has_districts:
             if len(districts) != 1:
+                # FIXME: better error messages
                 errors.append(FileImportError(_("No clear district")))
         else:
             if len(added_results) != 1:

onegov/election_day/tests/formats/election/test_wabstic_proporz.py

@@ -11,6 +11,77 @@
 from pytest import mark
 
 
+def help_print_errors(errors_list, max=20):
+    i = 0
+    while i < max:
+        try:
+            err = errors_list[i]
+            print('error in ', err.filename, ':',
+                  err.line, '-', err.error.interpolate())
+            i += 1
+        except Exception:
+            break
+
+
+@mark.parametrize("tar_file", [
+    module_path('onegov.election_day',
+                'tests/fixtures/wabstic_proporz_v2.3.tar.gz'),
+])
+def test_import_wabstic_proporz_v23(session, tar_file):
+    session.add(
+        ProporzElection(
+            title='test cantonal election',
+            domain='canton',
+            date=date(2016, 2, 28),
+            number_of_mandates=12,
+        )
+    )
+    session.flush()
+    election = session.query(Election).one()
+
+    principal = Canton(canton='sg')
+
+    # The tar file contains a test dataset
+
+    with tarfile.open(tar_file, 'r:gz') as f:
+        regional_wp_gemeinden = f.extractfile('WP_Gemeinden.csv').read()
+        regional_wp_kandidaten = f.extractfile(
+            'WP_Kandidaten.csv').read()
+        regional_wp_kandidatengde = f.extractfile(
+            'WP_KandidatenGde.csv').read()
+        regional_wp_listen = f.extractfile('WP_Listen.csv').read()
+        regional_wp_listengde = f.extractfile('WP_ListenGde.csv').read()
+        regional_wpstatic_gemeinden = f.extractfile(
+            'WPStatic_Gemeinden.csv').read()
+        regional_wpstatic_kandidaten = f.extractfile(
+            'WPStatic_Kandidaten.csv').read()
+        regional_wp_wahl = f.extractfile('WP_Wahl.csv').read()
+
+    # Test cantonal elections
+    election.expats = False
+
+    errors = import_election_wabstic_proporz(
+        election, principal, '1', None,
+        BytesIO(regional_wp_wahl), 'text/plain',
+        BytesIO(regional_wpstatic_gemeinden), 'text/plain',
+        BytesIO(regional_wp_gemeinden), 'text/plain',
+        BytesIO(regional_wp_listen), 'text/plain',
+        BytesIO(regional_wp_listengde), 'text/plain',
+        BytesIO(regional_wpstatic_kandidaten), 'text/plain',
+        BytesIO(regional_wp_kandidaten), 'text/plain',
+        BytesIO(regional_wp_kandidatengde), 'text/plain',
+    )
+
+    assert not errors
+    assert election.completed
+    # assert election.progress == (78, 78)
+    # assert election.absolute_majority is None
+    # assert election.eligible_voters == 317969
+    # assert election.accounted_ballots == 145631
+    # assert election.accounted_votes == 1732456
+    # assert election.allocated_mandates == 12
+
+
 @mark.parametrize("tar_file", [
     module_path('onegov.election_day',
                 'tests/fixtures/wabstic_proporz.tar.gz'),
@@ -384,14 +455,15 @@ def test_import_wabstic_proporz_invalid_values(session):
     ]) == [
         ('wp_gemeinden', 2, 'Invalid entity values'),
         ('wp_gemeinden', 2, 'Invalid entity values'),
-        ('wp_gemeinden', 2, 'Invalid entity values'),
+        ('wp_gemeinden', 2,
+            'Invalid integer: stimmberechtigte'),
         ('wp_kandidatengde', 2, 'Invalid candidate results'),
-        ('wp_listen', 2, 'Invalid list values'),
-        ('wp_listengde', 2, 'Invalid list results'),
-        ('wp_wahl', 2, 'Invalid values'),
+        ('wp_listen', 2, 'Invalid integer: sitze'),
+        ('wp_listengde', 2, 'Invalid integer: stimmentotal'),
+        ('wp_wahl', 2, 'Value ausmittlungsstand is not between 0 and 3'),
         ('wpstatic_gemeinden', 2, '100 is unknown'),
         ('wpstatic_kandidaten', 2, 'Invalid candidate values'),
-        ('wpstatic_kandidaten', 2, 'Invalid candidate values')
+        ('wpstatic_kandidaten', 2, 'Unknown derived list id')
     ]