Permalink
Browse files

Refactoring completed. Still one test failing. But the code it repres…

…ents should be removed.
  • Loading branch information...
1 parent 71b9194 commit cee604fadf319e6b5416fc9a4e76d2e26e22a71b @anagrius anagrius committed Jul 10, 2011
Showing with 227 additions and 1,680 deletions.
  1. +0 −846 subprojects/authorization-ws/logFile_IS_UNDEFINED
  2. +2 −1 subprojects/importer/src/main/java/com/trifork/stamdata/importer/jobs/FileParser.java
  3. +48 −32 .../src/main/java/com/trifork/stamdata/importer/jobs/autorisationsregister/AutorisationImporter.java
  4. +11 −7 subprojects/importer/src/main/java/com/trifork/stamdata/importer/jobs/cpr/CPRImporter.java
  5. +2 −2 .../src/main/java/com/trifork/stamdata/importer/jobs/dosagesuggestions/DosageSuggestionImporter.java
  6. +2 −1 subprojects/importer/src/main/java/com/trifork/stamdata/importer/jobs/sikrede/SikredeParser.java
  7. +2 −2 subprojects/importer/src/main/java/com/trifork/stamdata/importer/jobs/sks/SksImporter.java
  8. +2 −2 subprojects/importer/src/main/java/com/trifork/stamdata/importer/jobs/sor/SORImporter.java
  9. +4 −3 subprojects/importer/src/main/java/com/trifork/stamdata/importer/jobs/takst/TakstImporter.java
  10. +27 −22 subprojects/importer/src/main/java/com/trifork/stamdata/importer/jobs/takst/TakstParser.java
  11. +29 −50 .../importer/src/main/java/com/trifork/stamdata/importer/jobs/yderregister/YderregisterImporter.java
  12. +2 −2 subprojects/importer/src/main/resources/config.properties
  13. +0 −345 subprojects/importer/src/test/java/com/trifork/stamdata/importer/jobs/FileSpoolerImplTest.java
  14. +0 −67 subprojects/importer/src/test/java/com/trifork/stamdata/importer/jobs/FileSpoolerSetupTest.java
  15. +0 −59 subprojects/importer/src/test/java/com/trifork/stamdata/importer/jobs/JobSpoolerSetupTest.java
  16. +0 −86 subprojects/importer/src/test/java/com/trifork/stamdata/importer/jobs/SpoolerManagerTest.java
  17. +9 −31 ...er/src/test/java/com/trifork/stamdata/importer/jobs/autorisationsregister/AutIntegrationTest.java
  18. +11 −12 ...mporter/src/test/java/com/trifork/stamdata/importer/jobs/autorisationsregister/AutParserTest.java
  19. +30 −25 ...stamdata/importer/jobs/autorisationsregister/{AutImporterTest.java → AutorisationParserTest.java}
  20. +15 −17 subprojects/importer/src/test/java/com/trifork/stamdata/importer/jobs/cpr/CPRIntegrationTest.java
  21. +6 −8 ...c/test/java/com/trifork/stamdata/importer/jobs/doseringsforslag/DoseringsforslagImporterTest.java
  22. +2 −25 subprojects/importer/src/test/java/com/trifork/stamdata/importer/jobs/takst/TakstParserTest.java
  23. +19 −26 ...er/src/test/java/com/trifork/stamdata/importer/jobs/yderregister/YderregisterIntegrationTest.java
  24. +4 −9 ...mporter/src/test/java/com/trifork/stamdata/importer/jobs/yderregister/YderregisterParserTest.java

Large diffs are not rendered by default.

Oops, something went wrong.
@@ -3,6 +3,7 @@
import java.io.File;
import com.trifork.stamdata.importer.persistence.AuditingPersister;
+import com.trifork.stamdata.importer.persistence.Persister;
public interface FileParser
{
@@ -14,7 +15,7 @@
boolean ensureRequiredFileArePresent(File[] input);
- void importFiles(File[] input, AuditingPersister persister) throws Exception;
+ void importFiles(File[] input, Persister persister) throws Exception;
/**
* (non-javadoc)
@@ -25,25 +25,27 @@
import static com.trifork.stamdata.Preconditions.checkNotNull;
-import java.io.BufferedReader;
import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.io.InputStreamReader;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Timestamp;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.LineIterator;
import org.joda.time.DateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
import com.trifork.stamdata.importer.jobs.FileParser;
import com.trifork.stamdata.importer.jobs.autorisationsregister.model.Autorisation;
import com.trifork.stamdata.importer.jobs.autorisationsregister.model.Autorisationsregisterudtraek;
-import com.trifork.stamdata.importer.persistence.AuditingPersister;
+import com.trifork.stamdata.importer.persistence.Persister;
public class AutorisationImporter implements FileParser
{
+ private static final String FILENAME_DATE_FORMAT = "yyyyMMdd";
private static final String FILE_ENCODING = "ISO8859-15";
@Override
@@ -54,7 +56,7 @@ public boolean ensureRequiredFileArePresent(File[] input)
// make sure that there are some.
checkNotNull(input);
-
+
return (input.length > 0);
}
@@ -65,46 +67,60 @@ public String getIdentifier()
}
@Override
- public void importFiles(File[] files, AuditingPersister persister) throws Exception
+ public String getHumanName()
{
- for (File file : files)
+ return "Autorisationsregisteret Parser";
+ }
+
+ @Override
+ public void importFiles(File[] files, Persister persister) throws Exception
+ {
+ // Make sure the file set has not been imported before.
+ // Check what the previous highest version is (the ValidFrom column).
+
+ Connection connection = persister.getConnection();
+ ResultSet rows = connection.createStatement().executeQuery("SELECT MAX(ValidFrom) as version FROM Autorisation");
+
+ // There will always be a next here, but it might be null.
+
+ rows.next();
+ Timestamp previousVersion = rows.getTimestamp("version");
+
+ DateTime currentVersion = getDateFromFilename(files[0].getName());
+
+ if (previousVersion != null && !currentVersion.isAfter(previousVersion.getTime()))
{
- Date date = getDateFromFilename(file.getName());
+ throw new Exception("The version of autorisationsregister that was placed for import was out of order. current_version='" + previousVersion + "', new_version='" + currentVersion + "'.");
+ }
- Autorisationsregisterudtraek dataset = parse(file, date);
+ for (File file : files)
+ {
+ Autorisationsregisterudtraek dataset = parse(file, currentVersion);
persister.persistCompleteDataset(dataset);
}
}
- protected Date getDateFromFilename(String filename) throws ParseException
+ protected DateTime getDateFromFilename(String filename)
{
- SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd");
- return formatter.parse(filename.substring(0, 8));
+ DateTimeFormatter formatter = DateTimeFormat.forPattern(FILENAME_DATE_FORMAT);
+ return formatter.parseDateTime(filename.substring(0, 8));
}
- public Date getNextImportExpectedBefore(Date lastImport)
+ public Autorisationsregisterudtraek parse(File file, DateTime validFrom) throws IOException
{
- // Largest gap observed was 15 days from 2008-10-18 to 2008-11-01.
+ Autorisationsregisterudtraek dataset = new Autorisationsregisterudtraek(validFrom.toDate());
- return new DateTime(lastImport).plusMonths(1).toDate();
- }
-
- public Autorisationsregisterudtraek parse(File file, Date validFrom) throws IOException
- {
- Autorisationsregisterudtraek dataset = new Autorisationsregisterudtraek(validFrom);
- BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), FILE_ENCODING));
+ LineIterator lineIterator = FileUtils.lineIterator(file, FILE_ENCODING);
+
+ // TODO (thb): The parsing should not be handled in a constructor. That
+ // is bad design.
- while (reader.ready())
+ while (lineIterator.hasNext())
{
- dataset.addEntity(new Autorisation(reader.readLine()));
+ String line = lineIterator.nextLine();
+ dataset.addEntity(new Autorisation(line));
}
return dataset;
}
-
- @Override
- public String getHumanName()
- {
- return "Autorisationsregisteret Parser";
- }
}
@@ -23,6 +23,7 @@
package com.trifork.stamdata.importer.jobs.cpr;
+import static com.trifork.stamdata.Preconditions.checkNotNull;
import static com.trifork.stamdata.importer.util.DateUtils.yyyy_MM_dd;
import java.io.File;
@@ -39,8 +40,8 @@
import com.trifork.stamdata.importer.config.Configuration;
import com.trifork.stamdata.importer.jobs.FileParser;
import com.trifork.stamdata.importer.jobs.cpr.model.CPRDataset;
-import com.trifork.stamdata.importer.persistence.AuditingPersister;
import com.trifork.stamdata.importer.persistence.Dataset;
+import com.trifork.stamdata.importer.persistence.Persister;
import com.trifork.stamdata.importer.persistence.StamdataEntity;
import com.trifork.stamdata.importer.util.DateUtils;
@@ -71,15 +72,18 @@ public boolean ensureRequiredFileArePresent(File[] input)
}
@Override
- public void importFiles(File[] input, AuditingPersister persister) throws Exception
+ public void importFiles(File[] input, Persister persister) throws Exception
{
+ checkNotNull(input);
+ checkNotNull(persister);
+
logger.info("Starting to parse CPR file ");
for (File personFile : input)
{
if (!isPersonerFile(personFile))
{
- throw new Exception("File " + personFile.getAbsolutePath() + " is not a valid CPR file. Nothing is imported from the fileset");
+ throw new Exception("File " + personFile.getAbsolutePath() + " is not a valid CPR file. Nothing will be imported from the fileset.");
}
}
@@ -102,7 +106,7 @@ public void importFiles(File[] input, AuditingPersister persister) throws Except
if (previousVersion == null)
{
- logger.warn("could not get latestIKraft from database. Asuming empty database and skipping import sequence checks.");
+ logger.debug("Find any previous versions of CPR. Asuming an initial import and skipping sequence checks.");
}
else if (!cpr.getPreviousFileValidFrom().equals(previousVersion))
{
@@ -115,12 +119,12 @@ else if (!cpr.getPreviousFileValidFrom().equals(previousVersion))
persister.persistDeltaDataset(dataset);
}
- // Add latest 'ikraft' date to database if we are not importing
+ // Add latest 'version' date to database if we are not importing
// a full set.
if (isDeltaFile(personFile))
{
- insertIkraft(cpr.getValidFrom(), connection);
+ insertVersion(cpr.getValidFrom(), connection);
}
}
}
@@ -146,7 +150,7 @@ static public Date getLatestVersion(Connection con) throws SQLException
return null;
}
- void insertIkraft(Date calendar, Connection con) throws SQLException
+ void insertVersion(Date calendar, Connection con) throws SQLException
{
Statement stm = con.createStatement();
String query = "INSERT INTO PersonIkraft (IkraftDato) VALUES ('" + DateUtils.toMySQLdate(calendar) + "');";
@@ -53,8 +53,8 @@
import com.trifork.stamdata.importer.jobs.dosagesuggestions.models.DosageVersion;
import com.trifork.stamdata.importer.jobs.dosagesuggestions.models.Drug;
import com.trifork.stamdata.importer.jobs.dosagesuggestions.models.DrugDosageStructureRelation;
-import com.trifork.stamdata.importer.persistence.AuditingPersister;
import com.trifork.stamdata.importer.persistence.CompleteDataset;
+import com.trifork.stamdata.importer.persistence.Persister;
import com.trifork.stamdata.importer.persistence.StamdataEntity;
@@ -66,7 +66,7 @@
private static final Logger logger = getLogger(DosageSuggestionImporter.class);
@Override
- public void importFiles(File[] files, AuditingPersister persister) throws Exception
+ public void importFiles(File[] files, Persister persister) throws Exception
{
// METADATA FILE
//
@@ -5,6 +5,7 @@
import com.trifork.stamdata.importer.jobs.FileParser;
import com.trifork.stamdata.importer.persistence.AuditingPersister;
import com.trifork.stamdata.importer.persistence.Dataset;
+import com.trifork.stamdata.importer.persistence.Persister;
public class SikredeParser implements FileParser
{
@@ -31,7 +32,7 @@ public boolean ensureRequiredFileArePresent(File[] input)
}
@Override
- public void importFiles(File[] input, AuditingPersister persister) throws Exception
+ public void importFiles(File[] input, Persister persister) throws Exception
{
// 1. CHECK VERSIONS
//
@@ -32,8 +32,8 @@
import com.trifork.stamdata.importer.jobs.FileParser;
import com.trifork.stamdata.importer.jobs.sks.model.Organisation;
-import com.trifork.stamdata.importer.persistence.AuditingPersister;
import com.trifork.stamdata.importer.persistence.Dataset;
+import com.trifork.stamdata.importer.persistence.Persister;
/**
@@ -59,7 +59,7 @@ public boolean ensureRequiredFileArePresent(File[] input)
}
@Override
- public void importFiles(File[] files, AuditingPersister persister) throws Exception
+ public void importFiles(File[] files, Persister persister) throws Exception
{
for (File file : files)
{
@@ -34,7 +34,7 @@
import org.slf4j.LoggerFactory;
import com.trifork.stamdata.importer.jobs.FileParser;
-import com.trifork.stamdata.importer.persistence.AuditingPersister;
+import com.trifork.stamdata.importer.persistence.Persister;
/**
* Parser for the SOR register.
@@ -61,7 +61,7 @@ public boolean ensureRequiredFileArePresent(File[] input)
}
@Override
- public void importFiles(File[] files, AuditingPersister persister) throws Exception
+ public void importFiles(File[] files, Persister persister) throws Exception
{
for (File file : files)
{
@@ -40,7 +40,8 @@
import com.google.common.collect.Maps;
import com.trifork.stamdata.importer.config.MySQLConnectionManager;
import com.trifork.stamdata.importer.jobs.FileParser;
-import com.trifork.stamdata.importer.persistence.AuditingPersister;
+import com.trifork.stamdata.importer.persistence.CompleteDataset;
+import com.trifork.stamdata.importer.persistence.Persister;
/**
* Parser for the DKMA register. Also known as 'Taksten'.
@@ -72,11 +73,11 @@ public boolean ensureRequiredFileArePresent(File[] input)
return true;
}
- public void importFiles(File[] input, AuditingPersister persister) throws Exception
+ public void importFiles(File[] input, Persister persister) throws Exception
{
Takst takst = new TakstParser().parseFiles(input);
- persister.persistCompleteDataset(takst.getDatasets());
+ persister.persistCompleteDataset(takst.getDatasets().toArray(new CompleteDataset[] {}));
}
/**
Oops, something went wrong.

0 comments on commit cee604f

Please sign in to comment.