SO-2127 snomed rf2 importer issues

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/util/ImportUtil.java

@@ -528,7 +528,6 @@ public Boolean execute(RevisionSearcher index) throws IOException {
 					final String message = String.format("Validation encountered %s issue(s).", defects.size());
 					LogUtils.logImportActivity(IMPORT_LOGGER, requestingUserId, branchPath, message);
 
-					final String lineSeparator = System.getProperty("line.separator");
 					final Map<String, BufferedWriter> defectWriters = newHashMap();
 					try {
 						for (SnomedValidationDefect defect : defects) {
@@ -537,10 +536,7 @@ public Boolean execute(RevisionSearcher index) throws IOException {
 							if (!defectWriters.containsKey(defectsFile)) {
 								defectWriters.put(defectsFile, Files.newWriter(new File(defectsFile), Charsets.UTF_8));
 							}
-							for (String lineDefect : defect.getDefects()) {
-								defectWriters.get(defectsFile).write(defect.getDefectType().name() + "\t" + lineDefect);
-								defectWriters.get(defectsFile).write(lineSeparator);
-							}
+							defect.writeTo(defectWriters.get(defectsFile));
 						}
 					} finally {
 						for (BufferedWriter writer : defectWriters.values()) {

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/validation/SnomedTaxonomyValidator.java

@@ -16,14 +16,15 @@
 package com.b2international.snowowl.snomed.importer.rf2.validation;
 
 import static com.b2international.snowowl.snomed.common.ContentSubType.SNAPSHOT;
-import static com.google.common.collect.Sets.newHashSet;
+import static com.google.common.collect.Lists.newArrayListWithExpectedSize;
 import static java.util.Collections.emptySet;
 import static org.slf4j.LoggerFactory.getLogger;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 
@@ -36,12 +37,14 @@
 import com.b2international.snowowl.snomed.SnomedConstants.Concepts;
 import com.b2international.snowowl.snomed.datastore.index.entry.SnomedRelationshipIndexEntry;
 import com.b2international.snowowl.snomed.datastore.taxonomy.InvalidRelationship;
+import com.b2international.snowowl.snomed.datastore.taxonomy.InvalidRelationship.MissingConcept;
 import com.b2international.snowowl.snomed.datastore.taxonomy.SnomedTaxonomyBuilder;
 import com.b2international.snowowl.snomed.datastore.taxonomy.SnomedTaxonomyBuilderResult;
 import com.b2international.snowowl.snomed.importer.net4j.DefectType;
 import com.b2international.snowowl.snomed.importer.net4j.ImportConfiguration;
 import com.b2international.snowowl.snomed.importer.net4j.SnomedIncompleteTaxonomyValidationDefect;
 import com.b2international.snowowl.snomed.importer.net4j.SnomedValidationDefect;
+import com.b2international.snowowl.snomed.importer.net4j.TaxonomyDefect;
 import com.b2international.snowowl.snomed.importer.rf2.RepositoryState;
 import com.b2international.snowowl.snomed.importer.rf2.util.Rf2FileModifier;
 import com.google.common.collect.ArrayListMultimap;
@@ -56,6 +59,19 @@
 public class SnomedTaxonomyValidator {
 
 	private static final Logger LOGGER = getLogger(SnomedTaxonomyValidator.class);
+
+	private static final Comparator<String> EFFECTIVE_TIME_COMPARATOR = new Comparator<String>() {
+		@Override
+		public int compare(String o1, String o2) {
+			// consider empty greater than non-empty
+			if (o1.isEmpty() && !o2.isEmpty()) {
+				return 1;
+			} else if (!o1.isEmpty() && o2.isEmpty()) {
+				return -1;
+			}
+			return o1.compareTo(o2);
+		}
+	};
 
 	// new RF2 state
 	private final File conceptsFile;
@@ -112,105 +128,17 @@ public Collection<SnomedValidationDefect> validate() {
 	 */
 	private Collection<SnomedValidationDefect> doValidate() {
 		try {
-			final Rf2BasedSnomedTaxonomyBuilder builder = Rf2BasedSnomedTaxonomyBuilder.newInstance(new SnomedTaxonomyBuilder(conceptIds, statements), characteristicType);
-
-			final Multimap<String, InvalidRelationship> invalidRelationships = ArrayListMultimap.create();
-
-			if (snapshot) {
-
-				LOGGER.info("Validating SNOMED CT ontology based on the given RF2 release files...");
-
-
-				if (hasConceptImport()) {
-					final String conceptFilePath = removeConceptHeader();
-					builder.applyNodeChanges(conceptFilePath);
-				}
-
-				if (hasRelationshipImport()) {
-					final String relationshipFilePath = removeRelationshipHeader();
-					builder.applyEdgeChanges(relationshipFilePath);
-				}
-
-				final SnomedTaxonomyBuilderResult result = builder.build();
-				if (!result.getStatus().isOK()) {
-					invalidRelationships.putAll("", result.getInvalidRelationships());
-				}
-			} else {
-
-				LOGGER.info("Validating SNOMED CT ontology based on the given RF2 release files...");
-
-				final Map<String, File> conceptFiles = hasConceptImport() ? Rf2FileModifier.split(conceptsFile) : ImmutableMap.<String, File>of();
-				final Map<String, File> relationshipFiles = hasRelationshipImport() ? Rf2FileModifier.split(relationshipsFile) : ImmutableMap.<String, File>of();
-
-				final List<String> effectiveTimes = ImmutableSortedSet.<String>naturalOrder()
-						.addAll(conceptFiles.keySet())
-						.addAll(relationshipFiles.keySet())
-						.build()
-						.asList();
-
-
-				for (final String effectiveTime : effectiveTimes) {
-					LOGGER.info("Validating concepts and relationships from '" + effectiveTime + "'...");
-
-					final File conceptFile = conceptFiles.get(effectiveTime);
-					final File relationshipFile = relationshipFiles.get(effectiveTime);
-
-					builder.applyNodeChanges(getFilePath(conceptFile));
-					builder.applyEdgeChanges(getFilePath(relationshipFile));
-					final SnomedTaxonomyBuilderResult result = builder.build();
-					if (!result.getStatus().isOK()) {
-						invalidRelationships.putAll(effectiveTime, result.getInvalidRelationships());
-					}
-				}
-			}
-
+			final Multimap<String, InvalidRelationship> invalidRelationships = processTaxonomy();
 			if (!invalidRelationships.isEmpty()) {
-				final Collection<String> defects = newHashSet();
-				final Collection<String> conceptIdsToInactivate = newHashSet();
-
+				final Collection<TaxonomyDefect> defects = newArrayListWithExpectedSize(invalidRelationships.size());
 				for (final String effectiveTime : invalidRelationships.keySet()) {
 					for (final InvalidRelationship invalidRelationship: invalidRelationships.get(effectiveTime)) {
-						final String sourceId = Long.toString(invalidRelationship.getSourceId());
-						final String destinationId = Long.toString(invalidRelationship.getDestinationId());
-
-						if (conceptIds.contains(invalidRelationship.getDestinationId())) {
-							conceptIdsToInactivate.add(destinationId);
-						}
-
-						if (conceptIds.contains(invalidRelationship.getSourceId())) {
-							conceptIdsToInactivate.add(sourceId);
-						}
-
-						final StringBuilder sb = new StringBuilder();
-						sb.append("IS A relationship");
-						sb.append(" '" + invalidRelationship.getRelationshipId() + "'");
-						sb.append(" has a missing or inactive ");
-
-						switch (invalidRelationship.getMissingConcept()) {
-						case DESTINATION:
-							sb.append("destination concept");
-							sb.append(" '" + destinationId);
-							break;
-						case SOURCE:
-							sb.append("source concept");
-							sb.append(" '" + sourceId);
-							break;
-						default:
-							throw new IllegalStateException("Unexpected missing concept type '" + invalidRelationship.getMissingConcept() + "'.");					
-						}
-
-						sb.append("' in effectiveTime ");
-						sb.append("".equals(effectiveTime) ? "Unpublished/Undefined" : effectiveTime);
-						sb.append("'.");
-
-						defects.add(sb.toString());
+						defects.add(new TaxonomyDefect(invalidRelationship.getRelationshipId(), effectiveTime, invalidRelationship.getMissingConcept() == MissingConcept.DESTINATION ? TaxonomyDefect.Type.MISSING_DESTINATION : TaxonomyDefect.Type.MISSING_SOURCE, invalidRelationship.getMissingConceptId()));
 					}
 				}
 
-				final SnomedValidationDefect defect = new SnomedIncompleteTaxonomyValidationDefect(relationshipsFile.getName(), defects, conceptIdsToInactivate);
-				return Collections.<SnomedValidationDefect>singleton(defect);
+				return Collections.singleton(new SnomedIncompleteTaxonomyValidationDefect(relationshipsFile.getName(), defects));
 			}
-
 		} catch (final IOException e) {
 			LOGGER.error("Validation failed.", e);
 			return Collections.<SnomedValidationDefect>singleton(new SnomedValidationDefect(relationshipsFile.getName(), DefectType.IO_PROBLEM, Collections.<String>emptySet()));
@@ -220,6 +148,59 @@ private Collection<SnomedValidationDefect> doValidate() {
 		return emptySet();
 	}
 
+	private Multimap<String, InvalidRelationship> processTaxonomy() throws IOException {
+		final Rf2BasedSnomedTaxonomyBuilder builder = Rf2BasedSnomedTaxonomyBuilder.newInstance(new SnomedTaxonomyBuilder(conceptIds, statements), characteristicType);
+		final Multimap<String, InvalidRelationship> invalidRelationships = ArrayListMultimap.create();
+		if (snapshot) {
+
+			LOGGER.info("Validating SNOMED CT ontology based on the given RF2 release files...");
+
+
+			if (hasConceptImport()) {
+				final String conceptFilePath = removeConceptHeader();
+				builder.applyNodeChanges(conceptFilePath);
+			}
+
+			if (hasRelationshipImport()) {
+				final String relationshipFilePath = removeRelationshipHeader();
+				builder.applyEdgeChanges(relationshipFilePath);
+			}
+
+			final SnomedTaxonomyBuilderResult result = builder.build();
+			if (!result.getStatus().isOK()) {
+				invalidRelationships.putAll("", result.getInvalidRelationships());
+			}
+		} else {
+
+			LOGGER.info("Validating SNOMED CT ontology based on the given RF2 release files...");
+
+			final Map<String, File> conceptFiles = hasConceptImport() ? Rf2FileModifier.split(conceptsFile) : ImmutableMap.<String, File>of();
+			final Map<String, File> relationshipFiles = hasRelationshipImport() ? Rf2FileModifier.split(relationshipsFile) : ImmutableMap.<String, File>of();
+
+			final List<String> effectiveTimes = ImmutableSortedSet.orderedBy(EFFECTIVE_TIME_COMPARATOR)
+					.addAll(conceptFiles.keySet())
+					.addAll(relationshipFiles.keySet())
+					.build()
+					.asList();
+
+
+			for (final String effectiveTime : effectiveTimes) {
+				LOGGER.info("Validating taxonomy in '{}'...", effectiveTime);
+
+				final File conceptFile = conceptFiles.get(effectiveTime);
+				final File relationshipFile = relationshipFiles.get(effectiveTime);
+
+				builder.applyNodeChanges(getFilePath(conceptFile));
+				builder.applyEdgeChanges(getFilePath(relationshipFile));
+				final SnomedTaxonomyBuilderResult result = builder.build();
+				if (!result.getStatus().isOK()) {
+					invalidRelationships.putAll(effectiveTime, result.getInvalidRelationships());
+				}
+			}
+		}
+		return invalidRelationships;
+	}
+
 	private String getFilePath(@Nullable final File file) {
 		return null == file ?  null : file.getPath();
 	}

snomed/com.b2international.snowowl.snomed.importer/src/com/b2international/snowowl/snomed/importer/net4j/SnomedIncompleteTaxonomyValidationDefect.java

@@ -15,9 +15,10 @@
  */
 package com.b2international.snowowl.snomed.importer.net4j;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
+import java.io.IOException;
+import java.io.Writer;
 import java.util.Collection;
+import java.util.Collections;
 
 /**
  * Representation of a validation defect about incomplete taxonomy. 
@@ -26,23 +27,49 @@
 public class SnomedIncompleteTaxonomyValidationDefect extends SnomedValidationDefect {
 
 	private static final long serialVersionUID = -4826632877877405283L;
+	private final Collection<TaxonomyDefect> taxonomyDefects;
 
-	private Collection<String> conceptIdsToInactivate;
+	public SnomedIncompleteTaxonomyValidationDefect(final String filePath, final Collection<TaxonomyDefect> taxonomyDefects) {
+		super(filePath, DefectType.INCONSISTENT_TAXONOMY, Collections.singleton("fake"));
+		this.taxonomyDefects = taxonomyDefects;
+	}
 
-	public SnomedIncompleteTaxonomyValidationDefect(final String filePath, final Collection<String> defects, 
-			final Collection<String> conceptIdsToInactivate) {
-
-		super(filePath, DefectType.INCONSISTENT_TAXONOMY, defects);
-		this.conceptIdsToInactivate = checkNotNull(conceptIdsToInactivate, "conceptIdsToInactivate");
+	public Collection<TaxonomyDefect> getTaxonomyDefects() {
+		return taxonomyDefects;
 	}
 
-	/**
-	 * Returns with a collection of conflicting concept IDs. If these concepts
-	 * are inactivated one can fix the incomplete taxonomy defect.
-	 * @return the conceptIdsToInactivate the concept IDs to inactivate.
-	 */
-	public Collection<String> getConceptIdsToInactivate() {
-		return conceptIdsToInactivate;
+	@Override
+	public void writeTo(Writer writer) throws IOException {
+		// writer header
+		writer.write("defectType");
+		writer.write(TAB);
+		writer.write("id");
+		writer.write(TAB);
+		writer.write("effectiveTime");
+		writer.write(TAB);
+		writer.write("taxonomyDefectType");
+		writer.write(TAB);
+		writer.write("conceptId");
+		writer.write(TAB);
+		writer.write(LE);
+		for (TaxonomyDefect defect : getTaxonomyDefects()) {
+			// defectType
+			writer.write(getDefectType().name());
+			writer.write(TAB);
+			// id
+			writer.write(Long.toString(defect.getRelationshipId()));
+			writer.write(TAB);
+			// effectiveTime
+			writer.write(defect.getEffectiveTime());
+			writer.write(TAB);
+			// tax.def.type
+			writer.write(defect.getType().name());
+			writer.write(TAB);
+			// conceptId
+			writer.write(Long.toString(defect.getMissingConceptId()));
+			writer.write(TAB);
+			writer.write(LE);
+		}
 	}
-
+	
 }

snomed/com.b2international.snowowl.snomed.importer/src/com/b2international/snowowl/snomed/importer/net4j/SnomedValidationDefect.java

@@ -18,7 +18,9 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import java.io.IOException;
 import java.io.Serializable;
+import java.io.Writer;
 import java.util.Collection;
 
 import com.google.common.collect.ImmutableList;
@@ -29,6 +31,8 @@
 public class SnomedValidationDefect implements Serializable, Comparable<SnomedValidationDefect> {
 
 	private static final long serialVersionUID = 1L;
+	protected static final String TAB = "\t";
+	protected static final String LE = System.getProperty("line.separator");
 
 	private final String fileName;
 	private final DefectType defectType;
@@ -57,5 +61,19 @@ public final Collection<String> getDefects() {
 	public int compareTo(SnomedValidationDefect o) {
 		return getDefectType().compareTo(o.getDefectType());
 	}
+
+	/**
+	 * Writes this {@link SnomedValidationDefect SNOMED CT RF2 validation defect} to a writer.
+	 * @param writer
+	 * @throws IOException
+	 */
+	public void writeTo(Writer writer) throws IOException {
+		for (String lineDefect : getDefects()) {
+			writer.write(getDefectType().name());
+			writer.write(TAB);
+			writer.write(lineDefect);
+			writer.write(LE);
+		}
+	}
 
 }