SO-2127 snomed rf2 importer issues

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/util/ImportUtil.java

@@ -27,7 +27,9 @@
 import static com.b2international.snowowl.snomed.importer.rf2.util.RF2ReleaseRefSetFileCollector.collectUrlFromRelease;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.collect.Maps.newHashMap;
 
+import java.io.BufferedWriter;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
@@ -40,6 +42,7 @@
 import java.util.Enumeration;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
@@ -111,6 +114,7 @@
 import com.b2international.snowowl.snomed.importer.rf2.terminology.SnomedRelationshipImporter;
 import com.b2international.snowowl.snomed.importer.rf2.validation.SnomedValidationContext;
 import com.b2international.snowowl.terminologyregistry.core.request.CodeSystemRequests;
+import com.google.common.base.Charsets;
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Predicate;
@@ -515,23 +519,36 @@ private boolean isContentValid(final RepositoryState repositoryState,
 			@Override
 			public Boolean execute(RevisionSearcher index) throws IOException {
 				final SnomedValidationContext validator = new SnomedValidationContext(index, requestingUserId, configuration, IMPORT_LOGGER, repositoryState);
-				result.getValidationDefects().addAll(validator.validate(subMonitor.newChild(1)));
+				final Set<SnomedValidationDefect> defects = result.getValidationDefects();
+				defects.addAll(validator.validate(subMonitor.newChild(1)));
 
-				if (!isEmpty(result.getValidationDefects())) {
+				if (!isEmpty(defects)) {
 					// If only header differences exist, continue the import
-					final FluentIterable<String> defects = FluentIterable.from(result.getValidationDefects()).transformAndConcat(new Function<SnomedValidationDefect, Iterable<? extends String>>() {
-						@Override
-						public Iterable<? extends String> apply(SnomedValidationDefect input) {
-							return input.getDefects();
-						}
-					});
+
 					final String message = String.format("Validation encountered %s issue(s).", defects.size());
 					LogUtils.logImportActivity(IMPORT_LOGGER, requestingUserId, branchPath, message);
-					for (String defect : defects) {
-						LogUtils.logImportActivity(IMPORT_LOGGER, requestingUserId, branchPath, defect);
+
+					final String lineSeparator = System.getProperty("line.separator");
+					final Map<String, BufferedWriter> defectWriters = newHashMap();
+					try {
+						for (SnomedValidationDefect defect : defects) {
+							final String filePath = defect.getFileName();
+							final String defectsFile = String.format("d:/%s_%s_defects.txt", configuration.getArchiveFile().getName(), filePath);
+							if (!defectWriters.containsKey(defectsFile)) {
+								defectWriters.put(defectsFile, Files.newWriter(new File(defectsFile), Charsets.UTF_8));
+							}
+							for (String lineDefect : defect.getDefects()) {
+								defectWriters.get(defectsFile).write(defect.getDefectType().name() + "\t" + lineDefect);
+								defectWriters.get(defectsFile).write(lineSeparator);
+							}
+						}
+					} finally {
+						for (BufferedWriter writer : defectWriters.values()) {
+							writer.close();
+						}
 					}
 
-					return !Iterables.tryFind(result.getValidationDefects(), new Predicate<SnomedValidationDefect>() {
+					return !Iterables.tryFind(defects, new Predicate<SnomedValidationDefect>() {
 						@Override
 						public boolean apply(SnomedValidationDefect input) {
 							return input.getDefectType().isCritical();

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/util/Rf2FileModifier.java

@@ -38,6 +38,8 @@
  */
 public class Rf2FileModifier {
 
+	private static final Splitter ON = Splitter.on('\t');
+
 	/**
 	 * Splits the given RF2 files into RF2 file segments based on distinct effective times. The result will be a map
 	 * where keys are the effective times and values are the RF2 files. These files are being created to the temporary
@@ -66,7 +68,7 @@ public static Map<String, File> split(final File toSplit) throws IOException {
 					continue;
 				}
 
-				final List<String> values = Splitter.on('\t').splitToList(line);
+				final List<String> values = ON.splitToList(line);
 				final String effectiveTime = values.get(1);
 
 				final PrintWriter effectiveTimeWriter;

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/validation/AbstractSnomedValidator.java

@@ -183,8 +183,9 @@ private Path getEffectiveTimeFile(String effectiveTimeKey) {
 	 * @param monitor the SubMonitor instance to report progress on
 	 */
 	protected void doValidate(final String effectiveTime, IProgressMonitor monitor) {
-		if (!effectiveTimes.contains(effectiveTime)) return;
-
+		if (!effectiveTimes.contains(effectiveTime)) {
+			return;	
+		}
 		final Stopwatch watch = Stopwatch.createStarted();
 		final String effectiveTimeMessage = effectiveTime.length() == 0 ? "Unpublished" : effectiveTime;
 		final String message = String.format("Validating %s file in '%s'...", importType.getDisplayName(), effectiveTimeMessage);
@@ -249,7 +250,7 @@ protected void postValidate(final SubMonitor monitor) {
 	}
 
 	protected void addDefect(final DefectType type, String...defects) {
-		this.validationContext.addDefect(type, defects);
+		this.validationContext.addDefect(releaseFileName, type, defects);
 	}
 
 	/**
@@ -258,7 +259,7 @@ protected void addDefect(final DefectType type, String...defects) {
 	 * @param validationDefect the defect to be added
 	 */
 	protected void addDefect(final DefectType type, Iterable<String> defects) {
-		this.validationContext.addDefect(type, defects);
+		this.validationContext.addDefect(releaseFileName, type, defects);
 	}
 
 	/**
@@ -348,7 +349,7 @@ protected boolean isComponentActive(final String componentId) {
 	}
 
 	protected void registerComponent(ComponentCategory category, String componentId, boolean status) throws AlreadyExistsException {
-		validationContext.registerComponent(category, componentId, status);
+		validationContext.registerComponent(releaseFileName, category, componentId, status);
 	}
 
 	/**
@@ -422,7 +423,7 @@ private void validateModuleId(final List<String> row, final int lineNumber) {
 	}
 
 	private void validateEffectiveTime(String effectiveTime, final int lineNumber) {
-		if (ContentSubType.DELTA.equals(configuration.getVersion()) && effectiveTime.isEmpty()) {
+		if (effectiveTime.isEmpty()) {
 			return;
 		}
 		try {

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/validation/SnomedDescriptionValidator.java

@@ -105,6 +105,9 @@ protected void doValidate(String effectiveTime, IProgressMonitor monitor) {
 		addDefect(DefectType.DESCRIPTION_TYPE_NOT_EXIST, typeConceptNotExist);
 		addDefect(DefectType.DESCRIPTION_CASE_SIGNIFICANCE_NOT_EXIST, caseSignificanceConceptNotExist);
 
+		descriptionIdsWithEffectivetimeStatus.clear();
+		fullySpecifiedNames.clear();
+		descriptionIdNotUnique.clear();
 		fullySpecifiedNameNotUnique.clear();
 		descriptionConceptNotExist.clear();
 		typeConceptNotExist.clear();

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/validation/SnomedRefSetValidator.java

@@ -160,6 +160,7 @@ protected void doValidate(String effectiveTime, IProgressMonitor monitor) {
 		addDefect(DefectType.REFSET_MEMBER_COMPONENT_NOT_EXIST, referencedComponentNotExist);
 		uuidInvalid.clear();
 		referencedComponentNotExist.clear();
+		memberDataByUuid.clear();
 	}
 
 	/**returns with the proper import component type based on the component ID argument.*/

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/validation/SnomedRelationshipValidator.java

@@ -15,6 +15,7 @@
  */
 package com.b2international.snowowl.snomed.importer.rf2.validation;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.Collection;
 import java.util.List;
@@ -43,8 +44,8 @@ public class SnomedRelationshipValidator extends AbstractSnomedValidator {
 	private Collection<String> relationshipSourceAndDestinationAreEqual = Sets.newHashSet();
 	private Collection<String> missingReferencedConcepts = Sets.newHashSet();
 
-	public SnomedRelationshipValidator(final ImportConfiguration configuration, final SnomedValidationContext context) throws IOException {
-		super(configuration, configuration.toURL(configuration.getRelationshipsFile()), ComponentImportType.RELATIONSHIP, context, SnomedRf2Headers.RELATIONSHIP_HEADER);
+	public SnomedRelationshipValidator(final ImportConfiguration configuration, final SnomedValidationContext context, final File relationshipsFile) throws IOException {
+		super(configuration, configuration.toURL(relationshipsFile), ComponentImportType.RELATIONSHIP, context, SnomedRf2Headers.RELATIONSHIP_HEADER);
 	}
 
 	@Override
@@ -87,6 +88,8 @@ protected void doValidate(String effectiveTime, IProgressMonitor monitor) {
 		addDefect(DefectType.RELATIONSHIP_SOURCE_DESTINATION_EQUALS, relationshipSourceAndDestinationAreEqual);
 		addDefect(DefectType.RELATIONSHIP_REFERENCED_INVALID_CONCEPT, missingReferencedConcepts);
 
+		relationshipIdsWithEffectivetimeStatus.clear();
+		relationshipIdNotUnique.clear();
 		relationshipSourceAndDestinationAreEqual.clear();
 		missingReferencedConcepts.clear();
 	}

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/validation/SnomedTaxonomyValidator.java

@@ -35,7 +35,6 @@
 import com.b2international.snowowl.datastore.server.snomed.index.init.Rf2BasedSnomedTaxonomyBuilder;
 import com.b2international.snowowl.snomed.SnomedConstants.Concepts;
 import com.b2international.snowowl.snomed.datastore.index.entry.SnomedRelationshipIndexEntry;
-import com.b2international.snowowl.snomed.datastore.taxonomy.IncompleteTaxonomyException;
 import com.b2international.snowowl.snomed.datastore.taxonomy.InvalidRelationship;
 import com.b2international.snowowl.snomed.datastore.taxonomy.SnomedTaxonomyBuilder;
 import com.b2international.snowowl.snomed.datastore.taxonomy.SnomedTaxonomyBuilderResult;
@@ -45,9 +44,10 @@
 import com.b2international.snowowl.snomed.importer.net4j.SnomedValidationDefect;
 import com.b2international.snowowl.snomed.importer.rf2.RepositoryState;
 import com.b2international.snowowl.snomed.importer.rf2.util.Rf2FileModifier;
+import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSortedSet;
-import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
 
 /**
  * Class for validating the taxonomy of active concepts and active IS_A relationships.
@@ -113,6 +113,9 @@ public Collection<SnomedValidationDefect> validate() {
 	private Collection<SnomedValidationDefect> doValidate() {
 		try {
 			final Rf2BasedSnomedTaxonomyBuilder builder = Rf2BasedSnomedTaxonomyBuilder.newInstance(new SnomedTaxonomyBuilder(conceptIds, statements), characteristicType);
+
+			final Multimap<String, InvalidRelationship> invalidRelationships = ArrayListMultimap.create();
+
 			if (snapshot) {
 
 				LOGGER.info("Validating SNOMED CT ontology based on the given RF2 release files...");
@@ -130,7 +133,7 @@ private Collection<SnomedValidationDefect> doValidate() {
 
 				final SnomedTaxonomyBuilderResult result = builder.build();
 				if (!result.getStatus().isOK()) {
-					throw new IncompleteTaxonomyException(Lists.newArrayList(result.getInvalidRelationships()));
+					invalidRelationships.putAll("", result.getInvalidRelationships());
 				}
 			} else {
 
@@ -145,6 +148,7 @@ private Collection<SnomedValidationDefect> doValidate() {
 						.build()
 						.asList();
 
+
 				for (final String effectiveTime : effectiveTimes) {
 					LOGGER.info("Validating concepts and relationships from '" + effectiveTime + "'...");
 
@@ -153,54 +157,63 @@ private Collection<SnomedValidationDefect> doValidate() {
 
 					builder.applyNodeChanges(getFilePath(conceptFile));
 					builder.applyEdgeChanges(getFilePath(relationshipFile));
-					builder.build();
+					final SnomedTaxonomyBuilderResult result = builder.build();
+					if (!result.getStatus().isOK()) {
+						invalidRelationships.putAll(effectiveTime, result.getInvalidRelationships());
+					}
 				}
 			}
 
-		} catch (final IOException e) {
-			LOGGER.error("Validation failed with an IOException.", e);
-			return Collections.<SnomedValidationDefect>singleton(new SnomedValidationDefect(DefectType.IO_PROBLEM, Collections.<String>emptySet()));
-		} catch (final IncompleteTaxonomyException e) {
-			LOGGER.error("Validation failed.");
-			final Collection<String> defects = newHashSet();
-			final Collection<String> conceptIdsToInactivate = newHashSet();
-			for (final InvalidRelationship invalidRelationship: e.getInvalidRelationships()) {
-				final String sourceId = Long.toString(invalidRelationship.getSourceId());
-				final String destinationId = Long.toString(invalidRelationship.getDestinationId());
-
-				if (conceptIds.contains(invalidRelationship.getDestinationId())) {
-					conceptIdsToInactivate.add(destinationId);
-				}
+			if (!invalidRelationships.isEmpty()) {
+				final Collection<String> defects = newHashSet();
+				final Collection<String> conceptIdsToInactivate = newHashSet();
 
-				if (conceptIds.contains(invalidRelationship.getSourceId())) {
-					conceptIdsToInactivate.add(sourceId);
+				for (final String effectiveTime : invalidRelationships.keySet()) {
+					for (final InvalidRelationship invalidRelationship: invalidRelationships.get(effectiveTime)) {
+						final String sourceId = Long.toString(invalidRelationship.getSourceId());
+						final String destinationId = Long.toString(invalidRelationship.getDestinationId());
+
+						if (conceptIds.contains(invalidRelationship.getDestinationId())) {
+							conceptIdsToInactivate.add(destinationId);
+						}
+
+						if (conceptIds.contains(invalidRelationship.getSourceId())) {
+							conceptIdsToInactivate.add(sourceId);
+						}
+
+						final StringBuilder sb = new StringBuilder();
+						sb.append("IS A relationship");
+						sb.append(" '" + invalidRelationship.getRelationshipId() + "'");
+						sb.append(" has a missing or inactive ");
+
+						switch (invalidRelationship.getMissingConcept()) {
+						case DESTINATION:
+							sb.append("destination concept");
+							sb.append(" '" + destinationId);
+							break;
+						case SOURCE:
+							sb.append("source concept");
+							sb.append(" '" + sourceId);
+							break;
+						default:
+							throw new IllegalStateException("Unexpected missing concept type '" + invalidRelationship.getMissingConcept() + "'.");					
+						}
+
+						sb.append("' in effectiveTime ");
+						sb.append("".equals(effectiveTime) ? "Unpublished/Undefined" : effectiveTime);
+						sb.append("'.");
+
+						defects.add(sb.toString());
+					}
 				}
 
-				final StringBuilder sb = new StringBuilder();
-				sb.append("IS A relationship");
-				sb.append(" '" + invalidRelationship.getRelationshipId() + "'");
-				sb.append(" has a missing or inactive ");
-
-				switch (invalidRelationship.getMissingConcept()) {
-					case DESTINATION:
-						sb.append("destination concept");
-						sb.append(" '" + destinationId);
-						break;
-					case SOURCE:
-						sb.append("source concept");
-						sb.append(" '" + sourceId);
-						break;
-					default:
-						throw new IllegalStateException("Unexpected missing concept type '" + invalidRelationship.getMissingConcept() + "'.");					
-				}
-
-				sb.append("'.");
-
-				defects.add(sb.toString());
+				final SnomedValidationDefect defect = new SnomedIncompleteTaxonomyValidationDefect(relationshipsFile.getName(), defects, conceptIdsToInactivate);
+				return Collections.<SnomedValidationDefect>singleton(defect);
 			}
 
-			final SnomedValidationDefect defect = new SnomedIncompleteTaxonomyValidationDefect(defects, conceptIdsToInactivate);
-			return Collections.<SnomedValidationDefect>singleton(defect);
+		} catch (final IOException e) {
+			LOGGER.error("Validation failed.", e);
+			return Collections.<SnomedValidationDefect>singleton(new SnomedValidationDefect(relationshipsFile.getName(), DefectType.IO_PROBLEM, Collections.<String>emptySet()));
 		}
 
 		LOGGER.info("SNOMED CT ontology validation successfully finished. No errors were found.");