SO-2127 snomed rf2 importer issues

snomed/com.b2international.snowowl.snomed.datastore/src/com/b2international/snowowl/snomed/datastore/id/memory/DefaultSnomedIdentifierService.java

@@ -16,14 +16,17 @@
 package com.b2international.snowowl.snomed.datastore.id.memory;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.collect.Lists.newArrayListWithExpectedSize;
 import static com.google.common.collect.Sets.newHashSet;
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.eclipse.emf.cdo.spi.server.Store;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -213,18 +216,13 @@ public void register(final Collection<String> componentIds) {
 		LOGGER.debug(String.format("Registering %d component IDs.", componentIds.size()));
 
 		try {
-			final Collection<String> existingIds = getSctIds(componentIds).stream().filter(id -> id.matches(IdentifierStatus.AVAILABLE, IdentifierStatus.RESERVED)).map(SctId::getSctid).collect(Collectors.toSet());
-
-			for (final String componentId : componentIds) {
-				if (!existingIds.contains(componentId)) {
-					final SctId sctId = buildSctId(componentId, IdentifierStatus.ASSIGNED);
-					sctIds.put(componentId, sctId);
-					registeredComponentIds.add(componentId);
+			for (final SctId sctId : getSctIds(componentIds)) {
+				if (!sctId.matches(IdentifierStatus.AVAILABLE, IdentifierStatus.RESERVED)) {
+					LOGGER.warn("Cannot register ID {} as it is already present with status {}.", sctId.getSctid(), sctId.getStatus());
 				} else {
-					LOGGER.warn("Cannot register ID {} as it is already taken.", componentId);
+					sctId.setStatus(IdentifierStatus.ASSIGNED.getSerializedName());
 				}
 			}
-
 			putSctIds(sctIds);
 		} catch (Exception e) {
 			// remove the registered component IDs
@@ -307,7 +305,20 @@ public void publish(final Collection<String> componentIds) {
 
 	@Override
 	public Collection<SctId> getSctIds(final Collection<String> componentIds) {
-		return store.get().read(index -> index.search(Query.select(SctId.class).where(Expressions.matchAny(DocumentMapping._ID, componentIds)).limit(componentIds.size()).build()).getHits()); 
+		final List<SctId> existingIds = store.get().read(index -> index.search(Query.select(SctId.class).where(Expressions.matchAny(DocumentMapping._ID, componentIds)).limit(componentIds.size()).build()).getHits());
+		if (existingIds.size() == componentIds.size()) {
+			return existingIds;
+		} else {
+			final Set<String> existingIdentifiers = existingIds.stream().map(SctId::getSctid).collect(Collectors.toSet());
+			final List<SctId> ids = newArrayListWithExpectedSize(componentIds.size());
+			ids.addAll(existingIds);
+			for (String componentId : componentIds) {
+				if (!existingIdentifiers.contains(componentId)) {
+					ids.add(buildSctId(componentId, IdentifierStatus.AVAILABLE));
+				}
+			}
+			return ids;
+		}
 	}
 
 	private String generateId(final String namespace, final ComponentCategory category) {

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/model/AbstractSnomedImporter.java

@@ -403,7 +403,7 @@ private CellProcessor[] createValidatingCellProcessors() {
 	}
 
 	private ImportAction handlePreImportException(final SuperCSVException e) {
-		log("Exception caught while reading release file. Continuing with next row. {}", e.getMessage());
+		log("Exception caught while reading release file. Continuing with next row.", e);
 		importContext.getLogger().warn("Exception caught while reading release file. Continuing with next row.", e);
 		return ImportAction.CONTINUE;
 	}
@@ -437,7 +437,7 @@ private ComponentImportEntry getOrCreateImportEntry(final Map<String, ComponentI
 			try {
 				sliceStream = new FileOutputStream(sliceFile);
 			} catch (final FileNotFoundException e) {
-				log("SNOMED CT import failed. Couldn't open output file '{}' for writing. {}", sliceFile.getAbsolutePath(), e.getMessage());
+				log("SNOMED CT import failed. Couldn't open output file '{}' for writing.", sliceFile.getAbsolutePath(), e);
 				throw new ImportException("Couldn't open output file '" + sliceFile.getAbsolutePath() + "' for writing.", e);
 			}
 
@@ -534,7 +534,7 @@ public final void doImport(final SubMonitor subMonitor, final AbstractImportUnit
 					// process batch loaded rows and commit them
 					importRows(rows);
 					// reinit rows array
-					rows = Lists.newArrayListWithExpectedSize(COMMIT_EVERY_NUM_ELEMENTS);
+					rows.clear();
 					if (ImportAction.BREAK.equals(commit(subMonitor, effectiveTimeKey))) {
 						break;
 					}
@@ -557,8 +557,9 @@ protected int getImportWorkUnits(final int recordCount) {
 	}
 
 	private ImportAction handleImportException(final Throwable e) {
-		log("Exception caught while importing row from release file. Continuing with next row. {}", e.getMessage());
-		importContext.getLogger().warn("Exception caught while importing row from release file. Continuing with next row.", e);
+		final String message = "Exception caught while importing row from release file. Continuing with next row.";
+		log(message, e);
+		importContext.getLogger().warn(message, e);
 		return ImportAction.CONTINUE;
 	}
 
@@ -633,13 +634,13 @@ protected void preCommit(final InternalCDOTransaction transaction) throws Snowow
 	}
 
 	private ImportAction checkCommitException(final SnowowlServiceException e) {
-		log("SNOMED CT import failed. Caught exception while import, aborting. {}", e.getMessage());
+		log("SNOMED CT import failed. Caught exception while import, aborting. {}", e);
 		importContext.getLogger().warn("Caught exception while import, aborting.", e);
 		return ImportAction.BREAK;
 	}
 
 	private ImportAction checkCommitException(final CommitException e) {
-		log("SNOMED CT import failed. Caught exception while import, aborting. {}", e.getMessage());
+		log("SNOMED CT import failed. Caught exception while import, aborting. {}", e);
 		importContext.getLogger().warn("Caught exception while import, aborting.", e);
 		handleCommitException();
 		return ImportAction.BREAK;
@@ -665,10 +666,9 @@ private final void importRows(List<T> rows) {
 		final Map<String, C> existingComponents = loadComponents(rowsToImport.keySet()).stream().collect(Collectors.toMap(getComponentIdMapper(), c -> c));
 		// create or update components
 		final Collection<C> componentsToAttach = newHashSet();
-		for (Entry<String, T> rowToImport : rowsToImport.entrySet()) {
-			final String componentId = rowToImport.getKey();
+		for (final String componentId : rowsToImport.keySet()) {
+			final T row = rowsToImport.get(componentId);
 			C component = existingComponents.get(componentId);
-			final T row = rowToImport.getValue();
 			if (component == null) {
 				// XXX some RF2 rows might already introduced the component with just the ID
 				component = getOrCreateNew(componentId, componentsToAttach);

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/refset/AbstractSnomedRefSetImporter.java

@@ -217,12 +217,12 @@ protected void registerNewComponent(M component) {
 
 	@Override
 	protected final M createComponent(String memberId) {
-		final M member = createComponent();
+		final M member = createMember();
 		member.setUuid(memberId);
 		return member;
 	}
 
-	protected abstract M createComponent();
+	protected abstract M createMember();
 
 	protected SnomedRefSet getOrCreateRefSet(final String refSetSctId, final String referencedComponentId) {
 

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/refset/RefSetMemberLookup.java

@@ -125,7 +125,6 @@ private LongValueMap<String> getStorageKeys(final Collection<String> componentId
 			@Override
 			public LongValueMap<String> execute(RevisionSearcher index) throws IOException {
 				final LongValueMap<String> map = PrimitiveMaps.newObjectKeyLongOpenHashMapWithExpectedSize(componentIds.size());
-				// index componentIds by their category
 				final Query<SnomedRefSetMemberIndexEntry> query = Query.select(SnomedRefSetMemberIndexEntry.class)
 						.where(SnomedRefSetMemberIndexEntry.Expressions.ids(componentIds))
 						.limit(componentIds.size())

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/refset/SnomedAssociationRefSetImporter.java

@@ -81,7 +81,7 @@ protected SnomedRefSetType getRefSetType() {
 	}
 
 	@Override
-	protected SnomedAssociationRefSetMember createComponent() {
+	protected SnomedAssociationRefSetMember createMember() {
 		return SnomedRefSetFactory.eINSTANCE.createSnomedAssociationRefSetMember();
 	}
 

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/refset/SnomedAttributeValueRefSetImporter.java

@@ -116,7 +116,7 @@ protected String getIdentifierParentConceptId(final String refSetId) {
 	}
 
 	@Override
-	protected SnomedAttributeValueRefSetMember createComponent() {
+	protected SnomedAttributeValueRefSetMember createMember() {
 		return SnomedRefSetFactory.eINSTANCE.createSnomedAttributeValueRefSetMember();
 	}
 

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/refset/SnomedComplexMapTypeRefSetImporter.java

@@ -136,7 +136,7 @@ protected String getIdentifierParentConceptId(final String refSetId) {
 	}
 
 	@Override
-	protected SnomedComplexMapRefSetMember createComponent() {
+	protected SnomedComplexMapRefSetMember createMember() {
 		return SnomedRefSetFactory.eINSTANCE.createSnomedComplexMapRefSetMember();
 	}
 }

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/refset/SnomedConcreteDataTypeRefSetImporter.java

@@ -161,7 +161,7 @@ protected String getIdentifierParentConceptId(final String refSetId) {
 	}
 
 	@Override
-	protected SnomedConcreteDataTypeRefSetMember createComponent() {
+	protected SnomedConcreteDataTypeRefSetMember createMember() {
 		return SnomedRefSetFactory.eINSTANCE.createSnomedConcreteDataTypeRefSetMember();
 	}
 

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/refset/SnomedDescriptionTypeRefSetImporter.java

@@ -103,7 +103,7 @@ protected String getIdentifierParentConceptId(final String refSetId) {
 	}
 
 	@Override
-	protected SnomedDescriptionTypeRefSetMember createComponent() {
+	protected SnomedDescriptionTypeRefSetMember createMember() {
 		return SnomedRefSetFactory.eINSTANCE.createSnomedDescriptionTypeRefSetMember();
 	}
 }

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/refset/SnomedLanguageRefSetImporter.java

@@ -102,7 +102,7 @@ protected String getIdentifierParentConceptId(final String refSetId) {
 	}
 
 	@Override
-	protected SnomedLanguageRefSetMember createComponent() {
+	protected SnomedLanguageRefSetMember createMember() {
 		return SnomedRefSetFactory.eINSTANCE.createSnomedLanguageRefSetMember();
 	}
 

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/refset/SnomedModuleDependencyRefSetImporter.java

@@ -95,7 +95,7 @@ protected String getIdentifierParentConceptId(String refSetId) {
 	}
 
 	@Override
-	protected SnomedModuleDependencyRefSetMember createComponent() {
+	protected SnomedModuleDependencyRefSetMember createMember() {
 		return SnomedRefSetFactory.eINSTANCE.createSnomedModuleDependencyRefSetMember();
 	}
 

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/refset/SnomedQueryRefSetImporter.java

@@ -102,7 +102,7 @@ protected String getIdentifierParentConceptId(final String refSetId) {
 	}
 
 	@Override
-	protected SnomedQueryRefSetMember createComponent() {
+	protected SnomedQueryRefSetMember createMember() {
 		return SnomedRefSetFactory.eINSTANCE.createSnomedQueryRefSetMember(); 
 	}
 }

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/refset/SnomedSimpleMapTypeRefSetImporter.java

@@ -49,11 +49,11 @@ public class SnomedSimpleMapTypeRefSetImporter extends AbstractSnomedMapTypeRefS
 			.add(new IndexConfiguration("SNOMEDREFSET_SNOMEDSIMPLEMAPREFSETMEMBER_IDX1003", "SNOMEDREFSET_SNOMEDSIMPLEMAPREFSETMEMBER", "UUID/*!(255)*/", "CDO_BRANCH", "CDO_VERSION"))
 			.add(new IndexConfiguration("SNOMEDREFSET_SNOMEDSIMPLEMAPREFSETMEMBER_IDX1004", "SNOMEDREFSET_SNOMEDSIMPLEMAPREFSETMEMBER", "MAPTARGETCOMPONENTID/*!(255)*/", "CDO_BRANCH", "CDO_VERSION"))
 			.build();
-	private boolean extended;
+	private boolean hasMapTargetDescription;
 
-	public SnomedSimpleMapTypeRefSetImporter(final SnomedImportContext importContext, final InputStream releaseFileStream, boolean extended, final String releaseFileIdentifier) {
-		super(createImportConfiguration(extended), importContext, releaseFileStream, releaseFileIdentifier);
-		this.extended = extended;
+	public SnomedSimpleMapTypeRefSetImporter(final SnomedImportContext importContext, final InputStream releaseFileStream, boolean hasMapTargetDescription, final String releaseFileIdentifier) {
+		super(createImportConfiguration(hasMapTargetDescription), importContext, releaseFileStream, releaseFileIdentifier);
+		this.hasMapTargetDescription = hasMapTargetDescription;
 	}
 
 	private static SnomedImportConfiguration<SimpleMapRefSetRow> createImportConfiguration(boolean extended) {
@@ -109,7 +109,7 @@ protected void applyRow(SnomedSimpleMapRefSetMember member, SimpleMapRefSetRow r
 		member.setReferencedComponentId(row.getReferencedComponentId());
 		member.setMapTargetComponentId(row.getAssociatedComponentId());
 
-		if (extended) {
+		if (hasMapTargetDescription) {
 			member.setMapTargetComponentDescription(((SimpleMapRefSetRow) row).getMapTargetDescription());
 		}		
 	}
@@ -120,7 +120,7 @@ protected String getIdentifierParentConceptId(final String refSetId) {
 	}
 
 	@Override
-	protected SnomedSimpleMapRefSetMember createComponent() {
+	protected SnomedSimpleMapRefSetMember createMember() {
 		return SnomedRefSetFactory.eINSTANCE.createSnomedSimpleMapRefSetMember();
 	}
 }

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/refset/SnomedSimpleTypeRefSetImporter.java

@@ -93,7 +93,7 @@ protected String getIdentifierParentConceptId(final String refSetId) {
 	}
 
 	@Override
-	protected SnomedRefSetMember createComponent() {
+	protected SnomedRefSetMember createMember() {
 		return SnomedRefSetFactory.eINSTANCE.createSnomedRefSetMember();
 	}
 }

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/terminology/AbstractSnomedTerminologyImporter.java

@@ -116,12 +116,12 @@ protected final Function<C, String> getComponentIdMapper() {
 
 	@Override
 	protected C createComponent(String componentId) {
-		final C component = createComponent();
+		final C component = createCoreComponent();
 		component.setId(componentId);
 		return component;
 	}
 
-	protected abstract C createComponent();
+	protected abstract C createCoreComponent();
 
 	protected final Concept getConceptSafe(final String conceptId, final String conceptField, final String componentId) {
 		final Concept result = (Concept) Iterables.getOnlyElement(getComponents(Collections.singleton(conceptId)), null);

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/terminology/ComponentLookup.java

@@ -171,6 +171,7 @@ public LongValueMap<String> execute(RevisionSearcher index) throws IOException {
 				// index componentIds by their category
 				final Multimap<Class<? extends SnomedDocument>, String> idsByType = Multimaps.index(componentIds, id -> SnomedDocument.getType(SnomedIdentifiers.getComponentCategory(id)));
 				for (Class<? extends SnomedDocument> type : idsByType.keySet()) {
+					// execute queries for each type and based on the current clazz extract either refset or concept storage keys
 					final Query<? extends SnomedDocument> query = Query.select(type).where(SnomedDocument.Expressions.ids(componentIds)).limit(componentIds.size()).build();
 					final Hits<? extends SnomedDocument> hits = index.search(query);
 					for (SnomedDocument doc : hits) {
@@ -181,7 +182,6 @@ public LongValueMap<String> execute(RevisionSearcher index) throws IOException {
 						}
 					}
 				}
-				// execute queries for each type
 				return map;
 			}
 		});

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/terminology/SnomedConceptImporter.java

@@ -70,7 +70,7 @@ protected Class<? extends SnomedDocument> getType() {
 	}
 
 	@Override
-	protected Concept createComponent() {
+	protected Concept createCoreComponent() {
 		return SnomedFactory.eINSTANCE.createConcept();
 	}
 
@@ -81,7 +81,6 @@ protected void attach(Collection<Concept> componentsToAttach) {
 
 	@Override
 	protected void applyRow(Concept component, ConceptRow row, Collection<Concept> componentsToAttach) {
-		component.setId(row.getId());
 		if (row.getEffectiveTime() != null) {
 			component.setEffectiveTime(row.getEffectiveTime());
 			component.setReleased(true);

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/terminology/SnomedDescriptionImporter.java

@@ -78,7 +78,6 @@ protected Class<? extends SnomedDocument> getType() {
 
 	@Override
 	protected void applyRow(Description component, DescriptionRow row, Collection<Description> componentsToAttach) {
-		component.setId(row.getId());
 		if (row.getEffectiveTime() != null) {
 			component.setEffectiveTime(row.getEffectiveTime());
 			component.setReleased(true);
@@ -103,7 +102,7 @@ protected void attach(Collection<Description> componentsToAttach) {
 	}
 
 	@Override
-	protected Description createComponent() {
+	protected Description createCoreComponent() {
 		return SnomedFactory.eINSTANCE.createDescription();
 	}
 

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/terminology/SnomedRelationshipImporter.java

@@ -86,7 +86,6 @@ protected Class<? extends SnomedDocument> getType() {
 
 	@Override
 	protected void applyRow(Relationship component, RelationshipRow row, Collection<Relationship> componentsToAttach) {
-		component.setId(row.getId());
 		if (row.getEffectiveTime() != null) {
 			component.setEffectiveTime(row.getEffectiveTime());
 			component.setReleased(true);
@@ -122,7 +121,7 @@ protected void attach(Collection<Relationship> componentsToAttach) {
 	}
 
 	@Override
-	protected Relationship createComponent() {
+	protected Relationship createCoreComponent() {
 		return SnomedFactory.eINSTANCE.createRelationship();
 	}
 

snomed/com.b2international.snowowl.snomed.importer.rf2/src/com/b2international/snowowl/snomed/importer/rf2/util/Rf2FileModifier.java

@@ -38,7 +38,7 @@
  */
 public class Rf2FileModifier {
 
-	private static final Splitter ON = Splitter.on('\t');
+	private static final Splitter TAB_SPLITTER = Splitter.on('\t');
 
 	/**
 	 * Splits the given RF2 files into RF2 file segments based on distinct effective times. The result will be a map
@@ -68,7 +68,7 @@ public static Map<String, File> split(final File toSplit) throws IOException {
 					continue;
 				}
 
-				final List<String> values = ON.splitToList(line);
+				final List<String> values = TAB_SPLITTER.splitToList(line);
 				final String effectiveTime = values.get(1);
 
 				final PrintWriter effectiveTimeWriter;