SO-2127 snomed rf2 importer issues

core/com.b2international.snowowl.datastore.server/src/com/b2international/snowowl/datastore/server/CDOServerChangeManager.java

@@ -264,9 +264,19 @@ private Map<CDOID, CDORevisionDelta> getRevisionDeltas(final CDORevisionDelta[]
 
 	/*creates a change manager instance from the underlying transaction commit context*/
 	private DelegateCDOServerChangeManager createChangeManager(final TransactionCommitContext commitContext) {
-		return new DelegateCDOServerChangeManager(getCommitChangeSet(commitContext), factories, true);
+		final boolean isCommitNotificationEnabled = isCommitNotificationEnabled(commitContext);
+		return new DelegateCDOServerChangeManager(getCommitChangeSet(commitContext), factories, true, isCommitNotificationEnabled);
 	}
 
+	private boolean isCommitNotificationEnabled(TransactionCommitContext commitContext) {
+		if (commitContext instanceof CDOServerCommitBuilder.ServerTransactionCommitContext) {
+			return ((CDOServerCommitBuilder.ServerTransactionCommitContext) commitContext).isCommitNotificationEnabled();
+		} else {
+			// by default all non server commits should send out commit notifications
+			return true;
+		}
+	}
+
 	/*returns with the branch path extracted from the underlying transaction commit context.*/
 	private IBranchPath getBranchPath(final TransactionCommitContext commitContext) {
 		return createPath(checkNotNull(checkNotNull(checkNotNull(commitContext, "commitContext").getTransaction(), "transaction").getBranch()));

core/com.b2international.snowowl.datastore.server/src/com/b2international/snowowl/datastore/server/CDOServerCommitBuilder.java

@@ -52,7 +52,6 @@
 import org.eclipse.emf.cdo.common.revision.delta.CDORevisionDelta;
 import org.eclipse.emf.cdo.internal.common.commit.FailureCommitInfo;
 import org.eclipse.emf.cdo.internal.server.Repository;
-import org.eclipse.emf.internal.cdo.view.InternalCDOTransactionWrapper;
 import org.eclipse.emf.cdo.server.IStoreAccessor;
 import org.eclipse.emf.cdo.server.IView;
 import org.eclipse.emf.cdo.server.StoreThreadLocal;
@@ -75,6 +74,7 @@
 import org.eclipse.emf.cdo.view.CDOViewTargetChangedEvent;
 import org.eclipse.emf.internal.cdo.object.CDOObjectReferenceImpl;
 import org.eclipse.emf.internal.cdo.view.CDOStateMachine;
+import org.eclipse.emf.internal.cdo.view.InternalCDOTransactionWrapper;
 import org.eclipse.emf.spi.cdo.CDOSessionProtocol.CommitTransactionResult;
 import org.eclipse.emf.spi.cdo.InternalCDOObject;
 import org.eclipse.emf.spi.cdo.InternalCDOSession;
@@ -158,7 +158,7 @@ public void sendCommitNotification(final InternalSession sender, final CDOCommit
 	 * <p>
 	 * This class is non-static to allow direct access to fields in the containing builder class.
 	 */
-	private class ServerTransactionCommitContext extends CustomTransactionCommitContext {
+	/*package*/ class ServerTransactionCommitContext extends CustomTransactionCommitContext {
 
 		private ServerTransactionCommitContext(final InternalTransaction transaction, final IErrorLoggingStrategy strategy) {
 			super(transaction, strategy);
@@ -173,6 +173,10 @@ public String getUserID() {
 		protected String getParentContextDescription() { 
 			return parentContextDescription; 
 		}
+
+		public final boolean isCommitNotificationEnabled() {
+			return sendCommitNotification;
+		}
 	}
 
 	/**
@@ -195,7 +199,7 @@ private ServerTransaction(final InternalTransaction delegate, final CDOTransacti
 		public InternalRepository getRepository() {
 			return new ServerTransactionRepository(repository, transaction, notifyWriteAccessHandlers);
 		}
-
+		
 		@Override 
 		public ServerTransactionCommitContext createCommitContext() {
 			return new ServerTransactionCommitContext(this, FilteringErrorLoggingStrategy.INSTANCE);

core/com.b2international.snowowl.datastore.server/src/com/b2international/snowowl/datastore/server/DelegateCDOServerChangeManager.java

@@ -79,11 +79,14 @@ public class DelegateCDOServerChangeManager {
 	private final ICDOCommitChangeSet commitChangeSet;
 	private final IBranchPath branchPath;
 	private final String repositoryUuid;
+	private final boolean isCommitNotificationEnabled;
 
 	private @Nullable IOperationLockTarget lockTarget;
 	private @Nullable Collection<ICDOChangeProcessor> changeProcessors;
+
 
-	public DelegateCDOServerChangeManager(final ICDOCommitChangeSet commitChangeSet, final Collection<CDOChangeProcessorFactory> factories, final boolean copySession) {
+	public DelegateCDOServerChangeManager(final ICDOCommitChangeSet commitChangeSet, final Collection<CDOChangeProcessorFactory> factories, final boolean copySession, boolean isCommitNotificationEnabled) {
+		this.isCommitNotificationEnabled = isCommitNotificationEnabled;
 		this.commitChangeSet = Preconditions.checkNotNull(commitChangeSet, "Commit change set data argument cannot be null.");
 		final CDOView view = commitChangeSet.getView();
 		this.repositoryUuid = ApplicationContext.getInstance().getService(ICDOConnectionManager.class).get(view).getUuid();
@@ -232,10 +235,12 @@ public void handleTransactionAfterCommitted() {
 
 			ForkJoinUtils.runJobsInParallelWithErrorHandling(commitJobs, null);
 			// queue commit notification
-			final IndexCommitChangeSet mergedChangeSet = merge(indexCommitChangeSets);
-			getContext().getService(RepositoryManager.class)
-				.get(repositoryUuid)
-				.sendNotification(toCommitNotification(mergedChangeSet));
+			if (isCommitNotificationEnabled) {
+				final IndexCommitChangeSet mergedChangeSet = merge(indexCommitChangeSets);
+				getContext().getService(RepositoryManager.class)
+					.get(repositoryUuid)
+					.sendNotification(toCommitNotification(mergedChangeSet));
+			}
 		} catch (final Exception e) {
 			caughtException = new SnowowlRuntimeException("Error when committing change processors on branch: " + branchPath, e);
 		} finally {

core/com.b2international.snowowl.datastore.server/src/com/b2international/snowowl/datastore/server/internal/CDOBasedRepository.java

@@ -384,7 +384,7 @@ public void handleCommitInfo(CDOCommitInfo commitInfo) {
 			final long commitTimestamp = commitInfo.getTimeStamp();
 			((CDOBranchManagerImpl) service(BranchManager.class)).handleCommit(branch.getID(), commitTimestamp);
 			// send out the currently enqueued commit notification, if there is any (import might skip sending commit notifications until a certain point)
-			final RepositoryCommitNotification notification = commitNotifications.get(commitTimestamp);
+			final RepositoryCommitNotification notification = commitNotifications.remove(commitTimestamp);
 			if (notification != null) {
 				notification.publish(events());
 			}

snomed/com.b2international.snowowl.snomed.datastore.server/src/com/b2international/snowowl/datastore/server/snomed/index/SnomedCDOChangeProcessor.java

@@ -266,7 +266,10 @@ private void prepareTaxonomyBuilders(final ICDOCommitChangeSet commitChangeSet,
 
 		final FeatureToggles features = ApplicationContext.getServiceForClass(FeatureToggles.class);
 		final String reindexFeature = ReindexRequest.featureFor(SnomedDatastoreActivator.REPOSITORY_UUID);
-		final boolean checkCycles = features.exists(reindexFeature) ? !features.check(reindexFeature) : true;
+		final String importFeature = SnomedDatastoreActivator.REPOSITORY_UUID + ".import";
+		final boolean importRunning = features.exists(importFeature) ? features.check(importFeature) : false;
+		final boolean reindexRunning = features.exists(reindexFeature) ? features.check(reindexFeature) : false;
+		final boolean checkCycles = !importRunning && !reindexRunning;
 
 		final Runnable inferredRunnable = CDOServerUtils.withAccessor(new Runnable() {
 			@Override

snomed/com.b2international.snowowl.snomed.datastore.server/src/com/b2international/snowowl/datastore/server/snomed/index/init/Rf2BasedSnomedTaxonomyBuilder.java

@@ -88,10 +88,12 @@ public LongKeyMap<long[]> getEdges() {
 	public void applyNodeChanges(final String conceptFilePath) {
 		parseFile(conceptFilePath, 5, new RecordParserCallback<String>() {
 			@Override public void handleRecord(final int recordCount, final List<String> record) {
+				final String id = record.get(0);
+				final boolean active = ACTIVE_STATUS.equals(record.get(2));
 
 				final TaxonomyBuilderNode node = new TaxonomyBuilderNode() {
-					@Override public boolean isCurrent() { return ACTIVE_STATUS.equals(record.get(2)); }
-					@Override public String getId() { return record.get(0); }
+					@Override public boolean isCurrent() { return active; }
+					@Override public String getId() { return id; }
 				};
 
 				if (node.isCurrent()) {
@@ -107,22 +109,28 @@ public void applyNodeChanges(final String conceptFilePath) {
 	public void applyEdgeChanges(final String relationshipFilePath) {
 		parseFile(relationshipFilePath, 10, new RecordParserCallback<String>() {
 			@Override public void handleRecord(final int recordCount, final List<String> record) {
+				final String id = record.get(0);
+				final boolean active = ACTIVE_STATUS.equals(record.get(2));
+				final String sourceId = record.get(4);
+				final String destinationId = record.get(5);
+				final String typeId = record.get(7);
+				final String rowCharacteristicTypeId = record.get(8);
 
 				addEdge(new TaxonomyBuilderEdge() {
 					@Override public boolean isCurrent() {
-						return ACTIVE_STATUS.equals(record.get(2));
+						return active;
 					}
 					@Override public String getId() {
-						return record.get(0);
+						return id;
 					}
 					@Override public boolean isValid() {
-						return Concepts.IS_A.equals(record.get(7)) && characteristicTypeId.equals(record.get(8));
+						return Concepts.IS_A.equals(typeId) && characteristicTypeId.equals(rowCharacteristicTypeId);
 					}
 					@Override public String getSoureId() {
-						return record.get(4);
+						return sourceId;
 					}
 					@Override public String getDestinationId() {
-						return record.get(5);
+						return destinationId;
 					}
 				});
 

snomed/com.b2international.snowowl.snomed.datastore/src/com/b2international/snowowl/snomed/datastore/SnomedRefSetUtil.java

@@ -205,7 +205,7 @@ public static String getSpecificCondition(final SnomedRefSetType type) {
 	 * @return the identifier concept ID of the SNOMED CT concrete domain reference set.
 	 */
 	public static String getRefSetId(final DataType dataType) {
-		return DATATYPE_TO_REFSET_MAP.get(dataType);
+		return getConcreteDomainRefSetMap().get(dataType);
 	}
 
 	/**
@@ -214,8 +214,8 @@ public static String getRefSetId(final DataType dataType) {
 	 * @return the proper datatype for the specified reference set id
 	 */
 	public static DataType getDataType(String refsetId) {
-		if (DATATYPE_TO_REFSET_MAP.inverse().containsKey(refsetId)) {
-			return DATATYPE_TO_REFSET_MAP.inverse().get(refsetId);
+		if (getConcreteDomainRefSetMap().inverse().containsKey(refsetId)) {
+			return getConcreteDomainRefSetMap().inverse().get(refsetId);
 		} else {
 			return AMT_REFSET_TO_DATATYPE_MAP.get(refsetId);
 		}

snomed/com.b2international.snowowl.snomed.datastore/src/com/b2international/snowowl/snomed/datastore/id/memory/DefaultSnomedIdentifierService.java

@@ -16,13 +16,17 @@
 package com.b2international.snowowl.snomed.datastore.id.memory;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.collect.Lists.newArrayListWithExpectedSize;
 import static com.google.common.collect.Sets.newHashSet;
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
+import org.eclipse.emf.cdo.spi.server.Store;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -32,6 +36,7 @@
 import com.b2international.index.IndexWrite;
 import com.b2international.index.Searcher;
 import com.b2international.index.Writer;
+import com.b2international.index.mapping.DocumentMapping;
 import com.b2international.index.query.Expressions;
 import com.b2international.index.query.Query;
 import com.b2international.snowowl.core.exceptions.BadRequestException;
@@ -211,19 +216,13 @@ public void register(final Collection<String> componentIds) {
 		LOGGER.debug(String.format("Registering %d component IDs.", componentIds.size()));
 
 		try {
-			for (final String componentId : componentIds) {
-				final SctId sctId = getSctId(componentId);
-
+			for (final SctId sctId : getSctIds(componentIds)) {
 				if (!sctId.matches(IdentifierStatus.AVAILABLE, IdentifierStatus.RESERVED)) {
-					LOGGER.warn(String.format("Cannot register ID %s as it is already present with status %s.", componentId,
-							sctId.getStatus()));
+					LOGGER.warn("Cannot register ID {} as it is already present with status {}.", sctId.getSctid(), sctId.getStatus());
 				} else {
 					sctId.setStatus(IdentifierStatus.ASSIGNED.getSerializedName());
-					sctIds.put(componentId, sctId);
-					registeredComponentIds.add(componentId);
 				}
 			}
-
 			putSctIds(sctIds);
 		} catch (Exception e) {
 			// remove the registered component IDs
@@ -257,12 +256,10 @@ public void deprecate(final Collection<String> componentIds) {
 
 		LOGGER.debug(String.format("Deprecating %d component IDs.", componentIds.size()));
 
-		for (final String componentId : componentIds) {
-			final SctId sctId = getSctId(componentId);
-
+		for (final SctId sctId : getSctIds(componentIds)) {
 			if (sctId.matches(IdentifierStatus.ASSIGNED, IdentifierStatus.PUBLISHED)) {
 				sctId.setStatus(IdentifierStatus.DEPRECATED.getSerializedName());
-				deprecatedSctIds.put(componentId, sctId);
+				deprecatedSctIds.put(sctId.getSctid(), sctId);
 			} else if (!sctId.isDeprecated()) {
 				throw new BadRequestException(String.format("Cannot deprecate ID in state %s.", sctId.getStatus()));
 			}
@@ -277,11 +274,9 @@ public void release(final Collection<String> componentIds) {
 
 		LOGGER.debug(String.format("Releasing %d component IDs.", componentIds.size()));
 
-		for (final String componentId : componentIds) {
-			final SctId sctId = getSctId(componentId);
-
+		for (final SctId sctId : getSctIds(componentIds)) {
 			if (sctId.matches(IdentifierStatus.ASSIGNED, IdentifierStatus.RESERVED)) {
-				releasedComponentIds.add(componentId);
+				releasedComponentIds.add(sctId.getSctid());
 			} else if (!sctId.isAvailable()) {
 				throw new BadRequestException(String.format("Cannot release ID in state %s.", sctId.getStatus()));
 			}
@@ -296,12 +291,10 @@ public void publish(final Collection<String> componentIds) {
 
 		LOGGER.debug("Publishing {} component IDs.", componentIds.size());
 
-		for (final String componentId : componentIds) {
-			final SctId sctId = getSctId(componentId);
-
+		for (final SctId sctId : getSctIds(componentIds)) {
 			if (sctId.isAssigned()) {
 				sctId.setStatus(IdentifierStatus.PUBLISHED.getSerializedName());
-				publishedSctIds.put(componentId, sctId);
+				publishedSctIds.put(sctId.getSctid(), sctId);
 			} else if (!sctId.isPublished()) {
 				throw new BadRequestException("Cannot publish ID '%s' in state '%s'.", sctId.getSctid(), sctId.getStatus());
 			}
@@ -312,13 +305,20 @@ public void publish(final Collection<String> componentIds) {
 
 	@Override
 	public Collection<SctId> getSctIds(final Collection<String> componentIds) {
-		LOGGER.debug(String.format("Getting %d SctIds.", componentIds.size()));
-		final Collection<SctId> sctIds = Lists.newArrayList();
-		for (final String componentId : componentIds) {
-			sctIds.add(getSctId(componentId));
+		final List<SctId> existingIds = store.get().read(index -> index.search(Query.select(SctId.class).where(Expressions.matchAny(DocumentMapping._ID, componentIds)).limit(componentIds.size()).build()).getHits());
+		if (existingIds.size() == componentIds.size()) {
+			return existingIds;
+		} else {
+			final Set<String> existingIdentifiers = existingIds.stream().map(SctId::getSctid).collect(Collectors.toSet());
+			final List<SctId> ids = newArrayListWithExpectedSize(componentIds.size());
+			ids.addAll(existingIds);
+			for (String componentId : componentIds) {
+				if (!existingIdentifiers.contains(componentId)) {
+					ids.add(buildSctId(componentId, IdentifierStatus.AVAILABLE));
+				}
+			}
+			return ids;
 		}
-
-		return sctIds;
 	}
 
 	private String generateId(final String namespace, final ComponentCategory category) {
@@ -425,5 +425,5 @@ public SctId execute(Searcher index) throws IOException {
 			}
 		});
 	}
-
+	
 }

snomed/com.b2international.snowowl.snomed.datastore/src/com/b2international/snowowl/snomed/datastore/index/entry/SnomedConceptDocument.java

@@ -51,6 +51,7 @@
 import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
 import com.google.common.base.Function;
 import com.google.common.base.Objects.ToStringHelper;
+import com.google.common.base.Strings;
 import com.google.common.collect.FluentIterable;
 
 /**
@@ -297,10 +298,13 @@ public Builder refSet(final SnomedReferenceSet refSet) {
 				mapTargetComponentType(componentType);
 			}
 
-			return structural(SnomedRefSetUtil.isStructural(refSet.getId(), refSet.getType()))
+			Builder b = structural(SnomedRefSetUtil.isStructural(refSet.getId(), refSet.getType()))
 					.refSetType(refSet.getType())
-					.referencedComponentType(getValue(refSet.getReferencedComponentType()))
 					.refSetStorageKey(refSet.getStorageKey());
+			if (!Strings.isNullOrEmpty(refSet.getReferencedComponentType())) {
+				b.referencedComponentType(getValue(refSet.getReferencedComponentType()));
+			}
+			return b;
 		}
 
 		Builder mapTargetComponentType(int mapTargetComponentType) {

snomed/com.b2international.snowowl.snomed.datastore/src/com/b2international/snowowl/snomed/datastore/index/entry/SnomedDescriptionIndexEntry.java

@@ -22,7 +22,6 @@
 import static com.b2international.index.query.Expressions.matchTextFuzzy;
 import static com.b2international.index.query.Expressions.matchTextParsed;
 import static com.b2international.index.query.Expressions.matchTextPhrase;
-import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.collect.Sets.newHashSet;
 
 import java.util.Collection;
@@ -397,12 +396,12 @@ private SnomedDescriptionIndexEntry(final String id,
 				referringRefSets,
 				referringMappingRefSets);
 
-		this.conceptId = checkNotNull(conceptId, "Description concept identifier may not be null.");
-		this.languageCode = checkNotNull(languageCode, "Description language code may not be null.");
-		this.term = checkNotNull(term, "Description term may not be null.");
-		this.typeId = checkNotNull(typeId, "Description type identifier may not be null.");
+		this.conceptId = conceptId;
+		this.languageCode = languageCode;
+		this.term = term;
+		this.typeId = typeId;
 		this.typeLabel = typeLabel;
-		this.caseSignificanceId = checkNotNull(caseSignificanceId, "Description case significance identifier may not be null.");
+		this.caseSignificanceId = caseSignificanceId;
 		this.preferredIn = preferredIn == null ? Collections.<String>emptySet() : preferredIn;
 		this.acceptableIn = acceptableIn == null ? Collections.<String>emptySet() : acceptableIn;
 	}