Skip to content

Commit

Permalink
storage: Add status to SampleIndexConfiguration #1901
Browse files Browse the repository at this point in the history
  • Loading branch information
@j-coll
j-coll committed Mar 2, 2022
1 parent 947d2ba commit 09b075c
Show file tree
Hide file tree
Showing 14 changed files with 225 additions and 292 deletions.
2 changes: 1 addition & 1 deletion ...ysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,7 +497,7 @@ public OpenCGAResult<Job> configureSampleIndex(String studyStr, SampleIndexConfi
return secureOperation("configure", studyStr, new ObjectMap(), token, engine -> {
sampleIndexConfiguration.validate();
String studyFqn = getStudyFqn(studyStr, token);
engine.getMetadataManager().addSampleIndexConfiguration(studyFqn, sampleIndexConfiguration);
engine.getMetadataManager().addSampleIndexConfiguration(studyFqn, sampleIndexConfiguration, true);

catalogManager.getStudyManager()
.setVariantEngineConfigurationSampleIndex(studyStr, sampleIndexConfiguration, token);
Expand Down
3 changes: 2 additions & 1 deletion ...ava/org/opencb/opencga/app/migrations/v2_1_0/storage/DefaultSampleIndexConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,8 @@ protected void run() throws Exception {
if (CollectionUtils.isEmpty(studyMetadata.getSampleIndexConfigurations())) {
studyMetadata.setSampleIndexConfigurations(new ArrayList<>());
studyMetadata.getSampleIndexConfigurations().add(
new StudyMetadata.SampleIndexConfigurationVersioned(sampleIndexConfiguration, 1, Date.from(Instant.now())));
new StudyMetadata.SampleIndexConfigurationVersioned(sampleIndexConfiguration, 1, Date.from(Instant.now()),
StudyMetadata.SampleIndexConfigurationVersioned.Status.ACTIVE));
} else {
int size = studyMetadata.getSampleIndexConfigurations().size();
studyMetadata.getSampleIndexConfigurations().get(size - 1).setConfiguration(sampleIndexConfiguration);
Expand Down
20 changes: 15 additions & 5 deletions ...src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,23 +208,33 @@ public StudyMetadata createStudy(String studyName) throws StorageEngineException
}

public StudyMetadata.SampleIndexConfigurationVersioned addSampleIndexConfiguration(
String study, SampleIndexConfiguration configuration) throws StorageEngineException {
String study, SampleIndexConfiguration configuration, boolean staging) throws StorageEngineException {
Integer idOrNull = getStudyIdOrNull(study);
int studyId;
if (idOrNull == null) {
createStudy(study);
studyId = createStudy(study).getId();
} else {
studyId = idOrNull;
}
return updateStudyMetadata(study, studyMetadata -> {
StudyMetadata.SampleIndexConfigurationVersioned.Status status = staging
? StudyMetadata.SampleIndexConfigurationVersioned.Status.STAGING
: StudyMetadata.SampleIndexConfigurationVersioned.Status.ACTIVE;
return updateStudyMetadata(studyId, studyMetadata -> {
List<StudyMetadata.SampleIndexConfigurationVersioned> configurations = studyMetadata.getSampleIndexConfigurations();
if (configurations == null || configurations.isEmpty()) {
configurations = new ArrayList<>(2);
configurations.add(new StudyMetadata.SampleIndexConfigurationVersioned(
SampleIndexConfiguration.defaultConfiguration(),
StudyMetadata.DEFAULT_SAMPLE_INDEX_VERSION,
Date.from(Instant.now())));
Date.from(Instant.now()), StudyMetadata.SampleIndexConfigurationVersioned.Status.ACTIVE));
studyMetadata.setSampleIndexConfigurations(configurations);
}
int version = studyMetadata.getSampleIndexConfigurationLatest().getVersion() + 1;
configurations.add(new StudyMetadata.SampleIndexConfigurationVersioned(configuration, version, Date.from(Instant.now())));
configurations.add(new StudyMetadata.SampleIndexConfigurationVersioned(
configuration,
version,
Date.from(Instant.now()),
status));
}).getSampleIndexConfigurationLatest();
}

Expand Down
12 changes: 4 additions & 8 deletions ...ge-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/SampleMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,11 +175,10 @@ private TaskMetadata.Status getSampleIndexAnnotationStatus() {
}

@JsonIgnore
public TaskMetadata.Status getSampleIndexAnnotationStatus(int latestSampleIndexVersion) {
public TaskMetadata.Status getSampleIndexAnnotationStatus(int sampleIndexVersion) {
TaskMetadata.Status status = getSampleIndexAnnotationStatus();
if (status == TaskMetadata.Status.READY) {
int actualSampleIndexVersion = getSampleIndexAnnotationVersion();
if (actualSampleIndexVersion != latestSampleIndexVersion) {
if (!getSampleIndexAnnotationVersions().contains(sampleIndexVersion)) {
// logger.debug("Sample index annotation version outdated. Actual : " + actualSampleIndexVersion
// + " , expected : " + latestSampleIndexVersion);
status = TaskMetadata.Status.NONE;
Expand Down Expand Up @@ -251,13 +250,10 @@ public boolean hasSampleIndexStatus() {
}

@JsonIgnore
public TaskMetadata.Status getSampleIndexStatus(int latestSampleIndexVersion) {
public TaskMetadata.Status getSampleIndexStatus(int sampleIndexVersion) {
TaskMetadata.Status status = getSampleIndexStatus();
if (status == TaskMetadata.Status.READY) {
int actualSampleIndexVersion = getSampleIndexVersion();
if (actualSampleIndexVersion != latestSampleIndexVersion) {
// logger.debug("Sample index version outdated. Actual : " + actualSampleIndexVersion
// + " , expected : " + latestSampleIndexVersion);
if (!getSampleIndexVersions().contains(sampleIndexVersion)) {
status = TaskMetadata.Status.NONE;
}
}
Expand Down
51 changes: 32 additions & 19 deletions ...age-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/StudyMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public class StudyMetadata {
private Long timeStamp;
private VariantFileHeader variantHeader;
private List<VariantScoreMetadata> variantScores;
private List<SampleIndexConfigurationVersioned> sampleIndexConfigurations;
private List<SampleIndexConfigurationVersioned> sampleIndexConfigurations = Collections.emptyList();

private ObjectMap attributes;

Expand Down Expand Up @@ -136,14 +136,20 @@ public StudyMetadata setVariantScores(List<VariantScoreMetadata> variantScores)
}

public SampleIndexConfigurationVersioned getSampleIndexConfigurationLatest() {
return getSampleIndexConfigurationLatest(true);
}

public SampleIndexConfigurationVersioned getSampleIndexConfigurationLatest(boolean includeStagingSchemas) {
if (sampleIndexConfigurations == null || sampleIndexConfigurations.isEmpty()) {
return new SampleIndexConfigurationVersioned(SampleIndexConfiguration.defaultConfiguration(),
DEFAULT_SAMPLE_INDEX_VERSION, Date.from(Instant.now()));
DEFAULT_SAMPLE_INDEX_VERSION, Date.from(Instant.now()), SampleIndexConfigurationVersioned.Status.ACTIVE);
} else {
SampleIndexConfigurationVersioned conf = sampleIndexConfigurations.get(0);
for (SampleIndexConfigurationVersioned thisConf : sampleIndexConfigurations) {
if (thisConf.getVersion() > conf.getVersion()) {
conf = thisConf;
if (includeStagingSchemas || thisConf.getStatus() != SampleIndexConfigurationVersioned.Status.STAGING) {
if (thisConf.getVersion() > conf.getVersion()) {
conf = thisConf;
}
}
}
return conf;
Expand All @@ -154,17 +160,18 @@ public List<SampleIndexConfigurationVersioned> getSampleIndexConfigurations() {
return sampleIndexConfigurations;
}

public SampleIndexConfiguration getSampleIndexConfiguration(int version) {
public SampleIndexConfigurationVersioned getSampleIndexConfiguration(int version) {
if (sampleIndexConfigurations == null || sampleIndexConfigurations.isEmpty()) {
if (version == DEFAULT_SAMPLE_INDEX_VERSION) {
return SampleIndexConfiguration.defaultConfiguration();
return new SampleIndexConfigurationVersioned(SampleIndexConfiguration.defaultConfiguration(),
DEFAULT_SAMPLE_INDEX_VERSION, Date.from(Instant.now()), SampleIndexConfigurationVersioned.Status.ACTIVE);
} else {
return null;
}
} else {
for (SampleIndexConfigurationVersioned v : sampleIndexConfigurations) {
if (v.getVersion() == version) {
return v.getConfiguration();
return v;
}
}
return null;
Expand Down Expand Up @@ -243,18 +250,24 @@ public static class SampleIndexConfigurationVersioned {
private SampleIndexConfiguration configuration;
private int version;
private Date date;
// private boolean staging;
private Status status;
// private int numSamples;

public enum Status {
STAGING, // Index being built. Not ready. Not to be used.
ACTIVE, // Index ready to be used (if present)
DEPRECATED, // Index marked to be removed.
REMOVED // Index no longer exists.
}

public SampleIndexConfigurationVersioned() {
}

public SampleIndexConfigurationVersioned(SampleIndexConfiguration configuration, int version, Date date) {
public SampleIndexConfigurationVersioned(SampleIndexConfiguration configuration, int version, Date date, Status status) {
this.configuration = configuration;
this.version = version;
this.date = date;
// this.staging = false;
this.status = status;
}

public SampleIndexConfiguration getConfiguration() {
Expand Down Expand Up @@ -284,16 +297,16 @@ public SampleIndexConfigurationVersioned setDate(Date date) {
return this;
}

// public boolean isStaging() {
// return staging;
// }
//
// public SampleIndexConfigurationVersioned setStaging(boolean staging) {
// this.staging = staging;
// return this;
// }
public Status getStatus() {
return status;
}

public SampleIndexConfigurationVersioned setStatus(Status status) {
this.status = status;
return this;
}

// public int getNumSamples() {
// public int getNumSamples() {
// return numSamples;
// }
//
Expand Down
1 change: 1 addition & 0 deletions .../java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,7 @@ public void testGetByStudies() throws Exception {
.append(INCLUDE_SAMPLE.key(), ALL);
queryResult = query(query, options);
VariantQueryResult<Variant> allVariants = dbAdaptor.get(new Query()
.append(INCLUDE_SAMPLE.key(), ALL)
.append(VariantQueryParam.INCLUDE_STUDY.key(), study1), options);
assertThat(queryResult, everyResult(allVariants, withStudy(study1)));

Expand Down
2 changes: 1 addition & 1 deletion ...e/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,7 @@ public void sampleIndex(String study, List<String> samples, ObjectMap options) t
@Override
public void sampleIndexAnnotate(String study, List<String> samples, ObjectMap options) throws StorageEngineException {
options = getMergedOptions(options);
new SampleIndexAnnotationLoader(hBaseManager, getTableNameGenerator(), getMetadataManager(), getMRExecutor())
new SampleIndexAnnotationLoader(getSampleIndexDBAdaptor(), getMRExecutor())
.updateSampleAnnotation(study, samples, options);
}

Expand Down
16 changes: 4 additions & 12 deletions ...opencga/storage/hadoop/variant/index/annotation/mr/SampleIndexAnnotationLoaderDriver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import org.opencb.biodata.models.core.Region;
import org.opencb.commons.datastore.core.ObjectMap;
import org.opencb.opencga.core.config.storage.SampleIndexConfiguration;
import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager;
import org.opencb.opencga.storage.core.metadata.models.SampleMetadata;
import org.opencb.opencga.storage.core.metadata.models.StudyMetadata;
Expand All @@ -21,7 +20,6 @@
import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.PhoenixHelper;
import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema;
import org.opencb.opencga.storage.hadoop.variant.converters.HBaseToVariantConverter;
import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexAnnotationLoader;
import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexSchema;
import org.opencb.opencga.storage.hadoop.variant.mr.VariantAlignedInputFormat;
import org.opencb.opencga.storage.hadoop.variant.mr.VariantMapReduceUtil;
Expand All @@ -40,6 +38,7 @@ public class SampleIndexAnnotationLoaderDriver extends AbstractVariantsTableDriv

private static final Logger LOGGER = LoggerFactory.getLogger(SampleIndexAnnotationLoaderDriver.class);
public static final String OUTPUT = "output-table";
public static final String SAMPLE_INDEX_VERSION = "sample-index-version";
public static final String SAMPLES = "samples";
public static final String SAMPLE_IDS = "sampleIds";

Expand All @@ -59,6 +58,7 @@ protected Map<String, String> getParams() {
Map<String, String> params = new HashMap<>();
params.put("--" + SAMPLES, "<samples>");
params.put("--" + SAMPLE_IDS, "<sample-ids>");
params.put("--" + SAMPLE_INDEX_VERSION, "<version>");
params.put("--" + OUTPUT, "<output-table>");
params.put("--" + VariantQueryParam.REGION.key(), "<region>");
return params;
Expand Down Expand Up @@ -152,11 +152,11 @@ protected Job setupJob(Job job, String archiveTable, String variantTable) throws

VariantMapReduceUtil.setNoneReduce(job);

sampleIndexVersion = Integer.parseInt(getParam(SAMPLE_INDEX_VERSION));
StudyMetadata.SampleIndexConfigurationVersioned versioned = getMetadataManager()
.getStudyMetadata(getStudyId())
.getSampleIndexConfigurationLatest();
.getSampleIndexConfiguration(sampleIndexVersion);
SampleIndexConfiguration configuration = versioned.getConfiguration();
sampleIndexVersion = versioned.getVersion();
VariantMapReduceUtil.setSampleIndexConfiguration(job, configuration, sampleIndexVersion);

return job;
Expand All @@ -167,14 +167,6 @@ protected String getJobOperationName() {
return "sample_index_annotation_loader";
}

@Override
protected void postExecution(boolean succeed) throws IOException, StorageEngineException {
super.postExecution(succeed);
if (succeed && StringUtils.isEmpty(region)) {
SampleIndexAnnotationLoader.postAnnotationLoad(getStudyId(), sampleIds, getMetadataManager(), sampleIndexVersion);
}
}

public static void main(String[] args) throws Exception {
try {
System.exit(new SampleIndexAnnotationLoaderDriver().privateMain(args, null));
Expand Down
Loading

0 comments on commit 09b075c

Please sign in to comment.