Merge pull request #236 from pixlise/development

Releasing variogram and bug fixes

api/dataimport/for-trigger_test.go

@@ -234,6 +234,7 @@ func printArchiveOKLogOutput(logger *logger.StdOutLoggerForTest, db *mongo.Datab
 	}
 	// Clear the time stamp so it doesn't change next time we run test
 	summary.TimestampUnixSec = 0
+	summary.CompleteTimeStampUnixSec = 0
 
 	b, err := protojson.Marshal(summary)
 	if err != nil {
@@ -390,6 +391,7 @@ func printManualOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database,
 	} else {
 		// Clear the time stamp so it doesn't change next time we run test
 		summary.TimestampUnixSec = 0
+		summary.CompleteTimeStampUnixSec = 0
 	}
 
 	b, err := protojson.Marshal(summary)

api/dataimport/internal/output/output.go

@@ -38,6 +38,7 @@ import (
 	"github.com/pixlise/core/v4/core/fileaccess"
 	"github.com/pixlise/core/v4/core/gdsfilename"
 	"github.com/pixlise/core/v4/core/logger"
+	"github.com/pixlise/core/v4/core/scan"
 	"github.com/pixlise/core/v4/core/utils"
 	protos "github.com/pixlise/core/v4/generated-protos"
 	"go.mongodb.org/mongo-driver/bson"
@@ -361,7 +362,13 @@ func (s *PIXLISEDataSaver) Save(
 		return fmt.Errorf("Failed to get dataset file size for: %v", outFilePath)
 	}
 
-	summaryData := makeSummaryFileContent(&exp, data.DatasetID, data.Instrument, data.Meta, int(fi.Size()), creationUnixTimeSec, data.CreatorUserId)
+	var prevSavedScan *protos.ScanItem
+	if prevSavedScan, err = scan.ReadScanItem(data.DatasetID, db); err != nil {
+		// NOTE: this is largely paranoia, in case ReadScanItem changes and returns errors even on read
+		prevSavedScan = nil
+	}
+
+	summaryData := makeSummaryFileContent(&exp, prevSavedScan, data.DatasetID, data.Instrument, data.Meta /*int(fi.Size()),*/, creationUnixTimeSec, data.CreatorUserId)
 
 	jobLog.Infof("Writing summary to DB for %v...", summaryData.Id)
 

api/dataimport/internal/output/summary.go

@@ -27,10 +27,11 @@ import (
 
 func makeSummaryFileContent(
 	exp *protos.Experiment,
+	prevSavedScan *protos.ScanItem,
 	datasetID string,
 	sourceInstrument protos.ScanInstrument,
 	meta dataConvertModels.FileMetaData,
-	fileSize int,
+	//fileSize int,
 	creationUnixTimeSec int64,
 	creatorUserId string) *protos.ScanItem {
 	contextImgCount := len(exp.AlignedContextImages) + len(exp.UnalignedContextImages) + len(exp.MatchedAlignedContextImages)
@@ -93,16 +94,52 @@ func makeSummaryFileContent(
 	// what it's displaying
 
 	s := &protos.ScanItem{
-		Id:               datasetID,
-		Title:            meta.Title,
-		Description:      "",
-		DataTypes:        dataTypes,
-		Instrument:       sourceInstrument,
-		InstrumentConfig: exp.DetectorConfig,
-		TimestampUnixSec: uint32(creationUnixTimeSec),
-		Meta:             saveMeta,
-		ContentCounts:    contentCounts,
-		CreatorUserId:    creatorUserId,
+		Id:                         datasetID,
+		Title:                      meta.Title,
+		Description:                "",
+		DataTypes:                  dataTypes,
+		Instrument:                 sourceInstrument,
+		InstrumentConfig:           exp.DetectorConfig,
+		TimestampUnixSec:           uint32(creationUnixTimeSec),
+		Meta:                       saveMeta,
+		ContentCounts:              contentCounts,
+		CreatorUserId:              creatorUserId,
+		PreviousImportTimesUnixSec: []uint32{},
 	}
+
+	// If we've got a previously stored ScanItem, we are updating it, so read its time stamp into the array of previous time stamps
+	isComplete := exp.NormalSpectra == exp.PseudoIntensities*2
+
+	if prevSavedScan != nil {
+		// Build the list of previous import times
+		// Preserve the previous list...
+		s.PreviousImportTimesUnixSec = append(s.PreviousImportTimesUnixSec, prevSavedScan.PreviousImportTimesUnixSec...)
+
+		// Add new time at the end
+		s.PreviousImportTimesUnixSec = append(s.PreviousImportTimesUnixSec, prevSavedScan.TimestampUnixSec)
+	}
+
+	// Save a time stamp for completion
+	if isComplete {
+		// If previous scan was also complete, DON'T update the time stamp, just preserve it
+		if prevSavedScan != nil && prevSavedScan.CompleteTimeStampUnixSec > 0 {
+			s.CompleteTimeStampUnixSec = prevSavedScan.CompleteTimeStampUnixSec
+		} else {
+			// We must've just completed now, so save the time
+			s.CompleteTimeStampUnixSec = uint32(creationUnixTimeSec)
+		}
+	}
+
+	// Preserve user-editable fields
+	if prevSavedScan != nil {
+		s.Tags = prevSavedScan.Tags
+		s.Description = prevSavedScan.Description
+
+		if len(prevSavedScan.Description) > 0 {
+			// User has entered a description so perserve the title too
+			s.Title = prevSavedScan.Title
+		}
+	}
+
 	return s
 }

api/quantification/autoQuantification.go

@@ -3,10 +3,12 @@ package quantification
 import (
 	"context"
 	"fmt"
+	"strconv"
 
 	"github.com/pixlise/core/v4/api/dbCollections"
 	"github.com/pixlise/core/v4/api/services"
 	"github.com/pixlise/core/v4/api/specialUserIds"
+	"github.com/pixlise/core/v4/api/ws/wsHelpers"
 	protos "github.com/pixlise/core/v4/generated-protos"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo/options"
@@ -23,11 +25,14 @@ func RunAutoQuantifications(scanId string, svcs *services.APIServices) {
 	quantNames := []string{"AutoQuant-PDS", "AutoQuant-PIXL"}
 	quantModes := []string{quantModeCombinedAB, quantModeSeparateAB}
 	quantElements := [][]string{
-		// PDS
-		[]string{"Na2O", "MgO", "Al2O3", "SiO2", "P2O5", "SO3", "Cl", "K2O", "CaO", "TiO2", "Cr2O3", "MnO", "FeO-T", "NiO", "ZnO", "Br"},
-		[]string{"Na2O", "MgO", "Al2O3", "SiO2", "P2O5", "SO3", "Cl", "K2O", "CaO", "TiO2", "Cr2O3", "MnO", "FeO-T", "NiO", "ZnO", "GeO", "Br", "Rb2O", "SrO", "Y2O3", "ZrO2"},
+		// PDS: intended "Na2O", "MgO", "Al2O3", "SiO2", "P2O5", "SO3", "Cl", "K2O", "CaO", "TiO2", "Cr2O3", "MnO", "FeO-T", "NiO", "ZnO", "Br"
+		// But we must specify elements only! Expecting PIQUANT to determine the oxide states to write
+		[]string{"Na", "Mg", "Al", "Si", "P", "S", "Cl", "K", "Ca", "Ti", "Cr", "Mn", "Fe", "Ni", "Zn", "Br"},
+		// PIXL: intended "Na2O", "MgO", "Al2O3", "SiO2", "P2O5", "SO3", "Cl", "K2O", "CaO", "TiO2", "Cr2O3", "MnO", "FeO-T", "NiO", "ZnO", "GeO", "Br", "Rb2O", "SrO", "Y2O3", "ZrO2"
+		// But we must specify elements only! Expecting PIQUANT to determine the oxide states to write
+		[]string{"Na", "Mg", "Al", "Si", "P", "S", "Cl", "K", "Ca", "Ti", "Cr", "Mn", "Fe", "Ni", "Zn", "Ge", "Br", "Rb", "Sr", "Y", "Zr"},
 	}
-	detector := "PIXL/v7"
+	detector := "PIXL/PiquantConfigs/v7"
 
 	allNames := []string{}
 	for _, name := range quantNames {
@@ -52,18 +57,38 @@ func RunAutoQuantifications(scanId string, svcs *services.APIServices) {
 		return
 	}
 
+	exprPB, err := wsHelpers.ReadDatasetFile(scanId, svcs)
+	if err != nil {
+		svcs.Log.Errorf("AutoQuant failed to read scan %v to determine PMC list: %v", scanId, err)
+		return
+	}
+
+	pmcs := []int32{}
+	for _, loc := range exprPB.Locations {
+		pmc, err := strconv.Atoi(loc.Id)
+		if err != nil {
+			svcs.Log.Errorf("AutoQuant: Failed to read PMC %v from scan %v. Skipping...", loc.Id, scanId)
+			continue
+		}
+
+		if len(loc.PseudoIntensities) > 0 && len(loc.PseudoIntensities[0].ElementIntensities) > 0 {
+			// We have quantifiable data for this
+			pmcs = append(pmcs, int32(pmc))
+		}
+	}
+
 	// Start all the quants
 	for c, name := range quantNames {
 		for _, m := range quantModes {
 			params := &protos.QuantCreateParams{
 				Command:        "map",
 				Name:           makeAutoQuantName(name, m),
 				ScanId:         scanId,
-				Pmcs:           []int32{},
+				Pmcs:           pmcs,
 				Elements:       quantElements[c],
 				DetectorConfig: detector,
-				Parameters:     "",
-				RunTimeSec:     0,
+				Parameters:     "-Fe,1",
+				RunTimeSec:     300,
 				QuantMode:      m,
 				RoiIDs:         []string{},
 				IncludeDwells:  false,

api/ws/handlers/scan.go

@@ -15,6 +15,7 @@ import (
 	"github.com/pixlise/core/v4/api/dbCollections"
 	"github.com/pixlise/core/v4/api/filepaths"
 	"github.com/pixlise/core/v4/api/job"
+	"github.com/pixlise/core/v4/api/quantification"
 	"github.com/pixlise/core/v4/api/services"
 	"github.com/pixlise/core/v4/api/ws/wsHelpers"
 	"github.com/pixlise/core/v4/core/errorwithstatus"
@@ -708,3 +709,13 @@ func HandleScanAutoShareWriteReq(req *protos.ScanAutoShareWriteReq, hctx wsHelpe
 
 	return &protos.ScanAutoShareWriteResp{}, nil
 }
+
+func HandleScanTriggerAutoQuantReq(req *protos.ScanTriggerAutoQuantReq, hctx wsHelpers.HandlerContext) (*protos.ScanTriggerAutoQuantResp, error) {
+	if err := wsHelpers.CheckStringField(&req.ScanId, "ScanId", 1, wsHelpers.IdFieldMaxLength); err != nil {
+		return nil, err
+	}
+
+	quantification.RunAutoQuantifications(req.ScanId, hctx.Svcs)
+
+	return &protos.ScanTriggerAutoQuantResp{}, nil
+}

api/ws/handlers/selection-entry.go

@@ -94,7 +94,7 @@ func writeSelection(id string, idxs *protos.ScanEntryRange, db *mongo.Database,
 	}
 
 	// Modified and Upsert counts will be 0 if the selection hasn't changed, so we just check matched
-	if dbResult.MatchedCount != 1 {
+	if dbResult.MatchedCount != 1 && dbResult.UpsertedCount != 1 {
 		logger.Errorf("writeSelection (%v) UpdateByID result had unexpected counts %+v", id, dbResult)
 	}