Skip to content

Commit

Permalink
LAI INRA processor evolutions
Browse files Browse the repository at this point in the history
  • Loading branch information
@cosminudroiu
cosminudroiu committed Sep 7, 2018
1 parent 17ec646 commit c657ff7
Show file tree
Hide file tree
Showing 21 changed files with 2,043 additions and 2,112 deletions.
10 changes: 5 additions & 5 deletions sen2agri-orchestrator/processor/compositehandler.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,16 +249,16 @@ void CompositeHandler::HandleNewTilesList(EventProcessingContext &ctx,
bool isLastProduct = (i == (listProducts.size() - 1));
QStringList compositeSplitterArgs = { "CompositeSplitter2",
"-in", outL3AResultFile, "-xml", inputProduct, "-bmap", bandsMapping,
"-outweights", (isLastProduct ? (BuildProcessorOutputFileName(cfg.allCfgMap, outL3AResultWeightsFile)) : outL3AResultWeightsFile),
"-outdates", (isLastProduct ? (BuildProcessorOutputFileName(cfg.allCfgMap, outL3AResultDatesFile)) : outL3AResultDatesFile),
"-outrefls", (isLastProduct ? (BuildProcessorOutputFileName(cfg.allCfgMap, outL3AResultReflsFile)) : outL3AResultReflsFile),
"-outflags", (isLastProduct ? (BuildProcessorOutputFileName(cfg.allCfgMap, outL3AResultFlagsFile)) : outL3AResultFlagsFile),
"-outweights", outL3AResultWeightsFile,
"-outdates", outL3AResultDatesFile,
"-outrefls", outL3AResultReflsFile,
"-outflags", outL3AResultFlagsFile,
"-isfinal", (isLastProduct ? "1" : "0")
};
// we need to create the rgb file only if the last product
if(isLastProduct) {
compositeSplitterArgs.append("-outrgb");
compositeSplitterArgs.append(BuildProcessorOutputFileName(cfg.allCfgMap, outL3AResultRgbFile));
compositeSplitterArgs.append(outL3AResultRgbFile);
}
steps.append(compositeSplitter.CreateStep("CompositeSplitter", compositeSplitterArgs));

Expand Down
6 changes: 3 additions & 3 deletions sen2agri-orchestrator/processor/lairetrievalhandler_l3b.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,15 +282,15 @@ NewStepList LaiRetrievalHandlerL3B::GetStepsForMonodateLai(EventProcessingContex
const auto & quantifiedErrFileName = quantifyErrImageTask.GetFilePath("LAI_mono_date_ERR_img_16.tif");

QStringList genMonoDateMskFagsArgs = GetMonoDateMskFlagsArgs(prdTileInfo.tileFile, monoDateMskFlgsFileName,
BuildProcessorOutputFileName(configParameters, monoDateMskFlgsResFileName),
monoDateMskFlgsResFileName,
resolutionStr);

// add these steps to the steps list to be submitted
steps.append(genMonoDateMskFagsTask.CreateStep("GenerateLaiMonoDateMaskFlags", genMonoDateMskFagsArgs));

const QStringList &ndviRviExtractionArgs = GetNdviRviExtractionNewArgs(prdTileInfo.tileFile, monoDateMskFlgsFileName,
ftsFile,
BuildProcessorOutputFileName(configParameters, singleNdviFile),
singleNdviFile,
resolutionStr, laiCfgFile);
steps.append(ndviRviExtractorTask.CreateStep("NdviRviExtractionNew", ndviRviExtractionArgs));

Expand Down Expand Up @@ -579,7 +579,7 @@ QStringList LaiRetrievalHandlerL3B::GetQuantifyImageArgs(const std::map<QString,
;
return { "QuantifyImage",
"-in", inFileName,
"-out", BuildProcessorOutputFileName(configParams, outFileName)
"-out", outFileName
};
}

Expand Down
494 changes: 346 additions & 148 deletions sen2agri-orchestrator/processor/lairetrievalhandler_l3b_new.cpp
View file

Large diffs are not rendered by default.

67 changes: 60 additions & 7 deletions sen2agri-orchestrator/processor/lairetrievalhandler_l3b_new.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,32 @@ class LaiRetrievalHandlerL3BNew : public ProcessorHandler
QString tileFile;
} TileInfos;

typedef struct {
QString tileId;
QString tileFile;

QString ndviFile;
QString laiFile;
QString faparFile;
QString fcoverFile;

QString statusFlagsFile;
QString statusFlagsFileResampled;
QString inDomainFlagsFile;
QString laiDomainFlagsFile;
QString faparDomainFlagsFile;
QString fcoverDomainFlagsFile;

bool bHasNdvi;
bool bHasLai;
bool bHasFapar;
bool bHasFCover;

QString resolutionStr;
QString anglesFile;
} TileResultFiles;


private:

void HandleJobSubmittedImpl(EventProcessingContext &ctx,
Expand All @@ -25,22 +51,46 @@ class LaiRetrievalHandlerL3BNew : public ProcessorHandler

void CreateTasksForNewProduct(EventProcessingContext &ctx, const JobSubmittedEvent &event, QList<TaskToSubmit> &outAllTasksList,
const QList<TileInfos> &tileInfosList, bool bRemoveTempFiles);
int CreateAnglesTasks(int parentTaskId, QList<TaskToSubmit> &outAllTasksList, int nCurTaskIdx, int & nAnglesTaskId);
int CreateBiophysicalIndicatorTasks(int parentTaskId, QList<TaskToSubmit> &outAllTasksList,
QList<std::reference_wrapper<const TaskToSubmit>> &productFormatterParentsRefs,
int nCurTaskIdx);

void GetModelFileList(const QString &folderName, const QString &modelPrefix, QStringList &outModelsList);
void WriteExecutionInfosFile(const QString &executionInfosPath,
const QList<TileInfos> &tilesInfosList);
const QList<TileResultFiles> &tileResultFilesList);

QStringList GetCreateAnglesArgs(const QString &inputProduct, const QString &anglesFile);
QStringList GetGdalTranslateAnglesNoDataArgs(const QString &anglesFile, const QString &resultAnglesFile);
QStringList GetGdalBuildAnglesVrtArgs(const QString &anglesFile, const QString &resultVrtFile);
QStringList GetGdalTranslateResampleAnglesArgs(const QString &vrtFile, const QString &resultResampledAnglesFile);
QStringList GetGenerateInputDomainFlagsArgs(const QString &xmlFile, const QString &laiBandsCfg,
const QString &outFlagsFileName, const QString &outRes);
QStringList GetGenerateOutputDomainFlagsArgs(const QString &xmlFile, const QString &laiRasterFile,
const QString &laiBandsCfg, const QString &indexName,
const QString &outFlagsFileName, const QString &outCorrectedLaiFile, const QString &outRes);

// Arguments getters
QStringList GetNdviRviExtractionNewArgs(const QString &inputProduct, const QString &msksFlagsFile,
const QString &ndviFile, const QString &resolution, const QString &laiBandsCfg);
QStringList GetLaiProcessorArgs(const QString &xmlFile, const QString &resolution, const QString &laiBandsCfg, const QString &monoDateLaiFileName, const QString &indexName);
QStringList GetQuantifyImageArgs(const std::map<QString, QString> &configParameters, const QString &inFileName, const QString &outFileName);
QStringList GetLaiProcessorArgs(const QString &xmlFile, const QString &anglesFileName, const QString &resolution,
const QString &laiBandsCfg, const QString &monoDateLaiFileName, const QString &indexName);
QStringList GetQuantifyImageArgs(const QString &inFileName, const QString &outFileName);
QStringList GetMonoDateMskFlagsArgs(const QString &inputProduct, const QString &monoDateMskFlgsFileName, const QString &monoDateMskFlgsResFileName, const QString &resStr);
QStringList GetLaiMonoProductFormatterArgs(TaskToSubmit &productFormatterTask, EventProcessingContext &ctx, const JobSubmittedEvent &event,
const QList<TileInfos> &products, const QStringList &ndviList,
const QStringList &laiList, const QStringList &laiFlgsList, const QStringList &faparList, const QStringList &fcoverList);
QStringList GetLaiMonoProductFormatterArgs(TaskToSubmit &productFormatterTask, EventProcessingContext &ctx, const JobSubmittedEvent &event, const QList<TileResultFiles> &tileResultFilesList);
NewStepList GetStepsForMonodateLai(EventProcessingContext &ctx, const JobSubmittedEvent &event,
const QList<TileInfos> &prdTilesList, QList<TaskToSubmit> &allTasksList, bool bRemoveTempFiles, int tasksStartIdx);
int GetStepsForStatusFlags(QList<TaskToSubmit> &allTasksList, int curTaskIdx,
TileResultFiles &tileResultFileInfo, NewStepList &steps, QStringList &cleanupTemporaryFilesList);
int GetStepsForNdvi(QList<TaskToSubmit> &allTasksList, int curTaskIdx,
TileResultFiles &tileResultFileInfo, const QString &laiCfgFile, NewStepList &steps, QStringList &cleanupTemporaryFilesList);
int GetStepsForAnglesCreation(QList<TaskToSubmit> &allTasksList, int curTaskIdx, TileResultFiles &tileResultFileInfo, NewStepList &steps, QStringList &cleanupTemporaryFilesList);
int GetStepsForMonoDateBI(QList<TaskToSubmit> &allTasksList,
const QString &indexName, int curTaskIdx, const QString &laiCfgFile, TileResultFiles &tileResultFileInfo,
NewStepList &steps, QStringList &cleanupTemporaryFilesList);
int GetStepsForInDomainFlags(QList<TaskToSubmit> &allTasksList, int curTaskIdx,
const QString &laiCfgFile, TileResultFiles &tileResultFileInfo, NewStepList &steps,
QStringList &cleanupTemporaryFilesList);

const QString& GetDefaultCfgVal(std::map<QString, QString> &configParameters, const QString &key, const QString &defVal);

ProcessorJobDefinitionParams GetProcessingDefinitionImpl(SchedulingContext &ctx, int siteId, int scheduledDate,
Expand All @@ -49,6 +99,9 @@ class LaiRetrievalHandlerL3BNew : public ProcessorHandler
const QString &cmdLineParamName, const QString &cfgParamKey, bool defVal = true);
QSet<QString> GetTilesFilter(const QJsonObject &parameters, std::map<QString, QString> &configParameters);
bool FilterTile(const QSet<QString> &tilesSet, const QString &prdTileFile);
void InitTileResultFiles(bool bGenNdvi, bool bGenLai, bool bGenFapar, bool bGenFCover, const QString &resolutionStr,
const QString tileFileName, TileResultFiles &tileResultFileInfo);

void HandleProduct(EventProcessingContext &ctx, const JobSubmittedEvent &event, const QList<TileInfos> &prdTilesList,
QList<TaskToSubmit> &allTasksList);
void SubmitEndOfLaiTask(EventProcessingContext &ctx, const JobSubmittedEvent &event,
Expand Down
24 changes: 12 additions & 12 deletions sen2agri-orchestrator/processor/lairetrievalhandler_l3c.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,17 +168,17 @@ NewStepList LaiRetrievalHandlerL3C::GetStepsForMultiDateReprocessing(
QString allErrTimeSeriesFileName = errTimeSeriesBuilderTask.GetFilePath("Err_time_series.tif");
const auto & allMskFlagsTimeSeriesFileName = mskFlagsTimeSeriesBuilderTask.GetFilePath("Mask_Flags_time_series.tif");

const QStringList &timeSeriesBuilderArgs = GetTimeSeriesBuilderArgs(configParameters, quantifiedLaiFileNames2, allLaiTimeSeriesFileName, mainLaiImg);
const QStringList &timeSeriesBuilderArgs = GetTimeSeriesBuilderArgs(quantifiedLaiFileNames2, allLaiTimeSeriesFileName, mainLaiImg);
QStringList errTimeSeriesBuilderArgs;
if (errFilesUsed) {
errTimeSeriesBuilderArgs = GetTimeSeriesBuilderArgs(configParameters, quantifiedErrLaiFileNames2, allErrTimeSeriesFileName, mainLaiErrImg);
errTimeSeriesBuilderArgs = GetTimeSeriesBuilderArgs(quantifiedErrLaiFileNames2, allErrTimeSeriesFileName, mainLaiErrImg);
} else {
// do nothing in this case by executing the "true" command on linux that does nothing and will return success after the execution
allErrTimeSeriesFileName = "";
errTimeSeriesBuilderArgs.append("true");
}

const QStringList &mskFlagsTimeSeriesBuilderArgs = GetTimeSeriesBuilderArgs(configParameters, monoDateMskFlagsLaiFileNames2,
const QStringList &mskFlagsTimeSeriesBuilderArgs = GetTimeSeriesBuilderArgs(monoDateMskFlagsLaiFileNames2,
allMskFlagsTimeSeriesFileName, mainMsksImg, true);

steps.append(imgTimeSeriesBuilderTask.CreateStep("TimeSeriesBuilder", timeSeriesBuilderArgs));
Expand Down Expand Up @@ -322,7 +322,7 @@ NewStepList LaiRetrievalHandlerL3C::GetStepsForMultiDateReprocessing_New(std::ma
quantifiedErrLaiFileNames2, monoDateMskFlagsLaiFileNames2, mainLaiImg,
reprocTimeSeriesFileName, listDates);
} else {
profileReprocessingArgs = GetFittedProfileReprocArgs_New(configParameters, quantifiedLaiFileNames2,
profileReprocessingArgs = GetFittedProfileReprocArgs_New(quantifiedLaiFileNames2,
quantifiedErrLaiFileNames2, monoDateMskFlagsLaiFileNames2, mainLaiImg,
reprocTimeSeriesFileName, listDates);
}
Expand Down Expand Up @@ -989,12 +989,11 @@ void LaiRetrievalHandlerL3C::HandleTaskFinishedImpl(EventProcessingContext &ctx,
}
}

QStringList LaiRetrievalHandlerL3C::GetTimeSeriesBuilderArgs(const std::map<QString, QString> &configParameters,
const QStringList &monoDateFileNames,
QStringList LaiRetrievalHandlerL3C::GetTimeSeriesBuilderArgs(const QStringList &monoDateFileNames,
const QString &allTimeSeriesFileName, const QString &mainImg,
bool bIsFlg) {
QStringList timeSeriesBuilderArgs = { "TimeSeriesBuilder",
"-out", BuildProcessorOutputFileName(configParameters, allTimeSeriesFileName, true, !bIsFlg),
"-out", (bIsFlg ? allTimeSeriesFileName : (allTimeSeriesFileName + "&gdal:co:BIGTIFF=YES")),
"-main", mainImg,
"-il"
};
Expand All @@ -1017,7 +1016,7 @@ QStringList LaiRetrievalHandlerL3C::GetProfileReprocessingArgs(const std::map<QS
QStringList profileReprocessingArgs = { "ProfileReprocessing",
"-lai", allLaiTimeSeriesFileName,
"-msks", allMsksTimeSeriesFileName,
"-opf", BuildProcessorOutputFileName(configParameters, reprocTimeSeriesFileName, true, true),
"-opf", (reprocTimeSeriesFileName + "&gdal:co:BIGTIFF=YES"),
"-algo", "local",
"-algo.local.bwr", localWindowBwr,
"-algo.local.fwr", localWindowFwr,
Expand All @@ -1039,7 +1038,7 @@ QStringList LaiRetrievalHandlerL3C::GetProfileReprocessingArgs_New(const std::ma

QStringList profileReprocessingArgs = { "ProfileReprocessing",
"-main", mainImg,
"-opf", BuildProcessorOutputFileName(configParameters, reprocTimeSeriesFileName, true, true),
"-opf", (reprocTimeSeriesFileName + "&gdal:co:BIGTIFF=YES"),
"-algo", "local",
"-algo.local.bwr", localWindowBwr,
"-algo.local.fwr", localWindowFwr
Expand All @@ -1053,12 +1052,12 @@ QStringList LaiRetrievalHandlerL3C::GetProfileReprocessingArgs_New(const std::ma
return profileReprocessingArgs;
}

QStringList LaiRetrievalHandlerL3C::GetFittedProfileReprocArgs_New(const std::map<QString, QString> &configParameters,
QStringList &monoDateLaiFileNames, QStringList &errFileNames, QStringList &flgsFileNames,
QStringList LaiRetrievalHandlerL3C::GetFittedProfileReprocArgs_New(const QStringList &monoDateLaiFileNames,
const QStringList &errFileNames, const QStringList &flgsFileNames,
const QString &mainImg, const QString &reprocTimeSeriesFileName, const QStringList &listDates) {
QStringList profileReprocessingArgs = { "ProfileReprocessing",
"-main", mainImg,
"-opf", BuildProcessorOutputFileName(configParameters, reprocTimeSeriesFileName, true, true),
"-opf", (reprocTimeSeriesFileName + "&gdal:co:BIGTIFF=YES"),
"-genall", "1",
"-algo", "fit"
};
Expand Down Expand Up @@ -1125,6 +1124,7 @@ QStringList LaiRetrievalHandlerL3C::GetReprocProductFormatterArgs(TaskToSubmit &
"-siteid", QString::number(event.siteId),
"-processor", "vegetation",
"-gipp", executionInfosPath,
"-compress", "1",
"-outprops", outPropsPath};
productFormatterArgs += "-il";
productFormatterArgs += listProducts;
Expand Down
8 changes: 4 additions & 4 deletions sen2agri-orchestrator/processor/lairetrievalhandler_l3c.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ class LaiRetrievalHandlerL3C : public ProcessorHandler
const QStringList &listProducts, bool bIsReproc);

// Arguments getters
QStringList GetTimeSeriesBuilderArgs(const std::map<QString, QString> &configParameters, const QStringList &monoDateLaiFileNames,
QStringList GetTimeSeriesBuilderArgs(const QStringList &monoDateLaiFileNames,
const QString &allLaiTimeSeriesFileName, const QString &mainImg, bool bIsFlg = false);
QStringList GetProfileReprocessingArgs(const std::map<QString, QString> &configParameters, const QString &allLaiTimeSeriesFileName,
const QString &allErrTimeSeriesFileName, const QString &allMsksTimeSeriesFileName,
Expand Down Expand Up @@ -114,9 +114,9 @@ class LaiRetrievalHandlerL3C : public ProcessorHandler

QStringList GetProfileReprocessingArgs_New(const std::map<QString, QString> &configParameters, QStringList &monoDateLaiFileNames, QStringList &errFileNames, QStringList &flgsFileNames,
const QString &mainImg, const QString &reprocTimeSeriesFileName, const QStringList &listDates);
QStringList GetFittedProfileReprocArgs_New(const std::map<QString, QString> &configParameters,
QStringList &monoDateLaiFileNames, QStringList &errFileNames,
QStringList &flgsFileNames, const QString &mainImg, const QString &reprocTimeSeriesFileName, const QStringList &listDates);
QStringList GetFittedProfileReprocArgs_New(const QStringList &monoDateLaiFileNames, const QStringList &errFileNames,
const QStringList &flgsFileNames, const QString &mainImg, const QString &reprocTimeSeriesFileName,
const QStringList &listDates);
QMap<QString, TileTemporalFilesInfo> FilterSecondaryProductTiles(const QMap<QString, TileTemporalFilesInfo> &mapTiles,
const QMap<ProcessorHandlerHelper::SatelliteIdType, TileList> &siteTiles);
QStringList GetL3BProductRasterFiles(const TileTemporalFilesInfo &tileTemporalFilesInfo, LAI_RASTER_ADDITIONAL_INFO_IDX idx);
Expand Down
Loading

0 comments on commit c657ff7

Please sign in to comment.