From 8996a20ff1549b2d0239594cfcad69e6d1906381 Mon Sep 17 00:00:00 2001 From: mj-xmr Date: Wed, 19 Jan 2022 11:59:19 +0100 Subject: [PATCH 1/6] QT time shift fix; Preliminary scripted data generation --- src/lib-base/src/ConfigSym.cpp | 7 +- src/lib-base/src/ConfigTS.cpp | 6 +- src/lib-base/src/ConfigTS.h | 2 + src/lib-base/src/ITicksProvider.h | 2 +- src/lib-base/src/MainTester.cpp | 7 +- src/lib-base/src/OrderedSeries.cpp | 20 ++++- src/lib-base/src/OrderedSeries.h | 1 + src/lib-base/src/PriceFilterSpreadMul.cpp | 6 +- src/lib-base/src/TicksProviderBinary.cpp | 1 - src/lib-base/src/TicksProviderFake.h | 4 +- src/lib-base/src/TicksProviderGenerated.cpp | 84 +++++++++++++++++++ src/lib-base/src/TicksProviderGenerated.h | 6 +- src/lib-base/src/TradeUtil.cpp | 7 +- src/lib-base/src/TradeUtil.h | 2 +- .../static/scripts/data-generation.txt | 3 + src/wxConfQT/src/DialogTS.cpp | 41 +++++---- src/wxConfQT/src/DialogTS.h | 8 ++ src/wxConfQT/src/DialogTSImpl.cpp | 22 ++++- src/wxConfQT/src/wxUtil.cpp | 12 ++- src/wxConfQT/wxsmith/DialogTS.wxs | 64 +++++++++----- tests/test-base/CMakeLists.txt | 7 +- tests/test-tsqsim/CMakeLists.txt | 3 +- tests/test-tsqsim/src/TSXformImplTest.cpp | 23 +++-- util/prep-env.sh | 2 +- 24 files changed, 264 insertions(+), 76 deletions(-) create mode 100644 src/tsqsim-lib/static/scripts/data-generation.txt diff --git a/src/lib-base/src/ConfigSym.cpp b/src/lib-base/src/ConfigSym.cpp index 571d79f..515b00d 100644 --- a/src/lib-base/src/ConfigSym.cpp +++ b/src/lib-base/src/ConfigSym.cpp @@ -114,11 +114,16 @@ bool ConfigSym::ShiftRange(int shift) ConfigSym backup = *this; if (not ShiftRangeUpdate(change, &dates.yearStart, &dates.monthStart)) + { return false; + } if (not ShiftRangeUpdate(change, &dates.yearEnd, &dates.monthEnd)) + { return false; + } + - const Pair yearMonthMax = TradeUtil().GetYearMonthMax(); + const Pair yearMonthMax = TradeUtil().GetYearMonthMax(symbol); const int yearMax = yearMonthMax.first; const int monthMax = yearMonthMax.second; if ((dates.yearStart < YEAR_MIN) || diff --git a/src/lib-base/src/ConfigTS.cpp b/src/lib-base/src/ConfigTS.cpp index ebb3ef8..093ef12 100644 --- a/src/lib-base/src/ConfigTS.cpp +++ b/src/lib-base/src/ConfigTS.cpp @@ -13,6 +13,7 @@ using namespace EnjoLib; const char * ConfigTS::DEFAULT_SCRIPT_FILE_NAME = "ts-xform-script.txt"; const char * ConfigTS::DEFAULT_SCRIPT_FILE_NAME_R = "r-custom.R"; +const char * ConfigTS::DEFAULT_SCRIPT_FILE_NAME_GEN = "data-generation.txt"; ConfigTS::~ConfigTS(){} @@ -50,8 +51,9 @@ void ConfigTS::RegisterAndReadFloats(EnjoLib::Istream & f) void ConfigTS::RegisterAndReadStrs(EnjoLib::Istream & f) { const ConfigDirs dirs; - RegisterAndReadStr(f, m_scriptPathTxt, dirs.DIR_SCRIPTS2 + DEFAULT_SCRIPT_FILE_NAME); - RegisterAndReadStr(f, m_scriptPathTxtR, dirs.DIR_SCRIPTS2 + DEFAULT_SCRIPT_FILE_NAME_R); + RegisterAndReadStr(f, m_scriptPathTxt, dirs.DIR_SCRIPTS2 + DEFAULT_SCRIPT_FILE_NAME); + RegisterAndReadStr(f, m_scriptPathTxtR, dirs.DIR_SCRIPTS2 + DEFAULT_SCRIPT_FILE_NAME_R); + RegisterAndReadStr(f, m_scriptPathTxtGen, dirs.DIR_SCRIPTS2 + DEFAULT_SCRIPT_FILE_NAME_GEN); } PredictorType ConfigTS::GetPredType() const diff --git a/src/lib-base/src/ConfigTS.h b/src/lib-base/src/ConfigTS.h index 83a2237..4c8f0a2 100644 --- a/src/lib-base/src/ConfigTS.h +++ b/src/lib-base/src/ConfigTS.h @@ -29,9 +29,11 @@ class ConfigTS : public ConfigBase static const char * DEFAULT_SCRIPT_FILE_NAME; static const char * DEFAULT_SCRIPT_FILE_NAME_R; + static const char * DEFAULT_SCRIPT_FILE_NAME_GEN; EnjoLib::Str m_scriptPathTxt; EnjoLib::Str m_scriptPathTxtR; + EnjoLib::Str m_scriptPathTxtGen; bool crashOnRecoverableErrors = false; bool PLOT_SERIES = true; bool PLOT_BASELINE = true; diff --git a/src/lib-base/src/ITicksProvider.h b/src/lib-base/src/ITicksProvider.h index 0a404ed..2220fb1 100644 --- a/src/lib-base/src/ITicksProvider.h +++ b/src/lib-base/src/ITicksProvider.h @@ -14,7 +14,7 @@ class ITicksProvider virtual CorPtr GetTicks(const EnjoLib::Str & symbolName, const ConfigSym * confSym = nullptr) const = 0; virtual CorPtr GetPeriod(const EnjoLib::Str & symbolName, const EnjoLib::Str & periodName) const = 0; - virtual bool IsFake() const { return false; } + virtual bool IsGenerated() const { return false; } protected: private: diff --git a/src/lib-base/src/MainTester.cpp b/src/lib-base/src/MainTester.cpp index bad00b7..d28e040 100644 --- a/src/lib-base/src/MainTester.cpp +++ b/src/lib-base/src/MainTester.cpp @@ -112,9 +112,12 @@ CorPtr MainTester::GetSymbol(EnjoLib::Str symbolName, const VecStr & pe //lastSymbol = symbolName; //lastPeriod = p->GetName(); } - if (not tickProvider->IsFake()) // Don't filter generated data (for now) + if (not tickProvider->IsGenerated()) // Don't filter generated data (for now) { + ELO + //LOG << "Pre filter " << ticks.size() << Nl; ticks = mtu.Filter(symbolName, ticks, m_confSym)->GetTicksView(); + //LOG << "Post filter " << ticks.size() << Nl; } p->SetCandles(ticks.ToCandles()); } @@ -138,7 +141,7 @@ CorPtr MainTester::GetSymbol(EnjoLib::Str symbolName, const VecStr & pe void MainTester::ArchiveSymbol(EnjoLib::Str symbolName, const ITicksProvider * tickProvider) const { - if (tickProvider->IsFake()) + if (tickProvider->IsGenerated()) { m_log << symbolName << ": " << "Fake provider - not storing." << EnjoLib::Nl; return; diff --git a/src/lib-base/src/OrderedSeries.cpp b/src/lib-base/src/OrderedSeries.cpp index 648d3a2..5cc2b79 100644 --- a/src/lib-base/src/OrderedSeries.cpp +++ b/src/lib-base/src/OrderedSeries.cpp @@ -1,6 +1,8 @@ #include "OrderedSeries.h" #include "Tick.h" +#include + OrderedSeries::OrderedSeries(const ISymbol & sym) : PeriodClean(1, sym) { @@ -20,12 +22,12 @@ void OrderedSeries::FeedVals(const EnjoLib::VecD & vals) } } -void OrderedSeries::FeedVal(double val) +void OrderedSeries::FeedVal(double open, double close, double high, double low) { Tick tick; if (Len() == 0) { - tick.year = 1970; + tick.year = 2010; tick.month = 1; tick.day = 1; tick.hour = 0; @@ -44,10 +46,13 @@ void OrderedSeries::FeedVal(double val) { tick.hour = 0; tick.day++; + //LOGL << "New day = " << int(tick.day) << EnjoLib::Nl; if (tick.day == 29) // Dubious { tick.day = 1; tick.month++; + //LOGL << "New month = " << int(tick.month) << EnjoLib::Nl; + if (tick.month == 13) { tick.month = 1; @@ -57,6 +62,15 @@ void OrderedSeries::FeedVal(double val) } } } - tick.open = tick.close = tick.hi = tick.lo = val; + tick.open = open; + tick.close = close; + tick.hi = high; + tick.lo = low; + Feed(tick); +} + +void OrderedSeries::FeedVal(double val) +{ + FeedVal(val, val, val, val); } \ No newline at end of file diff --git a/src/lib-base/src/OrderedSeries.h b/src/lib-base/src/OrderedSeries.h index 805b3cc..0b97695 100644 --- a/src/lib-base/src/OrderedSeries.h +++ b/src/lib-base/src/OrderedSeries.h @@ -10,6 +10,7 @@ class OrderedSeries : public PeriodClean OrderedSeries(const ISymbol & sym); virtual ~OrderedSeries(); + void FeedVal(double open, double close, double high, double low); void FeedVal(double val); void FeedVals(const EnjoLib::VecD & vals); diff --git a/src/lib-base/src/PriceFilterSpreadMul.cpp b/src/lib-base/src/PriceFilterSpreadMul.cpp index 91cfcb6..8d982b7 100644 --- a/src/lib-base/src/PriceFilterSpreadMul.cpp +++ b/src/lib-base/src/PriceFilterSpreadMul.cpp @@ -9,7 +9,7 @@ #include #include -#include +//#include using namespace std; using namespace EnjoLib; @@ -22,7 +22,9 @@ VecCan PriceFilterSpreadMul::Filter(const PriceFilterConfig & cfg, const IDataPr //return canDef; /// DISABLED LOGL << "PriceFilterSpreadMul::Filter() " << per.GetName() << Nl; if (cfg.spreadMulFilter <= 0) + { return canDef; + } const CandleCalc ccc; VecCan ret; @@ -54,7 +56,7 @@ VecCan PriceFilterSpreadMul::Filter(const PriceFilterConfig & cfg, const IDataPr } } ret.push_back(cani); - + maxMinBody.Reset(); maxMinWick.Reset(); } diff --git a/src/lib-base/src/TicksProviderBinary.cpp b/src/lib-base/src/TicksProviderBinary.cpp index 27ba4fd..551c360 100644 --- a/src/lib-base/src/TicksProviderBinary.cpp +++ b/src/lib-base/src/TicksProviderBinary.cpp @@ -56,7 +56,6 @@ CorPtr TicksProviderBinary::GetTicks(const EnjoLib::Str & symbol, const } return CorPtr(new Ticks(ret)); } - } CorPtr TicksProviderBinary::GetPeriod(const EnjoLib::Str & symbolName, const EnjoLib::Str & periodName) const diff --git a/src/lib-base/src/TicksProviderFake.h b/src/lib-base/src/TicksProviderFake.h index ee76b84..6f1386c 100644 --- a/src/lib-base/src/TicksProviderFake.h +++ b/src/lib-base/src/TicksProviderFake.h @@ -15,7 +15,7 @@ class TicksProviderFake : public ITicksProvider CorPtr GetTicks(const EnjoLib::Str & symbolName, const ConfigSym * confSym = nullptr) const override; CorPtr GetPeriod(const EnjoLib::Str & symbolName, const EnjoLib::Str & periodName) const override; - bool IsFake() const override { return true; } + bool IsGenerated() const override { return true; } protected: private: }; @@ -29,7 +29,7 @@ class TicksProviderXMRFake : public ITicksProvider CorPtr GetTicks(const EnjoLib::Str & symbolName, const ConfigSym * confSym = nullptr) const override; CorPtr GetPeriod(const EnjoLib::Str & symbolName, const EnjoLib::Str & periodName) const override; - bool IsFake() const override { return true; } + bool IsGenerated() const override { return true; } protected: private: diff --git a/src/lib-base/src/TicksProviderGenerated.cpp b/src/lib-base/src/TicksProviderGenerated.cpp index 622c25f..73913b0 100644 --- a/src/lib-base/src/TicksProviderGenerated.cpp +++ b/src/lib-base/src/TicksProviderGenerated.cpp @@ -1,6 +1,20 @@ #include "TicksProviderGenerated.h" +#include "ConfigMan.h" +#include "ConfigTS.h" +#include "OrderedSeries.h" +#include "SymbolFactoryClean.h" +#include "Ticks.h" +#include +#include +#include +#include #include +#include +#include +#include + +#include using namespace EnjoLib; @@ -13,3 +27,73 @@ TicksProviderGenerated::~TicksProviderGenerated() { //dtor } + +CorPtr TicksProviderGenerated::GetPeriod(const Str & symbolName, const Str & periodName) const +{ + const SymbolFactoryClean symFact; + CorPtr ticks = GetTicks(symbolName); + CorPtr isym = symFact.Create(symbolName, VecStr { periodName } ); + const auto ticksView = ticks->GetTicksView(); + for (const Tick & tic : ticksView) + { + isym->Feed(tic); + } + const IPeriod & per = isym->GetPeriod(periodName); + + return CorPtr(new Ticks(*per.GetCandles().ToTicks())); +} + +CorPtr TicksProviderGenerated::GetTicks(const Str & symbolName, const ConfigSym * confSym) const +{ + const ConfigTS & cfgTS = *gcfgMan.cfgTS; + + const Tokenizer tok; + const Str & fname = cfgTS.m_scriptPathTxtGen; + Str line1st; + { Ifstream file(fname); } /// TODO: Move functionality to tok.GetLine + //LOG << "Reading script file: '" << fname << "'" << Nl; + const VecStr & lines = tok.GetLines(fname); + for (const Str & line : tok.FilterLines(lines)) + { + //LOG << "Script line: '" << line << "'" << Nl; + //m_xformMan.AddXForm(line); + line1st = line; + break; // only use the 1st line for now + } + + const VecStr & pars = tok.Tokenize(line1st); + if (pars.at(0) != "constant") + { + throw ExceptGeneral("Wrong type = " + pars.at(0)); + } + const CharManipulations cman; + const int len = cman.ToDouble(pars.at(1)); + const double mean = cman.ToDouble(pars.at(2)); + const double stdDev = cman.ToDouble(pars.at(3)); + + const SymbolFactoryClean symFact; + const auto isym = symFact.Create(symbolName); + OrderedSeries oser(*isym); + + const RandomMath rand; + const GeneralMath gmat; + double open = 0; + for (int i = 0; i < len; ++i) + { + const double close = rand.Rand(mean - stdDev, mean + stdDev); + const double stretch = 1.5; + const double high1 = rand.Rand(0, stdDev); + const double low1 = rand.Rand(0, stdDev); + const double high = close + high1; + const double low = close - low1; + if (i == 0) + { + open = close; + } + //oser.FeedVal(open, close, high, low); + oser.FeedVal(close); + + open = close; + } + return CorPtr(new Ticks(*oser.GetCandles().ToTicks())); +} diff --git a/src/lib-base/src/TicksProviderGenerated.h b/src/lib-base/src/TicksProviderGenerated.h index e74327e..5c4d49f 100644 --- a/src/lib-base/src/TicksProviderGenerated.h +++ b/src/lib-base/src/TicksProviderGenerated.h @@ -4,13 +4,15 @@ #include -class TicksProviderGenerated : public TicksProviderXMRFake +class TicksProviderGenerated : public ITicksProvider { public: TicksProviderGenerated(); virtual ~TicksProviderGenerated(); - bool IsFake() const override { return true; } + CorPtr GetTicks(const EnjoLib::Str & symbolName, const ConfigSym * confSym = nullptr) const override; + CorPtr GetPeriod(const EnjoLib::Str & symbolName, const EnjoLib::Str & periodName) const override; + bool IsGenerated() const override { return true; } protected: diff --git a/src/lib-base/src/TradeUtil.cpp b/src/lib-base/src/TradeUtil.cpp index 8fb1d27..4db3694 100644 --- a/src/lib-base/src/TradeUtil.cpp +++ b/src/lib-base/src/TradeUtil.cpp @@ -240,12 +240,11 @@ EnjoLib::Str TradeUtil::CutSymbolTestingPrefix(const EnjoLib::Str & symbolName) return symbolName; } -EnjoLib::Pair TradeUtil::GetYearMonthMax() const +EnjoLib::Pair TradeUtil::GetYearMonthMax(const EnjoLib::Str & symbolName) const { EnjoLib::Pair err(2018, 12); MaxMinFind maxMonth, maxYear; - const string sym = "EURUSD"; - const EnjoLib::Str dirBase = gcfgMan.cfgSym->GetDataBinDir() + sym + "/"; + const EnjoLib::Str dirBase = gcfgMan.cfgSym->GetDataBinDir() + symbolName + "/"; if (not FileUtils().DirExists(dirBase)) { return err; @@ -274,7 +273,7 @@ EnjoLib::Pair TradeUtil::GetYearMonthMax() const } const int year = maxYear .GetMax(); const int mont = maxMonth.GetMax(); - //cout << sym << ": Year/month max = " << year << "." << mont << endl; + LOGL << symbolName << ": Year/month max = " << year << "." << mont << Endl; return EnjoLib::Pair(year, mont); } diff --git a/src/lib-base/src/TradeUtil.h b/src/lib-base/src/TradeUtil.h index 7b9ddb4..1810a01 100644 --- a/src/lib-base/src/TradeUtil.h +++ b/src/lib-base/src/TradeUtil.h @@ -34,7 +34,7 @@ class TradeUtil DateInt DateZeroMinute(DateInt date) const; DateInt DateZeroHour(DateInt date) const; DateInt DateZeroNumSuffix(DateInt date, int numZeros) const; - EnjoLib::Pair GetYearMonthMax() const; + EnjoLib::Pair GetYearMonthMax(const EnjoLib::Str & symbolName) const; EnjoLib::Str MinutesToPeriodName(int minutes) const; diff --git a/src/tsqsim-lib/static/scripts/data-generation.txt b/src/tsqsim-lib/static/scripts/data-generation.txt new file mode 100644 index 0000000..c38a4ed --- /dev/null +++ b/src/tsqsim-lib/static/scripts/data-generation.txt @@ -0,0 +1,3 @@ +constant 1e5 3 0.5 +trend 1 +rounding -1 diff --git a/src/wxConfQT/src/DialogTS.cpp b/src/wxConfQT/src/DialogTS.cpp index e71fb67..c3c514e 100644 --- a/src/wxConfQT/src/DialogTS.cpp +++ b/src/wxConfQT/src/DialogTS.cpp @@ -21,6 +21,9 @@ const long DialogTS::ID_BUTTON2 = wxNewId(); const long DialogTS::ID_TEXTCTRL2 = wxNewId(); const long DialogTS::ID_BUTTON3 = wxNewId(); const long DialogTS::ID_BUTTON4 = wxNewId(); +const long DialogTS::ID_TEXTCTRL3 = wxNewId(); +const long DialogTS::ID_BUTTON5 = wxNewId(); +const long DialogTS::ID_BUTTON6 = wxNewId(); const long DialogTS::ID_CHOICE1 = wxNewId(); const long DialogTS::ID_CHOICE2 = wxNewId(); const long DialogTS::ID_CHECKBOX12 = wxNewId(); @@ -39,11 +42,11 @@ DialogTS::DialogTS(wxWindow* parent,wxWindowID id) wxBoxSizer* BoxSizer3; wxBoxSizer* BoxSizer4; wxBoxSizer* BoxSizer5; - wxBoxSizer* BoxSizer6; wxStaticBoxSizer* StaticBoxSizer1; wxStaticBoxSizer* StaticBoxSizer2; wxStaticBoxSizer* StaticBoxSizer3; wxStaticBoxSizer* StaticBoxSizer4; + wxStaticBoxSizer* StaticBoxSizer5; Create(parent, id, _T("Time Series Analysis"), wxDefaultPosition, wxDefaultSize, wxDEFAULT_DIALOG_STYLE, _T("id")); BoxSizer1 = new wxBoxSizer(wxVERTICAL); @@ -53,26 +56,32 @@ DialogTS::DialogTS(wxWindow* parent,wxWindowID id) m_txtScriptPath = new wxTextCtrl(this, ID_TEXTCTRL1, wxEmptyString, wxDefaultPosition, wxDefaultSize, 0, wxDefaultValidator, _T("ID_TEXTCTRL1")); m_txtScriptPath->SetToolTip(_T("Path to the text script")); BoxSizer4->Add(m_txtScriptPath, 1, wxALL|wxEXPAND, 5); - StaticBoxSizer1->Add(BoxSizer4, 1, wxALL|wxEXPAND, 5); - BoxSizer2 = new wxBoxSizer(wxHORIZONTAL); - m_butScriptText = new wxButton(this, ID_BUTTON1, _T("Find"), wxDefaultPosition, wxDefaultSize, 0, wxDefaultValidator, _T("ID_BUTTON1")); - BoxSizer2->Add(m_butScriptText, 0, wxALL|wxEXPAND, 5); - m_butScriptTextOpen = new wxButton(this, ID_BUTTON2, _T("Open"), wxDefaultPosition, wxDefaultSize, 0, wxDefaultValidator, _T("ID_BUTTON2")); - BoxSizer2->Add(m_butScriptTextOpen, 1, wxALL|wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL, 5); - StaticBoxSizer1->Add(BoxSizer2, 1, wxALL|wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL, 5); - m_sizerBools->Add(StaticBoxSizer1, 1, wxALL|wxEXPAND, 5); + m_butScriptText = new wxButton(this, ID_BUTTON1, _T("Find"), wxDefaultPosition, wxSize(50,-1), 0, wxDefaultValidator, _T("ID_BUTTON1")); + BoxSizer4->Add(m_butScriptText, 0, wxALL, 5); + m_butScriptTextOpen = new wxButton(this, ID_BUTTON2, _T("Open"), wxDefaultPosition, wxSize(50,-1), 0, wxDefaultValidator, _T("ID_BUTTON2")); + BoxSizer4->Add(m_butScriptTextOpen, 0, wxALL, 5); + StaticBoxSizer1->Add(BoxSizer4, 0, wxALL|wxEXPAND, 5); + m_sizerBools->Add(StaticBoxSizer1, 0, wxALL|wxEXPAND, 5); StaticBoxSizer4 = new wxStaticBoxSizer(wxVERTICAL, this, _T("Script R Custom")); BoxSizer5 = new wxBoxSizer(wxHORIZONTAL); m_txtScriptPathR = new wxTextCtrl(this, ID_TEXTCTRL2, wxEmptyString, wxDefaultPosition, wxDefaultSize, 0, wxDefaultValidator, _T("ID_TEXTCTRL2")); BoxSizer5->Add(m_txtScriptPathR, 1, wxALL|wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL, 5); + m_butScriptTextR = new wxButton(this, ID_BUTTON3, _T("Find"), wxDefaultPosition, wxSize(50,-1), 0, wxDefaultValidator, _T("ID_BUTTON3")); + BoxSizer5->Add(m_butScriptTextR, 0, wxALL|wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL, 5); + m_butScriptTextOpenR = new wxButton(this, ID_BUTTON4, _T("Open"), wxDefaultPosition, wxSize(50,-1), 0, wxDefaultValidator, _T("ID_BUTTON4")); + BoxSizer5->Add(m_butScriptTextOpenR, 0, wxALL|wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL, 5); StaticBoxSizer4->Add(BoxSizer5, 1, wxALL|wxEXPAND, 5); - BoxSizer6 = new wxBoxSizer(wxHORIZONTAL); - m_butScriptTextR = new wxButton(this, ID_BUTTON3, _T("Find"), wxDefaultPosition, wxDefaultSize, 0, wxDefaultValidator, _T("ID_BUTTON3")); - BoxSizer6->Add(m_butScriptTextR, 1, wxALL|wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL, 5); - m_butScriptTextOpenR = new wxButton(this, ID_BUTTON4, _T("Open"), wxDefaultPosition, wxDefaultSize, 0, wxDefaultValidator, _T("ID_BUTTON4")); - BoxSizer6->Add(m_butScriptTextOpenR, 1, wxALL|wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL, 5); - StaticBoxSizer4->Add(BoxSizer6, 1, wxALL|wxEXPAND, 5); m_sizerBools->Add(StaticBoxSizer4, 1, wxALL|wxEXPAND, 5); + StaticBoxSizer5 = new wxStaticBoxSizer(wxHORIZONTAL, this, _T("Script data generation")); + BoxSizer2 = new wxBoxSizer(wxHORIZONTAL); + m_txtScriptPathGen = new wxTextCtrl(this, ID_TEXTCTRL3, wxEmptyString, wxDefaultPosition, wxDefaultSize, 0, wxDefaultValidator, _T("ID_TEXTCTRL3")); + BoxSizer2->Add(m_txtScriptPathGen, 1, wxALL|wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL, 5); + m_butScriptTextGen = new wxButton(this, ID_BUTTON5, _T("Find"), wxDefaultPosition, wxSize(50,-1), 0, wxDefaultValidator, _T("ID_BUTTON5")); + BoxSizer2->Add(m_butScriptTextGen, 0, wxALL|wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL, 5); + m_butScriptTextOpenGen = new wxButton(this, ID_BUTTON6, _T("Open"), wxDefaultPosition, wxSize(50,-1), 0, wxDefaultValidator, _T("ID_BUTTON6")); + BoxSizer2->Add(m_butScriptTextOpenGen, 0, wxALL|wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL, 5); + StaticBoxSizer5->Add(BoxSizer2, 1, wxALL|wxEXPAND, 5); + m_sizerBools->Add(StaticBoxSizer5, 1, wxALL|wxEXPAND, 5); StaticBoxSizer3 = new wxStaticBoxSizer(wxVERTICAL, this, _T("Predictor type")); m_choicePredType = new MyChoiceEnum(this, ID_CHOICE1, wxDefaultPosition, wxDefaultSize, 0, 0, 0, wxDefaultValidator, _T("ID_CHOICE1")); m_choicePredType->SetToolTip(_T("Prediction type")); @@ -101,6 +110,8 @@ DialogTS::DialogTS(wxWindow* parent,wxWindowID id) Connect(ID_BUTTON2,wxEVT_COMMAND_BUTTON_CLICKED,(wxObjectEventFunction)&DialogTS::Onm_butScriptTextOpenClick); Connect(ID_BUTTON3,wxEVT_COMMAND_BUTTON_CLICKED,(wxObjectEventFunction)&DialogTS::Onm_butScriptTextRClick); Connect(ID_BUTTON4,wxEVT_COMMAND_BUTTON_CLICKED,(wxObjectEventFunction)&DialogTS::Onm_butScriptTextOpenRClick); + Connect(ID_BUTTON5,wxEVT_COMMAND_BUTTON_CLICKED,(wxObjectEventFunction)&DialogTS::Onm_butScriptTextGenClick); + Connect(ID_BUTTON6,wxEVT_COMMAND_BUTTON_CLICKED,(wxObjectEventFunction)&DialogTS::Onm_butScriptTextOpenGenClick); Connect(ID_CHOICE1,wxEVT_COMMAND_CHOICE_SELECTED,(wxObjectEventFunction)&DialogTS::OnTS); Connect(ID_CHOICE2,wxEVT_COMMAND_CHOICE_SELECTED,(wxObjectEventFunction)&DialogTS::OnTS); Connect(ID_CHECKBOX12,wxEVT_COMMAND_CHECKBOX_CLICKED,(wxObjectEventFunction)&DialogTS::OnTS); diff --git a/src/wxConfQT/src/DialogTS.h b/src/wxConfQT/src/DialogTS.h index 2ae4399..ffcd7a3 100644 --- a/src/wxConfQT/src/DialogTS.h +++ b/src/wxConfQT/src/DialogTS.h @@ -37,11 +37,14 @@ class DialogTS : public wxDialog wxButton* Button1; wxButton* m_butOK; wxButton* m_butScriptText; + wxButton* m_butScriptTextGen; wxButton* m_butScriptTextOpen; + wxButton* m_butScriptTextOpenGen; wxButton* m_butScriptTextOpenR; wxButton* m_butScriptTextR; wxCheckBox* m_chkTechs; wxTextCtrl* m_txtScriptPath; + wxTextCtrl* m_txtScriptPathGen; wxTextCtrl* m_txtScriptPathR; //*) @@ -56,6 +59,9 @@ class DialogTS : public wxDialog static const long ID_TEXTCTRL2; static const long ID_BUTTON3; static const long ID_BUTTON4; + static const long ID_TEXTCTRL3; + static const long ID_BUTTON5; + static const long ID_BUTTON6; static const long ID_CHOICE1; static const long ID_CHOICE2; static const long ID_CHECKBOX12; @@ -72,6 +78,8 @@ class DialogTS : public wxDialog void Onm_butScriptTextOpenClick(wxCommandEvent& event); void Onm_butScriptTextRClick(wxCommandEvent& event); void Onm_butScriptTextOpenRClick(wxCommandEvent& event); + void Onm_butScriptTextOpenGenClick(wxCommandEvent& event); + void Onm_butScriptTextGenClick(wxCommandEvent& event); //*) MapCheckbox m_mapCheckbox; MapChoiceEnum m_mapChoice; diff --git a/src/wxConfQT/src/DialogTSImpl.cpp b/src/wxConfQT/src/DialogTSImpl.cpp index a369ce1..adcd85b 100644 --- a/src/wxConfQT/src/DialogTSImpl.cpp +++ b/src/wxConfQT/src/DialogTSImpl.cpp @@ -36,8 +36,9 @@ void DialogTS::ReadSelections() { for (IMapControl * pmap : m_maps) pmap->FromCheckToVariable(); - m_confTS.m_scriptPathTxt = m_txtScriptPath-> GetValue().c_str().AsChar(); - m_confTS.m_scriptPathTxtR = m_txtScriptPathR->GetValue().c_str().AsChar(); + m_confTS.m_scriptPathTxt = m_txtScriptPath-> GetValue().c_str().AsChar(); + m_confTS.m_scriptPathTxtR = m_txtScriptPathR-> GetValue().c_str().AsChar(); + m_confTS.m_scriptPathTxtGen = m_txtScriptPathGen->GetValue().c_str().AsChar(); } /// Populate selections @@ -46,8 +47,9 @@ void DialogTS::RestoreConf() m_confTS.Read(); wxUtil().AutoGenCheckBox(this, m_confTS, m_sizerBools, &m_mapCheckbox, (wxObjectEventFunction)&DialogTS::OnTS); - m_txtScriptPath ->SetValue(m_confTS.m_scriptPathTxt. c_str()); - m_txtScriptPathR->SetValue(m_confTS.m_scriptPathTxtR.c_str()); + m_txtScriptPath ->SetValue(m_confTS.m_scriptPathTxt. c_str()); + m_txtScriptPathR ->SetValue(m_confTS.m_scriptPathTxtR. c_str()); + m_txtScriptPathGen->SetValue(m_confTS.m_scriptPathTxtGen.c_str()); for (IMapControl * pmap : m_maps) pmap->FromVariableToCheck(); @@ -77,3 +79,15 @@ void DialogTS::Onm_butScriptTextOpenRClick(wxCommandEvent& event) const wxString file_name = m_txtScriptPathR->GetValue(); EnjoLib::UtilWx::ExecuteDefaultEditorOnFile(file_name, ".txt"); } + +void DialogTS::Onm_butScriptTextGenClick(wxCommandEvent& event) +{ + EnjoLib::UtilWx::FindFileFillTxt(this, m_txtScriptPathGen); + OnTS(event); +} + +void DialogTS::Onm_butScriptTextOpenGenClick(wxCommandEvent& event) +{ + const wxString file_name = m_txtScriptPathGen->GetValue(); + EnjoLib::UtilWx::ExecuteDefaultEditorOnFile(file_name, ".txt"); +} diff --git a/src/wxConfQT/src/wxUtil.cpp b/src/wxConfQT/src/wxUtil.cpp index e8da6a3..fafdbcb 100644 --- a/src/wxConfQT/src/wxUtil.cpp +++ b/src/wxConfQT/src/wxUtil.cpp @@ -59,7 +59,17 @@ void wxUtil::SetSelectionInt(wxChoice* choice, long int sel) const void wxUtil::InitYears(wxChoice* choice) const { - InitChoice(choice, 2000, TradeUtil().GetYearMonthMax().first); + const int startYear = 2000; /// TODO: Make it dynamic + const VecStr & symbols = TradeUtil().GetAllSymbolsFromTxtDir(); + if (symbols.empty()) + { + InitChoice(choice, startYear, 2022); + } + else + { + InitChoice(choice, startYear, TradeUtil().GetYearMonthMax(symbols.at(0)).first); + } + } void wxUtil::InitMonths(wxChoice* choice) const { diff --git a/src/wxConfQT/wxsmith/DialogTS.wxs b/src/wxConfQT/wxsmith/DialogTS.wxs index 90e63ed..d4ee838 100644 --- a/src/wxConfQT/wxsmith/DialogTS.wxs +++ b/src/wxConfQT/wxsmith/DialogTS.wxs @@ -21,39 +21,31 @@ 5 - - wxALL|wxEXPAND - 5 - - - - + 50,-1 - wxALL|wxEXPAND + wxALL 5 + 50,-1 - wxALL|wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL + wxALL 5 - - wxALL|wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL + wxALL|wxEXPAND 5 - wxALL|wxEXPAND 5 - @@ -67,30 +59,62 @@ 5 + + + + 50,-1 + + + wxALL|wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL + 5 + + + + + 50,-1 + + + wxALL|wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL + 5 + wxALL|wxEXPAND 5 + + wxALL|wxEXPAND + 5 + + + + + - + - + + wxALL|wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL + 5 + + + + - + 50,-1 + wxALL|wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL 5 - - + - + 50,-1 + wxALL|wxALIGN_CENTER_HORIZONTAL|wxALIGN_CENTER_VERTICAL 5 - wxALL|wxEXPAND diff --git a/tests/test-base/CMakeLists.txt b/tests/test-base/CMakeLists.txt index ef4c89b..34983c4 100644 --- a/tests/test-base/CMakeLists.txt +++ b/tests/test-base/CMakeLists.txt @@ -11,12 +11,9 @@ add_executable(${PROJECT_NAME} ${SRCS}) add_test(${PROJECT_NAME} ${PROJECT_NAME}) enjoSetupTarget(${PROJECT_NAME}) target_link_libraries(${PROJECT_NAME} lib-base test-common) - +tsqSetupSymlinks(${PROJECT_NAME}) enjo_target_unity(${PROJECT_NAME}) -# A hack to make the tests run around OptiVar -add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/static/opti-release -) + # Make sure the compiler can find include files for our Hello library # when other libraries or executables link to Hello diff --git a/tests/test-tsqsim/CMakeLists.txt b/tests/test-tsqsim/CMakeLists.txt index 1b2adae..fa37590 100644 --- a/tests/test-tsqsim/CMakeLists.txt +++ b/tests/test-tsqsim/CMakeLists.txt @@ -11,9 +11,10 @@ add_executable(${PROJECT_NAME} ${SRCS}) add_test(${PROJECT_NAME} ${PROJECT_NAME}) enjoSetupTarget(${PROJECT_NAME}) target_link_libraries(${PROJECT_NAME} tsqsim-lib test-common) - +tsqSetupSymlinks(${PROJECT_NAME}) enjo_target_unity(${PROJECT_NAME}) + # Make sure the compiler can find include files for our Hello library # when other libraries or executables link to Hello target_include_directories (${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/tests/test-tsqsim/src/TSXformImplTest.cpp b/tests/test-tsqsim/src/TSXformImplTest.cpp index 6d89e82..0aa7681 100644 --- a/tests/test-tsqsim/src/TSXformImplTest.cpp +++ b/tests/test-tsqsim/src/TSXformImplTest.cpp @@ -9,6 +9,8 @@ #include "IHasCandles.h" #include "TSXformRes.h" #include "Candle.h" +#include "SymbolFactoryClean.h" +#include "OrderedSeries.h" #include "PredictorFactory.h" #include "PredictorType.h" @@ -125,9 +127,8 @@ TEST(Conv_inv_low_level_1_iter) switch (type) { - /// TODO: Service exceptions //case TSXformType::DIFF: - // break; /// TODO: Probably the most important + // break; // This is how you'd make exceptions default: { TestXformIteration(inp, xform.get(), verbose); @@ -245,7 +246,7 @@ static VecD TestXformArrayManPred(const IDataProvider & dat, const VecD & vecTru //verbose = true; TSXformDataMan dataMan; FillDataMan(vecTrue, man, &dataMan); - + CorPtr algo = PredictorFactory().Create(dat, type); const VecD & pred = algo->Predict(dataMan.converted); VecD reconstructed; @@ -256,7 +257,7 @@ static VecD TestXformArrayManPred(const IDataProvider & dat, const VecD & vecTru const TSRes & reconstr = man.Reconstruct(conv, lost); reconstructed.push_back(reconstr.val); } - + VecD vecExpected; const int lags = algo->GetLags(); for (int i = 0; i < lags; ++i) @@ -270,16 +271,19 @@ static VecD TestXformArrayManPred(const IDataProvider & dat, const VecD & vecTru CHECK_EQUAL(vecTrue.size(), reconstructed.size()); CHECK_EQUAL(vecExpected.size(), reconstructed.size()); CHECK_ARRAY_CLOSE(vecExpected, reconstructed, reconstructed.size(), 0.01); - + return reconstructed; } -TEST(Pred_xform_sqrt) /// TODO: Fixme +TEST(Pred_xform_sqrt) { TSXformMan man(PriceType::CLOSE); man.AddXForm(TSXformType::SQRTS); const VecD & vecTrue = TestXformGenInput(); - //const VecD & reconstrPred = TestXformArrayManPred(vecTrue, man, PredictorType::PRED_BASELINE); + CorPtr isym = SymbolFactoryClean().Create("Oser"); + OrderedSeries oser(*isym); + //oser.FeedVals(vecTrue); + const VecD & reconstrPred = TestXformArrayManPred(oser, vecTrue, man, PredictorType::PRED_BASELINE); //LOGL << reconstrPred.Print() << Nl; } @@ -289,6 +293,9 @@ TEST(Pred_xform_diff) /// TODO: FIXME man.AddXForm(TSXformType::DIFF); const VecD & vecTrue = TestXformGenInput(); /// TODO: Diff distorts the signal being inverted - //const VecD & reconstrPred = TestXformArrayManPred(vecTrue, man, PredictorType::PRED_BASELINE); + CorPtr isym = SymbolFactoryClean().Create("Oser"); + OrderedSeries oser(*isym); + //oser.FeedVals(vecTrue); + //const VecD & reconstrPred = TestXformArrayManPred(oser, vecTrue, man, PredictorType::PRED_BASELINE); //LOGL << reconstrPred.Print() << Nl; } diff --git a/util/prep-env.sh b/util/prep-env.sh index b6c0df9..274c226 100755 --- a/util/prep-env.sh +++ b/util/prep-env.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -DATA_FILE_BASE=TX-TS.csv +DATA_FILE_BASE=TXTS.csv URL="http://cryptog.hopto.org/monero/data/" if [[ "$(uname)" == "Linux" || "$(uname)" == "Darwin" ]]; then DATA_FILE=$DATA_FILE_BASE.gz From 911dba68df88059b771b8a847104e6c14afec7e0 Mon Sep 17 00:00:00 2001 From: mj-xmr Date: Wed, 19 Jan 2022 12:30:16 +0100 Subject: [PATCH 2/6] Fix default symbol --- src/lib-base/src/ConfigSym.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib-base/src/ConfigSym.cpp b/src/lib-base/src/ConfigSym.cpp index 515b00d..883fdeb 100644 --- a/src/lib-base/src/ConfigSym.cpp +++ b/src/lib-base/src/ConfigSym.cpp @@ -50,7 +50,7 @@ void ConfigSym::RegisterAndReadFloats(EnjoLib::Istream & f) } void ConfigSym::RegisterAndReadStrs(EnjoLib::Istream & f) { - RegisterAndReadStr(f, symbol, "TX-TS"); + RegisterAndReadStr(f, symbol, "TXTS"); RegisterAndReadStr(f, period, "h8"); } From 97f567d558aa8412db5ca0edecea455671c2c5f6 Mon Sep 17 00:00:00 2001 From: mj-xmr Date: Wed, 19 Jan 2022 12:31:14 +0100 Subject: [PATCH 3/6] REVERT ME: minimized CI --- .github/workflows/build.yml | 129 +----------------------------------- 1 file changed, 2 insertions(+), 127 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4dc923e..eba286c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,8 +27,8 @@ jobs: # max-parallel: 1 # Single threaded for now, because the jobs overwrite each other's data/bin/* files? matrix: os: [ubuntu-latest] #, ubuntu-18.04] # TODO: Consider(!) supporting the previous version. - compiler: [gcc, clang] - linkage: ["", "--shared", "--lto"] + compiler: [gcc] + linkage: ["--shared"] steps: - uses: actions/checkout@v1 with: @@ -75,128 +75,3 @@ jobs: # TODO: All the unmaintained denendencies should have their own CMakeLists.txt file (URT. Others?) and the util/deps-build.sh should not be used, if possible. # TODO: All the testing should be done in a separate job, AFTER building and downloading resources, in order to use caches better (short and easy. Needs to have the build scripts restructured - separate building and testing) - build-macos: - runs-on: ${{ matrix.os }} - env: - CCACHE_TEMPDIR: /tmp/.ccache-temp - strategy: - fail-fast: false - matrix: - #os: [macOS-latest, macOS-11] # prev - #os: [macOS-latest, macOS-10.15] # future - #os: [macOS-10.15, macOS-11] # transitionary - os: [macOS-11] - steps: - - uses: actions/checkout@v1 - with: - submodules: recursive - # Volatile cache - - name: ccache - uses: actions/cache@v2 - with: - path: /Users/runner/Library/Caches/ccache - key: ccache03-${{ runner.os }}-${{ matrix.os }}-build-${{ github.sha }} - restore-keys: ccache03-${{ runner.os }}-${{ matrix.os }}-build- - # Less volatile cache - - name: data cache - uses: actions/cache@v2 - with: - path: build/data/data/txt - key: data-${{ hashFiles('build/data/data/txt/**/*') }} - restore-keys: data-${{ hashFiles('build/data/data/txt/**/*') }} - - name: prepare environment - run: util/prep-env.sh - - name: install dependencies - run: util/deps-pull.sh - - name: build & install the unmanaged dependencies - run: util/deps-build.sh - - name: build - run: ./ci-default --unity --lto - - name: run demo - run: ./ci-default --unity --lto --run-demo - - uses: actions/upload-artifact@v2 - with: - name: tsqsim-${{ matrix.os }} - path: | - /Users/runner/work/tsqsim/tsqsim/build/default-static-release/bin/* - !/Users/runner/work/tsqsim/tsqsim/build/default-static-release/bin/lib/* - !/Users/runner/work/tsqsim/tsqsim/build/default-static-release/bin/data/data/bin/* - - build-windows: - runs-on: windows-latest - env: - CCACHE_TEMPDIR: C:\Users\runneradmin\.ccache-temp - CCACHE_DIR: C:\Users\runneradmin\.ccache - defaults: - run: - shell: msys2 {0} - steps: - - uses: actions/checkout@v1 - with: - submodules: recursive - # Volatile cache - - name: ccache - uses: actions/cache@v2 - with: - path: ~/.ccache - key: ccache03-${{ runner.os }}-build-${{ github.sha }} - restore-keys: ccache03-${{ runner.os }}-build- - # Less volatile cache - - name: data cache - uses: actions/cache@v2 - with: - path: build/data/data/txt - key: data-${{ hashFiles('build/data/data/txt/**/*') }} - restore-keys: data-${{ hashFiles('build/data/data/txt/**/*') }} - - uses: eine/setup-msys2@v2 - with: - update: false # TODO: Switch to true for master - release: false # TODO: Switch to true for Master - install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-gnuplot mingw-w64-x86_64-eigen3 mingw-w64-x86_64-unittest-cpp mingw-w64-x86_64-boost mingw-w64-x86_64-gsl mingw-w64-x86_64-wxWidgets mingw-w64-x86_64-winpthreads-git git # mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq - - name: prepare environment - run: util/prep-env.sh -# TODO: Use python later - - name: install dependencies - run: util/deps-pull.sh - - name: build & install the unmanaged dependencies - run: util/deps-build.sh - - name: build - run: ./ci-default --unity - - name: run demo - run: ./ci-default --unity --run-demo -# TODO: Make a separate package from GNUPLOT variable - - name: prep minimal - run: | - ls /mingw64/bin - libs="libwinpthread-1.dll libstdc++-6.dll libgcc_s_seh-1.dll libboost_program_options-mt.dll libboost_serialization-mt.dll libboost_system-mt.dll libboost_filesystem-mt.dll libboost_date_time-mt.dll libboost_iostreams-mt.dll libbz2-1.dll liblzma-5.dll libzstd.dll zlib1.dll" - for token in ${libs}; do - cp -v /mingw64/bin/"$token" ${{env.WIN_INSTALL_PATH}} - done - - uses: actions/upload-artifact@v2 - with: - name: tsqsim-Windows-minimal - path: | - ${{env.WIN_INSTALL_PATH}}/tsqsim.exe - ${{env.WIN_INSTALL_PATH}}/*.dll - !${{env.WIN_INSTALL_PATH}}/lib/* - !${{env.WIN_INSTALL_PATH}}/data/data/bin/* - !${{env.WIN_INSTALL_PATH}}/cfg/configTxt.txt - - name: prep full - run: | - GNUPLOT="gnuplot.exe libbrotlicommon.dll libbrotlidec.dll libbrotlienc.dll rav1e.dll libx265.dll libLerc.dll libjbig-0.dll libgmodule-2.0-0.dll libexpat-1.dll libdeflate.dll libde265-0.dll libdatrie-1.dll libdav1d.dll libbrotlidec.dll libaom.dll libffi-7.dll libpcre-1.dll libintl-8.dll libavif.dll libheif.dll libiconv-2.dll imagequant.dll libwebp-7.dll libthai-0.dll libgio-2.0-0.dll libtermcap-0.dll libjpeg-8.dll libpng16-16.dll libtiff-5.dll libfreetype-6.dll imagequant.dll libavif.dll libgd.dll libheif.dll libjpeg-8.dll libpangocairo-1.0-0.dll libpangoft2-1.0-0.dll libpangowin32-1.0-0.dll libreadline8.dll libtermcap-0.dll libtiff-5.dll libwebp-7.dll libXpm-noX4.dll libfontconfig-1.dll libreadline8.dll libgobject-2.0-0.dll libgd.dll libpango-1.0-0.dll libpangocairo-1.0-0.dll libpangoft2-1.0-0.dll libpangowin32-1.0-0.dll libharfbuzz-0.dll libharfbuzz-gobject-0.dll libharfbuzz-icu-0.dll libharfbuzz-subset-0.dll libgraphite2.dll libfribidi-0.dll libcairo-2.dll libcairo-gobject-2.dll libcairo-script-interpreter-2.dll libpixman-1-0.dll liblzo2-2.dll libglib-2.0-0.dll libcerf.dll libcaca-0.dll" - WX="wxbase30u_gcc_custom.dll wxmsw30u_core_gcc_custom.dll" - libs="$GNUPLOT $WX" - for token in ${libs}; do - cp -v /mingw64/bin/"$token" ${{env.WIN_INSTALL_PATH}} - done -# libjpeg-8.dll libpng16-16.dll libtiff-5.dll - - uses: actions/upload-artifact@v2 - with: - name: tsqsim-Windows - path: | - ${{env.WIN_INSTALL_PATH}}/* - !${{env.WIN_INSTALL_PATH}}/lib/* - !${{env.WIN_INSTALL_PATH}}/data/data/bin/* - !${{env.WIN_INSTALL_PATH}}/cfg/configTxt.txt - - From 194b3b367ba4deb2ab43152d5951279838b00417 Mon Sep 17 00:00:00 2001 From: mj-xmr Date: Wed, 19 Jan 2022 13:06:04 +0100 Subject: [PATCH 4/6] Creaded dir --- tests/test-base/CMakeLists.txt | 7 +++++-- tests/test-tsqsim/CMakeLists.txt | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test-base/CMakeLists.txt b/tests/test-base/CMakeLists.txt index 34983c4..ef4c89b 100644 --- a/tests/test-base/CMakeLists.txt +++ b/tests/test-base/CMakeLists.txt @@ -11,9 +11,12 @@ add_executable(${PROJECT_NAME} ${SRCS}) add_test(${PROJECT_NAME} ${PROJECT_NAME}) enjoSetupTarget(${PROJECT_NAME}) target_link_libraries(${PROJECT_NAME} lib-base test-common) -tsqSetupSymlinks(${PROJECT_NAME}) -enjo_target_unity(${PROJECT_NAME}) +enjo_target_unity(${PROJECT_NAME}) +# A hack to make the tests run around OptiVar +add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/static/opti-release +) # Make sure the compiler can find include files for our Hello library # when other libraries or executables link to Hello diff --git a/tests/test-tsqsim/CMakeLists.txt b/tests/test-tsqsim/CMakeLists.txt index fa37590..e16e6a8 100644 --- a/tests/test-tsqsim/CMakeLists.txt +++ b/tests/test-tsqsim/CMakeLists.txt @@ -11,9 +11,13 @@ add_executable(${PROJECT_NAME} ${SRCS}) add_test(${PROJECT_NAME} ${PROJECT_NAME}) enjoSetupTarget(${PROJECT_NAME}) target_link_libraries(${PROJECT_NAME} tsqsim-lib test-common) -tsqSetupSymlinks(${PROJECT_NAME}) enjo_target_unity(${PROJECT_NAME}) +# A hack to make the tests run around OptiVar +add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/static/opti-release +) + # Make sure the compiler can find include files for our Hello library # when other libraries or executables link to Hello From 5c8574fe57716c3da4720c3878f2b5aff8e13219 Mon Sep 17 00:00:00 2001 From: mj-xmr Date: Wed, 19 Jan 2022 13:10:13 +0100 Subject: [PATCH 5/6] Revert "REVERT ME: minimized CI" This reverts commit 97f567d558aa8412db5ca0edecea455671c2c5f6. --- .github/workflows/build.yml | 129 +++++++++++++++++++++++++++++++++++- 1 file changed, 127 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eba286c..4dc923e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,8 +27,8 @@ jobs: # max-parallel: 1 # Single threaded for now, because the jobs overwrite each other's data/bin/* files? matrix: os: [ubuntu-latest] #, ubuntu-18.04] # TODO: Consider(!) supporting the previous version. - compiler: [gcc] - linkage: ["--shared"] + compiler: [gcc, clang] + linkage: ["", "--shared", "--lto"] steps: - uses: actions/checkout@v1 with: @@ -75,3 +75,128 @@ jobs: # TODO: All the unmaintained denendencies should have their own CMakeLists.txt file (URT. Others?) and the util/deps-build.sh should not be used, if possible. # TODO: All the testing should be done in a separate job, AFTER building and downloading resources, in order to use caches better (short and easy. Needs to have the build scripts restructured - separate building and testing) + build-macos: + runs-on: ${{ matrix.os }} + env: + CCACHE_TEMPDIR: /tmp/.ccache-temp + strategy: + fail-fast: false + matrix: + #os: [macOS-latest, macOS-11] # prev + #os: [macOS-latest, macOS-10.15] # future + #os: [macOS-10.15, macOS-11] # transitionary + os: [macOS-11] + steps: + - uses: actions/checkout@v1 + with: + submodules: recursive + # Volatile cache + - name: ccache + uses: actions/cache@v2 + with: + path: /Users/runner/Library/Caches/ccache + key: ccache03-${{ runner.os }}-${{ matrix.os }}-build-${{ github.sha }} + restore-keys: ccache03-${{ runner.os }}-${{ matrix.os }}-build- + # Less volatile cache + - name: data cache + uses: actions/cache@v2 + with: + path: build/data/data/txt + key: data-${{ hashFiles('build/data/data/txt/**/*') }} + restore-keys: data-${{ hashFiles('build/data/data/txt/**/*') }} + - name: prepare environment + run: util/prep-env.sh + - name: install dependencies + run: util/deps-pull.sh + - name: build & install the unmanaged dependencies + run: util/deps-build.sh + - name: build + run: ./ci-default --unity --lto + - name: run demo + run: ./ci-default --unity --lto --run-demo + - uses: actions/upload-artifact@v2 + with: + name: tsqsim-${{ matrix.os }} + path: | + /Users/runner/work/tsqsim/tsqsim/build/default-static-release/bin/* + !/Users/runner/work/tsqsim/tsqsim/build/default-static-release/bin/lib/* + !/Users/runner/work/tsqsim/tsqsim/build/default-static-release/bin/data/data/bin/* + + build-windows: + runs-on: windows-latest + env: + CCACHE_TEMPDIR: C:\Users\runneradmin\.ccache-temp + CCACHE_DIR: C:\Users\runneradmin\.ccache + defaults: + run: + shell: msys2 {0} + steps: + - uses: actions/checkout@v1 + with: + submodules: recursive + # Volatile cache + - name: ccache + uses: actions/cache@v2 + with: + path: ~/.ccache + key: ccache03-${{ runner.os }}-build-${{ github.sha }} + restore-keys: ccache03-${{ runner.os }}-build- + # Less volatile cache + - name: data cache + uses: actions/cache@v2 + with: + path: build/data/data/txt + key: data-${{ hashFiles('build/data/data/txt/**/*') }} + restore-keys: data-${{ hashFiles('build/data/data/txt/**/*') }} + - uses: eine/setup-msys2@v2 + with: + update: false # TODO: Switch to true for master + release: false # TODO: Switch to true for Master + install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-gnuplot mingw-w64-x86_64-eigen3 mingw-w64-x86_64-unittest-cpp mingw-w64-x86_64-boost mingw-w64-x86_64-gsl mingw-w64-x86_64-wxWidgets mingw-w64-x86_64-winpthreads-git git # mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq + - name: prepare environment + run: util/prep-env.sh +# TODO: Use python later + - name: install dependencies + run: util/deps-pull.sh + - name: build & install the unmanaged dependencies + run: util/deps-build.sh + - name: build + run: ./ci-default --unity + - name: run demo + run: ./ci-default --unity --run-demo +# TODO: Make a separate package from GNUPLOT variable + - name: prep minimal + run: | + ls /mingw64/bin + libs="libwinpthread-1.dll libstdc++-6.dll libgcc_s_seh-1.dll libboost_program_options-mt.dll libboost_serialization-mt.dll libboost_system-mt.dll libboost_filesystem-mt.dll libboost_date_time-mt.dll libboost_iostreams-mt.dll libbz2-1.dll liblzma-5.dll libzstd.dll zlib1.dll" + for token in ${libs}; do + cp -v /mingw64/bin/"$token" ${{env.WIN_INSTALL_PATH}} + done + - uses: actions/upload-artifact@v2 + with: + name: tsqsim-Windows-minimal + path: | + ${{env.WIN_INSTALL_PATH}}/tsqsim.exe + ${{env.WIN_INSTALL_PATH}}/*.dll + !${{env.WIN_INSTALL_PATH}}/lib/* + !${{env.WIN_INSTALL_PATH}}/data/data/bin/* + !${{env.WIN_INSTALL_PATH}}/cfg/configTxt.txt + - name: prep full + run: | + GNUPLOT="gnuplot.exe libbrotlicommon.dll libbrotlidec.dll libbrotlienc.dll rav1e.dll libx265.dll libLerc.dll libjbig-0.dll libgmodule-2.0-0.dll libexpat-1.dll libdeflate.dll libde265-0.dll libdatrie-1.dll libdav1d.dll libbrotlidec.dll libaom.dll libffi-7.dll libpcre-1.dll libintl-8.dll libavif.dll libheif.dll libiconv-2.dll imagequant.dll libwebp-7.dll libthai-0.dll libgio-2.0-0.dll libtermcap-0.dll libjpeg-8.dll libpng16-16.dll libtiff-5.dll libfreetype-6.dll imagequant.dll libavif.dll libgd.dll libheif.dll libjpeg-8.dll libpangocairo-1.0-0.dll libpangoft2-1.0-0.dll libpangowin32-1.0-0.dll libreadline8.dll libtermcap-0.dll libtiff-5.dll libwebp-7.dll libXpm-noX4.dll libfontconfig-1.dll libreadline8.dll libgobject-2.0-0.dll libgd.dll libpango-1.0-0.dll libpangocairo-1.0-0.dll libpangoft2-1.0-0.dll libpangowin32-1.0-0.dll libharfbuzz-0.dll libharfbuzz-gobject-0.dll libharfbuzz-icu-0.dll libharfbuzz-subset-0.dll libgraphite2.dll libfribidi-0.dll libcairo-2.dll libcairo-gobject-2.dll libcairo-script-interpreter-2.dll libpixman-1-0.dll liblzo2-2.dll libglib-2.0-0.dll libcerf.dll libcaca-0.dll" + WX="wxbase30u_gcc_custom.dll wxmsw30u_core_gcc_custom.dll" + libs="$GNUPLOT $WX" + for token in ${libs}; do + cp -v /mingw64/bin/"$token" ${{env.WIN_INSTALL_PATH}} + done +# libjpeg-8.dll libpng16-16.dll libtiff-5.dll + - uses: actions/upload-artifact@v2 + with: + name: tsqsim-Windows + path: | + ${{env.WIN_INSTALL_PATH}}/* + !${{env.WIN_INSTALL_PATH}}/lib/* + !${{env.WIN_INSTALL_PATH}}/data/data/bin/* + !${{env.WIN_INSTALL_PATH}}/cfg/configTxt.txt + + From 85afe2e2e0e331b46fc827f7296fc6f6e11622a3 Mon Sep 17 00:00:00 2001 From: mj-xmr Date: Wed, 19 Jan 2022 13:24:54 +0100 Subject: [PATCH 6/6] Static data creation function --- cmake/util-tsqsim.cmake | 7 +++++++ tests/test-base/CMakeLists.txt | 7 ++----- tests/test-tsqsim/CMakeLists.txt | 7 +------ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/cmake/util-tsqsim.cmake b/cmake/util-tsqsim.cmake index 1424adc..a4ed870 100644 --- a/cmake/util-tsqsim.cmake +++ b/cmake/util-tsqsim.cmake @@ -12,3 +12,10 @@ function (tsqSetupSymlinks target) enjoSymlink(${target} ../../../static ${CMAKE_CURRENT_BINARY_DIR}/static) # Common static dir endfunction() + +function (tsqSetupDirStatic target) + # A hack to make the tests run around OptiVar + add_custom_command(TARGET ${target} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/static/opti-release + ) +endfunction() diff --git a/tests/test-base/CMakeLists.txt b/tests/test-base/CMakeLists.txt index ef4c89b..2cfc921 100644 --- a/tests/test-base/CMakeLists.txt +++ b/tests/test-base/CMakeLists.txt @@ -11,12 +11,9 @@ add_executable(${PROJECT_NAME} ${SRCS}) add_test(${PROJECT_NAME} ${PROJECT_NAME}) enjoSetupTarget(${PROJECT_NAME}) target_link_libraries(${PROJECT_NAME} lib-base test-common) - +tsqSetupDirStatic(${PROJECT_NAME}) enjo_target_unity(${PROJECT_NAME}) -# A hack to make the tests run around OptiVar -add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/static/opti-release -) + # Make sure the compiler can find include files for our Hello library # when other libraries or executables link to Hello diff --git a/tests/test-tsqsim/CMakeLists.txt b/tests/test-tsqsim/CMakeLists.txt index e16e6a8..4bc767a 100644 --- a/tests/test-tsqsim/CMakeLists.txt +++ b/tests/test-tsqsim/CMakeLists.txt @@ -12,12 +12,7 @@ add_test(${PROJECT_NAME} ${PROJECT_NAME}) enjoSetupTarget(${PROJECT_NAME}) target_link_libraries(${PROJECT_NAME} tsqsim-lib test-common) enjo_target_unity(${PROJECT_NAME}) - -# A hack to make the tests run around OptiVar -add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/static/opti-release -) - +tsqSetupDirStatic(${PROJECT_NAME}) # Make sure the compiler can find include files for our Hello library # when other libraries or executables link to Hello