Qt time shift fix

cmake/util-tsqsim.cmake

@@ -12,3 +12,10 @@ function (tsqSetupSymlinks target)
 	enjoSymlink(${target} ../../../static ${CMAKE_CURRENT_BINARY_DIR}/static)	# Common static dir
 endfunction()
 
+
+function (tsqSetupDirStatic target)
+	# A hack to make the tests run around OptiVar
+	add_custom_command(TARGET ${target} POST_BUILD
+		       COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/static/opti-release
+	)
+endfunction()

src/lib-base/src/ConfigSym.cpp

@@ -50,7 +50,7 @@ void ConfigSym::RegisterAndReadFloats(EnjoLib::Istream & f)
 }
 void ConfigSym::RegisterAndReadStrs(EnjoLib::Istream & f)
 {
-    RegisterAndReadStr(f, symbol, "TX-TS");
+    RegisterAndReadStr(f, symbol, "TXTS");
     RegisterAndReadStr(f, period, "h8");
 }
 
@@ -114,11 +114,16 @@ bool ConfigSym::ShiftRange(int shift)
 
     ConfigSym backup = *this;
     if (not ShiftRangeUpdate(change, &dates.yearStart, &dates.monthStart))
+    {
         return false;
+    }
     if (not ShiftRangeUpdate(change, &dates.yearEnd,   &dates.monthEnd))
+    {
         return false;
+    }
+
 
-    const Pair<int, int> yearMonthMax = TradeUtil().GetYearMonthMax();
+    const Pair<int, int> yearMonthMax = TradeUtil().GetYearMonthMax(symbol);
     const int yearMax = yearMonthMax.first;
     const int monthMax = yearMonthMax.second;
     if ((dates.yearStart < YEAR_MIN) ||

src/lib-base/src/ConfigTS.cpp

@@ -13,6 +13,7 @@ using namespace EnjoLib;
 
 const char * ConfigTS::DEFAULT_SCRIPT_FILE_NAME   = "ts-xform-script.txt";
 const char * ConfigTS::DEFAULT_SCRIPT_FILE_NAME_R = "r-custom.R";
+const char * ConfigTS::DEFAULT_SCRIPT_FILE_NAME_GEN = "data-generation.txt";
 
 
 ConfigTS::~ConfigTS(){}
@@ -50,8 +51,9 @@ void ConfigTS::RegisterAndReadFloats(EnjoLib::Istream & f)
 void ConfigTS::RegisterAndReadStrs(EnjoLib::Istream & f)
 {
     const ConfigDirs dirs;
-    RegisterAndReadStr(f, m_scriptPathTxt,  dirs.DIR_SCRIPTS2 + DEFAULT_SCRIPT_FILE_NAME);
-    RegisterAndReadStr(f, m_scriptPathTxtR, dirs.DIR_SCRIPTS2 + DEFAULT_SCRIPT_FILE_NAME_R);
+    RegisterAndReadStr(f, m_scriptPathTxt,      dirs.DIR_SCRIPTS2 + DEFAULT_SCRIPT_FILE_NAME);
+    RegisterAndReadStr(f, m_scriptPathTxtR,     dirs.DIR_SCRIPTS2 + DEFAULT_SCRIPT_FILE_NAME_R);
+    RegisterAndReadStr(f, m_scriptPathTxtGen,   dirs.DIR_SCRIPTS2 + DEFAULT_SCRIPT_FILE_NAME_GEN);
 }
 
 PredictorType ConfigTS::GetPredType() const

src/lib-base/src/ConfigTS.h

@@ -29,9 +29,11 @@ class ConfigTS : public ConfigBase
 
         static const char * DEFAULT_SCRIPT_FILE_NAME;
         static const char * DEFAULT_SCRIPT_FILE_NAME_R;
+        static const char * DEFAULT_SCRIPT_FILE_NAME_GEN;
 
         EnjoLib::Str m_scriptPathTxt;
         EnjoLib::Str m_scriptPathTxtR;
+        EnjoLib::Str m_scriptPathTxtGen;
         bool crashOnRecoverableErrors = false;
         bool PLOT_SERIES = true;
         bool PLOT_BASELINE = true;

src/lib-base/src/ITicksProvider.h

@@ -14,7 +14,7 @@ class ITicksProvider
         virtual CorPtr<ITicks> GetTicks(const EnjoLib::Str & symbolName, const ConfigSym * confSym = nullptr) const = 0;
         virtual CorPtr<ITicks> GetPeriod(const EnjoLib::Str & symbolName, const EnjoLib::Str & periodName) const = 0;
 
-        virtual bool IsFake() const { return false; }
+        virtual bool IsGenerated() const { return false; }
 
     protected:
     private:

src/lib-base/src/MainTester.cpp

@@ -112,9 +112,12 @@ CorPtr<ISymbol> MainTester::GetSymbol(EnjoLib::Str symbolName, const VecStr & pe
                 //lastSymbol = symbolName;
                 //lastPeriod = p->GetName();
             }
-            if (not tickProvider->IsFake()) // Don't filter generated data (for now)
+            if (not tickProvider->IsGenerated()) // Don't filter generated data (for now)
             {
+                ELO
+                //LOG << "Pre filter " << ticks.size() << Nl;
                 ticks = mtu.Filter(symbolName, ticks, m_confSym)->GetTicksView();
+                //LOG << "Post filter " << ticks.size() << Nl;
             }
             p->SetCandles(ticks.ToCandles());
         }
@@ -138,7 +141,7 @@ CorPtr<ISymbol> MainTester::GetSymbol(EnjoLib::Str symbolName, const VecStr & pe
 
 void MainTester::ArchiveSymbol(EnjoLib::Str symbolName, const ITicksProvider * tickProvider) const
 {
-    if (tickProvider->IsFake())
+    if (tickProvider->IsGenerated())
     {
         m_log << symbolName << ": " << "Fake provider - not storing." << EnjoLib::Nl;
         return;

src/lib-base/src/OrderedSeries.cpp

@@ -1,6 +1,8 @@
 #include "OrderedSeries.h"
 #include "Tick.h"
 
+#include <Util/CoutBuf.hpp>
+
 OrderedSeries::OrderedSeries(const ISymbol & sym)
 : PeriodClean(1, sym)
 {
@@ -20,12 +22,12 @@ void OrderedSeries::FeedVals(const EnjoLib::VecD & vals)
     }
 }
 
-void OrderedSeries::FeedVal(double val)
+void OrderedSeries::FeedVal(double open, double close, double high, double low)
 {
     Tick tick;
     if (Len() == 0)
     {
-        tick.year = 1970;
+        tick.year = 2010;
         tick.month = 1;
         tick.day = 1;
         tick.hour = 0;
@@ -44,10 +46,13 @@ void OrderedSeries::FeedVal(double val)
             {
                 tick.hour = 0;
                 tick.day++;
+                //LOGL << "New day = " << int(tick.day) << EnjoLib::Nl;
                 if (tick.day == 29) // Dubious
                 {
                     tick.day = 1;
                     tick.month++;
+                    //LOGL << "New month = " << int(tick.month) << EnjoLib::Nl;
+
                     if (tick.month == 13)
                     {
                         tick.month = 1;
@@ -57,6 +62,15 @@ void OrderedSeries::FeedVal(double val)
             }
         }
     }
-    tick.open = tick.close = tick.hi = tick.lo = val;
+    tick.open = open;
+    tick.close = close;
+    tick.hi = high;
+    tick.lo = low;
+
     Feed(tick);
+}
+
+void OrderedSeries::FeedVal(double val)
+{
+    FeedVal(val, val, val, val);
 }

src/lib-base/src/OrderedSeries.h

@@ -10,6 +10,7 @@ class OrderedSeries : public PeriodClean
         OrderedSeries(const ISymbol & sym);
         virtual ~OrderedSeries();
 
+        void FeedVal(double open, double close, double high, double low);
         void FeedVal(double val);
         void FeedVals(const EnjoLib::VecD & vals);
 

src/lib-base/src/PriceFilterSpreadMul.cpp

@@ -9,7 +9,7 @@
 #include <STD/VectorCpp.hpp>
 
 #include <Util/CoutBuf.hpp>
-#include <STD/Algorithm.hpp>
+//#include <STD/Algorithm.hpp>
 
 using namespace std;
 using namespace EnjoLib;
@@ -22,7 +22,9 @@ VecCan PriceFilterSpreadMul::Filter(const PriceFilterConfig & cfg, const IDataPr
     //return canDef; /// DISABLED
     LOGL << "PriceFilterSpreadMul::Filter() " << per.GetName() << Nl;
     if (cfg.spreadMulFilter <= 0)
+    {
         return canDef;
+    }
 
     const CandleCalc ccc;
     VecCan ret;
@@ -54,7 +56,7 @@ VecCan PriceFilterSpreadMul::Filter(const PriceFilterConfig & cfg, const IDataPr
                 }
             }
             ret.push_back(cani);
-            
+
             maxMinBody.Reset();
             maxMinWick.Reset();
         }

src/lib-base/src/TicksProviderBinary.cpp

@@ -56,7 +56,6 @@ CorPtr<ITicks> TicksProviderBinary::GetTicks(const EnjoLib::Str & symbol, const
         }
         return CorPtr<ITicks>(new Ticks(ret));
     }
-
 }
 
 CorPtr<ITicks> TicksProviderBinary::GetPeriod(const EnjoLib::Str & symbolName, const EnjoLib::Str & periodName) const

src/lib-base/src/TicksProviderFake.h

@@ -15,7 +15,7 @@ class TicksProviderFake : public ITicksProvider
         CorPtr<ITicks> GetTicks(const EnjoLib::Str & symbolName, const ConfigSym * confSym = nullptr) const override;
         CorPtr<ITicks> GetPeriod(const EnjoLib::Str & symbolName, const EnjoLib::Str & periodName) const override;
 
-        bool IsFake() const override { return true; }
+        bool IsGenerated() const override { return true; }
     protected:
     private:
 };
@@ -29,7 +29,7 @@ class TicksProviderXMRFake : public ITicksProvider
         CorPtr<ITicks> GetTicks(const EnjoLib::Str & symbolName, const ConfigSym * confSym = nullptr) const override;
         CorPtr<ITicks> GetPeriod(const EnjoLib::Str & symbolName, const EnjoLib::Str & periodName) const override;
 
-        bool IsFake() const override { return true; }
+        bool IsGenerated() const override { return true; }
 
     protected:
     private:

src/lib-base/src/TicksProviderGenerated.cpp

@@ -1,6 +1,20 @@
 #include "TicksProviderGenerated.h"
+#include "ConfigMan.h"
+#include "ConfigTS.h"
+#include "OrderedSeries.h"
+#include "SymbolFactoryClean.h"
+#include "Ticks.h"
 
+#include <Ios/Osstream.hpp>
+#include <Ios/Ifstream.hpp>
+#include <Math/RandomMath.hpp>
+#include <Math/GeneralMath.hpp>
 #include <Util/CoutBuf.hpp>
+#include <Util/Except.hpp>
+#include <Util/Tokenizer.hpp>
+#include <Util/CharManipulations.hpp>
+
+#include <STD/VectorCpp.hpp>
 
 using namespace EnjoLib;
 
@@ -13,3 +27,73 @@ TicksProviderGenerated::~TicksProviderGenerated()
 {
     //dtor
 }
+
+CorPtr<ITicks> TicksProviderGenerated::GetPeriod(const Str & symbolName, const Str & periodName) const
+{
+    const SymbolFactoryClean symFact;
+    CorPtr<ITicks> ticks = GetTicks(symbolName);
+    CorPtr<ISymbol> isym = symFact.Create(symbolName, VecStr { periodName } );
+    const auto ticksView = ticks->GetTicksView();
+    for (const Tick & tic : ticksView)
+    {
+        isym->Feed(tic);
+    }
+    const IPeriod & per = isym->GetPeriod(periodName);
+
+    return CorPtr<ITicks>(new Ticks(*per.GetCandles().ToTicks()));
+}
+
+CorPtr<ITicks> TicksProviderGenerated::GetTicks(const Str & symbolName, const ConfigSym * confSym) const
+{
+    const ConfigTS & cfgTS = *gcfgMan.cfgTS;
+
+    const Tokenizer tok;
+    const Str & fname = cfgTS.m_scriptPathTxtGen;
+    Str line1st;
+    { Ifstream file(fname); } /// TODO: Move functionality to tok.GetLine
+    //LOG << "Reading script file: '" << fname << "'" << Nl;
+    const VecStr & lines = tok.GetLines(fname);
+    for (const Str & line : tok.FilterLines(lines))
+    {
+        //LOG << "Script line: '" << line << "'" << Nl;
+        //m_xformMan.AddXForm(line);
+        line1st = line;
+        break; // only use the 1st line for now
+    }
+
+    const VecStr & pars = tok.Tokenize(line1st);
+    if (pars.at(0) != "constant")
+    {
+        throw ExceptGeneral("Wrong type = " + pars.at(0));
+    }
+    const CharManipulations cman;
+    const int    len    = cman.ToDouble(pars.at(1));
+    const double mean   = cman.ToDouble(pars.at(2));
+    const double stdDev = cman.ToDouble(pars.at(3));
+
+    const SymbolFactoryClean symFact;
+    const auto isym = symFact.Create(symbolName);
+    OrderedSeries oser(*isym);
+
+    const RandomMath rand;
+    const GeneralMath gmat;
+    double open = 0;
+    for (int i = 0; i < len; ++i)
+    {
+        const double close = rand.Rand(mean - stdDev, mean + stdDev);
+        const double stretch = 1.5;
+        const double high1  = rand.Rand(0, stdDev);
+        const double low1   = rand.Rand(0, stdDev);
+        const double high = close + high1;
+        const double low = close - low1;
+        if (i == 0)
+        {
+            open = close;
+        }
+        //oser.FeedVal(open, close, high, low);
+        oser.FeedVal(close);
+
+        open = close;
+    }
+    return CorPtr<ITicks>(new Ticks(*oser.GetCandles().ToTicks()));
+}

src/lib-base/src/TicksProviderGenerated.h

@@ -4,13 +4,15 @@
 #include <TicksProviderFake.h>
 
 
-class TicksProviderGenerated : public TicksProviderXMRFake
+class TicksProviderGenerated : public ITicksProvider
 {
     public:
         TicksProviderGenerated();
         virtual ~TicksProviderGenerated();
 
-        bool IsFake() const override { return true; }
+        CorPtr<ITicks> GetTicks(const EnjoLib::Str & symbolName, const ConfigSym * confSym = nullptr) const override;
+        CorPtr<ITicks> GetPeriod(const EnjoLib::Str & symbolName, const EnjoLib::Str & periodName) const override;
+        bool IsGenerated() const override { return true; }
 
     protected:
 

src/lib-base/src/TradeUtil.cpp

@@ -240,12 +240,11 @@ EnjoLib::Str TradeUtil::CutSymbolTestingPrefix(const EnjoLib::Str & symbolName)
     return symbolName;
 }
 
-EnjoLib::Pair<int, int> TradeUtil::GetYearMonthMax() const
+EnjoLib::Pair<int, int> TradeUtil::GetYearMonthMax(const EnjoLib::Str & symbolName) const
 {
     EnjoLib::Pair<int, int> err(2018, 12);
     MaxMinFind<int> maxMonth, maxYear;
-    const string sym = "EURUSD";
-    const EnjoLib::Str dirBase = gcfgMan.cfgSym->GetDataBinDir() + sym + "/";
+    const EnjoLib::Str dirBase = gcfgMan.cfgSym->GetDataBinDir() + symbolName + "/";
     if (not FileUtils().DirExists(dirBase))
     {
         return err;
@@ -274,7 +273,7 @@ EnjoLib::Pair<int, int> TradeUtil::GetYearMonthMax() const
     }
     const int year = maxYear .GetMax();
     const int mont = maxMonth.GetMax();
-    //cout << sym << ": Year/month max = " << year << "." << mont << endl;
+    LOGL << symbolName << ": Year/month max = " << year << "." << mont << Endl;
     return EnjoLib::Pair<int, int>(year, mont);
 
 }

src/lib-base/src/TradeUtil.h

@@ -34,7 +34,7 @@ class TradeUtil
         DateInt DateZeroMinute(DateInt date) const;
         DateInt DateZeroHour(DateInt date) const;
         DateInt DateZeroNumSuffix(DateInt date, int numZeros) const;
-        EnjoLib::Pair<int, int> GetYearMonthMax() const;
+        EnjoLib::Pair<int, int> GetYearMonthMax(const EnjoLib::Str & symbolName) const;
 
         EnjoLib::Str MinutesToPeriodName(int minutes) const;
 

src/tsqsim-lib/static/scripts/data-generation.txt

@@ -0,0 +1,3 @@
+constant 1e5 3 0.5
+trend 1
+rounding -1