refactor and support for pipeline with no need of macro

.travis.yml

@@ -5,7 +5,7 @@ os:
   - osx
   - windows
 julia:
-  - 1.5
+  - 1.6
   - nightly
 addons:
   apt:
@@ -17,6 +17,14 @@ env:
 notifications:
   email: false
 
+branches:
+   only:
+      - master
+      - main
+      - stable
+   except:
+      - experimental
+
 after_success:
   - if [[ $TRAVIS_OS_NAME = linux ]]; then
       julia --project=coverage/ -e 'using Pkg; Pkg.instantiate();

Project.toml

@@ -1,7 +1,7 @@
 name = "TSML"
 uuid = "198dc43e-9e51-5cd7-9d40-d9794d335912"
 authors = ["Paulito Palmes <ppalmes@gmail.com>"]
-version = "2.6.3"
+version = "2.7.0"
 
 [deps]
 AMLPipelineBase = "e3c3008a-8869-4d53-9f34-c96f99c8a2b6"

src/TSML.jl

@@ -25,6 +25,7 @@ export FeatureSelector, CatFeatureSelector, NumFeatureSelector, CatNumDiscrimina
 export crossvalidate
 export NARemover
 export @pipeline @pipelinex, @pipelinez
+export +, |>, *, |
 export Pipeline, ComboPipeline
 
 import AMLPipelineBase.AbsTypes: fit!, transform!

src/mlbase.jl

@@ -51,7 +51,7 @@ end
 Compute the parameters to center and scale.
 """
 function fit!(st::StandardScaler, features::DataFrame, labels::Vector=[]) 
-   mfeatures = convert(Matrix{Float64},features)
+   mfeatures = Matrix(features)
    pfeatures = mfeatures' |> collect |> Matrix{Float64}
    impl_args = st.model[:impl_args]
    st_transform = estimate(Standardize, Array(mfeatures'); impl_args...)
@@ -64,7 +64,7 @@ end
 Apply the computed parameters for centering and scaling to new data.
 """
 function transform!(st::StandardScaler, features::DataFrame)
-   mfeatures = convert(Matrix{Float64},features)
+   mfeatures = Matrix(features)
    st_transform = st.model[:standardize_transform]
    pfeatures = mfeatures' |> collect |> Matrix{Float64}
    transposed_instances = Array(pfeatures)

test/test_ensemble.jl

@@ -14,8 +14,8 @@ end
 
 function getprediction(model::Learner,data::Dict)
   Random.seed!(126)
-  trfeatures = data[:trfeatures] |> DataFrame
-  tstfeatures = data[:tstfeatures] |> DataFrame
+  trfeatures = data[:trfeatures] 
+  tstfeatures = data[:tstfeatures] 
   troutput = data[:troutput]
   tstoutput = data[:tstoutput]
   fit!(model,trfeatures,troutput)
@@ -29,11 +29,11 @@ function test_ensembles()
   tstdirname = joinpath(dirname(pathof(TSML)),"../data/realdatatsclassification/testing")
 
   frange = 5:20
-  trdata = getstats(trdirname) |> DataFrame
+  trdata = getstats(trdirname) 
   trfeatures = trdata[:,frange] 
   troutput = trdata[:,:dtype] |> Vector{String}
 
-  tstdata = getstats(tstdirname) |> DataFrame
+  tstdata = getstats(tstdirname) 
   tstfeatures = tstdata[:,frange] 
   tstoutput = tstdata[:,:dtype] |> Vector{String}
 

test/test_monotonicer.jl

@@ -26,16 +26,16 @@ function test_basicmonotonicer()
   stfier = Statifier(Dict(:processmissing=>true))
   mono = Monotonicer(Dict())
 
-  mpipeline1 = @pipeline csvfilter |> valgator |> mono |> stfier
+  mpipeline1 = csvfilter |> valgator |> mono |> stfier
   respipe1 = fit_transform!(mpipeline1)
 
-  mpipeline2 = @pipeline csvfilter |> valgator |> stfier
+  mpipeline2 = csvfilter |> valgator |> stfier
   respipe2 = fit_transform!(mpipeline2)
   @test (respipe1[1:1,:] .== respipe2[1:1,:]) |> Matrix |> sum  == ncol(respipe1)
 
-  mpipeline3 = @pipeline csvfilter |> valgator |> valnner |> mono |> stfier
+  mpipeline3 = csvfilter |> valgator |> valnner |> mono |> stfier
   respipe3 = fit_transform!(mpipeline3)
-  mpipeline5 = @pipeline csvfilter |> valgator |> valnner |> stfier
+  mpipeline5 = csvfilter |> valgator |> valnner |> stfier
   fit!(mpipeline5)
   respipe5 = transform!(mpipeline5)
   val1 = respipe3[1,3:end] |> Vector
@@ -62,13 +62,13 @@ function test_typesmonotonicer()
   stfier = Statifier(Dict(:processmissing=>true))
   mono = Monotonicer(Dict())
 
-  regpipeline = @pipeline regularfilecsv |> valgator |> valnner |> mono
+  regpipeline = regularfilecsv |> valgator |> valnner |> mono
   regulardf=fit_transform!(regpipeline)
 
-  monopipeline = @pipeline monofilecsv |> valgator |> valnner |> mono
+  monopipeline = monofilecsv |> valgator |> valnner |> mono
   monodf=fit_transform!(monopipeline)
 
-  dailymonopipeline = @pipeline dailymonofilecsv |> valgator |> valnner |> mono
+  dailymonopipeline = dailymonofilecsv |> valgator |> valnner |> mono
   dailymonodf=fit_transform!(dailymonopipeline)
 
   @test round(dailyflips(regulardf),digits=2) == 11.98

test/test_outliernicer.jl

@@ -33,15 +33,15 @@ function test_basicoutlier()
   mono = Monotonicer(Dict())
   outliernicer = Outliernicer(Dict(:dateinterval=>Dates.Hour(1)))
 
-  mpipeline1 = @pipeline csvfilter |> valgator |> mono |> valnner |> outliernicer |> stfier
+  mpipeline1 = csvfilter |> valgator |> mono |> valnner |> outliernicer |> stfier
   respipe1 = fit_transform!(mpipeline1)
   @test round(sum(respipe1[1,3:20])) == -206651.0
 
-  mpipeline2 = @pipeline csvfilter |> valgator |> mono |> outliernicer |> stfier
+  mpipeline2 = csvfilter |> valgator |> mono |> outliernicer |> stfier
   respipe2 = fit_transform!(mpipeline2)
   @test round(sum(respipe2[1,3:20])) == -213860.0
 
-  mpipeline3 = @pipeline csvfilter |> valgator |> valnner |> mono |> outliernicer |> stfier
+  mpipeline3 = csvfilter |> valgator |> valnner |> mono |> outliernicer |> stfier
   respipe3 = fit_transform!(mpipeline3)
   @test round(sum(respipe3[1,3:20])) == -206651.0  
 end
@@ -63,17 +63,17 @@ function test_typesoutliernicer()
   mono = Monotonicer(Dict())
   outliernicer = Outliernicer(Dict(:dateinterval=>Dates.Hour(1)))
 
-  regpipeline = @pipeline regularfilecsv |> valgator |> valnner |> mono |> outliernicer |> stfier
+  regpipeline = regularfilecsv |> valgator |> valnner |> mono |> outliernicer |> stfier
   fit!(regpipeline)
   regulardf=transform!(regpipeline)
   @test round(sum(regulardf[1,3:20])) == -60438.0 
 
-  monopipeline = @pipeline monofilecsv |> valgator |> valnner |> mono |> outliernicer |> stfier
+  monopipeline = monofilecsv |> valgator |> valnner |> mono |> outliernicer |> stfier
   fit!(monopipeline)
   monodf=transform!(monopipeline)
   @test round(sum(monodf[1,3:20])) == -884621.0
 
-  dailymonopipeline = @pipeline dailymonofilecsv |> valgator |> valnner |> mono |> outliernicer |> stfier
+  dailymonopipeline = dailymonofilecsv |> valgator |> valnner |> mono |> outliernicer |> stfier
   fit!(dailymonopipeline)
   dailymonodf=transform!(dailymonopipeline)
   @test round(sum(dailymonodf[1,3:20])) == -294446.0 

test/test_plotter.jl

@@ -45,7 +45,7 @@ function test_realdataplotter()
   fname = joinpath(dirname(pathof(TSML)),"../data/testdata.csv")
   csvfilter = CSVDateValReader(Dict(:filename=>fname,:dateformat=>"dd/mm/yyyy HH:MM"))
   pltr = TSML.Plotter(Dict(:interactive => false))
-  mpipeline1 = @pipeline csvfilter |> pltr
+  mpipeline1 = csvfilter |> pltr
   myplot = fit_transform!(mpipeline1);
   @test isa(myplot,Plots.Plot) == true;
 end

test/test_statifier.jl

@@ -28,32 +28,32 @@ function test_statifier()
   valnner = DateValNNer(Dict(:dateinterval=>Dates.Hour(1)))
 
   stfier = Statifier(Dict(:processmissing=>true))
-  mpipeline0 = @pipeline csvfilter |> stfier
+  mpipeline0 = csvfilter |> stfier
   respipe0 = fit_transform!(mpipeline0)
   @test round(respipe0[1,:sfreq],digits=2) ==  0.18
   vals = respipe0[1,3:end] |> Vector
   @test (vals[(!isnan).(vals)] |> sum |> x->round(x,sigdigits=2)) == -1.3e6
 
   stfier = Statifier(Dict(:processmissing=>true))
-  mpipeline1 = @pipeline csvfilter |> valgator |> stfier
+  mpipeline1 = csvfilter |> valgator |> stfier
   fit!(mpipeline1)
   respipe1 = transform!(mpipeline1)
   @test respipe1[1,3:end] |> Vector |> sum |> x->round(x,digits=5) == -102779.88226
 
-  mpipeline2 = @pipeline csvfilter |> valgator |> valnner |> stfier
+  mpipeline2 = csvfilter |> valgator |> valnner |> stfier
   fit!(mpipeline2)
   respipe2 = transform!(mpipeline2) 
   res2 = respipe2[1,3:end] |> Vector
   @test (res2[(!isnan).(res2)] |> sum |> x->round(x,sigdigits=2)) == -230000.00
 
   stfier = Statifier(Dict(:processmissing=>false))
-  mpipeline1 = @pipeline csvfilter |> valgator |> stfier
+  mpipeline1 = csvfilter |> valgator |> stfier
   respipe1 = fit_transform!(mpipeline1)
   res1 = respipe1[1,3:end] |> Vector
   @test res1 |> sum |> x->round(x,digits=5) == -105185.44114
 
   stfier = Statifier(Dict(:processmissing=>true))
-  mpipeline2 = @pipeline csvfilter |> valgator |> valnner |> stfier
+  mpipeline2 = csvfilter |> valgator |> valnner |> stfier
   respipe2 = fit_transform!(mpipeline2) 
   res2 = respipe2[1,3:end] |> Vector
   @test res2[(!isnan).(res2)] |> sum |> x->round(x,digits=5) == -227824.85354

test/test_timescaledb.jl

@@ -5,7 +5,7 @@ using TSML
 
 function test_timescaledb()
   tsdb = TimescaleDB()
-  mpipeline = @pipeline tsdb
+  mpipeline = tsdb
   try 
     respipe = fit_transform!(mpipeline)
     @test sum(respipe.Value) ≈ 560997.0

test/test_valdate.jl

@@ -218,7 +218,7 @@ function test_csvreaderwriter()
   csvwtr = CSVDateValWriter(Dict(:filename=>outputfile,:dateformat=>"d/m/y H:M"))
   filter1 = DateValgator()
   filter2 = DateValNNer(Dict(:nnsize=>1))
-  mypipeline = @pipeline csvreader |> filter1 |> filter2
+  mypipeline = csvreader |> filter1 |> filter2
   res=fit_transform!(mypipeline)
   @test nrow(res) == 8761
   @test ncol(res) == 2
@@ -231,7 +231,7 @@ function test_csvreaderwriter()
   @test ncol(res2) == 2
   @test sum(ismissing.(res2.Value)) == 0
   @test floor(sum(res2.Value)) == 97564.0
-  mypipeline = @pipeline csvreader |> filter1 |> filter2 |> csvwtr
+  mypipeline = csvreader |> filter1 |> filter2 |> csvwtr
   res=fit_transform!(mypipeline)
   @test nrow(res2) == 8761
   @test ncol(res2) == 2
@@ -249,7 +249,7 @@ end
 #  csvreader = BzCSVDateValReader(Dict(:filename=>inputfile,:dateformat=>"d/m/y H:M"))
 #  filter1 = DateValgator()
 #  filter2 = DateValNNer(Dict(:nnsize=>1))
-#  mypipeline = @pipeline csvreader |> filter1 |> filter2
+#  mypipeline = csvreader |> filter1 |> filter2
 #  res=fit_transform!(mypipeline)
 #  @test nrow(res) == 8761
 #  @test ncol(res) == 2
@@ -267,7 +267,7 @@ function test_statoutputwriter()
   filter1 = DateValgator(Dict(:nnsize=>1))
   filter2 = DateValNNer(Dict(:nnsize=>1))
 
-  mypipeline = @pipeline csvreader |> filter1  |> filter2 |> statfier  |> csvstatwtr
+  mypipeline = csvreader |> filter1  |> filter2 |> statfier  |> csvstatwtr
   res=fit_transform!(mypipeline)
 
   @test nrow(res) == 1