From cf38a535b228e585cd815800e5e1ab83e9360917 Mon Sep 17 00:00:00 2001 From: Diogo Netto Date: Mon, 6 Jul 2020 14:45:09 -0400 Subject: [PATCH 1/6] non-mutating munge_data --- src/interpolation_utils.jl | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/src/interpolation_utils.jl b/src/interpolation_utils.jl index 45333235..bb098270 100644 --- a/src/interpolation_utils.jl +++ b/src/interpolation_utils.jl @@ -33,38 +33,16 @@ end # helper function for data manipulation function munge_data(u::AbstractVector, t::AbstractVector) - Tu = Base.nonmissingtype(eltype(u)) - Tt = Base.nonmissingtype(eltype(t)) - newu = Tu[] - newt = Tt[] + newu = DataFrames.dropmissing(u) + newt = DataFrames.dropmissing(t) @assert length(t) == length(u) - @inbounds for i in eachindex(t) - ui = u[i] - ti = t[i] - if !ismissing(ui) && !ismissing(ti) - push!(newu, ui) - push!(newt, ti) - end - end return newu, newt end function munge_data(U::StridedMatrix, t::AbstractVector) - TU = Base.nonmissingtype(eltype(U)) - Tt = Base.nonmissingtype(eltype(t)) - newUs = AbstractVector{TU}[] - newt = Tt[] + tmpU = [U[:,j] for j un length(t)] + newUs = DataFrames.dropmissing(tmpU) + newt = DataFrames.dropmissing(t) @assert length(t) == size(U,2) - @inbounds for (j, tj) in enumerate(t) - - vUj = view(U, :, j) - if ismissing(tj) || any(ismissing, vUj) - continue - end - - push!(newt, tj) - push!(newUs, vUj) - end - return hcat(newUs...), newt end From f289f8b56544087870a001b8cc475e524d20f248 Mon Sep 17 00:00:00 2001 From: Diogo Netto Date: Wed, 8 Jul 2020 07:29:34 -0400 Subject: [PATCH 2/6] no dataframes --- src/interpolation_utils.jl | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/interpolation_utils.jl b/src/interpolation_utils.jl index bb098270..faa33bc9 100644 --- a/src/interpolation_utils.jl +++ b/src/interpolation_utils.jl @@ -33,16 +33,23 @@ end # helper function for data manipulation function munge_data(u::AbstractVector, t::AbstractVector) - newu = DataFrames.dropmissing(u) - newt = DataFrames.dropmissing(t) + Tu = Base.nonmissingtype(eltype(u)) + Tt = Base.nonmissingtype(eltype(t)) @assert length(t) == length(u) + non_missing_indices = collect(i for i in 1:kength(t) if !ismissing(ui) && !ismissing(ti)) + newu = Tu.([u[i] for i in non_missing_indices]) + newt = Tt.([t[i] for i in non_missing_indices]) + return newu, newt end function munge_data(U::StridedMatrix, t::AbstractVector) - tmpU = [U[:,j] for j un length(t)] - newUs = DataFrames.dropmissing(tmpU) - newt = DataFrames.dropmissing(t) + TU = Base.nonmissingtype(eltype(U)) + Tt = Base.nonmissingtype(eltype(t)) @assert length(t) == size(U,2) + non_missing_indices = collect(i for i in 1:kength(t) if !ismissing(U[:,i]) && !ismissing(t[i])) + newUs = TU.([U[:,i] for i in non_missing_indices]) + newt= Tt.([t[i] for i in non_missing_indices]) + return hcat(newUs...), newt end From 7d068e25cf24f6ad5faea80497f6b125d3a956d1 Mon Sep 17 00:00:00 2001 From: Diogo Netto Date: Wed, 8 Jul 2020 07:30:23 -0400 Subject: [PATCH 3/6] typo --- src/interpolation_utils.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/interpolation_utils.jl b/src/interpolation_utils.jl index faa33bc9..fc87302a 100644 --- a/src/interpolation_utils.jl +++ b/src/interpolation_utils.jl @@ -36,10 +36,10 @@ function munge_data(u::AbstractVector, t::AbstractVector) Tu = Base.nonmissingtype(eltype(u)) Tt = Base.nonmissingtype(eltype(t)) @assert length(t) == length(u) - non_missing_indices = collect(i for i in 1:kength(t) if !ismissing(ui) && !ismissing(ti)) + non_missing_indices = collect(i for i in 1:kength(t) if !ismissing(u[i]) && !ismissing(t[i])) newu = Tu.([u[i] for i in non_missing_indices]) newt = Tt.([t[i] for i in non_missing_indices]) - + return newu, newt end From c96e13731b11179f457175bc1d041a1558d8e707 Mon Sep 17 00:00:00 2001 From: Diogo Netto Date: Wed, 8 Jul 2020 07:35:18 -0400 Subject: [PATCH 4/6] typo --- src/interpolation_utils.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/interpolation_utils.jl b/src/interpolation_utils.jl index fc87302a..20d935e3 100644 --- a/src/interpolation_utils.jl +++ b/src/interpolation_utils.jl @@ -36,7 +36,7 @@ function munge_data(u::AbstractVector, t::AbstractVector) Tu = Base.nonmissingtype(eltype(u)) Tt = Base.nonmissingtype(eltype(t)) @assert length(t) == length(u) - non_missing_indices = collect(i for i in 1:kength(t) if !ismissing(u[i]) && !ismissing(t[i])) + non_missing_indices = collect(i for i in 1:length(t) if !ismissing(u[i]) && !ismissing(t[i])) newu = Tu.([u[i] for i in non_missing_indices]) newt = Tt.([t[i] for i in non_missing_indices]) @@ -47,7 +47,7 @@ function munge_data(U::StridedMatrix, t::AbstractVector) TU = Base.nonmissingtype(eltype(U)) Tt = Base.nonmissingtype(eltype(t)) @assert length(t) == size(U,2) - non_missing_indices = collect(i for i in 1:kength(t) if !ismissing(U[:,i]) && !ismissing(t[i])) + non_missing_indices = collect(i for i in 1:length(t) if !ismissing(U[:,i]) && !ismissing(t[i])) newUs = TU.([U[:,i] for i in non_missing_indices]) newt= Tt.([t[i] for i in non_missing_indices]) From c7897875106d49489248d85f90f174a502f5e6c8 Mon Sep 17 00:00:00 2001 From: Diogo Netto Date: Wed, 8 Jul 2020 07:43:11 -0400 Subject: [PATCH 5/6] other typo --- src/interpolation_utils.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interpolation_utils.jl b/src/interpolation_utils.jl index 20d935e3..1cb9b9d1 100644 --- a/src/interpolation_utils.jl +++ b/src/interpolation_utils.jl @@ -48,7 +48,7 @@ function munge_data(U::StridedMatrix, t::AbstractVector) Tt = Base.nonmissingtype(eltype(t)) @assert length(t) == size(U,2) non_missing_indices = collect(i for i in 1:length(t) if !ismissing(U[:,i]) && !ismissing(t[i])) - newUs = TU.([U[:,i] for i in non_missing_indices]) + newUs = [TU.(U[:,i]) for i in non_missing_indices] newt= Tt.([t[i] for i in non_missing_indices]) return hcat(newUs...), newt From 9182d37d6b3550aa3fd2d14c989e743bcbaf3293 Mon Sep 17 00:00:00 2001 From: Diogo Netto Date: Wed, 8 Jul 2020 09:58:55 -0400 Subject: [PATCH 6/6] new check for missing --- src/interpolation_utils.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interpolation_utils.jl b/src/interpolation_utils.jl index 1cb9b9d1..4258a980 100644 --- a/src/interpolation_utils.jl +++ b/src/interpolation_utils.jl @@ -47,7 +47,7 @@ function munge_data(U::StridedMatrix, t::AbstractVector) TU = Base.nonmissingtype(eltype(U)) Tt = Base.nonmissingtype(eltype(t)) @assert length(t) == size(U,2) - non_missing_indices = collect(i for i in 1:length(t) if !ismissing(U[:,i]) && !ismissing(t[i])) + non_missing_indices = collect(i for i in 1:length(t) if !any(ismissing,U[:,i]) && !ismissing(t[i])) newUs = [TU.(U[:,i]) for i in non_missing_indices] newt= Tt.([t[i] for i in non_missing_indices])