From cf22bbc0f2422050139a02050a444029a3fac1df Mon Sep 17 00:00:00 2001 From: odow Date: Tue, 2 Mar 2021 20:25:26 +1300 Subject: [PATCH 1/7] Fix complexity of getindex(::ScalarFunctionIterator{VAF} --- src/Utilities/functions.jl | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/Utilities/functions.jl b/src/Utilities/functions.jl index bebb872752..eac4bce7b3 100644 --- a/src/Utilities/functions.jl +++ b/src/Utilities/functions.jl @@ -372,22 +372,19 @@ function Base.getindex( ) return MOI.VectorOfVariables(it.f.variables[I]) end + function Base.getindex( it::ScalarFunctionIterator{VAF{T}}, I::AbstractVector, ) where {T} - terms = MOI.VectorAffineTerm{T}[] - # assume at least one term per index - sizehint!(terms, length(I)) - constant = it.f.constants[I] - for term in it.f.terms - idx = findfirst(Base.Fix1(==, term.output_index), I) - if idx !== nothing - push!(terms, MOI.VectorAffineTerm(idx, term.scalar_term)) - end - end - return VAF(terms, constant) + return VAF( + MOI.VectorAffineTerm{T}[ + term for term in it.f.terms if term.output_index in I + ], + it.f.constants[I], + ) end + function Base.getindex( it::ScalarFunctionIterator{VQF{T}}, I::AbstractVector, From 40e967edf6a1ff0b93292603590221d681a496ac Mon Sep 17 00:00:00 2001 From: odow Date: Tue, 2 Mar 2021 21:38:26 +1300 Subject: [PATCH 2/7] Fix implementation --- src/Utilities/functions.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Utilities/functions.jl b/src/Utilities/functions.jl index eac4bce7b3..66b9f4de9e 100644 --- a/src/Utilities/functions.jl +++ b/src/Utilities/functions.jl @@ -377,9 +377,11 @@ function Base.getindex( it::ScalarFunctionIterator{VAF{T}}, I::AbstractVector, ) where {T} + d = Dict(I[i] => i for i = 1:length(I)) return VAF( MOI.VectorAffineTerm{T}[ - term for term in it.f.terms if term.output_index in I + MOI.VectorAffineTerm(d[term.output_index], term.scalar_term) + for term in it.f.terms if haskey(d, term.output_index) ], it.f.constants[I], ) From 55b838b713049fa464b9aaecaa7d303174b64235 Mon Sep 17 00:00:00 2001 From: odow Date: Wed, 3 Mar 2021 08:56:37 +1300 Subject: [PATCH 3/7] Re-write getindex for ScalarFunctionIterator --- src/Utilities/functions.jl | 147 +++++++++++++++++++++++++------------ 1 file changed, 102 insertions(+), 45 deletions(-) diff --git a/src/Utilities/functions.jl b/src/Utilities/functions.jl index 66b9f4de9e..51c15b524d 100644 --- a/src/Utilities/functions.jl +++ b/src/Utilities/functions.jl @@ -320,9 +320,47 @@ function scalar_type(::Type{MOI.VectorQuadraticFunction{T}}) where {T} return MOI.ScalarQuadraticFunction{T} end +""" + ScalarFunctionIterator{F<:MOI.AbstractVectorFunction} + +A type that allows iterating over the scalar-functions that comprise an +`AbstractVectorFunction`. +""" struct ScalarFunctionIterator{F<:MOI.AbstractVectorFunction} f::F + # Dictionaries which map output indices to their terms. + affine::Dict{Int,Vector{Int}} + quadratic::Dict{Int,Vector{Int}} +end + +function ScalarFunctionIterator(f::MOI.VectorOfVariables) + return ScalarFunctionIterator( + f, + Dict{Int,Vector{Int}}(), + Dict{Int,Vector{Int}}(), + ) +end + +function ScalarFunctionIterator(f::MOI.VectorAffineFunction) + d = Dict(i => Int[] for i = 1:MOI.output_dimension(f)) + for (i, term) in enumerate(f.terms) + push!(d[term.output_index], i) + end + return ScalarFunctionIterator(f, d, Dict{Int,Vector{Int}}()) +end + +function ScalarFunctionIterator(f::MOI.VectorQuadraticFunction) + aff = Dict(i => Int[] for i = 1:MOI.output_dimension(f)) + quad = Dict(i => Int[] for i = 1:MOI.output_dimension(f)) + for (i, term) in enumerate(f.affine_terms) + push!(aff[term.output_index], i) + end + for (i, term) in enumerate(f.quadratic_terms) + push!(quad[term.output_index], i) + end + return ScalarFunctionIterator(f, aff, quad) end + eachscalar(f::MOI.AbstractVectorFunction) = ScalarFunctionIterator(f) eachscalar(f::AbstractVector) = f @@ -344,69 +382,88 @@ Base.lastindex(it::ScalarFunctionIterator) = length(it) # Define getindex for Vector functions +# VectorOfVariables + function Base.getindex( it::ScalarFunctionIterator{MOI.VectorOfVariables}, - i::Integer, + output_index::Integer, ) - return MOI.SingleVariable(it.f.variables[i]) + return MOI.SingleVariable(it.f.variables[output_index]) end -# Returns the scalar terms of output_index i -function scalar_terms_at_index( - terms::Vector{<:Union{MOI.VectorAffineTerm,MOI.VectorQuadraticTerm}}, - i::Int, + +function Base.getindex( + it::ScalarFunctionIterator{MOI.VectorOfVariables}, + output_indices::AbstractVector{<:Integer}, ) - return [term.scalar_term for term in terms if term.output_index == i] + return MOI.VectorOfVariables(it.f.variables[output_indices]) end -function Base.getindex(it::ScalarFunctionIterator{<:VAF}, i::Integer) - return SAF(scalar_terms_at_index(it.f.terms, i), it.f.constants[i]) -end -function Base.getindex(it::ScalarFunctionIterator{<:VQF}, i::Integer) - lin = scalar_terms_at_index(it.f.affine_terms, i) - quad = scalar_terms_at_index(it.f.quadratic_terms, i) - return SQF(lin, quad, it.f.constants[i]) + +# VectorAffineFunction + +function Base.getindex( + it::ScalarFunctionIterator{MOI.VectorAffineFunction{T}}, + output_index::Integer, +) where {T} + return MOI.ScalarAffineFunction{T}( + MOI.ScalarAffineTerm{T}[ + it.f.terms[i].scalar_term + for i in it.affine[output_index] + ], + it.f.constants[output_index], + ) end function Base.getindex( - it::ScalarFunctionIterator{MOI.VectorOfVariables}, - I::AbstractVector, -) - return MOI.VectorOfVariables(it.f.variables[I]) + it::ScalarFunctionIterator{MOI.VectorAffineFunction{T}}, + output_indices::AbstractVector{<:Integer}, +) where {T} + terms = MOI.VectorAffineTerm{T}[] + for (i, output_index) in enumerate(output_indices) + for j in it.affine[output_index] + push!(terms, MOI.VectorAffineTerm(i, it.f.terms[j].scalar_term)) + end + end + return MOI.VectorAffineFunction(terms, it.f.constants[output_indices]) end +# VectorQuadraticFunction + function Base.getindex( - it::ScalarFunctionIterator{VAF{T}}, - I::AbstractVector, -) where {T} - d = Dict(I[i] => i for i = 1:length(I)) - return VAF( - MOI.VectorAffineTerm{T}[ - MOI.VectorAffineTerm(d[term.output_index], term.scalar_term) - for term in it.f.terms if haskey(d, term.output_index) + it::ScalarFunctionIterator{MOI.VectorQuadraticFunction{T}}, + output_index::Integer, +) where {T} + return MOI.ScalarQuadraticFunction( + MOI.ScalarAffineTerm{T}[ + it.f.affine_terms[i].scalar_term for i in it.affine[output_index] ], - it.f.constants[I], + MOI.ScalarQuadraticTerm{T}[ + it.f.quadratic_terms[i].scalar_term for i in it.quadratic[output_index] + ], + it.f.constants[output_index], ) end function Base.getindex( - it::ScalarFunctionIterator{VQF{T}}, - I::AbstractVector, -) where {T} - affine_terms = MOI.VectorAffineTerm{T}[] - quadratic_terms = MOI.VectorQuadraticTerm{T}[] - constant = Vector{T}(undef, length(I)) - for (i, j) in enumerate(I) - g = it[j] - append!( - affine_terms, - map(t -> MOI.VectorAffineTerm(i, t), g.affine_terms), - ) - append!( - quadratic_terms, - map(t -> MOI.VectorQuadraticTerm(i, t), g.quadratic_terms), - ) - constant[i] = g.constant + it::ScalarFunctionIterator{MOI.VectorQuadraticFunction{T}}, + output_indices::AbstractVector{<:Integer}, +) where {T} + vat = MOI.VectorAffineTerm{T}[] + vqt = MOI.VectorQuadraticTerm{T}[] + for (i, output_index) in enumerate(output_indices) + for j in it.affine[output_index] + push!( + vat, + MOI.VectorAffineTerm(i, it.f.affine_terms[j].scalar_term), + ) + end + for j in it.quadratic[output_index] + push!( + vqt, + MOI.VectorQuadraticTerm(i, it.f.quadratic_terms[j].scalar_term), + ) + end end - return VQF(affine_terms, quadratic_terms, constant) + return MOI.VectorQuadraticFunction(vat, vqt, it.f.constants[output_indices]) end function zero_with_output_dimension(::Type{Vector{T}}, n::Integer) where {T} From 38c3dda4501bb8e14eb9c74c51076713b603b723 Mon Sep 17 00:00:00 2001 From: Oscar Dowson Date: Thu, 4 Mar 2021 06:21:24 +1300 Subject: [PATCH 4/7] Update functions.jl --- src/Utilities/functions.jl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Utilities/functions.jl b/src/Utilities/functions.jl index 51c15b524d..e4c1243714 100644 --- a/src/Utilities/functions.jl +++ b/src/Utilities/functions.jl @@ -329,29 +329,29 @@ A type that allows iterating over the scalar-functions that comprise an struct ScalarFunctionIterator{F<:MOI.AbstractVectorFunction} f::F # Dictionaries which map output indices to their terms. - affine::Dict{Int,Vector{Int}} - quadratic::Dict{Int,Vector{Int}} + affine::Vector{Vector{Int}} + quadratic:Vector{Vector{Int}} end function ScalarFunctionIterator(f::MOI.VectorOfVariables) return ScalarFunctionIterator( f, - Dict{Int,Vector{Int}}(), - Dict{Int,Vector{Int}}(), + Vector{Int}[], + Vector{Int}[], ) end function ScalarFunctionIterator(f::MOI.VectorAffineFunction) - d = Dict(i => Int[] for i = 1:MOI.output_dimension(f)) + d = [Int[] for i = 1:MOI.output_dimension(f)] for (i, term) in enumerate(f.terms) push!(d[term.output_index], i) end - return ScalarFunctionIterator(f, d, Dict{Int,Vector{Int}}()) + return ScalarFunctionIterator(f, d, Vector{Int}[]) end function ScalarFunctionIterator(f::MOI.VectorQuadraticFunction) - aff = Dict(i => Int[] for i = 1:MOI.output_dimension(f)) - quad = Dict(i => Int[] for i = 1:MOI.output_dimension(f)) + aff = [Int[] for i = 1:MOI.output_dimension(f)] + quad = [Int[] for i = 1:MOI.output_dimension(f)] for (i, term) in enumerate(f.affine_terms) push!(aff[term.output_index], i) end From 2819163faa2f55c9c6ed8d09f5a099dd4fc446c5 Mon Sep 17 00:00:00 2001 From: Oscar Dowson Date: Thu, 4 Mar 2021 06:21:54 +1300 Subject: [PATCH 5/7] Update functions.jl --- src/Utilities/functions.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Utilities/functions.jl b/src/Utilities/functions.jl index e4c1243714..fa5e18aceb 100644 --- a/src/Utilities/functions.jl +++ b/src/Utilities/functions.jl @@ -330,7 +330,7 @@ struct ScalarFunctionIterator{F<:MOI.AbstractVectorFunction} f::F # Dictionaries which map output indices to their terms. affine::Vector{Vector{Int}} - quadratic:Vector{Vector{Int}} + quadratic::Vector{Vector{Int}} end function ScalarFunctionIterator(f::MOI.VectorOfVariables) From b82a5bc16724bafaac2328c616857447630cb1fa Mon Sep 17 00:00:00 2001 From: Oscar Dowson Date: Thu, 4 Mar 2021 09:55:15 +1300 Subject: [PATCH 6/7] [skip ci] Fix typo in comment --- src/Utilities/functions.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Utilities/functions.jl b/src/Utilities/functions.jl index fa5e18aceb..7d3d475708 100644 --- a/src/Utilities/functions.jl +++ b/src/Utilities/functions.jl @@ -328,7 +328,7 @@ A type that allows iterating over the scalar-functions that comprise an """ struct ScalarFunctionIterator{F<:MOI.AbstractVectorFunction} f::F - # Dictionaries which map output indices to their terms. + # Vectors which map output indices to their terms. affine::Vector{Vector{Int}} quadratic::Vector{Vector{Int}} end From ef08ebd3d5b84f801c88c34207c60093fd5d72ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Legat?= Date: Thu, 4 Mar 2021 20:14:25 +0100 Subject: [PATCH 7/7] Cached getindex for scalar iterator (#1263) --- src/Utilities/functions.jl | 92 ++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 29 deletions(-) diff --git a/src/Utilities/functions.jl b/src/Utilities/functions.jl index 7d3d475708..8841388dc1 100644 --- a/src/Utilities/functions.jl +++ b/src/Utilities/functions.jl @@ -326,39 +326,73 @@ end A type that allows iterating over the scalar-functions that comprise an `AbstractVectorFunction`. """ -struct ScalarFunctionIterator{F<:MOI.AbstractVectorFunction} +struct ScalarFunctionIterator{F<:MOI.AbstractVectorFunction, C} f::F - # Vectors which map output indices to their terms. - affine::Vector{Vector{Int}} - quadratic::Vector{Vector{Int}} + # Cache that can be used to store a precomputed datastructure that allows + # an efficient implementation of `getindex`. + cache::C end - -function ScalarFunctionIterator(f::MOI.VectorOfVariables) +function ScalarFunctionIterator(func::MOI.AbstractVectorFunction) return ScalarFunctionIterator( - f, - Vector{Int}[], - Vector{Int}[], + func, + scalar_iterator_cache(func), ) end -function ScalarFunctionIterator(f::MOI.VectorAffineFunction) - d = [Int[] for i = 1:MOI.output_dimension(f)] - for (i, term) in enumerate(f.terms) - push!(d[term.output_index], i) +scalar_iterator_cache(func::MOI.AbstractVectorFunction) = nothing + +function output_index_iterator(terms::AbstractVector, output_dimension) + start = zeros(Int, output_dimension) + next = Vector{Int}(undef, length(terms)) + last = zeros(Int, output_dimension) + for i in eachindex(terms) + j = terms[i].output_index + if iszero(last[j]) + start[j] = i + else + next[last[j]] = i + end + last[j] = i end - return ScalarFunctionIterator(f, d, Vector{Int}[]) + for j in eachindex(last) + if !iszero(last[j]) + next[last[j]] = 0 + end + end + return ChainedIterator(start, next) +end +struct ChainedIterator + start::Vector{Int} + next::Vector{Int} +end +struct ChainedIteratorAtIndex + start::Int + next::Vector{Int} +end +function ChainedIteratorAtIndex(it::ChainedIterator, index::Int) + return ChainedIteratorAtIndex(it.start[index], it.next) +end +#TODO We could also precompute the length for each `output_index`, +# check that it's a win. +Base.IteratorSize(::ChainedIteratorAtIndex) = Base.SizeUnknown() +function Base.iterate(it::ChainedIteratorAtIndex, i = it.start) + if iszero(i) + return nothing + else + return i, it.next[i] + end +end + +function ScalarFunctionIterator(f::MOI.VectorAffineFunction) + return ScalarFunctionIterator(f, output_index_iterator(f.terms, MOI.output_dimension(f))) end function ScalarFunctionIterator(f::MOI.VectorQuadraticFunction) - aff = [Int[] for i = 1:MOI.output_dimension(f)] - quad = [Int[] for i = 1:MOI.output_dimension(f)] - for (i, term) in enumerate(f.affine_terms) - push!(aff[term.output_index], i) - end - for (i, term) in enumerate(f.quadratic_terms) - push!(quad[term.output_index], i) - end - return ScalarFunctionIterator(f, aff, quad) + return ScalarFunctionIterator( + f, + (output_index_iterator(f.affine_terms, MOI.output_dimension(f)), + output_index_iterator(f.quadratic_terms, MOI.output_dimension(f))), + ) end eachscalar(f::MOI.AbstractVectorFunction) = ScalarFunctionIterator(f) @@ -407,7 +441,7 @@ function Base.getindex( return MOI.ScalarAffineFunction{T}( MOI.ScalarAffineTerm{T}[ it.f.terms[i].scalar_term - for i in it.affine[output_index] + for i in ChainedIteratorAtIndex(it.cache, output_index) ], it.f.constants[output_index], ) @@ -419,7 +453,7 @@ function Base.getindex( ) where {T} terms = MOI.VectorAffineTerm{T}[] for (i, output_index) in enumerate(output_indices) - for j in it.affine[output_index] + for j in ChainedIteratorAtIndex(it.cache, output_index) push!(terms, MOI.VectorAffineTerm(i, it.f.terms[j].scalar_term)) end end @@ -434,10 +468,10 @@ function Base.getindex( ) where {T} return MOI.ScalarQuadraticFunction( MOI.ScalarAffineTerm{T}[ - it.f.affine_terms[i].scalar_term for i in it.affine[output_index] + it.f.affine_terms[i].scalar_term for i in ChainedIteratorAtIndex(it.cache[1], output_index) ], MOI.ScalarQuadraticTerm{T}[ - it.f.quadratic_terms[i].scalar_term for i in it.quadratic[output_index] + it.f.quadratic_terms[i].scalar_term for i in ChainedIteratorAtIndex(it.cache[2], output_index) ], it.f.constants[output_index], ) @@ -450,13 +484,13 @@ function Base.getindex( vat = MOI.VectorAffineTerm{T}[] vqt = MOI.VectorQuadraticTerm{T}[] for (i, output_index) in enumerate(output_indices) - for j in it.affine[output_index] + for j in ChainedIteratorAtIndex(it.cache[1], output_index) push!( vat, MOI.VectorAffineTerm(i, it.f.affine_terms[j].scalar_term), ) end - for j in it.quadratic[output_index] + for j in ChainedIteratorAtIndex(it.cache[2], output_index) push!( vqt, MOI.VectorQuadraticTerm(i, it.f.quadratic_terms[j].scalar_term),