From 9c1ca427551ad728c661926a6811dd018fb61edb Mon Sep 17 00:00:00 2001 From: odow Date: Sat, 27 Jun 2020 23:04:40 -0500 Subject: [PATCH 1/7] Improve efficiency of MPS file reader. - Function barriers for type-unstable hot loops - Deterministic ordering of rows/columns (Closes #1106) - Remove TempRow and TempColumn in favor of vectors - Use Int instead of String in coef matrix for lower memory usage --- src/FileFormats/MPS/MPS.jl | 558 ++++++++++++++++++++++--------------- 1 file changed, 332 insertions(+), 226 deletions(-) diff --git a/src/FileFormats/MPS/MPS.jl b/src/FileFormats/MPS/MPS.jl index 7cf800fd2e..d36e13d97f 100644 --- a/src/FileFormats/MPS/MPS.jl +++ b/src/FileFormats/MPS/MPS.jl @@ -32,7 +32,7 @@ Keyword arguments are: - `warn::Bool=false`: print a warning when variables or constraints are renamed. """ function Model(; - warn::Bool = false + warn::Bool = false ) model = Model{Float64}() model.ext[:MPS_OPTIONS] = Options(warn) @@ -94,18 +94,26 @@ const LINEAR_CONSTRAINTS = ( (MOI.Interval{Float64}, 'L') # See the note in the RANGES section. ) +function _write_rows(io, model, set_type, sense_char) + for index in MOI.get( + model, + MOI.ListOfConstraintIndices{ + MOI.ScalarAffineFunction{Float64}, set_type + }() + ) + row_name = MOI.get(model, MOI.ConstraintName(), index) + if row_name == "" + error("Row name is empty: $(index).") + end + println(io, " ", sense_char, " ", row_name) + end + return +end + function write_rows(io::IO, model::Model) println(io, "ROWS\n N OBJ") for (set_type, sense_char) in LINEAR_CONSTRAINTS - for index in MOI.get(model, MOI.ListOfConstraintIndices{ - MOI.ScalarAffineFunction{Float64}, - set_type}()) - row_name = MOI.get(model, MOI.ConstraintName(), index) - if row_name == "" - error("Row name is empty: $(index).") - end - println(io, " ", sense_char, " ", row_name) - end + _write_rows(io, model, set_type, sense_char) end return end @@ -114,15 +122,22 @@ end # COLUMNS # ============================================================================== +function _list_of_integer_variables(model, integer_variables, set_type) + for index in MOI.get( + model, + MOI.ListOfConstraintIndices{MOI.SingleVariable, set_type}() + ) + v_index = MOI.get(model, MOI.ConstraintFunction(), index) + v_name = MOI.get(model, MOI.VariableName(), v_index.variable) + push!(integer_variables, v_name) + end + return +end + function list_of_integer_variables(model::Model) integer_variables = Set{String}() for set_type in (MOI.ZeroOne, MOI.Integer) - for index in MOI.get(model, MOI.ListOfConstraintIndices{ - MOI.SingleVariable, set_type}()) - v_index = MOI.get(model, MOI.ConstraintFunction(), index) - v_name = MOI.get(model, MOI.VariableName(), v_index.variable) - push!(integer_variables, v_name) - end + _list_of_integer_variables(model, integer_variables, set_type) end return integer_variables end @@ -137,8 +152,12 @@ function add_coefficient(coefficients, variable_name, row_name, coefficient) end function extract_terms( - model::Model, coefficients, row_name::String, - func::MOI.ScalarAffineFunction, discovered_columns::Set{String}) + model::Model, + coefficients, + row_name::String, + func::MOI.ScalarAffineFunction, + discovered_columns::Set{String} +) for term in func.terms variable_name = MOI.get(model, MOI.VariableName(), term.variable_index) add_coefficient(coefficients, variable_name, row_name, term.coefficient) @@ -148,14 +167,34 @@ function extract_terms( end function extract_terms( - model::Model, coefficients, row_name::String, func::MOI.SingleVariable, - discovered_columns::Set{String}) + model::Model, + coefficients, + row_name::String, + func::MOI.SingleVariable, + discovered_columns::Set{String}, +) variable_name = MOI.get(model, MOI.VariableName(), func.variable) add_coefficient(coefficients, variable_name, row_name, 1.0) push!(discovered_columns, variable_name) return end +function _write_columns( + io, model, set_type, sense_char, coefficients, discovered_columns +) + for index in MOI.get( + model, + MOI.ListOfConstraintIndices{ + MOI.ScalarAffineFunction{Float64}, set_type + }() + ) + row_name = MOI.get(model, MOI.ConstraintName(), index) + func = MOI.get(model, MOI.ConstraintFunction(), index) + extract_terms( + model, coefficients, row_name, func, discovered_columns + ) + end +end function write_columns(io::IO, model::Model) # Many MPS readers (e.g., CPLEX and GAMS) will error if a variable (column) # appears in the BOUNDS section but did not appear in the COLUMNS section. @@ -168,14 +207,9 @@ function write_columns(io::IO, model::Model) println(io, "COLUMNS") coefficients = Dict{String, Vector{Tuple{String, Float64}}}() for (set_type, sense_char) in LINEAR_CONSTRAINTS - for index in MOI.get(model, MOI.ListOfConstraintIndices{ - MOI.ScalarAffineFunction{Float64}, - set_type}()) - row_name = MOI.get(model, MOI.ConstraintName(), index) - func = MOI.get(model, MOI.ConstraintFunction(), index) - extract_terms( - model, coefficients, row_name, func, discovered_columns) - end + _write_columns( + io, model, set_type, sense_char, coefficients, discovered_columns + ) end obj_func_type = MOI.get(model, MOI.ObjectiveFunctionType()) obj_func = MOI.get(model, MOI.ObjectiveFunction{obj_func_type}()) @@ -217,18 +251,25 @@ value(set::MOI.GreaterThan) = set.lower value(set::MOI.EqualTo) = set.value value(set::MOI.Interval) = set.upper # See the note in the RANGES section. +function _write_rhs(io, model, set_type, sense_char) + for index in MOI.get( + model, + MOI.ListOfConstraintIndices{ + MOI.ScalarAffineFunction{Float64}, set_type + }() + ) + row_name = MOI.get(model, MOI.ConstraintName(), index) + print(io, " rhs ", rpad(row_name, 8), " ") + set = MOI.get(model, MOI.ConstraintSet(), index) + Base.Grisu.print_shortest(io, value(set)) + println(io) + end +end + function write_rhs(io::IO, model::Model) println(io, "RHS") for (set_type, sense_char) in LINEAR_CONSTRAINTS - for index in MOI.get(model, MOI.ListOfConstraintIndices{ - MOI.ScalarAffineFunction{Float64}, - set_type}()) - row_name = MOI.get(model, MOI.ConstraintName(), index) - print(io, " rhs ", rpad(row_name, 8), " ") - set = MOI.get(model, MOI.ConstraintSet(), index) - Base.Grisu.print_shortest(io, value(set)) - println(io) - end + _write_rhs(io, model, set_type, sense_char) end return end @@ -252,9 +293,12 @@ end function write_ranges(io::IO, model::Model) println(io, "RANGES") - for index in MOI.get(model, MOI.ListOfConstraintIndices{ - MOI.ScalarAffineFunction{Float64}, - MOI.Interval{Float64}}()) + for index in MOI.get( + model, + MOI.ListOfConstraintIndices{ + MOI.ScalarAffineFunction{Float64}, MOI.Interval{Float64} + }() + ) row_name = MOI.get(model, MOI.ConstraintName(), index) print(io, " rhs ", rpad(row_name, 8), " ") set = MOI.get(model, MOI.ConstraintSet(), index)::MOI.Interval{Float64} @@ -325,21 +369,28 @@ function update_bounds(::Tuple{Float64, Float64}, set::MOI.EqualTo) return (set.value, set.value) end +function _collect_bounds(bounds, model, set_type) + for index in MOI.get( + model, + MOI.ListOfConstraintIndices{MOI.SingleVariable, set_type}() + ) + func = MOI.get(model, MOI.ConstraintFunction(), index) + variable_index = func.variable::MOI.VariableIndex + if !haskey(bounds, variable_index) + bounds[variable_index] = (-Inf, Inf) + end + set = MOI.get(model, MOI.ConstraintSet(), index)::set_type + bounds[variable_index] = update_bounds(bounds[variable_index], set) + end + return +end + function write_bounds(io::IO, model::Model, discovered_columns::Set{String}) println(io, "BOUNDS") free_variables = Set(MOI.get(model, MOI.ListOfVariableIndices())) bounds = Dict{MOI.VariableIndex, Tuple{Float64, Float64}}() - for (set_type, sense_char) in LINEAR_CONSTRAINTS - for index in MOI.get(model, MOI.ListOfConstraintIndices{ - MOI.SingleVariable, set_type}()) - func = MOI.get(model, MOI.ConstraintFunction(), index) - variable_index = func.variable::MOI.VariableIndex - if !haskey(bounds, variable_index) - bounds[variable_index] = (-Inf, Inf) - end - set = MOI.get(model, MOI.ConstraintSet(), index)::set_type - bounds[variable_index] = update_bounds(bounds[variable_index], set) - end + for (set_type, _) in LINEAR_CONSTRAINTS + _collect_bounds(bounds, model, set_type) end for (index, (lower, upper)) in bounds var_name = MOI.get(model, MOI.VariableName(), index) @@ -351,8 +402,10 @@ function write_bounds(io::IO, model::Model, discovered_columns::Set{String}) end pop!(free_variables, index) end - for index in MOI.get(model, MOI.ListOfConstraintIndices{ - MOI.SingleVariable, MOI.ZeroOne}()) + for index in MOI.get( + model, + MOI.ListOfConstraintIndices{MOI.SingleVariable, MOI.ZeroOne}() + ) func = MOI.get(model, MOI.ConstraintFunction(), index) variable_index = func.variable::MOI.VariableIndex var_name = MOI.get(model, MOI.VariableName(), variable_index) @@ -397,10 +450,14 @@ function write_sos_constraint(io::IO, model::Model, index) end function write_sos(io::IO, model::Model) - sos1_indices = MOI.get(model, MOI.ListOfConstraintIndices{ - MOI.VectorOfVariables, MOI.SOS1{Float64}}()) - sos2_indices = MOI.get(model, MOI.ListOfConstraintIndices{ - MOI.VectorOfVariables, MOI.SOS2{Float64}}()) + sos1_indices = MOI.get( + model, + MOI.ListOfConstraintIndices{MOI.VectorOfVariables, MOI.SOS1{Float64}}(), + ) + sos2_indices = MOI.get( + model, + MOI.ListOfConstraintIndices{MOI.VectorOfVariables, MOI.SOS2{Float64}}(), + ) if length(sos1_indices) + length(sos2_indices) > 0 println(io, "SOS") idx = 1 @@ -447,35 +504,48 @@ end # ENDATA # ============================================================================== -mutable struct TempColumn - lower::Float64 - upper::Float64 - is_int::Bool - TempColumn() = new(0.0, Inf, false) -end - -mutable struct TempRow - lower::Float64 - upper::Float64 - sense::String - terms::Dict{String, Float64} - TempRow() = new(-Inf, Inf, "E", Dict{String, Float64}()) -end - mutable struct TempMPSModel name::String obj_name::String - columns::Dict{String, TempColumn} - rows::Dict{String, TempRow} + c::Vector{Float64} + col_lower::Vector{Float64} + col_upper::Vector{Float64} + row_lower::Vector{Float64} + row_upper::Vector{Float64} + sense::Vector{String} + A::Vector{Vector{Tuple{Int, Float64}}} + is_int::Vector{Bool} + name_to_col::Dict{String, Int} + col_to_name::Vector{String} + name_to_row::Dict{String, Int} + row_to_name::Vector{String} intorg_flag::Bool # A flag used to parse COLUMNS section. - TempMPSModel() = new("", "", Dict{String, TempColumn}(), - Dict{String, TempRow}(), false) +end + +function TempMPSModel() + return TempMPSModel( + "", + "", + Float64[], # c + Float64[], # col_lower + Float64[], # col_upper + Float64[], # row_lower + Float64[], # row_upper + String[], # sense + Vector{Vector{Tuple{Int, Float64}}}[], # A + Bool[], + Dict{String, Int}(), + String[], + Dict{String, Int}(), + String[], + false, + ) end const HEADERS = ("ROWS", "COLUMNS", "RHS", "RANGES", "BOUNDS", "SOS", "ENDATA") """ - Base.read!(io::IO, model::FileFormats.CBF.Model) + Base.read!(io::IO, model::FileFormats.MPS.Model) Read `io` in the MPS file format and store the result in `model`. """ @@ -528,22 +598,22 @@ function bounds_to_set(lower, upper) elseif lower == upper return MOI.EqualTo(upper) end - return nothing + return end -function copy_to(model::Model, temp::TempMPSModel) - MOI.set(model, MOI.Name(), temp.name) +function copy_to(model::Model, data::TempMPSModel) + MOI.set(model, MOI.Name(), data.name) variable_map = Dict{String, MOI.VariableIndex}() # Add variables. - for (name, column) in temp.columns + for (i, name) in enumerate(data.col_to_name) x = MOI.add_variable(model) variable_map[name] = x MOI.set(model, MOI.VariableName(), x, name) - set = bounds_to_set(column.lower, column.upper) - if set === nothing && column.is_int + set = bounds_to_set(data.col_lower[i], data.col_upper[i]) + if set === nothing && data.is_int[i] # TODO: some solvers may interpret this as binary. MOI.add_constraint(model, MOI.SingleVariable(x), MOI.Integer()) - elseif set !== nothing && column.is_int + elseif set !== nothing && data.is_int[i] if set == MOI.Interval(0.0, 1.0) MOI.add_constraint(model, MOI.SingleVariable(x), MOI.ZeroOne()) else @@ -554,34 +624,38 @@ function copy_to(model::Model, temp::TempMPSModel) MOI.add_constraint(model, MOI.SingleVariable(x), set) end end + # Set objective. + MOI.set(model, MOI.ObjectiveSense(), MOI.MIN_SENSE) + MOI.set( + model, + MOI.ObjectiveFunction{MOI.ScalarAffineFunction{Float64}}(), + MOI.ScalarAffineFunction( + [ + MOI.ScalarAffineTerm(data.c[i], variable_map[v]) + for (i, v) in enumerate(data.col_to_name) if !iszero(data.c[i]) + ], + 0.0, + ) + ) # Add linear constraints. - for (c_name, row) in temp.rows - if c_name == temp.obj_name - # Set objective. - MOI.set(model, MOI.ObjectiveSense(), MOI.MIN_SENSE) - obj_func = if length(row.terms) == 1 && - first(row.terms).second == 1.0 - MOI.SingleVariable(variable_map[first(row.terms).first]) - else - MOI.ScalarAffineFunction([ - MOI.ScalarAffineTerm(coef, variable_map[v_name]) - for (v_name, coef) in row.terms], - 0.0) - end - MOI.set(model, MOI.ObjectiveFunction{typeof(obj_func)}(), obj_func) - else - constraint_function = MOI.ScalarAffineFunction([ - MOI.ScalarAffineTerm(coef, variable_map[v_name]) - for (v_name, coef) in row.terms], - 0.0) - set = bounds_to_set(row.lower, row.upper) - if set !== nothing - c = MOI.add_constraint(model, constraint_function, set) - MOI.set(model, MOI.ConstraintName(), c, c_name) - else - error("Expected a non-empty set for $(c_name). Got row=$(row)") - end - end + for (j, c_name) in enumerate(data.row_to_name) + set = bounds_to_set(data.row_lower[j], data.row_upper[j]) + if set === nothing + error("Expected a non-empty set for $(c_name). Got row=$(row)") + end + c = MOI.add_constraint( + model, + MOI.ScalarAffineFunction( + [ + MOI.ScalarAffineTerm( + coef, variable_map[data.col_to_name[i]] + ) for (i, coef) in data.A[j] + ], + 0.0 + ), + set, + ) + MOI.set(model, MOI.ConstraintName(), c, c_name) end return end @@ -611,30 +685,38 @@ function parse_rows_line(data::TempMPSModel, items::Vector{String}) error("Malformed ROWS line: $(join(items, " "))") end sense, name = items - if haskey(data.rows, name) + if haskey(data.name_to_row, name) error("Duplicate row encountered: $(line).") elseif sense != "N" && sense != "L" && sense != "G" && sense != "E" error("Invalid row sense: $(join(items, " "))") end - row = TempRow() - row.sense = sense if sense == "N" if data.obj_name != "" - # Detected a duplicate objective. Skip it. - return name + return name # Detected a duplicate objective. Skip it. end data.obj_name = name + return + end + if name == data.obj_name + error("Found row with same name as objective: $(line).") end # Add some default bounds for the constraints. + push!(data.row_to_name, name) + row = length(data.row_to_name) + data.name_to_row[name] = row + push!(data.sense, sense) + push!(data.A, Tuple{Int, Float64}[]) if sense == "G" - row.lower = 0.0 + push!(data.row_lower, 0.0) + push!(data.row_upper, Inf) + data.row_upper[row] = Inf elseif sense == "L" - row.upper = 0.0 + push!(data.row_lower, -Inf) + push!(data.row_upper, 0.0) elseif sense == "E" - row.lower = 0.0 - row.upper = 0.0 + push!(data.row_lower, 0.0) + push!(data.row_upper, 0.0) end - data.rows[name] = row return end @@ -642,58 +724,72 @@ end # COLUMNS # ============================================================================== -function parse_single_coefficient(data, row_name, column_name, value) - row = get(data.rows, row_name, nothing) +function parse_single_coefficient(data, row_name::String, column::Int, value) + if row_name == data.obj_name + data.c[column] += parse(Float64, value) + return + end + row = get(data.name_to_row, row_name, nothing) if row === nothing error("ROW name $(row_name) not recognised. Is it in the ROWS field?") end - if haskey(row.terms, column_name) - row.terms[column_name] += parse(Float64, value) - else - row.terms[column_name] = parse(Float64, value) + push!(data.A[row], (column, parse(Float64, value))) + return +end + +function _add_new_column(data, column_name) + if haskey(data.name_to_col, column_name) + return end + push!(data.col_to_name, column_name) + data.name_to_col[column_name] = length(data.col_to_name) + push!(data.c, 0.0) + push!(data.col_lower, 0.0) + push!(data.col_upper, Inf) + push!(data.is_int, false) return end -function parse_columns_line(data::TempMPSModel, items::Vector{String}, multi_objectives::Vector{String}) +function _set_intorg(data, column, column_name) + if data.is_int[column] && !data.intorg_flag + error( + "Variable $(column_name) appeared in COLUMNS outside an " * + "`INT` marker after already being declared as integer." + ) + end + data.is_int[column] = data.intorg_flag +end + +function parse_columns_line( + data::TempMPSModel, items::Vector{String}, multi_objectives::Vector{String} +) if length(items) == 3 # [column name] [row name] [value] column_name, row_name, value = items if uppercase(row_name) == "'MARKER'" && uppercase(value) == "'INTORG'" data.intorg_flag = true return - elseif uppercase(row_name) == "'MARKER'" && - uppercase(value) == "'INTEND'" + elseif uppercase(row_name) == "'MARKER'" && uppercase(value) == "'INTEND'" data.intorg_flag = false return elseif row_name in multi_objectives return end - if !haskey(data.columns, column_name) - data.columns[column_name] = TempColumn() - end - parse_single_coefficient(data, row_name, column_name, value) - if data.columns[column_name].is_int && !data.intorg_flag - error("Variable $(column_name) appeared in COLUMNS outside an" * - " `INT` marker after already being declared as integer.") - end - data.columns[column_name].is_int = data.intorg_flag + _add_new_column(data, column_name) + column = data.name_to_col[column_name] + parse_single_coefficient(data, row_name, column, value) + _set_intorg(data, column, column_name) elseif length(items) == 5 # [column name] [row name] [value] [row name 2] [value 2] column_name, row_name_1, value_1, row_name_2, value_2 = items - if !haskey(data.columns, column_name) - data.columns[column_name] = TempColumn() - end if row_name_1 in multi_objectives || row_name_2 in multi_objectives return end - parse_single_coefficient(data, row_name_1, column_name, value_1) - parse_single_coefficient(data, row_name_2, column_name, value_2) - if data.columns[column_name].is_int && !data.intorg_flag - error("Variable $(column_name) appeared in COLUMNS outside an" * - " `INT` marker after already being declared as integer.") - end - data.columns[column_name].is_int = data.intorg_flag + _add_new_column(data, column_name) + column = data.name_to_col[column_name] + parse_single_coefficient(data, row_name_1, column, value_1) + parse_single_coefficient(data, row_name_2, column, value_2) + _set_intorg(data, column, column_name) else error("Malformed COLUMNS line: $(join(items, " "))") end @@ -704,20 +800,22 @@ end # RHS # ============================================================================== -function parse_single_rhs(data, row_name, value, items::Vector{String}) - if !haskey(data.rows, row_name) +function parse_single_rhs( + data, row_name::String, value::Float64, items::Vector{String} +) + row = get(data.name_to_row, row_name, nothing) + if row === nothing error("ROW name $(row_name) not recognised. Is it in the ROWS field?") end - value = parse(Float64, value) - sense = data.rows[row_name].sense - if sense == "E" - data.rows[row_name].upper = value - data.rows[row_name].lower = value - elseif sense == "G" - data.rows[row_name].lower = value - elseif sense == "L" - data.rows[row_name].upper = value - elseif sense == "N" + if data.sense[row] == "E" + data.row_upper[row] = value + data.row_lower[row] = value + elseif data.sense[row] == "G" + data.row_lower[row] = value + elseif data.sense[row] == "L" + data.row_upper[row] = value + else + @assert data.sense[row] == "N" error("Cannot have RHS for objective: $(join(items, " "))") end return @@ -728,12 +826,12 @@ function parse_rhs_line(data::TempMPSModel, items::Vector{String}) if length(items) == 3 # [rhs name] [row name] [value] rhs_name, row_name, value = items - parse_single_rhs(data, row_name, value, items) + parse_single_rhs(data, row_name, parse(Float64, value), items) elseif length(items) == 5 # [rhs name] [row name 1] [value 1] [row name 2] [value 2] rhs_name, row_name_1, value_1, row_name_2, value_2 = items - parse_single_rhs(data, row_name_1, value_1, items) - parse_single_rhs(data, row_name_2, value_2, items) + parse_single_rhs(data, row_name_1, parse(Float64, value_1), items) + parse_single_rhs(data, row_name_2, parse(Float64, value_2), items) else error("Malformed RHS line: $(join(items, " "))") end @@ -755,20 +853,20 @@ end # ============================================================================== function parse_single_range(data, row_name, value) - if !haskey(data.rows, row_name) + row = get(data.name_to_row, row_name, nothing) + if row === nothing error("ROW name $(row_name) not recognised. Is it in the ROWS field?") end value = parse(Float64, value) - row = data.rows[row_name] - if row.sense == "G" - row.upper = row.lower + abs(value) - elseif row.sense == "L" - row.lower = row.upper - abs(value) - elseif row.sense == "E" + if data.sense[row] == "G" + data.row_upper[row] = data.row_lower[row] + abs(value) + elseif data.sense[row] == "L" + data.row_lower[row] = data.row_upper[row] - abs(value) + elseif data.sense[row] == "E" if value > 0.0 - row.upper = row.upper + value + data.row_upper[row] = data.row_lower[row] + value else - row.lower = row.lower + value + data.row_lower[row] = data.row_upper[row] + value end end return @@ -795,62 +893,70 @@ end # BOUNDS # ============================================================================== +function _parse_single_bound(data, column_name, bound_type) + col = get(data.name_to_col, column_name, nothing) + if col === nothing + error("Column name $(column_name) not found.") + end + if bound_type == "PL" + data.col_upper[col] = Inf + elseif bound_type == "MI" + data.col_lower[col] = -Inf + elseif bound_type == "FR" + data.col_lower[col] = -Inf + data.col_upper[col] = Inf + elseif bound_type == "BV" + data.col_lower[col] = 0.0 + data.col_upper[col] = 1.0 + data.is_int[col] = true + else + error("Invalid bound type $(bound_type): $(join(items, " "))") + end +end + +function _parse_single_bound(data, column_name, bound_type, value::Float64) + col = get(data.name_to_col, column_name, nothing) + if col === nothing + error("Column name $(column_name) not found.") + end + if bound_type == "FX" + data.col_lower[col] = value + data.col_upper[col] = value + elseif bound_type == "UP" + data.col_upper[col] = value + elseif bound_type == "LO" + data.col_lower[col] = value + elseif bound_type == "LI" + data.col_lower[col] = value + data.is_int[col] = true + elseif bound_type == "UI" + data.col_upper[col] = value + data.is_int[col] = true + elseif bound_type == "FR" + # So even though FR bounds should be of the form: + # FR BOUND1 VARNAME + # there are cases in MIPLIB2017 (e.g., leo1 and leo2) like so: + # FR BOUND1 C0000001 .000000 + # In these situations, just ignore the value. + data.col_lower[col] = -Inf + data.col_upper[col] = Inf + elseif bound_type == "BV" + # A similar situation happens with BV bounds in leo1 and leo2. + data.col_lower[col] = 0.0 + data.col_upper[col] = 1.0 + data.is_int[col] = true + else + error("Invalid bound type $(bound_type): $(join(items, " "))") + end +end + function parse_bounds_line(data::TempMPSModel, items::Vector{String}) if length(items) == 3 - bound_type, bound_name, column_name = items - if !haskey(data.columns, column_name) - error("Column name $(column_name) not found.") - end - column = data.columns[column_name] - if bound_type == "PL" - column.upper = Inf - elseif bound_type == "MI" - column.lower = -Inf - elseif bound_type == "FR" - column.lower = -Inf - column.upper = Inf - elseif bound_type == "BV" - column.lower = 0.0 - column.upper = 1.0 - column.is_int = true - else - error("Invalid bound type $(bound_type): $(join(items, " "))") - end + bound_type, _, column_name = items + _parse_single_bound(data, column_name, bound_type) elseif length(items) == 4 - bound_type, bound_name, column_name, value = items - if !haskey(data.columns, column_name) - error("Column name $(column_name) not found.") - end - column = data.columns[column_name] - value = parse(Float64, value) - if bound_type == "FX" - column.lower = column.upper = value - elseif bound_type == "UP" - column.upper = value - elseif bound_type == "LO" - column.lower = value - elseif bound_type == "LI" - column.lower = value - column.is_int = true - elseif bound_type == "UI" - column.upper = value - column.is_int = true - elseif bound_type == "FR" - # So even though FR bounds should be of the form: - # FR BOUND1 VARNAME - # there are cases in MIPLIB2017 (e.g., leo1 and leo2) like so: - # FR BOUND1 C0000001 .000000 - # In these situations, just ignore the value. - column.lower = -Inf - column.upper = Inf - elseif bound_type == "BV" - # A similar situation happens with BV bounds in leo1 and leo2. - column.lower = 0.0 - column.upper = 1.0 - column.is_int = true - else - error("Invalid bound type $(bound_type): $(join(items, " "))") - end + bound_type, _, column_name, value = items + _parse_single_bound(data, column_name, bound_type, parse(Float64, value)) else error("Malformed BOUNDS line: $(join(items, " "))") end From 4b143e0f262397c584620397a3200ac0af7177df Mon Sep 17 00:00:00 2001 From: odow Date: Mon, 29 Jun 2020 10:18:52 -0500 Subject: [PATCH 2/7] Allocation improvements. Move strings to enums --- src/FileFormats/MPS/MPS.jl | 243 ++++++++++++++++++++++++------------- 1 file changed, 161 insertions(+), 82 deletions(-) diff --git a/src/FileFormats/MPS/MPS.jl b/src/FileFormats/MPS/MPS.jl index d36e13d97f..9c4bb87829 100644 --- a/src/FileFormats/MPS/MPS.jl +++ b/src/FileFormats/MPS/MPS.jl @@ -504,6 +504,21 @@ end # ENDATA # ============================================================================== +@enum(Sense, SENSE_N, SENSE_G, SENSE_L, SENSE_E, SENSE_UNKNOWN) +function Sense(s::String) + if s == "G" + return SENSE_G + elseif s == "L" + return SENSE_L + elseif s == "E" + return SENSE_E + elseif s == "N" + return SENSE_N + else + return SENSE_UNKNOWN + end +end + mutable struct TempMPSModel name::String obj_name::String @@ -512,7 +527,7 @@ mutable struct TempMPSModel col_upper::Vector{Float64} row_lower::Vector{Float64} row_upper::Vector{Float64} - sense::Vector{String} + sense::Vector{Sense} A::Vector{Vector{Tuple{Int, Float64}}} is_int::Vector{Bool} name_to_col::Dict{String, Int} @@ -531,7 +546,7 @@ function TempMPSModel() Float64[], # col_upper Float64[], # row_lower Float64[], # row_upper - String[], # sense + Sense[], # sense Vector{Vector{Tuple{Int, Float64}}}[], # A Bool[], Dict{String, Int}(), @@ -542,7 +557,41 @@ function TempMPSModel() ) end -const HEADERS = ("ROWS", "COLUMNS", "RHS", "RANGES", "BOUNDS", "SOS", "ENDATA") +@enum( + Headers, + HEADER_NAME, + HEADER_ROWS, + HEADER_COLUMNS, + HEADER_RHS, + HEADER_RANGES, + HEADER_BOUNDS, + HEADER_SOS, + HEADER_ENDATA, + HEADER_UNKNOWN, +) + +function Headers(s::String) + s = uppercase(s) + if s == "ROWS" + return HEADER_ROWS + elseif s == "COLUMNS" + return HEADER_COLUMNS + elseif s == "RHS" + return HEADER_RHS + elseif s == "RANGES" + return HEADER_RANGES + elseif s == "BOUNDS" + return HEADER_BOUNDS + elseif s == "SOS" + return HEADER_SOS + elseif s == "ENDATA" + return HEADER_ENDATA + elseif s == "NAME" + return HEADER_NAME + else + return HEADER_UNKNOWN + end +end """ Base.read!(io::IO, model::FileFormats.MPS.Model) @@ -554,33 +603,35 @@ function Base.read!(io::IO, model::Model) error("Cannot read in file because model is not empty.") end data = TempMPSModel() - header = "NAME" + header = HEADER_NAME multi_objectives = String[] while !eof(io) && header != "ENDATA" - line = strip(readline(io)) + line = string(strip(readline(io))) if line == "" || startswith(line, "*") # Skip blank lines and comments. continue end - if uppercase(string(line)) in HEADERS - header = uppercase(string(line)) + h = Headers(line) + if h != HEADER_UNKNOWN + header = h continue end # TODO: split into hard fields based on column indices. items = String.(split(line, " ", keepempty = false)) - if header == "NAME" - # A special case. This only happens at the start. + if header == HEADER_NAME parse_name_line(data, items) - elseif header == "ROWS" + elseif header == HEADER_ROWS multi_obj = parse_rows_line(data, items) - multi_obj !== nothing && push!(multi_objectives, multi_obj) - elseif header == "COLUMNS" + if multi_obj !== nothing + push!(multi_objectives, multi_obj) + end + elseif header == HEADER_COLUMNS parse_columns_line(data, items, multi_objectives) - elseif header == "RHS" + elseif header == HEADER_RHS parse_rhs_line(data, items) - elseif header == "RANGES" + elseif header == HEADER_RANGES parse_ranges_line(data, items) - elseif header == "BOUNDS" + elseif header == HEADER_BOUNDS parse_bounds_line(data, items) end end @@ -598,7 +649,7 @@ function bounds_to_set(lower, upper) elseif lower == upper return MOI.EqualTo(upper) end - return + return # free variable end function copy_to(model::Model, data::TempMPSModel) @@ -606,25 +657,48 @@ function copy_to(model::Model, data::TempMPSModel) variable_map = Dict{String, MOI.VariableIndex}() # Add variables. for (i, name) in enumerate(data.col_to_name) - x = MOI.add_variable(model) - variable_map[name] = x - MOI.set(model, MOI.VariableName(), x, name) - set = bounds_to_set(data.col_lower[i], data.col_upper[i]) - if set === nothing && data.is_int[i] + _add_variable(model, data, variable_map, i, name) + end + # Set objective. + _add_objective(model, data, variable_map) + # Add linear constraints. + for (j, c_name) in enumerate(data.row_to_name) + set = bounds_to_set(data.row_lower[j], data.row_upper[j]) + if set === nothing + error("Expected a non-empty set for $(c_name).") + end + _add_linear_constraint(model, data, variable_map, j, c_name, set) + end + return +end + +function _add_variable(model, data, variable_map, i, name) + x = MOI.add_variable(model) + variable_map[name] = x + MOI.set(model, MOI.VariableName(), x, name) + set = bounds_to_set(data.col_lower[i], data.col_upper[i]) + if set === nothing + if data.is_int[i] # TODO: some solvers may interpret this as binary. MOI.add_constraint(model, MOI.SingleVariable(x), MOI.Integer()) - elseif set !== nothing && data.is_int[i] - if set == MOI.Interval(0.0, 1.0) - MOI.add_constraint(model, MOI.SingleVariable(x), MOI.ZeroOne()) - else - MOI.add_constraint(model, MOI.SingleVariable(x), MOI.Integer()) - MOI.add_constraint(model, MOI.SingleVariable(x), set) - end - elseif set !== nothing + else + # Free variable + end + elseif !data.is_int[i] + MOI.add_constraint(model, MOI.SingleVariable(x), set) + else + @assert data.is_int[i] + if set == MOI.Interval(0.0, 1.0) + MOI.add_constraint(model, MOI.SingleVariable(x), MOI.ZeroOne()) + else + MOI.add_constraint(model, MOI.SingleVariable(x), MOI.Integer()) MOI.add_constraint(model, MOI.SingleVariable(x), set) end end - # Set objective. + return +end + +function _add_objective(model, data, variable_map) MOI.set(model, MOI.ObjectiveSense(), MOI.MIN_SENSE) MOI.set( model, @@ -637,26 +711,23 @@ function copy_to(model::Model, data::TempMPSModel) 0.0, ) ) - # Add linear constraints. - for (j, c_name) in enumerate(data.row_to_name) - set = bounds_to_set(data.row_lower[j], data.row_upper[j]) - if set === nothing - error("Expected a non-empty set for $(c_name). Got row=$(row)") - end - c = MOI.add_constraint( - model, - MOI.ScalarAffineFunction( - [ - MOI.ScalarAffineTerm( - coef, variable_map[data.col_to_name[i]] - ) for (i, coef) in data.A[j] - ], - 0.0 - ), - set, - ) - MOI.set(model, MOI.ConstraintName(), c, c_name) - end + return +end + +function _add_linear_constraint(model, data, variable_map, j, c_name, set) + c = MOI.add_constraint( + model, + MOI.ScalarAffineFunction( + [ + MOI.ScalarAffineTerm( + coef, variable_map[data.col_to_name[i]] + ) for (i, coef) in data.A[j] + ], + 0.0 + ), + set, + ) + MOI.set(model, MOI.ConstraintName(), c, c_name) return end @@ -684,13 +755,13 @@ function parse_rows_line(data::TempMPSModel, items::Vector{String}) if length(items) != 2 error("Malformed ROWS line: $(join(items, " "))") end - sense, name = items + sense, name = Sense(items[1]), items[2] if haskey(data.name_to_row, name) - error("Duplicate row encountered: $(line).") - elseif sense != "N" && sense != "L" && sense != "G" && sense != "E" + error("Duplicate row encountered: $(join(items, " ")).") + elseif sense == SENSE_UNKNOWN error("Invalid row sense: $(join(items, " "))") end - if sense == "N" + if sense == SENSE_N if data.obj_name != "" return name # Detected a duplicate objective. Skip it. end @@ -698,7 +769,7 @@ function parse_rows_line(data::TempMPSModel, items::Vector{String}) return end if name == data.obj_name - error("Found row with same name as objective: $(line).") + error("Found row with same name as objective: $(join(items, " ")).") end # Add some default bounds for the constraints. push!(data.row_to_name, name) @@ -706,14 +777,15 @@ function parse_rows_line(data::TempMPSModel, items::Vector{String}) data.name_to_row[name] = row push!(data.sense, sense) push!(data.A, Tuple{Int, Float64}[]) - if sense == "G" + if sense == SENSE_G push!(data.row_lower, 0.0) push!(data.row_upper, Inf) data.row_upper[row] = Inf - elseif sense == "L" + elseif sense == SENSE_L push!(data.row_lower, -Inf) push!(data.row_upper, 0.0) - elseif sense == "E" + else + @assert sense == SENSE_E push!(data.row_lower, 0.0) push!(data.row_upper, 0.0) end @@ -724,16 +796,18 @@ end # COLUMNS # ============================================================================== -function parse_single_coefficient(data, row_name::String, column::Int, value) +function parse_single_coefficient( + data, row_name::String, column::Int, value::Float64 +) if row_name == data.obj_name - data.c[column] += parse(Float64, value) + data.c[column] += value return end row = get(data.name_to_row, row_name, nothing) if row === nothing error("ROW name $(row_name) not recognised. Is it in the ROWS field?") end - push!(data.A[row], (column, parse(Float64, value))) + push!(data.A[row], (column, value)) return end @@ -777,7 +851,7 @@ function parse_columns_line( end _add_new_column(data, column_name) column = data.name_to_col[column_name] - parse_single_coefficient(data, row_name, column, value) + parse_single_coefficient(data, row_name, column, parse(Float64, value)) _set_intorg(data, column, column_name) elseif length(items) == 5 # [column name] [row name] [value] [row name 2] [value 2] @@ -787,8 +861,12 @@ function parse_columns_line( end _add_new_column(data, column_name) column = data.name_to_col[column_name] - parse_single_coefficient(data, row_name_1, column, value_1) - parse_single_coefficient(data, row_name_2, column, value_2) + parse_single_coefficient( + data, row_name_1, column, parse(Float64, value_1) + ) + parse_single_coefficient( + data, row_name_2, column, parse(Float64, value_2) + ) _set_intorg(data, column, column_name) else error("Malformed COLUMNS line: $(join(items, " "))") @@ -807,15 +885,15 @@ function parse_single_rhs( if row === nothing error("ROW name $(row_name) not recognised. Is it in the ROWS field?") end - if data.sense[row] == "E" + if data.sense[row] == SENSE_E data.row_upper[row] = value data.row_lower[row] = value - elseif data.sense[row] == "G" + elseif data.sense[row] == SENSE_G data.row_lower[row] = value - elseif data.sense[row] == "L" + elseif data.sense[row] == SENSE_L data.row_upper[row] = value else - @assert data.sense[row] == "N" + @assert data.sense[row] == SENSE_N error("Cannot have RHS for objective: $(join(items, " "))") end return @@ -852,17 +930,16 @@ end # E | - | rhs + range | rhs # ============================================================================== -function parse_single_range(data, row_name, value) +function parse_single_range(data, row_name::String, value::Float64) row = get(data.name_to_row, row_name, nothing) if row === nothing error("ROW name $(row_name) not recognised. Is it in the ROWS field?") end - value = parse(Float64, value) - if data.sense[row] == "G" + if data.sense[row] == SENSE_G data.row_upper[row] = data.row_lower[row] + abs(value) - elseif data.sense[row] == "L" + elseif data.sense[row] == SENSE_L data.row_lower[row] = data.row_upper[row] - abs(value) - elseif data.sense[row] == "E" + elseif data.sense[row] == SENSE_E if value > 0.0 data.row_upper[row] = data.row_lower[row] + value else @@ -876,13 +953,13 @@ end function parse_ranges_line(data::TempMPSModel, items::Vector{String}) if length(items) == 3 # [rhs name] [row name] [value] - rhs_name, row_name, value = items - parse_single_range(data, row_name, value) + _, row_name, value = items + parse_single_range(data, row_name, parse(Float64, value)) elseif length(items) == 5 # [rhs name] [row name] [value] [row name 2] [value 2] - rhs_name, row_name, value, row_name_2, value_2 = items - parse_single_range(data, row_name, value) - parse_single_range(data, row_name_2, value_2) + _, row_name_1, value_1, row_name_2, value_2 = items + parse_single_range(data, row_name_1, parse(Float64, value_1)) + parse_single_range(data, row_name_2, parse(Float64, value_2)) else error("Malformed RANGES line: $(join(items, " "))") end @@ -893,7 +970,7 @@ end # BOUNDS # ============================================================================== -function _parse_single_bound(data, column_name, bound_type) +function _parse_single_bound(data, column_name::String, bound_type::String) col = get(data.name_to_col, column_name, nothing) if col === nothing error("Column name $(column_name) not found.") @@ -910,11 +987,13 @@ function _parse_single_bound(data, column_name, bound_type) data.col_upper[col] = 1.0 data.is_int[col] = true else - error("Invalid bound type $(bound_type): $(join(items, " "))") + error("Invalid bound type $(bound_type): $(column_name)") end end -function _parse_single_bound(data, column_name, bound_type, value::Float64) +function _parse_single_bound( + data, column_name::String, bound_type::String, value::Float64 +) col = get(data.name_to_col, column_name, nothing) if col === nothing error("Column name $(column_name) not found.") @@ -946,7 +1025,7 @@ function _parse_single_bound(data, column_name, bound_type, value::Float64) data.col_upper[col] = 1.0 data.is_int[col] = true else - error("Invalid bound type $(bound_type): $(join(items, " "))") + error("Invalid bound type $(bound_type): $(column_name)") end end From 8e6a07aa0fe1c61b07c1dd8e242bcae50a0a922b Mon Sep 17 00:00:00 2001 From: odow Date: Mon, 29 Jun 2020 12:00:32 -0500 Subject: [PATCH 3/7] Deterministic writing of coefficients and bounds --- src/FileFormats/MPS/MPS.jl | 330 +++++++++++++++++------------------- test/FileFormats/MPS/MPS.jl | 6 +- 2 files changed, 160 insertions(+), 176 deletions(-) diff --git a/src/FileFormats/MPS/MPS.jl b/src/FileFormats/MPS/MPS.jl index 9c4bb87829..23f335d6a4 100644 --- a/src/FileFormats/MPS/MPS.jl +++ b/src/FileFormats/MPS/MPS.jl @@ -44,6 +44,8 @@ function Base.show(io::IO, ::Model) return end +@enum(VType, VTYPE_CONTINUOUS, VTYPE_INTEGER, VTYPE_BINARY) + # ============================================================================== # # Base.write @@ -62,13 +64,22 @@ function Base.write(io::IO, model::Model) warn = options.warn, replacements = Function[s -> replace(s, ' ' => '_')] ) + + ordered_names = String[] + names = Dict{MOI.VariableIndex, String}() + for x in MOI.get(model, MOI.ListOfVariableIndices()) + n = MOI.get(model, MOI.VariableName(), x) + push!(ordered_names, n) + names[x] = n + end + write_model_name(io, model) write_rows(io, model) - discovered_columns = write_columns(io, model) + discovered_columns = write_columns(io, model, ordered_names, names) write_rhs(io, model) write_ranges(io, model) - write_bounds(io, model, discovered_columns) - write_sos(io, model) + write_bounds(io, model, discovered_columns, ordered_names, names) + write_sos(io, model, names) println(io, "ENDATA") return end @@ -87,11 +98,11 @@ end # ROWS # ============================================================================== -const LINEAR_CONSTRAINTS = ( +const SET_TYPES = ( (MOI.LessThan{Float64}, 'L'), (MOI.GreaterThan{Float64}, 'G'), (MOI.EqualTo{Float64}, 'E'), - (MOI.Interval{Float64}, 'L') # See the note in the RANGES section. + (MOI.Interval{Float64}, 'L'), # See the note in the RANGES section. ) function _write_rows(io, model, set_type, sense_char) @@ -112,7 +123,7 @@ end function write_rows(io::IO, model::Model) println(io, "ROWS\n N OBJ") - for (set_type, sense_char) in LINEAR_CONSTRAINTS + for (set_type, sense_char) in SET_TYPES _write_rows(io, model, set_type, sense_char) end return @@ -122,45 +133,39 @@ end # COLUMNS # ============================================================================== -function _list_of_integer_variables(model, integer_variables, set_type) +function _list_of_integer_variables(model, names, integer_variables, S) for index in MOI.get( - model, - MOI.ListOfConstraintIndices{MOI.SingleVariable, set_type}() + model, MOI.ListOfConstraintIndices{MOI.SingleVariable, S}() ) v_index = MOI.get(model, MOI.ConstraintFunction(), index) - v_name = MOI.get(model, MOI.VariableName(), v_index.variable) - push!(integer_variables, v_name) + push!(integer_variables, names[v_index.variable]) end return end -function list_of_integer_variables(model::Model) +function list_of_integer_variables(model::Model, names) integer_variables = Set{String}() - for set_type in (MOI.ZeroOne, MOI.Integer) - _list_of_integer_variables(model, integer_variables, set_type) + for S in (MOI.ZeroOne, MOI.Integer) + _list_of_integer_variables(model, names, integer_variables, S) end return integer_variables end -function add_coefficient(coefficients, variable_name, row_name, coefficient) - if haskey(coefficients, variable_name) - push!(coefficients[variable_name], (row_name, coefficient)) - else - coefficients[variable_name] = [(row_name, coefficient)] - end - return -end - function extract_terms( model::Model, - coefficients, + v_names::Dict{MOI.VariableIndex, String}, + coefficients::Dict{String, Vector{Tuple{String, Float64}}}, row_name::String, func::MOI.ScalarAffineFunction, - discovered_columns::Set{String} + discovered_columns::Set{String}, + multiplier::Float64 = 1.0, ) for term in func.terms - variable_name = MOI.get(model, MOI.VariableName(), term.variable_index) - add_coefficient(coefficients, variable_name, row_name, term.coefficient) + variable_name = v_names[term.variable_index] + push!( + coefficients[variable_name], + (row_name, multiplier * term.coefficient) + ) push!(discovered_columns, variable_name) end return @@ -168,34 +173,43 @@ end function extract_terms( model::Model, - coefficients, + v_names::Dict{MOI.VariableIndex, String}, + coefficients::Dict{String, Vector{Tuple{String, Float64}}}, row_name::String, func::MOI.SingleVariable, discovered_columns::Set{String}, + multiplier::Float64 = 1.0, ) - variable_name = MOI.get(model, MOI.VariableName(), func.variable) - add_coefficient(coefficients, variable_name, row_name, 1.0) + variable_name = v_names[func.variable] + push!(coefficients[variable_name], (row_name, multiplier)) push!(discovered_columns, variable_name) return end -function _write_columns( - io, model, set_type, sense_char, coefficients, discovered_columns +function _collect_coefficients( + model, + S, + v_names::Dict{MOI.VariableIndex, String}, + coefficients::Dict{String, Vector{Tuple{String, Float64}}}, + discovered_columns::Set{String}, ) for index in MOI.get( model, - MOI.ListOfConstraintIndices{ - MOI.ScalarAffineFunction{Float64}, set_type - }() + MOI.ListOfConstraintIndices{MOI.ScalarAffineFunction{Float64}, S}(), ) row_name = MOI.get(model, MOI.ConstraintName(), index) func = MOI.get(model, MOI.ConstraintFunction(), index) extract_terms( - model, coefficients, row_name, func, discovered_columns + model, v_names, coefficients, row_name, func, discovered_columns ) end + return end -function write_columns(io::IO, model::Model) + +function write_columns(io::IO, model::Model, ordered_names, names) + coefficients = Dict{String, Vector{Tuple{String, Float64}}}( + n => Tuple{String, Float64}[] for n in ordered_names + ) # Many MPS readers (e.g., CPLEX and GAMS) will error if a variable (column) # appears in the BOUNDS section but did not appear in the COLUMNS section. # This is likely because such variables are meaningless - they don't appear @@ -204,40 +218,37 @@ function write_columns(io::IO, model::Model) # names of all variables seen in COLUMNS into `discovered_columns`, and then # pass this set to `write_bounds` so that it can act appropriately. discovered_columns = Set{String}() - println(io, "COLUMNS") - coefficients = Dict{String, Vector{Tuple{String, Float64}}}() - for (set_type, sense_char) in LINEAR_CONSTRAINTS - _write_columns( - io, model, set_type, sense_char, coefficients, discovered_columns - ) + # Build constraint coefficients + for (S, _) in SET_TYPES + _collect_coefficients(model, S, names, coefficients, discovered_columns) end - obj_func_type = MOI.get(model, MOI.ObjectiveFunctionType()) - obj_func = MOI.get(model, MOI.ObjectiveFunction{obj_func_type}()) - extract_terms(model, coefficients, "OBJ", obj_func, discovered_columns) - if MOI.get(model, MOI.ObjectiveSense()) == MOI.MAX_SENSE - # MPS doesn't support maximization so we flip the sign on the objective - # coefficients. - for (v_name, terms) in coefficients - for (idx, (row_name, coef)) in enumerate(terms) - if row_name == "OBJ" - terms[idx] = (row_name, -coef) - end - end - end - end - integer_variables = list_of_integer_variables(model) - for (variable, terms) in coefficients - if variable in integer_variables + # Build objective + # MPS doesn't support maximization so we flip the sign on the objective + # coefficients. + s = MOI.get(model, MOI.ObjectiveSense()) == MOI.MAX_SENSE ? -1.0 : 1.0 + obj_func = MOI.get( + model, MOI.ObjectiveFunction{MOI.ScalarAffineFunction{Float64}}() + ) + extract_terms( + model, names, coefficients, "OBJ", obj_func, discovered_columns, s + ) + integer_variables = list_of_integer_variables(model, names) + println(io, "COLUMNS") + int_open = false + for variable in ordered_names + is_int = variable in integer_variables + if is_int && !int_open println(io, " MARKER 'MARKER' 'INTORG'") + int_open = true + elseif !is_int && int_open + println(io, " MARKER 'MARKER' 'INTEND'") + int_open = false end - for (constraint, coefficient) in terms + for (constraint, coefficient) in coefficients[variable] print(io, " ", rpad(variable, 8), " ", rpad(constraint, 8), " ") Base.Grisu.print_shortest(io, coefficient) println(io) end - if variable in integer_variables - println(io, " MARKER 'MARKER' 'INTEND'") - end end return discovered_columns end @@ -268,7 +279,7 @@ end function write_rhs(io::IO, model::Model) println(io, "RHS") - for (set_type, sense_char) in LINEAR_CONSTRAINTS + for (set_type, sense_char) in SET_TYPES _write_rhs(io, model, set_type, sense_char) end return @@ -333,7 +344,7 @@ function write_single_bound(io::IO, var_name::String, lower, upper) Base.Grisu.print_shortest(io, lower) println(io) elseif lower == -Inf && upper == Inf - # Skip this for now, we deal with it at the end of write_bounds. + println(io, " FR bounds ", var_name) else if lower == -Inf println(io, " MI bounds ", name) @@ -353,83 +364,70 @@ function write_single_bound(io::IO, var_name::String, lower, upper) return end -function update_bounds(current::Tuple{Float64, Float64}, set::MOI.GreaterThan) - return (max(current[1], set.lower), current[2]) +function update_bounds(x::Tuple{Float64, Float64, VType}, set::MOI.GreaterThan) + return (max(x[1], set.lower), x[2], x[3]) +end + +function update_bounds(x::Tuple{Float64, Float64, VType}, set::MOI.LessThan) + return (x[1], min(x[2], set.upper), x[3]) end -function update_bounds(current::Tuple{Float64, Float64}, set::MOI.LessThan) - return (current[1], min(current[2], set.upper)) +function update_bounds(x::Tuple{Float64, Float64, VType}, set::MOI.Interval) + return (set.lower, set.upper, x[3]) end -function update_bounds(::Tuple{Float64, Float64}, set::MOI.Interval) - return (set.lower, set.upper) +function update_bounds(x::Tuple{Float64, Float64, VType}, set::MOI.EqualTo) + return (set.value, set.value, x[3]) end -function update_bounds(::Tuple{Float64, Float64}, set::MOI.EqualTo) - return (set.value, set.value) +function update_bounds(x::Tuple{Float64, Float64, VType}, set::MOI.Integer) + return (x[1], x[2], VTYPE_INTEGER) end -function _collect_bounds(bounds, model, set_type) +function update_bounds(x::Tuple{Float64, Float64, VType}, set::MOI.ZeroOne) + return (x[1], x[2], VTYPE_BINARY) +end + +function _collect_bounds(bounds, model, S, names) for index in MOI.get( - model, - MOI.ListOfConstraintIndices{MOI.SingleVariable, set_type}() + model, MOI.ListOfConstraintIndices{MOI.SingleVariable, S}() ) func = MOI.get(model, MOI.ConstraintFunction(), index) - variable_index = func.variable::MOI.VariableIndex - if !haskey(bounds, variable_index) - bounds[variable_index] = (-Inf, Inf) - end - set = MOI.get(model, MOI.ConstraintSet(), index)::set_type - bounds[variable_index] = update_bounds(bounds[variable_index], set) + set = MOI.get(model, MOI.ConstraintSet(), index)::S + name = names[func.variable] + bounds[name] = update_bounds(bounds[name], set) end return end -function write_bounds(io::IO, model::Model, discovered_columns::Set{String}) +function write_bounds( + io::IO, model::Model, discovered_columns::Set{String}, ordered_names, names +) println(io, "BOUNDS") - free_variables = Set(MOI.get(model, MOI.ListOfVariableIndices())) - bounds = Dict{MOI.VariableIndex, Tuple{Float64, Float64}}() - for (set_type, _) in LINEAR_CONSTRAINTS - _collect_bounds(bounds, model, set_type) - end - for (index, (lower, upper)) in bounds - var_name = MOI.get(model, MOI.VariableName(), index) - if var_name in discovered_columns - write_single_bound(io, var_name, lower, upper) - else - @warn("Variable $var_name is mentioned in BOUNDS, but is not " * - "mentioned in the COLUMNS section. We are ignoring it.") - end - pop!(free_variables, index) - end - for index in MOI.get( - model, - MOI.ListOfConstraintIndices{MOI.SingleVariable, MOI.ZeroOne}() + bounds = Dict{String, Tuple{Float64, Float64, VType}}( + n => (-Inf, Inf, VTYPE_CONTINUOUS) for n in ordered_names ) - func = MOI.get(model, MOI.ConstraintFunction(), index) - variable_index = func.variable::MOI.VariableIndex - var_name = MOI.get(model, MOI.VariableName(), variable_index) - if var_name in discovered_columns - println(io, " BV bounds ", var_name) - else - @warn("Variable $var_name is mentioned in BOUNDS, but is not " * - "mentioned in the COLUMNS section. We are ignoring it.") - end - if variable_index in free_variables - # We can remove the variable because it has a bound, but first check - # that it is still there because some variables might have two - # bounds and so might have already been removed. - pop!(free_variables, variable_index) - end + for S in ( + MOI.LessThan{Float64}, + MOI.GreaterThan{Float64}, + MOI.EqualTo{Float64}, + MOI.Interval{Float64}, + MOI.ZeroOne, + ) + _collect_bounds(bounds, model, S, names) end - for variable_index in free_variables - var_name = MOI.get(model, MOI.VariableName(), variable_index) - if var_name in discovered_columns - println(io, " FR bounds ", var_name) - else - @warn("Variable $var_name is mentioned in BOUNDS, but is not " * - "mentioned in the COLUMNS section. We are ignoring it.") + for var_name in ordered_names + lower, upper, vtype = bounds[var_name] + if !(var_name in discovered_columns) + @warn( + "Variable $var_name is mentioned in BOUNDS, but is not " * + "mentioned in the COLUMNS section. We are ignoring it." + ) + continue + elseif vtype == VTYPE_BINARY + println(io, " BV bounds ", var_name) end + write_single_bound(io, var_name, lower, upper) end return end @@ -438,18 +436,17 @@ end # SOS # ============================================================================== -function write_sos_constraint(io::IO, model::Model, index) +function write_sos_constraint(io::IO, model::Model, index, names) func = MOI.get(model, MOI.ConstraintFunction(), index) set = MOI.get(model, MOI.ConstraintSet(), index) for (variable, weight) in zip(func.variables, set.weights) - var_name = MOI.get(model, MOI.VariableName(), variable) - print(io, " ", rpad(var_name, 8), " ") + print(io, " ", rpad(names[variable], 8), " ") Base.Grisu.print_shortest(io, weight) println(io) end end -function write_sos(io::IO, model::Model) +function write_sos(io::IO, model::Model, names) sos1_indices = MOI.get( model, MOI.ListOfConstraintIndices{MOI.VectorOfVariables, MOI.SOS1{Float64}}(), @@ -464,7 +461,7 @@ function write_sos(io::IO, model::Model) for (sos_type, indices) in enumerate([sos1_indices, sos2_indices]) for index in indices println(io, " S", sos_type, " SOS", idx) - write_sos_constraint(io, model, index) + write_sos_constraint(io, model, index, names) idx += 1 end end @@ -518,7 +515,6 @@ function Sense(s::String) return SENSE_UNKNOWN end end - mutable struct TempMPSModel name::String obj_name::String @@ -529,7 +525,7 @@ mutable struct TempMPSModel row_upper::Vector{Float64} sense::Vector{Sense} A::Vector{Vector{Tuple{Int, Float64}}} - is_int::Vector{Bool} + vtype::Vector{VType} name_to_col::Dict{String, Int} col_to_name::Vector{String} name_to_row::Dict{String, Int} @@ -607,9 +603,8 @@ function Base.read!(io::IO, model::Model) multi_objectives = String[] while !eof(io) && header != "ENDATA" line = string(strip(readline(io))) - if line == "" || startswith(line, "*") - # Skip blank lines and comments. - continue + if isempty(line) || startswith(line, "*") + continue # Skip blank lines and comments. end h = Headers(line) if h != HEADER_UNKNOWN @@ -677,23 +672,13 @@ function _add_variable(model, data, variable_map, i, name) variable_map[name] = x MOI.set(model, MOI.VariableName(), x, name) set = bounds_to_set(data.col_lower[i], data.col_upper[i]) - if set === nothing - if data.is_int[i] - # TODO: some solvers may interpret this as binary. - MOI.add_constraint(model, MOI.SingleVariable(x), MOI.Integer()) - else - # Free variable - end - elseif !data.is_int[i] + if set !== nothing MOI.add_constraint(model, MOI.SingleVariable(x), set) - else - @assert data.is_int[i] - if set == MOI.Interval(0.0, 1.0) - MOI.add_constraint(model, MOI.SingleVariable(x), MOI.ZeroOne()) - else - MOI.add_constraint(model, MOI.SingleVariable(x), MOI.Integer()) - MOI.add_constraint(model, MOI.SingleVariable(x), set) - end + end + if data.vtype[i] == VTYPE_INTEGER + MOI.add_constraint(model, MOI.SingleVariable(x), MOI.Integer()) + elseif data.vtype[i] == VTYPE_BINARY + MOI.add_constraint(model, MOI.SingleVariable(x), MOI.ZeroOne()) end return end @@ -715,18 +700,11 @@ function _add_objective(model, data, variable_map) end function _add_linear_constraint(model, data, variable_map, j, c_name, set) - c = MOI.add_constraint( - model, - MOI.ScalarAffineFunction( - [ - MOI.ScalarAffineTerm( - coef, variable_map[data.col_to_name[i]] - ) for (i, coef) in data.A[j] - ], - 0.0 - ), - set, - ) + terms = [ + MOI.ScalarAffineTerm(coef, variable_map[data.col_to_name[i]]) + for (i, coef) in data.A[j] + ] + c = MOI.add_constraint(model, MOI.ScalarAffineFunction(terms, 0.0), set) MOI.set(model, MOI.ConstraintName(), c, c_name) return end @@ -799,7 +777,9 @@ end function parse_single_coefficient( data, row_name::String, column::Int, value::Float64 ) - if row_name == data.obj_name + if iszero(value) + return + elseif row_name == data.obj_name data.c[column] += value return end @@ -820,18 +800,20 @@ function _add_new_column(data, column_name) push!(data.c, 0.0) push!(data.col_lower, 0.0) push!(data.col_upper, Inf) - push!(data.is_int, false) + push!(data.vtype, VTYPE_CONTINUOUS) return end function _set_intorg(data, column, column_name) - if data.is_int[column] && !data.intorg_flag + if data.intorg_flag + data.vtype[column] = VTYPE_INTEGER + elseif data.vtype[column] != VTYPE_CONTINUOUS error( "Variable $(column_name) appeared in COLUMNS outside an " * "`INT` marker after already being declared as integer." ) end - data.is_int[column] = data.intorg_flag + return end function parse_columns_line( @@ -983,9 +965,9 @@ function _parse_single_bound(data, column_name::String, bound_type::String) data.col_lower[col] = -Inf data.col_upper[col] = Inf elseif bound_type == "BV" - data.col_lower[col] = 0.0 - data.col_upper[col] = 1.0 - data.is_int[col] = true + data.col_lower[col] = -Inf + data.col_upper[col] = Inf + data.vtype[col] = VTYPE_BINARY else error("Invalid bound type $(bound_type): $(column_name)") end @@ -1007,10 +989,10 @@ function _parse_single_bound( data.col_lower[col] = value elseif bound_type == "LI" data.col_lower[col] = value - data.is_int[col] = true + data.vtype[col] = VTYPE_INTEGER elseif bound_type == "UI" data.col_upper[col] = value - data.is_int[col] = true + data.vtype[col] = VTYPE_INTEGER elseif bound_type == "FR" # So even though FR bounds should be of the form: # FR BOUND1 VARNAME @@ -1021,9 +1003,9 @@ function _parse_single_bound( data.col_upper[col] = Inf elseif bound_type == "BV" # A similar situation happens with BV bounds in leo1 and leo2. - data.col_lower[col] = 0.0 - data.col_upper[col] = 1.0 - data.is_int[col] = true + data.col_lower[col] = -Inf + data.col_upper[col] = Inf + data.vtype[col] = VTYPE_BINARY else error("Invalid bound type $(bound_type): $(column_name)") end diff --git a/test/FileFormats/MPS/MPS.jl b/test/FileFormats/MPS/MPS.jl index edc3dea333..3a49428223 100644 --- a/test/FileFormats/MPS/MPS.jl +++ b/test/FileFormats/MPS/MPS.jl @@ -52,8 +52,10 @@ end @testset "SOS" begin model = MPS.Model() x = MOI.add_variables(model, 3) + names = Dict{MOI.VariableIndex, String}() for i in 1:3 MOI.set(model, MOI.VariableName(), x[i], "x$(i)") + names[x[i]] = "x$(i)" end MOI.add_constraint(model, MOI.VectorOfVariables(x), @@ -63,7 +65,7 @@ end MOI.VectorOfVariables(x), MOI.SOS2([1.25, 2.25, 3.25]) ) - @test sprint(MPS.write_sos, model) == + @test sprint(MPS.write_sos, model, names) == "SOS\n" * " S1 SOS1\n" * " x1 1.5\n" * @@ -82,7 +84,7 @@ end MOI.set(model, MOI.ObjectiveSense(), MOI.MAX_SENSE) MOI.set(model, MOI.ObjectiveFunction{MOI.SingleVariable}(), MOI.SingleVariable(x)) - @test sprint(MPS.write_columns, model) == + @test sprint(MPS.write_columns, model, ["x"], Dict(x => "x")) == "COLUMNS\n x OBJ -1\n" end From 01e1f39c8eb796dbb96a7872d6df693631fa6f88 Mon Sep 17 00:00:00 2001 From: odow Date: Mon, 29 Jun 2020 14:19:15 -0500 Subject: [PATCH 4/7] Raise error when parsing SOS constraints --- src/FileFormats/MPS/MPS.jl | 53 ++++++++++++-------------------------- 1 file changed, 17 insertions(+), 36 deletions(-) diff --git a/src/FileFormats/MPS/MPS.jl b/src/FileFormats/MPS/MPS.jl index 23f335d6a4..244a7bb614 100644 --- a/src/FileFormats/MPS/MPS.jl +++ b/src/FileFormats/MPS/MPS.jl @@ -16,6 +16,8 @@ MOI.Utilities.@model(Model, () ) +MOI.supports(::Model, ::MOI.ObjectiveFunction{MOI.SingleVariable}) = false + struct Options warn::Bool end @@ -32,7 +34,7 @@ Keyword arguments are: - `warn::Bool=false`: print a warning when variables or constraints are renamed. """ function Model(; - warn::Bool = false + warn::Bool = false, ) model = Model{Float64}() model.ext[:MPS_OPTIONS] = Options(warn) @@ -64,7 +66,6 @@ function Base.write(io::IO, model::Model) warn = options.warn, replacements = Function[s -> replace(s, ' ' => '_')] ) - ordered_names = String[] names = Dict{MOI.VariableIndex, String}() for x in MOI.get(model, MOI.ListOfVariableIndices()) @@ -72,7 +73,6 @@ function Base.write(io::IO, model::Model) push!(ordered_names, n) names[x] = n end - write_model_name(io, model) write_rows(io, model) discovered_columns = write_columns(io, model, ordered_names, names) @@ -105,12 +105,10 @@ const SET_TYPES = ( (MOI.Interval{Float64}, 'L'), # See the note in the RANGES section. ) -function _write_rows(io, model, set_type, sense_char) +function _write_rows(io, model, S, sense_char) for index in MOI.get( model, - MOI.ListOfConstraintIndices{ - MOI.ScalarAffineFunction{Float64}, set_type - }() + MOI.ListOfConstraintIndices{MOI.ScalarAffineFunction{Float64}, S}() ) row_name = MOI.get(model, MOI.ConstraintName(), index) if row_name == "" @@ -152,7 +150,6 @@ function list_of_integer_variables(model::Model, names) end function extract_terms( - model::Model, v_names::Dict{MOI.VariableIndex, String}, coefficients::Dict{String, Vector{Tuple{String, Float64}}}, row_name::String, @@ -171,21 +168,6 @@ function extract_terms( return end -function extract_terms( - model::Model, - v_names::Dict{MOI.VariableIndex, String}, - coefficients::Dict{String, Vector{Tuple{String, Float64}}}, - row_name::String, - func::MOI.SingleVariable, - discovered_columns::Set{String}, - multiplier::Float64 = 1.0, -) - variable_name = v_names[func.variable] - push!(coefficients[variable_name], (row_name, multiplier)) - push!(discovered_columns, variable_name) - return -end - function _collect_coefficients( model, S, @@ -200,7 +182,7 @@ function _collect_coefficients( row_name = MOI.get(model, MOI.ConstraintName(), index) func = MOI.get(model, MOI.ConstraintFunction(), index) extract_terms( - model, v_names, coefficients, row_name, func, discovered_columns + v_names, coefficients, row_name, func, discovered_columns ) end return @@ -229,9 +211,7 @@ function write_columns(io::IO, model::Model, ordered_names, names) obj_func = MOI.get( model, MOI.ObjectiveFunction{MOI.ScalarAffineFunction{Float64}}() ) - extract_terms( - model, names, coefficients, "OBJ", obj_func, discovered_columns, s - ) + extract_terms(names, coefficients, "OBJ", obj_func, discovered_columns, s) integer_variables = list_of_integer_variables(model, names) println(io, "COLUMNS") int_open = false @@ -262,12 +242,10 @@ value(set::MOI.GreaterThan) = set.lower value(set::MOI.EqualTo) = set.value value(set::MOI.Interval) = set.upper # See the note in the RANGES section. -function _write_rhs(io, model, set_type, sense_char) +function _write_rhs(io, model, S, sense_char) for index in MOI.get( model, - MOI.ListOfConstraintIndices{ - MOI.ScalarAffineFunction{Float64}, set_type - }() + MOI.ListOfConstraintIndices{MOI.ScalarAffineFunction{Float64}, S}() ) row_name = MOI.get(model, MOI.ConstraintName(), index) print(io, " rhs ", rpad(row_name, 8), " ") @@ -380,10 +358,6 @@ function update_bounds(x::Tuple{Float64, Float64, VType}, set::MOI.EqualTo) return (set.value, set.value, x[3]) end -function update_bounds(x::Tuple{Float64, Float64, VType}, set::MOI.Integer) - return (x[1], x[2], VTYPE_INTEGER) -end - function update_bounds(x::Tuple{Float64, Float64, VType}, set::MOI.ZeroOne) return (x[1], x[2], VTYPE_BINARY) end @@ -601,7 +575,7 @@ function Base.read!(io::IO, model::Model) data = TempMPSModel() header = HEADER_NAME multi_objectives = String[] - while !eof(io) && header != "ENDATA" + while !eof(io) line = string(strip(readline(io))) if isempty(line) || startswith(line, "*") continue # Skip blank lines and comments. @@ -610,6 +584,8 @@ function Base.read!(io::IO, model::Model) if h != HEADER_UNKNOWN header = h continue + else + # Carry on with the previous header end # TODO: split into hard fields based on column indices. items = String.(split(line, " ", keepempty = false)) @@ -628,6 +604,11 @@ function Base.read!(io::IO, model::Model) parse_ranges_line(data, items) elseif header == HEADER_BOUNDS parse_bounds_line(data, items) + elseif header == HEADER_SOS + error("TODO(odow): implement parsing of SOS constraints.") + else + @assert header == HEADER_ENDATA + break end end copy_to(model, data) From 8731a4a729a86c2501fb709c7bbe4f091c5b5ca4 Mon Sep 17 00:00:00 2001 From: odow Date: Mon, 29 Jun 2020 21:06:24 -0500 Subject: [PATCH 5/7] Improve coverage --- src/FileFormats/MPS/MPS.jl | 13 ++++--------- .../MPS/failing_models/rows_duplicate_2.mps | 7 +++++++ test/FileFormats/MPS/failing_models/sos.mps | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 9 deletions(-) create mode 100644 test/FileFormats/MPS/failing_models/rows_duplicate_2.mps create mode 100644 test/FileFormats/MPS/failing_models/sos.mps diff --git a/src/FileFormats/MPS/MPS.jl b/src/FileFormats/MPS/MPS.jl index 244a7bb614..58745d9f14 100644 --- a/src/FileFormats/MPS/MPS.jl +++ b/src/FileFormats/MPS/MPS.jl @@ -631,19 +631,16 @@ end function copy_to(model::Model, data::TempMPSModel) MOI.set(model, MOI.Name(), data.name) variable_map = Dict{String, MOI.VariableIndex}() - # Add variables. for (i, name) in enumerate(data.col_to_name) _add_variable(model, data, variable_map, i, name) end - # Set objective. _add_objective(model, data, variable_map) - # Add linear constraints. for (j, c_name) in enumerate(data.row_to_name) set = bounds_to_set(data.row_lower[j], data.row_upper[j]) - if set === nothing - error("Expected a non-empty set for $(c_name).") + if set !== nothing + _add_linear_constraint(model, data, variable_map, j, c_name, set) end - _add_linear_constraint(model, data, variable_map, j, c_name, set) + # `else` is a free constraint. Don't add it. end return end @@ -758,9 +755,7 @@ end function parse_single_coefficient( data, row_name::String, column::Int, value::Float64 ) - if iszero(value) - return - elseif row_name == data.obj_name + if row_name == data.obj_name data.c[column] += value return end diff --git a/test/FileFormats/MPS/failing_models/rows_duplicate_2.mps b/test/FileFormats/MPS/failing_models/rows_duplicate_2.mps new file mode 100644 index 0000000000..63830fa1c2 --- /dev/null +++ b/test/FileFormats/MPS/failing_models/rows_duplicate_2.mps @@ -0,0 +1,7 @@ +NAME +ROWS + N obj + L c1 + G c1 +COLUMNS +ENDATA diff --git a/test/FileFormats/MPS/failing_models/sos.mps b/test/FileFormats/MPS/failing_models/sos.mps new file mode 100644 index 0000000000..a4ae8b8bd8 --- /dev/null +++ b/test/FileFormats/MPS/failing_models/sos.mps @@ -0,0 +1,14 @@ +NAME +ROWS + N obj +COLUMNS + x obj 1 + y obj 2 +BOUNDS + FR bounds x + FR bounds y +SOS + S1 sos_a + x 1 + y 2 +ENDATA From 9fff2e1848d14915fbbe4ec040035212d46f80d6 Mon Sep 17 00:00:00 2001 From: odow Date: Tue, 30 Jun 2020 10:48:18 -0500 Subject: [PATCH 6/7] Improve performance by removing --- src/FileFormats/MPS/MPS.jl | 70 +++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/src/FileFormats/MPS/MPS.jl b/src/FileFormats/MPS/MPS.jl index 58745d9f14..dfc4fd4382 100644 --- a/src/FileFormats/MPS/MPS.jl +++ b/src/FileFormats/MPS/MPS.jl @@ -540,27 +540,39 @@ end HEADER_UNKNOWN, ) -function Headers(s::String) - s = uppercase(s) - if s == "ROWS" - return HEADER_ROWS - elseif s == "COLUMNS" - return HEADER_COLUMNS - elseif s == "RHS" - return HEADER_RHS - elseif s == "RANGES" - return HEADER_RANGES - elseif s == "BOUNDS" - return HEADER_BOUNDS - elseif s == "SOS" - return HEADER_SOS - elseif s == "ENDATA" - return HEADER_ENDATA - elseif s == "NAME" - return HEADER_NAME - else - return HEADER_UNKNOWN +# Headers(s) gets called _alot_ (on every line), so we try very hard to be +# efficient.] +function Headers(s::AbstractString) + if length(s) == 3 + x = first(s) + if (x == 'R' || x == 'r') && uppercase(s) == "RHS" + return HEADER_RHS + elseif (x == 'S' || x == 's') && uppercase(s) == "SOS" + return HEADER_SOS + end + elseif length(s) == 4 + x = first(s) + if (x == 'R' || x == 'r') && uppercase(s) == "ROWS" + return HEADER_ROWS + # elseif (x == 'N' || x == 'n') && uppercase(s) == "NAME" + # return HEADER_NAME + end + elseif length(s) == 6 + x = first(s) + if (x == 'R' || x == 'r') && uppercase(s) == "RANGES" + return HEADER_RANGES + elseif (x == 'B' || x == 'b') && uppercase(s) == "BOUNDS" + return HEADER_BOUNDS + elseif (x == 'E' || x == 'e') && uppercase(s) == "ENDATA" + return HEADER_ENDATA + end + elseif length(s) == 7 + x = first(s) + if (x == 'C' || x == 'c') && (uppercase(s) == "COLUMNS") + return HEADER_COLUMNS + end end + return HEADER_UNKNOWN end """ @@ -577,7 +589,7 @@ function Base.read!(io::IO, model::Model) multi_objectives = String[] while !eof(io) line = string(strip(readline(io))) - if isempty(line) || startswith(line, "*") + if isempty(line) || first(line) == '*' continue # Skip blank lines and comments. end h = Headers(line) @@ -588,7 +600,7 @@ function Base.read!(io::IO, model::Model) # Carry on with the previous header end # TODO: split into hard fields based on column indices. - items = String.(split(line, " ", keepempty = false)) + items = String.(split(line, " "; keepempty = false)) if header == HEADER_NAME parse_name_line(data, items) elseif header == HEADER_ROWS @@ -798,12 +810,14 @@ function parse_columns_line( if length(items) == 3 # [column name] [row name] [value] column_name, row_name, value = items - if uppercase(row_name) == "'MARKER'" && uppercase(value) == "'INTORG'" - data.intorg_flag = true - return - elseif uppercase(row_name) == "'MARKER'" && uppercase(value) == "'INTEND'" - data.intorg_flag = false - return + if row_name == "'MARKER'" + if value == "'INTORG'" + data.intorg_flag = true + return + elseif value == "'INTEND'" + data.intorg_flag = false + return + end elseif row_name in multi_objectives return end From ec9962a34c147572c68c12cecadd726725c0ca56 Mon Sep 17 00:00:00 2001 From: odow Date: Wed, 1 Jul 2020 11:52:58 -0500 Subject: [PATCH 7/7] Few more changes --- src/FileFormats/MPS/MPS.jl | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/FileFormats/MPS/MPS.jl b/src/FileFormats/MPS/MPS.jl index dfc4fd4382..0c0461b002 100644 --- a/src/FileFormats/MPS/MPS.jl +++ b/src/FileFormats/MPS/MPS.jl @@ -225,7 +225,11 @@ function write_columns(io::IO, model::Model, ordered_names, names) int_open = false end for (constraint, coefficient) in coefficients[variable] - print(io, " ", rpad(variable, 8), " ", rpad(constraint, 8), " ") + print(io, " ") + print(io, rpad(variable, 8)) + print(io, " ") + print(io, rpad(constraint, 8)) + print(io, " ") Base.Grisu.print_shortest(io, coefficient) println(io) end @@ -543,21 +547,24 @@ end # Headers(s) gets called _alot_ (on every line), so we try very hard to be # efficient.] function Headers(s::AbstractString) - if length(s) == 3 + N = length(s) + if N > 7 || N < 3 + return HEADER_UNKNOWN + elseif N == 3 x = first(s) if (x == 'R' || x == 'r') && uppercase(s) == "RHS" return HEADER_RHS elseif (x == 'S' || x == 's') && uppercase(s) == "SOS" return HEADER_SOS end - elseif length(s) == 4 + elseif N == 4 x = first(s) if (x == 'R' || x == 'r') && uppercase(s) == "ROWS" return HEADER_ROWS - # elseif (x == 'N' || x == 'n') && uppercase(s) == "NAME" - # return HEADER_NAME end - elseif length(s) == 6 + elseif N == 5 + return HEADER_UNKNOWN + elseif N == 6 x = first(s) if (x == 'R' || x == 'r') && uppercase(s) == "RANGES" return HEADER_RANGES @@ -566,7 +573,7 @@ function Headers(s::AbstractString) elseif (x == 'E' || x == 'e') && uppercase(s) == "ENDATA" return HEADER_ENDATA end - elseif length(s) == 7 + elseif N == 7 x = first(s) if (x == 'C' || x == 'c') && (uppercase(s) == "COLUMNS") return HEADER_COLUMNS @@ -575,6 +582,11 @@ function Headers(s::AbstractString) return HEADER_UNKNOWN end +function line_to_items(line) + items = split(line, " "; keepempty = false) + return String.(items) +end + """ Base.read!(io::IO, model::FileFormats.MPS.Model) @@ -600,7 +612,7 @@ function Base.read!(io::IO, model::Model) # Carry on with the previous header end # TODO: split into hard fields based on column indices. - items = String.(split(line, " "; keepempty = false)) + items = line_to_items(line) if header == HEADER_NAME parse_name_line(data, items) elseif header == HEADER_ROWS @@ -690,7 +702,7 @@ function _add_objective(model, data, variable_map) end function _add_linear_constraint(model, data, variable_map, j, c_name, set) - terms = [ + terms = MOI.ScalarAffineTerm{Float64}[ MOI.ScalarAffineTerm(coef, variable_map[data.col_to_name[i]]) for (i, coef) in data.A[j] ]