diff --git a/src/Utilities/cachingoptimizer.jl b/src/Utilities/cachingoptimizer.jl
index 4401b16f8a..cc9124803e 100644
--- a/src/Utilities/cachingoptimizer.jl
+++ b/src/Utilities/cachingoptimizer.jl
@@ -145,17 +145,68 @@ errors can be thrown.
 """
 function attach_optimizer(model::CachingOptimizer)
     @assert model.state == EMPTY_OPTIMIZER
-    # We do not need to copy names because name-related operations are handled by `m.model_cache`
+    # We do not need to copy names because name-related operations are handled
+    # by `m.model_cache`
     indexmap = MOI.copy_to(model.optimizer, model.model_cache, copy_names=false)
     model.state = ATTACHED_OPTIMIZER
-    # MOI does not define the type of index_map, so we have to copy it into a
-    # concrete container. Also load the reverse map.
-    model.model_to_optimizer_map = IndexMap()
-    model.optimizer_to_model_map = IndexMap()
-    for k in keys(indexmap)
-        model.model_to_optimizer_map[k] = indexmap[k]
-        model.optimizer_to_model_map[indexmap[k]] = k
+    # MOI does not define the type of index_map, so we have to convert it
+    # into an actual IndexMap. Also load the reverse IndexMap.
+    model.model_to_optimizer_map = _standardize(indexmap)
+    model.optimizer_to_model_map = _reverse_index_map(indexmap)
+    return nothing
+end
+
+function _reverse_index_map(src::IndexMap)
+    dest = IndexMap()
+    sizehint!(dest.varmap, length(src.varmap))
+    _reverse_dict(dest.varmap, src.varmap)
+    _reverse_dict(dest.conmap, src.conmap)
+    return dest
+end
+
+"""
+    _reverse_dict(dest::AbstractDict, src::AbstractDict)
+
+Reverse dictionary so that values of `src` are key of `dest` and vice-versa.
+`dest` must be empty. Also the values of `src` are assumed to be unique.
+"""
+function _reverse_dict(dest::AbstractDict, src::AbstractDict)
+    for (k,v) in src
+        dest[v] = k
+    end
+end
+
+function _standardize(d::AbstractDict{MOI.Index, MOI.Index})
+    map = IndexMap()
+    for (k,v) in d
+        map[k] = v
+    end
+    return map
+end
+function _standardize(d::IndexMap)
+    # return d
+    # if we return d as is, its not possible to add variables
+    # if there was a dense dict inside
+    # the solution would be to allow automatically swtiching
+    # a ClevelDenseDict...
+    return IndexMap(_standard_dict(d.varmap), d.conmap)
+end
+function _standard_dict(
+    d::D
+)::D where {D<:Dict{MOI.VariableIndex, MOI.VariableIndex}}
+    return d
+end
+function _standard_dict(
+    d::D
+)::Dict{MOI.VariableIndex, MOI.VariableIndex} where {
+    D<:AbstractDict{MOI.VariableIndex, MOI.VariableIndex}
+}
+    ret = Dict{MOI.VariableIndex, MOI.VariableIndex}()
+    sizehint!(ret, length(d))
+    for (k,v) in d
+        ret[k] = v
     end
+    return d
 end
 
 function MOI.copy_to(m::CachingOptimizer, src::MOI.ModelLike; kws...)
diff --git a/src/Utilities/copy.jl b/src/Utilities/copy.jl
index 666a47039e..ee7bc7485b 100644
--- a/src/Utilities/copy.jl
+++ b/src/Utilities/copy.jl
@@ -63,9 +63,18 @@ error in case `copy_to` is called with `copy_names` equal to `true`.
 """
 supports_default_copy_to(model::MOI.ModelLike, copy_names::Bool) = false
 
-const DenseVariableDict{V} = DenseDict{MOI.VariableIndex, V, typeof(MOI.index_value), typeof(MOI.VariableIndex)}
+"""
+    _index_to_variable(i::Int)
+
+Simply returns `MOI.VariableIndex(i)`. This is necessary to pass a function that
+creates the `VariableIndex` from an integer. If we pass `MOI.VariableIndex`
+julia understands is as a `DataType` and not a function, this leads to type
+instability issues.
+"""
+_index_to_variable(i) = MOI.VariableIndex(i)
+const DenseVariableDict{V} = DenseDict{MOI.VariableIndex, V, typeof(MOI.index_value), typeof(_index_to_variable)}
 function dense_variable_dict(::Type{V}, n) where V
-    return DenseDict{MOI.VariableIndex, V}(MOI.index_value, MOI.VariableIndex, n)
+    return DenseDict{MOI.VariableIndex, V}(MOI.index_value, _index_to_variable, n)
 end
 
 struct IndexMap <: AbstractDict{MOI.Index, MOI.Index}
diff --git a/src/Utilities/dense_dict.jl b/src/Utilities/dense_dict.jl
index a7ac300baf..c38d3489b3 100644
--- a/src/Utilities/dense_dict.jl
+++ b/src/Utilities/dense_dict.jl
@@ -22,15 +22,16 @@ end
 
 # Implementation of the `AbstractDict` API.
 # Base.empty(::DenseDict, ::Type{K}, ::Type{V}) not implemented
-function Base.iterate(d::DenseDict, args...)
+function Base.iterate(d::DenseDict{K,V}, args...) where {K,V}
     itr = iterate(d.set, args...)
     if itr === nothing
         return nothing
     else
         el, i = itr
-        return d.inverse_hash(el) => d.map[el], i
+        return d.inverse_hash(el)::K => d.map[el]::V, i
     end
 end
+
 Base.length(d::DenseDict) = length(d.set)
 Base.haskey(dict::DenseDict, key) = dict.hash(key) in dict.set
 Base.getindex(dict::DenseDict, key) = dict.map[dict.hash(key)]