droplevels!(): more efficient implementation

src/array.jl

@@ -836,7 +836,30 @@ unique(A::CategoricalArray{T}) where {T} = _unique(T, A.refs, A.pool)
 Drop levels which do not appear in categorical array `A` (so that they will no longer be
 returned by [`levels`](@ref DataAPI.levels)).
 """
-droplevels!(A::CategoricalArray) = levels!(A, intersect(levels(A), unique(A)))
+function droplevels!(A::CategoricalArray)
+    arefs = refs(A)
+    nlevels = length(levels(A))
+    seen = fill(false, nlevels)
+    nseen = 0
+    @inbounds for ref in arefs
+        if (ref > 0) && !seen[ref]
+            seen[ref] = true
+            nseen += 1
+            (nseen == nlevels) && return A # all levels observed, nothing to drop
+        end
+    end
+    # recode refs to keep only the seen ones (optimized version of update_refs!())
+    newlv = 0
+    levelsmap = [s ? (newlv += 1) : 0 for s in seen]
+    @inbounds for i in eachindex(arefs)
+        if arefs[i] > 0
+            arefs[i] = levelsmap[arefs[i]]
+        end
+    end
+    # replace the pool
+    A.pool = typeof(pool(A))(@inbounds(levels(A)[seen]), isordered(A))
+    return A
+end
 
 """
     isordered(A::CategoricalArray)