fixed the overflow problem of powermod(x::Integer, p::Integer, m::T)

base/intfuncs.jl

@@ -369,8 +369,19 @@ julia> powermod(5, 3, 19)
 ```
 """
 function powermod(x::Integer, p::Integer, m::T) where T<:Integer
-    p < 0 && return powermod(invmod(x, m), -p, m)
     p == 0 && return mod(one(m),m)
+    # When the concrete type of p is sigened and has the lowerst value,
+    # `p != 0 && p == -p` is equivalent to `p == typemin(typeof(p))` for 2's complement representation.
+    # but will work for integer types like `BigInt` that don't have `typemin` defined
+    # It needs special handling otherwise will cause overflow problem.
+    if p == -p
+        t = powermod(invmod(x, m), -(p÷2), m)
+        t = mod(widemul(t, t), m)
+        iseven(p) && return t
+        #else odd
+        return mod(widemul(t, invmod(x, m)), m)
+    end
+    p < 0 && return powermod(invmod(x, m), -p, m)
     (m == 1 || m == -1) && return zero(m)
     b = oftype(m,mod(x,m))  # this also checks for divide by zero
 

test/intfuncs.jl

@@ -267,6 +267,12 @@ end
     @test powermod(2, -2, 5) == 4
     @test powermod(2, -1, -5) == -2
     @test powermod(2, -2, -5) == -1
+
+    @test powermod(2, typemin(Int128), 5) == 1
+    @test powermod(2, typemin(Int128), -5) == -4
+
+    @test powermod(2, big(3), 5) == 3
+    @test powermod(2, big(3), -5) == -2
 end
 
 @testset "nextpow/prevpow" begin