panic when using local plugin mode

[kk-kwok]

Welcome!

• Yes, I've searched similar issues on GitHub and didn't find any.
• Yes, I've searched similar issues on the Traefik community forum and didn't find any.

What did you do?

panic when using local plugin mode

What did you see instead?

No more panic

What version of Traefik are you using?

since v2.6.7 to v2.8.5
v2.6.6 is normal

What is your environment & configuration?

traefik.yaml

global:
  checkNewVersion: true
  sendAnonymousUsage: true

api:
  insecure: true

entryPoints:
  web:
    address: :80
    transport:
      respondingTimeouts:
        idleTimeout: 180
  websecure:
    address: :443
    transport:
      respondingTimeouts:
        idleTimeout: 180
  metrics:
    address: :8082

serversTransport:
  maxIdleConnsPerHost: 100
  forwardingTimeouts:
    dialTimeout: 30
    responseHeaderTimeout: 60

log:
  format: json
  level: INFO

accessLog:
  format: json
  bufferingsize: 100
  fields:
    defaultMode: keep
    names:
      StartUTC: drop
    headers:
      defaultMode: drop
      names:
        X-Forwarded-For: keep
        X-B3-Traceid: keep
        Ali-Cdn-Real-Ip: keep
        X-Uid: keep

metrics:
  prometheus:
    buckets:
      - 0.1
      - 0.3
      - 1.2
      - 5.0
    addEntryPointsLabels: true
    addRoutersLabels: true
    addServicesLabels: true
    entryPoint: metrics

providers:
  kubernetescrd: 
    allowCrossNamespace: true

tracing:
  serviceName: traefik
  jaeger:
    gen128Bit: true
    propagation: b3
    traceContextHeaderName: x-b3-traceid
    samplingType: const
    samplingParam: 1
    samplingServerURL: http://jaeger-agent.jaeger.svc.cluster.local:5778/sampling
    localAgentHostPort: jaeger-agent.jaeger.svc.cluster.local:6831

experimental:
  localPlugins:
    rateLimit:
      moduleName: code.xxx.com/traefik-plugins/ratelimit

middleware

apiVersion: traefik.containo.us/v1alpha1
kind: Middleware
metadata:
  name: ratelimit
spec:
  plugin:
    rateLimit:
      ipFromHeaderName: "Ali-Cdn-Real-Ip"
      uidFromHeaderName: "X-Uid"
      logLevel: DEBUG
      average: 5
      period: 1
      burst: 5
      excludedIPs:
        - 1.1.1.1

code.xxx.com/traefik-plugins/ratelimit/ratelimit.go

package ratelimit

import (
    "context"
    "errors"
    "fmt"
    "github.com/mailgun/holster/v4/clock"
    "github.com/mailgun/holster/v4/collections"
    "golang.org/x/time/rate"
    "net"
    "net/http"
    "strings"
    "time"
)

const (
	maxSources = 65536
)

type Checker struct {
	authorizedIPs    []*net.IP
	authorizedIPsNet []*net.IPNet
}

type Config struct {
	ExcludedIps       []string `json:"excludedIps,omitempty"`
	IpFromHeaderName  string   `json:"ipFromHeaderName,omitempty"`
	UidFromHeaderName string   `json:"uidFromHeaderName,omitempty"`
	Average           int64    `json:"average,omitempty"`
	Period            int      `json:"period,omitempty"`
	Burst             int64    `json:"burst,omitempty"`
}

// CreateConfig creates the default plugin configuration.
func CreateConfig() *Config {
	return &Config{}
}

type RateLimiter struct {
	name              string
	rate              rate.Limit // reqs/s
	burst             int64
	maxDelay          time.Duration
	ttl               int
	next              http.Handler
	buckets           *collections.TTLMap
	checker           *Checker
	ipFromHeaderName  string
	uidFromHeaderName string
}

func New(ctx context.Context, next http.Handler, config *Config, name string) (http.Handler, error) {
	buckets := collections.NewTTLMap(maxSources)
	clock.Freeze(time.Now())

	burst := config.Burst
	if burst < 1 {
		burst = 1
	}

	period := time.Duration(config.Period) * time.Second
	if period < 0 {
		return nil, fmt.Errorf("negative value not valid for period: %v", period)
	}
	if period == 0 {
		period = time.Second
	}

	// if config.Average == 0, in that case,
	// the value of maxDelay does not matter since the reservation will (buggily) give us a delay of 0 anyway.
	var maxDelay time.Duration
	var rtl float64
	if config.Average > 0 {
		rtl = float64(config.Average*int64(time.Second)) / float64(period)
		// maxDelay does not scale well for rates below 1,
		// so we just cap it to the corresponding value, i.e. 0.5s, in order to keep the effective rate predictable.
		// One alternative would be to switch to a no-reservation mode (Allow() method) whenever we are in such a low rate regime.
		if rtl < 1 {
			maxDelay = 500 * time.Millisecond
		} else {
			maxDelay = time.Second / (time.Duration(rtl) * 2)
		}
	}
	// Make the ttl inversely proportional to how often a rate limiter is supposed to see any activity (when maxed out),
	// for low rate limiters.
	// Otherwise, just make it a second for all the high rate limiters.
	// Add an extra second in both cases for continuity between the two cases.
	ttl := 1
	if rtl >= 1 {
		ttl++
	} else if rtl > 0 {
		ttl += int(1 / rtl)
	}

	checker := &Checker{}
	if config.ExcludedIps != nil || len(config.ExcludedIps) > 0 {
		checker, _ = NewChecker(config.ExcludedIps)
	}

	return &RateLimiter{
		name:              name,
		rate:              rate.Limit(rtl),
		burst:             burst,
		maxDelay:          maxDelay,
		ttl:               ttl,
		next:              next,
		buckets:           buckets,
		checker:           checker,
		ipFromHeaderName:  config.IpFromHeaderName,
		uidFromHeaderName: config.UidFromHeaderName,
	}, nil
}

func GetIP(req *http.Request, requestHeaderName string) string {
	ipAddr := req.Header.Get(requestHeaderName)
	ip, _, err := net.SplitHostPort(ipAddr)
	if err != nil {
		return ipAddr
	}
	if ip == "" {
		ip, _, err := net.SplitHostPort(req.RemoteAddr)
		if err != nil {
			return req.RemoteAddr
		}
		return ip
	}
	return ip
}

func (rl *RateLimiter) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	reqIp := GetIP(req, rl.ipFromHeaderName)
	if contain, _ := rl.checker.Contains(reqIp); contain {
		rl.next.ServeHTTP(rw, req)
	} else {
		var bucket *rate.Limiter
		uid := req.Header.Get(rl.uidFromHeaderName)
		source := fmt.Sprintf("%s%s", reqIp, req.URL.EscapedPath())
		if uid != "" {
			source = fmt.Sprintf("%s%s", uid, req.URL.EscapedPath())
		}

		if rlSource, exists := rl.buckets.Get(source); exists {
			bucket = rlSource.(*rate.Limiter)
		} else {
			bucket = rate.NewLimiter(rl.rate, int(rl.burst))
		}

		if err := rl.buckets.Set(source, bucket, rl.ttl); err != nil {
			fmt.Printf("could not insert/update bucket: %v\n", err)
			http.Error(rw, "could not insert/update bucket", http.StatusInternalServerError)
			return
		}

		res := bucket.Reserve()
		if !res.OK() {
			http.Error(rw, "No bursty traffic allowed", http.StatusTooManyRequests)
			return
		}
		delay := res.Delay()

		fmt.Println("RateLimited: #######################################################################################")
		fmt.Printf("RateLimited: source: %s, rl.rate: %v, burst=%d, res.ok=%v, res.delay=%v, "+
			"maxDelay=%v, ttl=%v\n", source, rl.rate, int(rl.burst), res.OK(), delay, rl.maxDelay, rl.ttl)
		if delay > rl.maxDelay {
			res.Cancel()
			rl.serveDelayError(rw, req, delay)
			return
		}
		time.Sleep(delay)
		rl.next.ServeHTTP(rw, req)
	}
}

func (rl *RateLimiter) serveDelayError(w http.ResponseWriter, r *http.Request, delay time.Duration) {
	w.Header().Add("Retry-After", fmt.Sprintf("%.0f", delay.Seconds()))
	w.Header().Add("X-Retry-In", delay.String())
	w.WriteHeader(http.StatusTooManyRequests)

	if _, err := w.Write([]byte(http.StatusText(http.StatusTooManyRequests))); err != nil {
		fmt.Printf("could not serve 429: %v\n", err)
	}
}

func NewChecker(trustedIPs []string) (*Checker, error) {
	if len(trustedIPs) == 0 {
		return nil, errors.New("no trusted IPs provided")
	}

	checker := &Checker{}

	for _, ipMask := range trustedIPs {
		if ipAddr := net.ParseIP(ipMask); ipAddr != nil {
			checker.authorizedIPs = append(checker.authorizedIPs, &ipAddr)
			continue
		}

		_, ipAddr, err := net.ParseCIDR(ipMask)
		if err != nil {
			return nil, fmt.Errorf("parsing CIDR trusted IPs %s: %w", ipAddr, err)
		}
		checker.authorizedIPsNet = append(checker.authorizedIPsNet, ipAddr)
	}

	return checker, nil
}

func (c *Checker) Contains(addr string) (bool, error) {
	if len(addr) == 0 {
		return false, errors.New("empty IP address")
	}

	ipAddr, err := parseIP(addr)
	if err != nil {
		return false, fmt.Errorf("unable to parse address: %s: %w", addr, err)
	}

	return c.ContainsIP(ipAddr), nil
}

func (c *Checker) ContainsIP(addr net.IP) bool {
	for _, authorizedIP := range c.authorizedIPs {
		if authorizedIP.Equal(addr) {
			return true
		}
	}

	for _, authorizedNet := range c.authorizedIPsNet {
		if authorizedNet.Contains(addr) {
			return true
		}
	}

	return false
}

func (c *Checker) IsAuthorized(addr string) error {
	var invalidMatches []string

	host, _, err := net.SplitHostPort(addr)
	if err != nil {
		host = addr
	}

	ok, err := c.Contains(host)
	if err != nil {
		return err
	}

	if !ok {
		invalidMatches = append(invalidMatches, addr)
		return fmt.Errorf("%q matched none of the trusted IPs", strings.Join(invalidMatches, ", "))
	}

	return nil
}

func parseIP(addr string) (net.IP, error) {
	ip := net.ParseIP(addr)
	if ip == nil {
		return nil, fmt.Errorf("can't parse IP from address %s", addr)
	}

	return ip, nil
}

If applicable, please paste the log output in DEBUG level

traefik-5dd575fb88-xc9cx traefik-ingress-lb {"level":"error","module":"code.xxx.com/traefik-plugins/ratelimit","msg":"plugins-local/src/code.xxx.com/traefik-plugins/ratelimit/ratelimit.go:131:25: panic","plugin":"plugin-rateLimit","time":"2022-09-22T14:22:30+08:00"}
traefik-5dd575fb88-xc9cx traefik-ingress-lb {"level":"error","middlewareName":"traefik-internal-recovery","middlewareType":"Recovery","msg":"Recovered from panic in HTTP handler [163.181.x.x:28101 - /cms_api/get_current_user_info?_t=1663827750641]: interface conversion: interface {} is interp.valueInterface, not *struct { Xmu sync.Mutex; Xlimit float64; Xburst int; Xtokens float64; Xlast time.Time; XlastEvent time.Time }","time":"2022-09-22T14:22:30+08:00"}
traefik-5dd575fb88-xc9cx traefik-ingress-lb {"level":"error","middlewareName":"traefik-internal-recovery","middlewareType":"Recovery","msg":"Stack: goroutine 124 [running]:\ngithub.com/traefik/traefik/v2/pkg/middlewares/recovery.recoverFunc({0x7f016c173ba0, 0xc00011e5b0}, 0xc001741100)
	github.com/traefik/traefik/v2/pkg/middlewares/recovery/recovery.go:46 +0x225\npanic({0x3075760, 0xc001e26c30})
	runtime/panic.go:884 +0x212\ngithub.com/traefik/yaegi/interp.runCfg.func1()
	github.com/traefik/yaegi@v0.14.2/interp/run.go:192 +0x148\npanic({0x3075760, 0xc001e26c30})
	runtime/panic.go:884 +0x212\ngithub.com/traefik/yaegi/interp.typeAssert.func3(0xc001a8c0b0)
	github.com/traefik/yaegi@v0.14.2/interp/run.go:440 +0x5c5\ngithub.com/traefik/yaegi/interp.runCfg(0xc001023680, 0xc001a8c0b0, 0x39?, 0x385fc40?)
	github.com/traefik/yaegi@v0.14.2/interp/run.go:200 +0x29d\ngithub.com/traefik/yaegi/interp.genFunctionWrapper.func2.1({0xc000e0a7b0, 0x2, 0x4?})
	github.com/traefik/yaegi@v0.14.2/interp/run.go:1022 +0x487\ngithub.com/traefik/yaegi/stdlib._net_http_Handler.ServeHTTP(...)
	github.com/traefik/yaegi@v0.14.2/stdlib/go1_19_net_http.go:290\ngithub.com/vulcand/oxy/buffer.(*Buffer).ServeHTTP(0xc001c11720, {0x7f016c173c80?, 0xc00011e5b8}, 0xc001741c00)
	github.com/vulcand/oxy@v1.4.1/buffer/buffer.go:280 +0x8f5\ngithub.com/traefik/traefik/v2/pkg/middlewares/buffering.(*buffer).ServeHTTP(0xc001741200?, {0x7f016c173c80?, 0xc00011e5b8?}, 0x398d21f?)
	github.com/traefik/traefik/v2/pkg/middlewares/buffering/buffering.go:54 +0x27\ngithub.com/traefik/traefik/v2/pkg/middlewares/tracing.(*Wrapper).ServeHTTP(0xc000fab440, {0x7f016c173c80, 0xc00011e5b8}, 0xc001741200)
	github.com/traefik/traefik/v2/pkg/middlewares/tracing/wrapper.go:66 +0xcc\ngithub.com/traefik/traefik/v2/pkg/middlewares/metrics.(*metricsMiddleware).ServeHTTP(0xc0029a7b20, {0x7f016c173ba0?, 0xc00011e5b0}, 0xc001741200)
	github.com/traefik/traefik/v2/pkg/middlewares/metrics/metrics.go:122 +0x6ec\ngithub.com/traefik/traefik/v2/pkg/middlewares/accesslog.(*FieldHandler).ServeHTTP(0xc0012b74c0, {0x7f016c173ba0, 0xc00011e5b0}, 0x323ab00?)
	github.com/traefik/traefik/v2/pkg/middlewares/accesslog/field_middleware.go:38 +0xf6\ngithub.com/gorilla/mux.(*Router).ServeHTTP(0xc001b78ba0, {0x7f016c173ba0, 0xc00011e5b0}, 0xc001741100)
	github.com/gorilla/mux@v1.8.0/mux.go:141 +0x24c\ngithub.com/traefik/traefik/v2/pkg/middlewares/recovery.(*recovery).ServeHTTP(0xc0002d86c8?, {0x7f016c173ba0?, 0xc00011e5b0?}, 0x13400c000580000?)
	github.com/traefik/traefik/v2/pkg/middlewares/recovery/recovery.go:32 +0x82\ngithub.com/traefik/traefik/v2/pkg/middlewares/accesslog.AddOriginFields({0x7f016c173c80?, 0xc00011e5a8}, 0x39ae391?, {0x4430b60, 0xc00181aa20}, 0xc00097e180)
	github.com/traefik/traefik/v2/pkg/middlewares/accesslog/field_middleware.go:55 +0x1bd\ngithub.com/traefik/traefik/v2/pkg/middlewares/accesslog.(*FieldHandler).ServeHTTP(0xc0012b7ec0, {0x7f016c173c80, 0xc00011e5a8}, 0xc001fdd400?)
	github.com/traefik/traefik/v2/pkg/middlewares/accesslog/field_middleware.go:36 +0xd8\ngithub.com/traefik/traefik/v2/pkg/middlewares/metrics.(*metricsMiddleware).ServeHTTP(0xc0029bcaf0, {0x7f016c173c30?, 0xc00011e5a0}, 0xc001741100)
	github.com/traefik/traefik/v2/pkg/middlewares/metrics/metrics.go:122 +0x6ec\ngithub.com/traefik/traefik/v2/pkg/middlewares/tracing.(*entryPointMiddleware).ServeHTTP(0xc00104f2c0, {0x7f016c173ba0?, 0xc00011e588}, 0xc001740700)
	github.com/traefik/traefik/v2/pkg/middlewares/tracing/entrypoint.go:52 +0x5a3\ngithub.com/traefik/traefik/v2/pkg/middlewares/accesslog.(*Handler).ServeHTTP(0xc0006723c0, {0x4458d08?, 0xc000bbe0e0}, 0xc001740300, {0x4430cc0, 0xc00104f2c0})
	github.com/traefik/traefik/v2/pkg/middlewares/accesslog/logger.go:227 +0xe48\ngithub.com/traefik/traefik/v2/pkg/middlewares/accesslog.WrapHandler.func1.1({0x4458d08?, 0xc000bbe0e0?}, 0xc000530600?)
	github.com/traefik/traefik/v2/pkg/middlewares/accesslog/logger.go:71 +0x3b\nnet/http.HandlerFunc.ServeHTTP(0xf8?, {0x4458d08?, 0xc000bbe0e0?}, 0xc000580000?)
	net/http/server.go:2109 +0x2f\ngithub.com/traefik/traefik/v2/pkg/middlewares.(*HTTPHandlerSwitcher).ServeHTTP(0x4108c7?, {0x4458d08, 0xc000bbe0e0}, 0x4428c01?)
	github.com/traefik/traefik/v2/pkg/middlewares/handler_switcher.go:23 +0x62\ngithub.com/traefik/traefik/v2/pkg/middlewares/requestdecorator.(*RequestDecorator).ServeHTTP(0xc00011f8b8, {0x4458d08, 0xc000bbe0e0}, 0xc001740000, 0xc000530820)
	github.com/traefik/traefik/v2/pkg/middlewares/requestdecorator/request_decorator.go:47 +0x30e\ngithub.com/traefik/traefik/v2/pkg/middlewares/requestdecorator.WrapHandler.func1.1({0x4458d08?, 0xc000bbe0e0?}, 0xc000e0a1b0?)
	github.com/traefik/traefik/v2/pkg/middlewares/requestdecorator/request_decorator.go:89 +0x68\nnet/http.HandlerFunc.ServeHTTP(0xc00012b680?, {0x4458d08?, 0xc000bbe0e0?}, 0x9?)
	net/http/server.go:2109 +0x2f\ngithub.com/traefik/traefik/v2/pkg/middlewares/forwardedheaders.(*XForwarded).ServeHTTP(0xc00012b680, {0x4458d08, 0xc000bbe0e0}, 0xc001740000)
	github.com/traefik/traefik/v2/pkg/middlewares/forwardedheaders/forwarded_header.go:192 +0xca\nnet/http.AllowQuerySemicolons.func1({0x4458d08, 0xc000bbe0e0}, 0xc001740000)
	net/http/server.go:2974 +0x223\nnet/http.HandlerFunc.ServeHTTP(0xc?, {0x4458d08?, 0xc000bbe0e0?}, 0xc000530990?)
	net/http/server.go:2109 +0x2f\ngolang.org/x/net/http2/h2c.h2cHandler.ServeHTTP({{0x44382e0?, 0xc0006fbcb0?}, 0xc00015c0c0?}, {0x4458d08, 0xc000bbe0e0}, 0xc001740000)
	golang.org/x/net@v0.0.0-20220624214902-1bab6f366d9e/http2/h2c/h2c.go:113 +0x49f\nnet/http.serverHandler.ServeHTTP({0xc000e0a120?}, {0x4458d08, 0xc000bbe0e0}, 0xc001740000)
	net/http/server.go:2947 +0x30c\nnet/http.(*conn).serve(0xc0004ae000, {0x445a990, 0xc00052c210})
	net/http/server.go:1991 +0x607\ncreated by net/http.(*Server).Serve
	net/http/server.go:3102 +0x4db\n","time":"2022-09-22T14:22:30+08:00"}
traefik-5dd575fb88-xc9cx traefik-ingress-lb {"ClientAddr":"163.181.x.x:28101","ClientHost":"163.181.x.x","ClientPort":"28101","ClientUsername":"-","DownstreamContentSize":22,"DownstreamStatus":500,"Duration":822314,"OriginContentSize":22,"OriginDuration":660212,"OriginStatus":500,"Overhead":162102,"RequestAddr":"dev.xxx.com","RequestContentSize":0,"RequestCount":24,"RequestHost":"dev.xxx.com","RequestMethod":"GET","RequestPath":"/cms_api/get_current_user_info?_t=1663827750641","RequestPort":"-","RequestProtocol":"HTTP/1.1","RequestScheme":"http","RetryAttempts":0,"RouterName":"istio-system-cms-api-f5e751664fa5918ff1bd@kubernetescrd","StartLocal":"2022-09-22T14:22:30.918693106+08:00","entryPointName":"web","level":"info","msg":"","request_Ali-Cdn-Real-Ip":"121.35.x.x","time":"2022-09-22T14:22:30+08:00"}

[kk-kwok]

After I pulled the traefik v2.6.7 code, I changed the yaegi version in the go.mod file from v0.12.0(traefik v2.6.7) to v0.11.3(traefik v2.6.6), re-run go mod tidy, and then compiled the new traefik, the same error

[kk-kwok]

It's normal working on traefik 2.9.4