/
skipper.go
1939 lines (1595 loc) · 66.6 KB
/
skipper.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
package skipper
import (
"context"
"crypto/tls"
"fmt"
"io"
"net"
"net/http"
"os"
"os/signal"
"path"
"regexp"
"strconv"
"strings"
"syscall"
"time"
stdlog "log"
ot "github.com/opentracing/opentracing-go"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"github.com/zalando/skipper/circuit"
"github.com/zalando/skipper/dataclients/kubernetes"
"github.com/zalando/skipper/dataclients/routestring"
"github.com/zalando/skipper/eskip"
"github.com/zalando/skipper/eskipfile"
"github.com/zalando/skipper/etcd"
"github.com/zalando/skipper/filters"
"github.com/zalando/skipper/filters/apiusagemonitoring"
"github.com/zalando/skipper/filters/auth"
block "github.com/zalando/skipper/filters/block"
"github.com/zalando/skipper/filters/builtin"
"github.com/zalando/skipper/filters/fadein"
logfilter "github.com/zalando/skipper/filters/log"
ratelimitfilters "github.com/zalando/skipper/filters/ratelimit"
"github.com/zalando/skipper/filters/shedder"
"github.com/zalando/skipper/innkeeper"
"github.com/zalando/skipper/loadbalancer"
"github.com/zalando/skipper/logging"
"github.com/zalando/skipper/metrics"
skpnet "github.com/zalando/skipper/net"
pauth "github.com/zalando/skipper/predicates/auth"
"github.com/zalando/skipper/predicates/cookie"
"github.com/zalando/skipper/predicates/cron"
"github.com/zalando/skipper/predicates/forwarded"
"github.com/zalando/skipper/predicates/host"
"github.com/zalando/skipper/predicates/interval"
"github.com/zalando/skipper/predicates/methods"
"github.com/zalando/skipper/predicates/primitive"
"github.com/zalando/skipper/predicates/query"
"github.com/zalando/skipper/predicates/source"
"github.com/zalando/skipper/predicates/tee"
"github.com/zalando/skipper/predicates/traffic"
"github.com/zalando/skipper/proxy"
"github.com/zalando/skipper/queuelistener"
"github.com/zalando/skipper/ratelimit"
"github.com/zalando/skipper/routing"
"github.com/zalando/skipper/scheduler"
"github.com/zalando/skipper/script"
"github.com/zalando/skipper/secrets"
"github.com/zalando/skipper/secrets/certregistry"
"github.com/zalando/skipper/swarm"
"github.com/zalando/skipper/tracing"
)
const (
defaultSourcePollTimeout = 30 * time.Millisecond
defaultRoutingUpdateBuffer = 1 << 5
)
const DefaultPluginDir = "./plugins"
type testOptions struct {
redisConnMetricsInterval time.Duration
redisUpdateInterval time.Duration
}
// Options to start skipper.
type Options struct {
// WaitForHealthcheckInterval sets the time that skipper waits
// for the loadbalancer in front to become unhealthy. Defaults
// to 0.
WaitForHealthcheckInterval time.Duration
// StatusChecks is an experimental feature. It defines a
// comma separated list of HTTP URLs to do GET requests to,
// that have to return 200 before skipper becomes ready
StatusChecks []string
// WhitelistedHealthcheckCIDR appends the whitelisted IP Range to the inernalIPS range for healthcheck purposes
WhitelistedHealthCheckCIDR []string
// Network address that skipper should listen on.
Address string
// Insecure network address skipper should listen on when TLS is enabled
InsecureAddress string
// EnableTCPQueue enables controlling the
// concurrently processed requests at the TCP listener.
EnableTCPQueue bool
// ExpectedBytesPerRequest is used by the TCP LIFO listener.
// It defines the expected average memory required to process an incoming
// request. It is used only when MaxTCPListenerConcurrency is not defined.
// It is used together with the memory limit defined in:
// cgroup v1 /sys/fs/cgroup/memory/memory.limit_in_bytes
// or
// cgroup v2 /sys/fs/cgroup/memory.max
//
// See also:
// cgroup v1: https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt
// cgroup v2: https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html#memory-interface-files
ExpectedBytesPerRequest int
// MaxTCPListenerConcurrency is used by the TCP LIFO listener.
// It defines the max number of concurrently accepted connections, excluding
// the pending ones in the queue.
//
// When undefined and the EnableTCPQueue is true,
MaxTCPListenerConcurrency int
// MaxTCPListenerQueue is used by the TCP LIFO listener.
// If defines the maximum number of pending connection waiting in the queue.
MaxTCPListenerQueue int
// List of custom filter specifications.
CustomFilters []filters.Spec
// Urls of nodes in an etcd cluster, storing route definitions.
EtcdUrls []string
// Path prefix for skipper related data in the etcd storage.
EtcdPrefix string
// Timeout used for a single request when querying for updates
// in etcd. This is independent of, and an addition to,
// SourcePollTimeout. When not set, the internally defined 1s
// is used.
EtcdWaitTimeout time.Duration
// Skip TLS certificate check for etcd connections.
EtcdInsecure bool
// If set this value is used as Bearer token for etcd OAuth authorization.
EtcdOAuthToken string
// If set this value is used as username for etcd basic authorization.
EtcdUsername string
// If set this value is used as password for etcd basic authorization.
EtcdPassword string
// If set enables skipper to generate based on ingress resources in kubernetes cluster
Kubernetes bool
// If set makes skipper authenticate with the kubernetes API server with service account assigned to the
// skipper POD.
// If omitted skipper will rely on kubectl proxy to authenticate with API server
KubernetesInCluster bool
// Kubernetes API base URL. Only makes sense if KubernetesInCluster is set to false. If omitted and
// skipper is not running in-cluster, the default API URL will be used.
KubernetesURL string
// KubernetesHealthcheck, when Kubernetes ingress is set, indicates
// whether an automatic healthcheck route should be generated. The
// generated route will report healthyness when the Kubernetes API
// calls are successful. The healthcheck endpoint is accessible from
// internal IPs, with the path /kube-system/healthz.
KubernetesHealthcheck bool
// KubernetesHTTPSRedirect, when Kubernetes ingress is set, indicates
// whether an automatic redirect route should be generated to redirect
// HTTP requests to their HTTPS equivalent. The generated route will
// match requests with the X-Forwarded-Proto and X-Forwarded-Port,
// expected to be set by the load-balancer.
KubernetesHTTPSRedirect bool
// KubernetesHTTPSRedirectCode overrides the default redirect code (308)
// when used together with -kubernetes-https-redirect.
KubernetesHTTPSRedirectCode int
// KubernetesIngressClass is a regular expression, that will make
// skipper load only the ingress resources that have a matching
// kubernetes.io/ingress.class annotation. For backwards compatibility,
// the ingresses without an annotation, or an empty annotation, will
// be loaded, too.
KubernetesIngressClass string
// KubernetesRouteGroupClass is a regular expression, that will make skipper
// load only the RouteGroup resources that have a matching
// zalando.org/routegroup.class annotation. Any RouteGroups without the
// annotation, or which an empty annotation, will be loaded too.
KubernetesRouteGroupClass string
// KubernetesIngressLabelSelectors is a map of kubernetes labels to their values that must be present on a resource to be loaded
// by the client. A label and its value on an Ingress must be match exactly to be loaded by Skipper.
// If the value is irrelevant for a given configuration, it can be left empty. The default
// value is no labels required.
// Examples:
// Config [] will load all Ingresses.
// Config ["skipper-enabled": ""] will load only Ingresses with a label "skipper-enabled", no matter the value.
// Config ["skipper-enabled": "true"] will load only Ingresses with a label "skipper-enabled: true"
// Config ["skipper-enabled": "", "foo": "bar"] will load only Ingresses with both labels while label "foo" must have a value "bar".
KubernetesIngressLabelSelectors map[string]string
// KubernetesServicesLabelSelectors is a map of kubernetes labels to their values that must be present on a resource to be loaded
// by the client. Read documentation for IngressLabelSelectors for examples and more details.
// The default value is no labels required.
KubernetesServicesLabelSelectors map[string]string
// KubernetesEndpointsLabelSelectors is a map of kubernetes labels to their values that must be present on a resource to be loaded
// by the client. Read documentation for IngressLabelSelectors for examples and more details.
// The default value is no labels required.
KubernetesEndpointsLabelSelectors map[string]string
// KubernetesSecretsLabelSelectors is a map of kubernetes labels to their values that must be present on a resource to be loaded
// by the client. Read documentation for IngressLabelSelectors for examples and more details.
// The default value is no labels required.
KubernetesSecretsLabelSelectors map[string]string
// KubernetesRouteGroupsLabelSelectors is a map of kubernetes labels to their values that must be present on a resource to be loaded
// by the client. Read documentation for IngressLabelSelectors for examples and more details.
// The default value is no labels required.
KubernetesRouteGroupsLabelSelectors map[string]string
// PathMode controls the default interpretation of ingress paths in cases
// when the ingress doesn't specify it with an annotation.
KubernetesPathMode kubernetes.PathMode
// KubernetesNamespace is used to switch between monitoring ingresses in the cluster-scope or limit
// the ingresses to only those in the specified namespace. Defaults to "" which means monitor ingresses
// in the cluster-scope.
KubernetesNamespace string
// *DEPRECATED* KubernetesEnableEastWest enables cluster internal service to service communication, aka east-west traffic
KubernetesEnableEastWest bool
// *DEPRECATED* KubernetesEastWestDomain sets the cluster internal domain used to create additional routes in skipper, defaults to skipper.cluster.local
KubernetesEastWestDomain string
// KubernetesEastWestRangeDomains set the the cluster internal domains for
// east west traffic. Identified routes to such domains will include
// the KubernetesEastWestRangePredicates.
KubernetesEastWestRangeDomains []string
// KubernetesEastWestRangePredicates set the Predicates that will be
// appended to routes identified as to KubernetesEastWestRangeDomains.
KubernetesEastWestRangePredicates []*eskip.Predicate
// KubernetesOnlyAllowedExternalNames will enable validation of ingress external names and route groups network
// backend addresses, explicit LB endpoints validation agains the list of patterns in
// AllowedExternalNames.
KubernetesOnlyAllowedExternalNames bool
// KubernetesAllowedExternalNames contains regexp patterns of those domain names that are allowed to be
// used with external name services (type=ExternalName).
KubernetesAllowedExternalNames []*regexp.Regexp
// KubernetesRedisServiceNamespace to be used to lookup ring shards dynamically
KubernetesRedisServiceNamespace string
// KubernetesRedisServiceName to be used to lookup ring shards dynamically
KubernetesRedisServiceName string
// *DEPRECATED* KubernetesRedisServicePort is not used anymore
KubernetesRedisServicePort int
// KubernetesForceService overrides the default Skipper functionality to route traffic using Kubernetes Endpoints,
// instead using Kubernetes Services.
KubernetesForceService bool
// *DEPRECATED* API endpoint of the Innkeeper service, storing route definitions.
InnkeeperUrl string
// *DEPRECATED* Fixed token for innkeeper authentication. (Used mainly in
// development environments.)
InnkeeperAuthToken string
// *DEPRECATED* Filters to be prepended to each route loaded from Innkeeper.
InnkeeperPreRouteFilters string
// *DEPRECATED* Filters to be appended to each route loaded from Innkeeper.
InnkeeperPostRouteFilters string
// *DEPRECATED* Skip TLS certificate check for Innkeeper connections.
InnkeeperInsecure bool
// *DEPRECATED* OAuth2 URL for Innkeeper authentication.
OAuthUrl string
// *DEPRECATED* Directory where oauth credentials are stored, with file names:
// client.json and user.json.
OAuthCredentialsDir string
// *DEPRECATED* The whitespace separated list of OAuth2 scopes.
OAuthScope string
// File containing static route definitions. Multiple may be given comma separated.
RoutesFile string
// File containing route definitions with file watch enabled.
// Multiple may be given comma separated. (For the skipper
// command this option is used when starting it with the -routes-file flag.)
WatchRoutesFile string
// RouteURLs are URLs pointing to route definitions, in eskip format, with change watching enabled.
RoutesURLs []string
// InlineRoutes can define routes as eskip text.
InlineRoutes string
// Polling timeout of the routing data sources.
SourcePollTimeout time.Duration
// DefaultFilters will be applied to all routes automatically.
DefaultFilters *eskip.DefaultFilters
// DisabledFilters is a list of filters unavailable for use
DisabledFilters []string
// CloneRoute is a slice of PreProcessors that will be applied to all routes
// automatically. They will clone all matching routes and apply changes to the
// cloned routes.
CloneRoute []*eskip.Clone
// EditRoute will be applied to all routes automatically and
// will apply changes to all matching routes.
EditRoute []*eskip.Editor
// A list of custom routing pre-processor implementations that will
// be applied to all routes.
CustomRoutingPreProcessors []routing.PreProcessor
// Deprecated. See ProxyFlags. When used together with ProxyFlags,
// the values will be combined with |.
ProxyOptions proxy.Options
// Flags controlling the proxy behavior.
ProxyFlags proxy.Flags
// Tells the proxy maximum how many idle connections can it keep
// alive.
IdleConnectionsPerHost int
// Defines the time period of how often the idle connections maintained
// by the proxy are closed.
CloseIdleConnsPeriod time.Duration
// Defines ReadTimeoutServer for server http connections.
ReadTimeoutServer time.Duration
// Defines ReadHeaderTimeout for server http connections.
ReadHeaderTimeoutServer time.Duration
// Defines WriteTimeout for server http connections.
WriteTimeoutServer time.Duration
// Defines IdleTimeout for server http connections.
IdleTimeoutServer time.Duration
// Defines MaxHeaderBytes for server http connections.
MaxHeaderBytes int
// Enable connection state metrics for server http connections.
EnableConnMetricsServer bool
// TimeoutBackend sets the TCP client connection timeout for
// proxy http connections to the backend.
TimeoutBackend time.Duration
// ResponseHeaderTimeout sets the HTTP response timeout for
// proxy http connections to the backend.
ResponseHeaderTimeoutBackend time.Duration
// ExpectContinueTimeoutBackend sets the HTTP timeout to expect a
// response for status Code 100 for proxy http connections to
// the backend.
ExpectContinueTimeoutBackend time.Duration
// KeepAliveBackend sets the TCP keepalive for proxy http
// connections to the backend.
KeepAliveBackend time.Duration
// DualStackBackend sets if the proxy TCP connections to the
// backend should be dual stack.
DualStackBackend bool
// TLSHandshakeTimeoutBackend sets the TLS handshake timeout
// for proxy connections to the backend.
TLSHandshakeTimeoutBackend time.Duration
// MaxIdleConnsBackend sets MaxIdleConns, which limits the
// number of idle connections to all backends, 0 means no
// limit.
MaxIdleConnsBackend int
// DisableHTTPKeepalives sets DisableKeepAlives, which forces
// a backend to always create a new connection.
DisableHTTPKeepalives bool
// Flag indicating to ignore trailing slashes in paths during route
// lookup.
IgnoreTrailingSlash bool
// Priority routes that are matched against the requests before
// the standard routes from the data clients.
PriorityRoutes []proxy.PriorityRoute
// Specifications of custom, user defined predicates.
CustomPredicates []routing.PredicateSpec
// Custom data clients to be used together with the default etcd and Innkeeper.
CustomDataClients []routing.DataClient
// CustomHttpHandlerWrap provides ability to wrap http.Handler created by skipper.
// http.Handler is used for accepting incoming http requests.
// It allows to add additional logic (for example tracing) by providing a wrapper function
// which accepts original skipper handler as an argument and returns a wrapped handler
CustomHttpHandlerWrap func(http.Handler) http.Handler
// CustomHttpRoundTripperWrap provides ability to wrap http.RoundTripper created by skipper.
// http.RoundTripper is used for making outgoing requests (backends)
// It allows to add additional logic (for example tracing) by providing a wrapper function
// which accepts original skipper http.RoundTripper as an argument and returns a wrapped roundtripper
CustomHttpRoundTripperWrap func(http.RoundTripper) http.RoundTripper
// WaitFirstRouteLoad prevents starting the listener before the first batch
// of routes were applied.
WaitFirstRouteLoad bool
// SuppressRouteUpdateLogs indicates to log only summaries of the routing updates
// instead of full details of the updated/deleted routes.
SuppressRouteUpdateLogs bool
// Dev mode. Currently this flag disables prioritization of the
// consumer side over the feeding side during the routing updates to
// populate the updated routes faster.
DevMode bool
// Network address for the support endpoints
SupportListener string
// Deprecated: Network address for the /metrics endpoint
MetricsListener string
// Skipper provides a set of metrics with different keys which are exposed via HTTP in JSON
// You can customize those key names with your own prefix
MetricsPrefix string
// EnableProfile exposes profiling information on /profile of the
// metrics listener.
EnableProfile bool
// BlockProfileRate calls runtime.SetBlockProfileRate(BlockProfileRate) if non zero value, deactivate with <0
BlockProfileRate int
// MutexProfileFraction calls runtime.SetMutexProfileFraction(MutexProfileFraction) if non zero value, deactivate with <0
MutexProfileFraction int
// MemProfileRate calls runtime.SetMemProfileRate(MemProfileRate) if non zero value, deactivate with <0
MemProfileRate int
// Flag that enables reporting of the Go garbage collector statistics exported in debug.GCStats
EnableDebugGcMetrics bool
// Flag that enables reporting of the Go runtime statistics exported in runtime and specifically runtime.MemStats
EnableRuntimeMetrics bool
// If set, detailed response time metrics will be collected
// for each route, additionally grouped by status and method.
EnableServeRouteMetrics bool
// If set, a counter for each route is generated, additionally
// grouped by status and method. It differs from the automatically
// generated counter from `EnableServeRouteMetrics` because it will
// always contain the status and method labels, independently of the
// `EnableServeMethodMetric` and `EnableServeStatusCodeMetric` flags.
EnableServeRouteCounter bool
// If set, detailed response time metrics will be collected
// for each host, additionally grouped by status and method.
EnableServeHostMetrics bool
// If set, a counter for each host is generated, additionally
// grouped by status and method. It differs from the automatically
// generated counter from `EnableServeHostMetrics` because it will
// always contain the status and method labels, independently of the
// `EnableServeMethodMetric` and `EnableServeStatusCodeMetric` flags.
EnableServeHostCounter bool
// If set, the detailed total response time metrics will contain the
// HTTP method as a domain of the metric. It affects both route and
// host splitted metrics.
EnableServeMethodMetric bool
// If set, the detailed total response time metrics will contain the
// HTTP Response status code as a domain of the metric. It affects
// both route and host splitted metrics.
EnableServeStatusCodeMetric bool
// If set, detailed response time metrics will be collected
// for each backend host
EnableBackendHostMetrics bool
// EnableAllFiltersMetrics enables collecting combined filter
// metrics per each route. Without the DisableMetricsCompatibilityDefaults,
// it is enabled by default.
EnableAllFiltersMetrics bool
// EnableCombinedResponseMetrics enables collecting response time
// metrics combined for every route.
EnableCombinedResponseMetrics bool
// EnableRouteResponseMetrics enables collecting response time
// metrics per each route. Without the DisableMetricsCompatibilityDefaults,
// it is enabled by default.
EnableRouteResponseMetrics bool
// EnableRouteBackendErrorsCounters enables counters for backend
// errors per each route. Without the DisableMetricsCompatibilityDefaults,
// it is enabled by default.
EnableRouteBackendErrorsCounters bool
// EnableRouteStreamingErrorsCounters enables counters for streaming
// errors per each route. Without the DisableMetricsCompatibilityDefaults,
// it is enabled by default.
EnableRouteStreamingErrorsCounters bool
// EnableRouteBackendMetrics enables backend response time metrics
// per each route. Without the DisableMetricsCompatibilityDefaults, it is
// enabled by default.
EnableRouteBackendMetrics bool
// EnableRouteCreationMetrics enables the OriginMarker to track route creation times. Disabled by default
EnableRouteCreationMetrics bool
// When set, makes the histograms use an exponentially decaying sample
// instead of the default uniform one.
MetricsUseExpDecaySample bool
// Use custom buckets for prometheus histograms.
HistogramMetricBuckets []float64
// The following options, for backwards compatibility, are true
// by default: EnableAllFiltersMetrics, EnableRouteResponseMetrics,
// EnableRouteBackendErrorsCounters, EnableRouteStreamingErrorsCounters,
// EnableRouteBackendMetrics. With this compatibility flag, the default
// for these options can be set to false.
DisableMetricsCompatibilityDefaults bool
// Implementation of a Metrics handler. If provided this is going to be used
// instead of creating a new one based on the Kind of metrics wanted. This
// is useful in case you want to report metrics to a custom aggregator.
MetricsBackend metrics.Metrics
// Output file for the application log. Default value: /dev/stderr.
//
// When /dev/stderr or /dev/stdout is passed in, it will be resolved
// to os.Stderr or os.Stdout.
//
// Warning: passing an arbitrary file will try to open it append
// on start and use it, or fail on start, but the current
// implementation doesn't support any more proper handling
// of temporary failures or log-rolling.
ApplicationLogOutput string
// Application log prefix. Default value: "[APP]".
ApplicationLogPrefix string
// Enables logs in JSON format
ApplicationLogJSONEnabled bool
// ApplicationLogJsonFormatter, when set and JSON logging is enabled, is passed along to to the underlying
// Logrus logger for application logs. To enable structured logging, use ApplicationLogJSONEnabled.
ApplicationLogJsonFormatter *log.JSONFormatter
// Output file for the access log. Default value: /dev/stderr.
//
// When /dev/stderr or /dev/stdout is passed in, it will be resolved
// to os.Stderr or os.Stdout.
//
// Warning: passing an arbitrary file will try to open for append
// it on start and use it, or fail on start, but the current
// implementation doesn't support any more proper handling
// of temporary failures or log-rolling.
AccessLogOutput string
// Disables the access log.
AccessLogDisabled bool
// Enables logs in JSON format
AccessLogJSONEnabled bool
// AccessLogStripQuery, when set, causes the query strings stripped
// from the request URI in the access logs.
AccessLogStripQuery bool
// AccessLogJsonFormatter, when set and JSON logging is enabled, is passed along to to the underlying
// Logrus logger for access logs. To enable structured logging, use AccessLogJSONEnabled.
AccessLogJsonFormatter *log.JSONFormatter
DebugListener string
// Path of certificate(s) when using TLS, mutiple may be given comma separated
CertPathTLS string
// Path of key(s) when using TLS, multiple may be given comma separated. For
// multiple keys, the order must match the one given in CertPathTLS
KeyPathTLS string
// TLS Settings for Proxy Server
ProxyTLS *tls.Config
// Client TLS to connect to Backends
ClientTLS *tls.Config
// TLSMinVersion to set the minimal TLS version for all TLS configurations
TLSMinVersion uint16
// Flush interval for upgraded Proxy connections
BackendFlushInterval time.Duration
// Experimental feature to handle protocol Upgrades for Websockets, SPDY, etc.
ExperimentalUpgrade bool
// ExperimentalUpgradeAudit enables audit log of both the request line
// and the response messages during web socket upgrades.
ExperimentalUpgradeAudit bool
// MaxLoopbacks defines the maximum number of loops that the proxy can execute when the routing table
// contains loop backends (<loopback>).
MaxLoopbacks int
// EnableBreakers enables the usage of the breakers in the route definitions without initializing any
// by default. It is a shortcut for setting the BreakerSettings to:
//
// []circuit.BreakerSettings{{Type: BreakerDisabled}}
//
EnableBreakers bool
// BreakerSettings contain global and host specific settings for the circuit breakers.
BreakerSettings []circuit.BreakerSettings
// EnableRatelimiters enables the usage of the ratelimiter in the route definitions without initializing any
// by default. It is a shortcut for setting the RatelimitSettings to:
//
// []ratelimit.Settings{{Type: DisableRatelimit}}
//
EnableRatelimiters bool
// RatelimitSettings contain global and host specific settings for the ratelimiters.
RatelimitSettings []ratelimit.Settings
// EnableRouteFIFOMetrics enables metrics for the individual route FIFO queues, if any.
EnableRouteFIFOMetrics bool
// EnableRouteLIFOMetrics enables metrics for the individual route LIFO queues, if any.
EnableRouteLIFOMetrics bool
// OpenTracing enables opentracing
OpenTracing []string
// OpenTracingInitialSpan can override the default initial, pre-routing, span name.
// Default: "ingress".
OpenTracingInitialSpan string
// OpenTracingExcludedProxyTags can disable a tag so that it is not recorded. By default every tag is included.
OpenTracingExcludedProxyTags []string
// OpenTracingDisableFilterSpans flag is used to disable creation of spans representing request and response filters.
OpenTracingDisableFilterSpans bool
// OpenTracingLogFilterLifecycleEvents flag is used to enable/disable the logs for events marking request and
// response filters' start & end times.
OpenTracingLogFilterLifecycleEvents bool
// OpenTracingLogStreamEvents flag is used to enable/disable the logs that marks the
// times when response headers & payload are streamed to the client
OpenTracingLogStreamEvents bool
// OpenTracingBackendNameTag enables an additional tracing tag containing a backend name
// for a route when it's available (e.g. for RouteGroups)
OpenTracingBackendNameTag bool
// PluginDir defines the directory to load plugins from, DEPRECATED, use PluginDirs
PluginDir string
// PluginDirs defines the directories to load plugins from
PluginDirs []string
// FilterPlugins loads additional filters from modules. The first value in each []string
// needs to be the plugin name (as on disk, without path, without ".so" suffix). The
// following values are passed as arguments to the plugin while loading, see also
// https://opensource.zalando.com/skipper/reference/plugins/
FilterPlugins [][]string
// PredicatePlugins loads additional predicates from modules. See above for FilterPlugins
// what the []string should contain.
PredicatePlugins [][]string
// DataClientPlugins loads additional data clients from modules. See above for FilterPlugins
// what the []string should contain.
DataClientPlugins [][]string
// Plugins combine multiple types of the above plugin types in one plugin (where
// necessary because of shared data between e.g. a filter and a data client).
Plugins [][]string
// DefaultHTTPStatus is the HTTP status used when no routes are found
// for a request.
DefaultHTTPStatus int
// EnablePrometheusMetrics enables Prometheus format metrics.
//
// This option is *deprecated*. The recommended way to enable prometheus metrics is to
// use the MetricsFlavours option.
EnablePrometheusMetrics bool
// An instance of a Prometheus registry. It allows registering and serving custom metrics when skipper is used as a
// library.
// A new registry is created if this option is nil.
PrometheusRegistry *prometheus.Registry
// MetricsFlavours sets the metrics storage and exposed format
// of metrics endpoints.
MetricsFlavours []string
// LoadBalancerHealthCheckInterval enables and sets the
// interval when to schedule health checks for dead or
// unhealthy routes
LoadBalancerHealthCheckInterval time.Duration
// ReverseSourcePredicate enables the automatic use of IP
// whitelisting in different places to use the reversed way of
// identifying a client IP within the X-Forwarded-For
// header. Amazon's ALB for example writes the client IP to
// the last item of the string list of the X-Forwarded-For
// header, in this case you want to set this to true.
ReverseSourcePredicate bool
// EnableOAuth2GrantFlow, enables OAuth2 Grant Flow filter
EnableOAuth2GrantFlow bool
// OAuth2AuthURL, the url to redirect the requests to when login is required.
OAuth2AuthURL string
// OAuth2TokenURL, the url where the access code should be exchanged for the
// access token.
OAuth2TokenURL string
// OAuth2RevokeTokenURL, the url where the access and refresh tokens can be
// revoked during a logout.
OAuth2RevokeTokenURL string
// OAuthTokeninfoURL sets the the URL to be queried for
// information for all auth.NewOAuthTokeninfo*() filters.
OAuthTokeninfoURL string
// OAuthTokeninfoTimeout sets timeout duration while calling oauth token service
OAuthTokeninfoTimeout time.Duration
// OAuthTokeninfoCacheSize configures the maximum number of cached tokens.
// Zero value disables tokeninfo cache.
OAuthTokeninfoCacheSize int
// OAuthTokeninfoCacheTTL limits the lifetime of a cached tokeninfo.
// Tokeninfo is cached for the duration of "expires_in" field value seconds or
// for the duration of OAuthTokeninfoCacheTTL if it is not zero and less than "expires_in" value.
OAuthTokeninfoCacheTTL time.Duration
// OAuth2SecretFile contains the filename with the encryption key for the
// authentication cookie and grant flow state stored in Secrets.
OAuth2SecretFile string
// OAuth2ClientID, the OAuth2 client id of the current service, used to exchange
// the access code.
OAuth2ClientID string
// OAuth2ClientSecret, the secret associated with the ClientID, used to exchange
// the access code.
OAuth2ClientSecret string
// OAuth2ClientIDFile, the path of the file containing the OAuth2 client id of
// the current service, used to exchange the access code.
OAuth2ClientIDFile string
// OAuth2ClientSecretFile, the path of the file containing the secret associated
// with the ClientID, used to exchange the access code.
OAuth2ClientSecretFile string
// OAuth2CallbackPath contains the path where the OAuth2 callback requests with the
// authorization code should be redirected to. Defaults to /.well-known/oauth2-callback
OAuth2CallbackPath string
// OAuthTokenintrospectionTimeout sets timeout duration while calling oauth tokenintrospection service
OAuthTokenintrospectionTimeout time.Duration
// OAuth2AuthURLParameters the additional parameters to send to OAuth2 authorize and token endpoints.
OAuth2AuthURLParameters map[string]string
// OAuth2AccessTokenHeaderName the name of the header to which the access token
// should be assigned after the oauthGrant filter.
OAuth2AccessTokenHeaderName string
// OAuth2TokeninfoSubjectKey the key of the subject ID attribute in the
// tokeninfo map. Used for downstream oidcClaimsQuery compatibility.
OAuth2TokeninfoSubjectKey string
// OAuth2TokenCookieName the name of the cookie that Skipper sets after a
// successful OAuth2 token exchange. Stores the encrypted access token.
OAuth2TokenCookieName string
// CompressEncodings, if not empty replace default compression encodings
CompressEncodings []string
// OIDCSecretsFile path to the file containing key to encrypt OpenID token
OIDCSecretsFile string
// OIDCCookieValidity sets validity time duration for Cookies to calculate expiration time. (default 1h).
OIDCCookieValidity time.Duration
// OIDCDistributedClaimsTimeout sets timeout duration while calling Distributed Claims endpoint.
OIDCDistributedClaimsTimeout time.Duration
// SecretsRegistry to store and load secretsencrypt
SecretsRegistry *secrets.Registry
// CredentialsPaths directories or files where credentials are stored one secret per file
CredentialsPaths []string
// CredentialsUpdateInterval sets the interval to update secrets
CredentialsUpdateInterval time.Duration
// API Monitoring feature is active (feature toggle)
ApiUsageMonitoringEnable bool
ApiUsageMonitoringRealmKeys string
ApiUsageMonitoringClientKeys string
ApiUsageMonitoringRealmsTrackingPattern string
// *DEPRECATED* ApiUsageMonitoringDefaultClientTrackingPattern
ApiUsageMonitoringDefaultClientTrackingPattern string
// Default filters directory enables default filters mechanism and sets the directory where the filters are located
DefaultFiltersDir string
// WebhookTimeout sets timeout duration while calling a custom webhook auth service
WebhookTimeout time.Duration
// MaxAuditBody sets the maximum read size of the body read by the audit log filter
MaxAuditBody int
// MaxMatcherBufferSize sets the maximum read buffer size of blockContent filter defaults to 2MiB
MaxMatcherBufferSize uint64
// EnableSwarm enables skipper fleet communication, required by e.g.
// the cluster ratelimiter
EnableSwarm bool
// redis based swarm
SwarmRedisURLs []string
SwarmRedisPassword string
SwarmRedisHashAlgorithm string
SwarmRedisDialTimeout time.Duration
SwarmRedisReadTimeout time.Duration
SwarmRedisWriteTimeout time.Duration
SwarmRedisPoolTimeout time.Duration
SwarmRedisMinIdleConns int
SwarmRedisMaxIdleConns int
// swim based swarm
SwarmKubernetesNamespace string
SwarmKubernetesLabelSelectorKey string
SwarmKubernetesLabelSelectorValue string
SwarmPort int
SwarmMaxMessageBuffer int
SwarmLeaveTimeout time.Duration
// swim based swarm for local testing
SwarmStaticSelf string // 127.0.0.1:9001
SwarmStaticOther string // 127.0.0.1:9002,127.0.0.1:9003
// SwarmRegistry specifies an optional callback function that is
// called after ratelimit registry is initialized
SwarmRegistry func(*ratelimit.Registry)
// ClusterRatelimitMaxGroupShards specifies the maximum number of group shards for the clusterRatelimit filter
ClusterRatelimitMaxGroupShards int
// KubernetesEnableTLS enables kubernetes to use resources to terminate tls
KubernetesEnableTLS bool
LuaModules []string
testOptions
}
type serverErrorLogWriter struct{}
func (*serverErrorLogWriter) Write(p []byte) (int, error) {
m := string(p)
if strings.HasPrefix(m, "http: TLS handshake error") && strings.HasSuffix(m, ": EOF\n") {
log.Debug(m) // https://github.com/golang/go/issues/26918
} else if strings.HasPrefix(m, "http: URL query contains semicolon") {
log.Debug(m) // https://github.com/golang/go/issues/25192
} else {
log.Error(m)
}
return len(p), nil
}
func newServerErrorLog() *stdlog.Logger {
return stdlog.New(&serverErrorLogWriter{}, "", 0)
}
func createDataClients(o Options, auth innkeeper.Authentication, cr *certregistry.CertRegistry) ([]routing.DataClient, error) {
var clients []routing.DataClient
if o.RoutesFile != "" {
for _, rf := range strings.Split(o.RoutesFile, ",") {
f, err := eskipfile.Open(rf)
if err != nil {
log.Error("error while opening eskip file", err)
return nil, err
}
clients = append(clients, f)
}
}
if o.WatchRoutesFile != "" {
for _, rf := range strings.Split(o.WatchRoutesFile, ",") {
clients = append(clients, eskipfile.Watch(rf))
}
}
if len(o.RoutesURLs) > 0 {
for _, url := range o.RoutesURLs {
client, err := eskipfile.RemoteWatch(&eskipfile.RemoteWatchOptions{
RemoteFile: url,
FailOnStartup: true,
HTTPTimeout: o.SourcePollTimeout,
})
if err != nil {
log.Errorf("error while loading routes from url %s: %s", url, err)
return nil, err
}
clients = append(clients, client)
}
}
if o.InlineRoutes != "" {
ir, err := routestring.New(o.InlineRoutes)
if err != nil {
log.Error("error while parsing inline routes", err)
return nil, err
}
clients = append(clients, ir)
}
if o.InnkeeperUrl != "" {
ic, err := innkeeper.New(innkeeper.Options{
Address: o.InnkeeperUrl,
Insecure: o.InnkeeperInsecure,
Authentication: auth,
PreRouteFilters: o.InnkeeperPreRouteFilters,
PostRouteFilters: o.InnkeeperPostRouteFilters,
})
if err != nil {
log.Error("error while initializing Innkeeper client", err)
return nil, err
}
clients = append(clients, ic)
}
if len(o.EtcdUrls) > 0 {
etcdClient, err := etcd.New(etcd.Options{
Endpoints: o.EtcdUrls,
Prefix: o.EtcdPrefix,
Timeout: o.EtcdWaitTimeout,
Insecure: o.EtcdInsecure,
OAuthToken: o.EtcdOAuthToken,
Username: o.EtcdUsername,
Password: o.EtcdPassword,
})
if err != nil {
return nil, err
}
clients = append(clients, etcdClient)
}
if o.Kubernetes {
kubernetesClient, err := kubernetes.New(kubernetes.Options{
AllowedExternalNames: o.KubernetesAllowedExternalNames,
BackendNameTracingTag: o.OpenTracingBackendNameTag,
DefaultFiltersDir: o.DefaultFiltersDir,
KubernetesInCluster: o.KubernetesInCluster,