diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..97e82cc56 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +fabio +dist/ diff --git a/.gitignore b/.gitignore index dbfffb934..0860c372a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,18 +1,21 @@ *-amd64 -*.out *.orig +*.out *.p12 *.pem *.pprof *.sha256 *.swp +*.tar.gz *.test *.un~ +*.zip .DS_Store .idea .vagrant build/builds/ fabio +fabio.exe fabio.sublime-* demo/cert/ pkg/ diff --git a/.goreleaser.yml b/.goreleaser.yml index 9dbd03e45..5627fab08 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -9,6 +9,7 @@ builds: - freebsd - netbsd - openbsd + - windows goarch: - 386 - amd64 @@ -19,7 +20,7 @@ archive: format: binary checksum: - name_template: '{{.ProjectName}}-{{ .Env.GOVERSION }}-{{.Version}}.sha256' + name_template: '{{.ProjectName}}-{{.Version}}-{{ .Env.GOVERSION }}.sha256' sign: artifacts: checksum diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 048a6f0ae..000000000 --- a/.travis.yml +++ /dev/null @@ -1,18 +0,0 @@ -dist: trusty - -language: go - -go: - - 1.8.x - - 1.9.x - -before_script: - - echo $HOSTNAME - - mkdir -p $GOPATH/bin - - wget --version - - wget https://releases.hashicorp.com/consul/1.0.0/consul_1.0.0_linux_amd64.zip - - wget https://releases.hashicorp.com/vault/0.8.3/vault_0.8.3_linux_amd64.zip - - unzip -d $GOPATH/bin consul_1.0.0_linux_amd64.zip - - unzip -d $GOPATH/bin vault_0.8.3_linux_amd64.zip - - vault --version - - consul --version diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f4db0dd9..b3b263097 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,151 @@ ## Changelog -### Unreleased +### [v1.5.9](https://github.com/fabiolb/fabio/releases/tag/v1.5.9) - 16 May 2018 + +#### Notes + + * [Issue #494](https://github.com/fabiolb/fabio/issues/494): Tests fail with Vault > 0.9.6 and Consul > 1.0.6 + + Needs more investigation. + +#### Breaking Changes + + * None + +#### Bug Fixes + + * [Issue #460](https://github.com/fabiolb/fabio/issues/460): Fix access logging when gzip is enabled + + Fabio was not writing access logs when the gzip compression was enabled. + + Thanks to [@tino](https://github.com/tino) for finding this and providing + and initial patch. + + * [PR #468](https://github.com/fabiolb/fabio/pull/468): Fix the regex of the example proxy.gzip.contenttype + + The example regexp for `proxy.gzip.contenttype` in `fabio.properties` was not properly escaped. + + Thanks to [@tino](https://github.com/tino) for the patch. + + * [Issue #421](https://github.com/fabiolb/fabio/issues/421): Fabio routing to wrong backend + + Fabio does not close websocket connections if the connection upgrade fails. This can lead to + connections being routed to the wrong backend if there is another HTTP router like nginx in + front of fabio. The failed websocket connection creates a direct TCP tunnel to the original + backend server and that connection is not closed properly. + + The patches detect an unsuccessful handshake and close the connection properly. + + Thanks to [@craigday](https://github.com/craigday) for the original reporting and debugging. + +#### Improvements + + * [Issue #427](https://github.com/fabiolb/fabio/issues/427): Fabio does not remove service when one of the registered health-checks fail + + If a service has more than one health check then the behavior in whether the + service is available differs between Consul and Fabio. Consul requires that + all health checks for a service need to pass in order to return a positive + DNS result. Fabio requires only one of the health checks to pass. + + A new config option `registry.consul.checksRequired` has been added which + defaults to the current fabio behavior of `one` passing health check for the + service to be added to the routing table. To make fabio behave like Consul + you can set the option to `all`. + + Fabio will make `all` the default as of version 1.6. + + Thanks to [@systemfreund](https://github.com/systemfreund) for the patch. + + * [Issue #448](https://github.com/fabiolb/fabio/issues/448): Redirect http to https on the same destination + + Fabio will now handle redirecting from http to https on the same destination + without a redirect loop. + + Thanks to [@leprechau](https://github.com/leprechau) for the patch and to + [@atillamas](https://github.com/atillamas) for the original PR and the + discussion. + + * [PR #453](https://github.com/fabiolb/fabio/pull/453): Handle proxy chains of any length + + Fabio will now validate that all elements of the `X-Forwarded-For` header + are allowed by the given ACL of the route. See discussion in + [PR #449](https://github.com/fabiolb/fabio/pull/449) for details. + + Thanks to [@leprechau](https://github.com/leprechau) for the patch and to + [@atillamas](https://github.com/atillamas) for the original PR and the + discussion. + + * [Issue #452](https://github.com/fabiolb/fabio/issues/452): Add improved glob matcher + + Fabio now uses the `github.com/gobaws/glob` package for glob matching which + allows more complex patterns. + + Thanks to [@sharbov](https://github.com/sharbov) for the patch. + +#### Features + + * None + +### [v1.5.8](https://github.com/fabiolb/fabio/releases/tag/v1.5.8) - 18 Feb 2018 + +#### Breaking Changes + + * None + +#### Bug Fixes + + * Fix windows build. + + fabio 1.5.7 broke the Windows build but this wasn't detected since the new + build process did not build the Windows binaries. This has been fixed. + + * [Issue #438](https://github.com/fabiolb/fabio/pull/438): Do not add separator to `noroute.html` page + + fabio 1.5.7 added support for multiple routing tables in Consul and added a + comment which described the origin to the output. The same comment was added + to the `noroute.html` page since the same code is used to fetch it. This + returned an invalid HTML page which has been fixed. + +#### Improvements + + * [PR #423](https://github.com/fabiolb/fabio/pull/423): TCP+SNI support arbitrary large Client Hello + + With this patch fabio correctly parses `ClientHello` messages on TLS + connections up to their maximum size. + + Thanks to [@DanSipola](https://github.com/DanSipola) for the patch. + +#### Features + + * [PR #426](https://github.com/fabiolb/fabio/pull/426): Add option to allow Fabio to register frontend services in Consul on behalf of user services + + With this patch fabio can register itself multiple times under different + names in Consul. By adding the `register=name` option to a route fabio will + register itself under that name as well. + + Thanks to [@rileyje](https://github.com/rileyje) for the patch. + + * [PR #442](https://github.com/fabiolb/fabio/pull/442): Add basic ip centric access control on routes + + With this patch fabio adds an `allow` and `deny` option to the routes which + allows for basic ip white and black listing of IPv4 and IPv6 addresses. See + http://fabiolb.net/feature/access-control/ for more details. + + Thanks to [@leprechau](https://github.com/leprechau) for the patch and + [@microadam](https://github.com/microadam) for the testing. + +### [v1.5.7](https://github.com/fabiolb/fabio/releases/tag/v1.5.7) - 6 Feb 2018 #### Breaking Changes * None +#### Bug Fixes + + * [Issue #434](https://github.com/fabiolb/fabio/issues/434): VaultPKI tests fail with go1.10rc1 + + All unit tests pass now on go1.10rc1. + #### Improvements * [Issue #369](https://github.com/fabiolb/fabio/issues/369): Warn if fabio is run as root @@ -21,9 +161,9 @@ #### Features - * [Issue #396](https://github.com/fabiolb/fabio/issue/396): treat `registry.consul.kvpath` as prefix + * [Issue #396](https://github.com/fabiolb/fabio/issues/396): treat `registry.consul.kvpath` as prefix - This patch allows fabio to have multiple manual routing tables stored in consul, e.g. + This patch allows fabio to have multiple manual routing tables stored in consul, e.g. under `fabio/config/foo` and `fabio/config/bar`. The routing table fragments are concatenated in lexicographical order of the keys and the log output contains comments to indicate to which key the segment belongs. @@ -32,7 +172,7 @@ fabio has now support for adding HSTS headers to the response. - Thanks to (@leprechau)[https://github.com/leprechau] for the patch. + Thanks to [@leprechau](https://github.com/leprechau) for the patch. ### [v1.5.6](https://github.com/fabiolb/fabio/releases/tag/v1.5.6) - 5 Jan 2018 diff --git a/Dockerfile-test b/Dockerfile-test new file mode 100644 index 000000000..c7d56d2fc --- /dev/null +++ b/Dockerfile-test @@ -0,0 +1,15 @@ +FROM ubuntu +RUN apt-get update && apt-get -y install unzip make git-core +ARG consul_version +ARG vault_version +ARG go_version +COPY consul_${consul_version}_linux_amd64.zip /tmp +COPY vault_${vault_version}_linux_amd64.zip /tmp +COPY go${go_version}.linux-amd64.tar.gz /tmp +RUN unzip /tmp/consul_${consul_version}_linux_amd64.zip -d /usr/local/bin +RUN unzip /tmp/vault_${vault_version}_linux_amd64.zip -d /usr/local/bin +RUN tar -C /usr/local -x -f /tmp/go${go_version}.linux-amd64.tar.gz +ENV PATH=/usr/local/go/bin:/root/go/bin:$PATH +WORKDIR /root/go/src/github.com/fabiolb/fabio +COPY . . +CMD "/bin/bash" diff --git a/Makefile b/Makefile index 866438c10..6d2ff11ba 100644 --- a/Makefile +++ b/Makefile @@ -10,10 +10,6 @@ LAST_TAG = $(shell git describe --abbrev=0) # e.g. 1.5.5 VERSION = $(shell git describe --abbrev=0 | cut -c 2-) -# GO runs the go binary with garbage collection disabled for faster builds. -# Do not specify a full path for go since travis will fail. -GO = GOGC=off go - # GOFLAGS is the flags for the go compiler. Currently, only the version number is # passed to the linker via the -ldflags. GOFLAGS = -ldflags "-X main.version=$(CUR_TAG)" @@ -30,8 +26,13 @@ GOVENDOR = $(shell which govendor) # VENDORFMT is the path to the vendorfmt binary. VENDORFMT = $(shell which vendorfmt) +# pin versions for CI builds +CI_CONSUL_VERSION=1.0.6 +CI_VAULT_VERSION=0.9.6 +CI_GO_VERSION=1.10.2 + # all is the default target -all: build test +all: test # help prints a help screen help: @@ -39,32 +40,27 @@ help: @echo "install - go install" @echo "test - go test" @echo "gofmt - go fmt" - @echo "vet - go vet" @echo "linux - go build linux/amd64" - @echo "release - build/release.sh" - @echo "gorelease - goreleaser" - @echo "homebrew - build/homebrew.sh" - @echo "buildpkg - build/build.sh" + @echo "release - tag, build and publish release with goreleaser" @echo "pkg - build, test and create pkg/fabio.tar.gz" @echo "clean - remove temp files" # build compiles fabio and the test dependencies -build: checkdeps vendorfmt vet gofmt - $(GO) build -i $(GOFLAGS) - $(GO) test -i ./... +build: checkdeps vendorfmt gofmt + go build # test runs the tests -test: checkdeps vendorfmt gofmt - $(GO) test -v -test.timeout 15s `go list ./... | grep -v '/vendor/'` +test: build + go test -v -test.timeout 15s `go list ./... | grep -v '/vendor/'` # checkdeps ensures that all required dependencies are vendored in checkdeps: - [ -x "$(GOVENDOR)" ] || $(GO) get -u github.com/kardianos/govendor + [ -x "$(GOVENDOR)" ] || go get -u github.com/kardianos/govendor govendor list +e | grep '^ e ' && { echo "Found missing packages. Please run 'govendor add +e'"; exit 1; } || : echo # vendorfmt ensures that the vendor/vendor.json file is formatted correctly vendorfmt: - [ -x "$(VENDORFMT)" ] || $(GO) get -u github.com/magiconair/vendorfmt/cmd/vendorfmt + [ -x "$(VENDORFMT)" ] || go get -u github.com/magiconair/vendorfmt/cmd/vendorfmt vendorfmt # gofmt runs gofmt on the code @@ -73,15 +69,11 @@ gofmt: # linux builds a linux binary linux: - GOOS=linux GOARCH=amd64 $(GO) build -i -tags netgo $(GOFLAGS) + GOOS=linux GOARCH=amd64 go build -tags netgo $(GOFLAGS) # install runs go install install: - $(GO) install $(GOFLAGS) - -# vet runs go vet -vet: - $(GO) vet ./... + go install $(GOFLAGS) # pkg builds a fabio.tar.gz package with only fabio in it pkg: build test @@ -89,13 +81,13 @@ pkg: build test mkdir pkg tar czf pkg/fabio.tar.gz fabio -# release executes a release -# this is deprecated since I'm switching to goreleaser -release: preflight test - build/release.sh - -# ship executes the steps for a release with goreleaser -ship: preflight test gorelease homebrew docker-aliases +# release tags, builds and publishes a build with goreleaser +# +# Run this in sub-shells instead of dependencies so that +# later targets can pick up the new tag value. +release: + $(MAKE) tag + $(MAKE) preflight docker-test gorelease homebrew docker-aliases # preflight runs some checks before a release preflight: @@ -116,11 +108,6 @@ gorelease: homebrew: build/homebrew.sh $(LAST_TAG) -# docker builds the docker containers and publishes them -# this is deprecated since goreleaser should handle that -docker: - build/docker.sh $(VERSION)-$(GOVERSION) - # docker-aliases creates aliases for the docker containers # since goreleaser doesn't handle that properly yet docker-aliases: @@ -129,12 +116,34 @@ docker-aliases: docker push magiconair/fabio:$(VERSION)-$(GOVERSION) docker push magiconair/fabio:latest +# docker-test runs make test in a Docker container with +# pinned versions of the external dependencies +# +# We download the binaries outside the Docker build to +# cache the binaries and prevent repeated downloads since +# ADD downloads the file every time. +docker-test: + test -r consul_$(CI_CONSUL_VERSION)_linux_amd64.zip || \ + wget https://releases.hashicorp.com/consul/$(CI_CONSUL_VERSION)/consul_$(CI_CONSUL_VERSION)_linux_amd64.zip + test -r vault_$(CI_VAULT_VERSION)_linux_amd64.zip || \ + wget https://releases.hashicorp.com/vault/$(CI_VAULT_VERSION)/vault_$(CI_VAULT_VERSION)_linux_amd64.zip + test -r go$(CI_GO_VERSION).linux-amd64.tar.gz || \ + wget https://dl.google.com/go/go$(CI_GO_VERSION).linux-amd64.tar.gz + docker build \ + --build-arg consul_version=$(CI_CONSUL_VERSION) \ + --build-arg vault_version=$(CI_VAULT_VERSION) \ + --build-arg go_version=$(CI_GO_VERSION) \ + -t test-fabio \ + -f Dockerfile-test \ + . + docker run -it test-fabio make test + # codeship runs the CI on codeship codeship: go version go env - wget -O ~/consul.zip https://releases.hashicorp.com/consul/1.0.0/consul_1.0.0_linux_amd64.zip - wget -O ~/vault.zip https://releases.hashicorp.com/vault/0.8.3/vault_0.8.3_linux_amd64.zip + wget -O ~/consul.zip https://releases.hashicorp.com/consul/$(CI_CONSUL_VERSION)/consul_$(CI_CONSUL_VERSION)_linux_amd64.zip + wget -O ~/vault.zip https://releases.hashicorp.com/vault/$(CI_VAULT_VERSION)/vault_$(CI_VAULT_VERSION)_linux_amd64.zip unzip -o -d ~/bin ~/consul.zip unzip -o -d ~/bin ~/vault.zip vault --version @@ -143,8 +152,8 @@ codeship: # clean removes intermediate files clean: - $(GO) clean + go clean rm -rf pkg dist fabio find . -name '*.test' -delete -.PHONY: build buildpkg clean docker gofmt homebrew install linux pkg release test vendorfmt vet +.PHONY: all build checkdeps clean codeship gofmt gorelease help homebrew install linux pkg preflight release tag test vendorfmt diff --git a/README.md b/README.md index a3bda98bc..040c36da0 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,6 @@ Release License MIT Codeship CI Status - Travis CI Status Downloads Docker Pulls magiconair Docker Pulls fabiolb @@ -51,7 +50,7 @@ It supports ([Full feature list](https://fabiolb.net/feature/)) * [Raw TCP proxy](https://fabiolb.net/feature/tcp-proxy/) * [TCP+SNI proxy for full end-to-end TLS](https://fabiolb.net/feature/tcp-sni-proxy/) without decryption * [HTTPS upstream support](https://fabiolb.net/feature/https-upstream/) -* [Websockets](https://fabiolb.net/feature/websocket-support/) and +* [Websockets](https://fabiolb.net/feature/websockets/) and [SSE](https://fabiolb.net/feature/sse/) * [Dynamic reloading without restart](https://fabiolb.net/feature/dynamic-reloading/) * [Traffic shaping](https://fabiolb.net/feature/traffic-shaping/) for "blue/green" deployments, @@ -70,8 +69,8 @@ The full documentation is on [fabiolb.net](https://fabiolb.net/) 1. Install from source, [binary](https://github.com/fabiolb/fabio/releases), [Docker](https://hub.docker.com/r/fabiolb/fabio/) or [Homebrew](http://brew.sh). ```shell - # go 1.8 or higher is required - go get github.com/fabiolb/fabio (>= go1.8) + # go 1.9 or higher is required + go get github.com/fabiolb/fabio (>= go1.9) brew install fabio (OSX/macOS stable) brew install --devel fabio (OSX/macOS devel) @@ -125,7 +124,7 @@ urlprefix-:3306 proto=tcp # route external port 3306 ### Contributors This project exists thanks to all the people who contribute. [[Contribute](CONTRIBUTING.md)]. - + ### Backers diff --git a/build/build.sh b/build/build.sh deleted file mode 100755 index 110bf5c46..000000000 --- a/build/build.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e - -readonly prgdir=$(cd $(dirname $0); pwd) -readonly basedir=$(cd $prgdir/..; pwd) -v=$1 - -[[ -n "$v" ]] || read -p "Enter version (e.g. 1.0.4): " v -if [[ -z "$v" ]] ; then - echo "Usage: $0 [] (e.g. 1.0.4)" - exit 1 -fi - -go get -u github.com/mitchellh/gox -for go in go1.9.2; do - echo "Building fabio with ${go}" - gox -gocmd ~/${go}/bin/go -tags netgo -output "${basedir}/build/builds/fabio-${v}/fabio-${v}-${go}-{{.OS}}_{{.Arch}}" -done - -( cd ${basedir}/build/builds/fabio-${v} && shasum -a 256 fabio-${v}-* > fabio-${v}.sha256 ) -( cd ${basedir}/build/builds/fabio-${v} && gpg --output fabio-${v}.sha256.sig --detach-sig fabio-${v}.sha256 ) diff --git a/build/docker.sh b/build/docker.sh deleted file mode 100755 index 240ad13ec..000000000 --- a/build/docker.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -e -# -# docker.sh will build docker images from -# the versions provided on the command line. -# The binaries must already exist in the build/builds -# directory and are usually built with the build.sh -# or the release.sh script. The last specified -# version will be used as the 'latest' image. -# -# Example: -# -# build/docker.sh 1.1-go1.5.4 1.1-go1.6 -# -# will build three containers -# -# * fabiolb/fabio:1.1-go1.5.4 -# * fabiolb/fabio:1.1-go1.6.2 -# * fabiolb/fabio (which contains 1.1-go1.6.2) -# -if [[ $# = 0 ]]; then - echo "Usage: docker.sh <1.x-go1.x.x> <1.x-go1.x.y>" - exit 1 -fi - -for v in "$@" ; do - echo "Building docker image fabiolb/fabio:$v" - ( - cp dist/linuxamd64/fabio fabio - docker build -q -t fabiolb/fabio:${v} . - ) - docker tag fabiolb/fabio:$v magiconair/fabio:$v - docker tag fabiolb/fabio:$v magiconair/fabio:latest - docker tag fabiolb/fabio:$v fabiolb/fabio:latest -done - -docker images | grep '/fabio' | egrep "($v|latest)" - -read -p "Push docker images? (y/N) " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "Not pushing images. Exiting" - exit 0 -fi - -echo "Pushing images..." -docker push fabiolb/fabio:$v -docker push fabiolb/fabio:latest -docker push magiconair/fabio:$v -docker push magiconair/fabio:latest diff --git a/build/release.sh b/build/release.sh deleted file mode 100755 index ce39144e4..000000000 --- a/build/release.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -e -# -# Script for replacing the version number -# in main.go, committing and tagging the code - -readonly prgdir=$(cd $(dirname $0); pwd) -readonly basedir=$(cd $prgdir/..; pwd) -v=$1 - -[[ -n "$v" ]] || read -p "Enter version (e.g. 1.0.4): " v -if [[ -z "$v" ]]; then - echo "Usage: $0 (e.g. 1.0.4)" - exit 1 -fi - -grep -q "$v" CHANGELOG.md || echo "CHANGELOG.md not updated" - -read -p "Release fabio version $v? (y/N) " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - exit 1 -fi - -sed -i '' -e "s|^var version .*$|var version = \"$v\"|" $basedir/main.go -git add $basedir/main.go -git commit -S -m "Release v$v" -git commit -S --amend -git tag -s v$v -m "Tag v${v}" - -$prgdir/build.sh $v diff --git a/config/config.go b/config/config.go index ca3f179ed..d6950e915 100644 --- a/config/config.go +++ b/config/config.go @@ -127,21 +127,23 @@ type File struct { } type Consul struct { - Addr string - Scheme string - Token string - KVPath string - NoRouteHTMLPath string - TagPrefix string - Register bool - ServiceAddr string - ServiceName string - ServiceTags []string - ServiceStatus []string - CheckInterval time.Duration - CheckTimeout time.Duration - CheckScheme string - CheckTLSSkipVerify bool + Addr string + Scheme string + Token string + KVPath string + NoRouteHTMLPath string + TagPrefix string + Register bool + ServiceAddr string + ServiceName string + ServiceTags []string + ServiceStatus []string + CheckInterval time.Duration + CheckTimeout time.Duration + CheckScheme string + CheckTLSSkipVerify bool + CheckDeregisterCriticalServiceAfter string + ChecksRequired string } type FastCGI struct { diff --git a/config/default.go b/config/default.go index 14b3f0195..0a10e036e 100644 --- a/config/default.go +++ b/config/default.go @@ -47,18 +47,20 @@ var defaultConfig = &Config{ Registry: Registry{ Backend: "consul", Consul: Consul{ - Addr: "localhost:8500", - Scheme: "http", - KVPath: "/fabio/config", - NoRouteHTMLPath: "/fabio/noroute.html", - TagPrefix: "urlprefix-", - Register: true, - ServiceAddr: ":9998", - ServiceName: "fabio", - ServiceStatus: []string{"passing"}, - CheckInterval: time.Second, - CheckTimeout: 3 * time.Second, - CheckScheme: "http", + Addr: "localhost:8500", + Scheme: "http", + KVPath: "/fabio/config", + NoRouteHTMLPath: "/fabio/noroute.html", + TagPrefix: "urlprefix-", + Register: true, + ServiceAddr: ":9998", + ServiceName: "fabio", + ServiceStatus: []string{"passing"}, + CheckInterval: time.Second, + CheckTimeout: 3 * time.Second, + CheckScheme: "http", + CheckDeregisterCriticalServiceAfter: "90m", + ChecksRequired: "one", }, Timeout: 10 * time.Second, Retry: 500 * time.Millisecond, diff --git a/config/load.go b/config/load.go index 3b98b6739..1d2c15e9b 100644 --- a/config/load.go +++ b/config/load.go @@ -175,7 +175,9 @@ func load(cmdline, environ, envprefix []string, props *properties.Properties) (c f.StringSliceVar(&cfg.Registry.Consul.ServiceStatus, "registry.consul.service.status", defaultConfig.Registry.Consul.ServiceStatus, "valid service status values") f.DurationVar(&cfg.Registry.Consul.CheckInterval, "registry.consul.register.checkInterval", defaultConfig.Registry.Consul.CheckInterval, "service check interval") f.DurationVar(&cfg.Registry.Consul.CheckTimeout, "registry.consul.register.checkTimeout", defaultConfig.Registry.Consul.CheckTimeout, "service check timeout") - f.BoolVar(&cfg.Registry.Consul.CheckTLSSkipVerify, "registry.consul.register.checkTLSSkipVerify", defaultConfig.Registry.Consul.CheckTLSSkipVerify, "service check TLS verifcation") + f.BoolVar(&cfg.Registry.Consul.CheckTLSSkipVerify, "registry.consul.register.checkTLSSkipVerify", defaultConfig.Registry.Consul.CheckTLSSkipVerify, "service check TLS verification") + f.StringVar(&cfg.Registry.Consul.CheckDeregisterCriticalServiceAfter, "registry.consul.register.checkDeregisterCriticalServiceAfter", defaultConfig.Registry.Consul.CheckDeregisterCriticalServiceAfter, "critical service deregistration timeout") + f.StringVar(&cfg.Registry.Consul.ChecksRequired, "registry.consul.checksRequired", defaultConfig.Registry.Consul.ChecksRequired, "number of checks which must pass: one or all") f.IntVar(&cfg.Runtime.GOGC, "runtime.gogc", defaultConfig.Runtime.GOGC, "sets runtime.GOGC") f.IntVar(&cfg.Runtime.GOMAXPROCS, "runtime.gomaxprocs", defaultConfig.Runtime.GOMAXPROCS, "sets runtime.GOMAXPROCS") f.StringVar(&cfg.UI.Access, "ui.access", defaultConfig.UI.Access, "access mode, one of [ro, rw]") diff --git a/config/load_test.go b/config/load_test.go index a58745f81..bf7c9f288 100644 --- a/config/load_test.go +++ b/config/load_test.go @@ -425,6 +425,13 @@ func TestLoad(t *testing.T) { return cfg }, }, + { + args: []string{"-proxy.gzip.contenttype", "^(text/.*|application/(javascript|json|font-woff|xml)|.*\\+(json|xml))(;.*)?$"}, + cfg: func(cfg *Config) *Config { + cfg.Proxy.GZIPContentTypes = regexp.MustCompile(`^(text/.*|application/(javascript|json|font-woff|xml)|.*\+(json|xml))(;.*)?$`) + return cfg + }, + }, { args: []string{"-proxy.log.routes", "foobar"}, cfg: func(cfg *Config) *Config { diff --git a/docs/content/cfg/_index.md b/docs/content/cfg/_index.md index 0b0e777a7..4d2a4791b 100644 --- a/docs/content/cfg/_index.md +++ b/docs/content/cfg/_index.md @@ -24,13 +24,16 @@ Add a route for a service `svc` for the `src` (e.g. `/path` or `:port`) to a `ds `route add [ weight ][ tags ",,..."][ opts "k1=v1 k2=v2 ..."]` -Option | Description --------------------- | ----------- -`strip=/path` | Forward `/path/to/file` as `/to/file` -`proto=tcp` | Upstream service is TCP, `dst` must be `:port` -`proto=https` | Upstream service is HTTPS -`tlsskipverify=true` | Disable TLS cert validation for HTTPS upstream -`host=name` | Set the `Host` header to `name`. If `name == 'dst'` then the `Host` header will be set to the registered upstream host name +Option | Description +------------------------------------------ | ----------- +`allow=ip:10.0.0.0/8,ip:fe80::/10` | Restrict access to source addresses within the `10.0.0.0/8` or `fe80::/10` CIDR mask. All other requests will be denied. +`deny=ip:10.0.0.0/8,ip:fe80::1234` | Deny requests that source from the `10.0.0.0/8` CIDR mask or `fe80::1234`. All other requests will be allowed. +`strip=/path` | Forward `/path/to/file` as `/to/file` +`proto=tcp` | Upstream service is TCP, `dst` must be `:port` +`proto=https` | Upstream service is HTTPS +`tlsskipverify=true` | Disable TLS cert validation for HTTPS upstream +`host=name` | Set the `Host` header to `name`. If `name == 'dst'` then the `Host` header will be set to the registered upstream host name +`register=name` | Register fabio as new service `name`. Useful for registering hostnames for host specific routes. ##### Example diff --git a/docs/content/deploy/amazon-api-gw.md b/docs/content/deploy/amazon-api-gw.md index 150cafbd4..c7eb64f3b 100644 --- a/docs/content/deploy/amazon-api-gw.md +++ b/docs/content/deploy/amazon-api-gw.md @@ -35,4 +35,4 @@ generated certificate you need to configure the `aws.apigw.cert.cn` as follows: `api-gw-cert.pem` is the certificate generated in the AWS Management Console. `your/cert.pem` and `your/key.pem` is the certificate/key pair for the HTTPS certificate. Since the Amazon API Gateway certificates don't have the `CA` flag set fabio needs to trust them for the client certificate authentication to work. Otherwise, you will get an `TLS handshake error: failed to verify client's certificate`. See [Issue 108](/eBay/fabio/issues/108) for details. -**Note:** The `aws.apigw.cert.cn` parameter will not be supported in version 1.2 and later which support dynamic certificate stores. You will have to add the `caupgcn=ApiGateway` parameter to the certificate source configuration instead. See [Certificate Stores](/#certificate-stores) for more detail. +**Note:** The `aws.apigw.cert.cn` parameter will not be supported in version 1.2 and later which support dynamic certificate stores. You will have to add the `caupgcn=ApiGateway` parameter to the certificate source configuration instead. See [Certificate Stores](/feature/certificate-stores/) for more detail. diff --git a/docs/content/feature/_index.md b/docs/content/feature/_index.md index 588074fa6..0a6a28984 100644 --- a/docs/content/feature/_index.md +++ b/docs/content/feature/_index.md @@ -6,6 +6,7 @@ weight: 200 The following list provides a list of features supported by fabio. * [Access Logging](/feature/access-logging/) - customizable access logs + * [Access Control](/feature/access-control/) - route specific access control * [Certificate Stores](/feature/certificate-stores/) - dynamic certificate stores like file system, HTTP server, [Consul](https://consul.io/) and [Vault](https://vaultproject.io/) * [Compression](/feature/compression/) - GZIP compression for HTTP responses * [Docker Support](/feature/docker/) - Official Docker image, Registrator and Docker Compose example diff --git a/docs/content/feature/access-control.md b/docs/content/feature/access-control.md new file mode 100644 index 000000000..90a4a195e --- /dev/null +++ b/docs/content/feature/access-control.md @@ -0,0 +1,54 @@ +--- +title: "Access Control" +since: "1.5.8" +--- + +fabio supports basic ip centric access control per route. You may +specify one of `allow` or `deny` options per route to control access. +Currently only source ip control is available. + + + +To allow access to a route from clients within the `192.168.1.0/24` +and `fe80::/10` subnet you would add the following option: + +``` +allow=ip:192.168.1.0/24,ip:fe80::/10 +``` + +With this specified only clients sourced from those two subnets will +be allowed. All other requests to that route will be denied. + + +Inversely, to deny a specific set of clients you can use the +following option syntax: + +``` +deny=ip:fe80::1234,100.123.0.0/16 +``` + +With this configuration access will be denied to any clients with +the `fe80::1234` address or coming from the `100.123.0.0/16` network. + +Single host addresses (addresses without a prefix) will have a +`/32` prefix, for IPv4, or a `/128` prefix, for IPv6, added automatically. +That means `1.2.3.4` is equivalent to `1.2.3.4/32` and `fe80::1234` +is equivalent to `fe80::1234/128` when specifying +address blocks for `allow` or `deny` rules. + +The source ip used for validation against the defined ruleset is +taken from information available in the request. + +For `HTTP` requests the client `RemoteAddr` is always validated +followed by the first element of the `X-Forwarded-For` header, if +present. When either of these elements match an `allow` the request +will be allowed; similarly when either element matches a `deny` the +request will be denied. + +For `TCP` requests the source address of the network socket +is used as the sole paramater for validation. + +If the inbound connection uses the [PROXY protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt) +to transmit the true source address of the client then it will +be used for both `HTTP` and `TCP` connections for validating access. + diff --git a/docs/content/feature/tcp-proxy.md b/docs/content/feature/tcp-proxy.md index db01fc694..0957a9c7c 100644 --- a/docs/content/feature/tcp-proxy.md +++ b/docs/content/feature/tcp-proxy.md @@ -12,7 +12,7 @@ Consul. In addition, fabio needs to be configured to listen on that port: fabio -proxy.addr ':1234;proto=tcp' ``` -TCP proxy support can be combined with [Certificate Stores](./certificate-stores) to provide TLS termination on fabio. +TCP proxy support can be combined with [Certificate Stores](/feature/certificate-stores/) to provide TLS termination on fabio. ``` fabio -proxy.cs 'cs=ssl;type=path;path=/etc/ssl' -proxy.addr ':1234;proto=tcp;cs=ssl' diff --git a/docs/content/ref/proxy.matcher.md b/docs/content/ref/proxy.matcher.md index 5051dea8d..f029f29ef 100644 --- a/docs/content/ref/proxy.matcher.md +++ b/docs/content/ref/proxy.matcher.md @@ -8,6 +8,17 @@ title: "proxy.matcher" * `prefix`: prefix matching * `glob`: glob matching +When `prefix` matching is enabled then the route path must be a +prefix of the request URI, e.g. `/foo` matches `/foo`, `/foot` but +not `/fo`. + +When `glob` matching is enabled the route is evaluated according to +globbing rules provided by the Go [`path.Match`](https://golang.org/pkg/path/#Match) +function. + +For example, `/foo*` matches `/foo`, `/fool` and `/fools`. Also, `/foo/*/bar` +matches `/foo/x/bar`. + The default is proxy.matcher = prefix diff --git a/docs/content/ref/registry.consul.checksRequired.md b/docs/content/ref/registry.consul.checksRequired.md new file mode 100644 index 000000000..4416a75b8 --- /dev/null +++ b/docs/content/ref/registry.consul.checksRequired.md @@ -0,0 +1,15 @@ +--- +title: "registry.consul.checksRequired" +--- + +`registry.consul.checksRequired` configures how many health checks +must pass in order for fabio to consider a service available. + +Possible values are: + +* `one`: at least one health check must pass +* `all`: all health checks must pass + +The default is + + registry.consul.checksRequired = one diff --git a/docs/content/ref/registry.consul.noroutehtmlpath.md b/docs/content/ref/registry.consul.noroutehtmlpath.md index 4d5f6eee0..307a2a890 100644 --- a/docs/content/ref/registry.consul.noroutehtmlpath.md +++ b/docs/content/ref/registry.consul.noroutehtmlpath.md @@ -8,4 +8,4 @@ The consul KV path is watched for changes. The default is - registry.consul.noroutehtmlpath = /fabio/noroutes.html + registry.consul.noroutehtmlpath = /fabio/noroute.html diff --git a/fabio.properties b/fabio.properties index 5b302a3e5..dae2caab0 100644 --- a/fabio.properties +++ b/fabio.properties @@ -441,7 +441,7 @@ # # A typical example is # -# proxy.gzip.contenttype = ^(text/.*|application/(javascript|json|font-woff|xml)|.*\+(json|xml))(;.*)?$ +# proxy.gzip.contenttype = ^(text/.*|application/(javascript|json|font-woff|xml)|.*\\+(json|xml))(;.*)?$ # # The default is # @@ -624,7 +624,7 @@ # # The default is # -# registry.consul.noroutehtmlpath = /fabio/noroutes.html +# registry.consul.noroutehtmlpath = /fabio/noroute.html # registry.consul.service.status configures the valid service status # values for services included in the routing table. @@ -718,6 +718,31 @@ # registry.consul.register.checkTLSSkipVerify = false +# registry.consul.register.checkDeregisterCriticalServiceAfter configures +# automatic deregistration of a service after the health check is critical for +# this length of time. +# +# Fabio registers an http health check on http(s)://${ui.addr}/health +# and this value tells consul to deregister the associated service if the check +# is critical for the specified duration. +# +# The default is +# +# registry.consul.register.checkDeregisterCriticalServiceAfter = 90m + + +# registry.consul.checksRequired configures how many health checks +# must pass in order for fabio to consider a service available. +# +# Possible values are: +# one: at least one health check must pass +# all: all health checks must pass +# +# The default is +# +# registry.consul.checksRequired = one + + # metrics.target configures the backend the metrics values are # sent to. # diff --git a/main.go b/main.go index 859beed6e..4edf5a0e2 100644 --- a/main.go +++ b/main.go @@ -43,7 +43,7 @@ import ( // It is also set by the linker when fabio // is built via the Makefile or the build/docker.sh // script to ensure the correct version nubmer -var version = "1.5.6" +var version = "1.5.9" var shuttingDown int32 @@ -110,7 +110,7 @@ func main() { if registry.Default == nil { return } - registry.Default.Deregister() + registry.Default.DeregisterAll() }) // init metrics early since that create the global metric registries @@ -362,7 +362,7 @@ func initBackend(cfg *config.Config) { } if err == nil { - if err = registry.Default.Register(); err == nil { + if err = registry.Default.Register(nil); err == nil { return } } @@ -404,6 +404,12 @@ func watchBackend(cfg *config.Config, first chan bool) { continue } + aliases, err := route.ParseAliases(next) + if err != nil { + log.Printf("[WARN]: %s", err) + } + registry.Default.Register(aliases) + t, err := route.NewTable(next) if err != nil { log.Printf("[WARN] %s", err) diff --git a/proxy/http_handler.go b/proxy/http_handler.go index 18281eecd..cfde275eb 100644 --- a/proxy/http_handler.go +++ b/proxy/http_handler.go @@ -24,20 +24,6 @@ func newHTTPProxy(target *url.URL, tr http.RoundTripper, flush time.Duration) ht } }, FlushInterval: flush, - Transport: &transport{tr, nil, nil}, + Transport: tr, } } - -// transport executes the roundtrip and captures the response. It is not -// safe for multiple or concurrent use since it only captures a single -// response. -type transport struct { - http.RoundTripper - resp *http.Response - err error -} - -func (t *transport) RoundTrip(r *http.Request) (*http.Response, error) { - t.resp, t.err = t.RoundTripper.RoundTrip(r) - return t.resp, t.err -} diff --git a/proxy/http_integration_test.go b/proxy/http_integration_test.go index 42ad53e9d..c1ae5a277 100644 --- a/proxy/http_integration_test.go +++ b/proxy/http_integration_test.go @@ -14,6 +14,7 @@ import ( "os" "regexp" "sort" + "strconv" "strings" "testing" "time" @@ -119,6 +120,35 @@ func TestProxySTSHeader(t *testing.T) { } } +func TestProxyChecksHeaderForAccessRules(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "OK") + })) + defer server.Close() + + proxy := httptest.NewServer(&HTTPProxy{ + Config: config.Proxy{}, + Transport: http.DefaultTransport, + Lookup: func(r *http.Request) *route.Target { + tgt := &route.Target{ + URL: mustParse(server.URL), + Opts: map[string]string{"allow": "ip:127.0.0.0/8,ip:fe80::/10,ip:::1"}, + } + tgt.ProcessAccessRules() + return tgt + }, + }) + defer proxy.Close() + + req, _ := http.NewRequest("GET", proxy.URL, nil) + req.Header.Set("X-Forwarded-For", "1.2.3.4") + resp, _ := mustDo(req) + + if got, want := resp.StatusCode, http.StatusForbidden; got != want { + t.Errorf("got %v want %v", got, want) + } +} + func TestProxyNoRouteHTML(t *testing.T) { want := "503" noroute.SetHTML(want) @@ -264,7 +294,7 @@ func TestRedirect(t *testing.T) { {req: "/", wantCode: 301, wantLoc: "http://a.com/"}, {req: "/aaa/bbb", wantCode: 301, wantLoc: "http://a.com/aaa/bbb"}, {req: "/foo", wantCode: 301, wantLoc: "http://a.com/abc"}, - {req: "/bar", wantCode: 302, wantLoc: "http://b.com"}, + {req: "/bar", wantCode: 302, wantLoc: "http://b.com/"}, {req: "/bar/aaa", wantCode: 302, wantLoc: "http://b.com/aaa"}, } @@ -286,6 +316,24 @@ func TestRedirect(t *testing.T) { } func TestProxyLogOutput(t *testing.T) { + t.Run("uncompressed response", func(t *testing.T) { + testProxyLogOutput(t, 73, config.Proxy{}) + }) + t.Run("compression enabled but no match", func(t *testing.T) { + testProxyLogOutput(t, 73, config.Proxy{ + GZIPContentTypes: regexp.MustCompile(`^$`), + }) + }) + t.Run("compression enabled and active", func(t *testing.T) { + testProxyLogOutput(t, 28, config.Proxy{ + GZIPContentTypes: regexp.MustCompile(`.*`), + }) + }) +} + +func testProxyLogOutput(t *testing.T, bodySize int, cfg config.Proxy) { + t.Helper() + // build a format string from all log fields and one header field fields := []string{"header.X-Foo:$header.X-Foo"} for _, k := range logger.Fields { @@ -302,7 +350,7 @@ func TestProxyLogOutput(t *testing.T) { // create an upstream server server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, "foo") + fmt.Fprint(w, "foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo") })) defer server.Close() @@ -360,7 +408,7 @@ func TestProxyLogOutput(t *testing.T) { "request_scheme:http", "request_uri:/foo?x=y", "request_url:http://example.com/foo?x=y", - "response_body_size:3", + "response_body_size:" + strconv.Itoa(bodySize), "response_status:200", "response_time_ms:1.111", "response_time_ns:1.111111111", diff --git a/proxy/http_proxy.go b/proxy/http_proxy.go index 605b38fc1..e0eb3010c 100644 --- a/proxy/http_proxy.go +++ b/proxy/http_proxy.go @@ -1,11 +1,12 @@ package proxy import ( + "bufio" "crypto/tls" + "errors" "io" "net" "net/http" - "net/http/httputil" "net/url" "strconv" "strings" @@ -85,6 +86,11 @@ func (p *HTTPProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + if t.AccessDeniedHTTP(r) { + http.Error(w, "access denied", http.StatusForbidden) + return + } + // build the request url since r.URL will get modified // by the reverse proxy and contains only the RequestURI anyway requestURL := &url.URL{ @@ -94,9 +100,8 @@ func (p *HTTPProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { RawQuery: r.URL.RawQuery, } - if t.RedirectCode != 0 { - redirectURL := t.GetRedirectURL(requestURL) - http.Redirect(w, r, redirectURL.String(), t.RedirectCode) + if t.RedirectCode != 0 && t.RedirectURL != nil { + http.Redirect(w, r, t.RedirectURL.String(), t.RedirectCode) if t.Timer != nil { t.Timer.Update(0) } @@ -171,11 +176,11 @@ func (p *HTTPProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { case upgrade == "websocket" || upgrade == "Websocket": r.URL = targetURL if targetURL.Scheme == "https" || targetURL.Scheme == "wss" { - h = newRawProxy(targetURL.Host, func(network, address string) (net.Conn, error) { + h = newWSHandler(targetURL.Host, func(network, address string) (net.Conn, error) { return tls.Dial(network, address, tr.(*http.Transport).TLSClientConfig) }) } else { - h = newRawProxy(targetURL.Host, net.Dial) + h = newWSHandler(targetURL.Host, net.Dial) } case accept == "text/event-stream": @@ -197,7 +202,8 @@ func (p *HTTPProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { } start := timeNow() - h.ServeHTTP(w, r) + rw := &responseWriter{w: w} + h.ServeHTTP(rw, r) end := timeNow() dur := end.Sub(start) @@ -207,28 +213,22 @@ func (p *HTTPProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { if t.Timer != nil { t.Timer.Update(dur) } - - // get response and update metrics - rp, ok := h.(*httputil.ReverseProxy) - if !ok { - return - } - rpt, ok := rp.Transport.(*transport) - if !ok { + if rw.code <= 0 { return } - if rpt.resp == nil { - return - } - metrics.DefaultRegistry.GetTimer(key(rpt.resp.StatusCode)).Update(dur) + + metrics.DefaultRegistry.GetTimer(key(rw.code)).Update(dur) // write access log if p.Logger != nil { p.Logger.Log(&logger.Event{ - Start: start, - End: end, - Request: r, - Response: rpt.resp, + Start: start, + End: end, + Request: r, + Response: &http.Response{ + StatusCode: rw.code, + ContentLength: int64(rw.size), + }, RequestURL: requestURL, UpstreamAddr: targetURL.Host, UpstreamService: t.Service, @@ -242,3 +242,36 @@ func key(code int) string { b = strconv.AppendInt(b, int64(code), 10) return string(b) } + +// responseWriter wraps an http.ResponseWriter to capture the status code and +// the size of the response. It also implements http.Hijacker to forward +// hijacking the connection to the wrapped writer if supported. +type responseWriter struct { + w http.ResponseWriter + code int + size int +} + +func (rw *responseWriter) Header() http.Header { + return rw.w.Header() +} + +func (rw *responseWriter) Write(b []byte) (int, error) { + n, err := rw.w.Write(b) + rw.size += n + return n, err +} + +func (rw *responseWriter) WriteHeader(statusCode int) { + rw.w.WriteHeader(statusCode) + rw.code = statusCode +} + +var errNoHijacker = errors.New("not a hijacker") + +func (rw *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + if hj, ok := rw.w.(http.Hijacker); ok { + return hj.Hijack() + } + return nil, nil, errNoHijacker +} diff --git a/proxy/http_raw_handler.go b/proxy/http_raw_handler.go deleted file mode 100644 index a2cf52e30..000000000 --- a/proxy/http_raw_handler.go +++ /dev/null @@ -1,67 +0,0 @@ -package proxy - -import ( - "io" - "log" - "net" - "net/http" - - "github.com/fabiolb/fabio/metrics" -) - -// conn measures the number of open web socket connections -var conn = metrics.DefaultRegistry.GetCounter("ws.conn") - -type dialFunc func(network, address string) (net.Conn, error) - -// newRawProxy returns an HTTP handler which forwards data between -// an incoming and outgoing TCP connection including the original request. -// This handler establishes a new outgoing connection per request. -func newRawProxy(host string, dial dialFunc) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - conn.Inc(1) - defer func() { conn.Inc(-1) }() - - hj, ok := w.(http.Hijacker) - if !ok { - http.Error(w, "not a hijacker", http.StatusInternalServerError) - return - } - - in, _, err := hj.Hijack() - if err != nil { - log.Printf("[ERROR] Hijack error for %s. %s", r.URL, err) - http.Error(w, "hijack error", http.StatusInternalServerError) - return - } - defer in.Close() - - out, err := dial("tcp", host) - if err != nil { - log.Printf("[ERROR] WS error for %s. %s", r.URL, err) - http.Error(w, "error contacting backend server", http.StatusInternalServerError) - return - } - defer out.Close() - - err = r.Write(out) - if err != nil { - log.Printf("[ERROR] Error copying request for %s. %s", r.URL, err) - http.Error(w, "error copying request", http.StatusInternalServerError) - return - } - - errc := make(chan error, 2) - cp := func(dst io.Writer, src io.Reader) { - _, err := io.Copy(dst, src) - errc <- err - } - - go cp(out, in) - go cp(in, out) - err = <-errc - if err != nil && err != io.EOF { - log.Printf("[INFO] WS error for %s. %s", r.URL, err) - } - }) -} diff --git a/proxy/tcp/sni_proxy.go b/proxy/tcp/sni_proxy.go index b75d06705..d9920876c 100644 --- a/proxy/tcp/sni_proxy.go +++ b/proxy/tcp/sni_proxy.go @@ -1,6 +1,7 @@ package tcp import ( + "bufio" "io" "log" "net" @@ -41,20 +42,40 @@ func (p *SNIProxy) ServeTCP(in net.Conn) error { p.Conn.Inc(1) } - // capture client hello - data := make([]byte, 1024) - n, err := in.Read(data) + tlsReader := bufio.NewReader(in) + tlsHeaders, err := tlsReader.Peek(9) if err != nil { + log.Print("[DEBUG] tcp+sni: TLS handshake failed (failed to peek data)") if p.ConnFail != nil { p.ConnFail.Inc(1) } return err } - data = data[:n] - host, ok := readServerName(data) + bufferSize, err := clientHelloBufferSize(tlsHeaders) + if err != nil { + log.Printf("[DEBUG] tcp+sni: TLS handshake failed (%s)", err) + if p.ConnFail != nil { + p.ConnFail.Inc(1) + } + return err + } + + data := make([]byte, bufferSize) + _, err = io.ReadFull(tlsReader, data) + if err != nil { + log.Printf("[DEBUG] tcp+sni: TLS handshake failed (%s)", err) + if p.ConnFail != nil { + p.ConnFail.Inc(1) + } + return err + } + + // readServerName wants only the handshake message so ignore the first + // 5 bytes which is the TLS record header + host, ok := readServerName(data[5:]) if !ok { - log.Print("[DEBUG] tcp+sni: TLS handshake failed") + log.Print("[DEBUG] tcp+sni: TLS handshake failed (unable to parse client hello)") if p.ConnFail != nil { p.ConnFail.Inc(1) } @@ -78,6 +99,10 @@ func (p *SNIProxy) ServeTCP(in net.Conn) error { } addr := t.URL.Host + if t.AccessDeniedTCP(in) { + return nil + } + out, err := net.DialTimeout("tcp", addr, p.DialTimeout) if err != nil { log.Print("[WARN] tcp+sni: cannot connect to upstream ", addr) @@ -88,8 +113,8 @@ func (p *SNIProxy) ServeTCP(in net.Conn) error { } defer out.Close() - // copy client hello - n, err = out.Write(data) + // write the data already read from the connection + n, err := out.Write(data) if err != nil { log.Print("[WARN] tcp+sni: copy client hello failed. ", err) if p.ConnFail != nil { diff --git a/proxy/tcp/tcp_proxy.go b/proxy/tcp/tcp_proxy.go index 5b14cc950..17bf69572 100644 --- a/proxy/tcp/tcp_proxy.go +++ b/proxy/tcp/tcp_proxy.go @@ -48,6 +48,10 @@ func (p *Proxy) ServeTCP(in net.Conn) error { } addr := t.URL.Host + if t.AccessDeniedTCP(in) { + return nil + } + out, err := net.DialTimeout("tcp", addr, p.DialTimeout) if err != nil { log.Print("[WARN] tcp: cannot connect to upstream ", addr) diff --git a/proxy/tcp/tls_clienthello.go b/proxy/tcp/tls_clienthello.go index 6988c307a..69f9fb468 100644 --- a/proxy/tcp/tls_clienthello.go +++ b/proxy/tcp/tls_clienthello.go @@ -1,73 +1,65 @@ package tcp -// record types -const ( - handshakeRecord = 0x16 - clientHelloType = 0x01 -) - -// readServerName returns the server name from a TLS ClientHello message which -// has the server_name extension (SNI). ok is set to true if the ClientHello -// message was parsed successfully. If the server_name extension was not set -// and empty string is returned as serverName. -func readServerName(data []byte) (serverName string, ok bool) { - if m, ok := readClientHello(data); ok { - return m.serverName, true - } - return "", false -} - -// readClientHello -func readClientHello(data []byte) (m *clientHelloMsg, ok bool) { - if len(data) < 9 { - // println("buf too short") - return nil, false - } +import "errors" +// Determines the required size of a buffer large enough to hold +// a client hello message including the tls record header and the +// handshake message header. +// The function requires at least the first 9 bytes of the tls conversation +// in "data". +// An error is returned if the data does not follow the +// specification (https://tools.ietf.org/html/rfc5246) or if the client hello +// is fragmented over multiple records. +func clientHelloBufferSize(data []byte) (int, error) { // TLS record header // ----------------- // byte 0: rec type (should be 0x16 == Handshake) // byte 1-2: version (should be 0x3000 < v < 0x3003) // byte 3-4: rec len - recType := data[0] - if recType != handshakeRecord { - // println("no handshake ") - return nil, false + if len(data) < 9 { + return 0, errors.New("At least 9 bytes required to determine client hello length") } - recLen := int(data[3])<<8 | int(data[4]) - if recLen == 0 || recLen > len(data)-5 { - // println("rec too short") - return nil, false + if data[0] != 0x16 { + return 0, errors.New("Not a TLS handshake") + } + + recordLength := int(data[3])<<8 | int(data[4]) + if recordLength <= 0 || recordLength > 16384 { + return 0, errors.New("Invalid TLS record length") } // Handshake record header // ----------------------- // byte 5: hs msg type (should be 0x01 == client_hello) // byte 6-8: hs msg len - hsType := data[5] - if hsType != clientHelloType { - // println("no client_hello") - return nil, false + if data[5] != 0x01 { + return 0, errors.New("Not a client hello") } - hsLen := int(data[6])<<16 | int(data[7])<<8 | int(data[8]) - if hsLen == 0 || hsLen > len(data)-9 { - // println("handshake rec too short") - return nil, false + handshakeLength := int(data[6])<<16 | int(data[7])<<8 | int(data[8]) + if handshakeLength <= 0 || handshakeLength > recordLength-4 { + return 0, errors.New("Invalid client hello length (fragmentation not implemented)") } - // byte 9- : client hello msg - // - // m.unmarshal parses the entire handshake message and - // not just the client hello. Therefore, we need to pass - // data from byte 5 instead of byte 9. (see comment below) - m = new(clientHelloMsg) - if !m.unmarshal(data[5:]) { - // println("client_hello unmarshal failed") - return nil, false + return handshakeLength + 9, nil //9 for the header bytes +} + +// readServerName returns the server name from a TLS ClientHello message which +// has the server_name extension (SNI). ok is set to true if the ClientHello +// message was parsed successfully. If the server_name extension was not set +// an empty string is returned as serverName. +// clientHelloHandshakeMsg must contain the full client hello handshake +// message including the 4 byte header. +// See: https://www.ietf.org/rfc/rfc5246.txt +func readServerName(clientHelloHandshakeMsg []byte) (serverName string, ok bool) { + m := new(clientHelloMsg) + if !m.unmarshal(clientHelloHandshakeMsg) { + //println("client_hello unmarshal failed") + return "", false } - return m, true + + return m.serverName, true } // The code below is a verbatim copy from go1.7/src/crypto/tls/handshake_messages.go diff --git a/proxy/tcp/tls_clienthello_test.go b/proxy/tcp/tls_clienthello_test.go new file mode 100644 index 000000000..9c947614d --- /dev/null +++ b/proxy/tcp/tls_clienthello_test.go @@ -0,0 +1,162 @@ +package tcp + +import ( + "encoding/hex" + "testing" +) + +func TestClientHelloBufferSize(t *testing.T) { + tests := []struct { + name string + data []byte + size int + fail bool + }{ + { + name: "valid data", + // Largest possible client hello message + // |- 16384 -| |----- 16380 ----| + data: []byte{0x16, 0x03, 0x01, 0x40, 0x00, 0x01, 0x00, 0x3f, 0xfc}, + size: 16384 + 5, // max record length + record header + fail: false, + }, + { + name: "not enough data", + data: []byte{0x16, 0x03, 0x01, 0x40, 0x00, 0x01, 0x00, 0x3f}, + size: 0, + fail: true, + }, + { + name: "not a TLS record", + data: []byte{0x15, 0x03, 0x01, 0x01, 0xF4, 0x01, 0x00, 0x01, 0xeb}, + size: 0, + fail: true, + }, + + { + name: "TLS record too large", + // | max + 1 | + data: []byte{0x16, 0x03, 0x01, 0x40, 0x01, 0x01, 0x00, 0x3f, 0xfc}, + size: 0, + fail: true, + }, + + { + name: "TLS record length zero", + // |----------| + data: []byte{0x16, 0x03, 0x01, 0x00, 0x00, 0x01, 0x00, 0x3f, 0xfc}, + size: 0, + fail: true, + }, + + { + name: "Not a client hello", + // |----| + data: []byte{0x16, 0x03, 0x01, 0x40, 0x00, 0x02, 0x00, 0x3f, 0xfc}, + size: 0, + fail: true, + }, + + { + name: "Invalid handshake message record length", + // |----- 0 --------| + data: []byte{0x16, 0x03, 0x01, 0x40, 0x00, 0x01, 0x00, 0x00, 0x00}, + size: 0, + fail: true, + }, + + { + name: "Fragmentation (handshake message larger than record)", + // |- 500 ---| |----- 497 ------| + data: []byte{0x16, 0x03, 0x01, 0x01, 0xF4, 0x01, 0x00, 0x01, 0xf1}, + size: 0, + fail: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := clientHelloBufferSize(tt.data) + + if tt.fail && err == nil { + t.Fatal("expected error, got nil") + } else if !tt.fail && err != nil { + t.Fatalf("expected error to be nil, got %s", err) + } + + if want := tt.size; got != want { + t.Fatalf("want size %d, got %d", want, got) + } + }) + } +} + +func TestReadServerName(t *testing.T) { + tests := []struct { + name string + servername string + ok bool + data string //Hex string, decoded by test + }{ + { + // Client hello from: + // openssl s_client -connect google.com:443 -servername google.com + name: "valid client hello with server name", + servername: "google.com", + ok: true, + data: "0100014803032657cacce41598fa82e5b75061050bc31c5affdba106b8e7431852" + + "24af0fa1aa000098cc14cc13cc15c030c02cc028c024c014c00a00a3009f00" + + "6b006a00390038ff8500c400c3008800870081c032c02ec02ac026c00fc005" + + "009d003d003500c00084c02fc02bc027c023c013c00900a2009e0067004000" + + "33003200be00bd00450044c031c02dc029c025c00ec004009c003c002f00ba" + + "0041c011c007c00cc00200050004c012c00800160013c00dc003000a001500" + + "12000900ff010000870000000f000d00000a676f6f676c652e636f6d000b00" + + "0403000102000a003a0038000e000d0019001c000b000c001b00180009000a" + + "001a0016001700080006000700140015000400050012001300010002000300" + + "0f0010001100230000000d00260024060106020603efef0501050205030401" + + "04020403eeeeeded030103020303020102020203", + }, + { + // Client hello from: + // openssl s_client -connect google.com:443 + name: "valid client hello but no server name extension", + servername: "", + ok: true, + data: "0100013503036dfb09de7b16503dd1bb304dcbe54079913b65abf53de997f73b26c99e" + + "67ba28000098cc14cc13cc15c030c02cc028c024c014c00a00a3009f006b006a00" + + "390038ff8500c400c3008800870081c032c02ec02ac026c00fc005009d003d0035" + + "00c00084c02fc02bc027c023c013c00900a2009e006700400033003200be00bd00" + + "450044c031c02dc029c025c00ec004009c003c002f00ba0041c011c007c00cc002" + + "00050004c012c00800160013c00dc003000a00150012000900ff01000074000b00" + + "0403000102000a003a0038000e000d0019001c000b000c001b00180009000a001a" + + "00160017000800060007001400150004000500120013000100020003000f001000" + + "1100230000000d00260024060106020603efef050105020503040104020403eeee" + + "eded030103020303020102020203", + }, + { + name: "invalid client hello", + servername: "", + ok: false, + data: "0100014c5768656e2070656f706c652073617920746f206d653a20776f756c6420796f" + + "75207261746865722062652074686f75676874206f6620617320612066756e6e79" + + "206d616e206f72206120677265617420626f73733f204d7920616e737765722773" + + "20616c77617973207468652073616d652c20746f206d652c207468657927726520" + + "6e6f74206d757475616c6c79206578636c75736976652e2d204461766964204272" + + "656e74", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientHelloMsg, _ := hex.DecodeString(tt.data) + servername, ok := readServerName(clientHelloMsg) + if got, want := servername, tt.servername; got != want { + t.Fatalf("%s: got servername \"%s\" want \"%s\"", tt.name, got, want) + } + + if got, want := ok, tt.ok; got != want { + t.Fatalf("%s: got ok %t want %t", tt.name, got, want) + } + }) + } +} diff --git a/proxy/ws_handler.go b/proxy/ws_handler.go new file mode 100644 index 000000000..830f6c15e --- /dev/null +++ b/proxy/ws_handler.go @@ -0,0 +1,105 @@ +package proxy + +import ( + "bytes" + "io" + "log" + "net" + "net/http" + "strings" + "time" + + "github.com/fabiolb/fabio/metrics" +) + +// conn measures the number of open web socket connections +var conn = metrics.DefaultRegistry.GetCounter("ws.conn") + +type dialFunc func(network, address string) (net.Conn, error) + +// newWSHandler returns an HTTP handler which forwards data between +// an incoming and outgoing websocket connection. It checks whether +// the handshake was completed successfully before forwarding data +// between the client and server. +func newWSHandler(host string, dial dialFunc) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn.Inc(1) + defer func() { conn.Inc(-1) }() + + hj, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "not a hijacker", http.StatusInternalServerError) + return + } + + in, _, err := hj.Hijack() + if err != nil { + log.Printf("[ERROR] Hijack error for %s. %s", r.URL, err) + http.Error(w, "hijack error", http.StatusInternalServerError) + return + } + defer in.Close() + + out, err := dial("tcp", host) + if err != nil { + log.Printf("[ERROR] WS error for %s. %s", r.URL, err) + http.Error(w, "error contacting backend server", http.StatusInternalServerError) + return + } + defer out.Close() + + err = r.Write(out) + if err != nil { + log.Printf("[ERROR] Error copying request for %s. %s", r.URL, err) + http.Error(w, "error copying request", http.StatusInternalServerError) + return + } + + // read the initial response to check whether we get an HTTP/1.1 101 ... response + // to determine whether the handshake worked. + b := make([]byte, 1024) + if err := out.SetReadDeadline(time.Now().Add(time.Second)); err != nil { + log.Printf("[ERROR] Error setting read timeout for %s: %s", r.URL, err) + http.Error(w, "error setting read timeout", http.StatusInternalServerError) + return + } + + n, err := out.Read(b) + if err != nil { + log.Printf("[ERROR] Error reading handshake for %s: %s", r.URL, err) + http.Error(w, "error reading handshake", http.StatusInternalServerError) + return + } + + b = b[:n] + if m, err := in.Write(b); err != nil || n != m { + log.Printf("[ERROR] Error sending handshake for %s: %s", r.URL, err) + http.Error(w, "error sending handshake", http.StatusInternalServerError) + return + } + + // https://tools.ietf.org/html/rfc6455#section-1.3 + // The websocket server must respond with HTTP/1.1 101 on successful handshake + if !bytes.HasPrefix(b, []byte("HTTP/1.1 101")) { + firstLine := strings.SplitN(string(b), "\n", 1) + log.Printf("[INFO] Websocket upgrade failed for %s: %s", r.URL, firstLine) + http.Error(w, "websocket upgrade failed", http.StatusInternalServerError) + return + } + + out.SetReadDeadline(time.Time{}) + + errc := make(chan error, 2) + cp := func(dst io.Writer, src io.Reader) { + _, err := io.Copy(dst, src) + errc <- err + } + + go cp(out, in) + go cp(in, out) + err = <-errc + if err != nil && err != io.EOF { + log.Printf("[INFO] WS error for %s. %s", r.URL, err) + } + }) +} diff --git a/registry/backend.go b/registry/backend.go index 152737bff..d47990e39 100644 --- a/registry/backend.go +++ b/registry/backend.go @@ -2,10 +2,13 @@ package registry type Backend interface { // Register registers fabio as a service in the registry. - Register() error + Register(services []string) error - // Deregister removes the service registration for fabio. - Deregister() error + // Deregister removes all service registrations for fabio. + DeregisterAll() error + + // Deregister removes the given service registration for fabio. + Deregister(service string) error // ManualPaths returns the list of paths for which there // are overrides. diff --git a/registry/consul/backend.go b/registry/consul/backend.go index 792643532..0d40448ca 100644 --- a/registry/consul/backend.go +++ b/registry/consul/backend.go @@ -15,7 +15,7 @@ type be struct { c *api.Client dc string cfg *config.Consul - dereg chan bool + dereg map[string](chan bool) } func NewBackend(cfg *config.Consul) (registry.Backend, error) { @@ -36,25 +36,66 @@ func NewBackend(cfg *config.Consul) (registry.Backend, error) { return &be{c: c, dc: dc, cfg: cfg}, nil } -func (b *be) Register() error { - if !b.cfg.Register { - log.Printf("[INFO] consul: Not registering fabio in consul") - return nil +func (b *be) Register(services []string) error { + if b.dereg == nil { + b.dereg = make(map[string](chan bool)) } - service, err := serviceRegistration(b.cfg) - if err != nil { - return err + if b.cfg.Register { + services = append(services, b.cfg.ServiceName) + } + + // deregister unneeded services + for service := range b.dereg { + if stringInSlice(service, services) { + continue + } + err := b.Deregister(service) + if err != nil { + return err + } + } + + // register new services + for _, service := range services { + if b.dereg[service] != nil { + log.Printf("[DEBUG] %q already registered", service) + continue + } + + serviceReg, err := serviceRegistration(b.cfg, service) + if err != nil { + return err + } + + b.dereg[service] = register(b.c, serviceReg) } - b.dereg = register(b.c, service) return nil } -func (b *be) Deregister() error { - if b.dereg != nil { - b.dereg <- true // trigger deregistration - <-b.dereg // wait for completion +func (b *be) Deregister(service string) error { + dereg := b.dereg[service] + if dereg == nil { + log.Printf("[WARN]: Attempted to deregister unknown service %q", service) + return nil + } + dereg <- true // trigger deregistration + <-dereg // wait for completion + delete(b.dereg, service) + + return nil +} + +func (b *be) DeregisterAll() error { + log.Printf("[DEBUG]: consul: Deregistering all registered aliases.") + for name, dereg := range b.dereg { + if dereg == nil { + continue + } + log.Printf("[INFO] consul: Deregistering %q", name) + dereg <- true // trigger deregistration + <-dereg // wait for completion } return nil } @@ -85,7 +126,7 @@ func (b *be) WatchServices() chan string { log.Printf("[INFO] consul: Using tag prefix %q", b.cfg.TagPrefix) svc := make(chan string) - go watchServices(b.c, b.cfg.TagPrefix, b.cfg.ServiceStatus, svc) + go watchServices(b.c, b.cfg, svc) return svc } @@ -93,7 +134,7 @@ func (b *be) WatchManual() chan string { log.Printf("[INFO] consul: Watching KV path %q", b.cfg.KVPath) kv := make(chan string) - go watchKV(b.c, b.cfg.KVPath, kv) + go watchKV(b.c, b.cfg.KVPath, kv, true) return kv } @@ -101,7 +142,7 @@ func (b *be) WatchNoRouteHTML() chan string { log.Printf("[INFO] consul: Watching KV path %q", b.cfg.NoRouteHTMLPath) html := make(chan string) - go watchKV(b.c, b.cfg.NoRouteHTMLPath, html) + go watchKV(b.c, b.cfg.NoRouteHTMLPath, html, false) return html } @@ -122,3 +163,12 @@ func datacenter(c *api.Client) (string, error) { } return dc, nil } + +func stringInSlice(str string, strSlice []string) bool { + for _, s := range strSlice { + if s == str { + return true + } + } + return false +} diff --git a/registry/consul/kv.go b/registry/consul/kv.go index 1c90aebf9..be73b3728 100644 --- a/registry/consul/kv.go +++ b/registry/consul/kv.go @@ -10,12 +10,12 @@ import ( // watchKV monitors a key in the KV store for changes. // The intended use case is to add additional route commands to the routing table. -func watchKV(client *api.Client, path string, config chan string) { +func watchKV(client *api.Client, path string, config chan string, separator bool) { var lastIndex uint64 var lastValue string for { - value, index, err := listKV(client, path, lastIndex) + value, index, err := listKV(client, path, lastIndex, separator) if err != nil { log.Printf("[WARN] consul: Error fetching config from %s. %v", path, err) time.Sleep(time.Second) @@ -46,7 +46,7 @@ func listKeys(client *api.Client, path string, waitIndex uint64) ([]string, uint return keys, meta.LastIndex, nil } -func listKV(client *api.Client, path string, waitIndex uint64) (string, uint64, error) { +func listKV(client *api.Client, path string, waitIndex uint64, separator bool) (string, uint64, error) { q := &api.QueryOptions{RequireConsistent: true, WaitIndex: waitIndex} kvpairs, meta, err := client.KV().List(path, q) if err != nil { @@ -57,7 +57,10 @@ func listKV(client *api.Client, path string, waitIndex uint64) (string, uint64, } var s []string for _, kvpair := range kvpairs { - val := "# --- " + kvpair.Key + "\n" + strings.TrimSpace(string(kvpair.Value)) + val := strings.TrimSpace(string(kvpair.Value)) + if separator { + val = "# --- " + kvpair.Key + "\n" + val + } s = append(s, val) } return strings.Join(s, "\n\n"), meta.LastIndex, nil diff --git a/registry/consul/passing.go b/registry/consul/passing.go index 77ea71882..45749174c 100644 --- a/registry/consul/passing.go +++ b/registry/consul/passing.go @@ -7,53 +7,97 @@ import ( "github.com/hashicorp/consul/api" ) -// passingServices filters out health checks for services which have -// passing health checks and where the neither the service instance itself -// nor the node is in maintenance mode. -func passingServices(checks []*api.HealthCheck, status []string) []*api.HealthCheck { +func passingServices(checks []*api.HealthCheck, status []string, strict bool) []*api.HealthCheck { var p []*api.HealthCheck for _, svc := range checks { - // first filter out non-service checks - if svc.ServiceID == "" || svc.CheckID == "serfHealth" || svc.CheckID == "_node_maintenance" || strings.HasPrefix("_service_maintenance:", svc.CheckID) { + if !isServiceCheck(svc) { continue } - - // then make sure the service health check is passing - if !contains(status, svc.Status) { + total, passing := countChecks(svc, checks, status) + if passing == 0 { + continue + } + if strict && total != passing { + continue + } + if isAgentCritical(svc, checks) { continue } + if isNodeInMaintenance(svc, checks) { + continue + } + if isServiceInMaintenance(svc, checks) { + continue + } + p = append(p, svc) + } - // then check whether the agent is still alive and both the - // node and the service are not in maintenance mode. - for _, c := range checks { - if c.Node != svc.Node { - continue - } - if c.CheckID == "serfHealth" && c.Status == "critical" { - log.Printf("[DEBUG] consul: Skipping service %q since agent on node %q is down: %s", svc.ServiceID, svc.Node, c.Output) - goto skip - } - if c.CheckID == "_node_maintenance" { - log.Printf("[DEBUG] consul: Skipping service %q since node %q is in maintenance mode: %s", svc.ServiceID, svc.Node, c.Output) - goto skip - } - if c.CheckID == "_service_maintenance:"+svc.ServiceID && c.Status == "critical" { - log.Printf("[DEBUG] consul: Skipping service %q since it is in maintenance mode: %s", svc.ServiceID, c.Output) - goto skip - } + return p +} + +// isServiceCheck returns true if the health check is a valid service check. +func isServiceCheck(c *api.HealthCheck) bool { + return c.ServiceID != "" && + c.CheckID != "serfHealth" && + c.CheckID != "_node_maintenance" && + !strings.HasPrefix("_service_maintenance:", c.CheckID) +} + +// isAgentCritical returns true if the agent on the node on which the service +// runs is critical. +func isAgentCritical(svc *api.HealthCheck, checks []*api.HealthCheck) bool { + for _, c := range checks { + if svc.Node == c.Node && c.CheckID == "serfHealth" && c.Status == "critical" { + log.Printf("[DEBUG] consul: Skipping service %q since agent on node %q is down: %s", c.ServiceID, c.Node, c.Output) + return true } + } + return false +} - p = append(p, svc) +// isNodeInMaintenance returns true if the node on which the service runs is in +// maintenance mode. +func isNodeInMaintenance(svc *api.HealthCheck, checks []*api.HealthCheck) bool { + for _, c := range checks { + if svc.Node == c.Node && c.CheckID == "_node_maintenance" { + log.Printf("[DEBUG] consul: Skipping service %q since node %q is in maintenance mode: %s", c.ServiceID, c.Node, c.Output) + return true + } + } + return false +} - skip: +// isServiceInMaintenance returns true if the service instance is in +// maintenance mode. +func isServiceInMaintenance(svc *api.HealthCheck, checks []*api.HealthCheck) bool { + for _, c := range checks { + if svc.Node == c.Node && c.CheckID == "_service_maintenance:"+svc.ServiceID && c.Status == "critical" { + log.Printf("[DEBUG] consul: Skipping service %q since it is in maintenance mode: %s", svc.ServiceID, c.Output) + return true + } } + return false +} - return p +// countChecks counts the number of service checks exist for a given service +// and how many of them are passing. +func countChecks(svc *api.HealthCheck, checks []*api.HealthCheck, status []string) (total int, passing int) { + for _, c := range checks { + if svc.Node == c.Node && svc.ServiceID == c.ServiceID { + total++ + if hasStatus(c, status) { + passing++ + } + } + } + return } -func contains(slice []string, item string) bool { - for _, a := range slice { - if a == item { +// hasStatus returns true if the health check status is one of the given +// values. +func hasStatus(c *api.HealthCheck, status []string) bool { + for _, s := range status { + if c.Status == s { return true } } diff --git a/registry/consul/passing_test.go b/registry/consul/passing_test.go index 944682303..51f083ba3 100644 --- a/registry/consul/passing_test.go +++ b/registry/consul/passing_test.go @@ -12,6 +12,7 @@ func TestPassingServices(t *testing.T) { serfPass = &api.HealthCheck{Node: "node", CheckID: "serfHealth", Status: "passing"} serfFail = &api.HealthCheck{Node: "node", CheckID: "serfHealth", Status: "critical"} svc1Pass = &api.HealthCheck{Node: "node", CheckID: "service:abc", Status: "passing", ServiceName: "abc", ServiceID: "abc-1"} + svc1Chk2Warn = &api.HealthCheck{Node: "node", CheckID: "service:abc", Status: "warning", ServiceName: "abc", ServiceID: "abc-1"} svc1Node2Pass = &api.HealthCheck{Node: "node2", CheckID: "service:abc", Status: "passing", ServiceName: "abc", ServiceID: "abc-1"} svc1Warn = &api.HealthCheck{Node: "node", CheckID: "service:abc", Status: "warning", ServiceName: "abc", ServiceID: "abc-2"} svc1Crit = &api.HealthCheck{Node: "node", CheckID: "service:abc", Status: "critical", ServiceName: "abc", ServiceID: "abc-3"} @@ -22,32 +23,106 @@ func TestPassingServices(t *testing.T) { ) tests := []struct { + name string + strict bool status []string in, out []*api.HealthCheck }{ - {[]string{"passing"}, nil, nil}, - {[]string{"passing"}, []*api.HealthCheck{}, nil}, - {[]string{"passing"}, []*api.HealthCheck{svc1Pass}, []*api.HealthCheck{svc1Pass}}, - {[]string{"passing"}, []*api.HealthCheck{svc1Pass, svc2Pass}, []*api.HealthCheck{svc1Pass, svc2Pass}}, - {[]string{"passing"}, []*api.HealthCheck{serfPass, svc1Pass}, []*api.HealthCheck{svc1Pass}}, - {[]string{"passing"}, []*api.HealthCheck{serfFail, svc1Pass}, nil}, - {[]string{"passing"}, []*api.HealthCheck{nodeMaint, svc1Pass}, nil}, - {[]string{"passing"}, []*api.HealthCheck{svc1Maint, svc1Pass}, nil}, - {[]string{"passing"}, []*api.HealthCheck{nodeMaint, svc1Maint, svc1Pass}, nil}, - {[]string{"passing"}, []*api.HealthCheck{serfFail, nodeMaint, svc1Maint, svc1Pass}, nil}, - {[]string{"passing"}, []*api.HealthCheck{svc1ID2Maint, svc1Pass}, []*api.HealthCheck{svc1Pass}}, - {[]string{"passing"}, []*api.HealthCheck{svc1Maint, svc1Pass, svc2Pass}, []*api.HealthCheck{svc2Pass}}, - {[]string{"passing"}, []*api.HealthCheck{svc1Crit, svc1Node2Pass}, []*api.HealthCheck{svc1Node2Pass}}, - {[]string{"passing"}, []*api.HealthCheck{svc1Maint, svc1Node2Pass}, []*api.HealthCheck{svc1Node2Pass}}, - {[]string{"passing", "warning"}, []*api.HealthCheck{serfPass, svc1Pass, svc1Crit}, []*api.HealthCheck{svc1Pass}}, - {[]string{"passing", "warning"}, []*api.HealthCheck{serfPass, svc1Warn, svc1Crit}, []*api.HealthCheck{svc1Warn}}, - {[]string{"passing", "warning"}, []*api.HealthCheck{serfPass, svc1Pass, svc1Warn}, []*api.HealthCheck{svc1Pass, svc1Warn}}, - {[]string{"passing", "warning"}, []*api.HealthCheck{serfPass, svc1Warn, svc1Crit, svc1Pass}, []*api.HealthCheck{svc1Warn, svc1Pass}}, + { + "expect no passing checks if checks array is nil", + false, []string{"passing"}, nil, nil, + }, + { + "expect no passing checks if checks array is empty", + false, []string{"passing"}, []*api.HealthCheck{}, nil, + }, + { + "expect check to pass if it has a matching status", + false, []string{"passing"}, []*api.HealthCheck{svc1Pass}, []*api.HealthCheck{svc1Pass}, + }, + { + "expect all checks to pass if they have a matching status", + false, []string{"passing"}, []*api.HealthCheck{svc1Pass, svc2Pass}, []*api.HealthCheck{svc1Pass, svc2Pass}, + }, + { + "expect that internal consul checks are filtered out", + false, []string{"passing"}, []*api.HealthCheck{serfPass, svc1Pass}, []*api.HealthCheck{svc1Pass}, + }, + { + "expect no passing checks if consul agent is unhealthy", + false, []string{"passing"}, []*api.HealthCheck{serfFail, svc1Pass}, nil, + }, + { + "expect no passing checks if node is in maintenance mode", + false, []string{"passing"}, []*api.HealthCheck{nodeMaint, svc1Pass}, nil, + }, + { + "expect no passing check if corresponding service is in maintenance mode", + false, []string{"passing"}, []*api.HealthCheck{svc1Maint, svc1Pass}, nil, + }, + { + "expect no passing check if node and service are in maintenance mode", + false, []string{"passing"}, []*api.HealthCheck{nodeMaint, svc1Maint, svc1Pass}, nil, + }, + { + "expect no passing check if agent is unhealthy or node and service are in maintenance mode", + false, []string{"passing"}, []*api.HealthCheck{serfFail, nodeMaint, svc1Maint, svc1Pass}, nil, + }, + { + "expect check of service which is not in maintenance mode to pass if another instance of same service is in maintenance mode", + false, []string{"passing"}, []*api.HealthCheck{svc1ID2Maint, svc1Pass}, []*api.HealthCheck{svc1Pass}, + }, + { + "expect that no checks of a service which is in maintenance mode are returned even if it has a passing check", + false, []string{"passing"}, []*api.HealthCheck{svc1Maint, svc1Pass, svc2Pass}, []*api.HealthCheck{svc2Pass}, + }, + { + "expect that a service's failing check does not affect a healthy instance of same service running on different node", + false, []string{"passing"}, []*api.HealthCheck{svc1Crit, svc1Node2Pass}, []*api.HealthCheck{svc1Node2Pass}, + }, + { + "service in maintenance mode does not affect healthy service running on different node", + false, []string{"passing"}, []*api.HealthCheck{svc1Maint, svc1Node2Pass}, []*api.HealthCheck{svc1Node2Pass}, + }, + { + "expect that internal consul check and failing check are not returned", + false, []string{"passing", "warning"}, []*api.HealthCheck{serfPass, svc1Pass, svc1Crit}, []*api.HealthCheck{svc1Pass}, + }, + { + "expect that internal consul check is filtered out and check with warning is passing", + false, []string{"passing", "warning"}, []*api.HealthCheck{serfPass, svc1Warn, svc1Crit}, []*api.HealthCheck{svc1Warn}, + }, + { + "expect that warning and passing non-internal checks are returned", + false, []string{"passing", "warning"}, []*api.HealthCheck{serfPass, svc1Pass, svc1Warn}, []*api.HealthCheck{svc1Pass, svc1Warn}, + }, + { + "expect that warning und passing non-internal checks are returned", + false, []string{"passing", "warning"}, []*api.HealthCheck{serfPass, svc1Warn, svc1Crit, svc1Pass}, []*api.HealthCheck{svc1Warn, svc1Pass}, + }, + { + "in non-strict mode, expect that checks which belong to same service are passing, if at least one of them is passing", + false, []string{"passing"}, []*api.HealthCheck{svc1Pass, svc1Chk2Warn}, []*api.HealthCheck{svc1Pass, svc1Chk2Warn}, + }, + { + "in strict mode, expect that no checks which belong to same service are passing, if not all of them are passing", + true, []string{"passing"}, []*api.HealthCheck{svc1Pass, svc1Chk2Warn}, nil, + }, + { + "in strict mode, expect that a failing check of one service does not affect a different service's passing check", + true, []string{"passing"}, []*api.HealthCheck{svc1Pass, svc1Chk2Warn, svc2Pass}, []*api.HealthCheck{svc2Pass}, + }, + { + "in strict mode, expect a check to pass if all of the other checks that belong to the same service are passing", + true, []string{"passing", "warning"}, []*api.HealthCheck{svc1Pass, svc1Chk2Warn}, []*api.HealthCheck{svc1Pass, svc1Chk2Warn}, + }, } - for i, tt := range tests { - if got, want := passingServices(tt.in, tt.status), tt.out; !reflect.DeepEqual(got, want) { - t.Errorf("%d: got %v want %v", i, got, want) - } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, want := passingServices(tt.in, tt.status, tt.strict), tt.out; !reflect.DeepEqual(got, want) { + t.Errorf("got %v want %v", got, want) + } + }) } } diff --git a/registry/consul/register.go b/registry/consul/register.go index 0ed7d01bc..e8aca4834 100644 --- a/registry/consul/register.go +++ b/registry/consul/register.go @@ -38,10 +38,11 @@ func register(c *api.Client, service *api.AgentServiceRegistration) chan bool { register := func() string { if err := c.Agent().ServiceRegister(service); err != nil { - log.Printf("[ERROR] consul: Cannot register fabio in consul. %s", err) + log.Printf("[ERROR] consul: Cannot register fabio [name:%q] in Consul. %s", service.Name, err) return "" } + log.Printf("[INFO] consul: Registered fabio as %q", service.Name) log.Printf("[INFO] consul: Registered fabio with id %q", service.ID) log.Printf("[INFO] consul: Registered fabio with address %q", service.Address) log.Printf("[INFO] consul: Registered fabio with tags %q", strings.Join(service.Tags, ",")) @@ -51,7 +52,7 @@ func register(c *api.Client, service *api.AgentServiceRegistration) chan bool { } deregister := func(serviceID string) { - log.Printf("[INFO] consul: Deregistering fabio") + log.Printf("[INFO] consul: Deregistering %s", service.Name) c.Agent().ServiceDeregister(serviceID) } @@ -76,7 +77,7 @@ func register(c *api.Client, service *api.AgentServiceRegistration) chan bool { return dereg } -func serviceRegistration(cfg *config.Consul) (*api.AgentServiceRegistration, error) { +func serviceRegistration(cfg *config.Consul, serviceName string) (*api.AgentServiceRegistration, error) { hostname, err := os.Hostname() if err != nil { return nil, err @@ -101,7 +102,7 @@ func serviceRegistration(cfg *config.Consul) (*api.AgentServiceRegistration, err } } - serviceID := fmt.Sprintf("%s-%s-%d", cfg.ServiceName, hostname, port) + serviceID := fmt.Sprintf("%s-%s-%d", serviceName, hostname, port) checkURL := fmt.Sprintf("%s://%s:%d/health", cfg.CheckScheme, ip, port) if ip.To16() != nil { @@ -110,15 +111,16 @@ func serviceRegistration(cfg *config.Consul) (*api.AgentServiceRegistration, err service := &api.AgentServiceRegistration{ ID: serviceID, - Name: cfg.ServiceName, + Name: serviceName, Address: ip.String(), Port: port, Tags: cfg.ServiceTags, Check: &api.AgentServiceCheck{ - HTTP: checkURL, - Interval: cfg.CheckInterval.String(), - Timeout: cfg.CheckTimeout.String(), - TLSSkipVerify: cfg.CheckTLSSkipVerify, + HTTP: checkURL, + Interval: cfg.CheckInterval.String(), + Timeout: cfg.CheckTimeout.String(), + TLSSkipVerify: cfg.CheckTLSSkipVerify, + DeregisterCriticalServiceAfter: cfg.CheckDeregisterCriticalServiceAfter, }, } diff --git a/registry/consul/service.go b/registry/consul/service.go index 2de1487f5..18bd4c38d 100644 --- a/registry/consul/service.go +++ b/registry/consul/service.go @@ -10,13 +10,15 @@ import ( "strings" "time" + "github.com/fabiolb/fabio/config" "github.com/hashicorp/consul/api" ) // watchServices monitors the consul health checks and creates a new configuration // on every change. -func watchServices(client *api.Client, tagPrefix string, status []string, config chan string) { +func watchServices(client *api.Client, config *config.Consul, svcConfig chan string) { var lastIndex uint64 + var strict bool = strings.EqualFold("all", config.ChecksRequired) for { q := &api.QueryOptions{RequireConsistent: true, WaitIndex: lastIndex} @@ -28,7 +30,7 @@ func watchServices(client *api.Client, tagPrefix string, status []string, config } log.Printf("[DEBUG] consul: Health changed to #%d", meta.LastIndex) - config <- servicesConfig(client, passingServices(checks, status), tagPrefix) + svcConfig <- servicesConfig(client, passingServices(checks, config.ServiceStatus, strict), config.TagPrefix) lastIndex = meta.LastIndex } } diff --git a/registry/static/backend.go b/registry/static/backend.go index 9ac4bc5b8..237ad2717 100644 --- a/registry/static/backend.go +++ b/registry/static/backend.go @@ -15,11 +15,15 @@ func NewBackend(cfg *config.Static) (registry.Backend, error) { return &be{cfg}, nil } -func (b *be) Register() error { +func (b *be) Register(services []string) error { return nil } -func (b *be) Deregister() error { +func (b *be) Deregister(serviceName string) error { + return nil +} + +func (b *be) DeregisterAll() error { return nil } diff --git a/rootwarn_windows.go b/rootwarn_windows.go index 771f0b892..3ce4a8c2e 100644 --- a/rootwarn_windows.go +++ b/rootwarn_windows.go @@ -2,6 +2,6 @@ package main -func CheckInsecure(allowRoot bool) { +func WarnIfRunAsRoot(allowRoot bool) { // windows not supported } diff --git a/route/access_rules.go b/route/access_rules.go new file mode 100644 index 000000000..4b73108f7 --- /dev/null +++ b/route/access_rules.go @@ -0,0 +1,182 @@ +package route + +import ( + "errors" + "fmt" + "log" + "net" + "net/http" + "strings" +) + +const ( + ipAllowTag = "allow:ip" + ipDenyTag = "deny:ip" +) + +// AccessDeniedHTTP checks rules on the target for HTTP proxy routes. +func (t *Target) AccessDeniedHTTP(r *http.Request) bool { + var ip net.IP + host, _, err := net.SplitHostPort(r.RemoteAddr) + + if err != nil { + log.Printf("[ERROR] failed to get host from remote header %s: %s", + r.RemoteAddr, err.Error()) + return false + } + + if ip = net.ParseIP(host); ip == nil { + log.Printf("[WARN] failed to parse remote address %s", host) + } + + // check remote source and return if denied + if t.denyByIP(ip) { + return true + } + + // check xff source if present + if xff := r.Header.Get("X-Forwarded-For"); xff != "" { + // Trusting XFF headers sent from clients is dangerous and generally + // bad practice. Therefore, we cannot assume which if any of the elements + // is the actual client address. To try and avoid the chance of spoofed + // headers and/or loose upstream proxies we validate all elements in the header. + // Specifically AWS does not strip XFF from anonymous internet sources: + // https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/x-forwarded-headers.html#x-forwarded-for + // See lengthy github discussion for more background: https://github.com/fabiolb/fabio/pull/449 + for _, xip := range strings.Split(xff, ",") { + xip = strings.TrimSpace(xip) + if xip == host { + continue + } + if ip = net.ParseIP(xip); ip == nil { + log.Printf("[WARN] failed to parse xff address %s", xip) + continue + } + if t.denyByIP(ip) { + return true + } + } + } + + // default allow + return false +} + +// AccessDeniedTCP checks rules on the target for TCP proxy routes. +func (t *Target) AccessDeniedTCP(c net.Conn) bool { + ip := net.ParseIP(c.RemoteAddr().String()) + if t.denyByIP(ip) { + return true + } + // default allow + return false +} + +func (t *Target) denyByIP(ip net.IP) bool { + if ip == nil || t.accessRules == nil { + return false + } + + // check allow (whitelist) first if it exists + if _, ok := t.accessRules[ipAllowTag]; ok { + var block *net.IPNet + for _, x := range t.accessRules[ipAllowTag] { + if block, ok = x.(*net.IPNet); !ok { + log.Print("[ERROR] failed to assert ip block while checking allow rule for ", t.Service) + continue + } + if block.Contains(ip) { + // specific allow matched - allow this request + return false + } + } + // we checked all the blocks - deny this request + log.Printf("[INFO] route rules denied access from %s to %s", + ip.String(), t.URL.String()) + return true + } + + // still going - check deny (blacklist) if it exists + if _, ok := t.accessRules[ipDenyTag]; ok { + var block *net.IPNet + for _, x := range t.accessRules[ipDenyTag] { + if block, ok = x.(*net.IPNet); !ok { + log.Print("[INFO] failed to assert ip block while checking deny rule for ", t.Service) + continue + } + if block.Contains(ip) { + // specific deny matched - deny this request + log.Printf("[INFO] route rules denied access from %s to %s", + ip.String(), t.URL.String()) + return true + } + } + } + + // default - do not deny + return false +} + +// ProcessAccessRules processes access rules from options specified on the target route +func (t *Target) ProcessAccessRules() error { + if t.Opts["allow"] != "" && t.Opts["deny"] != "" { + return errors.New("specifying allow and deny on the same route is not supported") + } + + for _, allowDeny := range []string{"allow", "deny"} { + if t.Opts[allowDeny] != "" { + if err := t.parseAccessRule(allowDeny); err != nil { + return err + } + } + } + return nil +} + +func (t *Target) parseAccessRule(allowDeny string) error { + var accessTag string + var temps []string + var value string + var ip net.IP + + // init rules if needed + if t.accessRules == nil { + t.accessRules = make(map[string][]interface{}) + } + + // loop over rule elements + for _, c := range strings.Split(t.Opts[allowDeny], ",") { + if temps = strings.SplitN(c, ":", 2); len(temps) != 2 { + return fmt.Errorf("invalid access item, expected :, got %s", temps) + } + + // form access type tag + accessTag = allowDeny + ":" + strings.ToLower(strings.TrimSpace(temps[0])) + + // switch on formed access tag - currently only ip types are implemented + switch accessTag { + case ipAllowTag, ipDenyTag: + if value = strings.TrimSpace(temps[1]); !strings.Contains(value, "/") { + if ip = net.ParseIP(value); ip == nil { + return fmt.Errorf("failed to parse IP %s", value) + } + if ip.To4() != nil { + value = ip.String() + "/32" + } else { + value = ip.String() + "/128" + } + } + _, net, err := net.ParseCIDR(value) + if err != nil { + return fmt.Errorf("failed to parse CIDR %s with error: %s", + c, err.Error()) + } + // add element to rule map + t.accessRules[accessTag] = append(t.accessRules[accessTag], net) + default: + return fmt.Errorf("unknown access item type: %s", temps[0]) + } + } + + return nil +} diff --git a/route/access_rules_test.go b/route/access_rules_test.go new file mode 100644 index 000000000..1f2e5c15a --- /dev/null +++ b/route/access_rules_test.go @@ -0,0 +1,233 @@ +package route + +import ( + "net" + "net/http" + "net/url" + "testing" +) + +func TestAccessRules_parseAccessRule(t *testing.T) { + tests := []struct { + desc string + allowDeny string + fail bool + }{ + { + desc: "valid ipv4 rule", + allowDeny: "ip:10.0.0.0/8,ip:192.168.0.0/24,ip:1.2.3.4/32", + }, + { + desc: "valid ipv6 rule", + allowDeny: "ip:1234:567:beef:cafe::/64,ip:1234:5678:dead:beef::/32", + }, + { + desc: "invalid rule type", + allowDeny: "xxx:10.0.0.0/8", + fail: true, + }, + { + desc: "ip rule with incomplete address", + allowDeny: "ip:10/8", + fail: true, + }, + { + desc: "ip rule with bad cidr mask", + allowDeny: "ip:10.0.0.0/255", + fail: true, + }, + { + desc: "single ipv4 with no mask", + allowDeny: "ip:1.2.3.4", + fail: false, + }, + { + desc: "single ipv6 with no mask", + allowDeny: "ip:fe80::1", + fail: false, + }, + } + + for i, tt := range tests { + tt := tt // capture loop var + + t.Run(tt.desc, func(t *testing.T) { + for _, ad := range []string{"allow", "deny"} { + tgt := &Target{Opts: map[string]string{ad: tt.allowDeny}} + err := tgt.parseAccessRule(ad) + if err != nil && !tt.fail { + t.Errorf("%d: %s\nfailed: %s", i, tt.desc, err.Error()) + return + } + } + }) + } +} + +func TestAccessRules_denyByIP(t *testing.T) { + tests := []struct { + desc string + target *Target + remote net.IP + denied bool + }{ + { + desc: "allow rule with included ipv4", + target: &Target{ + Opts: map[string]string{"allow": "ip:10.0.0.0/8,ip:192.168.0.0/24"}, + }, + remote: net.ParseIP("10.10.0.1"), + denied: false, + }, + { + desc: "allow rule with exluded ipv4", + target: &Target{ + Opts: map[string]string{"allow": "ip:10.0.0.0/8,ip:192.168.0.0/24"}, + }, + remote: net.ParseIP("1.2.3.4"), + denied: true, + }, + { + desc: "deny rule with included ipv4", + target: &Target{ + Opts: map[string]string{"deny": "ip:10.0.0.0/8,ip:192.168.0.0/24"}, + }, + remote: net.ParseIP("10.10.0.1"), + denied: true, + }, + { + desc: "deny rule with excluded ipv4", + target: &Target{ + Opts: map[string]string{"deny": "ip:10.0.0.0/8,ip:192.168.0.0/24"}, + }, + remote: net.ParseIP("1.2.3.4"), + denied: false, + }, + { + desc: "allow rule with included ipv6", + target: &Target{ + Opts: map[string]string{"allow": "ip:1234:dead:beef:cafe::/64"}, + }, + remote: net.ParseIP("1234:dead:beef:cafe::5678"), + denied: false, + }, + { + desc: "allow rule with exluded ipv6", + target: &Target{ + Opts: map[string]string{"allow": "ip:1234:dead:beef:cafe::/64"}, + }, + remote: net.ParseIP("1234:5678::1"), + denied: true, + }, + { + desc: "deny rule with included ipv6", + target: &Target{ + Opts: map[string]string{"deny": "ip:1234:dead:beef:cafe::/64"}, + }, + remote: net.ParseIP("1234:dead:beef:cafe::5678"), + denied: true, + }, + { + desc: "deny rule with excluded ipv6", + target: &Target{ + Opts: map[string]string{"deny": "ip:1234:dead:beef:cafe::/64"}, + }, + remote: net.ParseIP("1234:5678::1"), + denied: false, + }, + } + + for i, tt := range tests { + tt := tt // capture loop var + + t.Run(tt.desc, func(t *testing.T) { + if err := tt.target.ProcessAccessRules(); err != nil { + t.Errorf("%d: %s - failed to process access rules: %s", + i, tt.desc, err.Error()) + } + tt.target.URL, _ = url.Parse("http://testing.test/") + if deny := tt.target.denyByIP(tt.remote); deny != tt.denied { + t.Errorf("%d: %s\ngot denied: %t\nwant denied: %t\n", + i, tt.desc, deny, tt.denied) + return + } + }) + } +} + +func TestAccessRules_AccessDeniedHTTP(t *testing.T) { + req, _ := http.NewRequest("GET", "http://example.com/", nil) + tests := []struct { + desc string + target *Target + xff string + remote string + denied bool + }{ + { + desc: "single denied xff and allowed remote addr", + target: &Target{ + Opts: map[string]string{"allow": "ip:10.0.0.0/8,ip:192.168.0.0/24"}, + }, + xff: "10.11.12.13, 1.1.1.2, 10.11.12.14", + remote: "10.11.12.1:65500", + denied: true, + }, + { + desc: "allowed xff and denied remote addr", + target: &Target{ + Opts: map[string]string{"allow": "ip:10.0.0.0/8,ip:192.168.0.0/24"}, + }, + xff: "10.11.12.13, 1.2.3.4", + remote: "1.1.1.2:65500", + denied: true, + }, + { + desc: "single allowed xff and allowed remote addr", + target: &Target{ + Opts: map[string]string{"allow": "ip:10.0.0.0/8,ip:192.168.0.0/24"}, + }, + xff: "10.11.12.13, 1.2.3.4", + remote: "192.168.0.12:65500", + denied: true, + }, + { + desc: "denied xff and denied remote addr", + target: &Target{ + Opts: map[string]string{"allow": "ip:10.0.0.0/8,ip:192.168.0.0/24"}, + }, + xff: "1.2.3.4, 10.11.12.13, 10.11.12.14", + remote: "200.17.18.20:65500", + denied: true, + }, + { + desc: "all allowed xff and allowed remote addr", + target: &Target{ + Opts: map[string]string{"allow": "ip:10.0.0.0/8,ip:192.168.0.0/24"}, + }, + xff: "10.11.12.13, 10.110.120.130", + remote: "192.168.0.12:65500", + denied: false, + }, + } + + for i, tt := range tests { + tt := tt // capture loop var + + req.Header = http.Header{"X-Forwarded-For": []string{tt.xff}} + req.RemoteAddr = tt.remote + + t.Run(tt.desc, func(t *testing.T) { + if err := tt.target.ProcessAccessRules(); err != nil { + t.Errorf("%d: %s - failed to process access rules: %s", + i, tt.desc, err.Error()) + } + tt.target.URL = mustParse("http://testing.test/") + if deny := tt.target.AccessDeniedHTTP(req); deny != tt.denied { + t.Errorf("%d: %s\ngot denied: %t\nwant denied: %t\n", + i, tt.desc, deny, tt.denied) + return + } + }) + } +} diff --git a/route/matcher.go b/route/matcher.go index 710be013c..3a7274c66 100644 --- a/route/matcher.go +++ b/route/matcher.go @@ -1,8 +1,6 @@ package route import ( - "log" - "path" "strings" ) @@ -21,12 +19,7 @@ func prefixMatcher(uri string, r *Route) bool { return strings.HasPrefix(uri, r.Path) } -// globMatcher matches path to the routes' path using globbing. +// globMatcher matches path to the routes' path using gobwas/glob. func globMatcher(uri string, r *Route) bool { - var hasMatch, err = path.Match(r.Path, uri) - if err != nil { - log.Printf("[ERROR] Glob matching error %s for path %s route %s", err, uri, r.Path) - return false - } - return hasMatch + return r.Glob.Match(uri) } diff --git a/route/matcher_test.go b/route/matcher_test.go index d8e05160e..59a068c90 100644 --- a/route/matcher_test.go +++ b/route/matcher_test.go @@ -2,55 +2,61 @@ package route import ( "testing" + + "github.com/gobwas/glob" ) func TestPrefixMatcher(t *testing.T) { - routeFoo := &Route{Host: "www.example.com", Path: "/foo"} - tests := []struct { - uri string - want bool - route *Route + uri string + matches bool + route *Route }{ - {"/fo", false, routeFoo}, - {"/foo", true, routeFoo}, - {"/fools", true, routeFoo}, - {"/bar", false, routeFoo}, + {uri: "/foo", matches: true, route: &Route{Path: "/foo"}}, + {uri: "/fools", matches: true, route: &Route{Path: "/foo"}}, + {uri: "/fo", matches: false, route: &Route{Path: "/foo"}}, + {uri: "/bar", matches: false, route: &Route{Path: "/foo"}}, } for _, tt := range tests { - if got := prefixMatcher(tt.uri, tt.route); got != tt.want { - t.Errorf("%s: got %v want %v", tt.uri, got, tt.want) - } + t.Run(tt.uri, func(t *testing.T) { + if got, want := prefixMatcher(tt.uri, tt.route), tt.matches; got != want { + t.Fatalf("got %v want %v", got, want) + } + }) } } func TestGlobMatcher(t *testing.T) { - routeFoo := &Route{Host: "www.example.com", Path: "/foo"} - routeFooWild := &Route{Host: "www.example.com", Path: "/foo.*"} - tests := []struct { - uri string - want bool - route *Route + uri string + matches bool + route *Route }{ - {"/fo", false, routeFoo}, - {"/foo", true, routeFoo}, - {"/fools", false, routeFoo}, - {"/bar", false, routeFoo}, - - {"/fo", false, routeFooWild}, - {"/foo", false, routeFooWild}, - {"/fools", false, routeFooWild}, - {"/foo.", true, routeFooWild}, - {"/foo.a", true, routeFooWild}, - {"/foo.bar", true, routeFooWild}, - {"/foo.bar.baz", true, routeFooWild}, + // happy flows + {uri: "/foo", matches: true, route: &Route{Path: "/foo"}}, + {uri: "/fool", matches: true, route: &Route{Path: "/foo?"}}, + {uri: "/fool", matches: true, route: &Route{Path: "/foo*"}}, + {uri: "/fools", matches: true, route: &Route{Path: "/foo*"}}, + {uri: "/fools", matches: true, route: &Route{Path: "/foo*"}}, + {uri: "/foo/x/bar", matches: true, route: &Route{Path: "/foo/*/bar"}}, + {uri: "/foo/x/y/z/w/bar", matches: true, route: &Route{Path: "/foo/**"}}, + {uri: "/foo/x/y/z/w/bar", matches: true, route: &Route{Path: "/foo/**/bar"}}, + + // error flows + {uri: "/fo", matches: false, route: &Route{Path: "/foo"}}, + {uri: "/fools", matches: false, route: &Route{Path: "/foo"}}, + {uri: "/fo", matches: false, route: &Route{Path: "/foo*"}}, + {uri: "/fools", matches: false, route: &Route{Path: "/foo.*"}}, + {uri: "/foo/x/y/z/w/baz", matches: false, route: &Route{Path: "/foo/**/bar"}}, } for _, tt := range tests { - if got := globMatcher(tt.uri, tt.route); got != tt.want { - t.Errorf("%s: got %v want %v", tt.uri, got, tt.want) - } + t.Run(tt.uri, func(t *testing.T) { + tt.route.Glob = glob.MustCompile(tt.route.Path) + if got, want := globMatcher(tt.uri, tt.route), tt.matches; got != want { + t.Fatalf("got %v want %v", got, want) + } + }) } } diff --git a/route/parse_new.go b/route/parse_new.go index 225e9b837..4c6702e97 100644 --- a/route/parse_new.go +++ b/route/parse_new.go @@ -27,6 +27,8 @@ route add [ weight ][ tags ",,..."][ opts "k1=v1 k2= proto=tcp : upstream service is TCP, dst is ':port' proto=https : upstream service is HTTPS tlsskipverify=true : disable TLS cert validation for HTTPS upstream + host=name : set the Host header to 'name'. If 'name == "dst"' then the 'Host' header will be set to the registered upstream host name + register=name : register fabio as new service 'name'. Useful for registering hostnames for host specific routes. route del [ [ ]] - Remove route matching svc, src and/or dst @@ -87,6 +89,43 @@ func Parse(in string) (defs []*RouteDef, err error) { return defs, nil } +// ParseAliases scans a set of route commands for the "register" option and +// returns a list of services which should be registered by the backend. +func ParseAliases(in string) (names []string, err error) { + var defs []*RouteDef + var def *RouteDef + for i, s := range strings.Split(in, "\n") { + def, err = nil, nil + s = strings.TrimSpace(s) + switch { + case reComment.MatchString(s) || reBlankLine.MatchString(s): + continue + case reRouteAdd.MatchString(s): + def, err = parseRouteAdd(s) + case reRouteDel.MatchString(s): + def, err = parseRouteDel(s) + case reRouteWeight.MatchString(s): + def, err = parseRouteWeight(s) + default: + err = errors.New("syntax error: 'route' expected") + } + if err != nil { + return nil, fmt.Errorf("line %d: %s", i+1, err) + } + defs = append(defs, def) + } + + var aliases []string + + for _, d := range defs { + registerName, ok := d.Opts["register"] + if ok { + aliases = append(aliases, registerName) + } + } + return aliases, nil +} + // route add [ weight ][ tags ",,..."][ opts "k=v k=v ..."] // 1: service 2: src 3: dst 4: weight expr 5: weight val 6: tags expr 7: tags val 8: opts expr 9: opts val var reAdd = mustCompileWithFlexibleSpace(`^route add (\S+) (\S+) (\S+)( weight (\S+))?( tags "([^"]*)")?( opts "([^"]*)")?$`) diff --git a/route/parse_test.go b/route/parse_test.go index 1f2949681..49328409c 100644 --- a/route/parse_test.go +++ b/route/parse_test.go @@ -181,3 +181,72 @@ func TestParse(t *testing.T) { t.Run("Parse-"+tt.desc, func(t *testing.T) { run(tt.in, tt.out, tt.fail, Parse) }) } } + +func TestParseAliases(t *testing.T) { + tests := []struct { + desc string + in string + out []string + fail bool + }{ + // error flows + {"FailEmpty", ``, nil, false}, + {"FailNoRoute", `bang`, nil, true}, + {"FailRouteNoCmd", `route x`, nil, true}, + {"FailRouteAddNoService", `route add`, nil, true}, + {"FailRouteAddNoSrc", `route add svc`, nil, true}, + + // happy flows with and without aliases + { + desc: "RouteAddServiceWithoutAlias", + in: `route add alpha-be alpha/ http://1.2.3.4/ opts "strip=/path proto=https"`, + out: []string(nil), + }, + { + desc: "RouteAddServiceWithAlias", + in: `route add alpha-be alpha/ http://1.2.3.4/ opts "strip=/path proto=https register=alpha"`, + out: []string{"alpha"}, + }, + { + desc: "RouteAddServicesWithoutAliases", + in: `route add alpha-be alpha/ http://1.2.3.4/ opts "strip=/path proto=tcp" + route add bravo-be bravo/ http://1.2.3.5/ + route add charlie-be charlie/ http://1.2.3.6/ opts "host=charlie"`, + out: []string(nil), + }, + { + desc: "RouteAddServicesWithAliases", + in: `route add alpha-be alpha/ http://1.2.3.4/ opts "register=alpha" + route add bravo-be bravo/ http://1.2.3.5/ opts "strip=/foo register=bravo" + route add charlie-be charlie/ http://1.2.3.5/ opts "host=charlie proto=https" + route add delta-be delta/ http://1.2.3.5/ opts "host=delta proto=https register=delta"`, + out: []string{"alpha", "bravo", "delta"}, + }, + } + + reSyntaxError := regexp.MustCompile(`syntax error`) + + run := func(in string, aliases []string, fail bool, parseFn func(string) ([]string, error)) { + out, err := parseFn(in) + switch { + case err == nil && fail: + t.Errorf("got error nil want fail") + return + case err != nil && !fail: + t.Errorf("got error %v want nil", err) + return + case err != nil: + if !reSyntaxError.MatchString(err.Error()) { + t.Errorf("got error %q want 'syntax error.*'", err) + } + return + } + if got, want := out, aliases; !reflect.DeepEqual(got, want) { + t.Errorf("\ngot %#v\nwant %#v", got, want) + } + } + + for _, tt := range tests { + t.Run("ParseAliases-"+tt.desc, func(t *testing.T) { run(tt.in, tt.out, tt.fail, ParseAliases) }) + } +} diff --git a/route/route.go b/route/route.go index f0659c923..0f9e35dc9 100644 --- a/route/route.go +++ b/route/route.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/fabiolb/fabio/metrics" + "github.com/gobwas/glob" ) // Route maps a path prefix to one or more target URLs. @@ -36,6 +37,9 @@ type Route struct { // total contains the total number of requests for this route. // Used by the RRPicker total uint64 + + // Glob represents compiled pattern. + Glob glob.Glob } func (r *Route) addTarget(service string, targetURL *url.URL, fixedWeight float64, tags []string, opts map[string]string) { @@ -65,6 +69,7 @@ func (r *Route) addTarget(service string, targetURL *url.URL, fixedWeight float6 Timer: ServiceRegistry.GetTimer(name), TimerName: name, } + if opts != nil { t.StripPath = opts["strip"] t.TLSSkipVerify = opts["tlsskipverify"] == "true" @@ -79,6 +84,11 @@ func (r *Route) addTarget(service string, targetURL *url.URL, fixedWeight float6 log.Printf("[ERROR] redirect status code should be in 3xx range. Got: %s", opts["redirect"]) } } + + if err = t.ProcessAccessRules(); err != nil { + log.Printf("[ERROR] failed to process access rules: %s", + err.Error()) + } } r.Targets = append(r.Targets, t) diff --git a/route/table.go b/route/table.go index 6a23021d4..c417e2c34 100644 --- a/route/table.go +++ b/route/table.go @@ -13,7 +13,7 @@ import ( "sync/atomic" "github.com/fabiolb/fabio/metrics" - "github.com/ryanuber/go-glob" + "github.com/gobwas/glob" ) var errInvalidPrefix = errors.New("route: prefix must not be empty") @@ -153,13 +153,21 @@ func (t Table) addRoute(d *RouteDef) error { switch { // add new host case t[host] == nil: - r := &Route{Host: host, Path: path} + g, err := glob.Compile(path) + if err != nil { + return err + } + r := &Route{Host: host, Path: path, Glob: g} r.addTarget(d.Service, targetURL, d.Weight, d.Tags, d.Opts) t[host] = Routes{r} // add new route to existing host case t[host].find(path) == nil: - r := &Route{Host: host, Path: path} + g, err := glob.Compile(path) + if err != nil { + return err + } + r := &Route{Host: host, Path: path, Glob: g} r.addTarget(d.Service, targetURL, d.Weight, d.Tags, d.Opts) t[host] = append(t[host], r) sort.Sort(t[host]) @@ -290,7 +298,8 @@ func (t Table) matchingHosts(req *http.Request) (hosts []string) { host := normalizeHost(req.Host, req.TLS != nil) for pattern := range t { normpat := normalizeHost(pattern, req.TLS != nil) - if glob.Glob(normpat, host) { + g := glob.MustCompile(normpat) + if g.Match(host) { hosts = append(hosts, pattern) } } @@ -303,12 +312,11 @@ func (t Table) matchingHosts(req *http.Request) (hosts []string) { // and if none matches then it falls back to generic routes without // a host. This is useful for a catch-all '/' rule. func (t Table) Lookup(req *http.Request, trace string, pick picker, match matcher) (target *Target) { - path := req.URL.Path if trace != "" { if len(trace) > 16 { trace = trace[:15] } - log.Printf("[TRACE] %s Tracing %s%s", trace, req.Host, path) + log.Printf("[TRACE] %s Tracing %s%s", trace, req.Host, req.URL.Path) } // find matching hosts for the request @@ -316,7 +324,16 @@ func (t Table) Lookup(req *http.Request, trace string, pick picker, match matche hosts := t.matchingHosts(req) hosts = append(hosts, "") for _, h := range hosts { - if target = t.lookup(h, path, trace, pick, match); target != nil { + if target = t.lookup(h, req.URL.Path, trace, pick, match); target != nil { + if target.RedirectCode != 0 { + target.BuildRedirectURL(req.URL) // build redirect url and cache in target + if target.RedirectURL.Scheme == req.Header.Get("X-Forwarded-Proto") && + target.RedirectURL.Host == req.Host && + target.RedirectURL.Path == req.URL.Path { + log.Print("[INFO] Skipping redirect with same scheme, host and path") + continue + } + } break } } diff --git a/route/table_test.go b/route/table_test.go index eb0fdea71..ce8a0dca2 100644 --- a/route/table_test.go +++ b/route/table_test.go @@ -483,6 +483,80 @@ func TestNormalizeHost(t *testing.T) { } } +// see https://github.com/fabiolb/fabio/issues/448 +// for more information on the issue and purpose of this test +func TestTableLookupIssue448(t *testing.T) { + s := ` + route add mock foo.com:80/ https://foo.com/ opts "redirect=301" + route add mock aaa.com:80/ http://bbb.com/ opts "redirect=301" + route add mock ccc.com:443/bar https://ccc.com/baz opts "redirect=301" + route add mock / http://foo.com/ + ` + + tbl, err := NewTable(s) + if err != nil { + t.Fatal(err) + } + + var tests = []struct { + req *http.Request + dst string + }{ + { + req: &http.Request{ + Host: "foo.com", + URL: mustParse("/"), + }, + dst: "https://foo.com/", + // empty upstream header should follow redirect - standard behavior + }, + { + req: &http.Request{ + Host: "foo.com", + URL: mustParse("/"), + Header: http.Header{"X-Forwarded-Proto": {"http"}}, + }, + dst: "https://foo.com/", + // upstream http request to same host and path should follow redirect + }, + { + req: &http.Request{ + Host: "foo.com", + URL: mustParse("/"), + Header: http.Header{"X-Forwarded-Proto": {"https"}}, + TLS: &tls.ConnectionState{}, + }, + dst: "http://foo.com/", + // upstream https request to same host and path should NOT follow redirect" + }, + { + req: &http.Request{ + Host: "aaa.com", + URL: mustParse("/"), + Header: http.Header{"X-Forwarded-Proto": {"http"}}, + }, + dst: "http://bbb.com/", + // upstream http request to different http host should follow redirect + }, + { + req: &http.Request{ + Host: "ccc.com", + URL: mustParse("/bar"), + Header: http.Header{"X-Forwarded-Proto": {"https"}}, + TLS: &tls.ConnectionState{}, + }, + dst: "https://ccc.com/baz", + // upstream https request to same https host with different path should follow redirect" + }, + } + + for i, tt := range tests { + if got, want := tbl.Lookup(tt.req, "", rndPicker, prefixMatcher).URL.String(), tt.dst; got != want { + t.Errorf("%d: got %v want %v", i, got, want) + } + } +} + func TestTableLookup(t *testing.T) { s := ` route add svc / http://foo.com:800 diff --git a/route/target.go b/route/target.go index 6f5a70034..db9e5244b 100644 --- a/route/target.go +++ b/route/target.go @@ -38,6 +38,10 @@ type Target struct { // When set to a value > 0 the client is redirected to the target url. RedirectCode int + // RedirectURL is the redirect target based on the request. + // This is cached here to prevent multiple generations per request. + RedirectURL *url.URL + // FixedWeight is the weight assigned to this target. // If the value is 0 the targets weight is dynamic. FixedWeight float64 @@ -50,30 +54,35 @@ type Target struct { // TimerName is the name of the timer in the metrics registry TimerName string + + // accessRules is map of access information for the target. + accessRules map[string][]interface{} } -func (t *Target) GetRedirectURL(requestURL *url.URL) *url.URL { - redirectURL := &url.URL{ +func (t *Target) BuildRedirectURL(requestURL *url.URL) { + t.RedirectURL = &url.URL{ Scheme: t.URL.Scheme, Host: t.URL.Host, Path: t.URL.Path, RawQuery: t.URL.RawQuery, } - if strings.HasSuffix(redirectURL.Host, "$path") { - redirectURL.Host = redirectURL.Host[:len(redirectURL.Host)-len("$path")] - redirectURL.Path = "$path" + if strings.HasSuffix(t.RedirectURL.Host, "$path") { + t.RedirectURL.Host = t.RedirectURL.Host[:len(t.RedirectURL.Host)-len("$path")] + t.RedirectURL.Path = "$path" } - if strings.Contains(redirectURL.Path, "/$path") { - redirectURL.Path = strings.Replace(redirectURL.Path, "/$path", "$path", 1) + if strings.Contains(t.RedirectURL.Path, "/$path") { + t.RedirectURL.Path = strings.Replace(t.RedirectURL.Path, "/$path", "$path", 1) } - if strings.Contains(redirectURL.Path, "$path") { - redirectURL.Path = strings.Replace(redirectURL.Path, "$path", requestURL.Path, 1) - if t.StripPath != "" && strings.HasPrefix(redirectURL.Path, t.StripPath) { - redirectURL.Path = redirectURL.Path[len(t.StripPath):] + if strings.Contains(t.RedirectURL.Path, "$path") { + t.RedirectURL.Path = strings.Replace(t.RedirectURL.Path, "$path", requestURL.Path, 1) + if t.StripPath != "" && strings.HasPrefix(t.RedirectURL.Path, t.StripPath) { + t.RedirectURL.Path = t.RedirectURL.Path[len(t.StripPath):] } - if redirectURL.RawQuery == "" && requestURL.RawQuery != "" { - redirectURL.RawQuery = requestURL.RawQuery + if t.RedirectURL.RawQuery == "" && requestURL.RawQuery != "" { + t.RedirectURL.RawQuery = requestURL.RawQuery } } - return redirectURL + if t.RedirectURL.Path == "" { + t.RedirectURL.Path = "/" + } } diff --git a/route/target_test.go b/route/target_test.go index 7d9c7794b..21ad2bc42 100644 --- a/route/target_test.go +++ b/route/target_test.go @@ -5,7 +5,7 @@ import ( "testing" ) -func TestTarget_GetRedirectURL(t *testing.T) { +func TestTarget_BuildRedirectURL(t *testing.T) { type routeTest struct { req string want string @@ -95,8 +95,8 @@ func TestTarget_GetRedirectURL(t *testing.T) { target := route.Targets[0] for _, rt := range tt.tests { reqURL, _ := url.Parse("http://foo.com" + rt.req) - got := target.GetRedirectURL(reqURL) - if got.String() != rt.want { + target.BuildRedirectURL(reqURL) + if got := target.RedirectURL.String(); got != rt.want { t.Errorf("Got %s, wanted %s", got, rt.want) } } diff --git a/vendor/github.com/ryanuber/go-glob/LICENSE b/vendor/github.com/gobwas/glob/LICENSE similarity index 95% rename from vendor/github.com/ryanuber/go-glob/LICENSE rename to vendor/github.com/gobwas/glob/LICENSE index bdfbd9514..9d4735cad 100644 --- a/vendor/github.com/ryanuber/go-glob/LICENSE +++ b/vendor/github.com/gobwas/glob/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2014 Ryan Uber +Copyright (c) 2016 Sergey Kamardin Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/gobwas/glob/bench.sh b/vendor/github.com/gobwas/glob/bench.sh new file mode 100755 index 000000000..804cf22e6 --- /dev/null +++ b/vendor/github.com/gobwas/glob/bench.sh @@ -0,0 +1,26 @@ +#! /bin/bash + +bench() { + filename="/tmp/$1-$2.bench" + if test -e "${filename}"; + then + echo "Already exists ${filename}" + else + backup=`git rev-parse --abbrev-ref HEAD` + git checkout $1 + echo -n "Creating ${filename}... " + go test ./... -run=NONE -bench=$2 > "${filename}" -benchmem + echo "OK" + git checkout ${backup} + sleep 5 + fi +} + + +to=$1 +current=`git rev-parse --abbrev-ref HEAD` + +bench ${to} $2 +bench ${current} $2 + +benchcmp $3 "/tmp/${to}-$2.bench" "/tmp/${current}-$2.bench" diff --git a/vendor/github.com/gobwas/glob/compiler/compiler.go b/vendor/github.com/gobwas/glob/compiler/compiler.go new file mode 100644 index 000000000..02e7de80a --- /dev/null +++ b/vendor/github.com/gobwas/glob/compiler/compiler.go @@ -0,0 +1,525 @@ +package compiler + +// TODO use constructor with all matchers, and to their structs private +// TODO glue multiple Text nodes (like after QuoteMeta) + +import ( + "fmt" + "reflect" + + "github.com/gobwas/glob/match" + "github.com/gobwas/glob/syntax/ast" + "github.com/gobwas/glob/util/runes" +) + +func optimizeMatcher(matcher match.Matcher) match.Matcher { + switch m := matcher.(type) { + + case match.Any: + if len(m.Separators) == 0 { + return match.NewSuper() + } + + case match.AnyOf: + if len(m.Matchers) == 1 { + return m.Matchers[0] + } + + return m + + case match.List: + if m.Not == false && len(m.List) == 1 { + return match.NewText(string(m.List)) + } + + return m + + case match.BTree: + m.Left = optimizeMatcher(m.Left) + m.Right = optimizeMatcher(m.Right) + + r, ok := m.Value.(match.Text) + if !ok { + return m + } + + var ( + leftNil = m.Left == nil + rightNil = m.Right == nil + ) + if leftNil && rightNil { + return match.NewText(r.Str) + } + + _, leftSuper := m.Left.(match.Super) + lp, leftPrefix := m.Left.(match.Prefix) + la, leftAny := m.Left.(match.Any) + + _, rightSuper := m.Right.(match.Super) + rs, rightSuffix := m.Right.(match.Suffix) + ra, rightAny := m.Right.(match.Any) + + switch { + case leftSuper && rightSuper: + return match.NewContains(r.Str, false) + + case leftSuper && rightNil: + return match.NewSuffix(r.Str) + + case rightSuper && leftNil: + return match.NewPrefix(r.Str) + + case leftNil && rightSuffix: + return match.NewPrefixSuffix(r.Str, rs.Suffix) + + case rightNil && leftPrefix: + return match.NewPrefixSuffix(lp.Prefix, r.Str) + + case rightNil && leftAny: + return match.NewSuffixAny(r.Str, la.Separators) + + case leftNil && rightAny: + return match.NewPrefixAny(r.Str, ra.Separators) + } + + return m + } + + return matcher +} + +func compileMatchers(matchers []match.Matcher) (match.Matcher, error) { + if len(matchers) == 0 { + return nil, fmt.Errorf("compile error: need at least one matcher") + } + if len(matchers) == 1 { + return matchers[0], nil + } + if m := glueMatchers(matchers); m != nil { + return m, nil + } + + idx := -1 + maxLen := -1 + var val match.Matcher + for i, matcher := range matchers { + if l := matcher.Len(); l != -1 && l >= maxLen { + maxLen = l + idx = i + val = matcher + } + } + + if val == nil { // not found matcher with static length + r, err := compileMatchers(matchers[1:]) + if err != nil { + return nil, err + } + return match.NewBTree(matchers[0], nil, r), nil + } + + left := matchers[:idx] + var right []match.Matcher + if len(matchers) > idx+1 { + right = matchers[idx+1:] + } + + var l, r match.Matcher + var err error + if len(left) > 0 { + l, err = compileMatchers(left) + if err != nil { + return nil, err + } + } + + if len(right) > 0 { + r, err = compileMatchers(right) + if err != nil { + return nil, err + } + } + + return match.NewBTree(val, l, r), nil +} + +func glueMatchers(matchers []match.Matcher) match.Matcher { + if m := glueMatchersAsEvery(matchers); m != nil { + return m + } + if m := glueMatchersAsRow(matchers); m != nil { + return m + } + return nil +} + +func glueMatchersAsRow(matchers []match.Matcher) match.Matcher { + if len(matchers) <= 1 { + return nil + } + + var ( + c []match.Matcher + l int + ) + for _, matcher := range matchers { + if ml := matcher.Len(); ml == -1 { + return nil + } else { + c = append(c, matcher) + l += ml + } + } + return match.NewRow(l, c...) +} + +func glueMatchersAsEvery(matchers []match.Matcher) match.Matcher { + if len(matchers) <= 1 { + return nil + } + + var ( + hasAny bool + hasSuper bool + hasSingle bool + min int + separator []rune + ) + + for i, matcher := range matchers { + var sep []rune + + switch m := matcher.(type) { + case match.Super: + sep = []rune{} + hasSuper = true + + case match.Any: + sep = m.Separators + hasAny = true + + case match.Single: + sep = m.Separators + hasSingle = true + min++ + + case match.List: + if !m.Not { + return nil + } + sep = m.List + hasSingle = true + min++ + + default: + return nil + } + + // initialize + if i == 0 { + separator = sep + } + + if runes.Equal(sep, separator) { + continue + } + + return nil + } + + if hasSuper && !hasAny && !hasSingle { + return match.NewSuper() + } + + if hasAny && !hasSuper && !hasSingle { + return match.NewAny(separator) + } + + if (hasAny || hasSuper) && min > 0 && len(separator) == 0 { + return match.NewMin(min) + } + + every := match.NewEveryOf() + + if min > 0 { + every.Add(match.NewMin(min)) + + if !hasAny && !hasSuper { + every.Add(match.NewMax(min)) + } + } + + if len(separator) > 0 { + every.Add(match.NewContains(string(separator), true)) + } + + return every +} + +func minimizeMatchers(matchers []match.Matcher) []match.Matcher { + var done match.Matcher + var left, right, count int + + for l := 0; l < len(matchers); l++ { + for r := len(matchers); r > l; r-- { + if glued := glueMatchers(matchers[l:r]); glued != nil { + var swap bool + + if done == nil { + swap = true + } else { + cl, gl := done.Len(), glued.Len() + swap = cl > -1 && gl > -1 && gl > cl + swap = swap || count < r-l + } + + if swap { + done = glued + left = l + right = r + count = r - l + } + } + } + } + + if done == nil { + return matchers + } + + next := append(append([]match.Matcher{}, matchers[:left]...), done) + if right < len(matchers) { + next = append(next, matchers[right:]...) + } + + if len(next) == len(matchers) { + return next + } + + return minimizeMatchers(next) +} + +// minimizeAnyOf tries to apply some heuristics to minimize number of nodes in given tree +func minimizeTree(tree *ast.Node) *ast.Node { + switch tree.Kind { + case ast.KindAnyOf: + return minimizeTreeAnyOf(tree) + default: + return nil + } +} + +// minimizeAnyOf tries to find common children of given node of AnyOf pattern +// it searches for common children from left and from right +// if any common children are found – then it returns new optimized ast tree +// else it returns nil +func minimizeTreeAnyOf(tree *ast.Node) *ast.Node { + if !areOfSameKind(tree.Children, ast.KindPattern) { + return nil + } + + commonLeft, commonRight := commonChildren(tree.Children) + commonLeftCount, commonRightCount := len(commonLeft), len(commonRight) + if commonLeftCount == 0 && commonRightCount == 0 { // there are no common parts + return nil + } + + var result []*ast.Node + if commonLeftCount > 0 { + result = append(result, ast.NewNode(ast.KindPattern, nil, commonLeft...)) + } + + var anyOf []*ast.Node + for _, child := range tree.Children { + reuse := child.Children[commonLeftCount : len(child.Children)-commonRightCount] + var node *ast.Node + if len(reuse) == 0 { + // this pattern is completely reduced by commonLeft and commonRight patterns + // so it become nothing + node = ast.NewNode(ast.KindNothing, nil) + } else { + node = ast.NewNode(ast.KindPattern, nil, reuse...) + } + anyOf = appendIfUnique(anyOf, node) + } + switch { + case len(anyOf) == 1 && anyOf[0].Kind != ast.KindNothing: + result = append(result, anyOf[0]) + case len(anyOf) > 1: + result = append(result, ast.NewNode(ast.KindAnyOf, nil, anyOf...)) + } + + if commonRightCount > 0 { + result = append(result, ast.NewNode(ast.KindPattern, nil, commonRight...)) + } + + return ast.NewNode(ast.KindPattern, nil, result...) +} + +func commonChildren(nodes []*ast.Node) (commonLeft, commonRight []*ast.Node) { + if len(nodes) <= 1 { + return + } + + // find node that has least number of children + idx := leastChildren(nodes) + if idx == -1 { + return + } + tree := nodes[idx] + treeLength := len(tree.Children) + + // allocate max able size for rightCommon slice + // to get ability insert elements in reverse order (from end to start) + // without sorting + commonRight = make([]*ast.Node, treeLength) + lastRight := treeLength // will use this to get results as commonRight[lastRight:] + + var ( + breakLeft bool + breakRight bool + commonTotal int + ) + for i, j := 0, treeLength-1; commonTotal < treeLength && j >= 0 && !(breakLeft && breakRight); i, j = i+1, j-1 { + treeLeft := tree.Children[i] + treeRight := tree.Children[j] + + for k := 0; k < len(nodes) && !(breakLeft && breakRight); k++ { + // skip least children node + if k == idx { + continue + } + + restLeft := nodes[k].Children[i] + restRight := nodes[k].Children[j+len(nodes[k].Children)-treeLength] + + breakLeft = breakLeft || !treeLeft.Equal(restLeft) + + // disable searching for right common parts, if left part is already overlapping + breakRight = breakRight || (!breakLeft && j <= i) + breakRight = breakRight || !treeRight.Equal(restRight) + } + + if !breakLeft { + commonTotal++ + commonLeft = append(commonLeft, treeLeft) + } + if !breakRight { + commonTotal++ + lastRight = j + commonRight[j] = treeRight + } + } + + commonRight = commonRight[lastRight:] + + return +} + +func appendIfUnique(target []*ast.Node, val *ast.Node) []*ast.Node { + for _, n := range target { + if reflect.DeepEqual(n, val) { + return target + } + } + return append(target, val) +} + +func areOfSameKind(nodes []*ast.Node, kind ast.Kind) bool { + for _, n := range nodes { + if n.Kind != kind { + return false + } + } + return true +} + +func leastChildren(nodes []*ast.Node) int { + min := -1 + idx := -1 + for i, n := range nodes { + if idx == -1 || (len(n.Children) < min) { + min = len(n.Children) + idx = i + } + } + return idx +} + +func compileTreeChildren(tree *ast.Node, sep []rune) ([]match.Matcher, error) { + var matchers []match.Matcher + for _, desc := range tree.Children { + m, err := compile(desc, sep) + if err != nil { + return nil, err + } + matchers = append(matchers, optimizeMatcher(m)) + } + return matchers, nil +} + +func compile(tree *ast.Node, sep []rune) (m match.Matcher, err error) { + switch tree.Kind { + case ast.KindAnyOf: + // todo this could be faster on pattern_alternatives_combine_lite (see glob_test.go) + if n := minimizeTree(tree); n != nil { + return compile(n, sep) + } + matchers, err := compileTreeChildren(tree, sep) + if err != nil { + return nil, err + } + return match.NewAnyOf(matchers...), nil + + case ast.KindPattern: + if len(tree.Children) == 0 { + return match.NewNothing(), nil + } + matchers, err := compileTreeChildren(tree, sep) + if err != nil { + return nil, err + } + m, err = compileMatchers(minimizeMatchers(matchers)) + if err != nil { + return nil, err + } + + case ast.KindAny: + m = match.NewAny(sep) + + case ast.KindSuper: + m = match.NewSuper() + + case ast.KindSingle: + m = match.NewSingle(sep) + + case ast.KindNothing: + m = match.NewNothing() + + case ast.KindList: + l := tree.Value.(ast.List) + m = match.NewList([]rune(l.Chars), l.Not) + + case ast.KindRange: + r := tree.Value.(ast.Range) + m = match.NewRange(r.Lo, r.Hi, r.Not) + + case ast.KindText: + t := tree.Value.(ast.Text) + m = match.NewText(t.Text) + + default: + return nil, fmt.Errorf("could not compile tree: unknown node type") + } + + return optimizeMatcher(m), nil +} + +func Compile(tree *ast.Node, sep []rune) (match.Matcher, error) { + m, err := compile(tree, sep) + if err != nil { + return nil, err + } + + return m, nil +} diff --git a/vendor/github.com/gobwas/glob/glob.go b/vendor/github.com/gobwas/glob/glob.go new file mode 100644 index 000000000..2afde343a --- /dev/null +++ b/vendor/github.com/gobwas/glob/glob.go @@ -0,0 +1,80 @@ +package glob + +import ( + "github.com/gobwas/glob/compiler" + "github.com/gobwas/glob/syntax" +) + +// Glob represents compiled glob pattern. +type Glob interface { + Match(string) bool +} + +// Compile creates Glob for given pattern and strings (if any present after pattern) as separators. +// The pattern syntax is: +// +// pattern: +// { term } +// +// term: +// `*` matches any sequence of non-separator characters +// `**` matches any sequence of characters +// `?` matches any single non-separator character +// `[` [ `!` ] { character-range } `]` +// character class (must be non-empty) +// `{` pattern-list `}` +// pattern alternatives +// c matches character c (c != `*`, `**`, `?`, `\`, `[`, `{`, `}`) +// `\` c matches character c +// +// character-range: +// c matches character c (c != `\\`, `-`, `]`) +// `\` c matches character c +// lo `-` hi matches character c for lo <= c <= hi +// +// pattern-list: +// pattern { `,` pattern } +// comma-separated (without spaces) patterns +// +func Compile(pattern string, separators ...rune) (Glob, error) { + ast, err := syntax.Parse(pattern) + if err != nil { + return nil, err + } + + matcher, err := compiler.Compile(ast, separators) + if err != nil { + return nil, err + } + + return matcher, nil +} + +// MustCompile is the same as Compile, except that if Compile returns error, this will panic +func MustCompile(pattern string, separators ...rune) Glob { + g, err := Compile(pattern, separators...) + if err != nil { + panic(err) + } + + return g +} + +// QuoteMeta returns a string that quotes all glob pattern meta characters +// inside the argument text; For example, QuoteMeta(`{foo*}`) returns `\[foo\*\]`. +func QuoteMeta(s string) string { + b := make([]byte, 2*len(s)) + + // a byte loop is correct because all meta characters are ASCII + j := 0 + for i := 0; i < len(s); i++ { + if syntax.Special(s[i]) { + b[j] = '\\' + j++ + } + b[j] = s[i] + j++ + } + + return string(b[0:j]) +} diff --git a/vendor/github.com/gobwas/glob/match/any.go b/vendor/github.com/gobwas/glob/match/any.go new file mode 100644 index 000000000..514a9a5c4 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/any.go @@ -0,0 +1,45 @@ +package match + +import ( + "fmt" + "github.com/gobwas/glob/util/strings" +) + +type Any struct { + Separators []rune +} + +func NewAny(s []rune) Any { + return Any{s} +} + +func (self Any) Match(s string) bool { + return strings.IndexAnyRunes(s, self.Separators) == -1 +} + +func (self Any) Index(s string) (int, []int) { + found := strings.IndexAnyRunes(s, self.Separators) + switch found { + case -1: + case 0: + return 0, segments0 + default: + s = s[:found] + } + + segments := acquireSegments(len(s)) + for i := range s { + segments = append(segments, i) + } + segments = append(segments, len(s)) + + return 0, segments +} + +func (self Any) Len() int { + return lenNo +} + +func (self Any) String() string { + return fmt.Sprintf("", string(self.Separators)) +} diff --git a/vendor/github.com/gobwas/glob/match/any_of.go b/vendor/github.com/gobwas/glob/match/any_of.go new file mode 100644 index 000000000..8e65356cd --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/any_of.go @@ -0,0 +1,82 @@ +package match + +import "fmt" + +type AnyOf struct { + Matchers Matchers +} + +func NewAnyOf(m ...Matcher) AnyOf { + return AnyOf{Matchers(m)} +} + +func (self *AnyOf) Add(m Matcher) error { + self.Matchers = append(self.Matchers, m) + return nil +} + +func (self AnyOf) Match(s string) bool { + for _, m := range self.Matchers { + if m.Match(s) { + return true + } + } + + return false +} + +func (self AnyOf) Index(s string) (int, []int) { + index := -1 + + segments := acquireSegments(len(s)) + for _, m := range self.Matchers { + idx, seg := m.Index(s) + if idx == -1 { + continue + } + + if index == -1 || idx < index { + index = idx + segments = append(segments[:0], seg...) + continue + } + + if idx > index { + continue + } + + // here idx == index + segments = appendMerge(segments, seg) + } + + if index == -1 { + releaseSegments(segments) + return -1, nil + } + + return index, segments +} + +func (self AnyOf) Len() (l int) { + l = -1 + for _, m := range self.Matchers { + ml := m.Len() + switch { + case l == -1: + l = ml + continue + + case ml == -1: + return -1 + + case l != ml: + return -1 + } + } + + return +} + +func (self AnyOf) String() string { + return fmt.Sprintf("", self.Matchers) +} diff --git a/vendor/github.com/gobwas/glob/match/btree.go b/vendor/github.com/gobwas/glob/match/btree.go new file mode 100644 index 000000000..8302bf824 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/btree.go @@ -0,0 +1,185 @@ +package match + +import ( + "fmt" + "unicode/utf8" +) + +type BTree struct { + Value Matcher + Left Matcher + Right Matcher + ValueLengthRunes int + LeftLengthRunes int + RightLengthRunes int + LengthRunes int +} + +func NewBTree(Value, Left, Right Matcher) (tree BTree) { + tree.Value = Value + tree.Left = Left + tree.Right = Right + + lenOk := true + if tree.ValueLengthRunes = Value.Len(); tree.ValueLengthRunes == -1 { + lenOk = false + } + + if Left != nil { + if tree.LeftLengthRunes = Left.Len(); tree.LeftLengthRunes == -1 { + lenOk = false + } + } + + if Right != nil { + if tree.RightLengthRunes = Right.Len(); tree.RightLengthRunes == -1 { + lenOk = false + } + } + + if lenOk { + tree.LengthRunes = tree.LeftLengthRunes + tree.ValueLengthRunes + tree.RightLengthRunes + } else { + tree.LengthRunes = -1 + } + + return tree +} + +func (self BTree) Len() int { + return self.LengthRunes +} + +// todo? +func (self BTree) Index(s string) (index int, segments []int) { + //inputLen := len(s) + //// try to cut unnecessary parts + //// by knowledge of length of right and left part + //offset, limit := self.offsetLimit(inputLen) + //for offset < limit { + // // search for matching part in substring + // vi, segments := self.Value.Index(s[offset:limit]) + // if index == -1 { + // return -1, nil + // } + // if self.Left == nil { + // if index != offset { + // return -1, nil + // } + // } else { + // left := s[:offset+vi] + // i := self.Left.IndexSuffix(left) + // if i == -1 { + // return -1, nil + // } + // index = i + // } + // if self.Right != nil { + // for _, seg := range segments { + // right := s[:offset+vi+seg] + // } + // } + + // l := s[:offset+index] + // var left bool + // if self.Left != nil { + // left = self.Left.Index(l) + // } else { + // left = l == "" + // } + //} + + return -1, nil +} + +func (self BTree) Match(s string) bool { + inputLen := len(s) + // try to cut unnecessary parts + // by knowledge of length of right and left part + offset, limit := self.offsetLimit(inputLen) + + for offset < limit { + // search for matching part in substring + index, segments := self.Value.Index(s[offset:limit]) + if index == -1 { + releaseSegments(segments) + return false + } + + l := s[:offset+index] + var left bool + if self.Left != nil { + left = self.Left.Match(l) + } else { + left = l == "" + } + + if left { + for i := len(segments) - 1; i >= 0; i-- { + length := segments[i] + + var right bool + var r string + // if there is no string for the right branch + if inputLen <= offset+index+length { + r = "" + } else { + r = s[offset+index+length:] + } + + if self.Right != nil { + right = self.Right.Match(r) + } else { + right = r == "" + } + + if right { + releaseSegments(segments) + return true + } + } + } + + _, step := utf8.DecodeRuneInString(s[offset+index:]) + offset += index + step + + releaseSegments(segments) + } + + return false +} + +func (self BTree) offsetLimit(inputLen int) (offset int, limit int) { + // self.Length, self.RLen and self.LLen are values meaning the length of runes for each part + // here we manipulating byte length for better optimizations + // but these checks still works, cause minLen of 1-rune string is 1 byte. + if self.LengthRunes != -1 && self.LengthRunes > inputLen { + return 0, 0 + } + if self.LeftLengthRunes >= 0 { + offset = self.LeftLengthRunes + } + if self.RightLengthRunes >= 0 { + limit = inputLen - self.RightLengthRunes + } else { + limit = inputLen + } + return offset, limit +} + +func (self BTree) String() string { + const n string = "" + var l, r string + if self.Left == nil { + l = n + } else { + l = self.Left.String() + } + if self.Right == nil { + r = n + } else { + r = self.Right.String() + } + + return fmt.Sprintf("%s]>", l, self.Value, r) +} diff --git a/vendor/github.com/gobwas/glob/match/contains.go b/vendor/github.com/gobwas/glob/match/contains.go new file mode 100644 index 000000000..0998e95b0 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/contains.go @@ -0,0 +1,58 @@ +package match + +import ( + "fmt" + "strings" +) + +type Contains struct { + Needle string + Not bool +} + +func NewContains(needle string, not bool) Contains { + return Contains{needle, not} +} + +func (self Contains) Match(s string) bool { + return strings.Contains(s, self.Needle) != self.Not +} + +func (self Contains) Index(s string) (int, []int) { + var offset int + + idx := strings.Index(s, self.Needle) + + if !self.Not { + if idx == -1 { + return -1, nil + } + + offset = idx + len(self.Needle) + if len(s) <= offset { + return 0, []int{offset} + } + s = s[offset:] + } else if idx != -1 { + s = s[:idx] + } + + segments := acquireSegments(len(s) + 1) + for i := range s { + segments = append(segments, offset+i) + } + + return 0, append(segments, offset+len(s)) +} + +func (self Contains) Len() int { + return lenNo +} + +func (self Contains) String() string { + var not string + if self.Not { + not = "!" + } + return fmt.Sprintf("", not, self.Needle) +} diff --git a/vendor/github.com/gobwas/glob/match/every_of.go b/vendor/github.com/gobwas/glob/match/every_of.go new file mode 100644 index 000000000..7c968ee36 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/every_of.go @@ -0,0 +1,99 @@ +package match + +import ( + "fmt" +) + +type EveryOf struct { + Matchers Matchers +} + +func NewEveryOf(m ...Matcher) EveryOf { + return EveryOf{Matchers(m)} +} + +func (self *EveryOf) Add(m Matcher) error { + self.Matchers = append(self.Matchers, m) + return nil +} + +func (self EveryOf) Len() (l int) { + for _, m := range self.Matchers { + if ml := m.Len(); l > 0 { + l += ml + } else { + return -1 + } + } + + return +} + +func (self EveryOf) Index(s string) (int, []int) { + var index int + var offset int + + // make `in` with cap as len(s), + // cause it is the maximum size of output segments values + next := acquireSegments(len(s)) + current := acquireSegments(len(s)) + + sub := s + for i, m := range self.Matchers { + idx, seg := m.Index(sub) + if idx == -1 { + releaseSegments(next) + releaseSegments(current) + return -1, nil + } + + if i == 0 { + // we use copy here instead of `current = seg` + // cause seg is a slice from reusable buffer `in` + // and it could be overwritten in next iteration + current = append(current, seg...) + } else { + // clear the next + next = next[:0] + + delta := index - (idx + offset) + for _, ex := range current { + for _, n := range seg { + if ex+delta == n { + next = append(next, n) + } + } + } + + if len(next) == 0 { + releaseSegments(next) + releaseSegments(current) + return -1, nil + } + + current = append(current[:0], next...) + } + + index = idx + offset + sub = s[index:] + offset += idx + } + + releaseSegments(next) + + return index, current +} + +func (self EveryOf) Match(s string) bool { + for _, m := range self.Matchers { + if !m.Match(s) { + return false + } + } + + return true +} + +func (self EveryOf) String() string { + return fmt.Sprintf("", self.Matchers) +} diff --git a/vendor/github.com/gobwas/glob/match/list.go b/vendor/github.com/gobwas/glob/match/list.go new file mode 100644 index 000000000..7fd763ecd --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/list.go @@ -0,0 +1,49 @@ +package match + +import ( + "fmt" + "github.com/gobwas/glob/util/runes" + "unicode/utf8" +) + +type List struct { + List []rune + Not bool +} + +func NewList(list []rune, not bool) List { + return List{list, not} +} + +func (self List) Match(s string) bool { + r, w := utf8.DecodeRuneInString(s) + if len(s) > w { + return false + } + + inList := runes.IndexRune(self.List, r) != -1 + return inList == !self.Not +} + +func (self List) Len() int { + return lenOne +} + +func (self List) Index(s string) (int, []int) { + for i, r := range s { + if self.Not == (runes.IndexRune(self.List, r) == -1) { + return i, segmentsByRuneLength[utf8.RuneLen(r)] + } + } + + return -1, nil +} + +func (self List) String() string { + var not string + if self.Not { + not = "!" + } + + return fmt.Sprintf("", not, string(self.List)) +} diff --git a/vendor/github.com/gobwas/glob/match/match.go b/vendor/github.com/gobwas/glob/match/match.go new file mode 100644 index 000000000..f80e007fb --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/match.go @@ -0,0 +1,81 @@ +package match + +// todo common table of rune's length + +import ( + "fmt" + "strings" +) + +const lenOne = 1 +const lenZero = 0 +const lenNo = -1 + +type Matcher interface { + Match(string) bool + Index(string) (int, []int) + Len() int + String() string +} + +type Matchers []Matcher + +func (m Matchers) String() string { + var s []string + for _, matcher := range m { + s = append(s, fmt.Sprint(matcher)) + } + + return fmt.Sprintf("%s", strings.Join(s, ",")) +} + +// appendMerge merges and sorts given already SORTED and UNIQUE segments. +func appendMerge(target, sub []int) []int { + lt, ls := len(target), len(sub) + out := make([]int, 0, lt+ls) + + for x, y := 0, 0; x < lt || y < ls; { + if x >= lt { + out = append(out, sub[y:]...) + break + } + + if y >= ls { + out = append(out, target[x:]...) + break + } + + xValue := target[x] + yValue := sub[y] + + switch { + + case xValue == yValue: + out = append(out, xValue) + x++ + y++ + + case xValue < yValue: + out = append(out, xValue) + x++ + + case yValue < xValue: + out = append(out, yValue) + y++ + + } + } + + target = append(target[:0], out...) + + return target +} + +func reverseSegments(input []int) { + l := len(input) + m := l / 2 + + for i := 0; i < m; i++ { + input[i], input[l-i-1] = input[l-i-1], input[i] + } +} diff --git a/vendor/github.com/gobwas/glob/match/max.go b/vendor/github.com/gobwas/glob/match/max.go new file mode 100644 index 000000000..d72f69eff --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/max.go @@ -0,0 +1,49 @@ +package match + +import ( + "fmt" + "unicode/utf8" +) + +type Max struct { + Limit int +} + +func NewMax(l int) Max { + return Max{l} +} + +func (self Max) Match(s string) bool { + var l int + for range s { + l += 1 + if l > self.Limit { + return false + } + } + + return true +} + +func (self Max) Index(s string) (int, []int) { + segments := acquireSegments(self.Limit + 1) + segments = append(segments, 0) + var count int + for i, r := range s { + count++ + if count > self.Limit { + break + } + segments = append(segments, i+utf8.RuneLen(r)) + } + + return 0, segments +} + +func (self Max) Len() int { + return lenNo +} + +func (self Max) String() string { + return fmt.Sprintf("", self.Limit) +} diff --git a/vendor/github.com/gobwas/glob/match/min.go b/vendor/github.com/gobwas/glob/match/min.go new file mode 100644 index 000000000..db57ac8eb --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/min.go @@ -0,0 +1,57 @@ +package match + +import ( + "fmt" + "unicode/utf8" +) + +type Min struct { + Limit int +} + +func NewMin(l int) Min { + return Min{l} +} + +func (self Min) Match(s string) bool { + var l int + for range s { + l += 1 + if l >= self.Limit { + return true + } + } + + return false +} + +func (self Min) Index(s string) (int, []int) { + var count int + + c := len(s) - self.Limit + 1 + if c <= 0 { + return -1, nil + } + + segments := acquireSegments(c) + for i, r := range s { + count++ + if count >= self.Limit { + segments = append(segments, i+utf8.RuneLen(r)) + } + } + + if len(segments) == 0 { + return -1, nil + } + + return 0, segments +} + +func (self Min) Len() int { + return lenNo +} + +func (self Min) String() string { + return fmt.Sprintf("", self.Limit) +} diff --git a/vendor/github.com/gobwas/glob/match/nothing.go b/vendor/github.com/gobwas/glob/match/nothing.go new file mode 100644 index 000000000..0d4ecd36b --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/nothing.go @@ -0,0 +1,27 @@ +package match + +import ( + "fmt" +) + +type Nothing struct{} + +func NewNothing() Nothing { + return Nothing{} +} + +func (self Nothing) Match(s string) bool { + return len(s) == 0 +} + +func (self Nothing) Index(s string) (int, []int) { + return 0, segments0 +} + +func (self Nothing) Len() int { + return lenZero +} + +func (self Nothing) String() string { + return fmt.Sprintf("") +} diff --git a/vendor/github.com/gobwas/glob/match/prefix.go b/vendor/github.com/gobwas/glob/match/prefix.go new file mode 100644 index 000000000..a7347250e --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/prefix.go @@ -0,0 +1,50 @@ +package match + +import ( + "fmt" + "strings" + "unicode/utf8" +) + +type Prefix struct { + Prefix string +} + +func NewPrefix(p string) Prefix { + return Prefix{p} +} + +func (self Prefix) Index(s string) (int, []int) { + idx := strings.Index(s, self.Prefix) + if idx == -1 { + return -1, nil + } + + length := len(self.Prefix) + var sub string + if len(s) > idx+length { + sub = s[idx+length:] + } else { + sub = "" + } + + segments := acquireSegments(len(sub) + 1) + segments = append(segments, length) + for i, r := range sub { + segments = append(segments, length+i+utf8.RuneLen(r)) + } + + return idx, segments +} + +func (self Prefix) Len() int { + return lenNo +} + +func (self Prefix) Match(s string) bool { + return strings.HasPrefix(s, self.Prefix) +} + +func (self Prefix) String() string { + return fmt.Sprintf("", self.Prefix) +} diff --git a/vendor/github.com/gobwas/glob/match/prefix_any.go b/vendor/github.com/gobwas/glob/match/prefix_any.go new file mode 100644 index 000000000..8ee58fe1b --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/prefix_any.go @@ -0,0 +1,55 @@ +package match + +import ( + "fmt" + "strings" + "unicode/utf8" + + sutil "github.com/gobwas/glob/util/strings" +) + +type PrefixAny struct { + Prefix string + Separators []rune +} + +func NewPrefixAny(s string, sep []rune) PrefixAny { + return PrefixAny{s, sep} +} + +func (self PrefixAny) Index(s string) (int, []int) { + idx := strings.Index(s, self.Prefix) + if idx == -1 { + return -1, nil + } + + n := len(self.Prefix) + sub := s[idx+n:] + i := sutil.IndexAnyRunes(sub, self.Separators) + if i > -1 { + sub = sub[:i] + } + + seg := acquireSegments(len(sub) + 1) + seg = append(seg, n) + for i, r := range sub { + seg = append(seg, n+i+utf8.RuneLen(r)) + } + + return idx, seg +} + +func (self PrefixAny) Len() int { + return lenNo +} + +func (self PrefixAny) Match(s string) bool { + if !strings.HasPrefix(s, self.Prefix) { + return false + } + return sutil.IndexAnyRunes(s[len(self.Prefix):], self.Separators) == -1 +} + +func (self PrefixAny) String() string { + return fmt.Sprintf("", self.Prefix, string(self.Separators)) +} diff --git a/vendor/github.com/gobwas/glob/match/prefix_suffix.go b/vendor/github.com/gobwas/glob/match/prefix_suffix.go new file mode 100644 index 000000000..8208085a1 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/prefix_suffix.go @@ -0,0 +1,62 @@ +package match + +import ( + "fmt" + "strings" +) + +type PrefixSuffix struct { + Prefix, Suffix string +} + +func NewPrefixSuffix(p, s string) PrefixSuffix { + return PrefixSuffix{p, s} +} + +func (self PrefixSuffix) Index(s string) (int, []int) { + prefixIdx := strings.Index(s, self.Prefix) + if prefixIdx == -1 { + return -1, nil + } + + suffixLen := len(self.Suffix) + if suffixLen <= 0 { + return prefixIdx, []int{len(s) - prefixIdx} + } + + if (len(s) - prefixIdx) <= 0 { + return -1, nil + } + + segments := acquireSegments(len(s) - prefixIdx) + for sub := s[prefixIdx:]; ; { + suffixIdx := strings.LastIndex(sub, self.Suffix) + if suffixIdx == -1 { + break + } + + segments = append(segments, suffixIdx+suffixLen) + sub = sub[:suffixIdx] + } + + if len(segments) == 0 { + releaseSegments(segments) + return -1, nil + } + + reverseSegments(segments) + + return prefixIdx, segments +} + +func (self PrefixSuffix) Len() int { + return lenNo +} + +func (self PrefixSuffix) Match(s string) bool { + return strings.HasPrefix(s, self.Prefix) && strings.HasSuffix(s, self.Suffix) +} + +func (self PrefixSuffix) String() string { + return fmt.Sprintf("", self.Prefix, self.Suffix) +} diff --git a/vendor/github.com/gobwas/glob/match/range.go b/vendor/github.com/gobwas/glob/match/range.go new file mode 100644 index 000000000..ce30245a4 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/range.go @@ -0,0 +1,48 @@ +package match + +import ( + "fmt" + "unicode/utf8" +) + +type Range struct { + Lo, Hi rune + Not bool +} + +func NewRange(lo, hi rune, not bool) Range { + return Range{lo, hi, not} +} + +func (self Range) Len() int { + return lenOne +} + +func (self Range) Match(s string) bool { + r, w := utf8.DecodeRuneInString(s) + if len(s) > w { + return false + } + + inRange := r >= self.Lo && r <= self.Hi + + return inRange == !self.Not +} + +func (self Range) Index(s string) (int, []int) { + for i, r := range s { + if self.Not != (r >= self.Lo && r <= self.Hi) { + return i, segmentsByRuneLength[utf8.RuneLen(r)] + } + } + + return -1, nil +} + +func (self Range) String() string { + var not string + if self.Not { + not = "!" + } + return fmt.Sprintf("", not, string(self.Lo), string(self.Hi)) +} diff --git a/vendor/github.com/gobwas/glob/match/row.go b/vendor/github.com/gobwas/glob/match/row.go new file mode 100644 index 000000000..4379042e4 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/row.go @@ -0,0 +1,77 @@ +package match + +import ( + "fmt" +) + +type Row struct { + Matchers Matchers + RunesLength int + Segments []int +} + +func NewRow(len int, m ...Matcher) Row { + return Row{ + Matchers: Matchers(m), + RunesLength: len, + Segments: []int{len}, + } +} + +func (self Row) matchAll(s string) bool { + var idx int + for _, m := range self.Matchers { + length := m.Len() + + var next, i int + for next = range s[idx:] { + i++ + if i == length { + break + } + } + + if i < length || !m.Match(s[idx:idx+next+1]) { + return false + } + + idx += next + 1 + } + + return true +} + +func (self Row) lenOk(s string) bool { + var i int + for range s { + i++ + if i > self.RunesLength { + return false + } + } + return self.RunesLength == i +} + +func (self Row) Match(s string) bool { + return self.lenOk(s) && self.matchAll(s) +} + +func (self Row) Len() (l int) { + return self.RunesLength +} + +func (self Row) Index(s string) (int, []int) { + for i := range s { + if len(s[i:]) < self.RunesLength { + break + } + if self.matchAll(s[i:]) { + return i, self.Segments + } + } + return -1, nil +} + +func (self Row) String() string { + return fmt.Sprintf("", self.RunesLength, self.Matchers) +} diff --git a/vendor/github.com/gobwas/glob/match/segments.go b/vendor/github.com/gobwas/glob/match/segments.go new file mode 100644 index 000000000..9ea6f3094 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/segments.go @@ -0,0 +1,91 @@ +package match + +import ( + "sync" +) + +type SomePool interface { + Get() []int + Put([]int) +} + +var segmentsPools [1024]sync.Pool + +func toPowerOfTwo(v int) int { + v-- + v |= v >> 1 + v |= v >> 2 + v |= v >> 4 + v |= v >> 8 + v |= v >> 16 + v++ + + return v +} + +const ( + cacheFrom = 16 + cacheToAndHigher = 1024 + cacheFromIndex = 15 + cacheToAndHigherIndex = 1023 +) + +var ( + segments0 = []int{0} + segments1 = []int{1} + segments2 = []int{2} + segments3 = []int{3} + segments4 = []int{4} +) + +var segmentsByRuneLength [5][]int = [5][]int{ + 0: segments0, + 1: segments1, + 2: segments2, + 3: segments3, + 4: segments4, +} + +func init() { + for i := cacheToAndHigher; i >= cacheFrom; i >>= 1 { + func(i int) { + segmentsPools[i-1] = sync.Pool{New: func() interface{} { + return make([]int, 0, i) + }} + }(i) + } +} + +func getTableIndex(c int) int { + p := toPowerOfTwo(c) + switch { + case p >= cacheToAndHigher: + return cacheToAndHigherIndex + case p <= cacheFrom: + return cacheFromIndex + default: + return p - 1 + } +} + +func acquireSegments(c int) []int { + // make []int with less capacity than cacheFrom + // is faster than acquiring it from pool + if c < cacheFrom { + return make([]int, 0, c) + } + + return segmentsPools[getTableIndex(c)].Get().([]int)[:0] +} + +func releaseSegments(s []int) { + c := cap(s) + + // make []int with less capacity than cacheFrom + // is faster than acquiring it from pool + if c < cacheFrom { + return + } + + segmentsPools[getTableIndex(c)].Put(s) +} diff --git a/vendor/github.com/gobwas/glob/match/single.go b/vendor/github.com/gobwas/glob/match/single.go new file mode 100644 index 000000000..ee6e3954c --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/single.go @@ -0,0 +1,43 @@ +package match + +import ( + "fmt" + "github.com/gobwas/glob/util/runes" + "unicode/utf8" +) + +// single represents ? +type Single struct { + Separators []rune +} + +func NewSingle(s []rune) Single { + return Single{s} +} + +func (self Single) Match(s string) bool { + r, w := utf8.DecodeRuneInString(s) + if len(s) > w { + return false + } + + return runes.IndexRune(self.Separators, r) == -1 +} + +func (self Single) Len() int { + return lenOne +} + +func (self Single) Index(s string) (int, []int) { + for i, r := range s { + if runes.IndexRune(self.Separators, r) == -1 { + return i, segmentsByRuneLength[utf8.RuneLen(r)] + } + } + + return -1, nil +} + +func (self Single) String() string { + return fmt.Sprintf("", string(self.Separators)) +} diff --git a/vendor/github.com/gobwas/glob/match/suffix.go b/vendor/github.com/gobwas/glob/match/suffix.go new file mode 100644 index 000000000..85bea8c68 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/suffix.go @@ -0,0 +1,35 @@ +package match + +import ( + "fmt" + "strings" +) + +type Suffix struct { + Suffix string +} + +func NewSuffix(s string) Suffix { + return Suffix{s} +} + +func (self Suffix) Len() int { + return lenNo +} + +func (self Suffix) Match(s string) bool { + return strings.HasSuffix(s, self.Suffix) +} + +func (self Suffix) Index(s string) (int, []int) { + idx := strings.Index(s, self.Suffix) + if idx == -1 { + return -1, nil + } + + return 0, []int{idx + len(self.Suffix)} +} + +func (self Suffix) String() string { + return fmt.Sprintf("", self.Suffix) +} diff --git a/vendor/github.com/gobwas/glob/match/suffix_any.go b/vendor/github.com/gobwas/glob/match/suffix_any.go new file mode 100644 index 000000000..c5106f819 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/suffix_any.go @@ -0,0 +1,43 @@ +package match + +import ( + "fmt" + "strings" + + sutil "github.com/gobwas/glob/util/strings" +) + +type SuffixAny struct { + Suffix string + Separators []rune +} + +func NewSuffixAny(s string, sep []rune) SuffixAny { + return SuffixAny{s, sep} +} + +func (self SuffixAny) Index(s string) (int, []int) { + idx := strings.Index(s, self.Suffix) + if idx == -1 { + return -1, nil + } + + i := sutil.LastIndexAnyRunes(s[:idx], self.Separators) + 1 + + return i, []int{idx + len(self.Suffix) - i} +} + +func (self SuffixAny) Len() int { + return lenNo +} + +func (self SuffixAny) Match(s string) bool { + if !strings.HasSuffix(s, self.Suffix) { + return false + } + return sutil.IndexAnyRunes(s[:len(s)-len(self.Suffix)], self.Separators) == -1 +} + +func (self SuffixAny) String() string { + return fmt.Sprintf("", string(self.Separators), self.Suffix) +} diff --git a/vendor/github.com/gobwas/glob/match/super.go b/vendor/github.com/gobwas/glob/match/super.go new file mode 100644 index 000000000..3875950bb --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/super.go @@ -0,0 +1,33 @@ +package match + +import ( + "fmt" +) + +type Super struct{} + +func NewSuper() Super { + return Super{} +} + +func (self Super) Match(s string) bool { + return true +} + +func (self Super) Len() int { + return lenNo +} + +func (self Super) Index(s string) (int, []int) { + segments := acquireSegments(len(s) + 1) + for i := range s { + segments = append(segments, i) + } + segments = append(segments, len(s)) + + return 0, segments +} + +func (self Super) String() string { + return fmt.Sprintf("") +} diff --git a/vendor/github.com/gobwas/glob/match/text.go b/vendor/github.com/gobwas/glob/match/text.go new file mode 100644 index 000000000..0a17616d3 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/text.go @@ -0,0 +1,45 @@ +package match + +import ( + "fmt" + "strings" + "unicode/utf8" +) + +// raw represents raw string to match +type Text struct { + Str string + RunesLength int + BytesLength int + Segments []int +} + +func NewText(s string) Text { + return Text{ + Str: s, + RunesLength: utf8.RuneCountInString(s), + BytesLength: len(s), + Segments: []int{len(s)}, + } +} + +func (self Text) Match(s string) bool { + return self.Str == s +} + +func (self Text) Len() int { + return self.RunesLength +} + +func (self Text) Index(s string) (int, []int) { + index := strings.Index(s, self.Str) + if index == -1 { + return -1, nil + } + + return index, self.Segments +} + +func (self Text) String() string { + return fmt.Sprintf("", self.Str) +} diff --git a/vendor/github.com/gobwas/glob/readme.md b/vendor/github.com/gobwas/glob/readme.md new file mode 100644 index 000000000..f58144e73 --- /dev/null +++ b/vendor/github.com/gobwas/glob/readme.md @@ -0,0 +1,148 @@ +# glob.[go](https://golang.org) + +[![GoDoc][godoc-image]][godoc-url] [![Build Status][travis-image]][travis-url] + +> Go Globbing Library. + +## Install + +```shell + go get github.com/gobwas/glob +``` + +## Example + +```go + +package main + +import "github.com/gobwas/glob" + +func main() { + var g glob.Glob + + // create simple glob + g = glob.MustCompile("*.github.com") + g.Match("api.github.com") // true + + // quote meta characters and then create simple glob + g = glob.MustCompile(glob.QuoteMeta("*.github.com")) + g.Match("*.github.com") // true + + // create new glob with set of delimiters as ["."] + g = glob.MustCompile("api.*.com", '.') + g.Match("api.github.com") // true + g.Match("api.gi.hub.com") // false + + // create new glob with set of delimiters as ["."] + // but now with super wildcard + g = glob.MustCompile("api.**.com", '.') + g.Match("api.github.com") // true + g.Match("api.gi.hub.com") // true + + // create glob with single symbol wildcard + g = glob.MustCompile("?at") + g.Match("cat") // true + g.Match("fat") // true + g.Match("at") // false + + // create glob with single symbol wildcard and delimiters ['f'] + g = glob.MustCompile("?at", 'f') + g.Match("cat") // true + g.Match("fat") // false + g.Match("at") // false + + // create glob with character-list matchers + g = glob.MustCompile("[abc]at") + g.Match("cat") // true + g.Match("bat") // true + g.Match("fat") // false + g.Match("at") // false + + // create glob with character-list matchers + g = glob.MustCompile("[!abc]at") + g.Match("cat") // false + g.Match("bat") // false + g.Match("fat") // true + g.Match("at") // false + + // create glob with character-range matchers + g = glob.MustCompile("[a-c]at") + g.Match("cat") // true + g.Match("bat") // true + g.Match("fat") // false + g.Match("at") // false + + // create glob with character-range matchers + g = glob.MustCompile("[!a-c]at") + g.Match("cat") // false + g.Match("bat") // false + g.Match("fat") // true + g.Match("at") // false + + // create glob with pattern-alternatives list + g = glob.MustCompile("{cat,bat,[fr]at}") + g.Match("cat") // true + g.Match("bat") // true + g.Match("fat") // true + g.Match("rat") // true + g.Match("at") // false + g.Match("zat") // false +} + +``` + +## Performance + +This library is created for compile-once patterns. This means, that compilation could take time, but +strings matching is done faster, than in case when always parsing template. + +If you will not use compiled `glob.Glob` object, and do `g := glob.MustCompile(pattern); g.Match(...)` every time, then your code will be much more slower. + +Run `go test -bench=.` from source root to see the benchmarks: + +Pattern | Fixture | Match | Speed (ns/op) +--------|---------|-------|-------------- +`[a-z][!a-x]*cat*[h][!b]*eyes*` | `my cat has very bright eyes` | `true` | 432 +`[a-z][!a-x]*cat*[h][!b]*eyes*` | `my dog has very bright eyes` | `false` | 199 +`https://*.google.*` | `https://account.google.com` | `true` | 96 +`https://*.google.*` | `https://google.com` | `false` | 66 +`{https://*.google.*,*yandex.*,*yahoo.*,*mail.ru}` | `http://yahoo.com` | `true` | 163 +`{https://*.google.*,*yandex.*,*yahoo.*,*mail.ru}` | `http://google.com` | `false` | 197 +`{https://*gobwas.com,http://exclude.gobwas.com}` | `https://safe.gobwas.com` | `true` | 22 +`{https://*gobwas.com,http://exclude.gobwas.com}` | `http://safe.gobwas.com` | `false` | 24 +`abc*` | `abcdef` | `true` | 8.15 +`abc*` | `af` | `false` | 5.68 +`*def` | `abcdef` | `true` | 8.84 +`*def` | `af` | `false` | 5.74 +`ab*ef` | `abcdef` | `true` | 15.2 +`ab*ef` | `af` | `false` | 10.4 + +The same things with `regexp` package: + +Pattern | Fixture | Match | Speed (ns/op) +--------|---------|-------|-------------- +`^[a-z][^a-x].*cat.*[h][^b].*eyes.*$` | `my cat has very bright eyes` | `true` | 2553 +`^[a-z][^a-x].*cat.*[h][^b].*eyes.*$` | `my dog has very bright eyes` | `false` | 1383 +`^https:\/\/.*\.google\..*$` | `https://account.google.com` | `true` | 1205 +`^https:\/\/.*\.google\..*$` | `https://google.com` | `false` | 767 +`^(https:\/\/.*\.google\..*|.*yandex\..*|.*yahoo\..*|.*mail\.ru)$` | `http://yahoo.com` | `true` | 1435 +`^(https:\/\/.*\.google\..*|.*yandex\..*|.*yahoo\..*|.*mail\.ru)$` | `http://google.com` | `false` | 1674 +`^(https:\/\/.*gobwas\.com|http://exclude.gobwas.com)$` | `https://safe.gobwas.com` | `true` | 1039 +`^(https:\/\/.*gobwas\.com|http://exclude.gobwas.com)$` | `http://safe.gobwas.com` | `false` | 272 +`^abc.*$` | `abcdef` | `true` | 237 +`^abc.*$` | `af` | `false` | 100 +`^.*def$` | `abcdef` | `true` | 464 +`^.*def$` | `af` | `false` | 265 +`^ab.*ef$` | `abcdef` | `true` | 375 +`^ab.*ef$` | `af` | `false` | 145 + +[godoc-image]: https://godoc.org/github.com/gobwas/glob?status.svg +[godoc-url]: https://godoc.org/github.com/gobwas/glob +[travis-image]: https://travis-ci.org/gobwas/glob.svg?branch=master +[travis-url]: https://travis-ci.org/gobwas/glob + +## Syntax + +Syntax is inspired by [standard wildcards](http://tldp.org/LDP/GNU-Linux-Tools-Summary/html/x11655.htm), +except that `**` is aka super-asterisk, that do not sensitive for separators. \ No newline at end of file diff --git a/vendor/github.com/gobwas/glob/syntax/ast/ast.go b/vendor/github.com/gobwas/glob/syntax/ast/ast.go new file mode 100644 index 000000000..3220a694a --- /dev/null +++ b/vendor/github.com/gobwas/glob/syntax/ast/ast.go @@ -0,0 +1,122 @@ +package ast + +import ( + "bytes" + "fmt" +) + +type Node struct { + Parent *Node + Children []*Node + Value interface{} + Kind Kind +} + +func NewNode(k Kind, v interface{}, ch ...*Node) *Node { + n := &Node{ + Kind: k, + Value: v, + } + for _, c := range ch { + Insert(n, c) + } + return n +} + +func (a *Node) Equal(b *Node) bool { + if a.Kind != b.Kind { + return false + } + if a.Value != b.Value { + return false + } + if len(a.Children) != len(b.Children) { + return false + } + for i, c := range a.Children { + if !c.Equal(b.Children[i]) { + return false + } + } + return true +} + +func (a *Node) String() string { + var buf bytes.Buffer + buf.WriteString(a.Kind.String()) + if a.Value != nil { + buf.WriteString(" =") + buf.WriteString(fmt.Sprintf("%v", a.Value)) + } + if len(a.Children) > 0 { + buf.WriteString(" [") + for i, c := range a.Children { + if i > 0 { + buf.WriteString(", ") + } + buf.WriteString(c.String()) + } + buf.WriteString("]") + } + return buf.String() +} + +func Insert(parent *Node, children ...*Node) { + parent.Children = append(parent.Children, children...) + for _, ch := range children { + ch.Parent = parent + } +} + +type List struct { + Not bool + Chars string +} + +type Range struct { + Not bool + Lo, Hi rune +} + +type Text struct { + Text string +} + +type Kind int + +const ( + KindNothing Kind = iota + KindPattern + KindList + KindRange + KindText + KindAny + KindSuper + KindSingle + KindAnyOf +) + +func (k Kind) String() string { + switch k { + case KindNothing: + return "Nothing" + case KindPattern: + return "Pattern" + case KindList: + return "List" + case KindRange: + return "Range" + case KindText: + return "Text" + case KindAny: + return "Any" + case KindSuper: + return "Super" + case KindSingle: + return "Single" + case KindAnyOf: + return "AnyOf" + default: + return "" + } +} diff --git a/vendor/github.com/gobwas/glob/syntax/ast/parser.go b/vendor/github.com/gobwas/glob/syntax/ast/parser.go new file mode 100644 index 000000000..429b40943 --- /dev/null +++ b/vendor/github.com/gobwas/glob/syntax/ast/parser.go @@ -0,0 +1,157 @@ +package ast + +import ( + "errors" + "fmt" + "github.com/gobwas/glob/syntax/lexer" + "unicode/utf8" +) + +type Lexer interface { + Next() lexer.Token +} + +type parseFn func(*Node, Lexer) (parseFn, *Node, error) + +func Parse(lexer Lexer) (*Node, error) { + var parser parseFn + + root := NewNode(KindPattern, nil) + + var ( + tree *Node + err error + ) + for parser, tree = parserMain, root; parser != nil; { + parser, tree, err = parser(tree, lexer) + if err != nil { + return nil, err + } + } + + return root, nil +} + +func parserMain(tree *Node, lex Lexer) (parseFn, *Node, error) { + for { + token := lex.Next() + switch token.Type { + case lexer.EOF: + return nil, tree, nil + + case lexer.Error: + return nil, tree, errors.New(token.Raw) + + case lexer.Text: + Insert(tree, NewNode(KindText, Text{token.Raw})) + return parserMain, tree, nil + + case lexer.Any: + Insert(tree, NewNode(KindAny, nil)) + return parserMain, tree, nil + + case lexer.Super: + Insert(tree, NewNode(KindSuper, nil)) + return parserMain, tree, nil + + case lexer.Single: + Insert(tree, NewNode(KindSingle, nil)) + return parserMain, tree, nil + + case lexer.RangeOpen: + return parserRange, tree, nil + + case lexer.TermsOpen: + a := NewNode(KindAnyOf, nil) + Insert(tree, a) + + p := NewNode(KindPattern, nil) + Insert(a, p) + + return parserMain, p, nil + + case lexer.Separator: + p := NewNode(KindPattern, nil) + Insert(tree.Parent, p) + + return parserMain, p, nil + + case lexer.TermsClose: + return parserMain, tree.Parent.Parent, nil + + default: + return nil, tree, fmt.Errorf("unexpected token: %s", token) + } + } + return nil, tree, fmt.Errorf("unknown error") +} + +func parserRange(tree *Node, lex Lexer) (parseFn, *Node, error) { + var ( + not bool + lo rune + hi rune + chars string + ) + for { + token := lex.Next() + switch token.Type { + case lexer.EOF: + return nil, tree, errors.New("unexpected end") + + case lexer.Error: + return nil, tree, errors.New(token.Raw) + + case lexer.Not: + not = true + + case lexer.RangeLo: + r, w := utf8.DecodeRuneInString(token.Raw) + if len(token.Raw) > w { + return nil, tree, fmt.Errorf("unexpected length of lo character") + } + lo = r + + case lexer.RangeBetween: + // + + case lexer.RangeHi: + r, w := utf8.DecodeRuneInString(token.Raw) + if len(token.Raw) > w { + return nil, tree, fmt.Errorf("unexpected length of lo character") + } + + hi = r + + if hi < lo { + return nil, tree, fmt.Errorf("hi character '%s' should be greater than lo '%s'", string(hi), string(lo)) + } + + case lexer.Text: + chars = token.Raw + + case lexer.RangeClose: + isRange := lo != 0 && hi != 0 + isChars := chars != "" + + if isChars == isRange { + return nil, tree, fmt.Errorf("could not parse range") + } + + if isRange { + Insert(tree, NewNode(KindRange, Range{ + Lo: lo, + Hi: hi, + Not: not, + })) + } else { + Insert(tree, NewNode(KindList, List{ + Chars: chars, + Not: not, + })) + } + + return parserMain, tree, nil + } + } +} diff --git a/vendor/github.com/gobwas/glob/syntax/lexer/lexer.go b/vendor/github.com/gobwas/glob/syntax/lexer/lexer.go new file mode 100644 index 000000000..a1c8d1962 --- /dev/null +++ b/vendor/github.com/gobwas/glob/syntax/lexer/lexer.go @@ -0,0 +1,273 @@ +package lexer + +import ( + "bytes" + "fmt" + "github.com/gobwas/glob/util/runes" + "unicode/utf8" +) + +const ( + char_any = '*' + char_comma = ',' + char_single = '?' + char_escape = '\\' + char_range_open = '[' + char_range_close = ']' + char_terms_open = '{' + char_terms_close = '}' + char_range_not = '!' + char_range_between = '-' +) + +var specials = []byte{ + char_any, + char_single, + char_escape, + char_range_open, + char_range_close, + char_terms_open, + char_terms_close, +} + +func Special(c byte) bool { + return bytes.IndexByte(specials, c) != -1 +} + +type tokens []Token + +func (i *tokens) shift() (ret Token) { + ret = (*i)[0] + copy(*i, (*i)[1:]) + *i = (*i)[:len(*i)-1] + return +} + +func (i *tokens) push(v Token) { + *i = append(*i, v) +} + +func (i *tokens) empty() bool { + return len(*i) == 0 +} + +var eof rune = 0 + +type lexer struct { + data string + pos int + err error + + tokens tokens + termsLevel int + + lastRune rune + lastRuneSize int + hasRune bool +} + +func NewLexer(source string) *lexer { + l := &lexer{ + data: source, + tokens: tokens(make([]Token, 0, 4)), + } + return l +} + +func (l *lexer) Next() Token { + if l.err != nil { + return Token{Error, l.err.Error()} + } + if !l.tokens.empty() { + return l.tokens.shift() + } + + l.fetchItem() + return l.Next() +} + +func (l *lexer) peek() (r rune, w int) { + if l.pos == len(l.data) { + return eof, 0 + } + + r, w = utf8.DecodeRuneInString(l.data[l.pos:]) + if r == utf8.RuneError { + l.errorf("could not read rune") + r = eof + w = 0 + } + + return +} + +func (l *lexer) read() rune { + if l.hasRune { + l.hasRune = false + l.seek(l.lastRuneSize) + return l.lastRune + } + + r, s := l.peek() + l.seek(s) + + l.lastRune = r + l.lastRuneSize = s + + return r +} + +func (l *lexer) seek(w int) { + l.pos += w +} + +func (l *lexer) unread() { + if l.hasRune { + l.errorf("could not unread rune") + return + } + l.seek(-l.lastRuneSize) + l.hasRune = true +} + +func (l *lexer) errorf(f string, v ...interface{}) { + l.err = fmt.Errorf(f, v...) +} + +func (l *lexer) inTerms() bool { + return l.termsLevel > 0 +} + +func (l *lexer) termsEnter() { + l.termsLevel++ +} + +func (l *lexer) termsLeave() { + l.termsLevel-- +} + +var inTextBreakers = []rune{char_single, char_any, char_range_open, char_terms_open} +var inTermsBreakers = append(inTextBreakers, char_terms_close, char_comma) + +func (l *lexer) fetchItem() { + r := l.read() + switch { + case r == eof: + l.tokens.push(Token{EOF, ""}) + + case r == char_terms_open: + l.termsEnter() + l.tokens.push(Token{TermsOpen, string(r)}) + + case r == char_comma && l.inTerms(): + l.tokens.push(Token{Separator, string(r)}) + + case r == char_terms_close && l.inTerms(): + l.tokens.push(Token{TermsClose, string(r)}) + l.termsLeave() + + case r == char_range_open: + l.tokens.push(Token{RangeOpen, string(r)}) + l.fetchRange() + + case r == char_single: + l.tokens.push(Token{Single, string(r)}) + + case r == char_any: + if l.read() == char_any { + l.tokens.push(Token{Super, string(r) + string(r)}) + } else { + l.unread() + l.tokens.push(Token{Any, string(r)}) + } + + default: + l.unread() + + var breakers []rune + if l.inTerms() { + breakers = inTermsBreakers + } else { + breakers = inTextBreakers + } + l.fetchText(breakers) + } +} + +func (l *lexer) fetchRange() { + var wantHi bool + var wantClose bool + var seenNot bool + for { + r := l.read() + if r == eof { + l.errorf("unexpected end of input") + return + } + + if wantClose { + if r != char_range_close { + l.errorf("expected close range character") + } else { + l.tokens.push(Token{RangeClose, string(r)}) + } + return + } + + if wantHi { + l.tokens.push(Token{RangeHi, string(r)}) + wantClose = true + continue + } + + if !seenNot && r == char_range_not { + l.tokens.push(Token{Not, string(r)}) + seenNot = true + continue + } + + if n, w := l.peek(); n == char_range_between { + l.seek(w) + l.tokens.push(Token{RangeLo, string(r)}) + l.tokens.push(Token{RangeBetween, string(n)}) + wantHi = true + continue + } + + l.unread() // unread first peek and fetch as text + l.fetchText([]rune{char_range_close}) + wantClose = true + } +} + +func (l *lexer) fetchText(breakers []rune) { + var data []rune + var escaped bool + +reading: + for { + r := l.read() + if r == eof { + break + } + + if !escaped { + if r == char_escape { + escaped = true + continue + } + + if runes.IndexRune(breakers, r) != -1 { + l.unread() + break reading + } + } + + escaped = false + data = append(data, r) + } + + if len(data) > 0 { + l.tokens.push(Token{Text, string(data)}) + } +} diff --git a/vendor/github.com/gobwas/glob/syntax/lexer/token.go b/vendor/github.com/gobwas/glob/syntax/lexer/token.go new file mode 100644 index 000000000..2797c4e83 --- /dev/null +++ b/vendor/github.com/gobwas/glob/syntax/lexer/token.go @@ -0,0 +1,88 @@ +package lexer + +import "fmt" + +type TokenType int + +const ( + EOF TokenType = iota + Error + Text + Char + Any + Super + Single + Not + Separator + RangeOpen + RangeClose + RangeLo + RangeHi + RangeBetween + TermsOpen + TermsClose +) + +func (tt TokenType) String() string { + switch tt { + case EOF: + return "eof" + + case Error: + return "error" + + case Text: + return "text" + + case Char: + return "char" + + case Any: + return "any" + + case Super: + return "super" + + case Single: + return "single" + + case Not: + return "not" + + case Separator: + return "separator" + + case RangeOpen: + return "range_open" + + case RangeClose: + return "range_close" + + case RangeLo: + return "range_lo" + + case RangeHi: + return "range_hi" + + case RangeBetween: + return "range_between" + + case TermsOpen: + return "terms_open" + + case TermsClose: + return "terms_close" + + default: + return "undef" + } +} + +type Token struct { + Type TokenType + Raw string +} + +func (t Token) String() string { + return fmt.Sprintf("%v<%q>", t.Type, t.Raw) +} diff --git a/vendor/github.com/gobwas/glob/syntax/syntax.go b/vendor/github.com/gobwas/glob/syntax/syntax.go new file mode 100644 index 000000000..1d168b148 --- /dev/null +++ b/vendor/github.com/gobwas/glob/syntax/syntax.go @@ -0,0 +1,14 @@ +package syntax + +import ( + "github.com/gobwas/glob/syntax/ast" + "github.com/gobwas/glob/syntax/lexer" +) + +func Parse(s string) (*ast.Node, error) { + return ast.Parse(lexer.NewLexer(s)) +} + +func Special(b byte) bool { + return lexer.Special(b) +} diff --git a/vendor/github.com/gobwas/glob/util/runes/runes.go b/vendor/github.com/gobwas/glob/util/runes/runes.go new file mode 100644 index 000000000..a72355641 --- /dev/null +++ b/vendor/github.com/gobwas/glob/util/runes/runes.go @@ -0,0 +1,154 @@ +package runes + +func Index(s, needle []rune) int { + ls, ln := len(s), len(needle) + + switch { + case ln == 0: + return 0 + case ln == 1: + return IndexRune(s, needle[0]) + case ln == ls: + if Equal(s, needle) { + return 0 + } + return -1 + case ln > ls: + return -1 + } + +head: + for i := 0; i < ls && ls-i >= ln; i++ { + for y := 0; y < ln; y++ { + if s[i+y] != needle[y] { + continue head + } + } + + return i + } + + return -1 +} + +func LastIndex(s, needle []rune) int { + ls, ln := len(s), len(needle) + + switch { + case ln == 0: + if ls == 0 { + return 0 + } + return ls + case ln == 1: + return IndexLastRune(s, needle[0]) + case ln == ls: + if Equal(s, needle) { + return 0 + } + return -1 + case ln > ls: + return -1 + } + +head: + for i := ls - 1; i >= 0 && i >= ln; i-- { + for y := ln - 1; y >= 0; y-- { + if s[i-(ln-y-1)] != needle[y] { + continue head + } + } + + return i - ln + 1 + } + + return -1 +} + +// IndexAny returns the index of the first instance of any Unicode code point +// from chars in s, or -1 if no Unicode code point from chars is present in s. +func IndexAny(s, chars []rune) int { + if len(chars) > 0 { + for i, c := range s { + for _, m := range chars { + if c == m { + return i + } + } + } + } + return -1 +} + +func Contains(s, needle []rune) bool { + return Index(s, needle) >= 0 +} + +func Max(s []rune) (max rune) { + for _, r := range s { + if r > max { + max = r + } + } + + return +} + +func Min(s []rune) rune { + min := rune(-1) + for _, r := range s { + if min == -1 { + min = r + continue + } + + if r < min { + min = r + } + } + + return min +} + +func IndexRune(s []rune, r rune) int { + for i, c := range s { + if c == r { + return i + } + } + return -1 +} + +func IndexLastRune(s []rune, r rune) int { + for i := len(s) - 1; i >= 0; i-- { + if s[i] == r { + return i + } + } + + return -1 +} + +func Equal(a, b []rune) bool { + if len(a) == len(b) { + for i := 0; i < len(a); i++ { + if a[i] != b[i] { + return false + } + } + + return true + } + + return false +} + +// HasPrefix tests whether the string s begins with prefix. +func HasPrefix(s, prefix []rune) bool { + return len(s) >= len(prefix) && Equal(s[0:len(prefix)], prefix) +} + +// HasSuffix tests whether the string s ends with suffix. +func HasSuffix(s, suffix []rune) bool { + return len(s) >= len(suffix) && Equal(s[len(s)-len(suffix):], suffix) +} diff --git a/vendor/github.com/gobwas/glob/util/strings/strings.go b/vendor/github.com/gobwas/glob/util/strings/strings.go new file mode 100644 index 000000000..e8ee1920b --- /dev/null +++ b/vendor/github.com/gobwas/glob/util/strings/strings.go @@ -0,0 +1,39 @@ +package strings + +import ( + "strings" + "unicode/utf8" +) + +func IndexAnyRunes(s string, rs []rune) int { + for _, r := range rs { + if i := strings.IndexRune(s, r); i != -1 { + return i + } + } + + return -1 +} + +func LastIndexAnyRunes(s string, rs []rune) int { + for _, r := range rs { + i := -1 + if 0 <= r && r < utf8.RuneSelf { + i = strings.LastIndexByte(s, byte(r)) + } else { + sub := s + for len(sub) > 0 { + j := strings.IndexRune(s, r) + if j == -1 { + break + } + i = j + sub = sub[i+1:] + } + } + if i != -1 { + return i + } + } + return -1 +} diff --git a/vendor/github.com/ryanuber/go-glob/README.md b/vendor/github.com/ryanuber/go-glob/README.md deleted file mode 100644 index 48f7fcb05..000000000 --- a/vendor/github.com/ryanuber/go-glob/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# String globbing in golang [![Build Status](https://travis-ci.org/ryanuber/go-glob.svg)](https://travis-ci.org/ryanuber/go-glob) - -`go-glob` is a single-function library implementing basic string glob support. - -Globs are an extremely user-friendly way of supporting string matching without -requiring knowledge of regular expressions or Go's particular regex engine. Most -people understand that if you put a `*` character somewhere in a string, it is -treated as a wildcard. Surprisingly, this functionality isn't found in Go's -standard library, except for `path.Match`, which is intended to be used while -comparing paths (not arbitrary strings), and contains specialized logic for this -use case. A better solution might be a POSIX basic (non-ERE) regular expression -engine for Go, which doesn't exist currently. - -Example -======= - -``` -package main - -import "github.com/ryanuber/go-glob" - -func main() { - glob.Glob("*World!", "Hello, World!") // true - glob.Glob("Hello,*", "Hello, World!") // true - glob.Glob("*ello,*", "Hello, World!") // true - glob.Glob("World!", "Hello, World!") // false - glob.Glob("/home/*", "/home/ryanuber/.bashrc") // true -} -``` diff --git a/vendor/github.com/ryanuber/go-glob/glob.go b/vendor/github.com/ryanuber/go-glob/glob.go deleted file mode 100644 index d9d46379a..000000000 --- a/vendor/github.com/ryanuber/go-glob/glob.go +++ /dev/null @@ -1,51 +0,0 @@ -package glob - -import "strings" - -// The character which is treated like a glob -const GLOB = "*" - -// Glob will test a string pattern, potentially containing globs, against a -// subject string. The result is a simple true/false, determining whether or -// not the glob pattern matched the subject text. -func Glob(pattern, subj string) bool { - // Empty pattern can only match empty subject - if pattern == "" { - return subj == pattern - } - - // If the pattern _is_ a glob, it matches everything - if pattern == GLOB { - return true - } - - parts := strings.Split(pattern, GLOB) - - if len(parts) == 1 { - // No globs in pattern, so test for equality - return subj == pattern - } - - leadingGlob := strings.HasPrefix(pattern, GLOB) - trailingGlob := strings.HasSuffix(pattern, GLOB) - end := len(parts) - 1 - - // Check the first section. Requires special handling. - if !leadingGlob && !strings.HasPrefix(subj, parts[0]) { - return false - } - - // Go over the middle parts and ensure they match. - for i := 1; i < end; i++ { - if !strings.Contains(subj, parts[i]) { - return false - } - - // Trim evaluated text from subj as we loop over the pattern. - idx := strings.Index(subj, parts[i]) + len(parts[i]) - subj = subj[idx:] - } - - // Reached the last section. Requires special handling. - return trailingGlob || strings.HasSuffix(subj, parts[end]) -} diff --git a/vendor/vendor.json b/vendor/vendor.json index f31daa925..0625b8844 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -9,6 +9,14 @@ {"path":"github.com/circonus-labs/circonusllhist","checksumSHA1":"C4Z7+l5GOpOCW5DcvNYzheGvQRE=","revision":"d724266ae5270ae8b87a5d2e8081f04e307c3c18","revisionTime":"2016-05-26T04:38:13Z"}, {"path":"github.com/cyberdelia/go-metrics-graphite","checksumSHA1":"8/Q1JbAHUmL4sDURLq6yron4K/I=","revision":"b8345b7f01d571b05366d5791a034d872f1bb36f","revisionTime":"2015-08-25T20:22:00-07:00"}, {"path":"github.com/fatih/structs","checksumSHA1":"zKLQNNEpgfrJOwVrDDyBMYEIx/w=","revision":"5ada2f449b108d87dbd8c1e60c32cdd065c27886","revisionTime":"2016-06-01T09:31:17Z"}, + {"path":"github.com/gobwas/glob","checksumSHA1":"/RE0VzeaXdaBUNeNMUXWqzly7qY=","revision":"19c076cdf202b3d1c0489bdfa2f2f289f634474b","revisionTime":"2018-02-08T21:18:42Z"}, + {"path":"github.com/gobwas/glob/compiler","checksumSHA1":"UIxNdlqa2kcYi5EEe79/WzyPiz4=","revision":"19c076cdf202b3d1c0489bdfa2f2f289f634474b","revisionTime":"2018-02-08T21:18:42Z"}, + {"path":"github.com/gobwas/glob/match","checksumSHA1":"jNdbCxBabqjv9tREXA4Nx45Y4wg=","revision":"19c076cdf202b3d1c0489bdfa2f2f289f634474b","revisionTime":"2018-02-08T21:18:42Z"}, + {"path":"github.com/gobwas/glob/syntax","checksumSHA1":"2KhDNUE98XhHnIgg2S9E4gcWoig=","revision":"19c076cdf202b3d1c0489bdfa2f2f289f634474b","revisionTime":"2018-02-08T21:18:42Z"}, + {"path":"github.com/gobwas/glob/syntax/ast","checksumSHA1":"FPIMdPSazNZGNU+ZmPtIeZpVHFU=","revision":"19c076cdf202b3d1c0489bdfa2f2f289f634474b","revisionTime":"2018-02-08T21:18:42Z"}, + {"path":"github.com/gobwas/glob/syntax/lexer","checksumSHA1":"umyztSrQrQIl9JgwhDDb6wrSLeI=","revision":"19c076cdf202b3d1c0489bdfa2f2f289f634474b","revisionTime":"2018-02-08T21:18:42Z"}, + {"path":"github.com/gobwas/glob/util/runes","checksumSHA1":"vux29fN5O22F3QPLOnjQ37278vg=","revision":"19c076cdf202b3d1c0489bdfa2f2f289f634474b","revisionTime":"2018-02-08T21:18:42Z"}, + {"path":"github.com/gobwas/glob/util/strings","checksumSHA1":"Sxp2AHWkkLkIq6w9aMcLmzcwD6w=","revision":"19c076cdf202b3d1c0489bdfa2f2f289f634474b","revisionTime":"2018-02-08T21:18:42Z"}, {"path":"github.com/hashicorp/consul/api","checksumSHA1":"GsJ84gKbQno8KbojhVTgSVWNues=","revision":"402636ff2db998edef392ac6d59210d2170b3ebf","revisionTime":"2017-04-05T04:22:14Z","version":"v0.8.0","versionExact":"v0.8.0"}, {"path":"github.com/hashicorp/errwrap","checksumSHA1":"cdOCt0Yb+hdErz8NAQqayxPmRsY=","revision":"7554cd9344cec97297fa6649b055a8c98c2a1e55","revisionTime":"2014-10-27T22:47:10-07:00"}, {"path":"github.com/hashicorp/go-cleanhttp","checksumSHA1":"Uzyon2091lmwacNsl1hCytjhHtg=","revision":"ad28ea4487f05916463e2423a55166280e8254b5","revisionTime":"2016-04-07T17:41:26Z"}, @@ -34,7 +42,6 @@ {"path":"github.com/pkg/profile","checksumSHA1":"C3yiSMdTQxSY3xqKJzMV9T+KnIc=","revision":"5b67d428864e92711fcbd2f8629456121a56d91f","revisionTime":"2017-05-09T09:25:25Z"}, {"path":"github.com/rcrowley/go-metrics","checksumSHA1":"ODTWX4h8f+DW3oWZFT0yTmfHzdg=","revision":"3e5e593311103d49927c8d2b0fd93ccdfe4a525c","revisionTime":"2015-07-19T09:56:14-07:00"}, {"path":"github.com/rogpeppe/fastuuid","checksumSHA1":"ehRkDJisGCCSYdNgyvs1gSywSPE=","revision":"6724a57986aff9bff1a1770e9347036def7c89f6","revisionTime":"2015-01-06T09:31:45Z"}, - {"path":"github.com/ryanuber/go-glob","checksumSHA1":"EYkx4sXDLSEd1xUtGoXRsfd5cpw=","revision":"572520ed46dbddaed19ea3d9541bdd0494163693","revisionTime":"2016-02-26T08:37:05Z"}, {"path":"github.com/sergi/go-diff/diffmatchpatch","checksumSHA1":"iWCtyR1TkJ22Bi/ygzfKDvOQdQY=","revision":"24e2351369ec4949b2ed0dc5c477afdd4c4034e8","revisionTime":"2017-01-18T13:12:30Z"}, {"path":"golang.org/x/net/http2","checksumSHA1":"N1akwAdrHVfPPrsFOhG2ouP21VA=","revision":"f2499483f923065a842d38eb4c7f1927e6fc6e6d","revisionTime":"2017-01-14T04:22:49Z"}, {"path":"golang.org/x/net/http2/hpack","checksumSHA1":"HzuGD7AwgC0p1az1WAQnEFnEk98=","revision":"f2499483f923065a842d38eb4c7f1927e6fc6e6d","revisionTime":"2017-01-14T04:22:49Z"},