Skip to content

Commit

Permalink
ci: speed things up with restructuring jobs (#479)
Browse files Browse the repository at this point in the history 
Closes #477
  • Loading branch information
@zepatrik
zepatrik committed Mar 11, 2021
1 parent 9315a77 commit 7cfa9be
Show file tree
Hide file tree
Showing 3 changed files with 114 additions and 112 deletions.
133 changes: 69 additions & 64 deletions .circleci/config.yml
View file
Original file line number Diff line number Diff line change
@@ -1,138 +1,147 @@
version: 2.1

orbs:
sdk: ory/sdk@0.1.36
changelog: ory/changelog@0.1.2
goreleaser: ory/goreleaser@0.1.17
sdk: ory/sdk@0.1.40
changelog: ory/changelog@0.1.8
goreleaser: ory/goreleaser@0.1.28
slack: circleci/slack@3.4.2
nancy: ory/nancy@0.0.12
docs: ory/docs@0.0.8
golangci: ory/golangci@0.0.13
nancy: ory/nancy@0.0.17
docs: ory/docs@0.0.10
golangci: ory/golangci@0.0.18
go: circleci/go@1.5.0
prettier: ory/prettier@0.0.11

jobs:
test:
docker:
-
image: cimg/go:1.16-node
- image: cimg/go:1.16
environment:
TEST_DATABASE_POSTGRESQL: postgres://test:test@localhost:5432/keto?sslmode=disable
TEST_DATABASE_MYSQL: mysql://root:test@(localhost:3306)/mysql?parseTime=true&multiStatements=true
TEST_DATABASE_COCKROACHDB: cockroach://root@localhost:26257/defaultdb?sslmode=disable
-
image: postgres:11.8
- image: postgres:11.8
environment:
POSTGRES_USER: test
POSTGRES_PASSWORD: test
POSTGRES_DB: keto
-
image: mysql:8.0
- image: mysql:8.0
environment:
MYSQL_ROOT_PASSWORD: test
-
image: cockroachdb/cockroach:v20.2.4
- image: cockroachdb/cockroach:v20.2.4
command: start-single-node --insecure
steps:
- checkout
- setup_remote_docker
- run: sudo npm i -g npm@7

# Installation
-
run: go mod verify
-
run: go install github.com/ory/keto
-
run: go install github.com/go-swagger/go-swagger/cmd/swagger github.com/mattn/goveralls golang.org/x/tools/cmd/cover github.com/ory/go-acc
- go/load-cache
- go/mod-download
- run: |-
make .bin/goveralls
make .bin/go-acc
- run: go mod tidy
- go/save-cache

# Tests
-
run: go test -tags sqlite -race -short -v ./...
-
run: go-acc -o coverage.txt ./... -- -v -tags sqlite
- run: .bin/go-acc -o coverage.txt ./... -- -v -tags sqlite

# Submit coverage details
-
run: test -z "$CIRCLE_PR_NUMBER" && goveralls -service=circle-ci -coverprofile=coverage.txt -repotoken=$COVERALLS_REPO_TOKEN || echo "forks are not allowed to push to coveralls"
- run: test -z "$CIRCLE_PR_NUMBER" && .bin/goveralls -service=circle-ci -coverprofile=coverage.txt -repotoken=$COVERALLS_REPO_TOKEN || echo "forks are not allowed to push to coveralls"

test-race:
docker:
- image: cimg/go:1.16
parallelism: 4
steps:
- checkout
- go/load-cache
- go/mod-download
- run: go mod tidy
- go/save-cache
- run: go test -tags sqlite -race -short -v $(go list ./... | circleci tests split)

# Test documentation examples
-
run: make test-docs-samples
validate:
docker:
- image: cimg/go:1.16-node
environment:
GO111MODULE: 'on'
steps:
- checkout
- run: sudo npm i -g npm@7
- golangci/install
- golangci/lint
- docs/check-format
- run: npm ci && npm run format:check
- nancy/install
- nancy/check
- prettier/install
- prettier/check
- prettier/install:
dir: docs/
- prettier/check:
dir: docs/

# Test documentation examples
- run: make test-docs-samples

workflows:
version: 2
"test, build, and relase":
jobs:
-
nancy/test:
- validate:
filters:
tags:
only: /.*/
-
validate:
- test:
filters:
tags:
only: /.*/
-
test:
- test-race:
filters:
tags:
only: /.*/
-
changelog/generate:
- changelog/generate:
requires:
- test
- test-race
- validate
filters:
tags:
only: /.*/
branches:
only: master
-
docs/build:
- docs/build:
requires:
- test
- test-race
- validate
filters:
tags:
only: /.*/
branches:
only: master
# -
# sdk/generate:
# appname: Ory_Keto
# requires:
# - test
# - validate
-
sdk/release:
# -
# sdk/generate:
# appname: Ory_Keto
# requires:
# - test
# - validate
- sdk/release:
requires:
- test
- test-race
- validate
- goreleaser/release
filters:
tags:
only: /.*/
branches:
ignore: /.*/
-
goreleaser/test:
- goreleaser/test:
filters:
tags:
only: /.*/
-
goreleaser/release:
- goreleaser/release:
requires:
- goreleaser/test
- test
- test-race
- validate
filters:
branches:
Expand All @@ -145,34 +154,30 @@ workflows:
filters:
tags:
only: /.*/
-
goreleaser/newsletter-draft:
- goreleaser/newsletter-draft:
chimp-list: f605a41b53
chimp-segment: 6479489
requires:
- goreleaser/release
filters:
tags:
only: /.*/
-
slack/approval-notification:
- slack/approval-notification:
message: Pending approval
channel: release-automation
requires:
- goreleaser/newsletter-draft
filters:
tags:
only: /.*/
-
newsletter-approval:
- newsletter-approval:
type: approval
requires:
- goreleaser/newsletter-draft
filters:
tags:
only: /.*/
-
goreleaser/newsletter-send:
- goreleaser/newsletter-send:
chimp-list: f605a41b53
requires:
- newsletter-approval
Expand Down
73 changes: 28 additions & 45 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,52 +1,41 @@
SHELL=/bin/bash -o pipefail

EXECUTABLES = docker-compose docker node npm go
K := $(foreach exec,$(EXECUTABLES),\
$(if $(shell which $(exec)),some string,$(error "No $(exec) in PATH")))

export GO111MODULE := on
export PATH := .bin:${PATH}
export PWD := $(shell pwd)

GO_DEPENDENCIES = github.com/go-swagger/go-swagger/cmd/swagger \
golang.org/x/tools/cmd/goimports \
github.com/mattn/goveralls \
github.com/ory/cli \
github.com/ory/go-acc \
github.com/bufbuild/buf/cmd/buf \
google.golang.org/protobuf/cmd/protoc-gen-go \
google.golang.org/grpc/cmd/protoc-gen-go-grpc

define make-go-dependency
# go install is responsible for not re-building when the code hasn't changed
.bin/$(notdir $1): go.mod go.sum Makefile
GOBIN=$(PWD)/.bin/ go install $1
endef
$(foreach dep, $(GO_DEPENDENCIES), $(eval $(call make-go-dependency, $(dep))))
$(call make-lint-dependency)

.PHONY: deps
deps:
ifneq ("$(shell base64 Makefile) $(shell base64 go.mod) $(shell base64 go.sum)","$(shell cat .bin/.lock)")
go build -o .bin/go-acc github.com/ory/go-acc
go build -o .bin/goreturns github.com/sqs/goreturns
go build -o .bin/mockgen github.com/golang/mock/mockgen
go build -o .bin/swagger github.com/go-swagger/go-swagger/cmd/swagger
go build -o .bin/goimports golang.org/x/tools/cmd/goimports
go build -o .bin/ory github.com/ory/cli
go build -o .bin/buf github.com/bufbuild/buf/cmd/buf
go build -o .bin/protoc-gen-go google.golang.org/protobuf/cmd/protoc-gen-go
go build -o .bin/protoc-gen-go-grpc google.golang.org/grpc/cmd/protoc-gen-go-grpc
npm i
echo "v0" > .bin/.lock
echo "$$(base64 Makefile) $$(base64 go.mod) $$(base64 go.sum)" > .bin/.lock
endif
node_modules: package.json package-lock.json Makefile
npm ci

.PHONY: format
format:
format: .bin/goimports
goimports -w -local github.com/ory/keto *.go internal cmd contrib

.PHONY: install-stable
install-stable: deps
KETO_LATEST=$$(git describe --abbrev=0 --tags)
git checkout $$KETO_LATEST
GO111MODULE=on go install \
-ldflags "-X github.com/ory/keto/cmd.Version=$$KETO_LATEST -X github.com/ory/keto/cmd.Date=`TZ=UTC date -u '+%Y-%m-%dT%H:%M:%SZ'` -X github.com/ory/keto/cmd.Commit=`git rev-parse HEAD`" \
.
rm pkged.go
git checkout master

.PHONY: install
install: deps
GO111MODULE=on go install .
install:
go install -tags sqlite .

# Generates the SDKs
.PHONY: sdk
sdk: deps
sdk: .bin/swagger .bin/cli
swagger generate spec -m -o ./.schema/swagger.json -x internal/httpclient -x proto/ory/keto -x docker
ory dev swagger sanitize ./.schema/swagger.json
cli dev swagger sanitize ./.schema/swagger.json
swagger flatten --with-flatten=remove-unused -o ./.schema/swagger.json ./.schema/swagger.json
swagger validate ./.schema/swagger.json
rm -rf internal/httpclient
Expand All @@ -55,25 +44,23 @@ sdk: deps
make format

.PHONY: build
build: deps
build:
go build -tags sqlite

#
# Generate APIs and client stubs from the definitions
#
.PHONY: buf-gen
buf-gen: deps
buf-gen: .bin/buf .bin/protoc-gen-go .bin/protoc-gen-go-grpc node_modules
buf generate \
&& \
echo "TODO: generate gapic client at ./client" \
&& \
echo "All code was generated successfully!"

#
# Lint API definitions
#
.PHONY: buf-lint
buf-lint: deps
buf-lint: .bin/buf
buf check lint \
&& \
echo "All lint checks passed successfully!"
Expand All @@ -84,10 +71,6 @@ buf-lint: deps
.PHONY: buf
buf: buf-lint buf-gen

.PHONY: reset-testdb
reset-testdb:
source scripts/test-resetdb.sh

.PHONY: test-e2e
test-e2e:
go test -tags sqlite -failfast -v ./internal/e2e
Expand Down
20 changes: 17 additions & 3 deletions internal/x/dsn_testutils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,32 @@ func GetDSNs(t testing.TB) []*DsnT {
},
}
if !testing.Short() {
var mysql, postgres, cockroach string

dockertest.Parallel([]func(){
func() {
mysql = dockertest.RunTestMySQL(t)
},
func() {
postgres = dockertest.RunTestPostgreSQL(t)
},
func() {
cockroach = dockertest.RunTestCockroachDB(t)
},
})

dsns = append(dsns,
&DsnT{
Name: "mysql",
Conn: dockertest.RunTestMySQL(t),
Conn: mysql,
},
&DsnT{
Name: "postgres",
Conn: dockertest.RunTestPostgreSQL(t),
Conn: postgres,
},
&DsnT{
Name: "cockroach",
Conn: dockertest.RunTestCockroachDB(t),
Conn: cockroach,
},
)

Expand Down

0 comments on commit 7cfa9be

Please sign in to comment.