Skip to content

Commit

Permalink
Drop test_db
Browse files Browse the repository at this point in the history 
We are not using it much now, but it makes `task env-up` much slower.
  • Loading branch information
@AlekSi
AlekSi committed Jun 23, 2022
1 parent 712e940 commit 4306563
Show file tree
Hide file tree
Showing 5 changed files with 41 additions and 257 deletions.
22 changes: 11 additions & 11 deletions Taskfile.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ tasks:
env-setup:
deps: [gen-version]
cmds:
- go run ./cmd/envtool/main.go
- go run {{if ne OS "windows"}}-race{{end}} ./cmd/envtool/main.go

This comment has been minimized.

Sign in to view

Copy link
@noisersup

noisersup Jun 24, 2022

Member

isn't that related to a #753?

env-up-detach:
cmds:
Expand Down Expand Up @@ -90,13 +90,13 @@ tasks:
test-unit-short:
desc: "Run short unit tests"
cmds:
- go test -short -count=1 {{if ne OS "windows"}} -race {{end}} -shuffle=on -coverprofile=cover.txt ./...
- go test -short -count=1 {{if ne OS "windows"}}-race{{end}} -shuffle=on -coverprofile=cover.txt ./...

test-unit:
desc: "Run all unit tests"
cmds:
- go test -count=1 {{if ne OS "windows"}} -race {{end}} -shuffle=on -coverprofile=cover.txt -coverpkg=./... ./...
- go test -count=1 {{if ne OS "windows"}} -race {{end}} -shuffle=on -bench=. -benchtime=1x ./...
- go test -count=1 {{if ne OS "windows"}}-race{{end}} -shuffle=on -coverprofile=cover.txt -coverpkg=./... ./...
- go test -count=1 {{if ne OS "windows"}}-race{{end}} -shuffle=on -bench=. -benchtime=1x ./...

test-integration:
desc: "Run all integration tests in parallel"
Expand All @@ -121,7 +121,7 @@ tasks:
test-integration-db:
dir: integration
cmds:
- go test -count=1 {{if ne OS "windows"}} -race {{end}} -shuffle=on -coverprofile=integration-{{.DB}}.txt -coverpkg=../... -port={{.DB}}
- go test -count=1 {{if ne OS "windows"}}-race{{end}} -shuffle=on -coverprofile=integration-{{.DB}}.txt -coverpkg=../... -port={{.DB}}

bench-short:
desc: "Benchmark for about 20 seconds (with default BENCHTIME)"
Expand Down Expand Up @@ -160,15 +160,15 @@ tasks:
fuzz-corpus:
desc: "Sync seed and generated fuzz corpora with FUZZCORPUS"
cmds:
- go run ./cmd/fuzztool/fuzztool.go -dst={{.FUZZCORPUS}} -src=generated
- go run ./cmd/fuzztool/fuzztool.go -dst={{.FUZZCORPUS}} -src=seed
- go run ./cmd/fuzztool/fuzztool.go -src={{.FUZZCORPUS}} -dst=generated
- go run {{if ne OS "windows"}}-race{{end}} ./cmd/fuzztool/fuzztool.go -dst={{.FUZZCORPUS}} -src=generated
- go run {{if ne OS "windows"}}-race{{end}} ./cmd/fuzztool/fuzztool.go -dst={{.FUZZCORPUS}} -src=seed
- go run {{if ne OS "windows"}}-race{{end}} ./cmd/fuzztool/fuzztool.go -src={{.FUZZCORPUS}} -dst=generated

build-testcover:
desc: "Build bin/ferretdb-testcover"
deps: [gen-version]
cmds:
- go test -c -o=bin/ferretdb-testcover -trimpath -tags=testcover,tigris {{if ne OS "windows"}} -race {{end}} -coverpkg=./... ./cmd/ferretdb
- go test -c -o=bin/ferretdb-testcover -trimpath -tags=testcover,tigris {{if ne OS "windows"}}-race{{end}} -coverpkg=./... ./cmd/ferretdb

run:
desc: "Run FerretDB"
Expand Down Expand Up @@ -223,14 +223,14 @@ tasks:
desc: "Run MongoDB shell (`mongosh`)"
cmds:
- >
docker-compose exec mongodb mongosh mongodb://host.docker.internal:27017/monila?heartbeatFrequencyMS=300000
docker-compose exec mongodb mongosh mongodb://host.docker.internal:27017/test?heartbeatFrequencyMS=300000
--verbose --eval 'disableTelemetry()' --shell
mongo:
desc: "Run legacy `mongo` shell"
cmds:
- >
docker-compose exec mongodb mongo mongodb://host.docker.internal:27017/monila?heartbeatFrequencyMS=300000
docker-compose exec mongodb mongo mongodb://host.docker.internal:27017/test?heartbeatFrequencyMS=300000
--verbose
docker-init:
Expand Down
207 changes: 15 additions & 192 deletions cmd/envtool/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,47 +25,17 @@ import (
"os/exec"
"runtime"
"strings"
"sync"
"time"

"github.com/prometheus/client_golang/prometheus"
"go.uber.org/zap"

"github.com/FerretDB/FerretDB/internal/clientconn"
"github.com/FerretDB/FerretDB/internal/handlers"
"github.com/FerretDB/FerretDB/internal/handlers/pg"
"github.com/FerretDB/FerretDB/internal/handlers/pg/pgdb"
"github.com/FerretDB/FerretDB/internal/util/debug"
"github.com/FerretDB/FerretDB/internal/util/logging"
"github.com/FerretDB/FerretDB/internal/util/version"
)

var (
composeBin string

collections = []string{
"actor",
"address",
"category",
"city",
"country",
"customer",
"film_actor",
"film_category",
"film",
"inventory",
"language",
"rental",
"staff",
"store",
}
)

func runCompose(args []string, stdin io.Reader, logger *zap.SugaredLogger) {
if err := tryCommand(composeBin, args, stdin, nil, logger); err != nil {
logger.Fatal(err)
}
}
var composeBin string

func tryCommand(command string, args []string, stdin io.Reader, stdout io.Writer, logger *zap.SugaredLogger) error {
gitBin, err := exec.LookPath(command)
Expand Down Expand Up @@ -122,108 +92,6 @@ func waitForPostgresPort(ctx context.Context, port uint16) error {
return fmt.Errorf("failed to connect to 127.0.0.1:%d", port)
}

func setupMongoDB(ctx context.Context) {
start := time.Now()
logger := zap.S().Named("mongodb")

logger.Infof("Importing database...")

var wg sync.WaitGroup

for _, c := range collections {
args := fmt.Sprintf(
`exec -T mongodb mongoimport --uri mongodb://127.0.0.1:27017/monila `+
`--drop --maintainInsertionOrder --collection %[1]s /test_db/monila/%[1]s.json`,
c,
)

wg.Add(1)
go func() {
defer wg.Done()
runCompose(strings.Split(args, " "), nil, logger)
}()
}

{
args := `exec -T mongodb mongoimport --uri mongodb://127.0.0.1:27017/values ` +
`--drop --maintainInsertionOrder --collection values /test_db/values/values.json`

wg.Add(1)
go func() {
defer wg.Done()
runCompose(strings.Split(args, " "), nil, logger)
}()
}

wg.Wait()

logger.Infof("Done in %s.", time.Since(start))
}

func setupMonilaAndValues(ctx context.Context, handler handlers.Interface) {
start := time.Now()
logger := zap.S().Named("postgres.monila_and_values")

logger.Infof("Importing databases...")

// listen on all interfaces to make mongoimport below work from inside Docker
addr := ":27018"
if runtime.GOOS == "darwin" {
// do not trigger macOS firewall; it works with Docker Desktop
addr = "127.0.0.1:27018"
}

l := clientconn.NewListener(&clientconn.NewListenerOpts{
ListenAddr: addr,
Mode: "normal",
Handler: handler,
Logger: logger.Desugar(),
})

prometheus.DefaultRegisterer.MustRegister(l)

lCtx, lCancel := context.WithCancel(ctx)
lDone := make(chan struct{})
go func() {
defer close(lDone)
l.Run(lCtx)
}()

var wg sync.WaitGroup

for _, c := range collections {
cmd := fmt.Sprintf(
`exec -T mongodb mongoimport --uri mongodb://host.docker.internal:27018/monila `+
`--drop --maintainInsertionOrder --collection %[1]s /test_db/monila/%[1]s.json`,
c,
)

wg.Add(1)
go func() {
defer wg.Done()
runCompose(strings.Split(cmd, " "), nil, logger)
}()
}

{
cmd := `exec -T mongodb mongoimport --uri mongodb://host.docker.internal:27018/values ` +
`--drop --maintainInsertionOrder --collection values /test_db/values/values.json`

wg.Add(1)
go func() {
defer wg.Done()
runCompose(strings.Split(cmd, " "), nil, logger)
}()
}

wg.Wait()

lCancel()
<-lDone

logger.Infof("Done in %s.", time.Since(start))
}

//nolint:forbidigo // Printf used to make diagnostic data easier to copy.
func printDiagnosticData(runError error, logger *zap.SugaredLogger) {
buffer := bytes.NewBuffer([]byte{})
Expand Down Expand Up @@ -312,81 +180,36 @@ func run(ctx context.Context, logger *zap.SugaredLogger) error {
go debug.RunHandler(ctx, "127.0.0.1:8089", logger.Named("debug").Desugar())

var err error
composeBin, err = exec.LookPath("docker-compose")
if err != nil {
if composeBin, err = exec.LookPath("docker-compose"); err != nil {
return err
}

var wg sync.WaitGroup
portsCtx, portsCancel := context.WithTimeout(ctx, time.Minute)
defer portsCancel()

var portsCheckError error

wg.Add(1)
go func() {
defer wg.Done()

logger.Info("Waiting for port 37017 to be up...")
portsCheckError = waitForPort(portsCtx, 37017)
}()

wg.Add(1)
go func() {
defer wg.Done()

logger.Info("Waiting for port 5432 to be up...")
portsCheckError = waitForPostgresPort(portsCtx, 5432)
}()

wg.Wait()

if portsCheckError != nil {
return portsCheckError
logger.Info("Waiting for port MongoDB 37017 to be up...")
if err = waitForPort(ctx, 37017); err != nil {
return err
}

var pgPool *pgdb.Pool
pgPool, err = pgdb.NewPool(ctx, "postgres://postgres@127.0.0.1:5432/ferretdb", logger.Desugar(), false)
if err != nil {
logger.Info("Waiting for PostgreSQL port 5432 to be up...")
if err = waitForPostgresPort(ctx, 5432); err != nil {
return err
}
handlerOpts := &pg.NewOpts{
PgPool: pgPool,
L: logger.Desugar(),
StartTime: time.Now(),
}
h, err := pg.New(handlerOpts)

pgPool, err := pgdb.NewPool(ctx, "postgres://postgres@127.0.0.1:5432/ferretdb", logger.Desugar(), false)
if err != nil {
return err
}

for _, db := range []string{`monila`, `values`, `test`} {
if err := pgPool.CreateSchema(ctx, db); err != nil {
return err
}
if err = pgPool.CreateSchema(ctx, `test`); err != nil {
return err
}

wg.Add(1)
go func() {
defer wg.Done()
setupMongoDB(ctx)
}()

wg.Add(1)
go func() {
defer wg.Done()
setupMonilaAndValues(ctx, h)
}()

wg.Wait()

for _, q := range []string{
`CREATE ROLE readonly NOINHERIT LOGIN`,
`GRANT SELECT ON ALL TABLES IN SCHEMA monila, values, test TO readonly`,
`GRANT USAGE ON SCHEMA monila, values, test TO readonly`,
`GRANT SELECT ON ALL TABLES IN SCHEMA test TO readonly`,
`GRANT USAGE ON SCHEMA test TO readonly`,
`ANALYZE`, // to make tests more stable
} {
if _, err := pgPool.Exec(ctx, q); err != nil {
if _, err = pgPool.Exec(ctx, q); err != nil {
return err
}
}
Expand All @@ -400,7 +223,7 @@ func main() {

logger := setupLogger(*debugLevel)

ctx, cancel := context.WithCancel(context.Background())
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()

err := run(ctx, logger)
Expand Down
11 changes: 0 additions & 11 deletions docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ services:
context: ./build/deps
dockerfile: postgres.Dockerfile
container_name: ferretdb_postgres
depends_on: [test_db]
ports:
- 5432:5432
extra_hosts:
Expand Down Expand Up @@ -68,7 +67,6 @@ services:
context: ./build/deps
dockerfile: mongo.Dockerfile
container_name: ferretdb_mongodb
depends_on: [test_db]
command: --enableFreeMonitoring off
ports:
- 37017:27017
Expand All @@ -77,14 +75,6 @@ services:
environment:
# Always UTC+05:45. Set to catch timezone problems.
- TZ=Asia/Kathmandu
volumes:
- test_db_mongodb:/test_db/:ro

test_db:
image: aleksi/test_db:mongodb-values
container_name: ferretdb_test_db
volumes:
- test_db_mongodb:/test_db/mongodb:ro

markdownlint:
build:
Expand All @@ -95,5 +85,4 @@ services:
- .:/workdir

volumes:
test_db_mongodb:
tigrisdata:
Loading

0 comments on commit 4306563

Please sign in to comment.