Skip to content

Commit

Permalink
refactor: Splitting main into multiple files
Browse files Browse the repository at this point in the history 
Signed-off-by: Vincent Boutour <bob@vibioh.fr>
  • Loading branch information
@ViBiOh
ViBiOh committed Jun 27, 2024
1 parent 786f00d commit 8e15229
Show file tree
Hide file tree
Showing 11 changed files with 305 additions and 316 deletions.
113 changes: 53 additions & 60 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ Thanks to [FontAwesome](https://fontawesome.com) for icons.

## Features

- Timezone aware cron
- Simple and plain HTML interface, mobile ready
- Healthcheck available on `/health` endpoint
- Version available on `/version` endpoint
Expand All @@ -27,7 +26,7 @@ Thanks to [FontAwesome](https://fontawesome.com) for icons.

You need a [GitHub OAuth Token](https://github.com/settings/tokens), with no particular permission, for having a decent rate-limiting when querying GitHub. Configuration is done by passing `-githubToken` arg or setting equivalent environment variable (cf. [Usage](#usage) section)

You need a Postgres database for storing your data. I personally use free tier of [ElephantSQL](https://www.elephantsql.com). Once setup, you _have to_ to create schema with [Auth DDL](https://github.com/ViBiOh/auth/blob/main/ddl.sql) and [Ketchup DDL](sql/ddl.sql). Configuration is done by passing `-dbHost`, `-dbName`, `-dbUser`, `-dbPass` args or setting equivalent environment variables (cf. [Usage](#usage) section).
You need a Postgres database for storing your data. Once setup, you _have to_ to create schema with [Auth DDL](https://github.com/ViBiOh/auth/blob/main/ddl.sql) and [Ketchup DDL](sql/ddl.sql). Configuration is done by passing `-dbHost`, `-dbName`, `-dbUser`, `-dbPass` args or setting equivalent environment variables (cf. [Usage](#usage) section).

You need a Redis instance for storing captcha token and distributed locks across multiples instances. Configuration is done by passing `-redisAddress`, `-redisPassword`, `-redisDatabase` args or setting equivalent environment variables (cf. [Usage](#usage) section).

Expand Down Expand Up @@ -57,64 +56,58 @@ Be careful when using the CLI values, if someone list the processes on the syste

```bash
Usage of ketchup:
--address string [server] Listen address ${KETCHUP_ADDRESS}
--cert string [server] Certificate file ${KETCHUP_CERT}
--corsCredentials [cors] Access-Control-Allow-Credentials ${KETCHUP_CORS_CREDENTIALS} (default false)
--corsExpose string [cors] Access-Control-Expose-Headers ${KETCHUP_CORS_EXPOSE}
--corsHeaders string [cors] Access-Control-Allow-Headers ${KETCHUP_CORS_HEADERS} (default "Content-Type")
--corsMethods string [cors] Access-Control-Allow-Methods ${KETCHUP_CORS_METHODS} (default "GET")
--corsOrigin string [cors] Access-Control-Allow-Origin ${KETCHUP_CORS_ORIGIN} (default "*")
--csp string [owasp] Content-Security-Policy ${KETCHUP_CSP} (default "default-src 'self'; base-uri 'self'; script-src 'self' 'httputils-nonce'; style-src 'self' 'httputils-nonce'")
--dbHost string [db] Host ${KETCHUP_DB_HOST}
--dbMaxConn uint [db] Max Open Connections ${KETCHUP_DB_MAX_CONN} (default 5)
--dbMinConn uint [db] Min Open Connections ${KETCHUP_DB_MIN_CONN} (default 2)
--dbName string [db] Name ${KETCHUP_DB_NAME}
--dbPass string [db] Pass ${KETCHUP_DB_PASS}
--dbPort uint [db] Port ${KETCHUP_DB_PORT} (default 5432)
--dbSslmode string [db] SSL Mode ${KETCHUP_DB_SSLMODE} (default "disable")
--dbUser string [db] User ${KETCHUP_DB_USER}
--dockerPassword string [docker] Registry Password ${KETCHUP_DOCKER_PASSWORD}
--dockerUsername string [docker] Registry Username ${KETCHUP_DOCKER_USERNAME}
--frameOptions string [owasp] X-Frame-Options ${KETCHUP_FRAME_OPTIONS} (default "deny")
--githubToken string [github] OAuth Token ${KETCHUP_GITHUB_TOKEN}
--graceDuration duration [http] Grace duration when signal received ${KETCHUP_GRACE_DURATION} (default 30s)
--hsts [owasp] Indicate Strict Transport Security ${KETCHUP_HSTS} (default true)
--idleTimeout duration [server] Idle Timeout ${KETCHUP_IDLE_TIMEOUT} (default 2m0s)
--key string [server] Key file ${KETCHUP_KEY}
--loggerJson [logger] Log format as JSON ${KETCHUP_LOGGER_JSON} (default false)
--loggerLevel string [logger] Logger level ${KETCHUP_LOGGER_LEVEL} (default "INFO")
--loggerLevelKey string [logger] Key for level in JSON ${KETCHUP_LOGGER_LEVEL_KEY} (default "level")
--loggerMessageKey string [logger] Key for message in JSON ${KETCHUP_LOGGER_MESSAGE_KEY} (default "msg")
--loggerTimeKey string [logger] Key for timestamp in JSON ${KETCHUP_LOGGER_TIME_KEY} (default "time")
--mailerName string [mailer] HTTP Username or AMQP Exchange name ${KETCHUP_MAILER_NAME} (default "mailer")
--mailerPassword string [mailer] HTTP Pass ${KETCHUP_MAILER_PASSWORD}
--mailerURL string [mailer] URL (https?:// or amqps?://) ${KETCHUP_MAILER_URL}
--minify Minify HTML ${KETCHUP_MINIFY} (default true)
--name string [server] Name ${KETCHUP_NAME} (default "http")
--notifierPushUrl string [notifier] Pushgateway URL ${KETCHUP_NOTIFIER_PUSH_URL}
--okStatus int [http] Healthy HTTP Status code ${KETCHUP_OK_STATUS} (default 204)
--pathPrefix string Root Path Prefix ${KETCHUP_PATH_PREFIX}
--port uint [server] Listen port (0 to disable) ${KETCHUP_PORT} (default 1080)
--pprofAgent string [pprof] URL of the Datadog Trace Agent (e.g. http://datadog.observability:8126) ${KETCHUP_PPROF_AGENT}
--publicURL string Public URL ${KETCHUP_PUBLIC_URL} (default "https://ketchup.vibioh.fr")
--readTimeout duration [server] Read Timeout ${KETCHUP_READ_TIMEOUT} (default 5s)
--redisAddress string slice [redis] Redis Address host:port (blank to disable) ${KETCHUP_REDIS_ADDRESS}, as a string slice, environment variable separated by "," (default [127.0.0.1:6379])
--redisDatabase int [redis] Redis Database ${KETCHUP_REDIS_DATABASE} (default 0)
--redisMinIdleConn int [redis] Redis Minimum Idle Connections ${KETCHUP_REDIS_MIN_IDLE_CONN} (default 0)
--redisPassword string [redis] Redis Password, if any ${KETCHUP_REDIS_PASSWORD}
--redisPoolSize int [redis] Redis Pool Size (default GOMAXPROCS*10) ${KETCHUP_REDIS_POOL_SIZE} (default 0)
--redisUsername string [redis] Redis Username, if any ${KETCHUP_REDIS_USERNAME}
--schedulerEnabled [scheduler] Enable cron job ${KETCHUP_SCHEDULER_ENABLED} (default true)
--schedulerHour string [scheduler] Hour of cron, 24-hour format ${KETCHUP_SCHEDULER_HOUR} (default "08:00")
--schedulerTimezone string [scheduler] Timezone ${KETCHUP_SCHEDULER_TIMEZONE} (default "Europe/Paris")
--shutdownTimeout duration [server] Shutdown Timeout ${KETCHUP_SHUTDOWN_TIMEOUT} (default 10s)
--telemetryRate string [telemetry] OpenTelemetry sample rate, 'always', 'never' or a float value ${KETCHUP_TELEMETRY_RATE} (default "always")
--telemetryURL string [telemetry] OpenTelemetry gRPC endpoint (e.g. otel-exporter:4317) ${KETCHUP_TELEMETRY_URL}
--telemetryUint64 [telemetry] Change OpenTelemetry Trace ID format to an unsigned int 64 ${KETCHUP_TELEMETRY_UINT64} (default true)
--title string Application title ${KETCHUP_TITLE} (default "Ketchup")
--url string [alcotest] URL to check ${KETCHUP_URL}
--userAgent string [alcotest] User-Agent for check ${KETCHUP_USER_AGENT} (default "Alcotest")
--writeTimeout duration [server] Write Timeout ${KETCHUP_WRITE_TIMEOUT} (default 10s)
--address string [server] Listen address ${KETCHUP_ADDRESS}
--cert string [server] Certificate file ${KETCHUP_CERT}
--corsCredentials [cors] Access-Control-Allow-Credentials ${KETCHUP_CORS_CREDENTIALS} (default false)
--corsExpose string [cors] Access-Control-Expose-Headers ${KETCHUP_CORS_EXPOSE}
--corsHeaders string [cors] Access-Control-Allow-Headers ${KETCHUP_CORS_HEADERS} (default "Content-Type")
--corsMethods string [cors] Access-Control-Allow-Methods ${KETCHUP_CORS_METHODS} (default "GET")
--corsOrigin string [cors] Access-Control-Allow-Origin ${KETCHUP_CORS_ORIGIN} (default "*")
--csp string [owasp] Content-Security-Policy ${KETCHUP_CSP} (default "default-src 'self'; base-uri 'self'; script-src 'self' 'httputils-nonce'; style-src 'self' 'httputils-nonce'")
--dbHost string [db] Host ${KETCHUP_DB_HOST}
--dbMaxConn uint [db] Max Open Connections ${KETCHUP_DB_MAX_CONN} (default 5)
--dbMinConn uint [db] Min Open Connections ${KETCHUP_DB_MIN_CONN} (default 2)
--dbName string [db] Name ${KETCHUP_DB_NAME}
--dbPass string [db] Pass ${KETCHUP_DB_PASS}
--dbPort uint [db] Port ${KETCHUP_DB_PORT} (default 5432)
--dbSslmode string [db] SSL Mode ${KETCHUP_DB_SSLMODE} (default "disable")
--dbUser string [db] User ${KETCHUP_DB_USER}
--dockerPassword string [docker] Registry Password ${KETCHUP_DOCKER_PASSWORD}
--dockerUsername string [docker] Registry Username ${KETCHUP_DOCKER_USERNAME}
--frameOptions string [owasp] X-Frame-Options ${KETCHUP_FRAME_OPTIONS} (default "deny")
--githubToken string [github] OAuth Token ${KETCHUP_GITHUB_TOKEN}
--graceDuration duration [http] Grace duration when signal received ${KETCHUP_GRACE_DURATION} (default 30s)
--hsts [owasp] Indicate Strict Transport Security ${KETCHUP_HSTS} (default true)
--idleTimeout duration [server] Idle Timeout ${KETCHUP_IDLE_TIMEOUT} (default 2m0s)
--key string [server] Key file ${KETCHUP_KEY}
--loggerJson [logger] Log format as JSON ${KETCHUP_LOGGER_JSON} (default false)
--loggerLevel string [logger] Logger level ${KETCHUP_LOGGER_LEVEL} (default "INFO")
--loggerLevelKey string [logger] Key for level in JSON ${KETCHUP_LOGGER_LEVEL_KEY} (default "level")
--loggerMessageKey string [logger] Key for message in JSON ${KETCHUP_LOGGER_MESSAGE_KEY} (default "msg")
--loggerTimeKey string [logger] Key for timestamp in JSON ${KETCHUP_LOGGER_TIME_KEY} (default "time")
--minify Minify HTML ${KETCHUP_MINIFY} (default true)
--name string [server] Name ${KETCHUP_NAME} (default "http")
--okStatus int [http] Healthy HTTP Status code ${KETCHUP_OK_STATUS} (default 204)
--pathPrefix string Root Path Prefix ${KETCHUP_PATH_PREFIX}
--port uint [server] Listen port (0 to disable) ${KETCHUP_PORT} (default 1080)
--pprofAgent string [pprof] URL of the Datadog Trace Agent (e.g. http://datadog.observability:8126) ${KETCHUP_PPROF_AGENT}
--pprofPort int [pprof] Port of the HTTP server (0 to disable) ${KETCHUP_PPROF_PORT} (default 0)
--publicURL string Public URL ${KETCHUP_PUBLIC_URL} (default "https://ketchup.vibioh.fr")
--readTimeout duration [server] Read Timeout ${KETCHUP_READ_TIMEOUT} (default 5s)
--redisAddress string slice [redis] Redis Address host:port (blank to disable) ${KETCHUP_REDIS_ADDRESS}, as a string slice, environment variable separated by "," (default [127.0.0.1:6379])
--redisDatabase int [redis] Redis Database ${KETCHUP_REDIS_DATABASE} (default 0)
--redisMinIdleConn int [redis] Redis Minimum Idle Connections ${KETCHUP_REDIS_MIN_IDLE_CONN} (default 0)
--redisPassword string [redis] Redis Password, if any ${KETCHUP_REDIS_PASSWORD}
--redisPoolSize int [redis] Redis Pool Size (default GOMAXPROCS*10) ${KETCHUP_REDIS_POOL_SIZE} (default 0)
--redisUsername string [redis] Redis Username, if any ${KETCHUP_REDIS_USERNAME}
--shutdownTimeout duration [server] Shutdown Timeout ${KETCHUP_SHUTDOWN_TIMEOUT} (default 10s)
--telemetryRate string [telemetry] OpenTelemetry sample rate, 'always', 'never' or a float value ${KETCHUP_TELEMETRY_RATE} (default "always")
--telemetryURL string [telemetry] OpenTelemetry gRPC endpoint (e.g. otel-exporter:4317) ${KETCHUP_TELEMETRY_URL}
--telemetryUint64 [telemetry] Change OpenTelemetry Trace ID format to an unsigned int 64 ${KETCHUP_TELEMETRY_UINT64} (default true)
--title string Application title ${KETCHUP_TITLE} (default "Ketchup")
--url string [alcotest] URL to check ${KETCHUP_URL}
--userAgent string [alcotest] User-Agent for check ${KETCHUP_USER_AGENT} (default "Alcotest")
--writeTimeout duration [server] Write Timeout ${KETCHUP_WRITE_TIMEOUT} (default 10s)
```
## Contributing
Expand Down
139 changes: 12 additions & 127 deletions cmd/ketchup/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,148 +2,33 @@ package main

import (
"context"
"embed"
"flag"
"os"

authIdent "github.com/ViBiOh/auth/v2/pkg/ident/basic"
authMiddleware "github.com/ViBiOh/auth/v2/pkg/middleware"
authService "github.com/ViBiOh/auth/v2/pkg/service"
authStore "github.com/ViBiOh/auth/v2/pkg/store/db"
"github.com/ViBiOh/flags"
"github.com/ViBiOh/httputils/v4/pkg/alcotest"
"github.com/ViBiOh/httputils/v4/pkg/cors"
"github.com/ViBiOh/httputils/v4/pkg/db"
"github.com/ViBiOh/httputils/v4/pkg/health"
"github.com/ViBiOh/httputils/v4/pkg/httputils"
"github.com/ViBiOh/httputils/v4/pkg/logger"
"github.com/ViBiOh/httputils/v4/pkg/owasp"
"github.com/ViBiOh/httputils/v4/pkg/pprof"
"github.com/ViBiOh/httputils/v4/pkg/redis"
"github.com/ViBiOh/httputils/v4/pkg/renderer"
"github.com/ViBiOh/httputils/v4/pkg/request"
"github.com/ViBiOh/httputils/v4/pkg/server"
"github.com/ViBiOh/httputils/v4/pkg/telemetry"
"github.com/ViBiOh/ketchup/pkg/ketchup"
"github.com/ViBiOh/ketchup/pkg/notifier"
"github.com/ViBiOh/ketchup/pkg/provider/docker"
"github.com/ViBiOh/ketchup/pkg/provider/github"
"github.com/ViBiOh/ketchup/pkg/provider/helm"
"github.com/ViBiOh/ketchup/pkg/provider/npm"
"github.com/ViBiOh/ketchup/pkg/provider/pypi"
"github.com/ViBiOh/ketchup/pkg/scheduler"
ketchupService "github.com/ViBiOh/ketchup/pkg/service/ketchup"
repositoryService "github.com/ViBiOh/ketchup/pkg/service/repository"
userService "github.com/ViBiOh/ketchup/pkg/service/user"
ketchupStore "github.com/ViBiOh/ketchup/pkg/store/ketchup"
repositoryStore "github.com/ViBiOh/ketchup/pkg/store/repository"
userStore "github.com/ViBiOh/ketchup/pkg/store/user"
mailer "github.com/ViBiOh/mailer/pkg/client"
"go.opentelemetry.io/otel/trace"
)

//go:embed templates static
var content embed.FS

func initAuth(db db.Service, tracerProvider trace.TracerProvider) (authService.Service, authMiddleware.Service) {
authProvider := authStore.New(db)
identProvider := authIdent.New(authProvider, "ketchup")

return authService.New(authProvider, authProvider), authMiddleware.New(authProvider, tracerProvider, identProvider)
}

func main() {
fs := flag.NewFlagSet("ketchup", flag.ExitOnError)
fs.Usage = flags.Usage(fs)

appServerConfig := server.Flags(fs, "")
healthConfig := health.Flags(fs, "")

alcotestConfig := alcotest.Flags(fs, "")
loggerConfig := logger.Flags(fs, "logger")
telemetryConfig := telemetry.Flags(fs, "telemetry")
pprofConfig := pprof.Flags(fs, "pprof")
owaspConfig := owasp.Flags(fs, "", flags.NewOverride("Csp", "default-src 'self'; base-uri 'self'; script-src 'self' 'httputils-nonce'; style-src 'self' 'httputils-nonce'"))
corsConfig := cors.Flags(fs, "cors")
rendererConfig := renderer.Flags(fs, "", flags.NewOverride("Title", "Ketchup"), flags.NewOverride("PublicURL", "https://ketchup.vibioh.fr"))

dbConfig := db.Flags(fs, "db")
redisConfig := redis.Flags(fs, "redis")
mailerConfig := mailer.Flags(fs, "mailer")
githubConfig := github.Flags(fs, "github")
dockerConfig := docker.Flags(fs, "docker")
notifierConfig := notifier.Flags(fs, "notifier")
schedulerConfig := scheduler.Flags(fs, "scheduler")

_ = fs.Parse(os.Args[1:])

alcotest.DoAndExit(alcotestConfig)
config := newConfig()
alcotest.DoAndExit(config.alcotest)

ctx := context.Background()

logger.Init(ctx, loggerConfig)

telemetryService, err := telemetry.New(ctx, telemetryConfig)
logger.FatalfOnErr(ctx, err, "create telemetry")

defer telemetryService.Close(ctx)

service, version, env := telemetryService.GetServiceVersionAndEnv()
pprofService := pprof.New(pprofConfig, service, version, env)

logger.AddOpenTelemetryToDefaultLogger(telemetryService)
request.AddOpenTelemetryToDefaultClient(telemetryService.MeterProvider(), telemetryService.TracerProvider())

appServer := server.New(appServerConfig)

ketchupDb, err := db.New(ctx, dbConfig, telemetryService.TracerProvider())
logger.FatalfOnErr(ctx, err, "create database")

defer ketchupDb.Close()

redisClient, err := redis.New(ctx, redisConfig, telemetryService.MeterProvider(), telemetryService.TracerProvider())
logger.FatalfOnErr(ctx, err, "create redis")

defer redisClient.Close(ctx)

healthService := health.New(ctx, healthConfig, ketchupDb.Ping, redisClient.Ping)

go pprofService.Start(healthService.DoneCtx())

authServiceService, authMiddlewareApp := initAuth(ketchupDb, telemetryService.TracerProvider())

userServiceService := userService.New(userStore.New(ketchupDb), &authServiceService)
githubService := github.New(githubConfig, redisClient, telemetryService.MeterProvider(), telemetryService.TracerProvider())
dockerService := docker.New(dockerConfig)
helmService := helm.New()
npmService := npm.New()
pypiService := pypi.New()
repositoryServiceService := repositoryService.New(repositoryStore.New(ketchupDb), githubService, helmService, dockerService, npmService, pypiService)
ketchupServiceService := ketchupService.New(ketchupStore.New(ketchupDb), repositoryServiceService)

mailerService, err := mailer.New(ctx, mailerConfig, telemetryService.MeterProvider(), telemetryService.TracerProvider())
logger.FatalfOnErr(ctx, err, "create mailer")

defer mailerService.Close(ctx)

rendererService, err := renderer.New(ctx, rendererConfig, content, ketchup.FuncMap, telemetryService.MeterProvider(), telemetryService.TracerProvider())
logger.FatalfOnErr(ctx, err, "create renderer")

endCtx := healthService.EndCtx()
clients, err := newClients(ctx, config)
logger.FatalfOnErr(ctx, err, "clients")

notifierService := notifier.New(notifierConfig, repositoryServiceService, ketchupServiceService, userServiceService, mailerService, helmService)
schedulerService := scheduler.New(schedulerConfig, notifierService, redisClient, telemetryService.TracerProvider())
ketchupService := ketchup.New(endCtx, rendererService, ketchupServiceService, userServiceService, repositoryServiceService, redisClient, telemetryService.TracerProvider())
go clients.Start()
defer clients.Close(ctx)

doneCtx := healthService.DoneCtx()
services, err := newServices(ctx, config, clients)
logger.FatalfOnErr(ctx, err, "services")

if schedulerService != nil {
go schedulerService.Start(doneCtx)
}
port := newPort(config, services)

go appServer.Start(endCtx, httputils.Handler(newPort(ketchupService, authMiddlewareApp, userServiceService, rendererService), healthService, telemetryService.Middleware(appServerConfig.Name), owasp.New(owaspConfig).Middleware, cors.New(corsConfig).Middleware))
go services.server.Start(clients.health.EndCtx(), httputils.Handler(port, clients.health, clients.telemetry.Middleware("http"), services.owasp.Middleware, services.cors.Middleware))

healthService.WaitForTermination(appServer.Done())
clients.health.WaitForTermination(services.server.Done())

server.GracefulWait(appServer.Done())
server.GracefulWait(services.server.Done())
}
Loading

0 comments on commit 8e15229

Please sign in to comment.