Reorganize type declaration (#605)

* Reorganize type declaration

* Refactoring old code

* Rename types

* Fix tests

* Move files

* Rename unexported fields

* Refactoring dependencies

* Refactoring dependencies

* Refactor tests

* Refactoring dag package

* Simplify type naming

* Refactor mailer package

* Split file

* Rename package

* Refactor frontend server

* Refactor frontend server

* Fix frontend bug

* Update document

* Refactor frontend code

* Refactor frontend code

* Refactor frontend code

* Rename target name

* Update makefile

* Update contribution guide

* Move frontend package

* Move scheduler package

* Move scheduler service package

* Update make run command

* Move entry-reader pckage

* Refactor scheduler package

* Refactor scheduler package

* Refactor scheduler package

* Refactor scheduler package

* Update scheduler tests

* Update scheduler tests

* Remove unused function

* Update Makefile

* Update Makefile

* Move dag status type declaration

* Refactor dependency to config

* Refactor agent contstructor

* Refactor

* move grep package

* Rename package for tools.go

* Fix OpenAPI spec lint errors

* Refactor tests

* Fix lint error

* Move variable location

.github/workflows/release.yaml

@@ -39,7 +39,7 @@ jobs:
 
       - name: Copy web assets
         run: |
-          cp ui/dist/* ./internal/service/frontend/assets/
+          cp ui/dist/* ./internal/frontend/assets/
 
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v2

.gitignore

@@ -3,9 +3,9 @@ bin/
 dist/
 
 # frontend assets
-internal/service/frontend/assets/fonts/*
-internal/service/frontend/assets/*
-!internal/service/frontend/assets/.gitkeep
+internal/frontend/assets/fonts/*
+internal/frontend/assets/*
+!internal/frontend/assets/.gitkeep
 
 # NVM
 .nvmrc
@@ -19,6 +19,7 @@ tmp/*
 # Goland
 .idea
 
-# certs
-cert/*.pem
-cert/*.srl
+# local development
+!local/cert
+local/cert/*
+!local/cert/openssl.conf

.golangci.yml

@@ -61,8 +61,8 @@ issues:
     - .git
     - cert
     - bin
-    - internal/service/frontend/gen
-    - internal/service/scheduler/filenotify
+    - internal/frontend/gen
+    - internal/scheduler/filenotify
   exclude-rules:
     - path: _test\.go|tests/.+\.go
       linters:

.vscode/launch.json

@@ -12,7 +12,7 @@
             "program": "${workspaceFolder}/main.go",
             "args": [
                 "server",
-                "--port=9090",
+                "--port=8080",
                 "-s",
                 "localhost"
             ]

Dockerfile

@@ -20,8 +20,8 @@ ARG TARGETARCH
 WORKDIR /app
 COPY . .
 
-RUN go mod download && rm -rf service/frontend/assets
-COPY --from=ui-builder /app/dist/ ./internal/service/frontend/assets/
+RUN go mod download && rm -rf frontend/assets
+COPY --from=ui-builder /app/dist/ ./internal/frontend/assets/
 
 RUN GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags="${LDFLAGS}" -o ./bin/dagu .
 

Makefile

@@ -8,6 +8,7 @@ VERSION=
 
 # This Makefile's directory
 SCRIPT_DIR=$(abspath $(dir $(lastword $(MAKEFILE_LIST))))
+LOCAL_DIR=$(SCRIPT_DIR)/local
 
 SRC_DIR=$(SCRIPT_DIR)
 DST_DIR=$(SRC_DIR)/internal
@@ -29,13 +30,24 @@ PKG_gotestsum=gotest.tools/gotestsum
 COLOR_GREEN=\033[0;32m
 COLOR_RESET=\033[0m
 
-FE_DIR=./internal/service/frontend
+FE_DIR=./internal/frontend
 FE_GEN_DIR=${FE_DIR}/gen
 FE_ASSETS_DIR=${FE_DIR}/assets
 
-CERT_DIR=${SCRIPT_DIR}/cert
+CERT_DIR=${LOCAL_DIR}/cert
+
+CA_CERT_FILE=${CERT_DIR}/ca-cert.pem
+CA_KEY_FILE=${CERT_DIR}/ca-key.pem
+SERVER_CERT_REQ=${CERT_DIR}/server-req.pem
+SERVER_CERT_FILE=${CERT_DIR}/server-cert.pem
+SERVER_KEY_FILE=${CERT_DIR}/server-key.pem
+CLIENT_CERT_REQ=${CERT_DIR}/client-req.pem
+CLIENT_CERT_FILE=${CERT_DIR}/client-cert.pem
+CLIENT_KEY_FILE=${CERT_DIR}/client-key.pem
+OPENSSL_CONF=${CERT_DIR}/openssl.conf
 
 FE_BUILD_DIR=./ui/dist
+FE_BUNDLE_JS=${FE_ASSETS_DIR}/bundle.js
 
 APP_NAME=dagu
 BIN_DIR=${SCRIPT_DIR}/bin
@@ -46,15 +58,19 @@ GO_TEST_FLAGS=-v --race
 
 ########## Main Targets ##########
 
-# main starts the backend server.
-main: build-ui
-	go run . server
+# run starts the frontend server and the scheduler.
+run: ${FE_BUNDLE_JS}
+	go run . start-all
+
+# check if the frontend assets are built.
+${FE_BUNDLE_JS}:
+	echo "Please run 'make build-ui' to build the frontend assets."
 
 # https starts the server with the HTTPS protocol.
-https:
-	@DAGU_CERT_FILE=${CERT_DIR}/server-cert.pem \
-		DAGU_KEY_FILE=${CERT_DIR}/server-key.pem \
-		go run . server
+https: ${SERVER_CERT_FILE} ${SERVER_KEY_FILE}
+	@DAGU_CERT_FILE=${SERVER_CERT_FILE} \
+		DAGU_KEY_FILE=${SERVER_KEY_FILE} \
+		go run . start-all
 
 # watch starts development UI server.
 # The backend server should be running.
@@ -86,7 +102,7 @@ lint: golangci-lint
 swagger: clean-swagger gen-swagger
 
 # certs generates the certificates to use in the development environment.
-certs: cert-dir gencerts-ca gencerts-server gencerts-client gencert-check
+certs: ${SERVER_CERT_FILE} ${CLIENT_CERT_FILE} gencert-check
 
 # build build the binary.
 build: build-ui build-bin
@@ -147,40 +163,38 @@ gen-swagger:
 
 ########## Certificates ##########
 
-cert-dir:
-	@echo "${COLOR_GREEN}Creating cert directory...${COLOR_RESET}"
-	@mkdir -p ${CERT_DIR}
-
-gencerts-ca:
+${CA_CERT_FILE}:
 	@echo "${COLOR_GREEN}Generating CA certificates...${COLOR_RESET}"
 	@openssl req -x509 -newkey rsa:4096 \
-		-nodes -days 365 -keyout ${CERT_DIR}/ca-key.pem \
-		-out ${CERT_DIR}/ca-cert.pem \
+		-nodes -days 365 -keyout ${CA_KEY_FILE} \
+		-out ${CA_CERT_FILE} \
 		-subj "$(DEV_CERT_SUBJ_CA)"
 
-gencerts-server:
-	@echo "${COLOR_GREEN}Generating server certificates...${COLOR_RESET}"
-	@openssl req -newkey rsa:4096 -nodes -keyout ${CERT_DIR}/server-key.pem \
-		-out ${CERT_DIR}/server-req.pem \
+${SERVER_KEY_FILE}:
+	@echo "${COLOR_GREEN}Generating server key...${COLOR_RESET}"
+	@openssl req -newkey rsa:4096 -nodes -keyout ${SERVER_KEY_FILE} \
+		-out ${SERVER_CERT_REQ} \
 		-subj "$(DEV_CERT_SUBJ_SERVER)"
 
-	@echo "${COLOR_GREEN}Adding subjectAltName...${COLOR_RESET}"
-	@openssl x509 -req -in ${CERT_DIR}/server-req.pem -CA ${CERT_DIR}/ca-cert.pem -CAkey ${CERT_DIR}/ca-key.pem \
-		-CAcreateserial -out ${CERT_DIR}/server-cert.pem \
-		-extfile ${CERT_DIR}/openssl.conf
+${SERVER_CERT_FILE}: ${CA_CERT_FILE} ${SERVER_KEY_FILE}
+	@echo "${COLOR_GREEN}Generating server certificate...${COLOR_RESET}"
+	@openssl x509 -req -in ${SERVER_CERT_REQ} -CA ${CA_CERT_FILE} -CAkey ${CA_KEY_FILE} \
+		-CAcreateserial -out ${SERVER_CERT_FILE} \
+		-extfile ${OPENSSL_CONF}
 
-gencerts-client:
-	@echo "${COLOR_GREEN}Generating client certificates...${COLOR_RESET}"
-	@openssl req -newkey rsa:4096 -nodes -keyout cert/client-key.pem \
-		-out cert/client-req.pem \
+${CLIENT_KEY_FILE}:
+	@echo "${COLOR_GREEN}Generating client key...${COLOR_RESET}"
+	@openssl req -newkey rsa:4096 -nodes -keyout ${CLIENT_KEY_FILE} \
+		-out ${CLIENT_CERT_REQ} \
 		-subj "$(DEV_CERT_SUBJ_CLIENT)"
 
-	@echo "${COLOR_GREEN}Adding subjectAltName...${COLOR_RESET}"
-	@openssl x509 -req -in cert/client-req.pem -days 60 -CA cert/ca-cert.pem \
-		-CAkey cert/ca-key.pem -CAcreateserial -out cert/client-cert.pem \
-		-extfile cert/openssl.conf
+${CLIENT_CERT_FILE}: ${CA_CERT_FILE} ${CLIENT_KEY_FILE}
+	@echo "${COLOR_GREEN}Generating client certificate...${COLOR_RESET}"
+	@openssl x509 -req -in ${CLIENT_CERT_REQ} -days 60 -CA ${CA_CERT_FILE} \
+		-CAkey ${CA_KEY_FILE} -CAcreateserial -out ${CLIENT_CERT_FILE} \
+		-extfile ${OPENSSL_CONF}
 
 gencert-check:
 	@echo "${COLOR_GREEN}Checking CA certificate...${COLOR_RESET}"
-	@openssl x509 -in cert/server-cert.pem -noout -text
+	@openssl x509 -in ${SERVER_CERT_FILE} -noout -text
 

cmd/common_test.go

@@ -7,40 +7,51 @@ import (
 	"time"
 
 	"github.com/dagu-dev/dagu/internal/config"
+	"github.com/dagu-dev/dagu/internal/dag"
 	"github.com/dagu-dev/dagu/internal/persistence"
-	"github.com/dagu-dev/dagu/internal/persistence/client"
 
+	"github.com/dagu-dev/dagu/internal/dag/scheduler"
 	"github.com/dagu-dev/dagu/internal/engine"
-	"github.com/dagu-dev/dagu/internal/scheduler"
 	"github.com/dagu-dev/dagu/internal/util"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/require"
 )
 
+type testSetup struct {
+	homeDir   string
+	engine    engine.Engine
+	dataStore persistence.DataStores
+	cfg       *config.Config
+}
+
+func (t testSetup) cleanup() {
+	_ = os.RemoveAll(t.homeDir)
+}
+
 // setupTest is a helper function to setup the test environment.
 // This function does the following:
 // 1. It creates a temporary directory and returns the path to it.
 // 2. Sets the home directory to the temporary directory.
 // 3. Creates a new data store factory and engine.
-func setupTest(t *testing.T) (
-	string, engine.Engine, persistence.DataStoreFactory, *config.Config,
-) {
+func setupTest(t *testing.T) testSetup {
 	t.Helper()
 
 	tmpDir := util.MustTempDir("dagu_test")
 	err := os.Setenv("HOME", tmpDir)
 	require.NoError(t, err)
 
-	dataStore := client.NewDataStoreFactory(&config.Config{
-		DataDir: path.Join(tmpDir, ".dagu", "data"),
-	})
-
 	cfg, err := config.Load()
 	require.NoError(t, err)
 
-	return tmpDir, engine.New(
-		dataStore, new(engine.Config), cfg,
-	), dataStore, cfg
+	cfg.DataDir = path.Join(tmpDir, ".dagu", "data")
+	dataStore := newDataStores(cfg)
+
+	return testSetup{
+		homeDir:   tmpDir,
+		dataStore: dataStore,
+		engine:    newEngine(cfg),
+		cfg:       cfg,
+	}
 }
 
 // cmdTest is a helper struct to test commands.
@@ -135,7 +146,7 @@ func testStatusEventual(t *testing.T, e engine.Engine, dagFile string, expected
 	cfg, err := config.Load()
 	require.NoError(t, err)
 
-	dg, err := loadDAG(cfg, dagFile, "")
+	dg, err := dag.Load(cfg.BaseConfig, dagFile, "")
 	require.NoError(t, err)
 
 	require.Eventually(t, func() bool {