diff --git a/Makefile b/Makefile index 5170436d839f..6646f3b4c157 100644 --- a/Makefile +++ b/Makefile @@ -264,7 +264,7 @@ clean: .PHONY: fmt fmt: - GOFUMPT_PACKAGE=$(GOFUMPT_PACKAGE) $(GO) run build/code-batch-process.go gitea-fmt -w '{file-list}' + @GO=$(GO) ./build/gitea-fmt.sh -w $(eval TEMPLATES := $(shell find templates -type f -name '*.tmpl')) @# strip whitespace after '{{' and before `}}` unless there is only whitespace before it @$(SED_INPLACE) -e 's/{{[ ]\{1,\}/{{/g' -e '/^[ ]\{1,\}}}/! s/[ ]\{1,\}}}/}}/g' $(TEMPLATES) diff --git a/build.go b/build.go index d379745c6dfc..0c57ffe5a6d6 100644 --- a/build.go +++ b/build.go @@ -10,15 +10,8 @@ package main // These libraries will not be included in a normal compilation. import ( - // for embed - _ "github.com/shurcooL/vfsgen" - - // for cover merge - _ "golang.org/x/tools/cover" - - // for vet - _ "code.gitea.io/gitea-vet" - - // for swagger - _ "github.com/go-swagger/go-swagger/cmd/swagger" + _ "code.gitea.io/gitea-vet" // for vet + _ "github.com/go-swagger/go-swagger/cmd/swagger" // for swagger + _ "github.com/shurcooL/vfsgen" // for embed + _ "golang.org/x/tools/cover" // for cover merge ) diff --git a/build/code-batch-process.go b/build/code-batch-process.go deleted file mode 100644 index b6c4171ede71..000000000000 --- a/build/code-batch-process.go +++ /dev/null @@ -1,292 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -//go:build ignore - -package main - -import ( - "fmt" - "log" - "os" - "os/exec" - "path/filepath" - "regexp" - "strconv" - "strings" - - "code.gitea.io/gitea/build/codeformat" -) - -// Windows has a limitation for command line arguments, the size can not exceed 32KB. -// So we have to feed the files to some tools (like gofmt) batch by batch - -// We also introduce a `gitea-fmt` command, it does better import formatting than gofmt/goimports. `gitea-fmt` calls `gofmt` internally. - -var optionLogVerbose bool - -func logVerbose(msg string, args ...interface{}) { - if optionLogVerbose { - log.Printf(msg, args...) - } -} - -func passThroughCmd(cmd string, args []string) error { - foundCmd, err := exec.LookPath(cmd) - if err != nil { - log.Fatalf("can not find cmd: %s", cmd) - } - c := exec.Cmd{ - Path: foundCmd, - Args: append([]string{cmd}, args...), - Stdin: os.Stdin, - Stdout: os.Stdout, - Stderr: os.Stderr, - } - return c.Run() -} - -type fileCollector struct { - dirs []string - includePatterns []*regexp.Regexp - excludePatterns []*regexp.Regexp - batchSize int -} - -func newFileCollector(fileFilter string, batchSize int) (*fileCollector, error) { - co := &fileCollector{batchSize: batchSize} - if fileFilter == "go-own" { - co.dirs = []string{ - "build", - "cmd", - "contrib", - "tests", - "models", - "modules", - "routers", - "services", - "tools", - } - co.includePatterns = append(co.includePatterns, regexp.MustCompile(`.*\.go$`)) - - co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`.*\bbindata\.go$`)) - co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`tests/gitea-repositories-meta`)) - co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`tests/integration/migration-test`)) - co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`modules/git/tests`)) - co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`models/fixtures`)) - co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`models/migrations/fixtures`)) - co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`services/gitdiff/testdata`)) - } - - if co.dirs == nil { - return nil, fmt.Errorf("unknown file-filter: %s", fileFilter) - } - return co, nil -} - -func (fc *fileCollector) matchPatterns(path string, regexps []*regexp.Regexp) bool { - path = strings.ReplaceAll(path, "\\", "/") - for _, re := range regexps { - if re.MatchString(path) { - return true - } - } - return false -} - -func (fc *fileCollector) collectFiles() (res [][]string, err error) { - var batch []string - for _, dir := range fc.dirs { - err = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { - include := len(fc.includePatterns) == 0 || fc.matchPatterns(path, fc.includePatterns) - exclude := fc.matchPatterns(path, fc.excludePatterns) - process := include && !exclude - if !process { - if d.IsDir() { - if exclude { - logVerbose("exclude dir %s", path) - return filepath.SkipDir - } - // for a directory, if it is not excluded explicitly, we should walk into - return nil - } - // for a file, we skip it if it shouldn't be processed - logVerbose("skip process %s", path) - return nil - } - if d.IsDir() { - // skip dir, we don't add dirs to the file list now - return nil - } - if len(batch) >= fc.batchSize { - res = append(res, batch) - batch = nil - } - batch = append(batch, path) - return nil - }) - if err != nil { - return nil, err - } - } - res = append(res, batch) - return res, nil -} - -// substArgFiles expands the {file-list} to a real file list for commands -func substArgFiles(args, files []string) []string { - for i, s := range args { - if s == "{file-list}" { - newArgs := append(args[:i], files...) - newArgs = append(newArgs, args[i+1:]...) - return newArgs - } - } - return args -} - -func exitWithCmdErrors(subCmd string, subArgs []string, cmdErrors []error) { - for _, err := range cmdErrors { - if err != nil { - if exitError, ok := err.(*exec.ExitError); ok { - exitCode := exitError.ExitCode() - log.Printf("run command failed (code=%d): %s %v", exitCode, subCmd, subArgs) - os.Exit(exitCode) - } else { - log.Fatalf("run command failed (err=%s) %s %v", err, subCmd, subArgs) - } - } - } -} - -func parseArgs() (mainOptions map[string]string, subCmd string, subArgs []string) { - mainOptions = map[string]string{} - for i := 1; i < len(os.Args); i++ { - arg := os.Args[i] - if arg == "" { - break - } - if arg[0] == '-' { - arg = strings.TrimPrefix(arg, "-") - arg = strings.TrimPrefix(arg, "-") - fields := strings.SplitN(arg, "=", 2) - if len(fields) == 1 { - mainOptions[fields[0]] = "1" - } else { - mainOptions[fields[0]] = fields[1] - } - } else { - subCmd = arg - subArgs = os.Args[i+1:] - break - } - } - return -} - -func showUsage() { - fmt.Printf(`Usage: %[1]s [options] {command} [arguments] - -Options: - --verbose - --file-filter=go-own - --batch-size=100 - -Commands: - %[1]s gofmt ... - -Arguments: - {file-list} the file list - -Example: - %[1]s gofmt -s -d {file-list} - -`, "file-batch-exec") -} - -func getGoVersion() string { - goModFile, err := os.ReadFile("go.mod") - if err != nil { - log.Fatalf(`Faild to read "go.mod": %v`, err) - os.Exit(1) - } - goModVersionRegex := regexp.MustCompile(`go \d+\.\d+`) - goModVersionLine := goModVersionRegex.Find(goModFile) - return string(goModVersionLine[3:]) -} - -func newFileCollectorFromMainOptions(mainOptions map[string]string) (fc *fileCollector, err error) { - fileFilter := mainOptions["file-filter"] - if fileFilter == "" { - fileFilter = "go-own" - } - batchSize, _ := strconv.Atoi(mainOptions["batch-size"]) - if batchSize == 0 { - batchSize = 100 - } - - return newFileCollector(fileFilter, batchSize) -} - -func containsString(a []string, s string) bool { - for _, v := range a { - if v == s { - return true - } - } - return false -} - -func giteaFormatGoImports(files []string, doWriteFile bool) error { - for _, file := range files { - if err := codeformat.FormatGoImports(file, doWriteFile); err != nil { - log.Printf("failed to format go imports: %s, err=%v", file, err) - return err - } - } - return nil -} - -func main() { - mainOptions, subCmd, subArgs := parseArgs() - if subCmd == "" { - showUsage() - os.Exit(1) - } - optionLogVerbose = mainOptions["verbose"] != "" - - fc, err := newFileCollectorFromMainOptions(mainOptions) - if err != nil { - log.Fatalf("can not create file collector: %s", err.Error()) - } - - fileBatches, err := fc.collectFiles() - if err != nil { - log.Fatalf("can not collect files: %s", err.Error()) - } - - processed := 0 - var cmdErrors []error - for _, files := range fileBatches { - if len(files) == 0 { - break - } - substArgs := substArgFiles(subArgs, files) - logVerbose("batch cmd: %s %v", subCmd, substArgs) - switch subCmd { - case "gitea-fmt": - if containsString(subArgs, "-d") { - log.Print("the -d option is not supported by gitea-fmt") - } - cmdErrors = append(cmdErrors, giteaFormatGoImports(files, containsString(subArgs, "-w"))) - cmdErrors = append(cmdErrors, passThroughCmd("go", append([]string{"run", os.Getenv("GOFUMPT_PACKAGE"), "-extra", "-lang", getGoVersion()}, substArgs...))) - default: - log.Fatalf("unknown cmd: %s %v", subCmd, subArgs) - } - processed += len(files) - } - - logVerbose("processed %d files", processed) - exitWithCmdErrors(subCmd, subArgs, cmdErrors) -} diff --git a/build/codeformat.go b/build/codeformat.go new file mode 100644 index 000000000000..f7920cb37cfc --- /dev/null +++ b/build/codeformat.go @@ -0,0 +1,73 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +//go:build ignore + +// Gitea's code formatter: +// * Sort imports with 3 groups: std, gitea, other + +package main + +import ( + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "code.gitea.io/gitea/build/codeformat" +) + +func showUsage() { + fmt.Printf("Usage: codeformat {-l|-w} directory\n") +} + +var ignoreList = []string{ + "_bindata.go", + "tests/gitea-repositories-meta", + "tests/integration/migration-test", + "modules/git/tests", + "models/fixtures", + "models/migrations/fixtures", + "services/gitdiff/testdata", +} + +func main() { + if len(os.Args) != 3 { + showUsage() + os.Exit(1) + } + doList := os.Args[1] == "-l" + doWrite := os.Args[1] == "-w" + dir := os.Args[2] + if !doList && !doWrite { + showUsage() + fmt.Printf("You should set either '-l' or '-w'\n") + os.Exit(1) + } + + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + path = strings.ReplaceAll(path, "\\", "/") + for _, ignore := range ignoreList { + if strings.Contains(path, ignore) { + return filepath.SkipDir + } + } + if d.IsDir() { + return nil // walk into + } + if !strings.HasSuffix(path, ".go") { + return nil + } + if err := codeformat.FormatGoImports(path, doList, doWrite); err != nil { + log.Printf("Failed to format go imports: %s, err=%v", path, err) + return err + } + return nil + }) + if err != nil { + log.Printf("Failed to format code by walking directory: %s, err=%v", dir, err) + os.Exit(1) + } +} diff --git a/build/codeformat/formatimports.go b/build/codeformat/formatimports.go index 1076e3a0d1cd..e5ef6e423292 100644 --- a/build/codeformat/formatimports.go +++ b/build/codeformat/formatimports.go @@ -7,6 +7,7 @@ package codeformat import ( "bytes" "errors" + "fmt" "io" "os" "sort" @@ -18,6 +19,8 @@ var importPackageGroupOrders = map[string]int{ "code.gitea.io/gitea/": 2, } +// if comments are put between imports, the imports will be separated into different sorting groups, +// which is incorrect for most cases, so we forbid putting comments between imports var errInvalidCommentBetweenImports = errors.New("comments between imported packages are invalid, please move comments to the end of the package line") var ( @@ -158,7 +161,7 @@ func formatGoImports(contentBytes []byte) ([]byte, error) { } // FormatGoImports format the imports by our rules (see unit tests) -func FormatGoImports(file string, doWriteFile bool) error { +func FormatGoImports(file string, doListFile, doWriteFile bool) error { f, err := os.Open(file) if err != nil { return err @@ -182,6 +185,10 @@ func FormatGoImports(file string, doWriteFile bool) error { return nil } + if doListFile { + fmt.Println(file) + } + if doWriteFile { f, err = os.OpenFile(file, os.O_TRUNC|os.O_WRONLY, 0o644) if err != nil { diff --git a/build/gitea-fmt.sh b/build/gitea-fmt.sh new file mode 100755 index 000000000000..b5ba4b2d3913 --- /dev/null +++ b/build/gitea-fmt.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -e + +GO="${GO:=go}" + +# a simple self-check, make sure the current working directory is Gitea's repo +if [ ! -f ./build/gitea-fmt.sh ]; then + echo "$0 could only run in Gitea's source directory" + exit 1 +fi + +if [ "$1" != "-l" -a "$1" != "-w" ]; then + echo "$0 could only accept '-l' (list only) or '-w' (write to files) argument" + exit 1 +fi + +GO_VERSION=$(grep -Eo '^go\s+[0-9]+\.[0-9]+' go.mod | cut -d' ' -f2) + +echo "Run gofumpt with Go language version $GO_VERSION ..." +gofumpt -extra -lang "$GO_VERSION" "$1" . + +echo "Run codeformat ..." +"$GO" run ./build/codeformat.go "$1" . diff --git a/main.go b/main.go index 0e550f05ebca..8b36f47be7de 100644 --- a/main.go +++ b/main.go @@ -17,8 +17,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - // register supported doc types - _ "code.gitea.io/gitea/modules/markup/console" + _ "code.gitea.io/gitea/modules/markup/console" // register supported doc types _ "code.gitea.io/gitea/modules/markup/csv" _ "code.gitea.io/gitea/modules/markup/markdown" _ "code.gitea.io/gitea/modules/markup/orgmode"