-
-
Notifications
You must be signed in to change notification settings - Fork 1.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Add asset server and upload handling of binary files Co-authored-by: Björn Brauer <zaubernerd@zaubernerd.de> * Add asset download parts to the asset server Co-authored-by: Björn Brauer <zaubernerd@zaubernerd.de> * Add artifact-server-path flag If the flag is not defined, the artifact server isn't started. This includes the configuration of ACTIONS_RUNTIME_URL and ACTIONS_RUNTIME_TOKEN which are set if the server is started. Co-authored-by: Björn Brauer <zaubernerd@zaubernerd.de> * Move ACTIONS_RUNTIME_* vars into the withGithubEnv setup Co-authored-by: Björn Brauer <zaubernerd@zaubernerd.de> * feat: add artifact server port as flag This commits adds a flag to define the artifact server port. If not given, the port defaults to 34567. Co-authored-by: Björn Brauer <bjoern.brauer@new-work.se> * test: add artifact server tests Co-authored-by: Björn Brauer <bjoern.brauer@new-work.se> * refactor: use fs.FS This allows to add tests with in-memory file system * feat: add support for gzip encoded uploads Co-authored-by: Björn Brauer <bjoern.brauer@new-work.se> * test: add artifact integration test * chore: run act tests with asset server path Co-authored-by: Björn Brauer <bjoern.brauer@new-work.se> * docs: add new cli flags Co-authored-by: Björn Brauer <bjoern.brauer@new-work.se> * test: add test workflow to testdata * feat: add log output * refactor: log shutdown error instead of panic * feat: use outbound ip for the asset server This change should allow to use the host ip in macos and windows. Since docker is running in an intermediate vm, localhost is not sufficient to have the artifacts in the host system. * fix: do not use canceled context To shutdown artifact server, we should not use the already canceled context but the parent context instead. Co-authored-by: Björn Brauer <zaubernerd@zaubernerd.de> * feat: shutdown artifact server at end of pipeline When the pipeline is done the asset server should be shut down gracefully. Co-authored-by: Björn Brauer <zaubernerd@zaubernerd.de> * fix: close server if graceful shutdown failed Co-authored-by: Björn Brauer <zaubernerd@zaubernerd.de> * fix: ignore server closed error from listen call Co-authored-by: Björn Brauer <zaubernerd@zaubernerd.de> Co-authored-by: Björn Brauer <zaubernerd@zaubernerd.de> Co-authored-by: Björn Brauer <bjoern.brauer@new-work.se>
- Loading branch information
3 people
committed
Nov 10, 2021
1 parent
f8b3563
commit 11f6ee3
Showing
12 changed files
with
775 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,278 @@ | ||
| package artifacts | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "errors" | ||
| "fmt" | ||
| "io" | ||
| "io/fs" | ||
| "net/http" | ||
| "os" | ||
| "path" | ||
| "path/filepath" | ||
| "strings" | ||
|
|
||
| "github.com/julienschmidt/httprouter" | ||
| "github.com/nektos/act/pkg/common" | ||
| log "github.com/sirupsen/logrus" | ||
| ) | ||
|
|
||
| type FileContainerResourceURL struct { | ||
| FileContainerResourceURL string `json:"fileContainerResourceUrl"` | ||
| } | ||
|
|
||
| type NamedFileContainerResourceURL struct { | ||
| Name string `json:"name"` | ||
| FileContainerResourceURL string `json:"fileContainerResourceUrl"` | ||
| } | ||
|
|
||
| type NamedFileContainerResourceURLResponse struct { | ||
| Count int `json:"count"` | ||
| Value []NamedFileContainerResourceURL `json:"value"` | ||
| } | ||
|
|
||
| type ContainerItem struct { | ||
| Path string `json:"path"` | ||
| ItemType string `json:"itemType"` | ||
| ContentLocation string `json:"contentLocation"` | ||
| } | ||
|
|
||
| type ContainerItemResponse struct { | ||
| Value []ContainerItem `json:"value"` | ||
| } | ||
|
|
||
| type ResponseMessage struct { | ||
| Message string `json:"message"` | ||
| } | ||
|
|
||
| type MkdirFS interface { | ||
| fs.FS | ||
| MkdirAll(path string, perm fs.FileMode) error | ||
| Open(name string) (fs.File, error) | ||
| } | ||
|
|
||
| type MkdirFsImpl struct { | ||
| dir string | ||
| fs.FS | ||
| } | ||
|
|
||
| func (fsys MkdirFsImpl) MkdirAll(path string, perm fs.FileMode) error { | ||
| return os.MkdirAll(fsys.dir+"/"+path, perm) | ||
| } | ||
|
|
||
| func (fsys MkdirFsImpl) Open(name string) (fs.File, error) { | ||
| return os.OpenFile(fsys.dir+"/"+name, os.O_CREATE|os.O_RDWR, 0644) | ||
| } | ||
|
|
||
| var gzipExtension = ".gz__" | ||
|
|
||
| func uploads(router *httprouter.Router, fsys MkdirFS) { | ||
| router.POST("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { | ||
| runID := params.ByName("runId") | ||
|
|
||
| json, err := json.Marshal(FileContainerResourceURL{ | ||
| FileContainerResourceURL: fmt.Sprintf("http://%s/upload/%s", req.Host, runID), | ||
| }) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| _, err = w.Write(json) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| }) | ||
|
|
||
| router.PUT("/upload/:runId", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { | ||
| itemPath := req.URL.Query().Get("itemPath") | ||
| runID := params.ByName("runId") | ||
|
|
||
| if req.Header.Get("Content-Encoding") == "gzip" { | ||
| itemPath += gzipExtension | ||
| } | ||
|
|
||
| filePath := fmt.Sprintf("%s/%s", runID, itemPath) | ||
|
|
||
| err := fsys.MkdirAll(path.Dir(filePath), os.ModePerm) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| file, err := fsys.Open(filePath) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| defer file.Close() | ||
|
|
||
| writer, ok := file.(io.Writer) | ||
| if !ok { | ||
| panic(errors.New("File is not writable")) | ||
| } | ||
|
|
||
| if req.Body == nil { | ||
| panic(errors.New("No body given")) | ||
| } | ||
|
|
||
| _, err = io.Copy(writer, req.Body) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| json, err := json.Marshal(ResponseMessage{ | ||
| Message: "success", | ||
| }) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| _, err = w.Write(json) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| }) | ||
|
|
||
| router.PATCH("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { | ||
| json, err := json.Marshal(ResponseMessage{ | ||
| Message: "success", | ||
| }) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| _, err = w.Write(json) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| }) | ||
| } | ||
|
|
||
| func downloads(router *httprouter.Router, fsys fs.FS) { | ||
| router.GET("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { | ||
| runID := params.ByName("runId") | ||
|
|
||
| entries, err := fs.ReadDir(fsys, runID) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| var list []NamedFileContainerResourceURL | ||
| for _, entry := range entries { | ||
| list = append(list, NamedFileContainerResourceURL{ | ||
| Name: entry.Name(), | ||
| FileContainerResourceURL: fmt.Sprintf("http://%s/download/%s", req.Host, runID), | ||
| }) | ||
| } | ||
|
|
||
| json, err := json.Marshal(NamedFileContainerResourceURLResponse{ | ||
| Count: len(list), | ||
| Value: list, | ||
| }) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| _, err = w.Write(json) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| }) | ||
|
|
||
| router.GET("/download/:container", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { | ||
| container := params.ByName("container") | ||
| itemPath := req.URL.Query().Get("itemPath") | ||
| dirPath := fmt.Sprintf("%s/%s", container, itemPath) | ||
|
|
||
| var files []ContainerItem | ||
| err := fs.WalkDir(fsys, dirPath, func(path string, entry fs.DirEntry, err error) error { | ||
| if !entry.IsDir() { | ||
| rel, err := filepath.Rel(dirPath, path) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| // if it was upload as gzip | ||
| rel = strings.TrimSuffix(rel, gzipExtension) | ||
|
|
||
| files = append(files, ContainerItem{ | ||
| Path: fmt.Sprintf("%s/%s", itemPath, rel), | ||
| ItemType: "file", | ||
| ContentLocation: fmt.Sprintf("http://%s/artifact/%s/%s/%s", req.Host, container, itemPath, rel), | ||
| }) | ||
| } | ||
| return nil | ||
| }) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| json, err := json.Marshal(ContainerItemResponse{ | ||
| Value: files, | ||
| }) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| _, err = w.Write(json) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| }) | ||
|
|
||
| router.GET("/artifact/*path", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { | ||
| path := params.ByName("path")[1:] | ||
|
|
||
| file, err := fsys.Open(path) | ||
| if err != nil { | ||
| // try gzip file | ||
| file, err = fsys.Open(path + gzipExtension) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| w.Header().Add("Content-Encoding", "gzip") | ||
| } | ||
|
|
||
| _, err = io.Copy(w, file) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| }) | ||
| } | ||
|
|
||
| func Serve(ctx context.Context, artifactPath string, port string) context.CancelFunc { | ||
| serverContext, cancel := context.WithCancel(ctx) | ||
|
|
||
| if artifactPath == "" { | ||
| return cancel | ||
| } | ||
|
|
||
| router := httprouter.New() | ||
|
|
||
| log.Debugf("Artifacts base path '%s'", artifactPath) | ||
| fs := os.DirFS(artifactPath) | ||
| uploads(router, MkdirFsImpl{artifactPath, fs}) | ||
| downloads(router, fs) | ||
| ip := common.GetOutboundIP().String() | ||
|
|
||
| server := &http.Server{Addr: fmt.Sprintf("%s:%s", ip, port), Handler: router} | ||
|
|
||
| // run server | ||
| go func() { | ||
| log.Infof("Start server on http://%s:%s", ip, port) | ||
| if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { | ||
| log.Fatal(err) | ||
| } | ||
| }() | ||
|
|
||
| // wait for cancel to gracefully shutdown server | ||
| go func() { | ||
| <-serverContext.Done() | ||
|
|
||
| if err := server.Shutdown(ctx); err != nil { | ||
| log.Errorf("Failed shutdown gracefully - force shutdown: %v", err) | ||
| server.Close() | ||
| } | ||
| }() | ||
|
|
||
| return cancel | ||
| } |
Oops, something went wrong.