Container images

.dockerignore

@@ -0,0 +1,3 @@
+./build
+./.vscode
+./.github

.github/workflows/images.yml

@@ -0,0 +1,71 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# GitHub recommends pinning actions to a commit SHA.
+# To get a newer version, you will need to update the SHA.
+# You can also reference a tag or branch, but the action may change without warning.
+
+name: Create and publish a Docker images
+
+on:
+  push: {}
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_BASE_NAME: ${{ github.repository_owner }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    strategy:
+      matrix:
+        app:
+          - name: web_proxy
+            image: web_proxy
+          - name: static_datastore_builder
+            image: static_datastore_builder
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            "${{ env.REGISTRY }}/${{ env.IMAGE_BASE_NAME }}/${{ matrix.app.image }}"
+          tags: |
+            type=ref,event=branch,prefix=branch-
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: build/docker/Dockerfile
+          platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: APP_NAME=${{ matrix.app.name }}

.github/workflows/tests.yml

@@ -14,7 +14,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: "1.19"
+          go-version: "1.20"
       - uses: actions/checkout@v3
       - run: go vet ./...
       - run: go test -v ${{ matrix.env.coverage && '-coverprofile=profile.cov' }} ./...

build/docker/Dockerfile

@@ -0,0 +1,55 @@
+ARG GO_VERSION=1.20
+ARG TARGET_UID=9999
+ARG TARGET_GID=9999
+
+# Base image with go compiler and tested source code
+FROM --platform=$BUILDPLATFORM docker.io/library/golang:${GO_VERSION} as source
+
+ARG TARGET_UID TARGET_GID
+
+# Don't run the compiler as root
+RUN for dir in /.cache "/apps"; do \
+         mkdir -p "${dir}"; \
+         chown "${TARGET_UID}:${TARGET_GID}" "${dir}"; \
+    done
+USER "${TARGET_UID}:${TARGET_GID}"
+WORKDIR /usr/src/app
+
+# Fetch dependencies first
+COPY go.mod go.sum ./
+RUN go mod download && go mod verify
+
+# Fetch and test the source code
+COPY --chown=${TARGET_UID}:${TARGET_GID} . .
+RUN go vet -v ./...
+RUN go test -v ./...
+
+# Build binary
+FROM source AS binaries
+ARG APP_NAME TARGET_UID TARGET_GID TARGETOS TARGETARCH
+
+ENV CGO_ENABLED=0
+RUN mkdir -p "/apps/${APP_NAME}/"
+RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
+    go build \
+        -v \
+        -ldflags="-extldflags=-static" \
+        -o "/apps/${APP_NAME}/${APP_NAME}" \
+        "./cmd/${APP_NAME}"
+RUN ln -s "/${APP_NAME}" "/apps/${APP_NAME}/app"
+
+
+# Prepare target filesystems
+FROM alpine:latest as filesystem
+ARG TARGET_UID TARGET_GID APP_NAME
+COPY --from=binaries --chown=0:0 /apps/ /filesystem/
+COPY --from=alpine:latest /etc/ssl/certs/ca-certificates.crt /filesystem/web_proxy/etc/ssl/certs/
+RUN mkdir -p /filesystem/static_datastore_builder/data \
+ && chown "$TARGET_UID:$TARGET_GID" /filesystem/static_datastore_builder/data
+RUN find /filesystem/${APP_NAME}
+
+FROM scratch
+ARG APP_NAME TARGET_UID TARGET_GID
+COPY --from=filesystem /filesystem/${APP_NAME} /
+USER ${TARGET_UID}:${TARGET_GID}
+CMD [ "/${APP_NAME}" ]

pkg/cmd/cinode_web_proxy/root.go

@@ -20,7 +20,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"log"
 	"net/http"
 	"os"
@@ -132,43 +131,7 @@ func setupCinodeProxy(
 		IndexFile:        "index.html",
 	}
 
-	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.Method != "GET" {
-			http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
-			return
-		}
-
-		path := strings.TrimPrefix(r.URL.Path, "/")
-
-		fileEP, err := fs.FindEntrypoint(r.Context(), path)
-		switch {
-		case errors.Is(err, structure.ErrNotFound):
-			http.NotFound(w, r)
-			return
-		case err != nil:
-			log.Println("Error serving request:", err)
-			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
-			return
-		}
-
-		if fileEP.MimeType == structure.CinodeDirMimeType {
-			http.Redirect(w, r, r.URL.Path+"/", http.StatusPermanentRedirect)
-			return
-		}
-
-		w.Header().Set("Content-Type", fileEP.GetMimeType())
-		rc, err := fs.OpenContent(r.Context(), fileEP)
-		if err != nil {
-			log.Printf("Error sending file: %v", err)
-		}
-		defer rc.Close()
-
-		_, err = io.Copy(w, rc)
-		if err != nil {
-			log.Printf("Error sending file: %v", err)
-		}
-
-	})
-
-	return handler
+	return &structure.HTTPHandler{
+		FS: &fs,
+	}
 }

pkg/cmd/static_datastore/compile.go

@@ -18,10 +18,10 @@ package static_datastore
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"log"
 	"os"
-	"path"
 
 	"github.com/cinode/go/pkg/blenc"
 	"github.com/cinode/go/pkg/datastore"
@@ -37,6 +37,7 @@ func compileCmd() *cobra.Command {
 	var useStaticBlobs bool
 	var useRawFilesystem bool
 	var rootWriterInfoStr string
+	var rootWriterInfoFile string
 
 	cmd := &cobra.Command{
 		Use:   "compile --source <src_dir> --destination <dst_dir>",
@@ -45,52 +46,95 @@ func compileCmd() *cobra.Command {
 The compile command can be used to create an encrypted datastore from
 a content with static files that can then be used to serve through a
 simple http server.
-
-Files stored on disk are encrypted through the blenc layer. However
-this tool should not be considered secure since the encryption key
-for the root node is stored in plaintext in an 'entrypoint.txt' file.
 `,
 		Run: func(cmd *cobra.Command, args []string) {
 			if srcDir == "" || dstDir == "" {
 				cmd.Help()
 				return
 			}
 
+			enc := json.NewEncoder(os.Stdout)
+			enc.SetIndent("", "  ")
+
+			fatalResult := func(format string, args ...interface{}) {
+				msg := fmt.Sprintf(format, args...)
+
+				enc.Encode(map[string]string{
+					"result": "ERROR",
+					"msg":    msg,
+				})
+
+				log.Fatalf(msg)
+			}
+
 			var wi *protobuf.WriterInfo
+			if len(rootWriterInfoFile) > 0 {
+				data, err := os.ReadFile(rootWriterInfoFile)
+				if err != nil {
+					fatalResult("Couldn't read data from the writer info file at '%s': %v", rootWriterInfoFile, err)
+				}
+				if len(data) == 0 {
+					fatalResult("Writer info file at '%s' is empty", rootWriterInfoFile)
+				}
+				rootWriterInfoStr = string(data)
+			}
 			if len(rootWriterInfoStr) > 0 {
 				_wi, err := protobuf.WriterInfoFromBytes(base58.Decode(rootWriterInfoStr))
 				if err != nil {
-					log.Fatalf("Couldn't parse writer info: %v", err)
+					fatalResult("Couldn't parse writer info: %v", err)
 				}
 				wi = _wi
 			}
 
-			wi, err := compileFS(srcDir, dstDir, useStaticBlobs, wi, useRawFilesystem)
+			ep, wi, err := compileFS(srcDir, dstDir, useStaticBlobs, wi, useRawFilesystem)
+			if err != nil {
+				fatalResult("%s", err)
+			}
+
+			epBytes, err := ep.ToBytes()
 			if err != nil {
-				log.Fatal(err)
+				fatalResult("Couldn't serialize entrypoint: %v", err)
+			}
+
+			result := map[string]string{
+				"result":     "OK",
+				"entrypoint": base58.Encode(epBytes),
 			}
 			if wi != nil {
 				wiBytes, err := wi.ToBytes()
 				if err != nil {
-					log.Fatalf("Couldn't serialize writer info: %v", err)
+					fatalResult("Couldn't serialize writer info: %v", err)
 				}
-				log.Printf("Generated new root dynamic link, writer info: %s", base58.Encode(wiBytes))
+
+				result["writer-info"] = base58.Encode(wiBytes)
 			}
-			log.Println()
+			enc.Encode(result)
+
 			log.Println("DONE")
+
 		},
 	}
 
 	cmd.Flags().StringVarP(&srcDir, "source", "s", "", "Source directory with content to compile")
 	cmd.Flags().StringVarP(&dstDir, "destination", "d", "", "Destination directory for blobs")
-	cmd.Flags().BoolVarP(&useStaticBlobs, "static", "t", false, "If set to true, compile static dataset and entrypoint.txt file with static dataset")
+	cmd.Flags().BoolVarP(&useStaticBlobs, "static", "t", false, "If set to true, compile only the static dataset, do not create or update dynamic link")
 	cmd.Flags().BoolVarP(&useRawFilesystem, "raw-filesystem", "r", false, "If set to true, use raw filesystem instead of the optimized one, can be used to create dataset for a standard http server")
-	cmd.Flags().StringVarP(&rootWriterInfoStr, "writer-info", "w", "", "Writer info for the root dynamic link, if not specified, a random writer info will be generated and printed out")
+	cmd.Flags().StringVarP(&rootWriterInfoStr, "writer-info", "w", "", "Writer info for the root dynamic link, if neither writer info nor writer info file is specified, a random writer info will be generated and printed out")
+	cmd.Flags().StringVarP(&rootWriterInfoFile, "writer-info-file", "f", "", "Name of the file containing writer info for the root dynamic link, if neither writer info nor writer info file is specified, a random writer info will be generated and printed out")
 
 	return cmd
 }
 
-func compileFS(srcDir, dstDir string, static bool, writerInfo *protobuf.WriterInfo, useRawFS bool) (*protobuf.WriterInfo, error) {
+func compileFS(
+	srcDir, dstDir string,
+	static bool,
+	writerInfo *protobuf.WriterInfo,
+	useRawFS bool,
+) (
+	*protobuf.Entrypoint,
+	*protobuf.WriterInfo,
+	error,
+) {
 	var retWi *protobuf.WriterInfo
 
 	ds, err := func() (datastore.DS, error) {
@@ -100,43 +144,29 @@ func compileFS(srcDir, dstDir string, static bool, writerInfo *protobuf.WriterIn
 		return datastore.InFileSystem(dstDir)
 	}()
 	if err != nil {
-		return nil, fmt.Errorf("could not open datastore: %w", err)
+		return nil, nil, fmt.Errorf("could not open datastore: %w", err)
 	}
 
 	be := blenc.FromDatastore(ds)
 
 	ep, err := structure.UploadStaticDirectory(context.Background(), os.DirFS(srcDir), be)
 	if err != nil {
-		return nil, fmt.Errorf("couldn't upload directory content: %w", err)
+		return nil, nil, fmt.Errorf("couldn't upload directory content: %w", err)
 	}
 
 	if !static {
 		if writerInfo == nil {
 			ep, retWi, err = structure.CreateLink(context.Background(), be, ep)
 			if err != nil {
-				return nil, fmt.Errorf("failed to update root link: %w", err)
+				return nil, nil, fmt.Errorf("failed to update root link: %w", err)
 			}
 		} else {
 			ep, err = structure.UpdateLink(context.Background(), be, writerInfo, ep)
 			if err != nil {
-				return nil, fmt.Errorf("failed to update root link: %w", err)
+				return nil, nil, fmt.Errorf("failed to update root link: %w", err)
 			}
 		}
 	}
 
-	epBytes, err := ep.ToBytes()
-	if err != nil {
-		return nil, fmt.Errorf("failed to serialize entrypoint data: %w", err)
-	}
-
-	err = os.WriteFile(
-		path.Join(dstDir, "entrypoint.txt"),
-		[]byte(base58.Encode(epBytes)),
-		0666,
-	)
-	if err != nil {
-		return nil, fmt.Errorf("failed to write entrypoint data: %w", err)
-	}
-
-	return retWi, nil
+	return ep, retWi, nil
 }

pkg/cmd/static_datastore/root.go

@@ -44,7 +44,6 @@ node is stored in a plaintext in a file called 'entrypoint.txt'.
 	}
 
 	cmd.AddCommand(compileCmd())
-	cmd.AddCommand(serverCmd())
 
 	return cmd
 }