[Bounty] Headless browser for fetching Proof Post #40

Makefile

@@ -3,12 +3,18 @@
 bin_dir=build/
 commit=$$(git rev-parse HEAD)
 time=$$(date +%s)
+aws_registry_uri=${aws-account-id}.dkr.ecr.${aws-lambda-region}.amazonaws.com
+aws_docker_image=${aws_registry_uri}/${docker-image-name}:${commit}
 
 # Things in my.mk:
 # aws-lambda-function-staging=my-lambda-function-staging
 # aws-lambda-function-production=my-lambda-function-production
+# aws-lambda-headless-function-staging=my-lambda-headless-function-staging
 # aws-lambda-region=ap-east-1
 # aws-lambda-role=arn:aws:iam::xxxxx:....
+# aws-account-id=xxxxxxxxxx
+# docker-image-name=lambda_headless
+
 -include ./my.mk
 
 build:
@@ -78,3 +84,37 @@ lambda-pack-worker-staging: lambda-build-worker-staging
 
 lambda-update-worker-staging: lambda-pack-worker-staging
 	@aws lambda update-function-code --function-name ${aws-lambda-function-worker-staging} --zip-file 'fileb://./build/lambda.zip'
+
+lamda-create-registry-headless:
+	@aws ecr get-login-password \
+		--region ${aws-lambda-region} | docker login \
+		--username AWS \
+		--password-stdin ${aws_registry_uri}
+	@aws ecr describe-repositories \
+		--repository-names ${docker-image-name} || \
+		aws ecr create-repository \
+		--repository-name ${docker-image-name} \
+		--region ${aws-lambda-region} \
+		--image-scanning-configuration scanOnPush=true \
+		--image-tag-mutability MUTABLE
+
+lambda-build-headless-staging:
+	@docker build -f ./cmd/lambda_headless/Dockerfile -t ${docker-image-name}:${commit} .
+	@docker tag ${docker-image-name}:${commit} ${aws_docker_image}
+
+lambda-pack-headless-staging: lamda-create-registry-headless lambda-build-headless-staging
+	@docker push ${aws_docker_image}
+
+lambda-create-headless-staging: lambda-pack-headless-staging
+	aws lambda create-function \
+		--package-type Image \
+		--region ${aws-lambda-region} \
+		--function-name ${aws-lambda-headless-function-staging} \
+		--code ImageUri=${aws_docker_image} \
+		--memory-size 1200 \
+		--timeout 30 \
+		--architectures x86_64 \
+		--role ${aws-lambda-role}
+
+lambda-update-headless-staging: lambda-pack-headless-staging
+	@aws lambda update-function-code --function-name ${aws-lambda-headless-function-staging} --image-uri ${aws_docker_image}

cmd/headless/main.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+
+	"github.com/nextdotid/proof_server/headless"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	flagPort       = flag.Int("port", 9801, "Listen port")
+)
+
+func main() {
+	flag.Parse()
+	logrus.SetLevel(logrus.DebugLevel)
+	headless.Init("")
+
+	fmt.Printf("Server now running on 0.0.0.0:%d", *flagPort)
+	headless.Engine.Run(fmt.Sprintf("0.0.0.0:%d", *flagPort))
+}

cmd/lambda_headless/Dockerfile

@@ -0,0 +1,61 @@
+# syntax=docker/dockerfile:1
+
+# https://docs.docker.com/language/golang/build-images/
+FROM golang:1.18-buster AS build
+
+WORKDIR /app
+
+COPY go.mod ./
+COPY go.sum ./
+RUN go mod download
+COPY . ./
+
+# Remember to build your handler executable for Linux!
+# https://github.com/aws/aws-lambda-go/blob/main/README.md#building-your-function
+RUN env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \
+    go build -o /main ./cmd/lambda_headless/main.go
+
+
+# Install chromium
+FROM public.ecr.aws/lambda/provided:al2 as chromium
+
+# install brotli, so we can decompress chromium
+# we don't have access to brotli out of the box, to install we first need epel
+# https://docs.fedoraproject.org/en-US/epel/#what_is_extra_packages_for_enterprise_linux_or_epel
+RUN yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \
+    yum -y install brotli && \
+    yum clean all
+
+# download chromium
+# s/o to https://github.com/alixaxel/chrome-aws-lambda for the binary
+RUN yum -y install wget && \
+    wget --progress=dot:giga https://raw.githubusercontent.com/alixaxel/chrome-aws-lambda/master/bin/chromium.br -O /chromium.br && \
+    yum clean all
+
+# decompress chromium
+RUN brotli -d /chromium.br
+
+# copy artifacts to a clean image
+FROM public.ecr.aws/lambda/provided:al2
+
+# install chromium dependencies
+RUN yum -y install \
+    libX11 \
+    nano \
+    unzip \
+    wget \
+    xclock \
+    xorg-x11-xauth \
+    xterm && \
+    yum clean all
+
+# copy in chromium from chromium stage
+COPY --from=chromium /chromium /opt/chromium
+
+# grant our program access to chromium
+RUN chmod 777 /opt/chromium
+
+# copy in lambda fn from build stage
+COPY --from=build /main /main
+
+ENTRYPOINT ["/main"]

cmd/lambda_headless/main.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"github.com/akrylysov/algnhsa"
+	"github.com/sirupsen/logrus"
+	"github.com/nextdotid/proof_server/headless"
+)
+
+func init() {
+	logrus.SetLevel(logrus.WarnLevel)
+	headless.Init("/opt/chromium")
+}
+
+func main() {
+	algnhsa.ListenAndServe(headless.Engine, nil)
+}
+

go.mod

@@ -49,6 +49,7 @@ require (
 	github.com/go-playground/locales v0.14.0 // indirect
 	github.com/go-playground/universal-translator v0.18.0 // indirect
 	github.com/go-playground/validator/v10 v10.10.1 // indirect
+	github.com/go-rod/rod v0.112.0 // indirect
 	github.com/go-sql-driver/mysql v1.6.0 // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -88,12 +89,16 @@ require (
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/ssoroka/slice v0.0.0-20220402005549-78f0cea3df8b // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 // indirect
 	github.com/tidwall/gjson v1.9.3 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/ugorji/go/codec v1.2.7 // indirect
+	github.com/ysmood/goob v0.4.0 // indirect
+	github.com/ysmood/gson v0.7.2 // indirect
+	github.com/ysmood/leakless v0.8.0 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect

go.sum

@@ -231,6 +231,8 @@ github.com/go-playground/validator/v10 v10.10.1 h1:uA0+amWMiglNZKZ9FJRKUAe9U3RX9
 github.com/go-playground/validator/v10 v10.10.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU=
 github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY=
 github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I=
+github.com/go-rod/rod v0.112.0 h1:U9Yc+quw4hxZ6GrdbWFBeylvaYElEKM9ijFW2LYkGlA=
+github.com/go-rod/rod v0.112.0/go.mod h1:GZDtmEs6RpF6kBRYpGCZXxXlKNneKVPiKOjaMbmVVjE=
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -595,6 +597,8 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q
 github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/spf13/viper v1.11.0 h1:7OX/1FS6n7jHD1zGrZTM7WtY13ZELRyosK4k93oPr44=
 github.com/spf13/viper v1.11.0/go.mod h1:djo0X/bA5+tYVoCn+C7cAYJGcVn/qYLFTG8gdUsX7Zk=
+github.com/ssoroka/slice v0.0.0-20220402005549-78f0cea3df8b h1:nDFJ1KYD1CSRP3nHtkvCH+ztuoz+QW++OvCLgpS6kQE=
+github.com/ssoroka/slice v0.0.0-20220402005549-78f0cea3df8b/go.mod h1:l4Ov7Zo7X3/MCC+pefg/lN7x8X8FKb1Ub7oxosKKJa0=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
@@ -630,6 +634,15 @@ github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
+github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
+github.com/ysmood/got v0.31.3/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
+github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
+github.com/ysmood/gson v0.7.1/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
+github.com/ysmood/gson v0.7.2 h1:1iWUvpi5DPvd2j59W7ifRPR9DiAZ3Ga+fmMl1mJrRbM=
+github.com/ysmood/gson v0.7.2/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
+github.com/ysmood/leakless v0.8.0 h1:BzLrVoiwxikpgEQR0Lk8NyBN5Cit2b1z+u0mgL4ZJak=
+github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

headless/client.go

@@ -0,0 +1,72 @@
+package headless
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+
+	"golang.org/x/xerrors"
+)
+
+// HeadlessClient handles communication for headless browser service
+type HeadlessClient struct {
+	url    string
+	client *http.Client
+}
+
+// NewHeadlessClient creates a new headless client
+func NewHeadlessClient(url string) *HeadlessClient {
+	return &HeadlessClient{url, http.DefaultClient}
+}
+
+// Find find whether the target matching payload exists
+func (h *HeadlessClient) Find(ctx context.Context, payload *FindRequest) (bool, error) {
+	u, err := url.Parse(h.url)
+	if err != nil {
+		return false, xerrors.Errorf("%w", err)
+	}
+
+	u.Path = path.Join(u.Path, "/v1/find")
+	body, err := json.Marshal(payload)
+	if err != nil {
+		return false, xerrors.Errorf("%w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), bytes.NewReader(body))
+	if err != nil {
+		return false, xerrors.Errorf("%w", err)
+	}
+
+	req.Header.Add("Content-Type", "application/json")
+
+	res, err := h.client.Do(req)
+	if res != nil && err != nil {
+		if _, err := io.Copy(io.Discard, res.Body); err != nil {
+			return false, xerrors.Errorf("%w", err)
+		}
+	}
+
+	if res != nil {
+		defer res.Body.Close()
+	}
+
+	if err != nil {
+		return false, xerrors.Errorf("%w", err)
+	}
+
+	contents, err := io.ReadAll(res.Body)
+	if err != nil {
+		return false, xerrors.Errorf("%w", err)
+	}
+
+	var resBody FindRespond
+	if err := json.Unmarshal(contents, &resBody); err != nil {
+		return false, xerrors.Errorf("%w", err)
+	}
+
+	return resBody.Found, nil
+}