diff --git a/.dockerignore b/.dockerignore index de2a4c61182..bdd388f30fa 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,42 +1,3 @@ -.DS_Store -/.git -/.github -/.vscode -.idea -/AdGuardHome -/AdGuardHome.exe -/AdGuardHome.yaml -/AdGuardHome.log -/data -/build -/build2 -/dist -/client/node_modules -/client2/node_modules -/.gitattributes -/.gitignore -/.goreleaser.yml -/changelog.config.js -/coverage.txt -/Dockerfile -/LICENSE.txt -/Makefile -/querylog.json -/querylog.json.1 -/*.md - -# Test output -dnsfilter/tests/top-1m.csv -dnsfilter/tests/dnsfilter.TestLotsOfRules*.pprof - -# Snapcraft build temporary files -*.snap -launchpad_credentials -snapcraft_login -snapcraft.yaml.bak - -# IntelliJ IDEA project files -*.iml - -# Packr -*-packr.go +# Ignore everything except for explicitly allowed stuff. +* +!dist/docker diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bf4e0683e43..0bf8807b949 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,14 +55,14 @@ 'restore-keys': '${{ runner.os }}-node-' - 'name': 'Run make ci' 'shell': 'bash' - 'run': 'make ci' + 'run': 'make VERBOSE=1 ci' - 'name': 'Upload coverage' 'uses': 'codecov/codecov-action@v1' 'if': "success() && matrix.os == 'ubuntu-latest'" 'with': 'token': '${{ secrets.CODECOV_TOKEN }}' 'file': './coverage.txt' - 'app': + 'build-release': 'runs-on': 'ubuntu-latest' 'needs': 'test' 'steps': @@ -95,30 +95,16 @@ 'restore-keys': '${{ runner.os }}-node-' - 'name': 'Set up Snapcraft' 'run': 'sudo apt-get -yq --no-install-suggests --no-install-recommends install snapcraft' - - 'name': 'Set up GoReleaser' - 'run': 'curl -sfL https://install.goreleaser.com/github.com/goreleaser/goreleaser.sh | BINDIR="$(go env GOPATH)/bin" sh' - - 'name': 'Run snapshot build' - 'run': 'make release' - - 'docker': - 'runs-on': 'ubuntu-latest' - 'needs': 'test' - 'steps': - - 'name': 'Checkout' - 'uses': 'actions/checkout@v2' - 'with': - 'fetch-depth': 0 - 'name': 'Set up QEMU' 'uses': 'docker/setup-qemu-action@v1' - 'name': 'Set up Docker Buildx' 'uses': 'docker/setup-buildx-action@v1' - - 'name': 'Docker Buildx (build)' - 'run': 'make docker-multi-arch' + - 'name': 'Run snapshot build' + 'run': 'make SIGN=0 VERBOSE=1 js-deps js-build build-release build-docker' 'notify': 'needs': - - 'app' - - 'docker' + - 'build-release' # Secrets are not passed to workflows that are triggered by a pull request # from a fork. # @@ -139,7 +125,7 @@ 'uses': '8398a7/action-slack@v3' 'with': 'status': '${{ env.WORKFLOW_CONCLUSION }}' - 'fields': 'repo, message, commit, author, job' + 'fields': 'repo, message, commit, author, workflow' 'env': 'GITHUB_TOKEN': '${{ secrets.GITHUB_TOKEN }}' 'SLACK_WEBHOOK_URL': '${{ secrets.SLACK_WEBHOOK_URL }}' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d003dee9735..0ed7c02c6b1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -13,15 +13,15 @@ - 'uses': 'actions/checkout@v2' - 'name': 'run-lint' 'run': > - make go-install-tools go-lint + make go-deps go-tools go-lint 'eslint': 'runs-on': 'ubuntu-latest' 'steps': - 'uses': 'actions/checkout@v2' - 'name': 'Install modules' - 'run': 'npm --prefix client ci' + 'run': 'npm --prefix="./client" ci' - 'name': 'Run ESLint' - 'run': 'npm --prefix client run lint' + 'run': 'npm --prefix="./client" run lint' 'notify': 'needs': - 'go-lint' @@ -46,7 +46,7 @@ 'uses': '8398a7/action-slack@v3' 'with': 'status': '${{ env.WORKFLOW_CONCLUSION }}' - 'fields': 'repo, message, commit, author, job' + 'fields': 'repo, message, commit, author, workflow' 'env': 'GITHUB_TOKEN': '${{ secrets.GITHUB_TOKEN }}' 'SLACK_WEBHOOK_URL': '${{ secrets.SLACK_WEBHOOK_URL }}' diff --git a/.goreleaser.yml b/.goreleaser.yml deleted file mode 100644 index c533712b980..00000000000 --- a/.goreleaser.yml +++ /dev/null @@ -1,115 +0,0 @@ -'project_name': 'AdGuardHome' - -'env': -- 'GO111MODULE=on' -- 'GOPROXY=https://goproxy.io' - -'before': - 'hooks': - - 'go mod download' - - 'go generate ./...' - -'builds': -- 'main': './main.go' - 'ldflags': - - '-s -w -X main.version={{.Version}} -X main.channel={{.Env.CHANNEL}} -X main.goarm={{.Env.GOARM}}' - 'env': - - 'CGO_ENABLED=0' - 'goos': - - 'darwin' - - 'linux' - - 'freebsd' - - 'windows' - 'goarch': - - '386' - - 'amd64' - - 'arm' - - 'arm64' - - 'mips' - - 'mipsle' - - 'mips64' - - 'mips64le' - 'goarm': - - '5' - - '6' - - '7' - 'gomips': - - 'softfloat' - 'ignore': - - 'goos': 'freebsd' - 'goarch': 'mips' - - 'goos': 'freebsd' - 'goarch': 'mipsle' - -'archives': -- # Archive name template. - # Defaults: - # - if format is `tar.gz`, `tar.xz`, `gz` or `zip`: - # - `{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}{{ if .Mips }}_{{ .Mips }}{{ end }}` - # - if format is `binary`: - # - `{{ .Binary }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}{{ if .Mips }}_{{ .Mips }}{{ end }}` - 'name_template': '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}{{ if .Mips }}_{{ .Mips }}{{ end }}' - 'wrap_in_directory': 'AdGuardHome' - 'format_overrides': - - 'goos': 'windows' - 'format': 'zip' - - 'goos': 'darwin' - 'format': 'zip' - 'files': - - 'LICENSE.txt' - - 'README.md' - -'snapcrafts': -- 'name': 'adguard-home' - 'base': 'core20' - 'name_template': '{{ .ProjectName }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' - 'summary': 'Network-wide ads & trackers blocking DNS server' - 'description': | - AdGuard Home is a network-wide software for blocking ads & tracking. After - you set it up, it'll cover ALL your home devices, and you don't need any - client-side software for that. - - It operates as a DNS server that re-routes tracking domains to a "black hole," - thus preventing your devices from connecting to those servers. It's based - on software we use for our public AdGuard DNS servers -- both share a lot - of common code. - 'grade': 'stable' - 'confinement': 'strict' - 'publish': false - 'license': 'GPL-3.0' - 'extra_files': - - 'source': 'scripts/snap/local/adguard-home-web.sh' - 'destination': 'adguard-home-web.sh' - 'mode': 0755 - - 'source': 'scripts/snap/gui/adguard-home-web.desktop' - 'destination': 'meta/gui/adguard-home-web.desktop' - 'mode': 0644 - - 'source': 'scripts/snap/gui/adguard-home-web.png' - 'destination': 'meta/gui/adguard-home-web.png' - 'mode': 0644 - 'apps': - 'adguard-home': - 'command': 'AdGuardHome -w $SNAP_DATA --no-check-update' - 'plugs': - # Add the "netrwork-bind" plug to bind to interfaces. - - 'network-bind' - # Add the "netrwork-observe" plug to be able to bind to ports below 1024 - # (cap_net_bind_service) and also to bind to a particular interface using - # SO_BINDTODEVICE (cap_net_raw). - - 'network-observe' - 'daemon': 'simple' - 'adguard-home-web': - 'command': 'adguard-home-web.sh' - 'plugs': - - 'desktop' - -'checksum': - 'name_template': 'checksums.txt' - -'snapshot': - # TODO(a.garipov): A temporary solution to trim the prerelease versions. - # A real solution would consist of making a better versioning scheme that also - # doesn't break SemVer or Snapcraft. - # - # See https://github.com/AdguardTeam/AdGuardHome/issues/2412. - 'name_template': '{{ slice .Tag 0 8 }}-SNAPSHOT-{{ .ShortCommit }}' diff --git a/CHANGELOG.md b/CHANGELOG.md index 46c421ba554..e2fa248b297 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to ## [Unreleased] ### Added @@ -44,10 +44,11 @@ and this project adheres to improve error response ([#2358]). - Improved HTTP requests handling and timeouts ([#2343]). - Our snap package now uses the `core20` image as its base ([#2306]). -- Various internal improvements ([#2267], [#2271], [#2297]). +- New build system and various internal improvements ([#2276], [#2271], + [#2297]). [#2231]: https://github.com/AdguardTeam/AdGuardHome/issues/2231 -[#2267]: https://github.com/AdguardTeam/AdGuardHome/issues/2267 +[#2276]: https://github.com/AdguardTeam/AdGuardHome/issues/2276 [#2271]: https://github.com/AdguardTeam/AdGuardHome/issues/2271 [#2297]: https://github.com/AdguardTeam/AdGuardHome/issues/2297 [#2306]: https://github.com/AdguardTeam/AdGuardHome/issues/2306 @@ -81,6 +82,8 @@ and this project adheres to ### Removed +- `Dockerfile` ([#2276]). Replaced with the script + `scripts/make/build-docker.sh` which uses `scripts/make/Dockerfile`. - Support for pre-v0.99.3 format of query logs ([#2102]). ## [v0.104.3] - 2020-11-19 diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index a218f2d4131..00000000000 --- a/Dockerfile +++ /dev/null @@ -1,79 +0,0 @@ -FROM --platform=${BUILDPLATFORM:-linux/amd64} tonistiigi/xx:golang AS xgo -FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.14-alpine as builder - -ARG BUILD_DATE -ARG VCS_REF -ARG VERSION=dev -ARG CHANNEL=release - -ENV CGO_ENABLED 0 -ENV GO111MODULE on -ENV GOPROXY https://goproxy.io - -COPY --from=xgo / / -RUN go env - -RUN apk --update --no-cache add \ - build-base \ - gcc \ - git \ - npm \ - yarn \ - && rm -rf /tmp/* /var/cache/apk/* - -WORKDIR /app - -COPY . ./ - -# Prepare the client code -RUN npm --prefix client ci && npm --prefix client run build-prod -RUN yarn --cwd client2 build - -# Download go dependencies -RUN go mod download -RUN go generate ./... - -# It's important to place TARGET* arguments here to avoid running npm and go mod download for every platform -ARG TARGETPLATFORM -ARG TARGETOS -ARG TARGETARCH -RUN go build -ldflags="-s -w -X main.version=${VERSION} -X main.channel=${CHANNEL} -X main.goarm=${GOARM}" - -FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine:latest - -ARG BUILD_DATE -ARG VCS_REF -ARG VERSION -ARG CHANNEL - -LABEL maintainer="AdGuard Team " \ - org.opencontainers.image.created=$BUILD_DATE \ - org.opencontainers.image.url="https://adguard.com/adguard-home.html" \ - org.opencontainers.image.source="https://github.com/AdguardTeam/AdGuardHome" \ - org.opencontainers.image.version=$VERSION \ - org.opencontainers.image.revision=$VCS_REF \ - org.opencontainers.image.vendor="AdGuard" \ - org.opencontainers.image.title="AdGuard Home" \ - org.opencontainers.image.description="Network-wide ads & trackers blocking DNS server" \ - org.opencontainers.image.licenses="GPL-3.0" - -RUN apk --update --no-cache add \ - ca-certificates \ - libcap \ - libressl \ - && rm -rf /tmp/* /var/cache/apk/* - -COPY --from=builder --chown=nobody:nogroup /app/AdGuardHome /opt/adguardhome/AdGuardHome -COPY --from=builder --chown=nobody:nogroup /usr/local/go/lib/time/zoneinfo.zip /usr/local/go/lib/time/zoneinfo.zip - -RUN /opt/adguardhome/AdGuardHome --version \ - && mkdir -p /opt/adguardhome/conf /opt/adguardhome/work \ - && chown -R nobody: /opt/adguardhome \ - && setcap 'cap_net_bind_service=+eip' /opt/adguardhome/AdGuardHome - -EXPOSE 53/tcp 53/udp 67/udp 68/udp 80/tcp 443/tcp 853/tcp 3000/tcp -WORKDIR /opt/adguardhome/work -VOLUME ["/opt/adguardhome/conf", "/opt/adguardhome/work"] - -ENTRYPOINT ["/opt/adguardhome/AdGuardHome"] -CMD ["-h", "0.0.0.0", "-c", "/opt/adguardhome/conf/AdGuardHome.yaml", "-w", "/opt/adguardhome/work", "--no-check-update"] diff --git a/HACKING.md b/HACKING.md index 237e554f1df..750e5f47c20 100644 --- a/HACKING.md +++ b/HACKING.md @@ -1,4 +1,4 @@ - # *AdGuardHome* Developer Guidelines + # AdGuard Home Developer Guidelines As of **December 2020**, this document is partially a work-in-progress, but should still be followed. Some of the rules aren't enforced as thoroughly or diff --git a/Makefile b/Makefile index 68c7f9dcfe8..d9ce026dea9 100644 --- a/Makefile +++ b/Makefile @@ -1,355 +1,94 @@ +# Keep the Makefile POSIX-compliant. We currently allow hyphens in +# target names, but that may change in the future. # -# Available targets +# See https://pubs.opengroup.org/onlinepubs/9699919799/utilities/make.html. +.POSIX: + +CHANNEL = development +CLIENT_BETA_DIR = client2 +CLIENT_DIR = client +COMMIT = $$(git rev-parse --short HEAD) +DIST_DIR = dist +GO = go +# TODO(a.garipov): Add more default proxies using pipes after update to +# Go 1.15. # -# * build -- builds AdGuardHome for the current platform -# * client -- builds client-side code of AdGuard Home -# * client-watch -- builds client-side code of AdGuard Home and watches for changes there -# * docker -- builds a docker image for the current platform -# * clean -- clean everything created by previous builds -# * lint -- run all linters -# * test -- run all unit-tests -# * dependencies -- installs dependencies (go and npm modules) -# * ci -- installs dependencies, runs linters and tests, intended to be used by CI/CD -# -# Building releases: -# -# * release -- builds AdGuard Home distros. CHANNEL must be specified (edge, release or beta). -# * release_and_sign -- builds AdGuard Home distros and signs the binary files. -# CHANNEL must be specified (edge, release or beta). -# * sign -- Repacks all release archive files and signs the binary files inside them. -# For signing to work, the public+private key pair for $(GPG_KEY) must be imported: -# gpg --import public.txt -# gpg --import private.txt -# GPG_KEY_PASSPHRASE must contain the GPG key passphrase -# * docker-multi-arch -- builds a multi-arch image. If you want it to be pushed to docker hub, -# you must specify: -# * DOCKER_IMAGE_NAME - adguard/adguard-home -# * DOCKER_OUTPUT - type=image,name=adguard/adguard-home,push=true - -GO := go -GOPATH := $(shell $(GO) env GOPATH) -PWD := $(shell pwd) -TARGET=AdGuardHome -BASE_URL="https://static.adguard.com/adguardhome/$(CHANNEL)" -GPG_KEY := devteam@adguard.com -GPG_KEY_PASSPHRASE := -GPG_CMD := gpg --detach-sig --default-key $(GPG_KEY) --pinentry-mode loopback --passphrase $(GPG_KEY_PASSPHRASE) -VERBOSE := -v -REBUILD_CLIENT = 1 - -# See release target -DIST_DIR=dist - -# Update channel. Can be release, beta or edge. Uses edge by default. -CHANNEL ?= edge - -# Validate channel -ifneq ($(CHANNEL),release) -ifneq ($(CHANNEL),beta) -ifneq ($(CHANNEL),edge) -$(error CHANNEL value is not valid. Valid values are release,beta or edge) -endif -endif -endif - -# Version history URL (see -VERSION_HISTORY_URL="https://github.com/AdguardTeam/AdGuardHome/releases" -ifeq ($(CHANNEL),edge) - VERSION_HISTORY_URL="https://github.com/AdguardTeam/AdGuardHome/commits/master" -endif - -# goreleaser command depends on the $CHANNEL -GORELEASER_COMMAND=goreleaser release --rm-dist --skip-publish --snapshot --parallelism 1 -ifneq ($(CHANNEL),edge) - # If this is not an "edge" build, use normal release command - GORELEASER_COMMAND=goreleaser release --rm-dist --skip-publish --parallelism 1 -endif - -# Version properties -COMMIT=$(shell git rev-parse --short HEAD) -TAG_NAME=$(shell git describe --abbrev=0) -PRERELEASE_VERSION=$(shell git describe --abbrev=0) -# TODO(a.garipov): The cut call is a temporary solution to trim -# prerelease versions. See the comment in .goreleaser.yml. -RELEASE_VERSION=$(shell git describe --abbrev=0 | cut -c 1-8) -SNAPSHOT_VERSION=$(RELEASE_VERSION)-SNAPSHOT-$(COMMIT) - -# Set proper version -VERSION= -ifeq ($(TAG_NAME),$(shell git describe --abbrev=4)) - ifeq ($(CHANNEL),edge) - VERSION=$(SNAPSHOT_VERSION) - else ifeq ($(CHANNEL),beta) - VERSION=$(PRERELEASE_VERSION) - else - VERSION=$(RELEASE_VERSION) - endif -else - VERSION=$(SNAPSHOT_VERSION) -endif - -# Docker target parameters -DOCKER_IMAGE_NAME ?= adguardhome-dev -DOCKER_IMAGE_FULL_NAME = $(DOCKER_IMAGE_NAME):$(VERSION) -DOCKER_PLATFORMS=linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/386,linux/ppc64le -DOCKER_OUTPUT ?= type=image,name=$(DOCKER_IMAGE_NAME),push=false -BUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') - -# Docker tags (can be redefined) -DOCKER_TAGS ?= -ifndef DOCKER_TAGS - ifeq ($(CHANNEL),release) - DOCKER_TAGS := --tag $(DOCKER_IMAGE_NAME):latest - endif - ifeq ($(CHANNEL),beta) - DOCKER_TAGS := --tag $(DOCKER_IMAGE_NAME):beta - endif - ifeq ($(CHANNEL),edge) - # Don't set the version tag when pushing to "edge" - DOCKER_IMAGE_FULL_NAME := $(DOCKER_IMAGE_NAME):edge - # DOCKER_TAGS := --tag $(DOCKER_IMAGE_NAME):edge - endif -endif - -# Validate docker build arguments -ifndef DOCKER_IMAGE_NAME -$(error DOCKER_IMAGE_NAME value is not set) -endif - -# OS-specific flags -TEST_FLAGS := --race $(VERBOSE) -ifeq ($(OS),Windows_NT) - TEST_FLAGS := -endif - -.PHONY: all build client client-watch docker lint lint-js lint-go test dependencies clean release docker-multi-arch -all: build - -init: - git config core.hooksPath .githooks - -build: - test '$(REBUILD_CLIENT)' = '1' && $(MAKE) client_with_deps || exit 0 - $(GO) mod download - PATH=$(GOPATH)/bin:$(PATH) $(GO) generate ./... - CGO_ENABLED=0 $(GO) build -ldflags="-s -w -X main.version=$(VERSION) -X main.channel=$(CHANNEL) -X main.goarm=$(GOARM)" - PATH=$(GOPATH)/bin:$(PATH) packr clean - -client: - npm --prefix client run build-prod - yarn --cwd client2 build - -client_with_deps: - npm --prefix client ci - npm --prefix client run build-prod - yarn --cwd client2 build - -client-watch: - npm --prefix client run watch - yarn --cwd client2 start - -docker: - DOCKER_CLI_EXPERIMENTAL=enabled \ - docker buildx build \ - --build-arg VERSION=$(VERSION) \ - --build-arg CHANNEL=$(CHANNEL) \ - --build-arg VCS_REF=$(COMMIT) \ - --build-arg BUILD_DATE=$(BUILD_DATE) \ - $(DOCKER_TAGS) \ - --load \ - -t "$(DOCKER_IMAGE_NAME)" -f ./Dockerfile . - - @echo Now you can run the docker image: - @echo docker run --name "adguard-home" -p 53:53/tcp -p 53:53/udp -p 80:80/tcp -p 443:443/tcp -p 853:853/tcp -p 3000:3000/tcp $(DOCKER_IMAGE_NAME) - +# GOPROXY = https://goproxy.io|https://goproxy.cn|direct +GOPROXY = https://goproxy.cn,https://goproxy.io,direct +GPG_KEY_PASSPHRASE = not-a-real-password +NPM = npm +NPM_FLAGS = --prefix $(CLIENT_DIR) +SIGN = 1 +VERBOSE = 0 +VERSION = v0.0.0 +YARN = yarn +YARN_FLAGS = --cwd $(CLIENT_BETA_DIR) + +ENV = env\ + COMMIT='$(COMMIT)'\ + CHANNEL='$(CHANNEL)'\ + GPG_KEY_PASSPHRASE='$(GPG_KEY_PASSPHRASE)'\ + DIST_DIR='$(DIST_DIR)'\ + GO='$(GO)'\ + GOPROXY='$(GOPROXY)'\ + PATH="$${PWD}/bin:$$($(GO) env GOPATH)/bin:$${PATH}"\ + SIGN='$(SIGN)'\ + VERBOSE='$(VERBOSE)'\ + VERSION='$(VERSION)'\ + +# Keep the line above blank. + +# Keep this target first, so that a naked make invocation triggers +# a full build. +build: deps quick-build + +quick-build: js-build go-build + +ci: deps test + +deps: js-deps go-deps lint: js-lint go-lint - -js-lint: dependencies - npm --prefix client run lint - yarn --cwd client2 lint - -go-install-tools: - env GO=$(GO) sh ./scripts/go-install-tools.sh - -go-lint: - env GO=$(GO) PATH="$$PWD/bin:$$PATH" sh ./scripts/go-lint.sh - test: js-test go-test +# Here and below, keep $(SHELL) in quotes, because on Windows this will +# expand to something like "C:/Program Files/Git/usr/bin/sh.exe". +build-docker: ; $(ENV) "$(SHELL)" ./scripts/make/build-docker.sh + +build-release: deps js-build + $(ENV) "$(SHELL)" ./scripts/make/build-release.sh + +clean: ; $(ENV) "$(SHELL)" ./scripts/make/clean.sh +init: ; git config core.hooksPath ./scripts/hooks + +js-build: + $(NPM) $(NPM_FLAGS) run build-prod + $(YARN) $(YARN_FLAGS) build +js-deps: + $(NPM) $(NPM_FLAGS) ci + $(YARN) $(YARN_FLAGS) install +js-lint: + $(NPM) $(NPM_FLAGS) run lint + $(YARN) $(YARN_FLAGS) lint js-test: - npm run test --prefix client - -go-test: - $(GO) test $(TEST_FLAGS) --coverprofile coverage.txt ./... + $(NPM) $(NPM_FLAGS) run test -ci: client_with_deps - $(GO) mod download - $(MAKE) test +go-build: ; $(ENV) "$(SHELL)" ./scripts/make/go-build.sh +go-deps: ; $(ENV) "$(SHELL)" ./scripts/make/go-deps.sh +go-lint: ; $(ENV) "$(SHELL)" ./scripts/make/go-lint.sh +go-test: ; $(ENV) "$(SHELL)" ./scripts/make/go-test.sh +go-tools: ; $(ENV) "$(SHELL)" ./scripts/make/go-tools.sh +# TODO(a.garipov): Remove the legacy targets once the build +# infrastructure stops using them. dependencies: - npm --prefix client ci - yarn --cwd client2 install - $(GO) mod download - -clean: - rm -f ./AdGuardHome ./AdGuardHome.exe ./coverage.txt - rm -f -r ./build/ ./client/node_modules/ ./data/ ./$(DIST_DIR)/ -# Set the GOPATH explicitly in case make clean is called from under sudo -# after a Docker build. - env PATH="$(GOPATH)/bin:$$PATH" packr clean - rm -f -r ./bin/ - + @ echo "use make deps instead" + @ $(MAKE) deps docker-multi-arch: - DOCKER_CLI_EXPERIMENTAL=enabled \ - docker buildx build \ - --platform $(DOCKER_PLATFORMS) \ - --build-arg VERSION=$(VERSION) \ - --build-arg CHANNEL=$(CHANNEL) \ - --build-arg VCS_REF=$(COMMIT) \ - --build-arg BUILD_DATE=$(BUILD_DATE) \ - $(DOCKER_TAGS) \ - --output "$(DOCKER_OUTPUT)" \ - -t "$(DOCKER_IMAGE_FULL_NAME)" -f ./Dockerfile . - - @echo If the image was pushed to the registry, you can now run it: - @echo docker run --name "adguard-home" -p 53:53/tcp -p 53:53/udp -p 80:80/tcp -p 443:443/tcp -p 853:853/tcp -p 3000:3000/tcp $(DOCKER_IMAGE_NAME) - -release: client_with_deps - $(GO) mod download - @echo Starting release build: version $(VERSION), channel $(CHANNEL) - CHANNEL=$(CHANNEL) $(GORELEASER_COMMAND) - $(call write_version_file,$(VERSION)) - PATH=$(GOPATH)/bin:$(PATH) packr clean - -release_and_sign: client_with_deps - $(MAKE) release - $(call repack_dist) - -sign: - $(call repack_dist) - -define write_version_file - $(eval version := $(1)) - - @echo Writing version file: $(version) - - # Variables for CI - rm -f $(DIST_DIR)/version.txt - echo "version=$(version)" > $(DIST_DIR)/version.txt - - # Prepare the version.json file - rm -f $(DIST_DIR)/version.json - echo "{" >> $(DIST_DIR)/version.json - echo " \"version\": \"$(version)\"," >> $(DIST_DIR)/version.json - echo " \"announcement\": \"AdGuard Home $(version) is now available!\"," >> $(DIST_DIR)/version.json - echo " \"announcement_url\": \"$(VERSION_HISTORY_URL)\"," >> $(DIST_DIR)/version.json - echo " \"selfupdate_min_version\": \"0.0\"," >> $(DIST_DIR)/version.json - - # Windows builds - echo " \"download_windows_amd64\": \"$(BASE_URL)/AdGuardHome_windows_amd64.zip\"," >> $(DIST_DIR)/version.json - echo " \"download_windows_386\": \"$(BASE_URL)/AdGuardHome_windows_386.zip\"," >> $(DIST_DIR)/version.json - - # MacOS builds - echo " \"download_darwin_amd64\": \"$(BASE_URL)/AdGuardHome_darwin_amd64.zip\"," >> $(DIST_DIR)/version.json - echo " \"download_darwin_386\": \"$(BASE_URL)/AdGuardHome_darwin_386.zip\"," >> $(DIST_DIR)/version.json - - # Linux - echo " \"download_linux_amd64\": \"$(BASE_URL)/AdGuardHome_linux_amd64.tar.gz\"," >> $(DIST_DIR)/version.json - echo " \"download_linux_386\": \"$(BASE_URL)/AdGuardHome_linux_386.tar.gz\"," >> $(DIST_DIR)/version.json - - # Linux, all kinds of ARM - echo " \"download_linux_arm\": \"$(BASE_URL)/AdGuardHome_linux_armv6.tar.gz\"," >> $(DIST_DIR)/version.json - echo " \"download_linux_armv5\": \"$(BASE_URL)/AdGuardHome_linux_armv5.tar.gz\"," >> $(DIST_DIR)/version.json - echo " \"download_linux_armv6\": \"$(BASE_URL)/AdGuardHome_linux_armv6.tar.gz\"," >> $(DIST_DIR)/version.json - echo " \"download_linux_armv7\": \"$(BASE_URL)/AdGuardHome_linux_armv7.tar.gz\"," >> $(DIST_DIR)/version.json - echo " \"download_linux_arm64\": \"$(BASE_URL)/AdGuardHome_linux_arm64.tar.gz\"," >> $(DIST_DIR)/version.json - - # Linux, MIPS - echo " \"download_linux_mips\": \"$(BASE_URL)/AdGuardHome_linux_mips_softfloat.tar.gz\"," >> $(DIST_DIR)/version.json - echo " \"download_linux_mipsle\": \"$(BASE_URL)/AdGuardHome_linux_mipsle_softfloat.tar.gz\"," >> $(DIST_DIR)/version.json - echo " \"download_linux_mips64\": \"$(BASE_URL)/AdGuardHome_linux_mips64_softfloat.tar.gz\"," >> $(DIST_DIR)/version.json - echo " \"download_linux_mips64le\": \"$(BASE_URL)/AdGuardHome_linux_mips64le_softfloat.tar.gz\"," >> $(DIST_DIR)/version.json - - # FreeBSD - echo " \"download_freebsd_386\": \"$(BASE_URL)/AdGuardHome_freebsd_386.tar.gz\"," >> $(DIST_DIR)/version.json - echo " \"download_freebsd_amd64\": \"$(BASE_URL)/AdGuardHome_freebsd_amd64.tar.gz\"," >> $(DIST_DIR)/version.json - - # FreeBSD, all kinds of ARM - echo " \"download_freebsd_arm\": \"$(BASE_URL)/AdGuardHome_freebsd_armv6.tar.gz\"," >> $(DIST_DIR)/version.json - echo " \"download_freebsd_armv5\": \"$(BASE_URL)/AdGuardHome_freebsd_armv5.tar.gz\"," >> $(DIST_DIR)/version.json - echo " \"download_freebsd_armv6\": \"$(BASE_URL)/AdGuardHome_freebsd_armv6.tar.gz\"," >> $(DIST_DIR)/version.json - echo " \"download_freebsd_armv7\": \"$(BASE_URL)/AdGuardHome_freebsd_armv7.tar.gz\"," >> $(DIST_DIR)/version.json - echo " \"download_freebsd_arm64\": \"$(BASE_URL)/AdGuardHome_freebsd_arm64.tar.gz\"" >> $(DIST_DIR)/version.json - - # Finish - echo "}" >> $(DIST_DIR)/version.json -endef - -define repack_dist - # Repack archive files - # A temporary solution for our auto-update code to be able to unpack these archive files - # The problem is that goreleaser doesn't add directory AdGuardHome/ to the archive file - # and we can't create it - rm -rf $(DIST_DIR)/AdGuardHome - - # Windows builds - $(call zip_repack_windows,AdGuardHome_windows_amd64.zip) - $(call zip_repack_windows,AdGuardHome_windows_386.zip) - - # MacOS builds - $(call zip_repack,AdGuardHome_darwin_amd64.zip) - $(call zip_repack,AdGuardHome_darwin_386.zip) - - # Linux - $(call tar_repack,AdGuardHome_linux_amd64.tar.gz) - $(call tar_repack,AdGuardHome_linux_386.tar.gz) - - # Linux, all kinds of ARM - $(call tar_repack,AdGuardHome_linux_armv5.tar.gz) - $(call tar_repack,AdGuardHome_linux_armv6.tar.gz) - $(call tar_repack,AdGuardHome_linux_armv7.tar.gz) - $(call tar_repack,AdGuardHome_linux_arm64.tar.gz) - - # Linux, MIPS - $(call tar_repack,AdGuardHome_linux_mips_softfloat.tar.gz) - $(call tar_repack,AdGuardHome_linux_mipsle_softfloat.tar.gz) - $(call tar_repack,AdGuardHome_linux_mips64_softfloat.tar.gz) - $(call tar_repack,AdGuardHome_linux_mips64le_softfloat.tar.gz) - - # FreeBSD - $(call tar_repack,AdGuardHome_freebsd_386.tar.gz) - $(call tar_repack,AdGuardHome_freebsd_amd64.tar.gz) - - # FreeBSD, all kinds of ARM - $(call tar_repack,AdGuardHome_freebsd_armv5.tar.gz) - $(call tar_repack,AdGuardHome_freebsd_armv6.tar.gz) - $(call tar_repack,AdGuardHome_freebsd_armv7.tar.gz) - $(call tar_repack,AdGuardHome_freebsd_arm64.tar.gz) -endef - -define zip_repack_windows - $(eval ARC := $(1)) - cd $(DIST_DIR) && \ - unzip $(ARC) && \ - $(GPG_CMD) AdGuardHome/AdGuardHome.exe && \ - zip -r $(ARC) AdGuardHome/ && \ - rm -rf AdGuardHome -endef - -define zip_repack - $(eval ARC := $(1)) - cd $(DIST_DIR) && \ - unzip $(ARC) && \ - $(GPG_CMD) AdGuardHome/AdGuardHome && \ - zip -r $(ARC) AdGuardHome/ && \ - rm -rf AdGuardHome -endef - -define tar_repack - $(eval ARC := $(1)) - cd $(DIST_DIR) && \ - tar xzf $(ARC) && \ - $(GPG_CMD) AdGuardHome/AdGuardHome && \ - tar czf $(ARC) AdGuardHome/ && \ - rm -rf AdGuardHome -endef + @ echo "use make build-docker instead" + @ $(MAKE) build-docker +go-install-tools: + @ echo "use make go-tools instead" + @ $(MAKE) go-tools +release: + @ echo "use make build-release instead" + @ $(MAKE) build-release diff --git a/README.md b/README.md index 69ba6c20701..f09da329813 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ It operates as a DNS server that re-routes tracking domains to a "black hole," t * [Reporting issues](#reporting-issues) * [Help with translations](#translate) * [Other](#help-other) -* [Projects that use AdGuardHome](#uses) +* [Projects that use AdGuard Home](#uses) * [Acknowledgments](#acknowledgments) * [Privacy](#privacy) @@ -189,26 +189,28 @@ In order to do this, specify `GOOS` and `GOARCH` env variables before running ma For example: ``` -GOOS=linux GOARCH=arm64 make +env GOOS='linux' GOARCH='arm64' make +``` +Or: +``` +make GOOS='linux' GOARCH='arm64' ``` #### Preparing release You'll need this to prepare a release build: -* [goreleaser](https://goreleaser.com/) * [snapcraft](https://snapcraft.io/) Commands: -* `make release` - builds a snapshot build (CHANNEL=edge) -* `CHANNEL=beta make release` - builds beta version, tag is mandatory. -* `CHANNEL=release make release` - builds release version, tag is mandatory. +``` +make build-release CHANNEL='...' VERSION='...' +``` #### Docker image -* Run `make docker` to build the Docker image locally. -* Run `make docker-multi-arch` to build the multi-arch Docker image (the one that we publish to Docker Hub). +* Run `make build-docker` to build the Docker image locally (the one that we publish to DockerHub). Please note, that we're using [Docker Buildx](https://docs.docker.com/buildx/working-with-buildx/) to build our official image. @@ -322,7 +324,7 @@ This software wouldn't have been possible without: * And many more node.js packages. * [whotracks.me data](https://github.com/cliqz-oss/whotracks.me) -You might have seen that [CoreDNS](https://coredns.io) was mentioned here before — we've stopped using it in AdGuardHome. While we still use it on our servers for [AdGuard DNS](https://adguard.com/adguard-dns/overview.html) service, it seemed like an overkill for Home as it impeded with Home features that we plan to implement. +You might have seen that [CoreDNS](https://coredns.io) was mentioned here before — we've stopped using it in AdGuard Home. While we still use it on our servers for [AdGuard DNS](https://adguard.com/adguard-dns/overview.html) service, it seemed like an overkill for Home as it impeded with Home features that we plan to implement. For a full list of all node.js packages in use, please take a look at [client/package.json](https://github.com/AdguardTeam/AdGuardHome/blob/master/client/package.json) file. diff --git a/main.go b/main.go index ecbb3d3bc9b..be08b192451 100644 --- a/main.go +++ b/main.go @@ -1,4 +1,3 @@ -//go:generate go install -v github.com/gobuffalo/packr/packr //go:generate packr clean //go:generate packr -z package main @@ -17,8 +16,6 @@ var channel = "release" var goarm = "" // gomips is the GOMIPS value. It is set by the linker. -// -// TODO(a.garipov): Implement. var gomips = "" func main() { diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000000..1971f5cde08 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,159 @@ + # AdGuard Home Scripts + +## `hooks/`: Git Hooks + + ### Usage + +Run `make init` from the project root. + +## `querylog/`: Query Log Helpers + + ### Usage + + * `npm install`: install dependencies. Run this first. + * `npm run anonymize `: read the query log from the `` + and write anonymized version to ``. + +## `make/`: Makefile Scripts + +The release channels are: `development` (the default), `edge`, `beta`, and +`release`. If verbosity levels aren't documented here, there are only two: `0`, +don't print anything, and `1`, be verbose. + + ### `build-docker.sh`: Build A Multi-Architecture Docker Image + +Required environment: + + * `CHANNEL`: release channel, see above. + * `COMMIT`: current Git revision. + * `DIST_DIR`: the directory where a release has previously been built. + * `VERSION`: release version. + +Optional environment: + + * `DOCKER_IMAGE_NAME`: the name of the resulting Docker container. By default + it's `adguardhome-dev`. + * `DOCKER_OUTPUT`: the `--output` parameters. By default they are + `type=image,name=${DOCKER_IMAGE_NAME},push=false`. + * `SUDO`: allow users to use `sudo` or `doas` with `docker`. By default none + is used. + + ### `build-release.sh`: Build A Release For All Platforms + +Required environment: + * `CHANNEL`: release channel, see above. + * `GPG_KEY` and `GPG_KEY_PASSPHRASE`: data for `gpg`. Only required if `SIGN` + is `1`. + +Optional environment: + * `DIST_DIR`: the directory to build a release into. The default value is + `dist`. + * `GO`: set an alternarive name for the Go compiler. + * `SIGN`: `0` to not sign the resulting packages, `1` to sign. The default + value is `1`. + * `VERBOSE`: `1` to be verbose, `2` to also print environment. This script + calls `go-build.sh` with the verbosity level one level lower, so to get + verbosity level `2` in `go-build.sh`, set this to `3` when calling + `build-release.sh`. + * `VERSION`: release version. Will be set by `version.sh` if it is unset or + it has the default `Makefile` value of `v0.0.0`. + + ### `clean.sh`: Cleanup + +Optional environment: + * `GO`: set an alternarive name for the Go compiler. + +Required environment: + * `DIST_DIR`: the directory where a release has previously been built. + + ### `go-build.sh`: Build The Backend + +Optional environment: + * `GOARM`: ARM processor options for the Go compiler. + * `GOMIPS`: ARM processor options for the Go compiler. + * `GO`: set an alternarive name for the Go compiler. + * `OUT`: output binary name. + * `PARALLELISM`: set the maximum number of concurrently run build commands + (that is, compiler, linker, etc.). + * `VERBOSE`: verbosity level. `1` shows every command that is run and every + Go package that is processed. `2` also shows subcommands and environment. + The default value is `0`, don't be verbose. + +Required environment: + * `CHANNEL`: release channel, see above. + * `VERSION`: release version. + + ### `go-deps.sh`: Install Backend Dependencies + +Optional environment: + * `GO`: set an alternarive name for the Go compiler. + * `VERBOSE`: verbosity level. `1` shows every command that is run and every + Go package that is processed. `2` also shows subcommands and environment. + The default value is `0`, don't be verbose. + + ### `go-lint.sh`: Run Backend Static Analyzers + +Don't forget to run `make go-tools` once first! + +Optional environment: + * `EXIT_ON_ERROR`: if set to `0`, don't exit the script after the first + encountered error. The default value is `1`. + * `GO`: set an alternarive name for the Go compiler. + * `VERBOSE`: verbosity level. `1` shows every command that is run. `2` also + shows subcommands. The default value is `0`, don't be verbose. + + ### `go-test.sh`: Run Backend Tests + +Optional environment: + * `GO`: set an alternarive name for the Go compiler. + * `RACE`: set to `0` to not use the Go race detector. The default value is + `1`, use the race detector. + * `VERBOSE`: verbosity level. `1` shows every command that is run and every + Go package that is processed. `2` also shows subcommands. The default + value is `0`, don't be verbose. + + ### `go-tools.sh`: Install Backend Tooling + +Installs the Go static analysis and other tools into `${PWD}/bin`. Either add +`${PWD}/bin` to your `$PATH` before all other entries, or use the commands +directly, or use the commands through `make` (for example, `make go-lint`). + +Optional environment: + * `GO`: set an alternarive name for the Go compiler. + + ### `version.sh`: Print The Current Version + +Required environment: + * `CHANNEL`: release channel, see above. + +## `snap/`: Snap GUI Files + +App icons (see https://github.com/AdguardTeam/AdGuardHome/pull/1836), Snap +manifest file templates, and helper scripts. + +## `translations/`: Twosky Integration Script + + ### Usage + + * `npm install`: install dependencies. Run this first. + * `npm run locales:download`: download and save all translations. + * `npm run locales:upload`: upload the base `en` locale. + * `npm run locales:summary`: show the current locales summary. + * `npm run locales:unused`: show the list of unused strings. + +After the download you'll find the output locales in the `client/src/__locales/` +directory. + +## `whotracksme/`: Whotracks.me Database Converter + +A simple script that converts the Ghostery/Cliqz trackers database to a json format. + + ### Usage + +```sh +yarn install +node index.js +``` + +You'll find the output in the `whotracksmedb.json` file. Then, move it to +`client/src/helpers/trackers`. diff --git a/.githooks/pre-commit b/scripts/hooks/pre-commit similarity index 100% rename from .githooks/pre-commit rename to scripts/hooks/pre-commit diff --git a/scripts/install.sh b/scripts/install.sh index 6f1fa85fc11..1114c641f4b 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1,6 +1,6 @@ #!/bin/sh -# AdGuardHome installation script +# AdGuard Home Installation Script # # 1. Download the package # 2. Unpack it @@ -227,4 +227,4 @@ main() { log_info " sudo ${AGH_DIR}/AdGuardHome -s start|stop|restart|status|install|uninstall" } -main "$@" \ No newline at end of file +main "$@" diff --git a/scripts/make/Dockerfile b/scripts/make/Dockerfile new file mode 100644 index 00000000000..8494e7a5ccd --- /dev/null +++ b/scripts/make/Dockerfile @@ -0,0 +1,49 @@ +# A docker file for scripts/make/build-docker.sh. + +FROM alpine:3.12 + +ARG BUILD_DATE +ARG VERSION +ARG VCS_REF +LABEL maintainer="AdGuard Team " \ + org.opencontainers.image.created=$BUILD_DATE \ + org.opencontainers.image.url="https://adguard.com/adguard-home.html" \ + org.opencontainers.image.source="https://github.com/AdguardTeam/AdGuardHome" \ + org.opencontainers.image.version=$VERSION \ + org.opencontainers.image.revision=$VCS_REF \ + org.opencontainers.image.vendor="AdGuard" \ + org.opencontainers.image.title="AdGuard Home" \ + org.opencontainers.image.description="Network-wide ads & trackers blocking DNS server" \ + org.opencontainers.image.licenses="GPL-3.0" + +# Update certificates. +RUN apk --no-cache --update add ca-certificates libcap && \ + rm -rf /var/cache/apk/* && \ + mkdir -p /opt/adguardhome/conf /opt/adguardhome/work && \ + chown -R nobody: /opt/adguardhome + +ARG DIST_DIR +ARG TARGETARCH +ARG TARGETOS +ARG TARGETVARIANT + +COPY --chown=nobody:nogroup\ + ./${DIST_DIR}/docker/AdGuardHome_${TARGETOS}_${TARGETARCH}_${TARGETVARIANT}\ + /opt/adguardhome/AdGuardHome + +RUN setcap 'cap_net_bind_service=+eip' /opt/adguardhome/AdGuardHome + +EXPOSE 53/tcp 53/udp 67/udp 68/udp 80/tcp 443/tcp 853/tcp 3000/tcp + +VOLUME ["/opt/adguardhome/conf", "/opt/adguardhome/work"] + +WORKDIR /opt/adguardhome/work + +ENTRYPOINT ["/opt/adguardhome/AdGuardHome"] + +CMD [ \ + "--no-check-update", \ + "-c", "/opt/adguardhome/conf/AdGuardHome.yaml", \ + "-h", "0.0.0.0", \ + "-w", "/opt/adguardhome/work", \ +] diff --git a/scripts/make/build-docker.sh b/scripts/make/build-docker.sh new file mode 100644 index 00000000000..d8eb2737848 --- /dev/null +++ b/scripts/make/build-docker.sh @@ -0,0 +1,101 @@ +#!/bin/sh + +verbose="${VERBOSE:-0}" + +if [ "$verbose" -gt '0' ] +then + set -x + debug_flags='-D' +else + set +x + debug_flags='' +fi + +set -e -f -u + +# Require these to be set. The channel value is validated later. +readonly channel="$CHANNEL" +readonly commit="$COMMIT" +readonly dist_dir="$DIST_DIR" +readonly version="$VERSION" + +# Allow users to use sudo. +readonly sudo_cmd="${SUDO:-}" + +readonly docker_platforms="\ +linux/386,\ +linux/amd64,\ +linux/arm/v6,\ +linux/arm/v7,\ +linux/arm64,\ +linux/ppc64le" + +readonly build_date="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + +# Set DOCKER_IMAGE_NAME to 'adguard/adguard-home' if you want (and are +# allowed) to push to DockerHub. +readonly docker_image_name="${DOCKER_IMAGE_NAME:-adguardhome-dev}" + +# Set DOCKER_OUTPUT to 'type=image,name=adguard/adguard-home,push=true' +# if you want (and are allowed) to push to DockerHub. +readonly docker_output="${DOCKER_OUTPUT:-type=image,name=${docker_image_name},push=false}" + +case "$channel" +in +('release') + readonly docker_image_full_name="${docker_image_name}:${version}" + readonly docker_tags="--tag ${docker_image_name}:latest" + ;; +('beta') + readonly docker_image_full_name="${docker_image_name}:${version}" + readonly docker_tags="--tag ${docker_image_name}:beta" + ;; +('edge') + # Don't set the version tag when pushing to the edge channel. + readonly docker_image_full_name="${docker_image_name}:edge" + readonly docker_tags='' + ;; +('development') + readonly docker_image_full_name="${docker_image_name}" + readonly docker_tags='' + ;; +(*) + echo "invalid channel '$channel', supported values are\ + 'development', 'edge', 'beta', and 'release'" 1>&2 + exit 1 + ;; +esac + +# Copy the binaries into a new directory under new names, so that it's +# eaiser to COPY them later. DO NOT remove the trailing underscores. +# See scripts/make/Dockerfile. +readonly dist_docker="${dist_dir}/docker" +mkdir -p "$dist_docker" +cp "${dist_dir}/AdGuardHome_linux_386/AdGuardHome/AdGuardHome"\ + "${dist_docker}/AdGuardHome_linux_386_" +cp "${dist_dir}/AdGuardHome_linux_amd64/AdGuardHome/AdGuardHome"\ + "${dist_docker}/AdGuardHome_linux_amd64_" +cp "${dist_dir}/AdGuardHome_linux_arm64/AdGuardHome/AdGuardHome"\ + "${dist_docker}/AdGuardHome_linux_arm64_" +cp "${dist_dir}/AdGuardHome_linux_arm_6/AdGuardHome/AdGuardHome"\ + "${dist_docker}/AdGuardHome_linux_arm_v6" +cp "${dist_dir}/AdGuardHome_linux_arm_7/AdGuardHome/AdGuardHome"\ + "${dist_docker}/AdGuardHome_linux_arm_v7" +cp "${dist_dir}/AdGuardHome_linux_ppc64le/AdGuardHome/AdGuardHome"\ + "${dist_docker}/AdGuardHome_linux_ppc64le_" + +# Don't use quotes with $docker_tags and $debug_flags because we want +# word splitting and or an empty space if tags are empty. +$sudo_cmd docker\ + $debug_flags\ + buildx build\ + --build-arg BUILD_DATE="$build_date"\ + --build-arg DIST_DIR="$dist_dir"\ + --build-arg VCS_REF="$commit"\ + --build-arg VERSION="$version"\ + --output "$docker_output"\ + --platform "$docker_platforms"\ + $docker_tags\ + -t "$docker_image_full_name"\ + -f ./scripts/make/Dockerfile\ + . diff --git a/scripts/make/build-release.sh b/scripts/make/build-release.sh new file mode 100644 index 00000000000..e031d049be4 --- /dev/null +++ b/scripts/make/build-release.sh @@ -0,0 +1,325 @@ +#!/bin/sh + +# AdGuard Home Release Script +# +# The commentary in this file is written with the assumption that the +# reader only has superficial knowledge of the POSIX shell language and +# alike. Experienced readers may find it overly verbose. + +# The default verbosity level is 0. Show every command that is run if +# the caller requested verbosity level greater than 0. Show the +# environment if the callre requested verbosity level greater than 1. +# Otherwise, print nothing. +# +# The level of verbosity for the build script is the same minus one +# level. See below in build(). +readonly verbose="${VERBOSE:-0}" +if [ "$verbose" -gt '1' ] +then + env + set -x +elif [ "$verbose" -gt '0' ] +then + set -x +fi + +# By default, sign the packages, but allow users to skip that step. +readonly sign="${SIGN:-1}" + +# Exit the script if a pipeline fails (-e), prevent accidental filename +# expansion (-f), and consider undefined variables as errors (-u). +set -e -f -u + +# Function log is an echo wrapper that writes to stderr if the caller +# requested verbosity level greater than 0. Otherwise, it does nothing. +log() { + if [ "$verbose" -gt '0' ] + then + # Don't use quotes to get word splitting. + echo $@ 1>&2 + fi +} + +log 'starting to build AdGuard Home release' + +# Require the channel to be set. Additional validation is performed +# later by go-build.sh. +readonly channel="$CHANNEL" + +# Check VERSION against the default value from the Makefile. If it is +# that, use the version calculation script. +if [ "${VERSION:-}" = 'v0.0.0' -o "${VERSION:-}" = '' ] +then + readonly version="$(sh ./scripts/make/version.sh)" +else + readonly version="$VERSION" +fi + +log "channel '$channel'" +log "version '$version'" + +# Require the gpg key and passphrase to be set if the signing is +# required. +if [ "$sign" = '1' ] +then + readonly gpg_key_passphrase="$GPG_KEY_PASSPHRASE" + readonly gpg_key="$GPG_KEY" +fi + +# The default distribution files directory is dist. +readonly dist="${DIST_DIR:-dist}" + +# Give users the ability to override the go command from environment. +# For example, to build two releases with two different Go versions and +# test the difference. +readonly go="${GO:-go}" + +log "checking tools" + +# Make sure we fail gracefully if one of the tools we need is missing. +for tool in gpg gzip sed sha256sum snapcraft tar zip +do + which "$tool" >/dev/null ||\ + { log "pieces don't fit, '$tool' not found"; exit 1; } +done + +# Data section. Arrange data into space-separated tables for read -r to +# read. Use 0 for missing values. + +readonly arms='5 +6 +7' + +readonly mipses='softfloat' + +# os arch arm mips snap +readonly platforms="\ +darwin 386 0 0 0 +darwin amd64 0 0 0 +freebsd 386 0 0 0 +freebsd amd64 0 0 0 +freebsd arm 5 0 0 +freebsd arm 6 0 0 +freebsd arm 7 0 0 +linux 386 0 0 i386 +linux amd64 0 0 amd64 +linux arm 5 0 0 +linux arm 6 0 0 +linux arm 7 0 armhf +linux arm64 0 0 arm64 +linux mips 0 softfloat 0 +linux mips64 0 softfloat 0 +linux mips64le 0 softfloat 0 +linux mipsle 0 softfloat 0 +linux ppc64le 0 0 0 +windows 386 0 0 0 +windows amd64 0 0 0" + +# Function build builds the release for one platform. It builds +# a binary, an archive and, if needed, a snap package. +build() { + # Get the arguments. Here and below, use the "build_" prefix + # for all variables local to function build. + build_dir="${dist}/${1}/AdGuardHome"\ + build_ar="$2"\ + build_os="$3"\ + build_arch="$4"\ + build_arm="$5"\ + build_mips="$6"\ + build_snap="$7"\ + ; + + # Use the ".exe" filename extension if we build a Windows + # release. + if [ "$build_os" = 'windows' ] + then + build_output="./${build_dir}/AdGuardHome.exe" + else + build_output="./${build_dir}/AdGuardHome" + fi + + mkdir -p "./${build_dir}" + + # Build the binary. + # + # Set GOARM and GOMIPS to an empty string if $build_arm and + # $build_mips are zero by removing the zero as if it's a prefix. + # + # Don't use quotes with $build_par because we want an empty + # space if parallelism wasn't set. + env\ + GOARCH="$build_arch"\ + GOARM="${build_arm#0}"\ + GOMIPS="${build_mips#0}"\ + GOOS="$os"\ + VERBOSE="$(( verbose - 1 ))"\ + OUT="$build_output"\ + sh ./scripts/make/go-build.sh\ + ; + + log "$build_output" + + if [ "$sign" = '1' ] + then + gpg\ + --default-key "$gpg_key"\ + --detach-sig\ + --passphrase "$gpg_key_passphrase"\ + --pinentry-mode loopback\ + -q\ + "$build_output"\ + ; + fi + + # Prepare the build directory for archiving. + cp ./CHANGELOG.md ./LICENSE.txt ./README.md "$build_dir" + + # Make archives. Windows and macOS prefer ZIP archives; the + # rest, gzipped tarballs. + case "$build_os" + in + ('darwin'|'windows') + build_archive="./${dist}/${build_ar}.zip" + zip -9 -q "$build_archive" "$build_dir" + ;; + (*) + build_archive="./${dist}/${build_ar}.tar.gz" + tar -C "./${dist}/${1}" -c -f - "./AdGuardHome"\ + | gzip -9 - >"$build_archive" + ;; + esac + + log "$build_archive" + + if [ "$build_snap" = '0' ] + then + return + fi + + # Prepare snap build. + build_snap_output="./${dist}/AdGuardHome_${build_snap}.snap" + build_snap_dir="${build_snap_output}.dir" + + # Create the meta subdirectory and copy files there. + mkdir -p "${build_snap_dir}/meta" + cp "$build_output"\ + './scripts/snap/local/adguard-home-web.sh'\ + "$build_snap_dir" + cp -r './scripts/snap/gui'\ + "${build_snap_dir}/meta/" + + # Create a snap.yaml file, setting the values. + sed -e 's/%VERSION%/'"$version"'/'\ + -e 's/%ARCH%/'"$build_snap"'/'\ + ./scripts/snap/snap.tmpl.yaml\ + >"${build_snap_dir}/meta/snap.yaml" + + # TODO(a.garipov): The snapcraft tool will *always* write + # everything, including errors, to stdout. And there doesn't + # seem to be a way to change that. So, save the combined + # output, but only show it when snapcraft actually fails. + set +e + build_snapcraft_output="$( + snapcraft pack "$build_snap_dir"\ + --output "$build_snap_output" 2>&1 + )" + build_snapcraft_exit_code="$?" + set -e + if [ "$build_snapcraft_exit_code" != '0' ] + then + log "$build_snapcraft_output" + exit "$build_snapcraft_exit_code" + fi + + log "$build_snap_output" +} + +log "starting builds" + +# Go over all platforms defined in the space-separated table above, +# tweak the values where necessary, and feed to build. +echo "$platforms" | while read -r os arch arm mips snap +do + case "$arch" + in + (arm) + dir="AdGuardHome_${os}_${arch}_${arm}" + ar="AdGuardHome_${os}_${arch}v${arm}" + ;; + (mips*) + dir="AdGuardHome_${os}_${arch}_${mips}" + ar="$dir" + ;; + (*) + dir="AdGuardHome_${os}_${arch}" + ar="$dir" + ;; + esac + + build "$dir" "$ar" "$os" "$arch" "$arm" "$mips" "$snap" +done + +log "calculating checksums" + +# Calculate the checksums of the files in a subshell with file expansion +# enabled (+f) so that we don't need to use find or basename. +( + set +f + + cd "./${dist}" + + # Don't use quotes to get word splitting. + sha256sum $(ls *.tar.gz *.zip) > ./checksums.txt +) + +log "writing versions" + +echo "$version" > "./${dist}/version.txt" + +# Create the verison.json file. + +readonly version_download_url="https://static.adguard.com/adguardhome/${channel}" +readonly version_json="./${dist}/version.json" + +# Point users to the master branch if the channel is edge. +if [ "$channel" = 'edge' ] +then + readonly version_history_url='https://github.com/AdguardTeam/AdGuardHome/commits/master' +else + readonly version_history_url='https://github.com/AdguardTeam/AdGuardHome/releases' +fi + +rm -f "$version_json" +echo "{ + \"version\": \"${version}\", + \"announcement\": \"AdGuard Home ${version} is now available!\", + \"announcement_url\": \"${version_history_url}\", + \"selfupdate_min_version\": \"0.0\", +" >> "$version_json" + +( + # Use +f here so that ls works and we don't need to use find. + set +f + + # Don't use quotes to get word splitting. + for f in $(ls "./${dist}/"*.tar.gz "./${dist}/"*.zip) + do + platform="$f" + + # Remove the prefix. + platform="${platform#./${dist}/AdGuardHome_}" + + # Remove the filename extensions. + platform="${platform%.zip}" + platform="${platform%.tar.gz}" + + # Use the filename's base path. + filename="${f#./${dist}/}" + + echo " \"download_${platform}\": \"${version_download_url}/${filename}\"," >> "$version_json" + done +) + +echo '}' >> "$version_json" + +log "finished" diff --git a/scripts/make/clean.sh b/scripts/make/clean.sh new file mode 100644 index 00000000000..a366e79c4d8 --- /dev/null +++ b/scripts/make/clean.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +verbose="${VERBOSE:-0}" + +if [ "$verbose" -gt '0' ] +then + set -x +fi + +set -e -f -u + +dist_dir="$DIST_DIR" +go="${GO:-go}" + +# Set the GOPATH explicitly in case make clean is called from under sudo +# after a Docker build. +env PATH="$("$go" env GOPATH)/bin":"$PATH" packr clean + +rm -f\ + ./AdGuardHome\ + ./AdGuardHome.exe\ + ./coverage.txt\ + ; + +rm -f -r\ + ./bin/\ + ./build/\ + ./build2/\ + ./client/node_modules/\ + ./client2/node_modules/\ + ./data/\ + "./${dist_dir}/"\ + ; diff --git a/scripts/make/go-build.sh b/scripts/make/go-build.sh new file mode 100644 index 00000000000..a2a4a93a7a7 --- /dev/null +++ b/scripts/make/go-build.sh @@ -0,0 +1,98 @@ +#!/bin/sh + +# AdGuard Home Build Script +# +# The commentary in this file is written with the assumption that the +# reader only has superficial knowledge of the POSIX shell language and +# alike. Experienced readers may find it overly verbose. + +# The default verbosity level is 0. Show every command that is run and +# every package that is processed if the caller requested verbosity +# level greater than 0. Also show subcommands if the requested +# verbosity level is greater than 1. Otherwise, do nothing. +verbose="${VERBOSE:-0}" +if [ "$verbose" -gt '1' ] +then + env + set -x + readonly v_flags='-v' + readonly x_flags='-x' +elif [ "$verbose" -gt '0' ] +then + set -x + readonly v_flags='-v' + readonly x_flags='' +else + set +x + readonly v_flags='' + readonly x_flags='' +fi + +# Exit the script if a pipeline fails (-e), prevent accidental filename +# expansion (-f), and consider undefined variables as errors (-u). +set -e -f -u + +# Allow users to set the Go version. +go="${GO:-go}" + +# Require the channel to be set and validate the value. +channel="$CHANNEL" +case "$channel" +in +('development'|'edge'|'beta'|'release') + # All is well, go on. + ;; +(*) + echo "invalid channel '$channel', supported values are\ + 'development', 'edge', 'beta', and 'release'" 1>&2 + exit 1 + ;; +esac + +# Require the version to be set. +# +# TODO(a.garipov): Additional validation? +version="$VERSION" + +# Set the linker flags accordingly: set the channel and the versio as +# well as goarm and gomips variable values, if the variables are set and +# are not empty. +ldflags="-s -w -X main.version=${version}" +ldflags="${ldflags} -X main.channel=${channel}" +if [ "${GOARM:-}" != '' ] +then + ldflags="${ldflags} -X main.goarm=${GOARM}" +elif [ "${GOMIPS:-}" != '' ] +then + ldflags="${ldflags} -X main.gomips=${GOMIPS}" +fi + +# Allow users to limit the build's parallelism. +readonly parallelism="${PARALLELISM:-}" +if [ "$parallelism" != '' ] +then + readonly par_flags="-p ${parallelism}" +else + readonly par_flags='' +fi + +# Allow users to specify a different output name. +readonly out="${OUT:-}" +if [ "$out" != '' ] +then + readonly out_flags="-o ${out}" +else + readonly out_flags='' +fi + +# Don't use cgo. Use modules. +export CGO_ENABLED='0' GO111MODULE='on' + +readonly build_flags="${BUILD_FLAGS:-$out_flags $par_flags\ + $v_flags $x_flags}" + +# Don't use quotes with flag variables to get word splitting. +"$go" generate $v_flags $x_flags ./... + +# Don't use quotes with flag variables to get word splitting. +"$go" build --ldflags "$ldflags" $build_flags diff --git a/scripts/make/go-deps.sh b/scripts/make/go-deps.sh new file mode 100644 index 00000000000..f9bfb365c7d --- /dev/null +++ b/scripts/make/go-deps.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +verbose="${VERBOSE:-0}" + +if [ "$verbose" -gt '1' ] +then + env + set -x + readonly v_flags='-v' + readonly x_flags='-x' +elif [ "$verbose" -gt '0' ] +then + set -x + readonly v_flags='-v' + readonly x_flags='' +else + set +x + readonly v_flags='' + readonly x_flags='' +fi + +set -e -f -u + +go="${GO:-go}" + +# Don't use quotes with flag variables because we want an empty space if +# those aren't set. +"$go" mod download $x_flags + +env GOBIN="${PWD}/bin" "$go" install $v_flags $x_flags\ + github.com/gobuffalo/packr/packr diff --git a/scripts/go-lint.sh b/scripts/make/go-lint.sh similarity index 88% rename from scripts/go-lint.sh rename to scripts/make/go-lint.sh index ca30e1e78e3..22a576166c4 100644 --- a/scripts/go-lint.sh +++ b/scripts/make/go-lint.sh @@ -1,13 +1,23 @@ #!/bin/sh +verbose="${VERBOSE:-0}" + # Verbosity levels: # 0 = Don't print anything except for errors. # 1 = Print commands, but not nested commands. # 2 = Print everything. -test "${VERBOSE:=0}" -gt '0' && set -x - -# Set $EXITONERROR to zero to see all errors. -test "${EXITONERROR:=1}" = '0' && set +e || set -e +if [ "$verbose" -gt '0' ] +then + set -x +fi + +# Set $EXIT_ON_ERROR to zero to see all errors. +if [ "${EXIT_ON_ERROR:-1}" = '0' ] +then + set +e +else + set -e +fi # We don't need glob expansions and we want to see errors about unset # variables. @@ -17,7 +27,7 @@ not_found_msg=' looks like a binary not found error. make sure you have installed the linter binaries using: - $ make go-install-tools + $ make go-tools ' not_found() { @@ -95,7 +105,8 @@ ineffassign . unparam ./... -git ls-files -- '*.go' '*.md' '*.yaml' '*.yml' | xargs misspell --error +git ls-files -- '*.go' '*.md' '*.yaml' '*.yml' 'Makefile'\ + | xargs misspell --error looppointer ./... diff --git a/scripts/make/go-test.sh b/scripts/make/go-test.sh new file mode 100644 index 00000000000..098b56d8da8 --- /dev/null +++ b/scripts/make/go-test.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +verbose="${VERBOSE:-0}" + +# Verbosity levels: +# 0 = Don't print anything except for errors. +# 1 = Print commands, but not nested commands. +# 2 = Print everything. +if [ "$verbose" -gt '1' ] +then + set -x + v_flags='-v' + x_flags='-x' +elif [ "$verbose" -gt '0' ] +then + set -x + v_flags='-v' + x_flags='' +else + set +x + v_flags='' + x_flags='' +fi + +set -e -f -u + +race="${RACE:-1}" +if [ "$race" = '0' ] +then + race_flags='' +else + race_flags='--race' +fi + +go="${GO:-go}" +cover_flags='--coverprofile ./coverage.txt' +count_flags='--count 1' + +# Don't use quotes with flag variables because we want an empty space if +# those aren't set. +"$go" test $race_flags $count_flags $cover_flags $x_flags $v_flags ./... diff --git a/scripts/go-install-tools.sh b/scripts/make/go-tools.sh similarity index 89% rename from scripts/go-install-tools.sh rename to scripts/make/go-tools.sh index e75a54e4256..6c96e1cb8ce 100644 --- a/scripts/go-install-tools.sh +++ b/scripts/make/go-tools.sh @@ -1,6 +1,12 @@ #!/bin/sh -test "$VERBOSE" = '1' && set -x +verbose="${VERBOSE:-0}" + +if [ "$verbose" -gt '0' ] +then + set -x +fi + set -e -f -u # TODO(a.garipov): Add goconst? diff --git a/scripts/make/version.sh b/scripts/make/version.sh new file mode 100644 index 00000000000..08f9b48232c --- /dev/null +++ b/scripts/make/version.sh @@ -0,0 +1,57 @@ +#!/bin/sh + +readonly verbose="${VERBOSE:-0}" +if [ "$verbose" -gt '1' ] +then + set -x +fi + +set -e -f -u + +readonly awk_program='/^v[0-9]+\.[0-9]+\.[0-9]+.*$/ { + if (!$4) { + # The last tag is a full release version, so bump the + # minor one to get the next one. + $2++; + } + + print($1 "." $2 "." $3); + + next; +} + +{ + printf("invalid version: \"%s\"\n", $0); + + exit 1; +}' + +readonly last_tag="$(git describe --abbrev=0)" +readonly current_desc="$(git describe)" + +readonly channel="$CHANNEL" +case "$channel" +in +('development') + echo 'v0.0.0' + ;; +('edge') + next=$(echo $last_tag | awk -F '[.+-]' "$awk_program") + echo "${next}-SNAPSHOT-$(git rev-parse --short HEAD)" + ;; +('beta'|'release') + if [ "$current_desc" != "$last_tag" ] + then + echo 'need a tag' 1>&2 + + exit 1 + fi + + echo "$last_tag" + ;; +(*) + echo "invalid channel '$channel', supported values are\ + 'development', 'edge', 'beta', and 'release'" 1>&2 + exit 1 + ;; +esac diff --git a/scripts/querylog/README.md b/scripts/querylog/README.md deleted file mode 100644 index 219d95815b1..00000000000 --- a/scripts/querylog/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Helper tools to work with the Query log - -### Usage - -- `npm install` - Install the dependencies -- `npm run anonymize ` - Reads querylog from the `` and writes anonymized version to `` - -### Examples - -- `npm run anonymize test/querylog.json test/anonquerylog.json` - anonymizes the `test/querylog.json`. \ No newline at end of file diff --git a/scripts/snap/README.md b/scripts/snap/README.md deleted file mode 100644 index 6997dfee740..00000000000 --- a/scripts/snap/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Snap GUI - -These files are added to the AdGuard Home snap in order to add an app icon: -https://github.com/AdguardTeam/AdGuardHome/pull/1836 - -See .goreleaser.yml: snapcrafts.extra_files \ No newline at end of file diff --git a/scripts/snap/snap.tmpl.yaml b/scripts/snap/snap.tmpl.yaml new file mode 100644 index 00000000000..1dff31f6ef9 --- /dev/null +++ b/scripts/snap/snap.tmpl.yaml @@ -0,0 +1,37 @@ +# The %VARIABLES% are be replaced by actual values by the build script. + +'name': 'adguard-home' +'base': 'core20' +'version': '%VERSION%' +'summary': Network-wide ads & trackers blocking DNS server +'description': | + AdGuard Home is a network-wide software for blocking ads & tracking. After + you set it up, it'll cover ALL your home devices, and you don't need any + client-side software for that. + + It operates as a DNS server that re-routes tracking domains to a "black hole," + thus preventing your devices from connecting to those servers. It's based + on software we use for our public AdGuard DNS servers -- both share a lot + of common code. +'grade': 'stable' +'confinement': 'strict' + +'architectures': +- '%ARCH%' + +'apps': + 'adguard-home': + 'command': 'AdGuardHome --no-check-update -w $SNAP_DATA' + 'plugs': + # Add the "netrwork-bind" plug to bind to interfaces. + - 'network-bind' + # Add the "netrwork-observe" plug to be able to bind to ports below 1024 + # (cap_net_bind_service) and also to bind to a particular interface using + # SO_BINDTODEVICE (cap_net_raw). + - 'network-observe' + 'daemon': 'simple' + 'restart-condition': 'always' + 'adguard-home-web': + 'command': 'adguard-home-web.sh' + 'plugs': + - 'desktop' diff --git a/scripts/translations/README.md b/scripts/translations/README.md deleted file mode 100644 index 3a5e336cda5..00000000000 --- a/scripts/translations/README.md +++ /dev/null @@ -1,11 +0,0 @@ -## Twosky integration script - -### Usage - -- `npm install` Install the dependencies in the local node_modules folder -- `npm run locales:download` - Download and save all translations -- `npm run locales:upload` - Upload base `en` locale -- `npm run locales:summary` - Shows the current locales summary -- `npm run locales:unused` - Shows a list of unused strings - -After download you'll find the output locales in the `client/src/__locales/` folder. diff --git a/scripts/whotracksme/README.md b/scripts/whotracksme/README.md deleted file mode 100644 index cd5618ad1b6..00000000000 --- a/scripts/whotracksme/README.md +++ /dev/null @@ -1,13 +0,0 @@ -## Whotracks.me database converter - -A simple script that converts the Ghostery/Cliqz trackers database to a json format. - -### Usage - -``` -yarn install -node index.js -``` - -You'll find the output in the `whotracksmedb.json` file. -Move it to `client/src/helpers/trackers`. \ No newline at end of file