Skip to content

Commit

Permalink
Merge branch 'ayufan:master' into feat/fallback-for-o_tmpfile
Browse files Browse the repository at this point in the history
  • Loading branch information
jclab-joseph committed Nov 25, 2023
2 parents a350300 + 7da8692 commit 08b9c21
Show file tree
Hide file tree
Showing 64 changed files with 2,857 additions and 41 deletions.
2 changes: 2 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
tmp/
release/
**/Dockerfile
**/Dockerfile.client
1 change: 1 addition & 0 deletions .github/FUNDING.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ko_fi: ayufan
54 changes: 39 additions & 15 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
BUILD_ARCHS = amd64 arm64v8
CLIENT_BUILD_ARCHS = amd64 arm64v8 arm32v7
CLIENT_BUILD_ARCHS = amd64 arm64v8
REGISTRY ?= ayufan/proxmox-backup-server
VERSION ?= $(shell ls versions | grep -E -v '.(tmp|debug)' | sort -V | tail -n 1)

Expand All @@ -14,20 +14,20 @@ endif
arm32v7-%: DOCKER_ARCH=arm32v7
arm64v8-%: DOCKER_ARCH=arm64v8
amd64-%: DOCKER_ARCH=amd64
dev-%: DOCKER_ARCH=amd64
dev-%:

# Docker Images

%-docker-build:
docker build \
--tag $(REGISTRY):$(TAG)-$* \
--build-arg ARCH=$(DOCKER_ARCH)/ \
--build-arg ARCH=$(addsuffix /,$(DOCKER_ARCH)) \
--build-arg TAG=$(TAG) \
--build-arg VERSION=$(VERSION) \
-f Dockerfile \
-f versions/$(VERSION)/Dockerfile \
.

docker-build: $(addsuffix -build, $(BUILD_ARCHS))
docker-build: $(addsuffix -docker-build, $(BUILD_ARCHS))

# Docker Hub Images

Expand Down Expand Up @@ -66,7 +66,7 @@ dockerhub-latest-release: $(addsuffix -dockerhub-latest-release, $(BUILD_ARCHS))
--build-arg DOCKER_ARCH=$(DOCKER_ARCH) \
--build-arg TAG=$(TAG) \
--build-arg VERSION=$(VERSION) \
-f Dockerfile.client \
-f versions/$(VERSION)/Dockerfile.client \
.

mkdir -p release/$(TAG)
Expand All @@ -80,7 +80,7 @@ client: $(addsuffix -client, $(CLIENT_BUILD_ARCHS))
mkdir -p release/$(TAG)
-docker rm -f proxmox-backup-$(TAG)-$*
docker create --name=proxmox-backup-$(TAG)-$* $(REGISTRY):$(TAG)-$*
docker cp proxmox-backup-$(TAG)-$*:/src/. release/$(TAG)/$*
docker cp proxmox-backup-$(TAG)-$*:/deb/. release/$(TAG)/$*
-docker rm -f proxmox-backup-$(TAG)-$*

deb: $(addsuffix -deb, $(BUILD_ARCHS))
Expand All @@ -90,16 +90,38 @@ deb: $(addsuffix -deb, $(BUILD_ARCHS))
tmp-env:
mkdir -p "tmp/$(VERSION)"
cd "tmp/$(VERSION)" && ../../versions/$(VERSION)/clone.bash
cd "tmp/$(VERSION)" && ../../scripts/apply-patches.bash ../../versions/$(VERSION)/server/*.patch ../../versions/$(VERSION)/client*/*.patch
cd "tmp/$(VERSION)" && ../../scripts/apply-patches.bash ../../versions/$(VERSION)/server/*.patch
cd "tmp/$(VERSION)" && ../../scripts/strip-cargo.bash
cd "tmp/$(VERSION)" && ../../scripts/resolve-dependencies.bash

tmp-env-client:
mkdir -p "tmp/$(VERSION)-client"
cd "tmp/$(VERSION)-client" && ../../versions/$(VERSION)/clone.bash
cd "tmp/$(VERSION)-client" && ../../scripts/apply-patches.bash ../../versions/$(VERSION)/server/*.patch ./../versions/$(VERSION)/client*/*.patch
cd "tmp/$(VERSION)-client" && ../../scripts/strip-cargo.bash
cd "tmp/$(VERSION)-client" && ../../scripts/resolve-dependencies.bash

tmp-docker-shell:
docker build \
--tag tmp-docker-shell \
--build-arg ARCH=$(addsuffix /,$(DOCKER_ARCH)) \
--build-arg TAG=$(TAG) \
--build-arg VERSION=$(VERSION) \
--target toolchain \
-f versions/$(VERSION)/Dockerfile \
.
docker run --name=tmp-docker-shell --net=host --rm -it \
-v "$(CURDIR):$(CURDIR)" \
-w "$(CURDIR)/tmp/$(VERSION)" \
tmp-docker-shell

dev-run: dev-docker-build
-docker rm -f proxmox-backup
docker run --name=proxmox-backup --net=host --rm $(REGISTRY):$(TAG)-dev
docker run --name=proxmox-backup --net=host --tmpfs /run --rm $(REGISTRY):$(TAG)-dev

dev-shell: dev-docker-build
-docker rm -f proxmox-backup
docker run --name=proxmox-backup -it --rm $(REGISTRY):$(TAG)-dev /bin/bash
docker run --name=proxmox-backup -it --tmpfs /run --rm $(REGISTRY):$(TAG)-dev /bin/bash

# Version management

Expand Down Expand Up @@ -128,13 +150,15 @@ github-upload-all:
github-release upload -t $(TAG) -R -n $$(basename $$file) -f $$file; \
done

github-pre-release: release
go get github.com/github-release/github-release
github-pre-release:
rm -rf release/$(TAG)
make release
github-release --version || go get github.com/github-release/github-release
git push
github-release info -t $(TAG) || github-release release -t $(TAG) --draft
github-release info -t $(TAG) || github-release release -t $(TAG) --draft --description "$$(cat RELEASE.md)"
make github-upload-all
github-release edit -t $(TAG) --pre-release
github-release edit -t $(TAG) --pre-release --description "$$(cat RELEASE.md)"

github-latest-release:
github-release edit -t $(TAG)
github-release edit -t $(TAG) --description "$$(cat RELEASE.md)"
make dockerhub-latest-release
24 changes: 24 additions & 0 deletions PROCESS.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
## My own release procedure

```bash
# Fork the build
make fork-version NEW_VERSION=2.3.2 NEW_SHA=e6120a8f6ff36f627a4da3a1a51a1e47231f5cc8

# Try to naively apply patches
make tmp-env
make tmp-env-client

# Try to naively compile first
cd tmp/v2.3.2/proxmox-backup
cargo build

# Try to compile first locally
make amd64-docker-build
make amd64-client

# Create release package
make github-pre-release

# Mark the current version as latest
make github-latest-release
```
24 changes: 17 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,13 +1,27 @@
# Proxmox Backup Server in a Container

[![Docker Image Version (latest semver)](https://img.shields.io/docker/v/ayufan/proxmox-backup-server?sort=semver)](https://hub.docker.com/repository/docker/ayufan/proxmox-backup-server)
[![GitHub release (latest by date)](https://img.shields.io/github/v/release/ayufan/pve-backup-server-dockerfiles?label=GitHub%20STABLE)](https://github.com/ayufan/pve-backup-server-dockerfiles/releases) [![GitHub release (latest by date including pre-releases)](https://img.shields.io/github/v/release/ayufan/pve-backup-server-dockerfiles?include_prereleases&color=red&label=GitHub%20BETA)](https://github.com/ayufan/pve-backup-server-dockerfiles/releases/latest)

[![Docker Image Version (tag latest semver)](https://img.shields.io/docker/v/ayufan/proxmox-backup-server/latest?label=Docker%20LATEST)](https://hub.docker.com/r/ayufan/proxmox-backup-server/tags) [![Docker Image Version (latest semver)](https://img.shields.io/docker/v/ayufan/proxmox-backup-server?sort=semver&color=red&label=Docker%20BETA)](https://hub.docker.com/r/ayufan/proxmox-backup-server/tags)

This is an unofficial compilation of Proxmox Backup Server
to run it in a container for AMD64 and ARM64.

Running in a container might result in some functions not working
properly. Feel free to create an issue to debug those.

## Buy me a Coffee

<a href='https://ko-fi.com/ayufan' target='_blank'><img height='30' style='border:0px;height:40px;' src='https://az743702.vo.msecnd.net/cdn/kofi3.png?v=0' alt='Buy Me a Coffee at ko-fi.com' /></a>

If you found it useful :)

## Common problems

- Some people see authentication failure using `admin@pbs`: Ensure that `/run` is mounted to `tmpfs` which is requirement of `2.1.x`
- Some Synology devices use a really old kernel (3.1), for such the https://github.com/ayufan/pve-backup-server-dockerfiles/pull/15
is needed, and image needs to be manually recompiled.

## Pre-built images

For starting quickly all images are precompiled and hosted
Expand Down Expand Up @@ -38,11 +52,7 @@ The core features should work, but there are ones do not work due to container a

## Changelog

- v2.1.5 - 10 Feb, 2022 - also fixes bug with missing ksm
- v2.1.2 - 30 Nov, 2021 - this version requires `tmpfs` to be used for `/run`. Inspect `docker-compose.yml`
- v2.0.7 - Jul 31, 2021
- v2.0.4 - Jul 14, 2021
- v1.1.9 - Jun 14, 2021
See [Releases](https://github.com/ayufan/pve-backup-server-dockerfiles/releases).

## Configure

Expand Down Expand Up @@ -206,5 +216,5 @@ and install on Debian Bullseye.

## Author

This is just built by Kamil Trzciński, 2020-2021
This is just built by Kamil Trzciński, 2020-2023
from the sources found on http://git.proxmox.com/.
10 changes: 3 additions & 7 deletions RELEASE.md
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
## My own release procedure
## Buy me a Coffee

```bash
# Create release package
make github-pre-release
<a href='https://ko-fi.com/ayufan' target='_blank'><img height='30' style='border:0px;height:40px;' src='https://az743702.vo.msecnd.net/cdn/kofi3.png?v=0' alt='Buy Me a Coffee at ko-fi.com' /></a>

# Mark the current version as latest
make github-latest-release
```
If you found it useful :)
5 changes: 5 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ services:
- pbs_lib:/var/lib/proxmox-backup
tmpfs:
- /run
cap_add: # smartctl support
- SYS_RAWIO
devices:
- /dev/sda
- /dev/sdb
restart: unless-stopped
stop_signal: SIGHUP

Expand Down
13 changes: 13 additions & 0 deletions runit/proxmox-backup-api/run
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,24 @@ chmod 700 /var/lib/proxmox-backup
chown backup:backup /var/log/proxmox-backup
chmod 700 /var/log/proxmox-backup

# recycle lock files
rm /etc/proxmox-backup/.*.lck /etc/proxmox-backup/*.lock

if [[ ! -e /etc/proxmox-backup/csrf.key ]] && [[ ! -e /etc/proxmox-backup/.initialized ]]; then
echo 'Copying defaults into `/etc/proxmox-backup/`'
cp -rv /etc/proxmox-backup-default/. /etc/proxmox-backup/
touch /etc/proxmox-backup/.initialized
fi

for mp in /etc/proxmox-backup /var/lib/proxmox-backup /var/log/proxmox-backup; do
if ! mountpoint "$mp"; then
echo "WARNING: The '$mp' should be volume. PBS might not work correctly (data might be lost between runs)."
fi
done

if ! mountpoint /run; then
echo "WARNING: The '/run' should be 'tmpfs' mountpoint. PBS might not work correctly."
fi

echo "API: Starting..."
exec /usr/lib/*/proxmox-backup/proxmox-backup-api
19 changes: 19 additions & 0 deletions scripts/experimental-cargo.bash
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/bin/bash

set -eo pipefail

for package in *; do
[[ ! -d "$package" ]] && continue

# enable experimental features
for cargo in $(find "$package/" -name Cargo.toml); do
echo "Changing $cargo..."
sed -i '1s/^/cargo-features = ["workspace-inheritance"]\n\n/' "$cargo"
git -C "$package" add "$(realpath "$cargo")"
done

if ! git -C "$package" diff --cached --exit-code --quiet; then
git -C "$package" diff --cached | cat
git -C "$package" commit -m "experimental-cargo.bash"
fi
done
81 changes: 81 additions & 0 deletions scripts/resolve-dependencies.bash
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
#!/bin/bash

replace_patch_crates_io() {
sed -i -e '/^\[patch\.crates-io\]/,/^\[/{//!d}' -e '/^\[patch\.crates-io\]/d' "$1"
echo "[patch.crates-io]" >> "$1"
}

get_deps() {
local all=$(sed -n -e '/^\[dependencies\]/,/^\[/{//!p}' "$1")
echo "$all" | grep git | sed -n -e 's/^\([a-z0-9_-]*\)\(\.workspace\)*\s*=.*$/\1 git/p'
echo "$all" | grep -v git | sed -n -e 's/^\([a-z0-9_-]*\)\(\.workspace\)*\s*=.*$/\1/p'

local all=$(sed -n -e '/^\[workspace\.dependencies\]/,/^\[/{//!p}' "$1")
echo "$all" | grep git | sed -n -e 's/^\([a-z0-9_-]*\)\(\.workspace\)*\s*=.*$/\1 git/p'
echo "$all" | grep -v git | sed -n -e 's/^\([a-z0-9_-]*\)\(\.workspace\)*\s*=.*$/\1/p'
}

get_deps_with_path() {
local parent_dep_path="$1"
local parent_dep="$2"
local parent_dep_git="$3"

local cargo_dep
local cargo_git

while read cargo_dep cargo_git; do
local cargo_dep_path="${found_deps["$cargo_dep"]}"
[[ -z "$cargo_dep_path" ]] && cargo_dep_path="${found_deps["$cargo_dep-rs"]}"
[[ -z "$cargo_dep_path" ]] && continue

if [[ -n "$parent_dep_git" ]] && [[ "${git_deps["$parent_dep"]}" == "${git_deps["$cargo_dep"]}" ]]; then
get_deps_with_path "$cargo_dep_path/Cargo.toml" "$cargo_dep" "$parent_dep_git"
elif [[ "${git_deps["$parent_dep"]}" == "${git_deps["$cargo_dep"]}" ]]; then
get_deps_with_path "$cargo_dep_path/Cargo.toml" "$cargo_dep" "$parent_dep_git"
else
echo "$cargo_dep $cargo_dep_path"
get_deps_with_path "$cargo_dep_path/Cargo.toml" "$cargo_dep" "$cargo_git"
fi
done < <(get_deps "$parent_dep_path")
}

update_deps() {
local cargo_toml="$1"
local cargo_package=$(basename $(dirname "$cargo_toml"))

replace_patch_crates_io "$cargo_toml"

while read cargo_dep cargo_dep_path; do
echo "$cargo_dep => $cargo_dep_path"
grep -q "$cargo_dep.*git" "$cargo_toml" && continue # TODO: this is not fully working

echo "$cargo_package: Cargo dep: $cargo_dep => found => $cargo_dep_path"
echo "$cargo_dep = { path = \"$cargo_dep_path\" }" >> "$cargo_toml"
done < <(get_deps_with_path "$cargo_toml" "$cargo_package" "" | sort -u)
}

search_dir="${1:-.}"

declare -A found_deps
declare -A git_deps

while read cargo_path; do
cargo_path=$(dirname "$cargo_path")
cargo_dep=$(basename "$cargo_path")
git_repo=$(git -C "$cargo_path" rev-parse --show-toplevel)
found_deps[$cargo_dep]="$cargo_path"
git_deps[$cargo_dep]="$git_repo"
done < <(find "$PWD" -wholename "*/Cargo.toml" | sort)

while read CARGO_TOML; do
update_deps "$CARGO_TOML"

git_repo=$(git -C "$(dirname "$CARGO_TOML")" rev-parse --show-toplevel)
git -C "$git_repo" add "$(realpath "$CARGO_TOML")"
if ! git -C "$git_repo" diff --cached --exit-code --quiet; then
git -C "$git_repo" diff --cached | cat
git -C "$git_repo" commit -m "resolve-dependencies.bash"
fi
done < <(find $1 -name Cargo.toml)

echo "Done."
37 changes: 28 additions & 9 deletions scripts/strip-cargo.bash
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,37 @@
set -eo pipefail

for package in *; do
if [[ -f "$package/.cargo/config" ]]; then
rm -v "$package/.cargo/config"
fi
[[ ! -d "$package" ]] && continue

# remove ex.: librust-anyhow-1+default-dev
if [[ -f "$package/debian/control" ]]; then
echo "Stripping '$package/debian/control'..."
sed -i 's/librust-.*-dev[^,]*/libstd-rust-dev/g' "$package/debian/control"
fi
while read cargo_config; do
echo -n > "$cargo_config"
git -C "$package" add "$(realpath "$cargo_config")"
done < <(find "$package" -wholename '*/.cargo/config')

while read cargo_config; do
echo -n > "$cargo_config"
git -C "$package" add "$(realpath "$cargo_config")"
done < <(find "$package" -wholename '*/debian/cargo_home/config')

while read debian_control; do
echo "Stripping '$debian_control'..."

# remove ex.: librust-anyhow-1+default-dev (when it ends with `,`)
sed -i '/^\s*librust-.*-dev[^,]*,/d' "$debian_control"

# in some cases just replace to `libstd-rust-dev`
sed -i 's/^\s*librust-.*-dev[^,]*/ libstd-rust-dev/g' "$debian_control"

# change `cargo:native (>= 0.65.0-1),` to `cargo:native <!nocheck>`
sed -i 's/cargo:native\s*(>=.*)/cargo:native <!nocheck>/g' "$debian_control"

# change `dh-cargo (>= 25),` to `dh-cargo`
sed -i 's/dh-cargo\s*(>=.*)/dh-cargo/g' "$debian_control"
git -C "$package" add "$(realpath "$debian_control")" || true
done < <(find "$package" -wholename '*/debian/control')

git -C "$package" add .
if ! git -C "$package" diff --cached --exit-code --quiet; then
git -C "$package" diff --cached | cat
git -C "$package" commit -m "strip-cargo.bash"
fi
done
Loading

0 comments on commit 08b9c21

Please sign in to comment.