From b8d877920bd612867f72adffb486924859f572fe Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 6 Jun 2023 16:47:21 +0200 Subject: [PATCH 1/5] Monitor migrations to detect database changes --- update.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/update.sh b/update.sh index 5166878..01fb532 100755 --- a/update.sh +++ b/update.sh @@ -36,6 +36,7 @@ else fi CHECK_FILES=" + server/libs/infra/src/migrations server/Dockerfile server/start-server.sh server/start-microservices.sh From 4cce61b8fcb6567e2a93059e9fdcf32c1ea8d6cf Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 6 Jun 2023 17:21:07 +0200 Subject: [PATCH 2/5] Bump versions --- snap/snapcraft.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 8cca7a3..be69332 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -206,7 +206,7 @@ parts: plugin: npm npm-node-version: "18.16.0" source: https://github.com/immich-app/immich.git - source-tag: v1.57.1 + source-tag: v1.60.0 source-subdir: server override-build: | snapcraftctl set-version "$(git describe --tags)-dist1" @@ -233,7 +233,7 @@ parts: plugin: npm npm-node-version: "18.16.0" source: https://github.com/immich-app/immich.git - source-tag: v1.57.1 + source-tag: v1.60.0 source-subdir: web override-build: | patch -p0 -i $SNAPCRAFT_PART_SRC/../../patches/src/001-version-announcement-box.patch -d $SNAPCRAFT_PART_BUILD @@ -252,7 +252,7 @@ parts: machine-learning: plugin: python source: https://github.com/immich-app/immich.git - source-tag: v1.57.1 + source-tag: v1.60.0 source-subdir: machine-learning build-environment: - SNAPCRAFT_PYTHON_VENV_INTERP_PATH: /root/parts/machine-learning/install/bin/python3 @@ -271,7 +271,7 @@ parts: plugin: npm npm-node-version: "18.16.0" source: https://github.com/immich-app/CLI.git - source-tag: v0.36.0 + source-tag: v0.37.0 override-build: | npm install npm run build From 9d574e39e2df1afbede1cfe77cefa8569757a152 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Wed, 7 Jun 2023 22:06:39 +0200 Subject: [PATCH 3/5] Build and bundle python 3.10 inside snap --- snap/snapcraft.yaml | 34 +++++++++++++++++++++++++-------- src/bin/immich-machine-learning | 4 ++-- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index be69332..5c0bc28 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -189,6 +189,25 @@ parts: | tar xJf - -C "$SNAPCRAFT_PART_INSTALL" --no-same-owner --strip-components=1 fi + python: + source: https://www.python.org/ftp/python/3.10.11/Python-3.10.11.tgz + plugin: make + override-build: | + ./configure --enable-optimizations --with-ensurepip=install + snapcraftctl build + build-packages: + - libssl-dev + - zlib1g-dev + - libbz2-dev + - libreadline-dev + - libsqlite3-dev + - xz-utils + - libffi-dev + stage-packages: + - libtcl8.6 + - libtk8.6 + - libxft2 + ffmpeg: plugin: nil stage-packages: @@ -250,22 +269,21 @@ parts: - patches machine-learning: - plugin: python + plugin: nil source: https://github.com/immich-app/immich.git source-tag: v1.60.0 source-subdir: machine-learning - build-environment: - - SNAPCRAFT_PYTHON_VENV_INTERP_PATH: /root/parts/machine-learning/install/bin/python3 override-build: | - snapcraftctl build - pip install torch --index-url https://download.pytorch.org/whl/cpu - pip install transformers tqdm numpy scikit-learn scipy nltk sentencepiece fastapi Pillow uvicorn[standard] - pip install --no-deps sentence-transformers - pip install insightface onnxruntime + ../../python/install/usr/local/bin/python3 ../../python/install/usr/local/bin/pip3 install --target $SNAPCRAFT_PART_INSTALL/opt/piptarget torch --index-url https://download.pytorch.org/whl/cpu + ../../python/install/usr/local/bin/python3 ../../python/install/usr/local/bin/pip3 install --target $SNAPCRAFT_PART_INSTALL/opt/piptarget transformers tqdm numpy scikit-learn scipy nltk sentencepiece fastapi Pillow uvicorn[standard] + ../../python/install/usr/local/bin/python3 ../../python/install/usr/local/bin/pip3 install --target $SNAPCRAFT_PART_INSTALL/opt/piptarget --no-deps sentence-transformers + ../../python/install/usr/local/bin/python3 ../../python/install/usr/local/bin/pip3 install --target $SNAPCRAFT_PART_INSTALL/opt/piptarget insightface onnxruntime mkdir -p $SNAPCRAFT_PART_INSTALL/usr/src/ml cp -r machine-learning/src $SNAPCRAFT_PART_INSTALL/usr/src/ml stage-packages: - libgomp1 + after: + - python cli: plugin: npm diff --git a/src/bin/immich-machine-learning b/src/bin/immich-machine-learning index 68b1158..52ecd69 100755 --- a/src/bin/immich-machine-learning +++ b/src/bin/immich-machine-learning @@ -1,13 +1,13 @@ #!/bin/bash . $SNAP/bin/load-env -. $SNAP/bin/activate export PYTHONDONTWRITEBYTECODE=1 export PYTHONUNBUFFERED=1 +export PYTHONPATH="$SNAP/opt/piptarget" export MACHINE_LEARNING_CACHE_FOLDER="$SNAP_COMMON/cache" banner cd $SNAP/usr/src/ml -python3 $SNAP/usr/src/ml/src/main.py | grep -v "GET /ping" +$SNAP/usr/local/bin/python3 $SNAP/usr/src/ml/src/main.py | grep -v "GET /ping" From 6c64da33b8c3936246334250b9da93c3e930fa2d Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Wed, 7 Jun 2023 22:39:41 +0200 Subject: [PATCH 4/5] Fix path --- src/bin/sync-removed-watch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/sync-removed-watch b/src/bin/sync-removed-watch index 1381879..a0c7b77 100755 --- a/src/bin/sync-removed-watch +++ b/src/bin/sync-removed-watch @@ -22,7 +22,7 @@ watch_removed() { continue fi - if ! immich_api "$KEY" "/asset/assetById/$asset_id" | jq -e .id > /dev/null; then + if ! immich_api "$KEY" "asset/assetById/$asset_id" | jq -e .id > /dev/null; then echo "Asset $asset_id not found in Immich (user $user_id, path $path)" continue fi From 3801a5c98fefcf19015e35a8e43c98d5e213a623 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Wed, 7 Jun 2023 23:10:55 +0200 Subject: [PATCH 5/5] Wait on still growing files --- src/bin/sync-created-watch | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/bin/sync-created-watch b/src/bin/sync-created-watch index af48385..68396b2 100755 --- a/src/bin/sync-created-watch +++ b/src/bin/sync-created-watch @@ -12,6 +12,18 @@ block_immich_server_ready watch_created() { cat - | while read -r changed_path; do if ! ignore_path "$changed_path"; then + + init_file_size="$(stat --printf="%s" "$changed_path")" + while true; do + sleep 1 + current_file_size="$(stat --printf="%s" "$changed_path")" + if [ "$init_file_size" -eq "$current_file_size" ]; then + break + else + init_file_size="$current_file_size" + fi + done + file_hash="$(sha1sum "$changed_path")" file_hash="${file_hash% *}" user_id="$(extract_user_id_from_path "$changed_path")"