Update 4.1.3 (20240312)

.github/workflows/build-images.yml

@@ -3,7 +3,7 @@ name: Build Docker images for all OpenSlides services
 on:
   push:
     branches:
-      - main
+      - staging/4*
       - stable/4*
   workflow_dispatch:
 
@@ -33,7 +33,7 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v2
         with:
-          submodules: true
+          submodules: recursive
 
       - name: Read version
         id: readversion

Makefile

@@ -56,6 +56,9 @@ services-to-main-pull:
 staging-update:
 	$(SCRIPT_PATH)/make-update.sh staging $(ARGS)
 
+hotfix-update:
+	$(SCRIPT_PATH)/make-update.sh hotfix $(ARGS)
+
 stable-update:
 	$(SCRIPT_PATH)/make-update.sh stable $(ARGS)
 

VERSION

@@ -1 +1 @@
-4.1.2
+4.1.3

dev/docker/build.sh

@@ -18,12 +18,15 @@ TARGETS=(
   [vote]="$HOME/openslides-vote-service/"
   [icc]="$HOME/openslides-icc-service/"
 )
-CLIENT_VERSION_TXT="${TARGETS[client]}/client/src/assets/version.txt"
 
 DOCKER_REPOSITORY="openslides"
 DOCKER_TAG="$(cat VERSION)"
-[[ "$(git rev-parse --abbrev-ref HEAD)" != main ]] ||
+BRANCH="$(git rev-parse --abbrev-ref HEAD)"
+if [[ "$BRANCH" == staging/* ]]; then
   DOCKER_TAG="$DOCKER_TAG-staging-$(date +%Y%m%d)-$(git rev-parse HEAD | cut -c -7)"
+elif [[ "$BRANCH" != stable/* ]]; then
+  DOCKER_TAG="$DOCKER_TAG-$BRANCH"
+fi
 CONFIG="/etc/osinstancectl"
 OPTIONS=()
 BUILT_IMAGES=()
@@ -97,6 +100,8 @@ SELECTED_TARGETS=($@)
 [[ "${#SELECTED_TARGETS[@]}" -ge 1 ]] || SELECTED_TARGETS=("${DEFAULT_TARGETS[@]}")
 [[ "${SELECTED_TARGETS[@]}" != "all" ]] || SELECTED_TARGETS=("${!TARGETS[@]}")
 
+OPTIONS+=(--build-arg "VERSION=$DOCKER_TAG")
+
 for i in "${SELECTED_TARGETS[@]}"; do
 
   loc="${TARGETS[$i]}"
@@ -126,10 +131,6 @@ for i in "${SELECTED_TARGETS[@]}"; do
     printf '\t"commit-abbrev": "%s",\n' "$(git rev-parse --abbrev-ref HEAD)"
     printf '}\n'
   } > version.json
-  if [[ -w "$CLIENT_VERSION_TXT" ]]; then
-    client_dev_version="$(< "$CLIENT_VERSION_TXT")"
-    printf "$DOCKER_TAG ($(date +%Y-%m-%d))" > "$CLIENT_VERSION_TXT"
-  fi
 
   # Special instructions for local services
   build_script="${loc}/build.sh"
@@ -139,9 +140,6 @@ for i in "${SELECTED_TARGETS[@]}"; do
     docker build --tag "$img" --pull "${OPTIONS[@]}" "$loc"
   fi
   rm version.json
-  if [[ -w "$CLIENT_VERSION_TXT" ]]; then
-    echo "$client_dev_version" > "$CLIENT_VERSION_TXT"
-  fi
 
   BUILT_IMAGES+=("$img ON")
 done

dev/docker/dc.otel.dev.yml

@@ -23,3 +23,7 @@ services:
   datastore-writer:
     environment:
       - OPENTELEMETRY_ENABLED=1
+
+  auth:
+    environment:
+      - OPENTELEMETRY_ENABLED=1

dev/scripts/make-update.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -e
+set -ae
 
 ME=$(basename "$0")
 
@@ -39,6 +39,13 @@ MODES:
     merge commits are created in the main repository as well as every affected
     submodule. These will then be attempted to be pushed directly into upstream.
     Use --local to skip pushing.
+
+  hotfix
+    Create an update based on changes in submodules stable branches. Will
+    create a new commit in the main repositories stable analogous to the stable
+    workflow. These will then be attempted to be pushed directly into upstream.
+    Use --local to skip pushing.
+    Changes must manually be reflected into the main branch afterwards.
 EOF
 }
 
@@ -69,8 +76,8 @@ set_remote() {
 check_current_branch() {
   [ "$(git rev-parse --abbrev-ref HEAD)" == "$BRANCH_NAME" ] || {
     echo "ERROR: $BRANCH_NAME branch not checked out ($(basename $(realpath .)))"
-    ask y "Run \`git checkout $BRANCH_NAME\` now?" &&
-      git checkout $BRANCH_NAME ||
+    ask y "Run \`git checkout $BRANCH_NAME && git submodule update\` now?" &&
+      git checkout $BRANCH_NAME && git submodule update ||
       abort 0
   }
 
@@ -85,23 +92,71 @@ check_current_branch() {
   fi
 }
 
+check_submodules_intialized() {
+  # From `man git-submodule`:
+  #   Each SHA-1 will possibly be prefixed with - if the submodule is not initialized
+  git submodule status --recursive | awk '$1 ~ "^-" {print "  " $2; x=1} END {exit x}' || {
+    echo "ERROR: Found the above uninitialized submodules. Please correct before rerunning $ME."
+    abort 1
+  }
+}
+
+check_meta_consistency() {
+  local target_rev="$1"
+  local meta_sha=
+  local meta_sha_last=
+  local ret_code=0
+
+  echo "Checking openslides-meta consistency ..."
+
+  # Doing a nested loop rather than foreach --recursive as it's easier to get
+  # both the path of service submod and the (potential) meta submod in one
+  # iteration
+  while read mod; do
+    while read meta_name meta_path; do
+      [[ "$meta_name" == 'openslides-meta' ]] ||
+        continue
+
+      # If target_rev is not specified we check the status of the currently
+      # checked out HEAD in the service submod.
+      # Note that this is different from target_rev being specified as 'HEAD'
+      # as the service submod can be in a different state than recorded in HEAD
+      # (e.g. changed commit pointer during staging-update)
+      mod_target_rev=HEAD
+      [[ "$target_rev" == "" ]] ||
+        mod_target_rev="$(git rev-parse "${target_rev}:${mod}")"
+
+      meta_sha="$(git -C "$mod" rev-parse "${mod_target_rev}:${meta_path}")"
+      echo "  $meta_sha $mod"
+      [[ -z "$meta_sha_last" ]] || [[ "$meta_sha" == "$meta_sha_last" ]] ||
+        ret_code=1
+      meta_sha_last="$meta_sha"
+    done <<< "$(git -C $mod submodule foreach -q 'echo "$name $sm_path"')"
+  done <<< "$(git submodule foreach -q 'echo "$sm_path"')"
+
+  return $ret_code
+}
+
 pull_latest_commit() {
   if [ -z "$OPT_PULL" ]; then
-    echo "git fetch $REMOTE_NAME && git checkout $REMOTE_NAME/$BRANCH_NAME ..."
+    echo "git fetch $REMOTE_NAME && git checkout $REMOTE_NAME/$BRANCH_NAME && git submodule update ..."
     git fetch "$REMOTE_NAME" &&
     git checkout "$REMOTE_NAME/$BRANCH_NAME"
+    git submodule update
   else
-    echo "git checkout $BRANCH_NAME && git pull --ff-only $REMOTE_NAME $BRANCH_NAME ..."
+    echo "git checkout $BRANCH_NAME && git pull --ff-only $REMOTE_NAME $BRANCH_NAME && git submodule update ..."
     git checkout "$BRANCH_NAME" &&
     git pull --ff-only "$REMOTE_NAME" "$BRANCH_NAME" || {
       echo "ERROR: make sure a local branch $BRANCH_NAME exists and can be fast-forwarded to $REMOTE_NAME"
       abort 1
     }
+    git submodule update
   fi
 }
 
 increment_patch() {
   set_remote
+  git fetch $REMOTE_NAME $STABLE_BRANCH_NAME
   patch_upstream=$(git show $REMOTE_NAME/$STABLE_BRANCH_NAME:VERSION  | awk -F. '{print $3}')
   patch_local=$(git show HEAD:VERSION  | awk -F. '{print $3}')
 
@@ -127,15 +182,14 @@ fetch_all_changes() {
   echo "Successfully updated all submodules to latest commit."
 }
 
-
-make_staging_update() {
+add_changes() {
   local REPLY= diff_cmd=
 
   set_remote
   check_current_branch
 
   ask y "Fetch all submodules changes now?" &&
-    fetch_all_changes || :
+    fetch_all_changes
   diff_cmd="git diff --color=always --submodule=log"
   [[ "$($diff_cmd | grep -c .)" -gt 0 ]] ||
     abort 0
@@ -164,30 +218,77 @@ make_staging_update() {
           "") ;;
           *) target_sha="$REPLY" ;;
         esac
+	# If $mod is skipped it must be reset (i.e. checked out below) to the current commit in HEAD
         [[ "$target_sha" != '-' ]] ||
-          exit 0 # exit the subshell, acting like 'continue'
+          target_sha="$(git rev-parse "HEAD:$mod")"
+
         git -C "$mod" checkout "$target_sha"
+        git -C "$mod" submodule update
         git add "$mod"
       )
-    done || :
+    done
 
   echo ''
   diff_cmd="git diff --staged --submodule=log"
   [[ "$($diff_cmd | grep -c .)" -gt 0 ]] || {
     echo "No changes added."
     abort 0
   }
-  echo 'Changes to be included for the new staging update:'
+  echo 'Changes to be included for the new update:'
   echo '--------------------------------------------------------------------------------'
   $diff_cmd
   echo '--------------------------------------------------------------------------------'
-  ask y "Commit now?" && {
+}
+
+make_staging_update() {
+  local diff_cmd=
+
+  add_changes
+
+  check_meta_consistency || {
+    echo "WARN: openslides-meta is not consistent. Continuing may imply services are incompatible."
+    echo "WARN: Be sure to fix this until the next stable update."
+    ask y "Continue?" ||
+      abort 1
+  }
+
+  ask y "Commit on branch $BRANCH_NAME for a new staging update now?" && {
     increment_patch &&
       git add VERSION
     git commit --message "Staging update $(date +%Y%m%d)"
     git show --no-patch
     echo "Commit created. Push to $REMOTE_NAME remote and PR into main repo to bring it live."
-  } || :
+  }
+}
+
+make_hotfix_update() {
+  local diff_cmd=
+
+  add_changes
+
+  check_meta_consistency || {
+    echo "WARN: openslides-meta is not consistent. This is not a good sign for stable update."
+    echo "WARN: Only continue if you are sure services will be compatible."
+    ask n "Continue?" ||
+      abort 1
+  }
+
+  ask y "Commit on branch $BRANCH_NAME for a new stable (hotfix) update now?" && {
+    increment_patch &&
+      git add VERSION
+    git commit --message "Update $(cat VERSION) ($(date +%Y%m%d))"
+    git show --no-patch
+    echo "Commit created."
+  }
+
+  [[ -z "$OPT_LOCAL" ]] ||
+    return 0
+
+  echo "Attempting to push new $BRANCH_NAME commit into $REMOTE_NAME."
+  echo "Ensure you have proper access rights or use --local to skip pushing."
+  ask y "Continue?" ||
+    abort 0
+  git push "$REMOTE_NAME" "$BRANCH_NAME"
 }
 
 merge_stable_branches() {
@@ -212,6 +313,7 @@ merge_stable_branches() {
 
       set_remote
       git checkout "$BRANCH_NAME"
+      git submodule update
       check_current_branch
 
       git merge --no-ff -Xtheirs "$mod_target_sha" --log --message "Merge main into $STABLE_BRANCH_NAME. Update $(date +%Y%m%d)"
@@ -248,6 +350,12 @@ make_stable_update() {
     *) target_sha="$REPLY" ;;
   esac
 
+  check_meta_consistency "$target_sha" || {
+    echo "ERROR: openslides-meta is not consistent at $target_sha. This is not acceptable for a stable update."
+    echo "ERROR: Please fix this in a new staging update before trying again."
+    abort 1
+  }
+
   merge_stable_branches "$target_sha"
 
   # Merge, but don't commit yet ...
@@ -273,6 +381,7 @@ make_stable_update() {
   git push "$REMOTE_NAME" "$BRANCH_NAME"
 }
 
+
 shortopt='phl'
 longopt='pull,help,local'
 ARGS=$(getopt -o "$shortopt" -l "$longopt" -n "$ME" -- $@)
@@ -300,11 +409,16 @@ while true; do
   esac
 done
 
+# $ME assumes all submodules to be initialized, so check before doing anything else
+check_submodules_intialized
+
 for arg; do
   case $arg in
     fetch-all-changes)
       BRANCH_NAME=main
       fetch_all_changes
+      check_meta_consistency ||
+        echo "WARN: openslides-meta is not consistent."
       shift 1
       ;;
     staging)
@@ -317,5 +431,10 @@ for arg; do
       make_stable_update
       shift 1
       ;;
+    hotfix)
+      BRANCH_NAME=$STABLE_BRANCH_NAME
+      make_hotfix_update
+      shift 1
+      ;;
   esac
 done

dev/scripts/update_labels_and_milestones.py

@@ -11,6 +11,7 @@
     "openslides-icc-service",
     "openslides-manage-service",
     "openslides-media-service",
+    "openslides-meta",
     "openslides-proxy",
     "openslides-search-service",
     "openslides-vote-service",