Store build logs in subdirectories per branch and per month

Note this update requires an additional msys package to be installed (rsync2)
so you may have to re-run setup-mingw64.ps1 once before
starting buildserver/build_package.ps1.

buildserver/build_package.ps1

@@ -99,7 +99,7 @@ bash-command -command "cd $script_unix && git pull"
 bash-command -command "echo Build Started $time_stamp > $log_unix"
 #copy the file to the download server so that everyone can see we've started
 if ($hostname) {
-    bash-command -command "scp -p $log_unix $hostname/$log_dir/"
+    bash-command -command "$script_unix/buildserver/upload_build_log.sh $log_unix $hostname $log_dir $branch"
 }
 
 # Update MinGW-w64
@@ -142,7 +142,7 @@ bash-command -command "echo Build Ended $time_stamp >> $log_unix"
 
 # Copy the transcript and installer to the download server and delete them.
 if ($hostname) {
-	bash-command -command "scp -p $log_unix $hostname/$log_dir/"
+    bash-command -command "$script_unix/buildserver/upload_build_log.sh $log_unix $hostname $log_dir $branch"
     if ($new_file) {
 	bash-command -command "scp -p $setup_file $hostname/$branch"
     }

buildserver/upload_build_log.sh

@@ -0,0 +1,59 @@
+#! /bin/bash
+#
+# A function to upload build logs
+# If a branch is known it will upload to
+# build-logs/branch/yyyy-mm/logfile
+# otherwise it will upload to
+# build-logs/logfile
+
+log_file=$1
+host=$2
+log_dir=$3
+remote_branch_dir=$4
+base_dir=$(pwd) # A dummy base directory used in rsync
+
+# This function will create a remote directory
+# It will only run if $host is configured.
+# Note it assumes the parent directory exists and it won't
+# create intermediate subdirectories if that is not the case
+function create_remote_dir()
+{
+    if [[ -n "$host" ]]
+    then
+        # For this hack $base_dir is just an arbitrary existing directory
+        # It doesn't matter which one
+        # No files will be copied from it anyway because of the exclude parameter
+        rsync -a --exclude='*' "$base_dir"/ "$1"
+    fi
+}
+
+if [[ -n "$host" ]]
+then
+    echo "Uploading log file '$(basename $log_file)'"
+    create_remote_dir "$host"/build-logs
+    if [[ -z "$remote_branch_dir" ]]
+    then
+        # We don't know the build type yet, so we can't determine
+        # the final remote directory to store it
+        # So let's just store it in the top-level so we have a trace
+        # of the build start or very early failures
+        rsync -a "$log_file" "$host"/build-logs
+    else
+        # Now the subdirectory to store the log file is is known
+        # In addition group per month to simplify navigation even more
+        month_part=$(date +%Y-%m)
+
+        create_remote_dir "$host"/build-logs/$remote_branch_dir
+        create_remote_dir "$host"/build-logs/$remote_branch_dir/$month_part
+
+        rsync -a "$log_file" "$host"/build-logs/$remote_branch_dir/$month_part
+
+        # Finally remove the initially start build log uploaded earlier
+        # Disable fatal error handling though to prevent the complete script from exiting
+        # if no early build log exists
+        echo "Removing initial startup build log uploaded earlier"
+        set +ex
+        rsync -rv --delete --include="$(basename $log_file)" --exclude='*' "$base_dir"/ "$host"/build-logs/
+        set -ex
+    fi
+fi

setup-mingw64.ps1

@@ -234,7 +234,7 @@ bash-command -command "pacman -Syyuu --noconfirm"
 
 # Set up aliases for the parts of msys-devtools and mingw-w64-toolchain that
 # we need:
-$devel = "asciidoc autoconf autoconf2.13 autogen automake-wrapper automake1.10 automake1.11 automake1.12 automake1.13 automake1.14 automake1.15 automake1.6 automake1.7 automake1.8 automake1.9 bison diffstat diffutils dos2unix file flex gawk gettext gettext-devel gperf grep groff intltool libtool m4 make man-db pacman pactoys-git patch patchutils perl pkg-config sed swig texinfo texinfo-tex wget xmlto git jhbuild-git texinfo"
+$devel = "asciidoc autoconf autoconf2.13 autogen automake-wrapper automake1.10 automake1.11 automake1.12 automake1.13 automake1.14 automake1.15 automake1.6 automake1.7 automake1.8 automake1.9 bison diffstat diffutils dos2unix file flex gawk gettext gettext-devel gperf grep groff intltool libtool m4 make man-db pacman pactoys-git patch patchutils perl pkg-config rsync2 sed swig texinfo texinfo-tex wget xmlto git jhbuild-git texinfo"
 
 $toolchain = "binutils cmake crt-git gcc gcc-libs gdb headers-git libmangle-git libtool libwinpthread-git make pkg-config tools-git winpthreads-git"