grasslxd server: cronjobs hugo, snapshots, progman, addon manuals (#216)

scripts to compile on grasslxd server: - hugo based web pages - source code weekly snapshots - Linux binary weekly snapshots - addon manuals - programmer's manual rsync configuration for mirrors Fixes OSGeo/grass-website#171
OSGeo · Jun 20, 2020 · 0692cd4 · 0692cd4
1 parent e59ea8f
commit 0692cd4
Show file tree

Hide file tree

Showing 12 changed files with 1,303 additions and 0 deletions.
diff --git a/tools/cronjobs_osgeo_lxd/compile_addons_git.sh b/tools/cronjobs_osgeo_lxd/compile_addons_git.sh
@@ -0,0 +1,131 @@
+#!/bin/sh
+
+# Martin Landa, 2013
+# updated for GRASS GIS 7 (only) Addons by Markus Neteler 2020
+
+# This script compiles GRASS Addons, it's called by cron_grass78_releasebranch_78_build_bins.sh | cron_grass7_HEAD_build_bins.sh
+
+if [ -z "$3" ]; then
+    echo "usage: $0 git_path topdir addons_path [separate]"
+    echo "eg. $0 ~/src/grass_addons/grass7/ ~/src/grass_trunk/dist.x86_64-pc-linux-gnu ~/.grass7/addons"
+    exit 1
+fi
+
+GIT_PATH="$1"
+TOPDIR="$2"
+ADDON_PATH="$3"
+#GRASS_VERSION=`echo -n ${GIT_PATH} | tail -c 1`
+GRASS_VERSION=7
+INDEX_FILE="index"
+
+if [ ! -d "$3" ] ; then
+    mkdir -p "$3"
+fi
+
+if [ -n "$4" ] ; then
+    SEP=1 # useful for collecting files (see build-xml.py)
+else
+    SEP=0
+fi
+
+if [ `uname -m` = "x86_64" ] ; then
+    PLATFORM=x86_64
+else
+    PLATFORM=x86
+fi
+
+rm -rf "$ADDON_PATH"
+mkdir  "$ADDON_PATH"
+
+cd "$GIT_PATH"
+
+date=`date -R`
+uname=`uname`
+mkdir "$ADDON_PATH/logs"
+touch "$ADDON_PATH/logs/${INDEX_FILE}.log"
+
+echo "<!--<?xml-stylesheet href=\"style.css\" type=\"text/css\"?>-->
+<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.1//EN\"
+	  \"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd\">
+
+<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" >
+
+<head>
+<meta http-equiv=\"Content-Type\" content=\"application/xhtml+xml; charset=utf-8\" />
+<title>GRASS $ADDON_PATH AddOns Logs ($PLATFORM)</title>
+<style type=\"text/css\">
+h1 { font-size: 125%; font-weight: bold; }
+table
+{
+border-collapse:collapse;
+}
+table,th, td
+{
+border: 1px solid black;
+}
+</style>
+</head>
+<body>
+<h1>GRASS $GRASS_VERSION Addons ($PLATFORM) / $uname (logs generated $date)</h1>
+<hr /> 
+<table cellpadding=\"5\">
+<tr><th style=\"background-color: grey\">AddOns</th>
+<th style=\"background-color: grey\">Status</th>
+<th style=\"background-color: grey\">Log file</th></tr>" > "$ADDON_PATH/logs/${INDEX_FILE}.html"
+
+echo "-----------------------------------------------------"
+echo "Addons '$ADDON_PATH'..."
+echo "-----------------------------------------------------"
+
+pwd=`pwd`
+# .. "hadoop"
+# from ../../grass7/
+for c in "db" "display" "general" "gui/wxpython" "imagery" "misc" "raster" "raster3d" "temporal" "vector" ; do
+    if [ ! -d $c ]; then
+	continue
+    fi
+    cd $c
+    for m in `ls -d */Makefile 2>/dev/null` ; do
+	m="${m%%/Makefile}"
+	echo -n "Compiling $m..."
+	cd "$m"
+	if [ $SEP -eq 1 ] ; then
+	    path="$ADDON_PATH/$m"
+	else
+	    path="$ADDON_PATH"
+	fi
+
+	export GRASS_ADDON_BASE=$path
+	echo "<tr><td><tt>$c/$m</tt></td>" >> "$ADDON_PATH/logs/${INDEX_FILE}.html"
+	make MODULE_TOPDIR="$TOPDIR" clean > /dev/null 2>&1
+	make MODULE_TOPDIR="$TOPDIR" \
+	    BIN="$path/bin" \
+	    HTMLDIR="$path/docs/html" \
+	    MANBASEDIR="$path/docs/man" \
+	    SCRIPTDIR="$path/scripts" \
+	    ETC="$path/etc" \
+            SOURCE_URL="https://github.com/OSGeo/grass-addons/tree/master/grass${GRASS_VERSION}/" > \
+            "$ADDON_PATH/logs/$m.log" 2>&1
+	if [ `echo $?` -eq 0 ] ; then
+	    printf "%-30s%s\n" "$c/$m" "SUCCESS" >> "$ADDON_PATH/logs/${INDEX_FILE}.log"
+	    echo " SUCCESS"
+	    echo "<td style=\"background-color: green\">SUCCESS</td>" >> "$ADDON_PATH/logs/${INDEX_FILE}.html"
+	else
+	    printf "%-30s%s\n" "$c/$m" "FAILED" >> "$ADDON_PATH/logs/${INDEX_FILE}.log"
+	    echo " FAILED"
+	    echo "<td style=\"background-color: red\">FAILED</td>" >> "$ADDON_PATH/logs/${INDEX_FILE}.html"
+	fi
+	echo "<td><a href=\"$m.log\">log</a></td></tr>" >> "$ADDON_PATH/logs/${INDEX_FILE}.html"
+	cd ..
+    done
+    cd $pwd
+    unset GRASS_ADDON_BASE
+done
+
+echo "</table><hr />
+<div style=\"text-align: right\">Valid: <a href=\"http://validator.w3.org/check/referer\">XHTML</a></div>
+</body></html>" >> "$ADDON_PATH/logs/${INDEX_FILE}.html"
+
+echo "Log file written to <$ADDON_PATH/logs/>"
+exit 0
+
diff --git a/tools/cronjobs_osgeo_lxd/cron_grass78_releasebranch_78_build_bins.sh b/tools/cronjobs_osgeo_lxd/cron_grass78_releasebranch_78_build_bins.sh
@@ -0,0 +1,263 @@
+#!/bin/sh
+
+# script to build GRASS 7.x relbranch78 binaries (shared libs)
+# (c) GPL 2+ Markus Neteler <neteler@osgeo.org>
+# Sat Mar 29 13:05:53 PDT 2014
+# Tue Apr 22 11:23:12 PDT 2014
+# Sat Sep 19 03:43:34 PDT 2015
+# Wed May 25 10:05:05 PDT 2016
+# Sun Nov 12 22:02:53 CET 2017
+# Fri Aug 31 06:59:59 PDT 2018
+# Sun 02 Jun 2019 03:30:38 PM CEST
+# Sun 04 Aug 2019 11:02:52 AM CEST
+# Sun 27 Oct 2019 09:00:14 PM CET
+# Thu 16 Jan 2020 08:17:43 AM UTC
+# Fri 19 Jun 2020 07:56:20 PM UTC
+#
+# GRASS GIS github, https://github.com/OSGeo/grass
+#
+## prep, neteler@osgeo6:$
+# mkdir -p ~/src
+# cd ~/src
+# for i in 2 4 6 ; do git clone https://github.com/OSGeo/grass.git releasebranch_7_$i ; done
+# for i in 2 4 6 ; do (cd releasebranch_7_$i ;  git checkout releasebranch_7_$i ) ; done
+#
+###################################################################
+# how it works:
+# - it updates the GRASS source code from github server
+# - configures, compiles
+# - packages the binaries
+# - generated the install scripts
+# - generates the pyGRASS HTML manual
+# - generates the user HTML manuals
+
+# Preparations:
+#  - Install PROJ: http://trac.osgeo.org/proj/ incl Datum shift grids
+#     sh conf_proj4.sh
+#  - Install GDAL: http://trac.osgeo.org/gdal/wiki/DownloadSource
+#     sh conf_gdal.sh
+#  - Install apt-get install texlive-latex-extra python3-sphinxcontrib.apidoc
+#  - Clone source from github
+#################################
+PATH=/home/neteler/binaries/bin:/usr/bin:/bin:/usr/X11R6/bin:/usr/local/bin
+
+GMAJOR=7
+GMINOR=8
+DOTVERSION=$GMAJOR.$GMINOR
+VERSION=$GMAJOR$GMINOR
+GVERSION=$GMAJOR
+
+###################
+CFLAGSSTRING='-O2'
+CFLAGSSTRING='-Werror-implicit-function-declaration -fno-common'
+LDFLAGSSTRING='-s'
+
+#define several paths if required:
+
+MAINDIR=/home/neteler
+# where to find the GRASS sources (git clone):
+SOURCE=$MAINDIR/src/
+BRANCH=releasebranch_7_$GMINOR # master
+GRASSBUILDDIR=$SOURCE/$BRANCH
+TARGETMAIN=~/var/www/grass/grass-cms
+TARGETDIR=$TARGETMAIN/grass${VERSION}/binary/linux/snapshot
+TARGETHTMLDIR=$TARGETMAIN/grass${VERSION}/manuals/
+
+MYBIN=$MAINDIR/binaries
+
+############################## nothing to change below:
+
+MYMAKE="nice make LD_LIBRARY_PATH=$MYBIN/lib:/usr/lib:/usr/local/lib"
+
+# catch CTRL-C and other breaks:
+trap "echo 'user break.' ; exit" 2 3 9 15
+
+halt_on_error()
+{
+  echo "ERROR: $1"
+  exit 1
+}
+
+configure_grass()
+{
+
+#  --enable-64bit \
+#  --with-libs=/usr/lib64 \
+
+# which package?
+#   --with-mysql --with-mysql-includes=/usr/include/mysql --with-mysql-libs=/usr/lib/mysql \
+
+
+CFLAGS=$CFLAGSSTRING LDFLAGS=$LDFLAGSSTRING ./configure \
+  --with-cxx \
+  --with-sqlite \
+  --with-postgres \
+  --with-geos \
+  --with-odbc \
+  --with-cairo --with-cairo-ldflags=-lfontconfig \
+  --with-proj --with-proj-share=/usr/share/proj \
+  --with-postgres --with-postgres-includes=/usr/include/postgresql \
+  --with-freetype --with-freetype-includes=/usr/include/freetype2 \
+  --with-netcdf \
+  --with-pdal \
+  --with-fftw \
+  --with-nls \
+  --with-blas --with-blas-includes=/usr/include/atlas/ \
+  --with-lapack --with-lapack-includes=/usr/include/atlas/ \
+  --with-zstd \
+  --without-motif \
+  --without-liblas \
+  2>&1 | tee config_$DOTVERSION.git_log.txt
+
+ if [ $? -ne 0 ] ; then
+   halt_on_error "configure doesn't work."
+ fi
+}
+
+######## update from git:
+
+mkdir -p $TARGETDIR
+cd $GRASSBUILDDIR/
+
+$MYMAKE distclean > /dev/null 2>&1
+
+# cleanup leftover garbage
+git status | grep '.rst' | xargs rm -f
+rm -rf lib/python/docs/_build/ lib/python/docs/_templates/layout.html
+rm -f config_${DOTVERSION}.git_log.txt ChangeLog
+
+# be sure to be on branch
+git checkout $BRANCH
+
+echo "git update..."
+git fetch --all --prune && git checkout $BRANCH && git pull --rebase || halt_on_error "git update error!"
+git status
+
+# for the contributors list in CMS
+cp -f *.csv $TARGETMAIN/uploads/grass/
+
+#configure
+echo "configuring"
+$MYMAKE distclean  > /dev/null
+configure_grass || (echo "$0: an error occured" ; exit 1)
+pwd
+ARCH=`cat config.status |grep '@host@' | cut -d'%' -f3`
+
+########  now GRASS is prepared ####################
+
+#### next compile GRASS:
+$MYMAKE
+
+
+echo "GRASS $VERSION compilation done"
+
+# now GRASS is prepared ############################################
+
+#### create module overview (https://trac.osgeo.org/grass/ticket/1203)
+#sh tools/module_synopsis.sh
+
+#### generate developer stuff: pygrass docs + gunittest docs
+# generate pyGRASS sphinx manual (in docs/html/libpython/)
+# including source code
+$MYMAKE sphinxdoclib
+
+##
+echo "Copy over the manual + pygrass HTML pages:"
+mkdir -p $TARGETHTMLDIR
+# don't destroy the addons
+mv $TARGETHTMLDIR/addons /tmp
+rm -rf $TARGETHTMLDIR/*
+mv /tmp/addons $TARGETHTMLDIR
+
+cp -rp dist.$ARCH/docs/html/* $TARGETHTMLDIR/
+echo "Copied pygrass progman to http://grass.osgeo.org/grass${VERSION}/manuals/libpython/"
+
+cp -p AUTHORS CHANGES CITING COPYING GPL.TXT INSTALL REQUIREMENTS.html $TARGETDIR/
+
+# note: addons are in grass7x compilation scripts
+
+# clean wxGUI sphinx manual etc
+(cd $GRASSBUILDDIR/ ; $MYMAKE cleansphinx)
+
+##### generate i18N POT files, needed for https://www.transifex.com/grass-gis/grass7X/
+(cd locale ;
+$MYMAKE pot
+mkdir -p $TARGETDIR/transifex/
+cp templates/*.pot $TARGETDIR/transifex/
+)
+
+##### generate i18N stats for HTML page path (WebSVN):
+## Structure:  grasslibs_ar.po 144 translated messages 326 fuzzy translations 463 untranslated messages.
+cd $GRASSBUILDDIR
+(cd locale/ ;
+touch po/*.po ;
+$MYMAKE mo > out.i18n 2>&1 ;
+# libs, mods, wx
+# hack:
+cat out.i18n | tr '\n' ' ' | sed -e 's+ msgfmt+\
+msgfmt+g' | tr -s ' ' ' ' | cut -d' ' -f5,6- | sed 's+\,++g' | sed 's+^po/++g' | grep -v 'done for' | sort > $TARGETDIR/i18n_stats.txt
+rm -f out.i18n
+)
+
+cd $TARGETDIR/
+cat i18n_stats.txt | grep mod  > i18n_stats_mods.txt
+cat i18n_stats.txt | grep lib  > i18n_stats_libs.txt
+cat i18n_stats.txt | grep wxpy > i18n_stats_wxpy.txt
+cd $GRASSBUILDDIR
+
+# package the package
+$MYMAKE bindist
+if [ $? -ne 0 ] ; then
+   halt_on_error "make bindist."
+fi
+
+#report system:
+echo "System:
+$ARCH, compiled with:" > grass-$DOTVERSION\_$ARCH\_bin.txt
+## echo "Including precompiled $GDALVERSION library for r.in.gdal" >> grass-$DOTVERSION\_$ARCH\_bin.txt
+gcc -v 2>&1 | grep -v Reading >> grass-$DOTVERSION\_$ARCH\_bin.txt
+
+# clean old version off
+rm -f $TARGETDIR/grass-$DOTVERSION\_$ARCH\_bin.txt
+rm -f $TARGETDIR/grass-$DOTVERSION*.tar.gz
+rm -f $TARGETDIR/grass-$DOTVERSION*install.sh
+
+############################################
+echo "Copy new binary version into web space:"
+cp -p grass-$DOTVERSION\_$ARCH\_bin.txt grass-$DOTVERSION*.tar.gz grass-$DOTVERSION*install.sh $TARGETDIR
+rm -f grass-$DOTVERSION\_$ARCH\_bin.txt grass-$DOTVERSION*.tar.gz grass-$DOTVERSION*install.sh
+
+# generate manual ZIP package
+(cd $TARGETHTMLDIR/.. ; rm -f $TARGETHTMLDIR/*html_manual.zip ; zip -r /tmp/grass-${DOTVERSION}_html_manual.zip manuals/)
+mv /tmp/grass-${DOTVERSION}_html_manual.zip $TARGETHTMLDIR/
+
+# error log etc
+cp -f config_$DOTVERSION.git_log.txt $TARGETDIR
+cp -f error.log $TARGETDIR
+chmod -R a+r,g+w $TARGETDIR     2> /dev/null
+chmod -R a+r,g+w $TARGETHTMLDIR 2> /dev/null
+
+echo "Written to: $TARGETDIR"
+
+############################################
+# compile addons
+cd $GRASSBUILDDIR
+sh ~/cronjobs/compile_addons_git.sh ~/src/grass-addons/grass7/ ~/src/releasebranch_7_8/dist.x86_64-pc-linux-gnu/ ~/.grass7/addons
+mkdir $TARGETHTMLDIR/addons/
+cp ~/.grass7/addons/docs/html/* $TARGETHTMLDIR/addons/
+sh ~/cronjobs/grass-addons-index.sh $TARGETHTMLDIR/addons/
+chmod -R a+r,g+w $TARGETHTMLDIR 2> /dev/null
+
+# TODO: cp logs from ~/.grass7/addons/logs/
+
+############################################
+# cleanup
+cd $GRASSBUILDDIR
+$MYMAKE distclean  > /dev/null || (echo "$0: an error occured" ; exit 1)
+
+echo "Finished GRASS $VERSION $ARCH compilation."
+echo "Written to: $TARGETDIR"
+echo "Copied HTML manual to https://grass.osgeo.org/grass${VERSION}/manuals/"
+echo "Copied pygrass progman to https://grass.osgeo.org/grass${VERSION}/manuals/libpython/"
+exit 0
+