tools/cronjobs_osgeo_lxd: fix Add-On src and commits url in the manua…

…l html page (#307) * tools/cronjobs_osgeo_lxd: fix Add-On src and commits url in the manual html page * tools/cronjobs_osgeo_lxd: add checking Add-On source and commits url existence in the html manual page * tools/cronjobs_osgeo_lxd: fix call 'g.extension' module with '-j' flag
OSGeo · Oct 16, 2020 · abfd403 · abfd403
1 parent 3c668b1
commit abfd403
Show file tree

Hide file tree

Showing 3 changed files with 134 additions and 41 deletions.
diff --git a/tools/cronjobs_osgeo_lxd/check_addons_urls.sh b/tools/cronjobs_osgeo_lxd/check_addons_urls.sh
@@ -0,0 +1,63 @@
+#!/bin/sh
+
+# Tomas Zigo, 2020
+
+# This script check compiled Add-Ons html manual pages for existence
+# correct source and commits URL
+
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
+    echo "usage: $0 addons_docs_html_dir_path, log_file_path \
+log_html_file_path"
+    echo "eg. $0 ~/.grass7/addons/docs/html \
+~/.grass7/addons/logs/index_manual_pages.log \
+~/.grass7/addons/logs/index_manual_pages.html"
+    exit 1
+fi
+
+ADDONS_DOCS_HTML_DIR_PATH=$1
+LOG_FILE_PATH=$2
+LOG_HTML_FILE_PATH=$3
+
+SRC_URL_ID="tree"
+COMMITS_URL_ID="commits"
+
+NORMAL_COLOR=$(tput sgr0)
+RED_COLOR=$(tput setaf 1)
+
+check_addon_html_manual_page() {
+    for i in "${!urls[@]}"; do
+        IFS=':'
+        read -a template_page_url <<< "${urls[$i]}"
+
+        found_urls=$(grep -woe "$SRC_URL_ID" <<< "${urls[$i]}" \
+                          -woe "$COMMITS_URL_ID" <<< "${urls[$i]}" | wc -l)
+        pgm=$(basename "${template_page_url[0]}")
+        echo "<tr><td><tt>$pgm</tt></td>" \
+             >> "$LOG_HTML_FILE_PATH"
+        if [ "$found_urls" -eq 2 ]; then
+            # Stdout, log file
+            echo "${NORMAL_COLOR}Checking $pgm... \
+source and commits URL: CORRECT" | tee -a "$LOG_FILE_PATH"
+            # Html file
+            echo "<td style=\"background-color: green\">\
+source and commits URL: CORRECT</td>" >> "$LOG_HTML_FILE_PATH"
+        else
+            # Stdout, log file
+            echo "${RED_COLOR}Checking $pgm... source or \
+commits URL: INCORRECT" | tee -a "$LOG_FILE_PATH"
+            # Html file
+            echo "<td style=\"background-color: red\">\
+source or commits URL: INCORRECT</td>" >> "$LOG_HTML_FILE_PATH"
+        fi;
+        tput sgr0
+   done;
+}
+
+manual_html_pages=$(find "$ADDONS_DOCS_HTML_DIR_PATH" -type f \
+                         -regex ".*.html" | sort | \
+                        xargs grep -H "Available at:")
+
+echo "${manual_html_pages}" | \
+    { IFS=$'\n' read -rd '' -a urls; check_addon_html_manual_page; }
+
+exit 0
diff --git a/tools/cronjobs_osgeo_lxd/compile_addons_git.sh b/tools/cronjobs_osgeo_lxd/compile_addons_git.sh
@@ -6,23 +6,34 @@
 # This script compiles GRASS Addons, it's called by cron_grass78_releasebranch_78_build_bins.sh | cron_grass7_HEAD_build_bins.sh
 
 if [ -z "$3" ]; then
-    echo "usage: $0 git_path topdir addons_path [separate]"
-    echo "eg. $0 ~/src/grass_addons/grass7/ ~/src/grass_trunk/dist.x86_64-pc-linux-gnu ~/.grass7/addons"
+    echo "usage: $0 git_path topdir addons_path grass_startup_program [separate]"
+    echo "eg. $0 ~/src/grass_addons/grass7/ \
+~/src/releasebranch_7_8/dist.x86_64-pc-linux-gnu \
+~/.grass7/addons \
+~/src/releasebranch_7_8/bin.x86_64-pc-linux-gnu/grass78"
     exit 1
 fi
 
 GIT_PATH="$1"
 TOPDIR="$2"
 ADDON_PATH="$3"
+GRASS_STARTUP_PROGRAM="$4"
 #GRASS_VERSION=`echo -n ${GIT_PATH} | tail -c 1`
 GRASS_VERSION=7
 INDEX_FILE="index"
+INDEX_MANUAL_PAGES_FILE="index_manual_pages"
+ADDONS_PATHS_JSON_FILE="addons_paths.json"
 
 if [ ! -d "$3" ] ; then
     mkdir -p "$3"
 fi
 
-if [ -n "$4" ] ; then
+if [ -z "$4" ] ; then
+    echo "Set GRASS startup program arg parameter path"
+    exit 1
+fi
+
+if [ -n "$5" ] ; then
     SEP=1 # useful for collecting files (see build-xml.py)
 else
     SEP=0
@@ -44,9 +55,9 @@ uname=`uname`
 mkdir "$ADDON_PATH/logs"
 touch "$ADDON_PATH/logs/${INDEX_FILE}.log"
 
-echo "<!--<?xml-stylesheet href=\"style.css\" type=\"text/css\"?>-->
+html_template="<!--<?xml-stylesheet href=\"style.css\" type=\"text/css\"?>-->
 <!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.1//EN\"
-	  \"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd\">
+      \"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd\">
 
 <html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" >
 
@@ -67,11 +78,15 @@ border: 1px solid black;
 </head>
 <body>
 <h1>GRASS $GRASS_VERSION Addons ($PLATFORM) / $uname (logs generated $date)</h1>
-<hr /> 
+<hr />
 <table cellpadding=\"5\">
 <tr><th style=\"background-color: grey\">AddOns</th>
-<th style=\"background-color: grey\">Status</th>
-<th style=\"background-color: grey\">Log file</th></tr>" > "$ADDON_PATH/logs/${INDEX_FILE}.html"
+<th style=\"background-color: grey\">Status</th>"
+
+echo "$html_template" > "$ADDON_PATH/logs/${INDEX_FILE}.html"
+echo "<th style=\"background-color: grey\">Log file</th></tr>" > "$ADDON_PATH/logs/${INDEX_FILE}.html"
+
+echo "$html_template" > "$ADDON_PATH/logs/${INDEX_MANUAL_PAGES_FILE}.html"
 
 echo "-----------------------------------------------------"
 echo "Addons '$ADDON_PATH'..."
@@ -82,41 +97,45 @@ pwd=`pwd`
 # from ../../grass7/
 for c in "db" "display" "general" "gui/wxpython" "imagery" "misc" "raster" "raster3d" "temporal" "vector" ; do
     if [ ! -d $c ]; then
-	continue
+        continue
     fi
     cd $c
     for m in `ls -d */Makefile 2>/dev/null` ; do
-	m="${m%%/Makefile}"
-	echo -n "Compiling $m..."
-	cd "$m"
-	if [ $SEP -eq 1 ] ; then
-	    path="$ADDON_PATH/$m"
-	else
-	    path="$ADDON_PATH"
-	fi
-
-	export GRASS_ADDON_BASE=$path
-	echo "<tr><td><tt>$c/$m</tt></td>" >> "$ADDON_PATH/logs/${INDEX_FILE}.html"
-	make MODULE_TOPDIR="$TOPDIR" clean > /dev/null 2>&1
-	make MODULE_TOPDIR="$TOPDIR" \
-	    BIN="$path/bin" \
-	    HTMLDIR="$path/docs/html" \
-	    MANBASEDIR="$path/docs/man" \
-	    SCRIPTDIR="$path/scripts" \
-	    ETC="$path/etc" \
+    m="${m%%/Makefile}"
+    echo -n "Compiling $m..."
+    cd "$m"
+    if [ $SEP -eq 1 ] ; then
+        path="$ADDON_PATH/$m"
+    else
+        path="$ADDON_PATH"
+    fi
+
+    export GRASS_ADDON_BASE=$path
+    # Try download Add-Ons json file paths
+    if [ ! -f  "$GRASS_ADDON_BASE/$ADDONS_PATHS_JSON_FILE" ]; then
+        $GRASS_STARTUP_PROGRAM --tmp-location EPSG:4326 --exec g.extension -j > /dev/null 2>&1
+    fi
+    echo "<tr><td><tt>$c/$m</tt></td>" >> "$ADDON_PATH/logs/${INDEX_FILE}.html"
+    make MODULE_TOPDIR="$TOPDIR" clean > /dev/null 2>&1
+    make MODULE_TOPDIR="$TOPDIR" \
+        BIN="$path/bin" \
+        HTMLDIR="$path/docs/html" \
+        MANBASEDIR="$path/docs/man" \
+        SCRIPTDIR="$path/scripts" \
+        ETC="$path/etc" \
             SOURCE_URL="https://github.com/OSGeo/grass-addons/tree/master/grass${GRASS_VERSION}/" > \
             "$ADDON_PATH/logs/$m.log" 2>&1
-	if [ `echo $?` -eq 0 ] ; then
-	    printf "%-30s%s\n" "$c/$m" "SUCCESS" >> "$ADDON_PATH/logs/${INDEX_FILE}.log"
-	    echo " SUCCESS"
-	    echo "<td style=\"background-color: green\">SUCCESS</td>" >> "$ADDON_PATH/logs/${INDEX_FILE}.html"
-	else
-	    printf "%-30s%s\n" "$c/$m" "FAILED" >> "$ADDON_PATH/logs/${INDEX_FILE}.log"
-	    echo " FAILED"
-	    echo "<td style=\"background-color: red\">FAILED</td>" >> "$ADDON_PATH/logs/${INDEX_FILE}.html"
-	fi
-	echo "<td><a href=\"$m.log\">log</a></td></tr>" >> "$ADDON_PATH/logs/${INDEX_FILE}.html"
-	cd ..
+    if [ `echo $?` -eq 0 ] ; then
+        printf "%-30s%s\n" "$c/$m" "SUCCESS" >> "$ADDON_PATH/logs/${INDEX_FILE}.log"
+        echo " SUCCESS"
+        echo "<td style=\"background-color: green\">SUCCESS</td>" >> "$ADDON_PATH/logs/${INDEX_FILE}.html"
+    else
+        printf "%-30s%s\n" "$c/$m" "FAILED" >> "$ADDON_PATH/logs/${INDEX_FILE}.log"
+        echo " FAILED"
+        echo "<td style=\"background-color: red\">FAILED</td>" >> "$ADDON_PATH/logs/${INDEX_FILE}.html"
+    fi
+    echo "<td><a href=\"$m.log\">log</a></td></tr>" >> "$ADDON_PATH/logs/${INDEX_FILE}.html"
+    cd ..
     done
     cd $pwd
     unset GRASS_ADDON_BASE
@@ -126,6 +145,15 @@ echo "</table><hr />
 <div style=\"text-align: right\">Valid: <a href=\"http://validator.w3.org/check/referer\">XHTML</a></div>
 </body></html>" >> "$ADDON_PATH/logs/${INDEX_FILE}.html"
 
+echo ""
+sh ~/cronjobs/check_addons_urls.sh "$ADDON_PATH/docs/html" \
+   "$ADDON_PATH/logs/${INDEX_MANUAL_PAGES_FILE}.log" \
+   "$ADDON_PATH/logs/${INDEX_MANUAL_PAGES_FILE}.html"
+echo ""
+
+echo "</table><hr />
+<div style=\"text-align: right\">Valid: <a href=\"http://validator.w3.org/check/referer\">XHTML</a></div>
+</body></html>" >> "$ADDON_PATH/logs/${INDEX_MANUAL_PAGES_FILE}.html"
+
 echo "Log file written to <$ADDON_PATH/logs/>"
 exit 0
-
diff --git a/tools/cronjobs_osgeo_lxd/cron_grass78_releasebranch_78_build_bins.sh b/tools/cronjobs_osgeo_lxd/cron_grass78_releasebranch_78_build_bins.sh
@@ -248,7 +248,10 @@ echo "Written to: $TARGETDIR"
 (cd ~/src/grass-addons/grass7/ ; git pull origin master)
 # compile addons
 cd $GRASSBUILDDIR
-sh ~/cronjobs/compile_addons_git.sh ~/src/grass-addons/grass7/ ~/src/releasebranch_7_8/dist.x86_64-pc-linux-gnu/ ~/.grass7/addons
+sh ~/cronjobs/compile_addons_git.sh ~/src/grass-addons/grass7/ \
+   ~/src/releasebranch_7_8/dist.x86_64-pc-linux-gnu/ \
+   ~/.grass7/addons \
+   ~/src/releasebranch_7_8/bin.x86_64-pc-linux-gnu/grass$VERSION
 mkdir -p $TARGETHTMLDIR/addons/
 cp ~/.grass7/addons/docs/html/* $TARGETHTMLDIR/addons/
 sh ~/cronjobs/grass-addons-index.sh $TARGETHTMLDIR/addons/
@@ -271,4 +274,3 @@ echo "Written to: $TARGETDIR"
 echo "Copied HTML manual to https://grass.osgeo.org/grass${VERSION}/manuals/"
 echo "Copied pygrass progman to https://grass.osgeo.org/grass${VERSION}/manuals/libpython/"
 exit 0
-