introduced fast forward/resume

danker/danker.py

@@ -80,6 +80,7 @@
 """
 import sys
 import time
+import datetime
 import argparse
 
 # import memory_profiler
@@ -333,7 +334,7 @@ def _main():
         sys.exit(1)
     print(
         f"danker: starting computation of PageRank on '{args.left_sorted}' with parameters\n\t"
-        f"{param_out}",
+        f"{param_out} ({datetime.datetime.now()})",
         file=sys.stderr,
     )
     start = time.time()
@@ -356,7 +357,8 @@ def _main():
         danker_bigmem(dictionary, args.iterations, args.damping)
 
     print(
-        "danker: PageRank computation took " f"{time.time() - start:.2f} seconds.",
+        "danker: PageRank computation took "
+        f"{time.time() - start:.2f} seconds ({datetime.datetime.now()}).",
         file=sys.stderr,
     )
 

script/create_links.sh

@@ -30,7 +30,7 @@ while getopts ":d:f:k" a; do
         f)
             folder=${OPTARG}
             ;;
-	k)
+        k)
 	    keep_site_links=1
 	    ;;
         *)
@@ -111,6 +111,14 @@ if [ ! "$dump_date" ]; then
     dump_date=$(latest_dump) || exit 1
 fi
 
+# Skip if file exists
+if [ -f "$wiki-$dump_date"".links" ]; then
+    echo "$wiki-$dump_date.links exists... skipping" 1>&2
+    echo "$wiki-$dump_date"".links"
+    exit 0
+fi
+
+
 # File names are now fully specified
 page="$wiki-""$dump_date""-page.sql"
 pagelinks="$wiki-""$dump_date""-pagelinks.sql"

script/dank.sh

@@ -15,7 +15,6 @@
 #
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-set -e
 
 # Check for environment variables for sort
 if [ -z ${MEM_PERC+x} ]; then