improved formatting and filter options

cntpings.pl

@@ -9,7 +9,7 @@
 use TagTime qw(match); # should be in util.pl (or util.pl should all be here)
 
 my $start = -1;
-my $end = ts(time());
+my $end = -1; # ts(time());
 my $verbose = 0;
 GetOptions("start|s=s"=>\$start, "end|e=s"=>\$end, "verbose|v"=>\$verbose);
 $start = pd($start) unless isnum($start);
@@ -38,12 +38,14 @@
 #die "DEBUG: [", ts($start), "][", ts($end), "]\n";
 
 my %tc;            # tag counts -- hashes from tag to count.
-my $first = -1;    # timestamp of the first ping in time range.
+my $first = -1;    # timestamp of the first matching ping in time range.
+my $last  = -1;    # timestamp of the last  matching ping in time range.
 my $m = 0;         # number of pings in time range that match.
 my $n = 0;         # number of pings in time range that don't match.
 my $e = 0;         # number of lines with parse errors.
 my $toosoon = 0;   # number of pings before $start.
 my $toolate = 0;   # number of pings after $end.
+my $maybelate = 0; # number of pings since last match
 my $errstr = "";   # concatenation of bad lines from log file.
 
 my $logfile = shift;
@@ -59,41 +61,97 @@
 
   my @tags = split(/\s+/, $line);
   my $ts = shift(@tags);
-  if($first == -1 || $ts < $first) { $first = $ts; }
   if   ($ts < $start) { $toosoon++; }
-  elsif($ts > $end)   { $toolate++; }
+  elsif($ts > $end && $end != -1)   { $toolate++; }
   elsif(match($expr, $line)) {
+    if($first == -1 || $ts < $first) { $first = $ts; }
+    if($last  == -1 || $ts > $last ) { $last =  $ts; }
+    if($ts > $pingend) { $pingend = $ts; }
     $m++;
+    $maybelate = 0;
     for(@tags) { $tc{$_}++; }
-  } else { $n++; }
+  } else { 
+    $n++; 
+    $maybelate++; 
+    if($first == -1) { $toosoon++; }
+  }
 }
 $start = $first if $start == -1;
+$end   = $last  if $end   == -1;
+$toolate += $maybelate;
 
 if($e>0) { 
   print "Errors in log file: $e. ", 
         "They have to be fixed before pings can be counted:\n";
   print "\n$errstr";
   exit(1);
 }
-print "$m (", ss2($m*$gap), ") / ", $m+$n, 
+print "$m (", ss($m*$gap), ") / ", $m+$n, 
         " (", ss2(($m+$n)*$gap), " ~ ", 
-              ss2($end-$start), ") = ",
-  ($m+$n==0 ? "NaN" : round1(100*$m/($m+$n))), "% [rate: ",
-  ss(($end-$start)/($m+1)). "]\n";
+              ss2($last-$first), ") = ",
+        ($m+$n==0 ? "NaN" : round1(100*$m/($m+$n))), "%",
+        #" [rate: ", ss(($last-$first)/($m+1)). "]",
+        "\n";
   #"NomGap = ". ss2($gap) 
-if($verbose) {
+if($verbose and $end ne $start) {
+
+  my @pie = ();
+
+  my @pieline = ();
+  push @pieline, '*ALL*', $m, "100%";
+  push @pieline, ss($m*$gap),
+                 ss2($m*$gap/(($end-$start)/(24*3600))),
+                 ss2($m*$gap/(($end-$start)/(24*3600*7)));
+   push @pie, \@pieline;
+
   print "Start: ". ts($start). "  (pings before this: $toosoon)".
   "\n  End: ". ts($end).   "  (pings after this:  $toolate)\n";
   print lrjust("PIE:", "\n");
   for(sort {$tc{$b} <=> $tc{$a}} keys %tc) {
-    print "  $_ $tc{$_} = ". round1(100*$tc{$_}/$m). "% = ~";
-    if($n==0) {
-      print ss2($tc{$_}/$m*($end-$start)). " = ".
-            ss2($tc{$_}/$m*24*3600). "/day\n";
+
+   my @pieline = ();
+
+# FIXME something horrible happens if 100% of the lines match
+
+   push @pieline, $_, $tc{$_}, round1(100*$tc{$_}/$m)."%";
+   if($n==0) {
+      push @pieline, ss($tc{$_}/$m*($end-$start)),
+                     ss2($tc{$_}/$m*24*3600),
+                     ss2($tc{$_}/$m*24*3600*7);
     } else {
-      print ss($tc{$_}*$gap). " = ".
-           ss2($tc{$_}*$gap/(($end-$start)/(24*3600))). "/day\n";
+      push @pieline, ss($tc{$_}*$gap),
+                     ss2($tc{$_}*$gap/(($end-$start)/(24*3600))),
+		     ss2($tc{$_}*$gap/(($end-$start)/(24*3600*7)));
     }
+   push @pie, \@pieline;
+
+#    print "  $_ $tc{$_} = ". round1(100*$tc{$_}/$m). "% = ~";
+#    if($n==0) {
+#      print ss2($tc{$_}/$m*($last-$first)). " = ".
+#            ss2($tc{$_}/$m*24*3600). "/day = ",
+#            ss2($tc{$_}/$m*24*3600*7). "/week",
+#            "\n";
+#    } else {
+#      print ss($tc{$_}*$gap). " = ".
+#            ss2($tc{$_}*$gap/(($last-$first)/(24*3600))). "/day = ",
+#            ss2($tc{$_}*$gap/(($last-$first)/(24*3600*7))). "/week",
+#            "\n";
+#    }
+
+
+  }
+
+  format HEADER =
+  tag         pings  pct   total time  time/day  time/week
+.
+  $^ = 'HEADER';
+  for my $pieline (@pie) {
+    format STDOUT =
+  @<<<<<<<<<< ^#### @>>> @>>>>>>>>>>> @>>>>>>>> @>>>>>>>>> 
+          @$pieline
+.
+    write;
+    $- = 10000000; # only one header please
   }
 }
 close(LOG);

cntpings.sh

@@ -0,0 +1,193 @@
+#!/bin/sh 
+
+# USAGE: cntpings.sh --help
+#        Additional wrapper around cntpings.pl
+#
+# AUTHOR:  Philip Hellyer 
+# URL:     https://github.com/pjjh/TagTime
+# FILE:    cntpings.sh
+# LICENSE: Licensed on the same terms as TagTime itself
+#          Copyright 2014 Philip Hellyer
+
+# DEPENDENCIES & LIMITATIONS
+# Relies on newstyle formatted output from cntpings.pl
+# Parameters intended for the .sh must appear before ones for the .pl
+# --help tails the output from cntpings.pl, to include relevant usage
+# TagTime :)
+
+## CONFIG ##
+
+# Default tagtime log file
+# N.B. If you're merging files, any unmerged pings won't be reflected
+LOG='merged.log'
+
+# Exclude these tags unless --all
+FILTER='afk|off|RETRO|prowl'
+
+# A predefined tag set, e.g. for notionally productive time
+# TODO somehow reuse bmndr tags defined in settings.pl
+NWO='(nnj|gc|profdev|bmndr|ntwk|tock|mit|conf|fv|dj) & !(social|idle|avoid)'
+# Exclude these percentages by default 
+SHARE='01234'
+
+### END CONFIG ###
+
+
+if [ "$1" = '--help' ] ; then
+  echo "USAGE: $0 [logfile] [--timeline] [options] [boolean expression]"
+  echo "    (automatically filters $FILTER unless --all)"
+  echo "Available options:"
+  echo "    --help: this text"
+  echo "    --nwo:  report on my predefined tag set"
+  echo "    --all:  report on all tags regardless of % share"
+  echo "    --zero: only include tags with at least 0% share (i.e. --all)"
+  echo "    --one:  only include tags with at least 1% share"
+  echo "    --two:  only include tags with at least 2% share"
+  echo "    --five: only include tags with at least 5% share (DEFAULT)"
+  echo "    --today:     report on today's pings"
+  echo "    --yesterday: report on yesterday's pings"
+  echo "    --week:      report on the last week"
+  echo "    --month:     report on the last month"
+  echo "    --timeline:  report on multiple periods"
+  echo "                 (reveals all --time-period options)"
+  cntpings.pl 2>&1 | tail
+  exit 0
+fi
+
+
+# Start date, if specified by --week --month --quarter --year
+START=''
+
+# Predefined Tag sets. TODO move to user defined variable in .profile
+TAGS=''
+
+# Logfile specified on command line
+if [ -f "$1" ] ; then
+  LOG="$1"
+  shift
+fi
+
+# Timeline mode to show history of pings
+if [ "$1" = '--timeline' ] ; then
+  shift
+  # headers
+  $0 --today "$@" | grep 'time/day' | sed 's/tag   /period/'
+  # periods
+  for period in 'week' 'fortnight' 'month' 'quarter' 'halfyear' 'year'; do
+    # SED magic adjusts the spacing in the output depending on the length of $period
+    $0 "--$period" "$@" | grep '*ALL*' | sed "s/ALL/$period/" | sed "s/\* \{$((${#period}-3))\}/*/"
+  done
+  # epoch - all available data
+  $0 "$@" | grep '*ALL*' 
+  exit 0
+fi
+
+
+# Order-Independent Parameter Processing
+
+while true ; do
+  if [ "$1" = '--nwo' ] ; then
+    TAGS="$NWO"
+    shift
+  elif [ "$1" = '--all' -o "$1" = '--zero' ] ; then
+    SHARE=''
+    FILTER=''
+    shift
+  elif [ "$1" = '--one' ] ; then
+    SHARE='0'
+    shift
+  elif [ "$1" = '--two' ] ; then
+    SHARE='01'
+    shift
+  elif [ "$1" = '--three' ] ; then
+    SHARE='012'
+    shift
+  elif [ "$1" = '--four' ] ; then
+    SHARE='0123'
+    shift
+  elif [ "$1" = '--five' ] ; then
+    SHARE='01234'
+    shift
+  elif [ "$1" = '--today' ] ; then
+    START="-s $(date +%C%y-%m-%d)"
+    shift
+  elif [ "$1" = '--yesterday' ] ; then
+    START="-s $(date -v-1d +%C%y-%m-%d) -e $(date +%C%y-%m-%d)"
+    shift
+  elif [ "$1" = '--week' ] ; then
+    START="-s $(date -v-7d +%C%y-%m-%d)"
+    shift
+  elif [ "$1" = '--fortnight' ] ; then
+    START="-s $(date -v-14d +%C%y-%m-%d)"
+    shift
+  elif [ "$1" = '--month' ] ; then
+    START="-s $(date -v-1m +%C%y-%m-%d)"
+    shift
+  elif [ "$1" = '--quarter' ] ; then
+    START="-s $(date -v-3m +%C%y-%m-%d)"
+    shift
+  elif [ "$1" = '--halfyear' ] ; then
+    START="-s $(date -v-6m +%C%y-%m-%d)"
+    shift
+  elif [ "$1" = '--year' ] ; then
+    START="-s $(date -v-1y +%C%y-%m-%d)"
+    shift
+  else
+    break
+  fi
+done
+
+
+if   [ -n "$SHARE" -a -n "$FILTER" ] ; then
+  cntpings.pl -v "$LOG" $START "$TAGS" "$@" | grep -vE " [$SHARE]% " | grep -vE "$FILTER"
+elif [ -z "$SHARE" -a -n "$FILTER" ] ; then
+  cntpings.pl -v "$LOG" $START "$TAGS" "$@" |                          grep -vE "$FILTER"
+elif [ -n "$SHARE" -a -z "$FILTER" ] ; then
+  cntpings.pl -v "$LOG" $START "$TAGS" "$@" | grep -vE " [$SHARE]% " 
+else
+  cntpings.pl -v "$LOG" $START "$TAGS" "$@"
+fi
+
+exit
+
+
+if [ "$1" = '--all' ] ; then
+  shift
+  cntpings.pl -v "$LOG" "$@" 
+elif [ "$1" = '--zero' ] ; then
+  shift
+  SHARE='0'
+  cntpings.pl -v "$LOG" "$@" | grep -vE ' 0% ' | grep -vE "$FILTER"
+elif [ "$1" = '--one' ] ; then
+  shift
+  SHARE='01'
+  cntpings.pl -v "$LOG" "$@" | grep -vE ' [01]% ' | grep -vE "$FILTER"
+elif [ "$1" = '--week' ] ; then
+  shift
+  START="-s $(date -v-7d +%C%y-%m-%d)"
+  cntpings.pl -v "$LOG" -s $(date -v-7d +%C%y-%m-%d) "$@" | grep -vE ' [01234]% ' | grep -vE "$FILTER"
+elif [ "$1" = '--month' ] ; then
+  shift
+  cntpings.pl -v "$LOG" -s $(date -v-30d +%C%y-%m-%d) "$@" | grep -vE ' [01234]% ' | grep -vE "$FILTER"
+elif [ "$1" = '--quarter' ] ; then
+  shift
+  cntpings.pl -v "$LOG" -s $(date -v-90d +%C%y-%m-%d) "$@" | grep -vE ' [01234]% ' | grep -vE "$FILTER"
+else
+  cntpings.pl -v "$LOG" "$@" | grep -vE ' [01234]% ' | grep -vE "$FILTER"
+fi
+
+
+# This finds all tags registered in the last week...
+# cntpings.sh --all -s $(date -v-7d +%C%y-%m-%d) | awk '{print $1}' | grep -v '[:*+]' | sort
+
+# The harder question is what to do with this, and how to see changes.
+
+# Taxonomy demainds monitoring for:
+# - tags never seen before
+# - tags not seen for some time
+# - tags with the wrong year(?)
+
+# man comm
+
+
+