Item12739: Item12888: Spreadsheet enhancements

The TestCases/SpreadSheetPluginTestCases are failing: - There were some ISO-8858-1 characters that need to be utf-8 - Tests were added for functions that were not implemented in Foswiki - Several tests showed a lack of checking, resulted in warnings / crashes This adds missing functions, and unit tests: - FILTER - ISDIGIT - ISLOWER - ISUPPER - ISWIKIWORD The BITXOR test cases in TestCases web still fails - It's probably a bogus function on a unicode core. We should probabaly remove the function. The TIMEADD tests fail, might be due to local timezone?
foswiki · May 29, 2015 · b5f5d88 · b5f5d88
1 parent fac6135
commit b5f5d88
Show file tree

Hide file tree

Showing 5 changed files with 145 additions and 32 deletions.
diff --git a/SpreadSheetPlugin/data/System/SpreadSheetPlugin.txt b/SpreadSheetPlugin/data/System/SpreadSheetPlugin.txt
@@ -1,4 +1,4 @@
-%META:TOPICINFO{author="ProjectContributor" date="1426439233" format="1.1" version="1"}%
+%META:TOPICINFO{author="ProjectContributor" date="1432867135" format="1.1" version="1"}%
 %META:TOPICPARENT{name="Plugins"}%
 ---+!! Foswiki Spreadsheet Plugin
 
@@ -249,6 +249,19 @@ This function has two modes of operation.
    * Example: ==%<nop>CALC{"$FIND(@, fluffy, 1)"}%== returns ==0==
    * Related: =[[#FuncINSERTSTRING][$INSERTSTRING()]]=, =[[#FuncLEFTSTRING][$LEFTSTRING()]]=, =[[#FuncREPLACE][$REPLACE()]]=, =[[#FuncRIGHTSTRING][$RIGHTSTRING()]]=, =[[#FuncSUBSTRING][$SUBSTRING()]]=, =[[#FuncSEARCH][$SEARCH()]]=
 
+#FuncFILTER
+---+++ FILTER( expression, text ) -- filter out characters from text
+   * Remove characters from a =text= string. The filter is applied multiple times.
+   * The =expression= can be a sequence of characters or a %SYSTEMWEB%.RegularExpression. Use tokens in the =expression= if needed: =$comma= for comma, =$sp= for space. The =text= may contain commas.
+   * Syntax: ==$FILTER( chars, text )==
+   * Example: ==%<nop>CALCULATE{$FILTER(f, fluffy)}%== returns ==luy== - filter out a character multiple times
+   * Example: ==%<nop>CALCULATE{$FILTER(an Franc, San Francisco)}%== returns ==Sisco== - cut a string
+   * Example: ==%<nop>CALCULATE{$FILTER($sp, Cat and Mouse)}%== returns ==CatandMouse== - remove all spaces
+   * Example: ==%<nop>CALCULATE{$FILTER([^0-9], Project-ID-1234)}%== returns ==1234== - filter in digits, e.g. keep only digits
+   * Example: ==%<nop>CALCULATE{$FILTER([^a-zA-Z0-9 ], Stupid mistake*%@^! Fixed)}%== returns ==Stupid mistake Fixed== - keep only alphanumeric characters and spaces
+   * Example: ==%<nop>CALCULATE{$FILTER([^a-zA-Z0-9], $PROPER(an EXELLENT idea.))}%== returns ==AnExcellentIdea== - turn a string into a %SYSTEMWEB%.WikiWord topic name
+   * Related: =[[#FuncFIND][$FIND()]]=, =[[#FuncINSERTSTRING][$INSERTSTRING()]]=, =[[#FuncLEFTSTRING][$LEFTSTRING()]]=, =[[#FuncPROPER][$PROPER()]]=, =[[#FuncREPLACE][$REPLACE()]]=, =[[#FuncRIGHTSTRING][$RIGHTSTRING()]]=, =[[#FuncSEARCH][$SEARCH()]]=, =[[#FuncSUBSTITUTE][$SUBSTITUTE()]]=, =[[#FuncSUBSTRING][$SUBSTRING()]]=, =[[#FuncTRANSLATE][$TRANSLATE()]]=, =[[#FuncTRIM][$TRIM()]]=
+
 #FuncFLOOR
 ---+++ FLOOR( num ) -- return the largest integer preceding a number
    * The floor(x) is the largest integer not greater than x
@@ -385,6 +398,40 @@ This function has two modes of operation.
    * Example: ==%<nop>CALC{"$INT($VALUE(09))"}%== returns ==9==
    * Related: =[[#FuncCEILING][$CEILING()]]=, =[[#FuncEVAL][$EVAL()]]=, =[[#FuncFLOOR][$FLOOR()]]=, =[[#FuncROUND][$ROUND()]]=, =[[#FuncVALUE][$VALUE()]]=
 
+#FuncISDIGIT
+---+++ ISDIGIT( text ) -- test for digits
+   * Test for one or more digits (0...9)
+   * Syntax: ==$ISDIGIT( text )==
+   * Example: ==%<nop>CALCULATE{$ISDIGIT(123)}%== returns ==1==
+   * Example: ==%<nop>CALCULATE{$ISDIGIT(-7)}%== returns ==0==
+   * Related: =[[#FuncABS][$ABS()]]=, =[[#FuncEMPTY][$EMPTY()]]=, =[[#FuncEVEN][$EVEN()]]=, =[[#FuncEXACT][$EXACT()]]=, =[[#FuncIF][$IF()]]=, =[[#FuncISDIGIT][$ISDIGIT()]]=, =[[#FuncMOD][$MOD()]]=, =[[#FuncODD][$ODD()]]=, =[[#FuncSIGN][$SIGN()]]=, =[[#FuncVALUE][$VALUE()]]=
+
+#FuncISLOWER
+---+++ ISLOWER( text ) -- test for lower case text
+   * Syntax: ==$ISLOWER( text )==
+   * Example: ==%<nop>CALCULATE{$ISLOWER(apple)}%== returns ==1==
+   * Example: ==%<nop>CALCULATE{$ISLOWER(apple tree)}%== returns ==0== (text contains a space character)
+   * Example: ==%<nop>CALCULATE{$ISLOWER(ORANGE)}%== returns ==0==
+   * Related: =[[#FuncEMPTY][$EMPTY()]]=, =[[#FuncEXACT][$EXACT()]]=, =[[#FuncLOWER][$LOWER()]]=, =[[#FuncIF][$IF()]]=, =[[#FuncISDIGIT][$ISDIGIT()]]=, =[[#FuncISUPPER][$ISUPPER()]]=, =[[#FuncISWIKIWORD][$ISWIKIWORD()]]=, =[[#FuncUPPER][$UPPER()]]=
+
+#FuncISUPPER
+---+++ ISUPPER( text ) -- test for upper case text
+   * Can be used to test for ACRONYMS
+   * Syntax: ==$ISUPPER( text )==
+   * Example: ==%<nop>CALCULATE{$ISUPPER(apple)}%== returns ==0==
+   * Example: ==%<nop>CALCULATE{$ISUPPER(ORANGE)}%== returns ==1==
+   * Example: ==%<nop>CALCULATE{$ISUPPER(ORANGE GARDEN)}%== returns ==0== (text contains a space character)
+   * Related: =[[#FuncEMPTY][$EMPTY()]]=, =[[#FuncEXACT][$EXACT()]]=, =[[#FuncLOWER][$LOWER()]]=, =[[#FuncIF][$IF()]]=, =[[#FuncISDIGIT][$ISDIGIT()]]=, =[[#FuncISLOWER][$ISLOWER()]]=, =[[#FuncISWIKIWORD][$ISWIKIWORD()]]=, =[[#FuncUPPER][$UPPER()]]=
+
+#FuncISWIKIWORD
+---+++ ISWIKIWORD( text ) -- test for !WikiWord
+   * A %SYSTEMWEB%.WikiWord has a sequence of UPPER, lower/digit, UPPER, optional mixed case alphanumeric characters
+   * Can be used together with =$ISUPPER()= to test for valid topic names
+   * Syntax: ==$ISWIKIWORD( text )==
+   * Example: ==%<nop>CALCULATE{$ISWIKIWORD(<nop>GoldenGate)}%== returns ==1==
+   * Example: ==%<nop>CALCULATE{$ISWIKIWORD(whiteRafting)}%== returns ==0==
+   * Related: =[[#FuncEMPTY][$EMPTY()]]=, =[[#FuncEXISTS][$EXISTS()]]=, =[[#FuncEXACT][$EXACT()]]=, =[[#FuncIF][$IF()]]=, =[[#FuncISDIGIT][$ISDIGIT()]]=, =[[#FuncISLOWER][$ISLOWER()]]=, =[[#FuncISUPPER][$ISUPPER()]]=, =[[#FuncPROPER][$PROPER()]]=, =[[#FuncPROPERSPACE][$PROPERSPACE()]]=
+
 #FuncLEFT
 ---+++ LEFT( ) -- address range of cells to the left of the current cell
    * Syntax: ==$LEFT( )==
@@ -947,7 +994,8 @@ Note that the =DONTSPACE= global preference overrides the =SPREADSHEETPLUGIN_DON
 ---++ Info
 
 |  Change History: | <!-- specify latest version first -->&nbsp; |
-|  02 Jan 2015: | Foswikitask:Item12739: fixed use of uninitialized value, Add triple-quote for escaped strings,  Add RANDSTRING() function.<br />\
+|  28 May 2015: (1.19) | Foswikitask:item12739: Added ISDIGIT(), ISLOWER(), ISUPPER(), ISWIKIWORD() and FILTER() functions.
+|  02 Jan 2015: (1.18) | Foswikitask:Item12739: fixed use of uninitialized value, Add triple-quote for escaped strings,  Add RANDSTRING() function.<br />\
                   Add the base conversion functions: <noautolink> DEC2BIN() BIN2DEC() DEC2HEX() HEX2DEC() DEC2OCT() and OCT2DEC().</noautolink> <br />\
                   Add LISTEACH() as an alias for LISTMAP() |
 |  05 Nov 2012: | Foswikitask:Item8417: add VarCALCULATE macro to be used within the normal macro evaluation order. (name chosen for comaptibility ) |

diff --git a/SpreadSheetPlugin/lib/Foswiki/Plugins/SpreadSheetPlugin.pm b/SpreadSheetPlugin/lib/Foswiki/Plugins/SpreadSheetPlugin.pm
@@ -12,8 +12,8 @@ use vars qw(
   $web $topic $user $installWeb $debug $skipInclude $doInit
 );
 
-our $VERSION           = '1.18';
-our $RELEASE           = '1.18';
+our $VERSION           = '1.19';
+our $RELEASE           = '1.19';
 our $NO_PREFS_IN_TOPIC = 1;
 our $SHORTDESCRIPTION =
 'Add spreadsheet calculations like "$SUM($ABOVE())" to Foswiki tables and other topic text';

diff --git a/SpreadSheetPlugin/lib/Foswiki/Plugins/SpreadSheetPlugin/Calc.pm b/SpreadSheetPlugin/lib/Foswiki/Plugins/SpreadSheetPlugin/Calc.pm
@@ -150,6 +150,7 @@ sub _doCalc {
         $recurseFunc = \&_recurseFuncCutWhitespace;
     }
     else {
+
 # recursively evaluate functions without removing white space (compatible with old spec)
         $recurseFunc = \&_recurseFunc;
     }
@@ -205,8 +206,10 @@ sub _addNestingLevel {
 sub _recurseFunc {
 
     # Handle functions recursively
+    no warnings 'uninitialized';
     $_[0] =~
 s/\$([A-Z]+[A-Z0-9]*)$escToken([0-9]+)\((.*?)$escToken\2\)/_doFunc($1,$3)/geos;
+    use warnings 'uninitialized';
 
     # Clean up unbalanced mess
     $_[0] =~ s/$escToken\-*[0-9]+([\(\)])/$1/go;
@@ -250,7 +253,7 @@ my $Function = {
     EXEC             => \&_EXEC,
     EXISTS           => sub { ( Foswiki::Func::topicExists( $web, $_[0] ) ) ? 1 : 0 },
     EXP              => sub { exp( _getNumber($_[0]) ) },
-    # FILTER - Filter out characters from a string.
+    FILTER           => \&_FILTER,
     FIND             => \&_FIND,
     FLOOR            => \&_FLOOR,
     FORMAT           => \&_FORMAT,
@@ -274,10 +277,10 @@ my $Function = {
                         my $rslt = _safeEvalPerl($_[0]);
                         return ( $rslt =~ /^ERROR/ ) ? $rslt : int( _getNumber($rslt) );
                        },
-    # ISDIGIT
-    # ISLOWER
-    # ISUPPER
-    # ISWIKIWORD
+    ISDIGIT          => sub { ($_[0] =~ m/^[[:digit:]]+$/ ) ? 1 : 0 },
+    ISLOWER          => sub { ($_[0] =~ m/^[[:lower:]]+$/ ) ? 1 : 0 },
+    ISUPPER          => sub { ($_[0] =~ m/^[[:upper:]]+$/ ) ? 1 : 0 },
+    ISWIKIWORD       => sub { (Foswiki::isValidWikiWord( $_[0] ) ) ? 1 : 0 },
     LEFT             => sub { my $i = $rPos + 1; return "R$i:C1..R$i:C$cPos" },
     LEFTSTRING       => \&_LEFTSTRING,
     LENGTH           => sub { length( $_[0] ) },
@@ -380,7 +383,6 @@ my $Function = {
     XOR              => \&_XOR,
 };
 #>>>
-
 $Function->{MIDSTRING} = $Function->{SUBSTRING};    # MIDSTRING Undocumented
 $Function->{DURATION} = $Function->{SUMDAYS};  # DURATION undocumented, for Sven
 $Function->{MULT}     = $Function->{PRODUCT};  # MULT deprecated
@@ -492,7 +494,6 @@ sub _NOP {
 
     # pass everything through, this will allow plugins to defy plugin order
     # for example the %SEARCH{}% variable
-    $_[0] =~ s/\$per/%/g;
     $_[0] =~ s/\$per(cnt)?/%/g;
     $_[0] =~ s/\$quot/"/g;
     return $_[0];
@@ -503,6 +504,7 @@ sub _WHILE {
 
     # WHILE(condition, do something)
     my ( $condition, $str ) = _properSplit( $_[0], 2 );
+    return '' unless defined $condition;
     my $result;
     my $i = 0;
     while (1) {
@@ -794,6 +796,9 @@ sub _PERCENTILE {
 sub _PRODUCT {
     my $result = 0;
     my @arr    = _getListAsFloat( $_[0] );
+
+    # no arguments,  return 0.
+    return 0 unless scalar @arr;
     $result = 1;
     foreach my $i (@arr) {
         $result *= $i if defined $i;
@@ -886,6 +891,7 @@ sub _GET {
 # =========================
 sub _SET {
     my ( $name, $value ) = split( /,\s*/, $_[0], 2 );
+    return '' unless defined $name;
     $name =~ s/[^a-zA-Z0-9\_]//g;
     if ( $name && defined($value) ) {
         $value =~ s/\s*$//;
@@ -897,6 +903,7 @@ sub _SET {
 # =========================
 sub _SETIFEMPTY {
     my ( $name, $value ) = split( /,\s*/, $_[0], 2 );
+    return '' unless defined $name;
     $name =~ s/[^a-zA-Z0-9\_]//g;
     if ( $name && defined($value) && !$varStore{$name} ) {
         $value =~ s/\s*$//;
@@ -908,11 +915,13 @@ sub _SETIFEMPTY {
 # =========================
 sub _SETM {
     my ( $name, $value ) = split( /,\s*/, $_[0], 2 );
+    return '' unless defined $name;
     $name =~ s/[^a-zA-Z0-9\_]//g;
     if ($name) {
         my $old = $varStore{$name};
-        $old             = "" unless ( defined($old) );
-        $value           = _safeEvalPerl("$old $value");
+        $old   = "" unless ( defined($old) );
+        $value = "" unless ( defined($value) );
+        $value = _safeEvalPerl("$old $value");
         $varStore{$name} = $value;
     }
     return '';
@@ -1178,7 +1187,8 @@ sub _BITXOR {
 
     # This is a standard bit-wise xor of a list of integers.
     else {
-        @arr    = _getListAsInteger( $_[0] );
+        @arr = _getListAsInteger( $_[0] );
+        return '' unless scalar @arr;
         $result = int( shift(@arr) );
         if ( scalar(@arr) > 0 ) {
             foreach my $i (@arr) {
@@ -1444,6 +1454,19 @@ sub _EXACT {
     return ( $str1 eq $str2 ) ? 1 : 0;
 }
 
+# =========================
+sub _FILTER {
+    my $result = '';
+    my ( $filter, $string ) = split( /,\s*/, $_[0], 2 );
+    if ( defined $string ) {
+        $filter =~ s/\$comma/,/g;
+        $filter =~ s/\$sp/ /g;
+        eval '$string =~ s/$filter//go';
+        $result = $string;
+    }
+    return $result;
+}
+
 # ========================
 sub _FIND {
     return _SEARCH( $_[0], 'FIND' );
@@ -1472,6 +1495,7 @@ sub _REPLACE {
     my ( $string, $start, $num, $replace ) = split( /,\s*/, $_[0], 4 );
     $string = "" unless ( defined $string );
     my $result = $string;
+    $start ||= 0;
     $start-- unless ( $start < 1 );
     $num     = 0  unless ($num);
     $replace = "" unless ( defined $replace );
@@ -1683,6 +1707,7 @@ sub _getNumber {
 # =========================
 sub _safeEvalPerl {
     my ($theText) = @_;
+    $theText = '' unless defined $theText;
 
     # Allow only simple math with operators - + * / % ( )
     $theText =~ s/\%\s*[^\-\+\*\/0-9\.\(\)]+//g; # defuse %hash but keep modulus
@@ -1794,6 +1819,7 @@ sub _getList {
     my ($theAttr) = @_;
 
     my @list = ();
+    return @list unless $theAttr;
     $theAttr =~ s/^\s*//;    # Drop leading / trailing spaces
     $theAttr =~ s/\s*$//;
     foreach ( split( /\s*,\s*/, $theAttr ) ) {

diff --git a/SpreadSheetPlugin/test/unit/SpreadSheetPlugin/SpreadSheetPluginTests.pm b/SpreadSheetPlugin/test/unit/SpreadSheetPlugin/SpreadSheetPluginTests.pm
@@ -460,6 +460,17 @@ sub test_FIND {
     $this->assert( $this->CALC('$FIND(@, fluffy, 1)') == 0 );
 }
 
+sub test_FILTER {
+    my ($this) = @_;
+    $this->assert( $this->CALC('$FILTER(f, fluffy)') eq 'luy' );
+    $this->assert_equals( $this->CALC('$FILTER(an Franc, San Francisco)'),
+        'Sisco' );
+    $this->assert(
+        $this->CALC('$FILTER($sp, The quick brown)') eq 'Thequickbrown' );
+    $this->assert(
+        $this->CALC('$FILTER([^a-zA-Z0-9 ], Stupid mistake*%@^! Fixed)') );
+}
+
 sub test_FLOOR {
     my ($this) = @_;
     $this->assert( $this->CALC('$FLOOR(5)') == 5 );
@@ -668,6 +679,34 @@ sub test_INT {
         $this->CALC('$INT(10 / 0)') eq 'ERROR: Illegal division by zero' );
 }
 
+sub test_ISDIGIT {
+    my ($this) = @_;
+    $this->assert( $this->CALC('$ISDIGIT(123)') );
+    $this->assert( !$this->CALC('$ISDIGIT(-34)') );
+    $this->assert( !$this->CALC('$ISDIGIT(18.23)') );
+}
+
+sub test_ISLOWER {
+    my ($this) = @_;
+    $this->assert( $this->CALC('$ISLOWER(abcdefg)') );
+    $this->assert( !$this->CALC('$ISLOWER(abUPcdefg)') );
+    $this->assert( !$this->CALC('$ISLOWER(ab12cdefg)') );
+}
+
+sub test_ISUPPER {
+    my ($this) = @_;
+    $this->assert( $this->CALC('$ISUPPER(ASDFASD)') );
+    $this->assert( !$this->CALC('$ISUPPER(WikiWord)') );
+    $this->assert( !$this->CALC('$ISUPPER(123ABC)') );
+}
+
+sub test_ISWIKIWORD {
+    my ($this) = @_;
+    $this->assert( $this->CALC('$ISWIKIWORD(MyWikiWord)') );
+    $this->assert( $this->CALC('$ISWIKIWORD(MyWord23)') );
+    $this->assert( !$this->CALC('$ISWIKIWORD(fooBar)') );
+}
+
 sub test_LEFT {
     my ($this) = @_;
 
@@ -956,9 +995,8 @@ sub test_MOD {
 
 sub test_NOP {
     my ($this) = @_;
-    $this->assert( $this->CALC('$NOP(abcd)') eq 'abcd' );
-    $this->assert(
-        $this->CALC('$NOP($perabc$percntdef$quot)') eq '%abc%cntdef"' );
+    $this->assert( $this->CALC('$NOP(abcd)')                   eq 'abcd' );
+    $this->assert( $this->CALC('$NOP($perabc$percntdef$quot)') eq '%abc%def"' );
 }
 
 sub test_NOT {