gtools-0.13.1 (2018-05-02); major bug fixes

Pending - For now, throws error for strL variable: #39 Plugin version 2.0 makes no mention of strL variables; however, plugin version 3.0 notes that they are actually not supported by the macros that I use to access strings. I think I'll have to compile a sepparate version of the plugin for Stata 13 and Stata 14 and above. The Stata 13 version will throw an error for strL variables. This is currently the only version until I figure out where to test Stata 14 and how to implement this switch. Bug fixes - Fixes #38 When the quantile requested was N - 1 out of an array of length N, I makde an exception so the function picked up the largest value, instead of passing N - 1 to the selection algorithm. However, I made a mistake and quantiles between 100 - 150 / N and 100 - 100 / N, left-inclusive right-exclusive, would give the wrong quantile.
mcaceresb · May 2, 2018 · 2c50d5d · 2c50d5d
1 parent 48ada5a
commit 2c50d5d
Show file tree

Hide file tree

Showing 71 changed files with 307 additions and 84 deletions.
diff --git a/.appveyor.yml b/.appveyor.yml
@@ -1,4 +1,4 @@
-version: "generic-0.6.0-{build}"
+version: "generic-0.7.0-{build}"
 
 environment:
   matrix:

diff --git a/.travis.yml b/.travis.yml
@@ -16,4 +16,4 @@ script:
 - ALL_CFLAGS+=-fPIC make
 - cd -
 - make clean && make SPOOKYPATH=$(dirname `find ./lib/spookyhash/ -name "*libspookyhash.a"`)
-# - "./lib/travis_build_push.sh"
+- "./lib/travis_build_push.sh"
diff --git a/README.md b/README.md
@@ -13,7 +13,7 @@ implementation of collapse, pctile, xtile, contract, egen, isid,
 levelsof, and unique/distinct using C plugins for a massive speed
 improvement.
 
-`version 0.13.0 24Apr2018`
+`version 0.13.1 02May2018`
 Builds: Linux, OSX [![Travis Build Status](https://travis-ci.org/mcaceresb/stata-gtools.svg?branch=develop)](https://travis-ci.org/mcaceresb/stata-gtools),
 Windows (Cygwin) [![Appveyor Build status](https://ci.appveyor.com/api/projects/status/2bh1q9bulx3pl81p/branch/develop?svg=true)](https://ci.appveyor.com/project/mcaceresb/stata-gtools)
 
@@ -27,6 +27,8 @@ caveats and details on the plugin (including some extra features!).
 
 __*Gtools commands with a Stata equivalent*__
 
+(_**NOTE:**_ `strL` variables are not yet supported; see [issue 39](https://github.com/mcaceresb/stata-gtools/issues/39))
+
 | Function     | Replaces | Speedup (IC / MP)        | Unsupported     | Extras                                  |
 | ------------ | -------- | ------------------------ | --------------- | --------------------------------------- |
 | gcollapse    | collapse |  9 to 300 / 4 to 120 (+) |                 | Quantiles, merge, nunique, label output |

diff --git a/build/_gtools_internal.ado b/build/_gtools_internal.ado
@@ -1,6 +1,14 @@
-*! version 0.6.0 24Apr2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
+*! version 0.7.1 02May2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
 *! Encode varlist using Jenkin's 128-bit spookyhash via C plugins
 
+* rc 17000
+* rc 17001
+* rc 17002 - strL variables and version < 14
+* rc 17003 - strL variables and version >= 14
+* rc 17459
+* rc 17900
+* rc 17999
+
 capture program drop _gtools_internal
 program _gtools_internal, rclass
     version 13
@@ -2040,21 +2048,21 @@ program _gtools_internal, rclass
             disp as txt "{hline 72}"
             disp as txt `""'
             disp as txt `"    gcall:            `gcall'"'
-            disp as txt `""'                    
+            disp as txt `""'
             disp as txt `"    contractvars:     `contractvars'"'
-            disp as txt `""'                    
+            disp as txt `""'
             disp as txt `"    nolocalvar:       `nolocalvar'"'
             disp as txt `"    freq:             `freq'"'
             disp as txt `"    store:            `store'"'
-            disp as txt `""'                    
+            disp as txt `""'
             disp as txt `"    ntop:             `ntop'"'
             disp as txt `"    pct:              `pct'"'
             disp as txt `"    freq:             `freq'"'
             disp as txt `"    misslab:          `misslab'"'
             disp as txt `"    otherlab:         `otherlab'"'
             disp as txt `"    groupmiss:        `groupmiss'"'
             disp as txt `"    nrows:            `nrows'"'
-            disp as txt `""'                    
+            disp as txt `""'
             disp as txt `"    xvars:            `xvars'"'
             disp as txt `"    xsources:         `xsources'"'
             disp as txt `"    nquantiles:       `nquantiles'"'
@@ -2079,7 +2087,7 @@ program _gtools_internal, rclass
             disp as txt `"    altdef:           `altdef'"'
             disp as txt `"    strict:           `strict'"'
             disp as txt `"    minmax:           `minmax'"'
-            disp as txt `""'                    
+            disp as txt `""'
             disp as txt `"    xhow_nq:          `xhow_nq'"'
             disp as txt `"    xhow_cutvars:     `xhow_cutvars'"'
             disp as txt `"    xhow_qvars:       `xhow_qvars'"'
@@ -2457,9 +2465,10 @@ program parse_by_types, rclass
     local kstr  = 0
     local kvars = 0
 
-    local varint ""
-    local varnum ""
-    local varstr ""
+    local varint  ""
+    local varnum  ""
+    local varstr  ""
+    local varstrL ""
 
     if ( "`varlist'" != "" ) {
         cap confirm variable `varlist'
@@ -2487,6 +2496,7 @@ program parse_by_types, rclass
                 local varstr `varstr' `byvar'
                 if regexm("`:type `byvar''", "str([1-9][0-9]*|L)") {
                     if (regexs(1) == "L") {
+                        local varstrL `varstrL' `byvar'
                         tempvar strlen
                         gen long `strlen' = length(`byvar')
                         qui sum `strlen', meanonly
@@ -2514,6 +2524,30 @@ program parse_by_types, rclass
         }
     }
 
+    * if ( ("`varstrL'" != "") & (_caller() < 14) ) {
+    *     disp as err _n(1) "gtools for Stata 13 and earlier does not support strL variables. If your" ///
+    *                 _n(1) "strL variables are string-only, try"                                      ///
+    *                 _n(2) "    {stata compress `varstrL'}"                                           ///
+    *                 _n(2) "If this does not work or if you have binary data, you will need to use"   ///
+    *                 _n(1) "Stata 14 or newer. This limitation comes from the Stata Plugin Interface" ///
+    *                 _n(1) "(SPI) 2.0 that was used to write gtools. gtools 0.14 integrated 3.0"      ///
+    *                 _n(1) "(Stata 14 and above only), which added support for strL variables."
+    *     exit 17002
+    * }
+    * else if ( "`varstrL'" != "" ) {
+    if ( "`varstrL'" != "" ) {
+        disp as err _n(1) "gtools 0.13.x does not support strL variables. If your strL variables"   ///
+                    _n(1) "are string-only, try"                                                    ///
+                    _n(2) "    {stata compress `varstrL'}"                                          ///
+                    _n(2) "If this does not work or if you have binary data, then you will have to" ///
+                    _n(1) "wait for the next release of gtools (0.14)."                             ///
+                    _n(2) "This limitation comes from the Stata Plugin Interface (SPI) 2.0"         ///
+                    _n(1) "that was used to write gtools. 3.0 (Stata 14 and above only) added"      ///
+                    _n(1) "support for strL variables. gtools will add strL support in its next"    ///
+                    _n(1) "relase (0.14)."
+        exit 17003
+    }
+
     * Parse which hashing strategy to use
     * -----------------------------------
 

diff --git a/build/changelog.md b/build/changelog.md
@@ -1,6 +1,30 @@
 Change Log
 ==========
 
+## gtools-0.13.1 (2018-05-02)
+
+### Pending
+
+- For now, throws error for strL variable: https://github.com/mcaceresb/stata-gtools/issues/39
+
+Plugin version 2.0 makes no mention of strL variables; however, plugin
+version 3.0 notes that they are actually not supported by the macros
+that I use to access strings. I think I'll have to compile a sepparate
+version of the plugin for Stata 13 and Stata 14 and above. The Stata
+13 version will throw an error for strL variables. This is currently
+the only version until I figure out where to test Stata 14 and how to
+implement this switch.
+
+### Bug fixes
+
+- Fixes https://github.com/mcaceresb/stata-gtools/issues/38
+
+When the quantile requested was N - 1 out of an array of length N, I
+makde an exception so the function picked up the largest value, instead
+of passing N - 1 to the selection algorithm. However, I made a mistake
+and quantiles between 100 - 150 / N and 100 - 100 / N, left-inclusive
+right-exclusive, would give the wrong quantile.
+
 ## gtools-0.13.0 (2018-04-24)
 
 ### Enhancements

diff --git a/build/env_set_windows.plugin b/build/env_set_windows.plugin
diff --git a/build/fasterxtile.ado b/build/fasterxtile.ado
@@ -1,4 +1,4 @@
-*! version 0.5.0 24Apr2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
+*! version 0.6.1 02May2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
 *! faster implementation of xtile and fastxtile using C for faster processing
 *! (note: this is a wrapper for gquantiles)
 

diff --git a/build/gcollapse.ado b/build/gcollapse.ado
@@ -1,4 +1,4 @@
-*! version 0.13.0 24Apr2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
+*! version 0.13.1 02May2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
 *! -collapse- implementation using C for faster processing
 
 capture program drop gcollapse

diff --git a/build/gcollapse.sthlp b/build/gcollapse.sthlp
@@ -1,5 +1,5 @@
 {smcl}
-{* *! version 0.13.0 24Apr2018}{...}
+{* *! version 0.13.1 02May2018}{...}
 {viewerdialog gcollapse "dialog gcollapse"}{...}
 {vieweralsosee "[R] gcollapse" "mansection R gcollapse"}{...}
 {viewerjumpto "Syntax" "gcollapse##syntax"}{...}

diff --git a/build/gcontract.ado b/build/gcontract.ado
@@ -1,4 +1,4 @@
-*! version 0.6.0 24Apr2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
+*! version 0.7.1 02May2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
 *! Frequency counts using C-plugins for a speedup.
 
 cap program drop gcontract

diff --git a/build/gcontract.sthlp b/build/gcontract.sthlp
@@ -1,5 +1,5 @@
 {smcl}
-{* *! version 0.6.0 24Apr2018}{...}
+{* *! version 0.7.1 02May2018}{...}
 {viewerdialog gcontract "dialog gcontract"}{...}
 {vieweralsosee "[R] gcontract" "mansection R gcontract"}{...}
 {viewerjumpto "Syntax" "gcontract##syntax"}{...}

diff --git a/build/gdistinct.ado b/build/gdistinct.ado
@@ -1,4 +1,4 @@
-*! version 0.6.0 24Apr2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
+*! version 0.7.1 02May2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
 *! -distinct- implementation using C for faster processing
 
 capture program drop gdistinct

diff --git a/build/gdistinct.sthlp b/build/gdistinct.sthlp
@@ -1,5 +1,5 @@
 {smcl}
-{* *! version 0.6.0  24Apr2018}{...}
+{* *! version 0.7.1  02May2018}{...}
 {viewerdialog gdistinct "dialog gdistinct"}{...}
 {vieweralsosee "[D] gdistinct" "mansection D gdistinct"}{...}
 {viewerjumpto "Syntax" "gdistinct##syntax"}{...}

diff --git a/build/gegen.ado b/build/gegen.ado
@@ -1,4 +1,4 @@
-*! version 0.13.0 24Apr2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
+*! version 0.13.1 02May2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
 *! implementation -egen- using C for faster processing
 
 /*

diff --git a/build/gegen.sthlp b/build/gegen.sthlp
@@ -1,5 +1,5 @@
 {smcl}
-{* *! version 0.13.0 24Apr2018}{...}
+{* *! version 0.13.1 02May2018}{...}
 {viewerdialog gegen "dialog gegen"}{...}
 {vieweralsosee "[R] gegen" "mansection R gegen"}{...}
 {viewerjumpto "Syntax" "gegen##syntax"}{...}

diff --git a/build/gisid.ado b/build/gisid.ado
@@ -1,4 +1,4 @@
-*! version 0.7.0 24Apr2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
+*! version 0.8.1 02May2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
 *! -isid- implementation using C for faster processing
 
 capture program drop gisid

diff --git a/build/gisid.sthlp b/build/gisid.sthlp
@@ -1,5 +1,5 @@
 {smcl}
-{* *! version 0.7.0  24Apr2018}{...}
+{* *! version 0.8.1  02May2018}{...}
 {viewerdialog gisid "dialog gisid"}{...}
 {vieweralsosee "[D] gisid" "mansection D gisid"}{...}
 {viewerjumpto "Syntax" "gisid##syntax"}{...}

diff --git a/build/glevelsof.ado b/build/glevelsof.ado
@@ -1,4 +1,4 @@
-*! version 0.7.0 24Apr2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
+*! version 0.8.1 02May2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
 *! -isid- implementation using C for faster processing
 
 capture program drop glevelsof

diff --git a/build/glevelsof.sthlp b/build/glevelsof.sthlp
@@ -1,5 +1,5 @@
 {smcl}
-{* *! version 0.7.0  24Apr2018}{...}
+{* *! version 0.8.1  02May2018}{...}
 {vieweralsosee "[P] glevelsof" "mansection P glevelsof"}{...}
 {vieweralsosee "" "--"}{...}
 {vieweralsosee "[P] foreach" "help foreach"}{...}

diff --git a/build/gquantiles.ado b/build/gquantiles.ado
@@ -1,4 +1,4 @@
-*! version 0.5.0 24Apr2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
+*! version 0.6.1 02May2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
 *! faster implementation of pctile, xtile, and _pctile using C plugins
 
 capture program drop gquantiles

diff --git a/build/gquantiles.sthlp b/build/gquantiles.sthlp
@@ -1,5 +1,5 @@
 {smcl}
-{* *! version 0.5.0 24Apr2018}{...}
+{* *! version 0.6.1 02May2018}{...}
 {viewerdialog gquantiles "dialog gquantiles"}{...}
 {vieweralsosee "[R] gquantiles" "mansection R gquantiles"}{...}
 {viewerjumpto "Syntax" "gquantiles##syntax"}{...}

diff --git a/build/gtools.ado b/build/gtools.ado
@@ -1,4 +1,4 @@
-*! version 0.13.0 24Apr2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
+*! version 0.13.1 02May2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
 *! Program for managing the gtools package installation
 
 capture program drop gtools

diff --git a/build/gtools.pkg b/build/gtools.pkg
@@ -1,4 +1,4 @@
-v 0.13.0
+v 0.13.1
 d
 d 'GTOOLS': Faster group operations using C plugins
 d
@@ -40,7 +40,7 @@ d
 d Author: Mauricio Caceres Bravo
 d Support: email mauricio.caceres.bravo@gmail.com
 d
-d Distribution-Date: 20180424
+d Distribution-Date: 20180502
 d
 f _gtools_internal.ado
 f gcollapse.ado

diff --git a/build/gtools.sthlp b/build/gtools.sthlp
@@ -1,5 +1,5 @@
 {smcl}
-{* *! version 0.13.0 24Apr2018}{...}
+{* *! version 0.13.1 02May2018}{...}
 {viewerdialog gtools "dialog gtools"}{...}
 {vieweralsosee "[R] gtools" "mansection R gtools"}{...}
 {viewerjumpto "Syntax" "gtools##syntax"}{...}

diff --git a/build/gtools_tests.do b/build/gtools_tests.do
@@ -3,9 +3,9 @@
 * Program: gtools_tests.do
 * Author:  Mauricio Caceres Bravo <mauricio.caceres.bravo@gmail.com>
 * Created: Tue May 16 07:23:02 EDT 2017
-* Updated: Tue Apr 24 15:08:03 EDT 2018
+* Updated: Wed May  2 02:29:56 EDT 2018
 * Purpose: Unit tests for gtools
-* Version: 0.12.8
+* Version: 0.13.1
 * Manual:  help gtools
 
 * Stata start-up options
@@ -22,6 +22,9 @@ set type double
 * Main program wrapper
 * --------------------
 
+* TODO: cap repalce random1 = runiform() in 1 / `=ceil(_N / 2)'
+* TODO: cap repalce random2 = rnormal()  in `=ceil(_N / 2)' / _N
+
 program main
     syntax, [NOIsily *]
 
@@ -523,6 +526,45 @@ program checks_corners
     syntax, [*]
     di _n(1) "{hline 80}" _n(1) "checks_corners `options'" _n(1) "{hline 80}" _n(1)
 
+    * https://github.com/mcaceresb/stata-gtools/issues/38
+    qui {
+        clear
+        set obs 5
+        gen x = _n
+        gcollapse (p70) x
+        assert x == 4
+
+        clear
+        set obs 5
+        gen x = _n
+        gcollapse (p80) x
+        assert x == 4.5
+
+        clear
+        set obs 5
+        gen x = _n
+        gcollapse (p80.0001) x
+        assert x == 5
+
+        clear
+        set obs 3
+        gen x = _n
+        gcollapse (p50) x
+        assert x == 2
+
+        clear
+        set obs 3
+        gen x = _n
+        gcollapse (p66.6) x
+        assert x == 2
+
+        clear
+        set obs 3
+        gen x = _n
+        gcollapse (p66.7) x
+        assert x == 3
+    }
+
     * https://github.com/mcaceresb/stata-gtools/issues/32
     qui {
         clear

diff --git a/build/gtools_unix.plugin b/build/gtools_unix.plugin
diff --git a/build/gtools_windows.plugin b/build/gtools_windows.plugin
diff --git a/build/gtop.ado b/build/gtop.ado
@@ -1,4 +1,4 @@
-*! version 0.6.0 24Apr2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
+*! version 0.7.1 02May2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
 *! Calculate the top groups by count of a varlist (jointly).
 
 cap program drop gtop

diff --git a/build/gtoplevelsof.ado b/build/gtoplevelsof.ado
@@ -1,4 +1,4 @@
-*! version 0.6.0 24Apr2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
+*! version 0.7.1 02May2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
 *! Calculate the top groups by count of a varlist (jointly).
 
 * TODO: do not replace value if it does not have a label // 2017-11-09 21:43 EST

diff --git a/build/gtoplevelsof.sthlp b/build/gtoplevelsof.sthlp
@@ -1,5 +1,5 @@
 {smcl}
-{* *! version 0.7.0  24Apr2018}{...}
+{* *! version 0.8.1  02May2018}{...}
 {vieweralsosee "[P] gtoplevelsof" "mansection P gtoplevelsof"}{...}
 {viewerjumpto "Syntax" "gtoplevelsof##syntax"}{...}
 {viewerjumpto "Description" "gtoplevelsof##description"}{...}

diff --git a/build/gunique.ado b/build/gunique.ado
@@ -1,4 +1,4 @@
-*! version 0.6.0 24Apr2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
+*! version 0.7.1 02May2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
 *! -unique- implementation using C for faster processing
 
 capture program drop gunique

diff --git a/build/gunique.sthlp b/build/gunique.sthlp
@@ -1,5 +1,5 @@
 {smcl}
-{* *! version 0.6.0  24Apr2018}{...}
+{* *! version 0.7.1  02May2018}{...}
 {viewerdialog gunique "dialog gunique"}{...}
 {vieweralsosee "[D] gunique" "mansection D gunique"}{...}
 {viewerjumpto "Syntax" "gunique##syntax"}{...}

diff --git a/build/hashsort.ado b/build/hashsort.ado
@@ -1,4 +1,4 @@
-*! version 0.7.0 24Apr2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
+*! version 0.8.1 02May2018 Mauricio Caceres Bravo, mauricio.caceres.bravo@gmail.com
 *! Hash-based implementation of -sort- and -gsort- using C-plugins
 
 capture program drop hashsort