Merge pull request #138 from jameslamb/bugfix/parsing

changed strategy for dropping duplicates (fixes #137)
uptake · Jan 30, 2019 · ced8bb4 · ced8bb4
2 parents 98172a9 + 9101883
commit ced8bb4
Show file tree

Hide file tree

Showing 3 changed files with 99 additions and 44 deletions.
diff --git a/.ci/before_install.sh b/.ci/before_install.sh
@@ -0,0 +1,10 @@
+
+# failure is a natural part of life
+set -e
+
+# If language: r,
+# install these testing packages we need
+if [[ "$TASK" == "rpkg" ]];
+then
+  Rscript -e "install.packages(c('devtools', 'knitr', 'testthat', 'rmarkdown'), repos = 'http://cran.rstudio.com')"
+fi
diff --git a/.travis.yml b/.travis.yml
@@ -5,6 +5,9 @@ addons:
     packages:
       - oracle-java8-set-default
 
+before_install:
+  - .ci/before_install.sh
+
 # Manually specifying each build configuration.
 # Would be better to figure out how to build a matrix with two
 # languages but share the ES_VERSION matrix across all builds.
@@ -30,9 +33,10 @@ matrix:
     - language: r
       warnings_are_errors: true
       cache: packages
-      env: ES_VERSION=1.0.0
+      env:
+        - ES_VERSION=1.0.0
+        - TASK=rpkg
       install:
-        - Rscript -e "install.packages('devtools', repos = 'http://cran.rstudio.com')"
         - Rscript -e "devtools::install('r-pkg')"
       script:
         - R CMD build r-pkg/
@@ -42,9 +46,10 @@ matrix:
     - language: r
       warnings_are_errors: true
       cache: packages
-      env: ES_VERSION=1.4.4
+      env:
+        - ES_VERSION=1.4.4
+        - TASK=rpkg
       install:
-        - Rscript -e "install.packages('devtools', repos = 'http://cran.rstudio.com')"
         - Rscript -e "devtools::install('r-pkg')"
       script:
         - R CMD build r-pkg/
@@ -54,9 +59,10 @@ matrix:
     - language: r
       warnings_are_errors: true
       cache: packages
-      env: ES_VERSION=1.7.2
+      env:
+        - ES_VERSION=1.7.2
+        - TASK=rpkg
       install:
-        - Rscript -e "install.packages('devtools', repos = 'http://cran.rstudio.com')"
         - Rscript -e "devtools::install('r-pkg')"
       script:
         - R CMD build r-pkg/
@@ -66,9 +72,10 @@ matrix:
     - language: r
       warnings_are_errors: true
       cache: packages
-      env: ES_VERSION=2.0.2
+      env:
+        - ES_VERSION=2.0.2
+        - TASK=rpkg
       install:
-        - Rscript -e "install.packages('devtools', repos = 'http://cran.rstudio.com')"
         - Rscript -e "devtools::install('r-pkg')"
       script:
         - R CMD build r-pkg/
@@ -78,9 +85,10 @@ matrix:
     - language: r
       warnings_are_errors: true
       cache: packages
-      env: ES_VERSION=2.1.2
+      env:
+        - ES_VERSION=2.1.2
+        - TASK=rpkg
       install:
-        - Rscript -e "install.packages('devtools', repos = 'http://cran.rstudio.com')"
         - Rscript -e "devtools::install('r-pkg')"
       script:
         - R CMD build r-pkg/
@@ -90,9 +98,10 @@ matrix:
     - language: r
       warnings_are_errors: true
       cache: packages
-      env: ES_VERSION=2.2.2
+      env:
+        - ES_VERSION=2.2.2
+        - TASK=rpkg
       install:
-        - Rscript -e "install.packages('devtools', repos = 'http://cran.rstudio.com')"
         - Rscript -e "devtools::install('r-pkg')"
       script:
         - R CMD build r-pkg/
@@ -102,9 +111,10 @@ matrix:
     - language: r
       warnings_are_errors: true
       cache: packages
-      env: ES_VERSION=2.3.5
+      env:
+        - ES_VERSION=2.3.5
+        - TASK=rpkg
       install:
-        - Rscript -e "install.packages('devtools', repos = 'http://cran.rstudio.com')"
         - Rscript -e "devtools::install('r-pkg')"
       script:
         - R CMD build r-pkg/
@@ -114,9 +124,10 @@ matrix:
     - language: r
       warnings_are_errors: true
       cache: packages
-      env: ES_VERSION=5.0.2
+      env:
+        - ES_VERSION=5.0.2
+        - TASK=rpkg
       install:
-        - Rscript -e "install.packages('devtools', repos = 'http://cran.rstudio.com')"
         - Rscript -e "devtools::install('r-pkg')"
       script:
         - R CMD build r-pkg/
@@ -126,9 +137,10 @@ matrix:
     - language: r
       warnings_are_errors: true
       cache: packages
-      env: ES_VERSION=5.3.3
+      env:
+        - ES_VERSION=5.3.3
+        - TASK=rpkg
       install:
-        - Rscript -e "install.packages('devtools', repos = 'http://cran.rstudio.com')"
         - Rscript -e "devtools::install('r-pkg')"
       script:
         - R CMD build r-pkg/
@@ -138,9 +150,10 @@ matrix:
     - language: r
       warnings_are_errors: true
       cache: packages
-      env: ES_VERSION=5.4.3
+      env:
+        - ES_VERSION=5.4.3
+        - TASK=rpkg
       install:
-        - Rscript -e "install.packages('devtools', repos = 'http://cran.rstudio.com')"
         - Rscript -e "devtools::install('r-pkg')"
       script:
         - R CMD build r-pkg/
@@ -150,9 +163,10 @@ matrix:
     - language: r
       warnings_are_errors: true
       cache: packages
-      env: ES_VERSION=5.6.9
+      env:
+        - ES_VERSION=5.6.9
+        - TASK=rpkg
       install:
-        - Rscript -e "install.packages('devtools', repos = 'http://cran.rstudio.com')"
         - Rscript -e "devtools::install('r-pkg')"
       script:
         - R CMD build r-pkg/
@@ -162,9 +176,10 @@ matrix:
     - language: r
       warnings_are_errors: true
       cache: packages
-      env: ES_VERSION=6.0.1
+      env:
+        - ES_VERSION=6.0.1
+        - TASK=rpkg
       install:
-        - Rscript -e "install.packages('devtools', repos = 'http://cran.rstudio.com')"
         - Rscript -e "devtools::install('r-pkg')"
       script:
         - R CMD build r-pkg/
@@ -174,9 +189,10 @@ matrix:
     - language: r
       warnings_are_errors: true
       cache: packages
-      env: ES_VERSION=6.1.4
+      env:
+        - ES_VERSION=6.1.4
+        - TASK=rpkg
       install:
-        - Rscript -e "install.packages('devtools', repos = 'http://cran.rstudio.com')"
         - Rscript -e "devtools::install('r-pkg')"
       script:
         - R CMD build r-pkg/
@@ -186,9 +202,10 @@ matrix:
     - language: r
       warnings_are_errors: true
       cache: packages
-      env: ES_VERSION=6.2.4
+      env:
+        - ES_VERSION=6.2.4
+        - TASK=rpkg
       install:
-        - Rscript -e "install.packages('devtools', repos = 'http://cran.rstudio.com')"
         - Rscript -e "devtools::install('r-pkg')"
       script:
         - R CMD build r-pkg/
@@ -204,98 +221,126 @@ matrix:
         pip install py-pkg/
       script:
         - pytest --verbose py-pkg/
-      env: ES_VERSION=1.0.0
+      env:
+        - ES_VERSION=1.0.0
+        - TASK=pypkg
     - language: python
       python: 3.5
       install:
         pip install py-pkg/
       script:
         - pytest --verbose py-pkg/
-      env: ES_VERSION=1.4.4
+      env:
+        - ES_VERSION=1.4.4
+        - TASK=pypkg
     - language: python
       python: 3.5
       install:
         pip install py-pkg/
       script:
         - pytest --verbose py-pkg/
-      env: ES_VERSION=1.7.2
+      env:
+        - ES_VERSION=1.7.2
+        - TASK=pypkg
     - language: python
       python: 3.5
       install:
         pip install py-pkg/
       script:
         - pytest --verbose py-pkg/
-      env: ES_VERSION=2.0.2
+      env:
+        - ES_VERSION=2.0.2
+        - TASK=pypkg
     - language: python
       python: 3.5
       install:
         pip install py-pkg/
       script:
         - pytest --verbose py-pkg/
-      env: ES_VERSION=2.1.2
+      env:
+        - ES_VERSION=2.1.2
+        - TASK=pypkg
     - language: python
       python: 3.5
       install:
         pip install py-pkg/
       script:
         - pytest --verbose py-pkg/
-      env: ES_VERSION=2.2.2
+      env:
+        - ES_VERSION=2.2.2
+        - TASK=pypkg
     - language: python
       python: 3.5
       install:
         pip install py-pkg/
       script:
         - pytest --verbose py-pkg/
-      env: ES_VERSION=2.3.5
+      env:
+        - ES_VERSION=2.3.5
+        - TASK=pypkg
     - language: python
       python: 3.5
       install:
         pip install py-pkg/
       script:
         - pytest --verbose py-pkg/
-      env: ES_VERSION=5.0.2
+      env:
+        - ES_VERSION=5.0.2
+        - TASK=pypkg
     - language: python
       python: 3.5
       install:
         pip install py-pkg/
       script:
         - pytest --verbose py-pkg/
-      env: ES_VERSION=5.3.3
+      env:
+        - ES_VERSION=5.3.3
+        - TASK=pypkg
     - language: python
       python: 3.5
       install:
         pip install py-pkg/
       script:
         - pytest --verbose py-pkg/
-      env: ES_VERSION=5.4.3
+      env:
+        - ES_VERSION=5.4.3
+        - TASK=pypkg
     - language: python
       python: 3.5
       install:
         pip install py-pkg/
       script:
         - pytest --verbose py-pkg/
-      env: ES_VERSION=5.6.9
+      env:
+        - ES_VERSION=5.6.9
+        - TASK=pypkg
     - language: python
       python: 3.5
       install:
         pip install py-pkg/
       script:
         - pytest --verbose py-pkg/
-      env: ES_VERSION=6.0.1
+      env:
+        - ES_VERSION=6.0.1
+        - TASK=pypkg
     - language: python
       python: 3.5
       install:
         pip install py-pkg/
       script:
         - pytest --verbose py-pkg/
-      env: ES_VERSION=6.1.4
+      env:
+        - ES_VERSION=6.1.4
+        - TASK=pypkg
     - language: python
       python: 3.5
       install:
         pip install py-pkg/
       script:
         - pytest --verbose py-pkg/
-      env: ES_VERSION=6.2.4
+      env:
+        - ES_VERSION=6.2.4
+        - TASK=pypkg
 
 before_script:
   - case "$ES_VERSION" in

diff --git a/r-pkg/R/es_search.R b/r-pkg/R/es_search.R
@@ -100,7 +100,7 @@ es_search <- function(es_host
         )
         log_fatal(msg)
     }
-  
+
     # assign 1 core by default, if the number of cores is NA
     if (is.na(n_cores) || !assertthat::is.count(n_cores)){
       msg <- "detectCores() returned NA. Assigning number of cores to be 1."
@@ -397,7 +397,7 @@ es_search <- function(es_host
 
     # It's POSSIBLE that the parallel process gave us duplicates. Correct for that
     data.table::setkeyv(outDT, NULL)
-    outDT <- unique(outDT)
+    outDT <- unique(outDT, by = "_id")
 
     # Check we got the number of unique records we expected
     if (nrow(outDT) < hits_to_pull && break_on_duplicates){