Skip to content

Commit

Permalink
Fix out-of-dateness propagation
Browse files Browse the repository at this point in the history
Out-of-dateness was not being propagated correctly for deviant pipeline

Also:
* Replace installed files with temp files
* Update README
* Increment version number
  • Loading branch information
kinto-b committed Nov 3, 2021
1 parent 9bd144d commit a605957
Show file tree
Hide file tree
Showing 16 changed files with 78 additions and 138 deletions.
2 changes: 0 additions & 2 deletions CRAN-RELEASE

This file was deleted.

2 changes: 1 addition & 1 deletion DESCRIPTION
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
Package: makepipe
Title: Pipeline Tools Inspired by 'GNU Make'
Version: 0.0.4
Version: 0.0.5
Authors@R:
person(given = "Kinto",
family = "Behr",
Expand Down
5 changes: 5 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
# makepipe 0.0.5

* Fixed minor bug having to do with the propagation of out-of-dateness in
Pipeline visualizations.

# makepipe 0.0.4

* Renamed package from `piper` to `makepipe` (#11).
Expand Down
3 changes: 2 additions & 1 deletion R/Pipeline.R
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ Pipeline <- R6::R6Class(classname = "Pipeline", list(

# Propagate out-of-dateness
for (i in seq_along(edges$to)) {
if (edges$.source[i]) next
edges$out_of_date[i] <- propagate_outofdateness(edges$to[i], edges)
}

Expand Down Expand Up @@ -376,7 +377,7 @@ propagate_outofdateness <- function(initial_node, edges, next_node = NULL) {

inputs <- edges[edges$to == next_node, ]
for (i in seq_along(inputs$from)) {
outdated <- inputs$from_mtime[i] > target_mtime
outdated <- inputs$out_of_date[i] | inputs$from_mtime[i] > target_mtime
next_node <- inputs$from[i]

# Base case
Expand Down
15 changes: 12 additions & 3 deletions README.Rmd
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,21 @@ of data science workflows.

## Installation

You can install the development version from [GitHub](https://github.com/) with:
You can install the released version of `makepipe` from
[CRAN](https://CRAN.R-project.org) with:

``` r
install.packages("makepipe")
```

And the development version from [GitHub](https://github.com/) with:

``` r
# install.packages("devtools")
devtools::install_github("kinto-b/makepipe")
```


## Building a pipeline

To construct a pipeline, one simply needs to chain together a number of
Expand All @@ -50,8 +59,8 @@ visualisation of the pipeline. For example:
library(makepipe)
make_with_source(
dependencies = c("data/0_raw_data.csv", "lookup/concordance.csv"),
source = c("1 data_prep.R"),
targets = c("data/1_data.Rds")
source = "1 data_prep.R",
targets = "data/1_data.Rds"
)
make_with_recipe(
dependencies = c("data/1_data.Rds", "data/0_pop.Rds"),
Expand Down
14 changes: 10 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,14 @@ easy-going, being adaptable to a wide range of data science workflows.

## Installation

You can install the development version from
[GitHub](https://github.com/) with:
You can install the released version of `makepipe` from
[CRAN](https://CRAN.R-project.org) with:

``` r
install.packages("makepipe")
```

And the development version from [GitHub](https://github.com/) with:

``` r
# install.packages("devtools")
Expand All @@ -44,8 +50,8 @@ For example:
library(makepipe)
make_with_source(
dependencies = c("data/0_raw_data.csv", "lookup/concordance.csv"),
source = c("1 data_prep.R"),
targets = c("data/1_data.Rds")
source = "1 data_prep.R",
targets = "data/1_data.Rds"
)
make_with_recipe(
dependencies = c("data/1_data.Rds", "data/0_pop.Rds"),
Expand Down
6 changes: 3 additions & 3 deletions cran-comments.md
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
## Resubmission
This is a resubmission (thank you for taking the time to read my last
submission). In this version I have:
This is a re-submission. In this version I have:

* Removed package name and all backticks from Description text.
* Replaced installed test files with tempfiles so that the tests only write to
the temporary directory

## Test environments
* local R installation, R 4.1.0
Expand Down
33 changes: 0 additions & 33 deletions inst/tests/mtcars.csv

This file was deleted.

33 changes: 0 additions & 33 deletions inst/tests/mtcars.txt

This file was deleted.

3 changes: 0 additions & 3 deletions inst/tests/mtcars1.R

This file was deleted.

7 changes: 0 additions & 7 deletions inst/tests/mtcars2.R

This file was deleted.

33 changes: 0 additions & 33 deletions inst/tests/mtcars_src.csv

This file was deleted.

1 change: 1 addition & 0 deletions inst/tests/source1.R
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
makepipe::make_register(5, 'five')
1 change: 1 addition & 0 deletions inst/tests/source2.R
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
2+2
23 changes: 18 additions & 5 deletions tests/testthat/test-Pipeline.R
Original file line number Diff line number Diff line change
@@ -1,9 +1,19 @@
# Test files -------------------------------------------------------------------
dependency <- system.file("tests", "mtcars_src.csv", package = "makepipe")
source1 <- system.file("tests", "mtcars1.R", package = "makepipe")
source2 <- system.file("tests", "mtcars2.R", package = "makepipe")
target1 <- system.file("tests", "mtcars.csv", package = "makepipe")
target2 <- system.file("tests", "mtcars.txt", package = "makepipe")
dependency <- tempfile(fileext=".csv")
write.csv(mtcars, dependency)

target1 <- tempfile(fileext=".csv")
write.csv(mtcars, target1)

target2 <- tempfile(fileext=".txt")
write.table(mtcars, target2)

source1 <- tempfile(fileext=".R")
file.copy(system.file("tests", "source1.R", package = "makepipe"), source1)

source2 <- tempfile(fileext=".R")
file.copy(system.file("tests", "source2.R", package = "makepipe"), source2)


# Functions --------------------------------------------------------------------
order_filetimes <- function(...) {
Expand Down Expand Up @@ -166,3 +176,6 @@ test_that("annotations cannot must be character", {
})


# Unlink ------------------------------------------------------------------

unlink(c("dependency", "target1", "target2", "source1", "source2"))
35 changes: 25 additions & 10 deletions tests/testthat/test-make.R
Original file line number Diff line number Diff line change
@@ -1,9 +1,18 @@
# Test files -------------------------------------------------------------------
dependency <- system.file("tests", "mtcars_src.csv", package = "makepipe")
source1 <- system.file("tests", "mtcars1.R", package = "makepipe")
source2 <- system.file("tests", "mtcars2.R", package = "makepipe")
target1 <- system.file("tests", "mtcars.csv", package = "makepipe")
target2 <- system.file("tests", "mtcars.txt", package = "makepipe")
dependency <- tempfile(fileext=".csv")
write.csv(mtcars, dependency)

target1 <- tempfile(fileext=".csv")
write.csv(mtcars, target1)

target2 <- tempfile(fileext=".txt")
write.table(mtcars, target2)

source1 <- tempfile(fileext=".R")
file.copy(system.file("tests", "source1.R", package = "makepipe"), source1)

source2 <- tempfile(fileext=".R")
file.copy(system.file("tests", "source2.R", package = "makepipe"), source2)

# Functions --------------------------------------------------------------------
order_filetimes <- function(...) {
Expand Down Expand Up @@ -240,6 +249,8 @@ test_that("make_with_source returns what's registered", {
expect_equal(res$result$five, 5)
})


# Printing ---------------------------------------------------------------------
test_that("make_with_recipe result prints nicely", {
order_filetimes(target1, dependency)
x <- make_with_recipe({
Expand All @@ -249,8 +260,8 @@ test_that("make_with_recipe result prints nicely", {

expect_output(print(x), regexp = "# makepipe segment")
expect_output(print(x), regexp = "* Recipe: ")
expect_output(print(x), regexp = "* Targets: '.*/mtcars.csv'")
expect_output(print(x), regexp = "* File dependencies: '.*/mtcars_src.csv'")
expect_output(print(x), regexp = "* Targets: '.*'")
expect_output(print(x), regexp = "* File dependencies: '.*'")
expect_output(print(x), regexp = "* Executed: TRUE")
expect_output(print(x), regexp = "* Result: 1 object")
expect_output(print(x), regexp = "* Environment: ")
Expand All @@ -270,17 +281,21 @@ test_that("make_with_source result prints nicely", {
x <- make_with_source(source1, target1, dependency, quiet = TRUE)

expect_output(print(x), regexp = "# makepipe segment")
expect_output(print(x), regexp = "* Source: '.*/mtcars1.R'")
expect_output(print(x), regexp = "* Targets: '.*/mtcars.csv'")
expect_output(print(x), regexp = "* File dependencies: '.*/mtcars_src.csv'")
expect_output(print(x), regexp = "* Source: '.*'")
expect_output(print(x), regexp = "* Targets: '.*'")
expect_output(print(x), regexp = "* File dependencies: '.*'")
expect_output(print(x), regexp = "* Executed: TRUE")
expect_output(print(x), regexp = "* Result: 1 object")
expect_output(print(x), regexp = "* Environment: ")

order_filetimes(dependency, source1, target1)
x <- make_with_source(source1, target1, dependency, quiet = TRUE)
expect_output(print(x), regexp = "* Executed: FALSE")
})


# Unlink ------------------------------------------------------------------

unlink(c("dependency", "target1", "target2", "source1", "source2"))


0 comments on commit a605957

Please sign in to comment.