[help] Using tar_combine() for a nested static pipeline

[ginolhac]

Help

• I understand and agree to https://books.ropensci.org/targets/help.html.

Description

Hello,

thanks again for {targets}, a package and maintainer that never ceases to amaze me.

I would like to combine the nested tar_map() inside one tar_map(). I can achieve this with dynamic-within-static but for the sake of clean target names, would be great to have this full static working.
Through reprexes I would show the different steps. The key part I think is the unlist = FALSE that does not seem to be active.

Nested static without combine step

First static is for 2 models of distributions (1 and 2) and for each (nested tar_map() 2 estimators A and B:

# modified from https://github.com/ropensci/targets/discussions/348#discussioncomment-500050
targets::tar_dir({
  library(targets)
  
  tar_script({
    library(targets)
    library(tarchetypes)
    tar_map(
      values = list(model = c("mod_1", "mod_2")),
      tar_target(
        distrib,
        tar_name(),
      ),
      # static in static: nested tar_map()
      tar_map(
        values = list(sim = c("A", "B")),
        tar_target(
          estim,
          paste(distrib, tar_name()),
        )
      )
    )
  })
  
  tar_make()
  tar_read(estim_B_mod_2)
  tar_visnetwork()
})
#> ▶ dispatched target distrib_mod_1
#> ● completed target distrib_mod_1 [0.001 seconds]
#> ▶ dispatched target distrib_mod_2
#> ● completed target distrib_mod_2 [0 seconds]
#> ▶ dispatched target estim_A_mod_1
#> ● completed target estim_A_mod_1 [0 seconds]
#> ▶ dispatched target estim_B_mod_1
#> ● completed target estim_B_mod_1 [0 seconds]
#> ▶ dispatched target estim_A_mod_2
#> ● completed target estim_A_mod_2 [0 seconds]
#> ▶ dispatched target estim_B_mod_2
#> ● completed target estim_B_mod_2 [0 seconds]
#> ▶ ended pipeline [0.281 seconds]

^{Created on 2024-06-05 with reprex v2.1.0}

Nested static with a combination for all estimators

Works but I would like a combination per distribution, not all 4 together

targets::tar_dir({
  library(targets)
  
  tar_script({
    library(targets)
    library(tarchetypes)
    mapped <- tar_map(
      #unlist = FALSE, # Return a nested list from tar_map()
      values = list(model = c("mod_1", "mod_2")),
      tar_target(
        distrib,
        tar_name(),
      ),
      # static in static
      tar_map(
        values = list(sim = c("A", "B")),
        tar_target(
          estim,
          paste(distrib, tar_name()),
        )
      )
    )
    combined <- tar_combine(combi, 
                            # select both _A and _B while we want per model
                            tar_select_targets(mapped, starts_with("estim")), 
                            command = paste(!!!.x))
    list(mapped, combined)
  })
  
  tar_make()
  tar_read(combi)
  tar_visnetwork()
})
#> ▶ dispatched target distrib_mod_1
#> ● completed target distrib_mod_1 [0.001 seconds]
#> ▶ dispatched target distrib_mod_2
#> ● completed target distrib_mod_2 [0 seconds]
#> ▶ dispatched target estim_A_mod_1
#> ● completed target estim_A_mod_1 [0 seconds]
#> ▶ dispatched target estim_B_mod_1
#> ● completed target estim_B_mod_1 [0 seconds]
#> ▶ dispatched target estim_A_mod_2
#> ● completed target estim_A_mod_2 [0 seconds]
#> ▶ dispatched target estim_B_mod_2
#> ● completed target estim_B_mod_2 [0 seconds]
#> ▶ dispatched target combi
#> ● completed target combi [0 seconds]
#> ▶ ended pipeline [0.32 seconds]

^{Created on 2024-06-05 with reprex v2.1.0}

Trying the unlist = FALSE option of tar_map()

> waldo::compare(mapped, mapped_unlistF)
✔ No differences

I was expecting to see the nesting pattern here. Like mapped["distrib"]][["estim_A"]]

Manual 2 tar_combine does the job

Works but if I have more estimators, it could get verbose.

targets::tar_dir({
  library(targets)
  
  tar_script({
    library(targets)
    library(tarchetypes)
    mapped <- tar_map(
      #unlist = FALSE, # Return a nested list from tar_map()
      values = list(model = c("mod_1", "mod_2")),
      tar_target(
        distrib,
        tar_name(),
      ),
      # static in static
      tar_map(
        values = list(sim = c("A", "B")),
        tar_target(
          estim,
          paste(distrib, tar_name()),
        )
      )
    )
    combined1 <- tar_combine(combi1, 
                            # select both _A and _B while we want per model
                            tar_select_targets(mapped, tidyselect::matches("estim_._mod_1")), 
                            command = paste(!!!.x))
    combined2 <- tar_combine(combi2, 
                             # select both _A and _B while we want per model
                             tar_select_targets(mapped, tidyselect::matches("estim_._mod_2")), 
                             command = paste(!!!.x))
    list(mapped, combined1, combined2)
  })
  
  tar_make()
  tar_read(combi1)
  tar_visnetwork()
})
#> ▶ dispatched target distrib_mod_1
#> ● completed target distrib_mod_1 [0.001 seconds]
#> ▶ dispatched target distrib_mod_2
#> ● completed target distrib_mod_2 [0 seconds]
#> ▶ dispatched target estim_A_mod_1
#> ● completed target estim_A_mod_1 [0 seconds]
#> ▶ dispatched target estim_B_mod_1
#> ● completed target estim_B_mod_1 [0 seconds]
#> ▶ dispatched target estim_A_mod_2
#> ● completed target estim_A_mod_2 [0 seconds]
#> ▶ dispatched target estim_B_mod_2
#> ● completed target estim_B_mod_2 [0 seconds]
#> ▶ dispatched target combi1
#> ● completed target combi1 [0 seconds]
#> ▶ dispatched target combi2
#> ● completed target combi2 [0.001 seconds]
#> ▶ ended pipeline [0.34 seconds]

^{Created on 2024-06-05 with reprex v2.1.0}

Dynamic-within-static solve it thanks to the auto-aggregation

targets::tar_dir({
  library(targets)
  
  tar_script({
    library(targets)
    library(tarchetypes)
    tar_map(
      unlist = FALSE, # Return a nested list from tar_map()
      values = list(model = c("mod_1", "mod_2")),
      tar_target(
        distrib,
        tar_name(),
      ),
      tar_target(sims, c("A", "B")),
      tar_target(
        estim,
        paste(distrib, tar_name()),
        pattern = map(sims),
      )
    )
  })
  tar_make()
  tar_visnetwork(label = "branches")
})
#> ▶ dispatched target distrib_mod_1
#> ● completed target distrib_mod_1 [0.001 seconds]
#> ▶ dispatched target sims_mod_1
#> ● completed target sims_mod_1 [0 seconds]
#> ▶ dispatched target distrib_mod_2
#> ● completed target distrib_mod_2 [0 seconds]
#> ▶ dispatched target sims_mod_2
#> ● completed target sims_mod_2 [0 seconds]
#> ▶ dispatched branch estim_mod_1_d7ba40b079f612aa
#> ● completed branch estim_mod_1_d7ba40b079f612aa [0 seconds]
#> ▶ dispatched branch estim_mod_1_bd3a9abfe88d1ac8
#> ● completed branch estim_mod_1_bd3a9abfe88d1ac8 [0 seconds]
#> ● completed pattern estim_mod_1
#> ▶ dispatched branch estim_mod_2_f6fa6d3af286dc68
#> ● completed branch estim_mod_2_f6fa6d3af286dc68 [0 seconds]
#> ▶ dispatched branch estim_mod_2_9bead65bf54d6b30
#> ● completed branch estim_mod_2_9bead65bf54d6b30 [0 seconds]
#> ● completed pattern estim_mod_2
#> ▶ ended pipeline [0.307 seconds]

tar_read(estim_mod_1)
#>                estim_mod_1_d7ba40b079f612aa                 estim_mod_1_bd3a9abfe88d1ac8 
#> "distrib_mod_1 estim_mod_1_d7ba40b079f612aa" "distrib_mod_1 estim_mod_1_bd3a9abfe88d1ac8" 

^{Created on 2024-06-05 with reprex v2.1.0}

Question

Last reprex works great, scaling up will be fine, but if we want a full nested static pipeline, can we automatize the
combination to keep clear target names?

Versions

Platform: Arch GNU/Linux

• R 4.4.0
• {targets} 1.7.0
• {tarchetypes} 0.9.0

[wlandau]

You're looking for this graph, right?

If so, it may help to generate the tar_combine() target separately just using the pre-mapped estim targets:

library(targets)
library(tarchetypes)

inner <- tar_map(
  values = list(sim = c("A", "B")),
  tar_target(
    estim,
    paste(distrib, tar_name()),
  )
)

combine <- tar_combine(combi, inner, command = paste(!!!.x))

tar_map(
  values = list(model = c("mod_1", "mod_2")),
  tar_target(distrib, tar_name()),
  inner,
  combine
)

[ginolhac]

fantastic William, just brilliant. Just tested it with 3 inner and it looks perfect, thanks a ton!