Add chunk_readylist

Reports disabled chunks’ inputs (number and how many available) as well
as dependencies (number and total downstream number)

NAMESPACE

@@ -5,6 +5,7 @@ export("module_gcam-usa_LA100.Socioeconomics")
 export(approx_fun)
 export(chunk_inputs)
 export(chunk_outputs)
+export(chunk_readylist)
 export(downscale_Maddison_country)
 export(driver)
 export(dstrace)

R/admin.R

@@ -0,0 +1,153 @@
+
+#' chunk_readylist
+#'
+#' @return Returns a list of disabled chunks: their lines of code, inputs and available inputs, dependencies
+#' @importFrom dplyr filter mutate select
+#' @importFrom tidyr gather spread
+#' @export
+chunk_readylist <- function() {
+  chunklist <- find_chunks(include_disabled = TRUE)
+  ci <- chunk_inputs(chunklist$name)
+  co <- chunk_outputs(chunklist$name)
+
+  filter(chunklist, !disabled) %>%
+    left_join(co, by = "name") %>%
+    select(output) %>%
+    mutate(available = TRUE) ->
+    enabled_outputs
+
+  # 'Ready list' is disabled chunks all of whose inputs are supplied by an enabled chunk
+  chunklist %>%
+    filter(disabled) %>%
+    left_join(ci, by = "name") %>%
+    filter(!from_file) %>%
+    select(-name) %>%
+    left_join(enabled_outputs, by = c("input" = "output")) %>%
+    mutate(available = if_else(is.na(available), FALSE, TRUE)) %>%
+    group_by(module, chunk, disabled) %>%
+    summarise(n_inputs = length(available),
+              n_avail = sum(available),
+              all_avail = all(available),
+              n_deps = count_dependencies(chunk[1], chunklist, ci, co),
+              n_deps_total = count_dependencies(chunk[1], chunklist, ci, co, TRUE)) ->
+    readylist
+
+  # Add number of code lines
+  admin.LINEDATA %>%
+    mutate(chunk = gsub("chunk_", "", filename) %>%
+             gsub("\\.R$", "_DISABLED", .)) %>%
+    select(-filename) %>%
+    right_join(readylist, by = "chunk")
+}
+
+
+# internal function, used by chunk_readylist above
+count_dependencies <- function(chunkname, chunklist, ci, co, recurse = FALSE) {
+  chunklist %>%
+    select(name, chunk) %>%
+    right_join(co, by = "name") %>%
+    filter(chunk == chunkname) %>%
+    select(output) %>%
+    inner_join(ci, by = c("output" = "input")) %>%
+    left_join(chunklist, by = "name") ->
+    outputlist
+
+  deps <- length(unique(outputlist$name))
+
+  if(recurse) {
+    for(i in unique(outputlist$chunk)) {
+      deps <- deps + count_dependencies(i, chunklist, ci, co, recurse)
+    }
+  }
+
+  deps
+}
+
+
+# This list of file line counts is generated by `chunk-generator.R`
+# Just pasted in here for use by chunk_readylist above
+admin.LINEDATA <- structure(list(
+  filename = c("chunk_LA100.FAO_downscale_ctry.R",
+               "chunk_LA100.GTAP_downscale_ctry.R", "chunk_LA100.IMAGE_downscale_ctry_yr.R",
+               "chunk_LA101.ag_FAO_R_C_Y.R", "chunk_LA102.ag_GTAP_R_C_AEZ.R",
+               "chunk_LA103.ag_R_C_Y_AEZ.R", "chunk_LA104.ag_Yield_Prod_adj.R",
+               "chunk_LA105.an_FAO_R_C_Y.R", "chunk_LA106.ag_an_NetExp_FAO_R_C_Y.R",
+               "chunk_LA107.an_IMAGE_R_C_Sys_Fd_Y.R", "chunk_LA108.ag_Feed_R_C_Y.R",
+               "chunk_LB109.ag_an_ALL_R_C_Y.R", "chunk_LB110.For_FAO_R_Y.R",
+               "chunk_LB111.ag_resbio_R_C.R", "chunk_LB112.ag_prodchange_R_C_Y.R",
+               "chunk_LB113.bio_Yield_R_AEZ.R", "chunk_LB114.ag_prodchange_scenarios.R",
+               "chunk_LB115.ag_CCI_ISIMIP.R", "chunk_LB120.LC_GIS_R_LTgis_Yh_AEZ.R",
+               "chunk_LB121.Carbon_LT.R", "chunk_LB122.LC_R_Cropland_Yh_AEZ.R",
+               "chunk_LB123.LC_R_MgdPastFor_Yh_AEZ.R", "chunk_LB124.LC_R_UnMgd_Yh_AEZ.R",
+               "chunk_LB125.LC_tot.R", "chunk_LB131.LV_R_AEZ.R", "chunk_LB132.ag_an_For_Prices_USA_C_2005.R",
+               "chunk_LB133.ag_Costs_USA_C_2005.R", "chunk_LB134.Diet_Rfao.R",
+               "chunk_LB141.ag_Fert_IFA_ctry_crop.R", "chunk_LB142.ag_Fert_IO_R_C_Y_AEZ.R",
+               "chunk_L201.ag_For_Past_bio_input.R", "chunk_L202.an_input.R",
+               "chunk_L203.demand_input.R", "chunk_L204.resbio_input.R", "chunk_L205.ag_prodchange_cost_input.R",
+               "chunk_L206.ag_Fert.R", "chunk_L211.land_input_1.R", "chunk_L212.land_input_2.R",
+               "chunk_L213.land_input_3.R", "chunk_L214.protected_land_input_2.R",
+               "chunk_L215.protected_land_input_3.R", "chunk_L101.nonghg_en_USA_S_T_Y.R",
+               "chunk_L102.ghg_en_USA_S_T_Y.R", "chunk_L103.ghg_an_USA_S_T_Y.R",
+               "chunk_L104.bcoc_en_USA_S_T_Y.R", "chunk_L105.nh3_an_USA_S_T_Y.R",
+               "chunk_L111.nonghg_en_R_S_T_Y.R", "chunk_L112.ghg_en_R_S_T_Y.R",
+               "chunk_L113.ghg_an_R_S_T_Y.R", "chunk_L114.bcoc_en_R_S_T_Y.R",
+               "chunk_L115.nh3_an_R_S_T_Y.R", "chunk_L121.nonco2_awb_R_S_T_Y.R",
+               "chunk_L122.ghg_agr_R_S_T_Y.R", "chunk_L123.bcoc_awb_R_S_T_Y.R",
+               "chunk_L124.nonco2_unmgd_R_S_T_Y.R", "chunk_L125.bcoc_unmgd_R_S_T_Y.R",
+               "chunk_L131.nonco2_proc_R_S_T_Y.R", "chunk_L141.hfc_R_S_T_Y.R",
+               "chunk_L142.pfc_R_S_T_Y.R", "chunk_L151.ctrl_R_en_S_T.R", "chunk_L152.MACC.R",
+               "chunk_L161.nonghg_en_ssp_R_S_T_Y.R", "chunk_L201.en_nonco2.R",
+               "chunk_L211.ag_nonco2.R", "chunk_L212.unmgd_nonco2.R", "chunk_L213.protected_unmgd_nonco2.R",
+               "chunk_L231.proc_sector.R", "chunk_L232.prc_nonco2.R", "chunk_L241.en_newtech_nonco2.R",
+               "chunk_L241.fgas.R", "chunk_L251.en_ssp_nonco2.R", "chunk_L252.MACC.R",
+               "chunk_LA100.CDIAC_downscale_ctry.R", "chunk_LA100.IEA_downscale_ctry.R",
+               "chunk_LA101.en_bal_IEA.R", "chunk_LA1011.en_bal_adj.R", "chunk_LA102.en_emiss_CDIAC.R",
+               "chunk_LA111.rsrc_fos_Prod.R", "chunk_LA112.U_DEMO.R", "chunk_LA112.U.R",
+               "chunk_LA113.MSW.R", "chunk_LA114.wind.R", "chunk_LA115.roofPV.R",
+               "chunk_LA116.geo.R", "chunk_LA117.tradbio.R", "chunk_LA118.hydro.R",
+               "chunk_LA119.solar.R", "chunk_LA121.oil.R", "chunk_LA122.gasproc_refining.R",
+               "chunk_LA123.electricity.R", "chunk_LA1231.elec_tech.R", "chunk_LA124.heat.R",
+               "chunk_LA126.distribution.R", "chunk_LA131.enduse.R", "chunk_LA132.industry.R",
+               "chunk_LA1321.cement.R", "chunk_LA142.building_agg.R", "chunk_LA143.HDDCDD.R",
+               "chunk_LA144.building_det_en.R", "chunk_LA144.building_det_flsp.R",
+               "chunk_LA152.transportation.R", "chunk_LA154.transportation_UCD.R",
+               "chunk_LA161.Cstorage.R", "chunk_LB1322.Fert.R", "chunk_L202.Ccoef.R",
+               "chunk_L210.resources.R", "chunk_L221.en_supply.R", "chunk_L222.en_transformation.R",
+               "chunk_L223.electricity.R", "chunk_L224.heat.R", "chunk_L225.hydrogen.R",
+               "chunk_L226.en_distribution.R", "chunk_L232.industry.R", "chunk_L2321.cement.R",
+               "chunk_L2322.Fert.R", "chunk_L242.building_agg.R", "chunk_L244.building_det.R",
+               "chunk_L252.transportation.R", "chunk_L254.transportation_UCD.R",
+               "chunk_L261.Cstorage.R", "chunk_LA100.Socioeconomics.R", "chunk_LA101.EIA_SEDS.R",
+               "chunk_LA114.Wind.R", "chunk_LA115.RooftopPV.R", "chunk_LA119.Solar.R",
+               "chunk_LA122.Refining.R", "chunk_LA132.Industry.R", "chunk_LA1321.Cement.R",
+               "chunk_LA1322.Fert.R", "chunk_LA142.Building.R", "chunk_LA144.Commercial.R",
+               "chunk_LA144.Residential.R", "chunk_LA154.Transport.R", "chunk_LB123.Electricity.R",
+               "chunk_LB1231.Elec_tech.R", "chunk_LB1232.Elec_subregions.R",
+               "chunk_LB1233.Elec_water.R", "chunk_LB126.Gas_ElecTD.R", "chunk_L201.socioeconomics_USA.R",
+               "chunk_L210.resources_USA.R", "chunk_L222.en_transformation_USA.R",
+               "chunk_L223.electricity_USA.R", "chunk_L2232.electricity_FERC_USA.R",
+               "chunk_L225.hydrogen_USA.R", "chunk_L226.en_distribution_USA.R",
+               "chunk_L232.industry_USA.R", "chunk_L2321.cement_USA.R", "chunk_L2322.Fert_USA.R",
+               "chunk_L244.building_USA.R", "chunk_L254.transportation_USA.R",
+               "chunk_L261.carbon_storage_USA.R", "chunk_L200.modeltime.R",
+               "chunk_L100.GDP_hist.R", "chunk_L100.Population_downscale_ctry.R",
+               "chunk_L101.Population.R", "chunk_L102.GDP.R", "chunk_L201.Pop_GDP_scenarios.R",
+               "chunk_L242.Bld_Inc_Elas_scenarios.R", "chunk_L252.Trn_Inc_Elas_scenarios.R"),
+  lines = c(225L, 77L, 59L, 118L, 65L, 85L, 99L, 106L, 140L,
+            94L, 183L, 156L, 89L, 73L, 165L, 130L, 98L, 208L, 135L, 189L,
+            212L, 221L, 99L, 87L, 64L, 161L, 205L, 166L, 119L, 177L, 294L,
+            396L, 504L, 192L, 281L, 138L, 159L, 157L, 368L, 126L, 133L, 265L,
+            79L, 72L, 83L, 82L, 177L, 130L, 100L, 124L, 98L, 92L, 141L, 69L,
+            140L, 95L, 112L, 121L, 94L, 78L, 97L, 318L, 163L, 203L, 237L,
+            265L, 233L, 79L, 134L, 110L, 82L, 141L, 93L, 185L, 164L, 122L,
+            98L, 154L, 64L, 56L, 73L, 138L, 75L, 114L, 60L, 128L, 116L, 99L,
+            210L, 89L, 127L, 117L, 96L, 119L, 109L, 236L, 69L, 137L, 377L,
+            193L, 44L, 366L, 73L, 289L, 73L, 319L, 351L, 266L, 584L, 221L,
+            149L, 197L, 422L, 294L, 270L, 231L, 660L, 180L, 469L, 179L, 89L,
+            73L, 69L, 142L, 58L, 167L, 159L, 66L, 66L, 89L, 319L, 366L, 103L,
+            131L, 72L, 47L, 181L, 237L, 85L, 164L, 268L, 475L, 304L, 37L,
+            228L, 236L, 197L, 214L, 541L, 243L, 140L, 86L, 43L, 139L, 95L,
+            189L, 211L, 92L, 92L)),
+  .Names = c("filename", "lines"),
+  row.names = c(NA, -159L),
+  class = c("tbl_df", "tbl", "data.frame"))

chunk-generator/chunk-generator.R

@@ -189,6 +189,7 @@ files <- list.files("~/Documents/Work/Code/gcam-data-system-OLD/",
 # Limit to scripts in the processing code folers
 files <- files[grepl("processing-code", files, fixed = TRUE)]
 
+linedata <- list()
 
 for(fn in files) {
   # Isolate the module and level information from the filename
@@ -206,4 +207,10 @@ for(fn in files) {
     newfn <- paste0("chunk-generator/outputs/chunk_", basename(fn))
     cat(out, "\n", file = newfn, sep = "\n", append = FALSE)
   }
+  linedata[[newfn]] <- tibble(filename = basename(newfn),
+                              lines = length(readLines(fn)))
+
 }
+
+linedata <- dplyr::bind_rows(linedata)
+readr::write_csv(linedata, "chunk-generator/linedata.csv")

chunk-generator/linedata.csv

@@ -0,0 +1,160 @@
+filename,lines
+chunk_LA100.FAO_downscale_ctry.R,225
+chunk_LA100.GTAP_downscale_ctry.R,77
+chunk_LA100.IMAGE_downscale_ctry_yr.R,59
+chunk_LA101.ag_FAO_R_C_Y.R,118
+chunk_LA102.ag_GTAP_R_C_AEZ.R,65
+chunk_LA103.ag_R_C_Y_AEZ.R,85
+chunk_LA104.ag_Yield_Prod_adj.R,99
+chunk_LA105.an_FAO_R_C_Y.R,106
+chunk_LA106.ag_an_NetExp_FAO_R_C_Y.R,140
+chunk_LA107.an_IMAGE_R_C_Sys_Fd_Y.R,94
+chunk_LA108.ag_Feed_R_C_Y.R,183
+chunk_LB109.ag_an_ALL_R_C_Y.R,156
+chunk_LB110.For_FAO_R_Y.R,89
+chunk_LB111.ag_resbio_R_C.R,73
+chunk_LB112.ag_prodchange_R_C_Y.R,165
+chunk_LB113.bio_Yield_R_AEZ.R,130
+chunk_LB114.ag_prodchange_scenarios.R,98
+chunk_LB115.ag_CCI_ISIMIP.R,208
+chunk_LB120.LC_GIS_R_LTgis_Yh_AEZ.R,135
+chunk_LB121.Carbon_LT.R,189
+chunk_LB122.LC_R_Cropland_Yh_AEZ.R,212
+chunk_LB123.LC_R_MgdPastFor_Yh_AEZ.R,221
+chunk_LB124.LC_R_UnMgd_Yh_AEZ.R,99
+chunk_LB125.LC_tot.R,87
+chunk_LB131.LV_R_AEZ.R,64
+chunk_LB132.ag_an_For_Prices_USA_C_2005.R,161
+chunk_LB133.ag_Costs_USA_C_2005.R,205
+chunk_LB134.Diet_Rfao.R,166
+chunk_LB141.ag_Fert_IFA_ctry_crop.R,119
+chunk_LB142.ag_Fert_IO_R_C_Y_AEZ.R,177
+chunk_L201.ag_For_Past_bio_input.R,294
+chunk_L202.an_input.R,396
+chunk_L203.demand_input.R,504
+chunk_L204.resbio_input.R,192
+chunk_L205.ag_prodchange_cost_input.R,281
+chunk_L206.ag_Fert.R,138
+chunk_L211.land_input_1.R,159
+chunk_L212.land_input_2.R,157
+chunk_L213.land_input_3.R,368
+chunk_L214.protected_land_input_2.R,126
+chunk_L215.protected_land_input_3.R,133
+chunk_L101.nonghg_en_USA_S_T_Y.R,265
+chunk_L102.ghg_en_USA_S_T_Y.R,79
+chunk_L103.ghg_an_USA_S_T_Y.R,72
+chunk_L104.bcoc_en_USA_S_T_Y.R,83
+chunk_L105.nh3_an_USA_S_T_Y.R,82
+chunk_L111.nonghg_en_R_S_T_Y.R,177
+chunk_L112.ghg_en_R_S_T_Y.R,130
+chunk_L113.ghg_an_R_S_T_Y.R,100
+chunk_L114.bcoc_en_R_S_T_Y.R,124
+chunk_L115.nh3_an_R_S_T_Y.R,98
+chunk_L121.nonco2_awb_R_S_T_Y.R,92
+chunk_L122.ghg_agr_R_S_T_Y.R,141
+chunk_L123.bcoc_awb_R_S_T_Y.R,69
+chunk_L124.nonco2_unmgd_R_S_T_Y.R,140
+chunk_L125.bcoc_unmgd_R_S_T_Y.R,95
+chunk_L131.nonco2_proc_R_S_T_Y.R,112
+chunk_L141.hfc_R_S_T_Y.R,121
+chunk_L142.pfc_R_S_T_Y.R,94
+chunk_L151.ctrl_R_en_S_T.R,78
+chunk_L152.MACC.R,97
+chunk_L161.nonghg_en_ssp_R_S_T_Y.R,318
+chunk_L201.en_nonco2.R,163
+chunk_L211.ag_nonco2.R,203
+chunk_L212.unmgd_nonco2.R,237
+chunk_L213.protected_unmgd_nonco2.R,265
+chunk_L231.proc_sector.R,233
+chunk_L232.prc_nonco2.R,79
+chunk_L241.en_newtech_nonco2.R,134
+chunk_L241.fgas.R,110
+chunk_L251.en_ssp_nonco2.R,82
+chunk_L252.MACC.R,141
+chunk_LA100.CDIAC_downscale_ctry.R,93
+chunk_LA100.IEA_downscale_ctry.R,185
+chunk_LA101.en_bal_IEA.R,164
+chunk_LA1011.en_bal_adj.R,122
+chunk_LA102.en_emiss_CDIAC.R,98
+chunk_LA111.rsrc_fos_Prod.R,154
+chunk_LA112.U_DEMO.R,64
+chunk_LA112.U.R,56
+chunk_LA113.MSW.R,73
+chunk_LA114.wind.R,138
+chunk_LA115.roofPV.R,75
+chunk_LA116.geo.R,114
+chunk_LA117.tradbio.R,60
+chunk_LA118.hydro.R,128
+chunk_LA119.solar.R,116
+chunk_LA121.oil.R,99
+chunk_LA122.gasproc_refining.R,210
+chunk_LA123.electricity.R,89
+chunk_LA1231.elec_tech.R,127
+chunk_LA124.heat.R,117
+chunk_LA126.distribution.R,96
+chunk_LA131.enduse.R,119
+chunk_LA132.industry.R,109
+chunk_LA1321.cement.R,236
+chunk_LA142.building_agg.R,69
+chunk_LA143.HDDCDD.R,137
+chunk_LA144.building_det_en.R,377
+chunk_LA144.building_det_flsp.R,193
+chunk_LA152.transportation.R,44
+chunk_LA154.transportation_UCD.R,366
+chunk_LA161.Cstorage.R,73
+chunk_LB1322.Fert.R,289
+chunk_L202.Ccoef.R,73
+chunk_L210.resources.R,319
+chunk_L221.en_supply.R,351
+chunk_L222.en_transformation.R,266
+chunk_L223.electricity.R,584
+chunk_L224.heat.R,221
+chunk_L225.hydrogen.R,149
+chunk_L226.en_distribution.R,197
+chunk_L232.industry.R,422
+chunk_L2321.cement.R,294
+chunk_L2322.Fert.R,270
+chunk_L242.building_agg.R,231
+chunk_L244.building_det.R,660
+chunk_L252.transportation.R,180
+chunk_L254.transportation_UCD.R,469
+chunk_L261.Cstorage.R,179
+chunk_LA100.Socioeconomics.R,89
+chunk_LA101.EIA_SEDS.R,73
+chunk_LA114.Wind.R,69
+chunk_LA115.RooftopPV.R,142
+chunk_LA119.Solar.R,58
+chunk_LA122.Refining.R,167
+chunk_LA132.Industry.R,159
+chunk_LA1321.Cement.R,66
+chunk_LA1322.Fert.R,66
+chunk_LA142.Building.R,89
+chunk_LA144.Commercial.R,319
+chunk_LA144.Residential.R,366
+chunk_LA154.Transport.R,103
+chunk_LB123.Electricity.R,131
+chunk_LB1231.Elec_tech.R,72
+chunk_LB1232.Elec_subregions.R,47
+chunk_LB1233.Elec_water.R,181
+chunk_LB126.Gas_ElecTD.R,237
+chunk_L201.socioeconomics_USA.R,85
+chunk_L210.resources_USA.R,164
+chunk_L222.en_transformation_USA.R,268
+chunk_L223.electricity_USA.R,475
+chunk_L2232.electricity_FERC_USA.R,304
+chunk_L225.hydrogen_USA.R,37
+chunk_L226.en_distribution_USA.R,228
+chunk_L232.industry_USA.R,236
+chunk_L2321.cement_USA.R,197
+chunk_L2322.Fert_USA.R,214
+chunk_L244.building_USA.R,541
+chunk_L254.transportation_USA.R,243
+chunk_L261.carbon_storage_USA.R,140
+chunk_L200.modeltime.R,86
+chunk_L100.GDP_hist.R,43
+chunk_L100.Population_downscale_ctry.R,139
+chunk_L101.Population.R,95
+chunk_L102.GDP.R,189
+chunk_L201.Pop_GDP_scenarios.R,211
+chunk_L242.Bld_Inc_Elas_scenarios.R,92
+chunk_L252.Trn_Inc_Elas_scenarios.R,92