Skip to content

Commit

Permalink
Snakefile: Comment out group settings
Browse files Browse the repository at this point in the history
They are still experimental and lead to difficult to understand bugs.
For now we are advising to use submit the whole model generation up until the
prepare_network step on one compute node, i.e. submit a job running

snakemake --until prepare_network --resources mem=60000 -j4

for a 64GB compute node.

And only run the solve_network rule in parallel on the cluster.
  • Loading branch information
coroa committed Oct 26, 2018
1 parent c0fe95a commit ee6085e
Showing 1 changed file with 14 additions and 14 deletions.
28 changes: 14 additions & 14 deletions Snakefile
Expand Up @@ -24,7 +24,7 @@ if config['enable']['prepare_links_p_nom']:
output: 'data/links_p_nom.csv'
threads: 1
resources: mem=500
group: 'nonfeedin_preparation'
# group: 'nonfeedin_preparation'
script: 'scripts/prepare_links_p_nom.py'

if config['enable']['powerplantmatching']:
Expand All @@ -33,7 +33,7 @@ if config['enable']['powerplantmatching']:
output: "resources/powerplants.csv"
threads: 1
resources: mem=500
group: 'nonfeedin_preparation'
# group: 'nonfeedin_preparation'
script: "scripts/build_powerplants.py"

rule base_network:
Expand All @@ -53,7 +53,7 @@ rule base_network:
benchmark: "benchmarks/base_network"
threads: 1
resources: mem=500
group: 'nonfeedin_preparation'
# group: 'nonfeedin_preparation'
script: "scripts/base_network.py"

rule build_shapes:
Expand All @@ -72,7 +72,7 @@ rule build_shapes:
nuts3_shapes='resources/nuts3_shapes.geojson'
threads: 1
resources: mem=500
group: 'nonfeedin_preparation'
# group: 'nonfeedin_preparation'
script: "scripts/build_shapes.py"

rule build_bus_regions:
Expand All @@ -84,15 +84,15 @@ rule build_bus_regions:
regions_onshore="resources/regions_onshore.geojson",
regions_offshore="resources/regions_offshore.geojson"
resources: mem=1000
group: 'nonfeedin_preparation'
# group: 'nonfeedin_preparation'
script: "scripts/build_bus_regions.py"

rule build_cutout:
output: "cutouts/{cutout}"
resources: mem=5000
threads: config['atlite'].get('nprocesses', 4)
benchmark: "benchmarks/build_cutout_{cutout}"
group: 'feedin_preparation'
# group: 'feedin_preparation'
script: "scripts/build_cutout.py"

def memory_build_renewable_potentials(wildcards):
Expand All @@ -107,7 +107,7 @@ rule build_renewable_potentials:
output: "resources/potentials_{technology}.nc"
resources: mem=memory_build_renewable_potentials
benchmark: "benchmarks/build_renewable_potentials_{technology}"
group: 'feedin_preparation'
# group: 'feedin_preparation'
script: "scripts/build_renewable_potentials.py"

rule build_renewable_profiles:
Expand All @@ -122,7 +122,7 @@ rule build_renewable_profiles:
profile="resources/profile_{technology}.nc",
resources: mem=5000
benchmark: "benchmarks/build_renewable_profiles_{technology}"
group: 'feedin_preparation'
# group: 'feedin_preparation'
script: "scripts/build_renewable_profiles.py"

rule build_hydro_profile:
Expand All @@ -132,7 +132,7 @@ rule build_hydro_profile:
cutout="cutouts/" + config["renewable"]['hydro']['cutout']
output: 'resources/profile_hydro.nc'
resources: mem=5000
group: 'feedin_preparation'
# group: 'feedin_preparation'
script: 'scripts/build_hydro_profile.py'

rule add_electricity:
Expand All @@ -150,7 +150,7 @@ rule add_electricity:
benchmark: "benchmarks/add_electricity"
threads: 1
resources: mem=3000
group: 'build_pypsa_networks'
# group: 'build_pypsa_networks'
script: "scripts/add_electricity.py"

rule simplify_network:
Expand All @@ -166,7 +166,7 @@ rule simplify_network:
benchmark: "benchmarks/simplify_network/{network}_s{simpl}"
threads: 1
resources: mem=4000
group: 'build_pypsa_networks'
# group: 'build_pypsa_networks'
script: "scripts/simplify_network.py"

rule cluster_network:
Expand All @@ -183,7 +183,7 @@ rule cluster_network:
benchmark: "benchmarks/cluster_network/{network}_s{simpl}_{clusters}"
threads: 1
resources: mem=3000
group: 'build_pypsa_networks'
# group: 'build_pypsa_networks'
script: "scripts/cluster_network.py"

# rule add_sectors:
Expand All @@ -201,7 +201,7 @@ rule prepare_network:
output: 'networks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}.nc'
threads: 1
resources: mem=1000
benchmark: "benchmarks/prepare_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}"
# benchmark: "benchmarks/prepare_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}"
script: "scripts/prepare_network.py"

def memory(w):
Expand Down Expand Up @@ -244,7 +244,7 @@ rule solve_operations_network:
benchmark: "benchmarks/solve_operations_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}"
threads: 4
resources: mem=(lambda w: 5000 + 372 * int(w.clusters))
group: "solve_operations"
# group: "solve_operations"
script: "scripts/solve_operations_network.py"

rule plot_network:
Expand Down

0 comments on commit ee6085e

Please sign in to comment.