Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow sequential_distibute to save/load large domains. #154

Merged
merged 2 commits into from
May 7, 2019
Merged

Allow sequential_distibute to save/load large domains. #154

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e747efa
Allow sequential_distibute to save/load large domains.
ninnghazad Oct 17, 2017
10eba3a
Removed dumb test-thing i forgot was in there.
ninnghazad Oct 18, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
32 changes: 24 additions & 8 deletions anuga/parallel/sequential_distribute.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@

import numpy as num


from anuga import Domain

from anuga.parallel.distribute_mesh import send_submesh
Expand Down Expand Up @@ -217,7 +216,7 @@ def sequential_distribute_dump(domain, numprocs=1, verbose=False, partition_dir=
"""

from os.path import join

partition = Sequential_distribute(domain, verbose, debug, parameters)

partition.distribute(numprocs)
Expand All @@ -234,17 +233,29 @@ def sequential_distribute_dump(domain, numprocs=1, verbose=False, partition_dir=
if exception.errno != errno.EEXIST:
raise


import cPickle
for p in range(0, numprocs):

tostore = partition.extract_submesh(p)

import cPickle
pickle_name = partition.domain_name + '_P%g_%g.pickle'% (numprocs,p)
pickle_name = join(partition_dir,pickle_name)
f = file(pickle_name, 'wb')
cPickle.dump( tostore, f, protocol=cPickle.HIGHEST_PROTOCOL)

lst = list(tostore)

# Write points and triangles to their own files
num.save(pickle_name+".np1",tostore[1],allow_pickle=False) # this append .npy to filename
lst[1] = pickle_name+".np1.npy"
num.save(pickle_name+".np2",tostore[2],allow_pickle=False)
lst[2] = pickle_name+".np2.npy"

# Write each quantity to it's own file
for k in tostore[4]:
num.save(pickle_name+".np4."+k,num.array(tostore[4][k]),allow_pickle=False)
lst[4][k] = pickle_name+".np4."+k+".npy"

cPickle.dump( tuple(lst), f, protocol=cPickle.HIGHEST_PROTOCOL)
return


Expand All @@ -257,17 +268,17 @@ def sequential_distribute_load(filename = 'domain', partition_dir = '.', verbose

pickle_name = filename+'_P%g_%g.pickle'% (numprocs,myid)
pickle_name = join(partition_dir,pickle_name)

return sequential_distribute_load_pickle_file(pickle_name, numprocs, verbose = verbose)


def sequential_distribute_load_pickle_file(pickle_name, np=1, verbose = False):
"""
Open pickle files
"""

import cPickle

f = file(pickle_name, 'rb')
import cPickle

kwargs, points, vertices, boundary, quantities, boundary_map, \
domain_name, domain_dir, domain_store, domain_store_centroids, \
Expand All @@ -276,6 +287,11 @@ def sequential_distribute_load_pickle_file(pickle_name, np=1, verbose = False):
domain_quantities_to_be_stored, domain_smooth = cPickle.load(f)
f.close()

for k in quantities:
quantities[k] = num.load(quantities[k])
points = num.load(points)
vertices = num.load(vertices)

#---------------------------------------------------------------------------
# Create domain (parallel if np>1)
#---------------------------------------------------------------------------
Expand Down