Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Ref/remote download #598

Merged
merged 7 commits into from
May 5, 2017
Merged

Ref/remote download #598

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
97e4488
update remote download to handle more of the path wrangling as per #596
lheagy May 3, 2017
102d76f
fix filepaths in casing example
lheagy May 4, 2017
e5c149f
rename `remoteDownload` to `download`, only overwrite files not direc…
lheagy May 4, 2017
9d4172e
add cleanup test of downloads
lheagy May 4, 2017
88348c2
fix tests and examples that I missed in the first go-around
lheagy May 4, 2017
cc27951
clean up how url is printed so that it is clickable in the notebook
lheagy May 5, 2017
2818916
path --> folder
lheagy May 5, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
8 changes: 5 additions & 3 deletions SimPEG/EM/Base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,9 @@ def MeSigmaI(self):
Inverse of the edge inner product matrix for \\(\\sigma\\).
"""
if getattr(self, '_MeSigmaI', None) is None:
self._MeSigmaI = self.mesh.getEdgeInnerProduct(self.sigma, invMat=True)
self._MeSigmaI = self.mesh.getEdgeInnerProduct(
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is just white-space

self.sigma, invMat=True
)
return self._MeSigmaI

# TODO: This should take a vector
Expand Down Expand Up @@ -425,7 +427,7 @@ def s_e(self, prob):
"""
return Utils.Zero()

def s_mDeriv(self, prob, v, adjoint = False):
def s_mDeriv(self, prob, v, adjoint=False):
"""
Derivative of magnetic source term with respect to the inversion model

Expand All @@ -438,7 +440,7 @@ def s_mDeriv(self, prob, v, adjoint = False):

return Utils.Zero()

def s_eDeriv(self, prob, v, adjoint = False):
def s_eDeriv(self, prob, v, adjoint=False):
"""
Derivative of electric source term with respect to the inversion model

Expand Down
68 changes: 53 additions & 15 deletions SimPEG/Utils/io_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from SimPEG import Mesh
import time as tm
import re
import warnings


def read_GOCAD_ts(tsfile):
Expand Down Expand Up @@ -146,35 +147,72 @@ def surface2inds(vrtx, trgl, mesh, boundaries=True, internal=True):
return insideGrid


def remoteDownload(url, remoteFiles, basePath=None):
def remoteDownload(
url, remoteFiles, path='.', directory='SimPEGtemp', rm_previous=False
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

see what you think of these input names @fourndo, @rowanc1

):
"""
Function to download all files stored in a cloud directory
var: url ("http:\\...")
list: List of file names to download

:param str url: url of the storage bucket ("http://...")
:param list remoteFiles: List of file names to download from the storate bucket
:param str path: path to where the directory is created and files downloaded (default is the current directory)
:param str directory: name of the directory to be created and have content downloaded to
:param bool rm_previous: remove file and contents if a directory with the specified name already exists
"""

# Download from cloud
import urllib
import shutil
import os
import sys

def rename_path(downloadpath):
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if we start to make heavy use of this function, I can see a number of scenarios where we do not necessarily want to over-write the contents of SimPEGtemp, so here it will add SimPEGtemp(1) (or up the integer value until we hit a unique name, similar to what you would see with a web browser)

splitfullpath = downloadpath.split(os.path.sep)
curdir = splitfullpath[-1]
splitdir = curdir.split('(')
rootdir = splitdir[0]

# add (num) to the end of the filename
if len(splitdir) == 1:
num = 1
else:
num = int(splitdir[-1][:-1])
num += 1

return os.path.sep.join(
splitfullpath[:-1] + [rootdir + '({})'.format(num)]
)

# grab the correct url retriever
if sys.version_info < (3,):
urlretrieve = urllib.urlretrieve
else:
urlretrieve = urllib.request.urlretrieve

if basePath is None:
basePath = os.curdir+os.path.sep+'SimPEGtemp'+os.path.sep

if os.path.exists(basePath):
shutil.rmtree(basePath)

os.makedirs(basePath)

print("Download files from URL...")
# ensure we are working with absolute paths
path = os.path.abspath(path)
downloadpath = os.path.sep.join([path]+[directory])
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[path, directory]? Same difference.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, this is likely confusing. path is the path to your working directory, directory is the name of the new folder we are adding where we download things.

Thoughts on alternatives?

# check if the directory already exists
while os.path.exists(downloadpath):
if rm_previous is True:
print("removing previous contents of {}".format(downloadpath))
shutil.rmtree(downloadpath)
elif rm_previous is False:
downloadpath = rename_path(downloadpath)
print(
"directory already exists, new downloads will be in {}".format(
downloadpath
)
)

# create the directory
os.makedirs(downloadpath+os.path.sep)

# download files
print("Download files from {}...".format(url))
for file in remoteFiles:
print("Retrieving: " + file)
urlretrieve(url + file, basePath+file)
print(" Retrieving: " + file)
urlretrieve(url + file, os.path.sep.join([downloadpath]+[file]))

print("Download completed!")
return basePath
return downloadpath
20 changes: 12 additions & 8 deletions examples/04-grav/plot_laguna_del_maule_inversion.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,18 @@ def run(plotIt=True, cleanAfterRun=True):

# Start by downloading files from the remote repository
url = "https://storage.googleapis.com/simpeg/Chile_GRAV_4_Miller/"
cloudfiles = ['LdM_grav_obs.grv', 'LdM_mesh.mesh',
'LdM_topo.topo', 'LdM_input_file.inp']

basePath = os.path.sep.join(os.path.abspath(os.getenv('HOME')).split
(os.path.sep)+['Downloads']+['SimPEGtemp'])
basePath = os.path.abspath(remoteDownload(url,
cloudfiles,
basePath=basePath+os.path.sep))
cloudfiles = [
'LdM_grav_obs.grv', 'LdM_mesh.mesh',
'LdM_topo.topo', 'LdM_input_file.inp'
]

# Download to Downloads/SimPEGtemp
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here we are not specifying where things get put? I think they should just go directly into pwd? Unless you specify a folder or something?

Can we just rename this to download? seems a bit redundant.

file_names = download([url1, url2], folder='~/Downloads/mag_stuff', overwrite=True)
# or 
file_name = download(url1)
# where
assert isinstance(file_names, list)
assert len(file_names) == 2
assert isinstance(file_name, str)

I think that the default should the the pwd of where the python file is run from, appending a SimPEGtemp seems odd to me. Downloads are downloads.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This also would mean that you don't have to reconstruct the file name after the fact, you just loop through the list:

with open(file_name, 'r') as f:
    f.read()

with open(file_names[0], 'r') as f2:
    f2.read()

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  • download works for me. @fourndo ?
  • SimPEGtemp was mainly motivated because we make heavy use of this in examples, so for someone unfamiliar it is easy to find (we can still do this and specify it in the example script rather than the util). However, I agree that can be cleared up by specifying the folder (I am tempted to call it path or directory instead as it is not just a name of a folder but does include path information)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that it could be both? You can call abspath on the thing and it should expand it out regardless.

folder='here'
folder='or/here'
folder='/Users/rowan/downloads/or/even/here'

I see all of these as folders?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, fair enough, folder can work

basePath = remoteDownload(
url, cloudfiles,
path=os.path.sep.join([os.getenv('HOME'), 'Downloads']),
rm_previous=True
)

input_file = basePath + os.path.sep + 'LdM_input_file.inp'
# %% User input
# Plotting parameters, max and min densities in g/cc
Expand Down
7 changes: 4 additions & 3 deletions examples/07-fdem/plot_heagyetal2016_casing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1246,16 +1246,17 @@ class PrimSecCasingStoredResults(PrimSecCasingExample):
@property
def filepath(self):
return os.path.sep.join(
os.path.abspath(os.getenv('HOME')).split(os.path.sep) +
['Downloads'] + ['SimPEGtemp']
[os.path.abspath(os.getenv('HOME')), 'Downloads', 'SimPEGtemp']
)

def downloadStoredResults(self):
# download the results from where they are stored on google app engine

return os.path.abspath(
remoteDownload(
self.url, [self.cloudfile], basePath=self.filepath+os.path.sep
self.url, [self.cloudfile], path=os.path.sep.join(
[os.path.abspath(os.getenv('HOME')), 'Downloads']
), rm_previous=True
)
)

Expand Down
15 changes: 11 additions & 4 deletions tests/pf/test_gravity_IO.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from SimPEG.Utils import io_utils
from scipy.constants import mu_0
import shutil
import os


class MagSensProblemTests(unittest.TestCase):
Expand All @@ -13,11 +14,13 @@ def setUp(self):
cloudfiles = ['GravData.obs', 'Gaussian.topo', 'Mesh_10m.msh',
'ModelStart.sus', 'SimPEG_Grav_Input.inp']

self.basePath = io_utils.remoteDownload(url, cloudfiles)
self.basePath = io_utils.remoteDownload(
url, cloudfiles, rm_previous=True
)

def test_magnetics_inversion(self):

inp_file = self.basePath + 'SimPEG_Grav_Input.inp'
inp_file = os.path.sep.join([self.basePath, 'SimPEG_Grav_Input.inp'])

driver = PF.GravityDriver.GravityDriver_Inv(inp_file)

Expand All @@ -36,8 +39,12 @@ def test_magnetics_inversion(self):
print(driver.eps)

# Write obs to file
PF.Gravity.writeUBCobs(self.basePath + 'FWR_data.dat',
driver.survey, driver.survey.dobs)
PF.Gravity.writeUBCobs(
os.path.sep.join(
[self.basePath, 'FWR_data.dat']
),
driver.survey, driver.survey.dobs
)

# Clean up the working directory
shutil.rmtree(self.basePath)
Expand Down
9 changes: 6 additions & 3 deletions tests/pf/test_magnetics_IO.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from SimPEG.Utils import io_utils
from scipy.constants import mu_0
import shutil
import os


class MagSensProblemTests(unittest.TestCase):
Expand All @@ -17,7 +18,7 @@ def setUp(self):

def test_magnetics_inversion(self):

inp_file = self.basePath + 'SimPEG_Mag_Input.inp'
inp_file = os.path.sep.join([self.basePath, 'SimPEG_Mag_Input.inp'])

driver = PF.MagneticsDriver.MagneticsDriver_Inv(inp_file)

Expand All @@ -36,8 +37,10 @@ def test_magnetics_inversion(self):
print(driver.eps)

# Write obs to file
PF.Magnetics.writeUBCobs(self.basePath + 'FWR_data.dat',
driver.survey, driver.survey.dobs)
PF.Magnetics.writeUBCobs(
os.path.sep.join([self.basePath, 'FWR_data.dat']),
driver.survey, driver.survey.dobs
)

# Clean up the working directory
shutil.rmtree(self.basePath)
Expand Down