Skip to content

Commit

Permalink
Refactor common stereo commands (#27)
Browse files Browse the repository at this point in the history 
* need to simplify stereo command to single call with different kwargs
* some code de-duplication for parallel_stereo calls and improved common stereo step
* simplifications for step 10/11, optional expanding utility function
* added postfix configuration and fixups for hirise side
  • Loading branch information
@AndrewAnnex
AndrewAnnex committed Jul 7, 2021
1 parent 5f4f8c5 commit 614e913
Showing 1 changed file with 93 additions and 99 deletions.
192 changes: 93 additions & 99 deletions asap_stereo/asap.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from string import Template

import affine
import struct
import csv
Expand Down Expand Up @@ -62,6 +64,31 @@
_threads_multiprocess = _threads_singleprocess // 2 if _threads_singleprocess > 1 else 1 # 12, 8
_processes = _threads_multiprocess // 4 if _threads_multiprocess > 3 else 1 # 3, 2

# defaults for first 3 steps parallel stereo
defaults_ps1 = {
'--processes' : _processes,
'--threads-singleprocess': _threads_singleprocess,
'--threads-multiprocess' : _threads_multiprocess,
'--stop-point' : 4,
'--bundle-adjust-prefix' : 'adjust/ba'
}

# defaults for first last step parallel stereo (triangulation)
defaults_ps2 = {
'--processes' : _threads_singleprocess, # use more cores for triangulation!
'--threads-singleprocess': _threads_singleprocess,
'--threads-multiprocess' : _threads_multiprocess,
'--entry-point' : 4,
'--bundle-adjust-prefix' : 'adjust/ba'
}

# default eqc Iau projections, eventually replace with proj4 lookups
projections = {
"IAU_Mars" : "+proj=eqc +lat_ts=0 +lat_0=0 +lon_0=0 +x_0=0 +y_0=0 +a=3396190 +b=3396190 +units=m +no_defs",
"IAU_Moon" : "+proj=eqc +lat_ts=0 +lat_0=0 +lon_0=0 +x_0=0 +y_0=0 +a=1737400 +b=1737400 +units=m +no_defs",
"IAU_Mercury": "+proj=eqc +lat_ts=0 +lat_0=0 +lon_0=0 +x_0=0 +y_0=0 +a=2439700 +b=2439700 +units=m +no_defs"
}


pool = Semaphore(cores)

Expand Down Expand Up @@ -90,6 +117,14 @@ def silent_cd(newdir):
finally:
os.chdir(prevdir)

def optional(variable, null=''):
# TODO: this is equivalent to something from functional programming that I am forgetting the name of
if isinstance(variable, (bool, int, float, str, Path)):
variable = [variable]
for _ in variable:
if _ != null:
yield _

def cmd_to_string(command: sh.RunningCommand) -> str:
"""
Converts the running command into a single string of the full command call for easier logging
Expand Down Expand Up @@ -235,31 +270,6 @@ class CommonSteps(object):
█████████████████████████████████████████████████████████████
"""

# defaults for first 3 steps parallel stereo
defaults_ps1 = {
'--processes': _processes,
'--threads-singleprocess': _threads_singleprocess,
'--threads-multiprocess': _threads_multiprocess,
'--stop-point': 4,
'--bundle-adjust-prefix': 'adjust/ba'
}

# defaults for first last step parallel stereo (triangulation)
defaults_ps2 = {
'--processes' : _threads_singleprocess, # use more cores for triangulation!
'--threads-singleprocess': _threads_singleprocess,
'--threads-multiprocess' : _threads_multiprocess,
'--entry-point' : 4,
'--bundle-adjust-prefix' : 'adjust/ba'
}

# default eqc Iau projections, eventually replace with proj4 lookups
projections = {
"IAU_Mars": "+proj=eqc +lat_ts=0 +lat_0=0 +lon_0=0 +x_0=0 +y_0=0 +a=3396190 +b=3396190 +units=m +no_defs",
"IAU_Moon": "+proj=eqc +lat_ts=0 +lat_0=0 +lon_0=0 +x_0=0 +y_0=0 +a=1737400 +b=1737400 +units=m +no_defs",
"IAU_Mercury": "+proj=eqc +lat_ts=0 +lat_0=0 +lon_0=0 +x_0=0 +y_0=0 +a=2439700 +b=2439700 +units=m +no_defs"
}

def __init__(self):
self.parallel_stereo = Command('parallel_stereo').bake(_out=sys.stdout, _err=sys.stderr)
self.point2dem = Command('point2dem').bake(_out=sys.stdout, _err=sys.stderr)
Expand Down Expand Up @@ -587,7 +597,7 @@ def get_pedr_4_pcalign(self, cub_path, pedr_path, proj):
sh.awk(sh.sed(proj_tab, 's/\\t/,/g'),'-F,','{print($5","$4","$3","$1","$2","$6)}', _out=f'./{out_name}_pedr4align.csv')

@rich_logger
def bundle_adjust(self, *vargs, postfix='_RED.map.cub', bundle_adjust_prefix='adjust/ba', **kwargs):
def bundle_adjust(self, *vargs, postfix='_RED.map.cub', bundle_adjust_prefix='adjust/ba', **kwargs)-> sh.RunningCommand:
"""
Bundle adjustment wrapper
Expand All @@ -609,36 +619,29 @@ def bundle_adjust(self, *vargs, postfix='_RED.map.cub', bundle_adjust_prefix='ad
return self.ba(f'{left}{postfix}', f'{right}{postfix}', *vargs, '-o', bundle_adjust_prefix, '--save-cnet-as-csv', *args)

@rich_logger
def stereo_1(self, stereo_conf: str, postfix='.lev1eo.cub', **kwargs):
def stereo_asap(self, stereo_conf: str, refdem: str = '', postfix='.lev1eo.cub', output_file_prefix='results_ba/${both}_ba', posargs: str = '', **kwargs):
"""
Step 1 of parallel stereo
:param postfix:
:param stereo_conf:
:param kwargs:
"""
left, right, both = self.parse_stereopairs()
assert both is not None
stereo_conf = Path(stereo_conf).absolute()
with cd(Path.cwd() / both):
args = kwargs_to_args({**self.defaults_ps1, **clean_kwargs(kwargs)})
return self.parallel_stereo(*args, f'{left}{postfix}', f'{right}{postfix}', '-s', stereo_conf, f'results_ba/{both}_ba')

@rich_logger
def stereo_2(self, stereo_conf: str, postfix='.lev1eo.cub', **kwargs):
"""
Step 2 of parallel stereo
:param postfix:
:param stereo_conf:
:param kwargs:
parallel stereo common step
:param output_file_prefix: template string for output file prefix
:param refdem: optional reference DEM for 2nd pass stereo
:param posargs: additional positional args
:param postfix: postfix(s) to use for input images
:param stereo_conf: stereo config file
:param kwargs: keyword arguments for parallel_stereo
"""
left, right, both = self.parse_stereopairs()
assert both is not None
output_file_prefix = Template(output_file_prefix).safe_substitute(both=both)
stereo_conf = Path(stereo_conf).absolute()
with cd(Path.cwd() / both):
args = kwargs_to_args({**self.defaults_ps2, **clean_kwargs(kwargs)})
return self.parallel_stereo(*args, f'{left}{postfix}', f'{right}{postfix}', '-s', stereo_conf, f'results_ba/{both}_ba')
kwargs['--stereo-file'] = stereo_conf
_kwargs = kwargs_to_args(clean_kwargs(kwargs))
_posargs = posargs.split(' ')
if isinstance(postfix, str):
postfix = [postfix]
imgs = list(itertools.chain([[f'{left}{px}', f'{right}{px}'] for px in postfix]))
return self.parallel_stereo(*optional(_posargs), *_kwargs, *imgs, output_file_prefix, *optional(refdem))

@rich_logger
def rescale_cub(self, src_file: str, factor=4, overwrite=False, dst_file=None):
Expand Down Expand Up @@ -746,7 +749,12 @@ def __init__(self, https=False, datum="D_MARS", proj: Optional[str] = None):
self.datum = datum
# if proj is not none, get the corresponding proj or else override with proj,
# otherwise it's a none so remain a none
self.proj = CommonSteps.projections.get(proj, proj)
self.proj = projections.get(proj, proj)

def get_first_pass_refdem(self)-> str:
left, right, both = self.cs.parse_stereopairs()
refdem = Path.cwd() / both / 'results_ba' / 'dem' / f'{both}_ba_100_0-DEM.tif'
return str(refdem)

def get_full_ctx_id(self, pid):
res = str(moody.ODE(self.https).get_ctx_meta_by_key(pid, 'ProductURL'))
Expand Down Expand Up @@ -874,22 +882,22 @@ def step_four(self, *vargs, bundle_adjust_prefix='adjust/ba', **kwargs)-> sh.Run
return self.cs.bundle_adjust(*vargs, postfix='.lev1eo.cub', bundle_adjust_prefix=bundle_adjust_prefix, **kwargs)

@rich_logger
def step_five(self, stereo_conf, **kwargs):
def step_five(self, stereo_conf, posargs='', **kwargs):
"""
Parallel Stereo Part 1
Run first part of parallel_stereo asp_ctx_lev1eo2dem.sh
"""
return self.cs.stereo_1(stereo_conf, postfix='.lev1eo.cub', **kwargs)
return self.cs.stereo_asap(stereo_conf, postfix='.lev1eo.cub', posargs=posargs, **{**defaults_ps1, **kwargs})

@rich_logger
def step_six(self, stereo_conf, **kwargs):
def step_six(self, stereo_conf, posargs='', **kwargs):
"""
Parallel Stereo Part 2
Run second part of parallel_stereo, asp_ctx_lev1eo2dem.sh stereo is completed after this step
"""
return self.cs.stereo_2(stereo_conf, postfix='.lev1eo.cub', **kwargs)
return self.cs.stereo_asap(stereo_conf, postfix='.lev1eo.cub', posargs=posargs, **{**defaults_ps2, **kwargs})

@rich_logger
def step_seven(self, mpp=24, just_dem=False, folder='results_ba', **kwargs):
Expand Down Expand Up @@ -960,44 +968,30 @@ def step_nine(self, refdem=None, mpp=6):
self.cs.mapproject('-t', 'isis', refdem, f'{right}.lev1eo.cub', f'{right}.ba.map.tif', '--mpp', mpp, '--bundle-adjust-prefix', 'adjust/ba')

@rich_logger
def step_ten(self, stereo_conf, refdem=None, **kwargs):
def step_ten(self, stereo_conf, refdem=None, posargs='', **kwargs):
"""
Second stereo first step
:param stereo_conf:
:param refdem: path to reference DEM or PEDR csv file
:param posargs: additional positional args
:param kwargs:
"""
left, right, both = self.cs.parse_stereopairs()
stereo_conf = Path(stereo_conf).absolute()
if not refdem:
refdem = Path.cwd() / both / 'results_ba' / 'dem' / f'{both}_ba_100_0-DEM.tif'
else:
refdem = Path(refdem).absolute()
with cd(Path.cwd() / both):
args = kwargs_to_args({**self.cs.defaults_ps1, **clean_kwargs(kwargs)})
return self.cs.parallel_stereo(*args, f'{left}.ba.map.tif', f'{right}.ba.map.tif', f'{left}.lev1eo.cub', f'{right}.lev1eo.cub',
'-s', stereo_conf, f'results_map_ba/{both}_ba', refdem)
refdem = str(Path(self.get_first_pass_refdem() if not refdem else refdem).absolute())
return self.cs.stereo_asap(stereo_conf=stereo_conf, refdem=refdem, postfix=['.ba.map.tif', '.lev1eo.cub'], output_file_prefix='results_map_ba/${both}_ba', posargs=posargs, **{**defaults_ps1, **kwargs})

@rich_logger
def step_eleven(self, stereo_conf, refdem=None, **kwargs):
def step_eleven(self, stereo_conf, refdem=None, posargs='', **kwargs):
"""
Second stereo second step
:param stereo_conf:
:param refdem: path to reference DEM or PEDR csv file
:param posargs: additional positional args
:param kwargs:
"""
left, right, both = self.cs.parse_stereopairs()
stereo_conf = Path(stereo_conf).absolute()
if not refdem:
refdem = Path.cwd() / both / 'results_ba' / 'dem' / f'{both}_ba_100_0-DEM.tif'
else:
refdem = Path(refdem).absolute()
with cd(Path.cwd() / both):
args = kwargs_to_args({**self.cs.defaults_ps2, **clean_kwargs(kwargs)})
return self.cs.parallel_stereo(*args, f'{left}.ba.map.tif', f'{right}.ba.map.tif', f'{left}.lev1eo.cub', f'{right}.lev1eo.cub',
'-s', stereo_conf, f'results_map_ba/{both}_ba', refdem)
refdem = str(Path(self.get_first_pass_refdem() if not refdem else refdem).absolute())
return self.cs.stereo_asap(stereo_conf=stereo_conf, refdem=refdem, postfix=['.ba.map.tif', '.lev1eo.cub'], output_file_prefix='results_map_ba/${both}_ba', posargs=posargs, **{**defaults_ps2, **kwargs})

@rich_logger
def step_twelve(self, pedr_list=None, postfix='.lev1eo'):
Expand Down Expand Up @@ -1127,7 +1121,7 @@ def __init__(self, https=False, threads=cores, datum="D_MARS", proj: Optional[st
self.cam2map4stereo = sh.Command('cam2map4stereo.py')
# if proj is not none, get the corresponding proj or else override with proj,
# otherwise it's a none so remain a none
self.proj = CommonSteps.projections.get(proj, proj)
self.proj = projections.get(proj, proj)

def get_hirise_emission_angle(self, pid: str)-> float:
"""
Expand Down Expand Up @@ -1271,19 +1265,19 @@ def hiedr2mosaic(*im):
_ = [p.wait() for p in procs]

@rich_logger
def step_four(self):
def step_four(self, postfix='*.mos_hijitreged.norm.cub'):
"""
Move hieder2mosaic files
Move the hiedr2mosaic output to the location needed for cam2map4stereo
"""
left, right, both = self.cs.parse_stereopairs()
sh.mv(next(Path(f'./{left}/').glob(f'{left}*.mos_hijitreged.norm.cub')), both)
sh.mv(next(Path(f'./{right}/').glob(f'{right}*.mos_hijitreged.norm.cub')), both)
sh.mv(next(Path(f'./{left}/').glob(f'{left}{postfix}')), both)
sh.mv(next(Path(f'./{right}/').glob(f'{right}{postfix}')), both)

@rich_logger
def step_five(self, gsd: float = None):
def step_five(self, gsd: float = None, postfix='*.mos_hijitreged.norm.cub'):
"""
Map project HiRISE data for stereo processing
Expand All @@ -1299,8 +1293,8 @@ def par_cam2map(argstring):

left, right, both = self.cs.parse_stereopairs()
with cd(both):
left_im = next(Path('.').glob(f'{left}*.mos_hijitreged.norm.cub'))
right_im = next(Path('.').glob(f'{right}*.mos_hijitreged.norm.cub'))
left_im = next(Path('.').glob(f'{left}{postfix}'))
right_im = next(Path('.').glob(f'{right}{postfix}'))
cam2map_args = ['-n', left_im, right_im]
if gsd is not None:
cam2map_args = ['--pixres', 'MPP', '--resolution', str(gsd), *cam2map_args]
Expand All @@ -1318,7 +1312,7 @@ def par_cam2map(argstring):
_ = [p.wait() for p in procs]

@rich_logger
def step_six(self, *vargs, bundle_adjust_prefix='adjust/ba', **kwargs)-> sh.RunningCommand:
def step_six(self, *vargs, postfix='_RED.map.cub', bundle_adjust_prefix='adjust/ba', **kwargs)-> sh.RunningCommand:
"""
Bundle Adjust HiRISE
Expand All @@ -1327,28 +1321,28 @@ def step_six(self, *vargs, bundle_adjust_prefix='adjust/ba', **kwargs)-> sh.Runn
:param vargs: variable length additional positional arguments to pass to bundle adjust
:param bundle_adjust_prefix:
"""
return self.cs.bundle_adjust(*vargs, postfix='_RED.map.cub', bundle_adjust_prefix=bundle_adjust_prefix, **kwargs)
return self.cs.bundle_adjust(*vargs, postfix=postfix, bundle_adjust_prefix=bundle_adjust_prefix, **kwargs)

@rich_logger
def step_seven(self, stereo_conf, **kwargs):
def step_seven(self, stereo_conf, postfix='_RED.map.cub', posargs='', **kwargs):
"""
Parallel Stereo Part 1
Run first part of parallel_stereo
"""
return self.cs.stereo_1(stereo_conf, postfix='_RED.map.cub', **kwargs)
return self.cs.stereo_asap(stereo_conf, postfix=postfix, posargs=posargs, **{**defaults_ps1, **kwargs})

@rich_logger
def step_eight(self, stereo_conf, **kwargs):
def step_eight(self, stereo_conf, postfix='_RED.map.cub', posargs='', **kwargs):
"""
Parallel Stereo Part 2
Run second part of parallel_stereo, stereo is completed after this step
"""
return self.cs.stereo_2(stereo_conf, postfix='_RED.map.cub', **kwargs)
return self.cs.stereo_asap(stereo_conf, postfix=postfix, posargs=posargs, **{**defaults_ps2, **kwargs})

@rich_logger
def step_nine(self, mpp=2, just_dem=False, **kwargs):
def step_nine(self, mpp=2, just_dem=False, postfix='_RED.map.cub', **kwargs):
"""
Produce preview DEMs/Orthos
Expand All @@ -1364,8 +1358,8 @@ def step_nine(self, mpp=2, just_dem=False, **kwargs):
post_args.extend(['-n', '--errorimage', '--orthoimage', f'{both}_ba-L.tif'])
with cd(Path.cwd() / both / 'results_ba'):
# check the GSD against the MPP
self.cs.check_mpp_against_true_gsd(f'../{left}_RED.map.cub', mpp)
proj = self.cs.get_srs_info(f'../{left}_RED.map.cub', use_eqc=self.proj)
self.cs.check_mpp_against_true_gsd(f'../{left}{postfix}', mpp)
proj = self.cs.get_srs_info(f'../{left}{postfix}', use_eqc=self.proj)
defaults = {
'--t_srs' : proj,
'-r' : 'mars',
Expand All @@ -1376,7 +1370,7 @@ def step_nine(self, mpp=2, just_dem=False, **kwargs):
pre_args = kwargs_to_args({**defaults, **clean_kwargs(kwargs)})
return self.cs.point2dem(*pre_args, f'{both}_ba-PC.tif', *post_args)

def _gdal_hirise_rescale(self, mpp):
def _gdal_hirise_rescale(self, mpp, postfix='_RED.map.cub'):
"""
Hillshade using gdaldem instead of asp
Expand All @@ -1386,7 +1380,7 @@ def _gdal_hirise_rescale(self, mpp):
mpp_postfix = self.cs.get_mpp_postfix(mpp)
with cd(Path.cwd() / both / 'results_ba' / 'dem'):
# check the GSD against the MPP
self.cs.check_mpp_against_true_gsd(f'../../{left}_RED.map.cub', mpp)
self.cs.check_mpp_against_true_gsd(f'../../{left}{postfix}', mpp)
in_dem = next(Path.cwd().glob('*-DEM.tif')) # todo: this might not always be the right thing to do...
return sh.gdal_translate('-r', 'cubic', '-tr', float(mpp), float(mpp), in_dem, f'./{both}_{mpp_postfix}-DEM.tif')

Expand Down Expand Up @@ -1482,7 +1476,7 @@ def step_ten(self, maxd, refdem, highest_accuracy=True, **kwargs):
return self.cs.pc_align(*args, *hq, f'{both}_ba-PC.tif', refdem)

@rich_logger
def step_eleven(self, mpp=1.0, just_ortho=False, output_folder='dem_align', **kwargs):
def step_eleven(self, mpp=1.0, just_ortho=False, postfix='_RED.map.cub', output_folder='dem_align', **kwargs):
"""
Produce final DEMs/Orthos
Expand All @@ -1502,10 +1496,10 @@ def step_eleven(self, mpp=1.0, just_ortho=False, output_folder='dem_align', **kw
add_params.extend(['-n', '--errorimage',])

with cd(Path.cwd() / both / 'results_ba'):
proj = self.cs.get_srs_info(f'../{left}_RED.map.cub', use_eqc=self.proj)
proj = self.cs.get_srs_info(f'../{left}{postfix}', use_eqc=self.proj)
if not just_ortho:
# check the GSD against the MPP
self.cs.check_mpp_against_true_gsd(f'../{left}_RED.map.cub', mpp)
self.cs.check_mpp_against_true_gsd(f'../{left}{postfix}', mpp)
with cd(output_folder):
point_cloud = next(Path.cwd().glob('*trans_reference.tif'))
defaults = {
Expand Down

0 comments on commit 614e913

Please sign in to comment.