fba_launch

#!/usr/bin/env python

import os
import sys
import fitsio
from fiberassign.utils import Logger
from time import time
from datetime import datetime, timezone
from astropy.time import Time
import tempfile
import shutil
from desiutil.redirect import stdouterr_redirected
from fiberassign.fba_launch_io import (
    get_program_latest_timestamp,
    assert_env_vars,
    get_latest_rundate,
    assert_svn_tileid,
    assert_arg_dates,
    print_config_infos,
    get_desitarget_paths,
    create_tile,
    create_sky,
    create_targ_nomtl,
    create_mtl,
    create_too,
    launch_onetile_fa,
    update_fiberassign_header,
    secure_gzip,
    make_qa,
    rmv_nonsvn,
    mv_temp2final,
    copy_to_svn,
)
from argparse import ArgumentParser

# AR allowed steps in fba_launch
steps_all = ["tiles", "sky", "gfa", "targ", "scnd", "too", "fa", "zip", "move", "qa"]


# AR goal effective time
# AR https://desi.lbl.gov/svn/data/surveyops/trunk/ops/config-sv2.yaml
# AR https://desi.lbl.gov/svn/data/surveyops/trunk/ops/config-sv3.yaml as of Apr. 5th
# AR main: [desi-survey 2390] Minimum exposure time fraction and requested effective times
goaltimes_all = {
    "sv1": {"DARK": 1000.0, "BRIGHT": 150.0, "BACKUP": 30.0},
    "sv2": {"DARK": 1000.0, "BRIGHT": 150.0, "BACKUP": 30.0},
    "sv3": {"DARK": 1200.0, "BRIGHT": 220.0, "BACKUP": 30.0},
    "main": {"DARK": 1000.0, "BRIGHT": 180.0, "BACKUP": 60.0},
}

# AR surface brightness profile to be used for computing time -> efftime relation
sbprofs_all = {
    "sv1": {"DARK": "ELG", "BRIGHT": "BGS", "BACKUP": "PSF"},
    "sv2": {"DARK": "ELG", "BRIGHT": "BGS", "BACKUP": "PSF"},
    "sv3": {"DARK": "ELG", "BRIGHT": "BGS", "BACKUP": "PSF"},
    "main": {"DARK": "ELG", "BRIGHT": "BGS", "BACKUP": "PSF"},
}

# AR minimum exposure time fraction needed for this tile to be considered done
# AR main: [desi-survey 2390] Minimum exposure time fraction and requested effective times
mintfracs_all = {
    "sv1": 0.9,
    "sv2": 0.9,
    "sv3": 0.9,
    "main": 0.85,
}


def main():
    #
    start = time()
    log.info("{:.1f}s\tstart\tTIMESTAMP={}".format(time() - start, Time.now().isot))

    # AR if for development, print it big at the beginning
    if args.custom_too_development:
        log.warning("")
        log.warning("\t\t================================================================================================")
        log.warning("\t\t================================= THIS IS FOR DEVELOPMENT ONLY =================================")
        log.warning("\t\t================================================================================================")
        log.warning("")

    # AR rundate: if None, set to the latest time
    # AR    in the latest desi-state*ecsv file
    if args.rundate is None:
        log.info("")
        log.info("")
        log.info("{:.1f}s\trundate\tTIMESTAMP={}".format(time() - start, Time.now().isot))
        args.rundate = get_latest_rundate(log=log, step="rundate", start=start)
        log.info("{:.1f}s\trundate\tsetting args.rundate={}".format(time() - start, args.rundate))

    # AR mtltime
    # AR if possible: setting to the latest of:
    # AR - latest timestamp for the program
    # AR - latest timestamp in the ledgers for pixels touching the tile
    # AR else: setting to utc_time_now_str
    if args.mtltime is None:
        log.info("")
        log.info("")
        log.info("{:.1f}s\tmtltime\tTIMESTAMP={}".format(time() - start, Time.now().isot))
        args.mtltime = utc_time_now_str
        timestamp = get_program_latest_timestamp(
            args.survey,
            args.program,
            args.tilera,
            args.tiledec,
            too_tile=args.too_tile,
            log=log,
            step="mtltime",
            start=start,
        )
        if timestamp is not None:
            args.mtltime = timestamp
            log.info("{:.1f}s\tmtltime\tsetting args.mtltime={}, based on the latest timestamp".format(time() - start, args.mtltime))
        else:
            log.info("{:.1f}s\tmtltime\tsetting args.mtltime={}, based on current time".format(time() - start, args.mtltime))

    log.info("")
    log.info("")
    log.info("{:.1f}s\tsettings\tTIMESTAMP={}".format(time() - start, Time.now().isot))
    # AR printing settings
    tmpstr = " , ".join(
        [kwargs[0] + "=" + str(kwargs[1]) for kwargs in args._get_kwargs()]
    )
    log.info("{:.1f}s\tsettings\targs: {}".format(time() - start, tmpstr))

    # AR steps to execute in fba_launch
    steps = [step for step in args.steps.split(",")]
    if args.nosteps is not None:
        steps = [step for step in steps if step not in args.nosteps.split(",")]
        log.info("{:.1f}s\tsettings\tsteps to exclude: {}".format(time() - start, args.nosteps.split(",")))
    else:
        log.info("{:.1f}s\tsettings\tsteps to exclude: -".format(time() - start))
    log.info("{:.1f}s\tsettings\tsteps to execute: {}".format(time() - start, steps))

    # AR files we expect to be created, given args.steps, for the "move" step
    expected_files = {}
    for step in steps:
        if step not in ["zip", "move", "qa"]:
            if step == "fa":
                expected_files["fba"], expected_files["fiberassign"] = True, True
            else:
                expected_files[step] = True

    # AR safe: DESI environment variables defined?
    assert_env_vars(log=log, step="settings", start=start)

    # AR safe: TILEID already existing in SVN?
    assert_svn_tileid(
        args.tileid, forcetileid=args.forcetileid, log=log, step="settings", start=start
    )

    # AR safe: argument dates correctly formatted?
    assert_arg_dates(args, log=log, step="settings", start=start)

    # AR mjd_now: setting to pmtime_utc_str if not set
    # AR    see https://github.com/desihub/fiberassign/issues/395
    # AR    (and we just assessed args.pmtime_utc_str has the correct formatting)
    # AR    (instead of utc_time_now_str, to allow reproducibility)
    # AR    (purposely not tying to rundate)
    mjd_now = Time(datetime.strptime(args.pmtime_utc_str, "%Y-%m-%dT%H:%M:%S%z")).mjd
    log.info(
        "{:.1f}s\tsettings\tsetting mjd_now={}, corresponding to args.pmtime_utc_str={}".format(
            time() - start, mjd_now, args.pmtime_utc_str
        )
    )

    # AR print general configuration informations
    print_config_infos(log=log, step="settings", start=start)

    # AR (temporary) output files
    for key in list(myouts.keys()):
        log.info(
            "{:.1f}s\tsettings\toutput file for {}: {}".format(
                time() - start, key, myouts[key]
            )
        )
    for key in list(mytmpouts.keys()):
        log.info(
            "{:.1f}s\tsettings\ttemporary output file for {}: {}".format(
                time() - start, key, mytmpouts[key]
            )
        )

    # AR faflavor
    faflavor = "{}{}".format(args.hdr_survey, args.hdr_faprgrm)
    log.info(
        "{:.1f}s\tsettings\theader: args.hdr_survey={} , args.hdr_faprgrm={} , faflavor={}".format(
            time() - start, args.hdr_survey, args.hdr_faprgrm, faflavor
        )
    )

    # AR obscon : for the tile observing conditions, permissive value
    obscon = "DARK|GRAY|BRIGHT|BACKUP"
    log.info("{:.1f}s\tsettings\ttile obscon={}".format(time() - start, obscon))

    # AR desitarget folders/files
    # AR folder architecture is now the same at NERSC/KPNO (https://github.com/desihub/fiberassign/issues/302)
    # AR DESI_ROOT : NERSC: '/global/cfs/cdirs/desi' ; KPNO: '/data/datasystems'
    mydirs = get_desitarget_paths(
        args.dtver,
        args.survey,
        args.program,
        too_tile=args.too_tile,
        dr=args.dr,
        gaiadr=args.gaiadr,
        custom_too_file=args.custom_too_file,
        custom_too_development=args.custom_too_development,
        log=log,
        step="settings",
        start=start,
    )

    # AR tiles
    if "tiles" in steps:
        create_tile(
            args.tileid,
            args.tilera,
            args.tiledec,
            mytmpouts["tiles"],
            args.survey,
            obscon=obscon,
            log=log,
            step="tiles",
            start=start,
        )
    else:
        log.info("")
        log.info("")
        log.info("{:.1f}s\t{}\tnot executing, as per args.steps/args.nosteps".format(time() - start, "tiles"))

    # AR sky
    if "sky" in steps:
        create_sky(
            mytmpouts["tiles"],
            mydirs["sky"],
            mytmpouts["sky"],
            suppskydir=mydirs["skysupp"],
            tmpoutdir=tmpoutdir,
            log=log,
            step="sky",
            start=start,
        )
    else:
        log.info("")
        log.info("")
        log.info("{:.1f}s\t{}\tnot executing, as per args.steps/args.nosteps".format(time() - start, "sky"))

    # AR gfa
    if "gfa" in steps:
        create_targ_nomtl(
            mytmpouts["tiles"],
            mydirs["gfa"],
            args.survey,
            args.gaiadr.replace("gaia", ""),
            args.pmcorr,
            mytmpouts["gfa"],
            tmpoutdir=tmpoutdir,
            pmtime_utc_str=args.pmtime_utc_str,
            log=log,
            step="gfa",
            start=start,
            add_plate_cols=False,
        )
    else:
        log.info("")
        log.info("")
        log.info("{:.1f}s\t{}\tnot executing, as per args.steps/args.nosteps".format(time() - start, "gfa"))

    # AR targ
    if "targ" in steps:
        create_mtl(
            mytmpouts["tiles"],
            mydirs["mtl"],
            args.mtltime,
            mydirs["targ"],
            args.survey,
            args.gaiadr.replace("gaia", ""),
            args.pmcorr,
            mytmpouts["targ"],
            tmpoutdir=tmpoutdir,
            pmtime_utc_str=args.pmtime_utc_str,
            std_only=args.targ_std_only,
            log=log,
            step="targ",
            start=start,
        )
    else:
        log.info("")
        log.info("")
        log.info("{:.1f}s\t{}\tnot executing, as per args.steps/args.nosteps".format(time() - start, "targ"))

    # AR secondary targets
    # AR if not backup
    if ("scnd" in steps):
        if ("scnd" in list(mydirs.keys())):
            targdirs = [mydirs["scnd"]]
            for key in sorted(list(mydirs.keys())):
                if (key[:4] == "scnd") & (key != "scnd") & (key != "scndmtl"):
                    targdirs.append(mydirs[key])
            create_mtl(
                mytmpouts["tiles"],
                mydirs["scndmtl"],
                args.mtltime,
                targdirs,
                args.survey,
                args.gaiadr.replace("gaia", ""),
                args.pmcorr,
                mytmpouts["scnd"],
                tmpoutdir=tmpoutdir,
                pmtime_utc_str=args.pmtime_utc_str,
                log=log,
                step="scnd",
                start=start,
            )
        else:
            expected_files["scnd"] = False
            log.info("{:.1f}s\tscnd\tno secondary here".format(time() - start))
    else:
        log.info("")
        log.info("")
        log.info("{:.1f}s\t{}\tnot executing, as per args.steps/args.nosteps".format(time() - start, "scnd"))


    # AR ToO targets
    # AR 20210902: setting mjd_min = mjd_max = mjd_now
    # AR        so that we select: (d["MJD_BEGIN"] < mjd_now) & (d["MJD_END"] > mjd_now)
    if "too" in steps:
        toofns = mydirs["too"]
        for key in sorted(list(mydirs.keys())):
            if (key[:3] == "too") & (key != "too"):
                toofns += "," + mydirs[key]
        expected_files["too"] = create_too(
            mytmpouts["tiles"],
            toofns,
            mjd_now,
            mjd_now,
            args.survey,
            args.gaiadr.replace("gaia", ""),
            args.pmcorr,
            mytmpouts["too"],
            mtltime=args.mtltime,
            tmpoutdir=tmpoutdir,
            pmtime_utc_str=args.pmtime_utc_str,
            too_tile=args.too_tile,
            log=log,
            step="too",
            start=start,
        )
    else:
        log.info("")
        log.info("")
        log.info("{:.1f}s\t{}\tnot executing, as per args.steps/args.nosteps".format(time() - start, "too"))

    # AR fiberassign
    if "fa" in steps:
        # AR prepare targfns
        targfns = [mytmpouts["targ"]]
        if ("scnd" in steps) & (os.path.isfile(mytmpouts["scnd"])):
            targfns.append(mytmpouts["scnd"])
        if ("too" in steps) & (os.path.isfile(mytmpouts["too"])):
            targfns.append(mytmpouts["too"])
        # AR sky?
        if "sky" in steps:
            skyfn = mytmpouts["sky"]
        else:
            skyfn = None
        # AR gfa?
        if "gfa" in steps:
            gfafn = mytmpouts["gfa"]
        else:
            gfafn = None

        # AR launch_onetile_fa
        launch_onetile_fa(
            args,
            mytmpouts["tiles"],
            targfns,
            mytmpouts["fba"],
            mytmpouts["fiberassign"],
            skyfn=skyfn,
            gfafn=gfafn,
            log=log,
            step="fa",
            start=start,
        )

        # AR propagating some settings into the PRIMARY header
        ebv = fitsio.read(mytmpouts["targ"], columns=["EBV"])["EBV"]
        fascript = sys.argv[0]
        update_fiberassign_header(
            mytmpouts["fiberassign"],
            args,
            mydirs,
            args.hdr_survey,
            args.hdr_faprgrm,
            faflavor,
            ebv,
            obscon,
            fascript,
            nowtime=utc_time_now_str,
            log=log,
            step="fa",
            start=start,
        )
    else:
        log.info("")
        log.info("")
        log.info("{:.1f}s\t{}\tnot executing, as per args.steps/args.nosteps".format(time() - start, "fa"))

    # AR gzip all fiberassign files
    if "zip" in steps:
        secure_gzip(
            mytmpouts["fiberassign"], log=log, step="zip", start=start,
        )
        # AR updating the path
        mytmpouts["fiberassign"] += ".gz"
        myouts["fiberassign"] += ".gz"
    else:
        log.info("")
        log.info("")
        log.info("{:.1f}s\t{}\tnot executing, as per args.steps/args.nosteps".format(time() - start, "zip"))

    # AR move tmpoutdir -> args.outdir
    if "move" in steps:
        # AR files we expect to be created
        expected_keys = [key for key in list(expected_files.keys()) if expected_files[key]]
        mv_temp2final(
            mytmpouts, myouts, expected_keys, log=log, step="move", start=start,
        )
    else:
        log.info("")
        log.info("")
        log.info("{:.1f}s\t{}\tnot executing, as per args.steps/args.nosteps".format(time() - start, "move"))

    # AR QA plots
    if "qa" in steps:

        log.info("")
        log.info("")
        log.info("{:.1f}s\t{}\tTIMESTAMP={}".format(time() - start, "qa", Time.now().isot))
        # AR used targfns
        targfns = [myouts["targ"]]
        if os.path.isfile(myouts["scnd"]):
            targfns.append(myouts["scnd"])
        if os.path.isfile(myouts["too"]):
            targfns.append(myouts["too"])
        # AR fiberassign, updating path
        if "zip" not in steps:
            myouts["fiberassign"] += ".gz"

        make_qa(
            myouts["png"],
            args.survey,
            args.program,
            faflavor,
            targfns,
            myouts["fiberassign"],
            args.tileid,
            args.tilera,
            args.tiledec,
            obscon,
            args.rundate,
            tmpoutdir=tmpoutdir,
            width_deg=4,
        )
    else:
        log.info("")
        log.info("")
        log.info("{:.1f}s\t{}\tnot executing, as per args.steps/args.nosteps".format(time() - start, "qa"))

    # AR do clean?
    if args.doclean == "y":
        rmv_nonsvn(myouts, log=log, step="clean", start=start)

    # AR and we re done
    if not args.log_stdout:
        log.info("")
        log.info("")
        log.info(
            "{:.1f}s\t{}\tmoving {} to {}".format(
                time() - start, "log", mytmpouts["log"], myouts["log"]
            )
        )
    log.info("")
    log.info("")
    log.info("{:.1f}s\tend\tTIMESTAMP={}".format(time() - start, Time.now().isot))


if __name__ == "__main__":

    # AR reading arguments
    parser = ArgumentParser()
    parser.add_argument(
        "--outdir", help="output directory", type=str, default=None, required=True,
    )
    parser.add_argument(
        "--tileid",
        help="output TILEID (e.g., 63142)",
        type=int,
        default=None,
        required=True,
    )
    parser.add_argument(
        "--forcetileid",
        help="y/n, if y, allows to generate tileids which already exist in svn (default=n)",
        type=str,
        default="n",
        required=False,
        choices=["y", "n"],
    )
    parser.add_argument(
        "--tilera", help="tile centre ra", type=float, default=None, required=True,
    )
    parser.add_argument(
        "--tiledec", help="tile centre dec", type=float, default=None, required=True,
    )
    parser.add_argument("--ha", type=float, required=False, default=0.,
                        help="Design for the given Hour Angle in degrees.")
    parser.add_argument(
        "--survey",
        help="survey",
        type=str,
        default=None,
        required=True,
        choices=["sv1", "sv2", "sv3", "main"],
    )
    parser.add_argument(
        "--program",
        help="program",
        type=str,
        default=None,
        required=True,
        choices=["DARK", "BRIGHT", "BACKUP"],
    )
    parser.add_argument(
        "--goaltype",
        help="goaltype (default=args.program)",
        type=str,
        default=None,
        required=False,
        choices=["DARK", "BRIGHT", "BACKUP"],
    )
    parser.add_argument(
        "--goaltime",
        help="goal effective time (default for main: {})".format(
            ", ".join(
                [
                    "{}={}".format(key, goaltimes_all["main"][key])
                    for key in ["DARK", "BRIGHT", "BACKUP"]
                ]
            )
        ),
        type=float,
        default=None,
        required=False,
    )
    parser.add_argument(
        "--sbprof",
        help="surface brightness profile to be used for computing time -> efftime relation (default for main: {})".format(
            ", ".join(
                [
                    "{}={}".format(key, sbprofs_all["main"][key])
                    for key in ["DARK", "BRIGHT", "BACKUP"]
                ]
            )
        ),
        type=str,
        default=None,
        required=False,
        choices=["ELG", "BGS", "PSF", "FLT"],
    )
    parser.add_argument(
        "--mintfrac",
        help="minimum exposure time fraction needed for this tile to be considered done (default for main: {}".format(
            mintfracs_all["main"],
        ),
        type=float,
        default=None,
        required=False,
    )
    parser.add_argument(
        "--rundate",
        help="yyyy-mm-ddThh:mm:ss+00:00 rundate for focalplane with UTC timezone formatting (default=current UTC time)",
        type=str,
        default=None,
        required=False,
    )
    parser.add_argument(
        "--mtltime",
        help="yyyy-mm-ddThh:mm:ss+00:00 MTL isodate, with UTC timezone formatting (default=latest MTL timestamp for args.program if available, if not current UTC time)",
        type=str,
        default=None,
        required=False,
    )
    parser.add_argument(
        "--standards_per_petal",
        help="required number of standards per petal (default=10)",
        type=str,
        default="10",
        required=False,
    )
    parser.add_argument(
        "--sky_per_petal",
        help="required number of sky targets per petal (default=40)",
        type=str,
        default="40",
        required=False,
    )
    parser.add_argument(
        "--sky_per_slitblock",
        help="Required number of sky targets per fiber slitblock (default=1)",
        type=int,
        default=1,
        required=False,
    )
    parser.add_argument("--margin-pos", "--margin_pos", type=float, required=False, default=0.05,
                        help="Add margin (in mm) around positioner keep-out polygons (default: 0.05)")
    parser.add_argument("--margin-petal", "--margin_petal", type=float, required=False, default=0.4,
                        help="Add margin (in mm) around petal-boundary keep-out polygons (default: 0.4)")
    parser.add_argument("--margin-gfa", "--margin_gfa", type=float, required=False, default=0.4,
                        help="Add margin (in mm) around GFA keep-out polygons (default: 0.4)")
    parser.add_argument(
        "--dr",
        help="legacypipe dr (default=dr9)",
        type=str,
        default="dr9",
        required=False,
        choices=["dr9"],
    )
    parser.add_argument(
        "--gaiadr",
        help="gaia dr (default=gaiadr2)",
        type=str,
        default="gaiadr2",
        required=False,
        choices=["gaiadr2"],
    )
    parser.add_argument(
        "--dtver",
        help="desitarget catalogue version",
        type=str,
        default=None,
        required=True,
    )
    parser.add_argument(
        "--pmcorr",
        help="apply proper-motion-correction before fiber assignment? (y/n) (default=n)",
        type=str,
        default="n",
        required=False,
    )
    parser.add_argument(
        "--pmtime_utc_str",
        help="yyyy-mm-ddThh:mm:ss+00:00, UTC time use to compute new coordinates after applying proper motion since REF_EPOCH (default=current UTC time)",
        type=str,
        default=None,
        required=False,
    )
    parser.add_argument(
        "--doclean",
        help="delete TILEID-{tiles,sky,std,gfa,targ,scnd,too}.fits files (y/n)",
        type=str,
        default="n",
        required=False,
    )
    parser.add_argument(
        "--log-stdout", "--log_stdout",
        action="store_true",
        help="log to stdout instead of redirecting to a file",
    )
    #
    parser.add_argument(
        "--hdr_survey",
        help="SURVEY keyword in fiberassign-TILEID.fits.gz file (default=args.survey); FAFLAVOR will be {args.hdr_survey}{args.hdr_faprgrm}",
        type=str,
        default=None,
        required=False,
    )
    parser.add_argument(
        "--hdr_faprgrm",
        help="FAPRGRM keyword in fiberassign-TILEID.fits.gz file (default=args.program.lower()); FAFLAVOR will be {args.hdr_survey}{args.hdr_faprgrm}",
        type=str,
        default=None,
        required=False,
    )
    parser.add_argument(
        "--steps",
        help="comma-separated list of steps to accomplish, amongst {} (defaults to {} if args.nosteps=None)".format(steps_all, ",".join(steps_all)),
        type=str,
        default=None,
        required=False,
    )
    parser.add_argument(
        "--nosteps",
        help="comma-separated list of steps *not* to accomplish, amongst {} (default=None); cannot be used if some args.steps values are also provided".format(steps_all),
        type=str,
        default=None,
        required=False,
    )
    parser.add_argument(
        "--svntiledir",
        help="SVN tile directory; fiberassign-* files created should be copied here.",
        type=str,
        default=None,
        required=False,
    )
    parser.add_argument(
        "--worldreadable",
        help="Permissions should be world readable.",
        action='store_true',
        default=False,
    )
    parser.add_argument(
        "--lookup_sky_source",
        help="'ls' or 'gaia' (default='gaia' if main/backup, 'ls' otherwise)",
        type=str,
        choices=["ls", "gaia"],
        default=None,
    )
    parser.add_argument(
        "--targ_std_only",
        help="only select standard stars for TILEID-targ.fits (e.g., for ToO-dedicated tiles); can only be used with args.survey='main'",
        action="store_true",
    )
    parser.add_argument(
        "--too_tile",
        help='do not cut on TOO_TYPE, hence keeping both TOO_TYPE="FIBER" *and* TOO_TYPE="TILE" for ToO dedicated tiles,',
        action="store_true",
    )
    parser.add_argument(
        "--custom_too_file",
        help="full path to a custom ToO file, or a comma-separated list of custom ToO files, for tertiary program or development work, which overrides the official one (default=None)",
        type=str,
        default=None,
    )
    parser.add_argument(
        "--custom_too_development",
        help="is this for development? (allows args.custom_too_file to be outside of $DESI_SURVEYOPS)",
        action="store_true",
        default=False,
    )

    args = parser.parse_args()
    log = Logger.get()
    start = time()

    # AR safe: outdir
    if args.outdir[-1] != "/":
        args.outdir += "/"
    subdir = ('%06d' % args.tileid)[:3]
    if not os.path.isdir(os.path.join(args.outdir, subdir)):
        dirmode = 0o2775 if args.worldreadable else 0o2770
        os.makedirs(os.path.join(args.outdir, subdir), exist_ok=True,
                    mode=dirmode)

    # AR utc_time_now, rundate, pmtime
    utc_time_now = datetime.now(tz=timezone.utc)
    utc_time_now_str = utc_time_now.isoformat(timespec="seconds")
    if args.pmtime_utc_str is None:
        args.pmtime_utc_str = utc_time_now_str

    # AR rundate (2021-09-01): if set to None, now setting it inside main()
    #          so that it is recorded in the fiberassign-TILEID.log file

    # AR mtltime (2021-09-01): if set to None, now setting it inside main()
    # AR          so that it is recorded in the fiberassign-TILEID.log file

    # AR goaltype
    if args.goaltype is None:
        args.goaltype = args.program

    # AR goaltime
    if args.goaltime is None:
        args.goaltime = goaltimes_all[args.survey][args.program]

    # AR sbprof
    if args.sbprof is None:
        args.sbprof = sbprofs_all[args.survey][args.program]

    # AR mintfrac
    if args.mintfrac is None:
        args.mintfrac = mintfracs_all[args.survey]

    # AR hdr_survey, hdr_faprgrm
    if args.hdr_survey is None:
        args.hdr_survey = args.survey
    if args.hdr_faprgrm is None:
        args.hdr_faprgrm = args.program.lower()

    # AR steps/nosteps
    if args.steps is not None and args.nosteps is not None:
        log.error("only *one* of args.steps and args.nosteps has to be set to None; exiting")
        sys.exit(1)
    if args.steps is None:
        args.steps = ",".join(steps_all)
        if args.nosteps is not None:
            wrong_nosteps = [step for step in args.nosteps.split(",") if step not in steps_all]
            if len(wrong_nosteps) > 0:
                log.error("args.nosteps have the following not authorized steps : {}; exiting".format(
                    wrong_nosteps
                    )
                )
                sys.exit(1)
    else:
        wrong_steps = [step for step in args.steps.split(",") if step not in steps_all]
        if len(wrong_steps) > 0:
            log.error("args.steps have the following not authorized steps : {}; exiting".format(
                wrong_steps
                )
            )
            sys.exit(1)

    # AR lookup_sky_source
    if args.lookup_sky_source is None:
        if (args.survey == "main") & (args.program == "BACKUP"):
            args.lookup_sky_source = "gaia"
            if os.getenv("SKYHEALPIXS_DIR") is None:
                log.warning("SKYHEALPIXS_DIR not set: stuck positioners will not be used for skies.")
        else:
            args.lookup_sky_source = "ls"
            if os.getenv("SKYBRICKS_DIR") is None:
                log.warning("SKYBRICKS_DIR not set: stuck positioners will not be used for skies.")

    # AR custom_too_file
    if args.custom_too_file is not None:
        log.warning("args.custom_too_file is set ({}) -> this is for tertiary program or development only!".format(args.custom_too_file))

    # AR create a temporary directory for generated files
    tmpoutdir = tempfile.mkdtemp()

    if tmpoutdir[-1] != "/":
        tmpoutdir += "/"

    # AR output files
    myouts = {}
    #
    for key in ["tiles", "sky", "gfa", "targ", "scnd", "too"]:
        myouts[key] = os.path.join(
            args.outdir, subdir, "{:06d}-{}.fits".format(args.tileid, key)
        )
    myouts["fba"] = os.path.join(
        args.outdir, subdir, "fba-{:06d}.fits".format(args.tileid))
    myouts["fiberassign"] = os.path.join(
        args.outdir, subdir, "fiberassign-{:06d}.fits".format(args.tileid)
    )
    myouts["png"] = os.path.join(
        args.outdir, subdir,  "fiberassign-{:06d}.png".format(args.tileid)
    )
    myouts["log"] = os.path.join(
        args.outdir, subdir, "fiberassign-{:06d}.log".format(args.tileid)
    )

    # AR temporary output files
    # AR    except for png, as doqa is after domove
    mytmpouts = {}
    for key in list(myouts.keys()):
        if key != "png":
            mytmpouts[key] = myouts[key].replace(
                os.path.join(args.outdir, subdir), tmpoutdir)

    # AR temporary log file
    if os.path.isfile(mytmpouts["log"]):
        os.remove(mytmpouts["log"])
    if args.log_stdout:
        main()
    else:
        with stdouterr_redirected(to=mytmpouts["log"]):
            main()

        # AR/SB moving the log at last if it was sent to a file
        # EFS remove file first if it exists; otherwise permissions
        # errors can occur if the writer user is different from
        # the original writer, even if the writer has permission
        # to delete the file!
        try:
            os.remove(myouts["log"])
        except FileNotFoundError:
            pass
        _ = shutil.move(mytmpouts["log"], myouts["log"])

    # must occur outside of main to get log.
    if args.svntiledir is not None:
        copy_to_svn(args.svntiledir, args.tileid, myouts,
                    worldreadable=args.worldreadable)

    # AR deleting folder if empty
    tmpfiles = os.listdir(tmpoutdir)
    if len(tmpfiles) == 0:
        os.rmdir(tmpoutdir)
    else:
        log.error("temp directory not empty: {}".format(tmpfiles))