canomate.py

#!/usr/bin/env python3

#
#############################################################################
#
# canomate.py - Automation of Canon camera operations
# using Canon's Camera Control API (CCAPI) over WiFi
# Copyright (c) 2020, Horshack
#
# This module is licensed under GPL v3: http://www.gnu.org/licenses/gpl-3.0.html
#
#############################################################################
#

from applog import *
from strutil import *
import argparse
import ast
import errno
import json
import math
import os
import pathlib
import platform
import re
import requests
import shutil
import subprocess
import sys
import time
import traceback

#
#############################################################################
#
# constants
#
#############################################################################
#

APP_NAME                                    = "canomate"
APP_VERSION                                 = "0.02"

# default values
DEFAULT_CCAPI_HTTP_PORT                     = 8080
DEFAULT_CCAPI_TIMEOUT_SECS                  = 5.0
DEFAULT_CCAPI_DOWNLOAD_TIMEOUT_SECS         = 0
DEFAULT_CCAPI_TRANSPORT_ERROR_RETRIES       = 2
DEFAULT_CCAPI_CMD_ERROR_RETRIES             = 2
DEFAULT_CCAPI_MAX_BUSY_RETRY_SECS           = 10
DEFAULT_CCAPI_RETRY_DELAY_SECS              = 0
DEFAULT_EXIT_ON_CMD_ERR                     = "Yes"

# app exit codes
ERRNO_OK                                    = 0
ERRNO_BAD_PYTHON_VER                        = 1
ERRNO_CANT_OPEN_OP_FILE                     = 2
ERRNO_CANT_READ_OP_FILE                     = 3
ERRNO_FILE_EXISTS_USER_SPECIFIED_EXIT       = 4
ERRNO_CTRL_C_INTERRUPT                      = 5
ERRNO_CODE_EXCEPTION                        = 6
ERRNO_BAD_CMD_LINE_ARG                      = 100
ERRNO_BAD_CMD_LINE_ARG_VALUE                = 101
ERRNO_ERROR_PARSING_AUTOMATION_OP_PARAMS    = 102
ERRNO_UNKNOWN_AUTOMATION_OP                 = 103
ERRNO_MISSING_AUTOMATION_PARAM              = 104
ERRNO_RUNEXECUTABLE_OP_MISSING_ARGUMENT_LINE= 105
ERRNO_INVALID_TIME_VALUE_SPECIFIED          = 106
ERRNO_CANT_CONVERT_BOOL_VAL                 = 200
ERRNO_CANT_CONVERT_INT_VAL                  = 201
ERRNO_CANT_CONVERT_STR_VAL                  = 202
ERRNO_INVALID_OP_PARAM_CHOICE               = 203
ERRNO_UNMATCHED_END_GROUP_OP                = 204
ERRNO_UNMATCHED_GROUP_OP                    = 205
ERRNO_CANT_CONVERT_STR_VAL_TO_TIME          = 206
ERRNO_UNKNOWN_OP_PARAM                      = 207
ERRNO_OP_PARAM_INVALID_FOR_OTHER_PARAMS     = 208
ERRNO_CCAPI_TRANSPORT_ERROR                 = 1100
ERRNO_CCAPI_CMD_ERROR                       = 1101
ERRNO_HTTP_DOWNLOAD_FAILED                  = 1102
ERRNO_RUN_EXECUTABLE_LAUNCH_FAILED          = 1200
ERRNO_RUN_EXECUTABLE_OUTPUT_COPY_FAILED     = 1201
ERRNO_RUN_EXECUTABLE_EXIT_MISMATCH_RET_CODE = 1202
ERRNO_RUN_EXECUTABLE_FAILED                 = 1203
ERRNO_ASSERT_CAMERA_SETTING_WRONG_VALUE     = 1300

#
# class to contain most of our application-wide global vars
#
class GlobalVarsStruct:
    def __init__(self):
        self.appStartTime = None                        # time app was started (epoch)
        self.isWin32 = None                             # True if we're running on a Windows platform
        self.isOSX = None                               # True if we're running on OSX (Mac)
        self.appDir = None                              # directory this script is running from
        self.appDataDir = None                          # directory where our internal data/log files go
        self.args = None                                # dictionary of command-line arguments (generated by argparse)
        self.mainAutomationGroup = None                 # first/container group for automation operations
        self.noYes = ['no', 'yes']                      # boolean conversion to no/yes
        self.offOn = ['off', 'on']                      # boolean conversion to off/on

#
# Class for spwaning a child app including command-line parameters, with
# the output copied to a caller-specified file
#
class RunExecutable:

    #
    # creates an array of arguments from a space-delimited sequence of arguments
    # in a single-string. handles the case of quoted arguments as well
    #
    @staticmethod
    def createArgListFromArgStr(argStr):
        return [x.strip('"') for x in re.split('( |".*?")', argStr) if x.strip()]

    #
    # executes process, waits for completions, optionally copies stdout/stderr
    # @return (fExecuteSuccessful, errno) Tuple
    #
    @staticmethod
    def execute(executableNameIncludingPath, cmdLineAsStr, fWaitForCompletion = True, outputFilename=None, outputFileWriteMethod=None, msgDuringExec=""):

        # build list containing all command-line parameters, the first of which is the executable we'll be spanwing
        cmdLineAsStr = executableNameIncludingPath + ' ' + cmdLineAsStr
        cmdLineArgArray = RunExecutable.createArgListFromArgStr(cmdLineAsStr)

        # generate filenames that will hold the stdout and stderr of the spwaned process
        stdoutFilename = os.path.join(g.appDataDir, "RunExecutable_StdOut.txt")
        stderrFilename = os.path.join(g.appDataDir, "RunExecutable_StdErr.txt")

        #
        #  run executable (wait for completion case)
        #
        if fWaitForCompletion:
            if msgDuringExec:
                consoleWriteLine(msgDuringExec)
            try:
               with open(stdoutFilename, "w") as stdoutFile, open(stderrFilename, "w") as stderrFile:
                   processRunResult = subprocess.run(cmdLineArgArray, stdout=stdoutFile, stderr=stderrFile)
            except Exception as e:
                applog_e("RunExecutable: Launch failed - \"{:s}\"".format(str(e)))
                return (False, ERRNO_RUN_EXECUTABLE_LAUNCH_FAILED)
            if msgDuringExec:
                consoleClearLine()

            #
            # copy stdout and stderr out of process to caller-specified filename
            #
            if outputFilename:
                outputFileOpenModeStr = "w"  # assume we'll be overwriting the file
                if outputFileWriteMethod == "overwrite_first_time":
                    # overwrite file if this is the first time we're writing to it during this app session
                    if os.path.exists(outputFilename):
                        if os.path.getmtime(outputFilename) >= g.appStartTime:
                            # file last modification is equal to or newer than our app start time, meaning
                            # we've already written to it once this session. use append instead
                            outputFileOpenModeStr = "a"
                elif outputFileWriteMethod == "append":
                    outputFileOpenModeStr = "a"
                else: assert outputFileWriteMethod == "overwrite"
                try:
                    with open(outputFilename, outputFileOpenModeStr) as outputFile, open(stdoutFilename, "r") as stdoutFile, open(stderrFilename, "r") as stderrFile:
                        outputFile.write(stdoutFile.read())
                        outputFile.write(stderrFile.read())
                except Exception as e:
                    applog_e("RunExecutable: Error copying RunExecutable output to \"{:s}\" {:s}".format(outputFilename, str(e)))
                    return (False, ERRNO_RUN_EXECUTABLE_OUTPUT_COPY_FAILED)

            return (True, processRunResult.returncode)

        #
        # run executable (don't wait for completion)
        #
        try:
            process = subprocess.Popen(cmdLineArgArray)
        except Exception as e:
            applog_e("RunExecutable: Launch failed - \"{:s}\"".format(str(e)))
            return (False, ERRNO_RUN_EXECUTABLE_LAUNCH_FAILED)
        return (True, ERRNO_OK)

#
# base class for automation operations. Each derived class corresponds to a user-specified automation
# op, which is specified on either the command line via --op or inside a file via --opfile
#
class AutomationOp:

    #
    # static class variables
    #
    lastDownloadedFiles_FullUrl = None         # last downloaded file from camera (URL path)
    lastDownloadedFiles_LocalPath = None       # last downloaded file from camera (local path file written to)
    lastFilesFoundDuringPolling = None         # list of files last found when polling for new files on camera

    #
    # base implementation that handles op parameters common to all operations
    #
    def __init__(self, paramDict):

        #
        # process parameters common to all ops
        #
        if not self.isGroupOp():
            # process parameters that are only valid for non-group ops
            self.repeatCount = self.getDictValueAsScalar(paramDict, 'repeatcount', default=1)
            self.delayAfterSecs = self.getDictValueAsSeconds(paramDict, 'delayafter', 0)
        else:
            #
            # set values on group for same parameters not allowed on group. this allow common
            # code to operate on groups and non-group items without the need for conditional
            # logic that checks whether or not a given op is a non-group. We also need delete
            # the keys for these params so we don't interpret them as invalid in self.unusedParamDictKeys;
            # they only exist in the group to support the inheritance into the child ops of the group
            #
            paramDict.pop('repeatcount', None)
            paramDict.pop('delayafter', None)
            self.repeatCount = 1
            self.delayAfterSecs = 0


        retryArgsDict = dict()
        retryArgsDict['timeout'] = self.getDictValueAsScalar(paramDict, 'timeout', g.args['timeout'], float)
        retryArgsDict['downloadtimeout'] = self.getDictValueAsScalar(paramDict, 'downloadtimeout', g.args['downloadtimeout'], float)
        retryArgsDict['transportretries'] = self.getDictValueAsScalar(paramDict, 'transportretries', g.args['transportretries'])
        retryArgsDict['cmdretries'] = self.getDictValueAsScalar(paramDict, 'cmdretries', g.args['cmdretries'])
        retryArgsDict['retrydelay'] = self.getDictValueAsScalar(paramDict, 'retrydelay', g.args['retrydelay'], float)
        retryArgsDict['maxbusyretrytime' ] = self.getDictValueAsScalar(paramDict, 'maxbusyretrytime', g.args['maxbusyretrytime'])
        retryArgsDict['exitoncmderr'] = self.getDictValueAsBool(paramDict, 'exitoncmderr', g.args['exitoncmderr'])
        self.retryInfo = CCAPI.RetryInfo(retryArgsDict)

        #
        # the superclass and baseclass have now processed (and removed) all the paramDict key values
        # they recognize. if we have any keys (parameters) left over then those represent unsupported
        # parameters
        #
        if len(paramDict) > 0:
            self.unusedParamDictKeys = paramDict
        else:
            self.unusedParamDictKeys = None

    def getUnusedParamDictKeys(self):
        return self.unusedParamDictKeys

    def isGroupOp(self):
        return self.getClassSuffix() == 'Group'

    def execute(self):
        raise AssertionError("No execute() method in AutomationOp base class")

    #
    # returns this instance's class name. Example: "AutomationOp_Delay"
    #
    def getClassSuffix(self):
        return AutomationOp.getAutomationOpClassSuffix(type(self).__name__)

    #
    # returns the suffix of the instance's class name. Example: "Delay" for AutomationOp_Delay
    #
    @staticmethod
    def getAutomationOpClassSuffix(fullClassName):
        return fullClassName[fullClassName.find('_')+1:]

    #
    # Converts a string value for key into a boolean
    # @param paramDict Dictionary containing keys:values
    # @param key Key we are to convert from a string to a boolean
    # @param default Default value if the key is not found in the dictionary.
    # Specify None if the key must exist - if the key is not found the app exits
    # with a message
    # @return String converted to a boolean
    #
    def getDictValueAsBool(self, paramDict, key, default=False, fRequireValue=False):
        if key in paramDict:
            val = paramDict[key]
            del paramDict[key]
            if type(val).__name__ == 'bool':
                return val
            if type(val).__name__ == 'int':
                return bool(val)
            if type(val).__name__ == 'str':
                if val.lower() in ['1', 'true', 't', 'yes', 'y', 'on', 'enabled']:
                    return True
                if val.lower() in ['0', 'false', 'f', 'no', 'n', 'off', 'disabled']:
                    return False
            applog_e("For op '{:s}', unable to convert value '{:s}' for parameter '{:s}' to a boolean".format(self.getClassSuffix(), val, key))
            exit(ERRNO_CANT_CONVERT_BOOL_VAL)
        if fRequireValue:
            applog_e("Missing required parameter '{:s}' for op '{:s}'".format(key, self.getClassSuffix()))
            exit(ERRNO_MISSING_AUTOMATION_PARAM)
        return default

    #
    # Converts a string value for key into a scalar value (int or float)
    # @param paramDict Dictionary containing keys:values
    # @param key Key we are to convert from a string to a scalar
    # @param default Default value if the key is not found in the dictionary.
    # Specify None if the key must exist - if the key is not found the app exits
    # with a message
    # @param scalarClass Scalar we are to conver to. Supports int and float
    # @return String converted to a scalar
    #
    def getDictValueAsScalar(self, paramDict, key, default=None, scalarClass=int, fRequiredValue=False):
        if key in paramDict:
            keyStr = paramDict[key]
            del paramDict[key]
            try:
               val = scalarClass(keyStr)
            except ValueError:
                applog_e("For op '{:s}', unable to convert value '{:s}' for parameter '{:s}' to a {:s}".format(self.getClassSuffix(), keyStr, key, type(scalarClass).__name__))
                exit(ERRNO_CANT_CONVERT_INT_VAL)
            return val
        if fRequiredValue:
            applog_e("Missing required parameter '{:s}' for op '{:s}'".format(key, self.getClassSuffix()))
            exit(ERRNO_MISSING_AUTOMATION_PARAM)
        return default

    #
    # Gets a string value from the dictionary, applying a default if none is provided
    # @param paramDict Dictionary containing keys:values
    # @param key Key holding the desired string value
    # @param default Default value if the key is not found in the dictionary.
    # Specify None if the key must exist - if the key is not found the app exits
    # with a message
    # @return String, from either the dictionary if present or the default value passed
    #
    def getDictValueAsStr(self, paramDict, key, default=None, fRequiredValue=False):
        if key in paramDict:
            val = paramDict[key]
            del paramDict[key]
            try:
                strVal = str(val)
            except ValueError:
                applog_e("For op '{:s}', unable to convert value '{:s}' for parameter '{:s}' to a string".format(self.getClassSuffix(), val, key))
                exit(ERRNO_CANT_CONVERT_STR_VAL)
            return strVal
        if fRequiredValue:
            applog_e("Missing required parameter '{:s}' for op '{:s}'".format(key, self.getClassSuffix()))
            exit(ERRNO_MISSING_AUTOMATION_PARAM)
        return default

    #
    # Gets a string value from the dictionary, matching it to a predefined set of choices
    # @param paramDict Dictionary containing keys:values
    # @param key Key holding the desired string value
    # @param listChoices List containing valid choices. Comparison is case-insensitive
    # @param default Default value if the key is not found in the dictionary.
    # Specify None if the key must exist - if the key is not found the app exits
    # with a message
    # @return String, from either the dictionary if present or the default value passed
    #
    def getDictValueStrChoice(self, paramDict, key, listChoices, default=None, fRequiredValue=False):
        if key in paramDict:
            val = paramDict[key]
            del paramDict[key]
            if val.lower() in listChoices:
                return val.lower()
            applog_e("For op '{:s}', invalid option '{:s}' for parameter '{:s}'. Choices: {}".format(self.getClassSuffix(), val, key, listChoices))
            exit(ERRNO_INVALID_OP_PARAM_CHOICE)
        if fRequiredValue:
            applog_e("Missing required parameter '{:s}' for op '{:s}'".format(key, self.getClassSuffix()))
            exit(ERRNO_MISSING_AUTOMATION_PARAM)
        return default

    #
    # Converts a string value to a time in seconds. The string can
    # be a simple value (seconds) or a time-code in the form of
    # hh:mm:ss or mm:ss.
    # @param paramDict Dictionary containing keys:values
    # @param key Key holding the desired string value
    # @param default Default value if the key is not found in the dictionary.
    # Specify None if the key must exist - if the key is not found the app exits
    # with a message
    # @return Convert time value string into seconds
    #
    def getDictValueAsSeconds(self, paramDict, key, default=None, fRequiredValue=False):
        if key in paramDict:
            val = paramDict[key]
            del paramDict[key]
            try:
                seconds = convertTimeStrToSeconds(val, False)
            except Exception as e:
                applog_e("For op '{:s}', unable to convert time value '{:s}' for parameter '{:s}' to a valid time".format(self.getClassSuffix(), val, key))
                exit(ERRNO_INVALID_TIME_VALUE_SPECIFIED)
            return seconds
        if fRequiredValue:
            applog_e("Missing required parameter '{:s}' for op '{:s}'".format(key, self.getClassSuffix()))
            exit(ERRNO_MISSING_AUTOMATION_PARAM)
        return default

    def printGetResponse(self, resp, respKeyVal='value'):
        fListAbilities = self.fListAbilities if hasattr(self, 'fListAbilities') else False
        if resp['success']:
            availableStr = " Available: {}".format(resp['ability']) if fListAbilities else ""
            infoStr = "{:s}{:s}".format(resp[respKeyVal], availableStr)
        else:
            infoStr = "N/A"
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), infoStr))

class AutomationOp_Delay(AutomationOp):
    def __init__(self, paramDict):
        self.delaySecs = self.getDictValueAsSeconds(paramDict, 'delaytime', default=None, fRequiredValue=True)
        return super().__init__(paramDict)
    def execute(self):
        delayWithConsoleMessage(self.delaySecs, 'Delay op')

class AutomationOp_WaitForEnterKeyToContinue(AutomationOp):
    def __init__(self, paramDict):
        self.fBeep = self.getDictValueAsBool(paramDict, 'beep', default=False)
        return super().__init__(paramDict)
    def execute(self):
        if self.fBeep:
            beep()
        applog_i_nc("{:s}: Press <Enter> to continue...".format(self.getClassSuffix()))
        print("{:s}: Press <Enter> to continue...".format(self.getClassSuffix()), end='')
        input() # todo-todo: Flush input before checking. sys.stdin.flush() doesn't work

class AutomationOp_Beep(AutomationOp):
    def __init__(self, paramDict):
        return super().__init__(paramDict)
    def execute(self):
        beep()

class AutomationOp_PrintMovieMode(AutomationOp):
    def __init__(self, paramDict):
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getMovieMode(self.retryInfo)
        self.printGetResponse(resp, respKeyVal='status')

class AutomationOp_PrintMovieQuality(AutomationOp):
    def __init__(self, paramDict):
        self.fListAbilities = self.getDictValueAsBool(paramDict, 'listavailable', default=False)
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getMovieQuality(self.retryInfo)
        self.printGetResponse(resp)

class AutomationOp_SetMovieQuality(AutomationOp):
    def __init__(self, paramDict):
        self.movieQualityStr = self.getDictValueAsStr(paramDict, 'moviequality', None, fRequiredValue=True)
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), self.movieQualityStr))
        ccapi.setMovieQuality(self.retryInfo, self.movieQualityStr)

class AutomationOp_EnterMovieMode(AutomationOp):
    def __init__(self, paramDict):
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}".format(self.getClassSuffix()))
        ccapi.setMovieMode(self.retryInfo, True)

class AutomationOp_ExitMovieMode(AutomationOp):
    def __init__(self, paramDict):
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}".format(self.getClassSuffix()))
        ccapi.setMovieMode(self.retryInfo, False)

class AutomationOp_StartMovieRecord(AutomationOp):
    def __init__(self, paramDict):
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}".format(self.getClassSuffix()))
        ccapi.startStopMovieRecord(self.retryInfo, True)

class AutomationOp_StopMovieRecord(AutomationOp):
    def __init__(self, paramDict):
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}".format(self.getClassSuffix()))
        ccapi.startStopMovieRecord(self.retryInfo, False)

class AutomationOp_PrintCameraInfo(AutomationOp):
    def __init__(self, paramDict):
        return super().__init__(paramDict)
    def execute(self):
        cameraInfo = ccapi.getCameraInfo(self.retryInfo)
        if cameraInfo['success']:
            cameraInfoStr = "Model: {:s}, S/N: {:s}, Firmware: {:s}".format(
                cameraInfo['productname'], cameraInfo['serialnumber'], cameraInfo['firmwareversion'])
        else:
            cameraInfoStr = "N/A"
        infoStr = cameraInfoStr
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), infoStr))

class AutomationOp_PrintTemperatureStatus(AutomationOp):
    def __init__(self, paramDict):
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getTemperatureStatus(self.retryInfo)
        infoStr = resp['status'] if resp['success'] else "N/A"
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), infoStr))


class AutomationOp_TakePhoto(AutomationOp):
    def __init__(self, paramDict):
        self.fPerformAF = self.getDictValueAsBool(paramDict, 'autofocus', default=False)
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}".format(self.getClassSuffix()))
        ccapi.takeStillPhoto(self.retryInfo, self.fPerformAF)

class AutomationOp_PrintBatteryInfo(AutomationOp):
    def __init__(self, paramDict):
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getBatteryInfo(self.retryInfo)
        if resp['success']:
            infoStr = "Name: {:s}, Kind: {:s}, Level: {:s}, Quality: {:s}".format(
                resp['name'], resp['kind'], resp['level'], resp['quality'])
        else:
            infoStr = "N/A"
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), infoStr))

class AutomationOp_PrintShootingSettings(AutomationOp):
    def __init__(self, paramDict):
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getShootingSettings(self.retryInfo)
        if resp['success']:
            infoStr = "Mode: {:s}, Aperture: {:s}, SS: {:s}, ISO: {:s}, EC: {:s}".format(
                resp['shootingmodedial']['value'], resp['av']['value'], resp['tv']['value'],
                resp['iso']['value'], resp['exposure']['value'])
        else:
            infoStr = "N/A"
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), infoStr))

class AutomationOp_GetInfoByUrl(AutomationOp):
    def __init__(self, paramDict):
        self.url = self.getDictValueAsStr(paramDict, 'url', default=None, fRequiredValue=True)
        self.fullUrl = ccapi.genFullUrl(self.url, verPrefix=None)
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.get(self.retryInfo, self.fullUrl)
        infoStr = "'{:s}': {}".format(self.url, resp) if resp['success'] else "N/A"
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), infoStr))

class AutomationOp_DisconnectWireless(AutomationOp):
    def __init__(self, paramDict):
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}".format(self.getClassSuffix()))
        return ccapi.disconnectWirelss(self.retryInfo)

class AutomationOp_PrintAPI(AutomationOp):
    def __init__(self, paramDict):
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getApiInfo(self.retryInfo)
        infoStr = "{}".format(resp) if resp['success'] else "N/A"
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), infoStr))

class AutomationOp_DownloadFileByUrl(AutomationOp):
    def __init__(self, paramDict):
        self.url = self.getDictValueAsStr(paramDict, 'url', default=None, fRequiredValue=True)
        self.fullUrl = ccapi.genFullUrl(self.url)
        self.outputDir = self.getDictValueAsStr(paramDict, 'outputdir', g.args['outputdir'])
        self.ifExists = self.getDictValueAsStr(paramDict, 'ifexists', g.args['ifexists'])
        return super().__init__(paramDict)
    def execute(self):

        AutomationOp.lastDownloadedFiles_FullUrl = None
        AutomationOp.lastDownloadedFiles_LocalPath = None

        #
        # formualte the full local path+filename to hold the file and handle
        # the case of what to do if the local file already exists.
        #
        filenameFromUrl = CCAPI.getFilenameFromUrl(self.fullUrl)
        localFileFullPath = os.path.join(self.outputDir, filenameFromUrl)
        localFileFullPath = checkWritingNewFileExists(localFileFullPath, self.ifExists)

        if localFileFullPath:
            #
            # file didn't exist or we're configured to generate a unique name. in either case,
            # 'localFileFullPath' holds the local path+filename we're to write the file to
            #
            (fFailed, downloadTimeSecs) = ccapi.downloadFile(self.retryInfo, self.fullUrl, localFileFullPath)
            if fFailed == False:
                AutomationOp.lastDownloadedFiles_FullUrl = [self.fullUrl]
                AutomationOp.lastDownloadedFiles_LocalPath = [localFileFullPath]
                applog_i("{:s} From {:s} stored at {:s} [download time = {:.2f} (s)]".format(self.getClassSuffix(), self.url, localFileFullPath, downloadTimeSecs))

class AutomationOp_GetPendingEvents(AutomationOp):
    def __init__(self, paramDict):
        self.fPrintEvents = self.getDictValueAsBool(paramDict, 'printevents', default=False)
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getPolledUpdate(self.retryInfo)
        if self.fPrintEvents:
            infoStr = "{}".format(resp) if resp['success'] else "N/A"
        else:
            infoStr = "Cleared-only (not printed)" if resp['success'] else "N/A"
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), infoStr))

class AutomationOp_WaitForNewFilesOnCamera(AutomationOp):
    def __init__(self, paramDict):
        self.maxWaitSecs = self.getDictValueAsScalar(paramDict, 'maxwaittime', default=None, fRequiredValue=True)
        return super().__init__(paramDict)
    def execute(self):
        AutomationOp.lastFilesFoundDuringPolling = None
        newFileList = ccapi.pollForNewFilesOnCamera(self.retryInfo, self.maxWaitSecs)
        applog_i("{:s}: {}".format(self.getClassSuffix(), newFileList))
        AutomationOp.lastFilesFoundDuringPolling = newFileList

class AutomationOp_DownloadNewFilesPolled(AutomationOp):
    def __init__(self, paramDict):
        self.outputDir = self.getDictValueAsStr(paramDict, 'outputdir', g.args['outputdir'])
        self.ifExists = self.getDictValueAsStr(paramDict, 'ifexists', g.args['ifexists'])
        return super().__init__(paramDict)
    def execute(self):

        AutomationOp.lastDownloadedFiles_FullUrl = []
        AutomationOp.lastDownloadedFiles_LocalPath = []

        if AutomationOp.lastFilesFoundDuringPolling == None:
            applog_i("{:s}: No files from previous WaitForNewFilesOnCamera".format(self.getClassSuffix()))
            return
        applog_i("{:s}: File(s):".format(self.getClassSuffix()))
        for index, fullUrl in enumerate(AutomationOp.lastFilesFoundDuringPolling):

            #
            # formualte the full local path+filename to hold the file and handle
            # the case of what to do if the local file already exists.
            #
            filenameFromUrl = CCAPI.getFilenameFromUrl(fullUrl)
            localFileFullPath = os.path.join(self.outputDir, filenameFromUrl)
            localFileFullPath = checkWritingNewFileExists(localFileFullPath, self.ifExists)

            if localFileFullPath:
                #
                # file didn't exist or we're configured to generate a unique name. in either case,
                # 'localFileFullPath' holds the local path+filename we're to write the file to
                #
                (fFailed, downloadTimeSecs) = ccapi.downloadFile(self.retryInfo, fullUrl, localFileFullPath)
                if fFailed == False:
                    AutomationOp.lastDownloadedFiles_FullUrl.append(fullUrl)
                    AutomationOp.lastDownloadedFiles_LocalPath.append(localFileFullPath)
                    applog_i("{:s}: {:s} stored at {:s} [download time = {:.2f} (s)]".format(self.getClassSuffix(), filenameFromUrl, localFileFullPath, downloadTimeSecs))

        if len(AutomationOp.lastDownloadedFiles_FullUrl) == 0:
            # if no files were downloaded then reset our last-downloaded vars to None (instead of an empty list)
            AutomationOp.lastDownloadedFiles_FullUrl = None
            AutomationOp.lastDownloadedFiles_LocalPath = None

class AutomationOp_GetInfoOnNewFilesPolled(AutomationOp):
    def __init__(self, paramDict):
        return super().__init__(paramDict)
    def execute(self):
        if AutomationOp.lastFilesFoundDuringPolling == None:
            applog_i("{:s}: No files from previous WaitForNewFilesOnCamera".format(self.getClassSuffix()))
            return
        applog_i("{:s}: File(s):".format(self.getClassSuffix()))
        for index, fullUrl in enumerate(AutomationOp.lastFilesFoundDuringPolling):
            resp = ccapi.getFileInfo(self.retryInfo, fullUrl)
            if resp['success']:
                infoStr = "Size: {:s}, Duration: {:s}, Date: {:s}".format(getHumanReadableSize(int(resp['filesize'])),
                    str(resp['playtime']) + ' (s)' if type(resp['playtime']).__name__ == 'int' else 'N/A',
                    resp['lastmodifieddate'])
            else:
                infoStr = "N/A"
            applog_i("File #{:d}: Name: {:s} {:s}".format(index+1, CCAPI.getFilenameFromUrl(fullUrl), infoStr))

class AutomationOp_RunExecutable(AutomationOp):
    def __init__(self, paramDict):
        self.executable = self.getDictValueAsStr(paramDict, 'executable', None, fRequiredValue=True)
        self.args = self.getDictValueAsStr(paramDict, 'args', None, fRequiredValue=True) # required, even if blank
        self.fAppendLastDownloadedFilenames = self.getDictValueAsBool(paramDict, 'appendlastdownloadstoargs', False)
        self.fExitOnLaunchErr = self.getDictValueAsBool(paramDict, 'exitonlauncherr', True)
        self.fWaitForCompletion = self.getDictValueAsBool(paramDict, 'waitforcompletion', True)
        # following options are only valid when waiting for completion. we still call getDictXXX since we pass default values for them to RunExecutable.execute()
        if not self.fWaitForCompletion:
            if 'outputfile' in paramDict or 'writemode' in paramDict or 'assertexitcode' in paramDict:
                applog_i("{:s}: outputfile/writemode/assertexitcode not valid when 'waitforcompletion' is false".format(self.getClassSuffix()))
                # don't exit, so that the line # of the above can be reported before we exit due to unusedparams
                return super().__init__(paramDict)
        self.outputFilename = self.getDictValueAsStr(paramDict, 'outputfile', "")
        self.outputFileWriteMethod = self.getDictValueStrChoice(paramDict, 'writemode', ['append', 'overwrite', 'overwrite_first_time'], 'append')
        self.assertExitCode = self.getDictValueAsScalar(paramDict, 'assertexitcode', None, int, fRequiredValue=False)
        return super().__init__(paramDict)
    def execute(self):
        args = self.args
        if self.fAppendLastDownloadedFilenames and AutomationOp.lastDownloadedFiles_LocalPath != None:
            filenameArgsList = ['"{:s}"'.format(x) for x in AutomationOp.lastDownloadedFiles_LocalPath] # build list with each filename in quotes (in case there are spaces in filename)
            filenameArgsStr = " ".join(filenameArgsList) # convert list to a single string, with a space between each quoted filename
            args += ' ' + filenameArgsStr
        applog_i("{:s}: exec={:s} args={:s} [{:s}]".format(self.getClassSuffix(), self.executable, args, ['NO_WAIT', 'WAIT'][self.fWaitForCompletion]))
        (fExecuteSuccessful, _errno) = RunExecutable.execute(self.executable, args, self.fWaitForCompletion, self.outputFilename, self.outputFileWriteMethod, "Running: {:s}".format(self.executable))
        if self.fExitOnLaunchErr and fExecuteSuccessful == False:
            exit(_errno)
        if self.fWaitForCompletion:
            if self.assertExitCode != None and _errno != self.assertExitCode:
                applog_i("{:s}: Terminating due exit code of {:d} not matching 'assertExitCode' value of {:d}".format(self.getClassSuffix(), _errno, self.assertExitCode))
                exit(ERRNO_RUN_EXECUTABLE_EXIT_MISMATCH_RET_CODE)

class AutomationOp_PrintMessageToLog(AutomationOp):
    def __init__(self, paramDict):
        self.message = self.getDictValueAsStr(paramDict, 'message', None, fRequiredValue=True)
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), self.message))

class AutomationOp_ExitApp(AutomationOp):
    def __init__(self, paramDict):
        self.exitCode = self.getDictValueAsScalar(paramDict, 'exitcode', 0, int)
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}: exitcode={:d}".format(self.getClassSuffix(), self.exitCode))
        exit(self.exitCode)

class AutomationOp_PrintAperture(AutomationOp):
    def __init__(self, paramDict):
        self.fListAbilities = self.getDictValueAsBool(paramDict, 'listavailable', default=False)
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getAperture(self.retryInfo)
        self.printGetResponse(resp)

class AutomationOp_SetAperture(AutomationOp):
    def __init__(self, paramDict):
        self.apertureStr = self.getDictValueAsStr(paramDict, 'aperture', None, fRequiredValue=True)
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), self.apertureStr))
        ccapi.setAperture(self.retryInfo, self.apertureStr)

class AutomationOp_PrintShutterSpeed(AutomationOp):
    def __init__(self, paramDict):
        self.fListAbilities = self.getDictValueAsBool(paramDict, 'listavailable', default=False)
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getShutterSpeed(self.retryInfo)
        self.printGetResponse(resp)

class AutomationOp_SetShutterSpeed(AutomationOp):
    def __init__(self, paramDict):
        self.shutterSpeedStr = self.getDictValueAsStr(paramDict, 'shutterspeed', None, fRequiredValue=True)
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), self.shutterSpeedStr))
        ccapi.setShutterSpeed(self.retryInfo, self.shutterSpeedStr)

class AutomationOp_PrintIso(AutomationOp):
    def __init__(self, paramDict):
        self.fListAbilities = self.getDictValueAsBool(paramDict, 'listavailable', default=False)
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getIso(self.retryInfo)
        self.printGetResponse(resp)

class AutomationOp_SetIso(AutomationOp):
    def __init__(self, paramDict):
        self.isoStr = self.getDictValueAsStr(paramDict, 'iso', None, fRequiredValue=True)
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), self.isoStr))
        ccapi.setIso(self.retryInfo, self.isoStr)

class AutomationOp_PrintExposureCompensation(AutomationOp):
    def __init__(self, paramDict):
        self.fListAbilities = self.getDictValueAsBool(paramDict, 'listavailable', default=False)
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getExposureCompensation(self.retryInfo)
        self.printGetResponse(resp)

class AutomationOp_SetExposureCompensation(AutomationOp):
    def __init__(self, paramDict):
        self.exposureCompensationStr = self.getDictValueAsStr(paramDict, 'exposurecompensation', None, fRequiredValue=True)
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), self.exposureCompensationStr))
        ccapi.setExposureCompensation(self.retryInfo, self.exposureCompensationStr)

class AutomationOp_PrintWhiteBalance(AutomationOp):
    def __init__(self, paramDict):
        self.fListAbilities = self.getDictValueAsBool(paramDict, 'listavailable', default=False)
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getWhiteBalance(self.retryInfo)
        self.printGetResponse(resp)

class AutomationOp_SetWhiteBalance(AutomationOp):
    def __init__(self, paramDict):
        self.whiteBalanceStr = self.getDictValueAsStr(paramDict, 'whitebalance', None, fRequiredValue=True)
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), self.whiteBalanceStr))
        ccapi.setWhiteBalance(self.retryInfo, self.whiteBalanceStr)

class AutomationOp_PrintShootingModeDial(AutomationOp):
    def __init__(self, paramDict):
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getShootingModeDial(self.retryInfo)
        self.printGetResponse(resp)

class AutomationOp_PrintDriveMode(AutomationOp):
    def __init__(self, paramDict):
        self.fListAbilities = self.getDictValueAsBool(paramDict, 'listavailable', default=False)
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getDriveMode(self.retryInfo)
        self.printGetResponse(resp)

class AutomationOp_SetDriveMode(AutomationOp):
    def __init__(self, paramDict):
        self.driveModeStr = self.getDictValueAsStr(paramDict, 'drivemode', None, fRequiredValue=True)
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), self.driveModeStr))
        ccapi.setDriveMode(self.retryInfo, self.driveModeStr)

class AutomationOp_PrintAfOperation(AutomationOp):
    def __init__(self, paramDict):
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getAfOperation(self.retryInfo)
        self.printGetResponse(resp)

class AutomationOp_PrintAfMethod(AutomationOp):
    def __init__(self, paramDict):
        self.fListAbilities = self.getDictValueAsBool(paramDict, 'listavailable', default=False)
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getAfMethod(self.retryInfo)
        self.printGetResponse(resp)

class AutomationOp_SetAfMethod(AutomationOp):
    def __init__(self, paramDict):
        self.afMethodStr = self.getDictValueAsStr(paramDict, 'afmethod', None, fRequiredValue=True)
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), self.afMethodStr))
        ccapi.setAfMethod(self.retryInfo, self.afMethodStr)

class AutomationOp_PrintMeteringMode(AutomationOp):
    def __init__(self, paramDict):
        self.fListAbilities = self.getDictValueAsBool(paramDict, 'listavailable', default=False)
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getMeteringMode(self.retryInfo)
        self.printGetResponse(resp)

class AutomationOp_SetMeteringMode(AutomationOp):
    def __init__(self, paramDict):
        self.meteringModeStr = self.getDictValueAsStr(paramDict, 'meteringmode', None, fRequiredValue=True)
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), self.meteringModeStr))
        ccapi.setMeteringMode(self.retryInfo, self.meteringModeStr)

class AutomationOp_PrintStillImageQuality(AutomationOp):
    def __init__(self, paramDict):
        self.fListAbilities = self.getDictValueAsBool(paramDict, 'listavailable', default=False)
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getStillImageQuality(self.retryInfo)
        if resp['success']:
            currentStr = "Raw: {:s}, JPEG: {:s}".format(resp['value']['raw'], resp['value']['jpeg'])
            if self.fListAbilities:
                availableStr = " Available: Raw: {}, JPEG: {}".format(resp['ability']['raw'], resp['ability']['jpeg'])
            else:
                availableStr = ""
            infoStr = "{:s}{:s}".format(currentStr, availableStr)
        print(resp['value']['jpeg'])
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), infoStr))

class AutomationOp_SetStillImageQuality(AutomationOp):
    def __init__(self, paramDict):
        self.rawQualityStr = self.getDictValueAsStr(paramDict, 'rawquality', None, fRequiredValue=True)
        self.jpegQualityStr = self.getDictValueAsStr(paramDict, 'jpegquality', None, fRequiredValue=True)
        return super().__init__(paramDict)
    def execute(self):
        applog_i("{:s}: Raw: {:s}, JPEG: {:s}".format(self.getClassSuffix(), self.rawQualityStr, self.jpegQualityStr))
        ccapi.setStillImageQuality(self.retryInfo, self.rawQualityStr, self.jpegQualityStr)

class AutomationOp_PrintLensInfo(AutomationOp):
    def __init__(self, paramDict):
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getLensInfo(self.retryInfo)
        if resp['success']:
            infoStr = "{:s}".format(resp['name'] if resp['mount'] else "No lens mounted")
        else:
            infoStr = "N/A"
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), infoStr))

class AutomationOp_AssertCameraSettings(AutomationOp):
    def __init__(self, paramDict):
        self.modeDialSetting = self.getDictValueAsStr(paramDict, 'modedial', default=None)
        self.fInMovieMode = self.getDictValueAsBool(paramDict, 'moviemode', default=None)
        self.afOperation = self.getDictValueAsStr(paramDict, 'afoperation', default=None)
        return super().__init__(paramDict)
    def execute(self):
        if self.modeDialSetting != None:
            resp = ccapi.getShootingModeDial(self.retryInfo)
            if resp['success'] and resp['value'] != self.modeDialSetting:
                applog_i("{:s}: Mode dial setting is \"{:s}\" but must be \"{:s}\"".format(self.getClassSuffix(), resp['value'], self.modeDialSetting))
                exit(ERRNO_ASSERT_CAMERA_SETTING_WRONG_VALUE)
        if self.fInMovieMode != None:
            resp = ccapi.getMovieMode(self.retryInfo)
            if resp['success']:
                fInMovieMode = resp['status'] == 'on'
                if fInMovieMode != self.fInMovieMode:
                    applog_i("{:s}: Movie Mode is \"{:s}\" but must be \"{:s}\"".format(self.getClassSuffix(), g.offOn[fInMovieMode], g.offOn[self.fInMovieMode]) )
                    exit(ERRNO_ASSERT_CAMERA_SETTING_WRONG_VALUE)
        if self.afOperation != None:
            resp = ccapi.getAfOperation(self.retryInfo)
            if resp['success'] and resp['value'] != self.afOperation:
                applog_i("{:s}: AF operation is \"{:s}\" but must be \"{:s}\"".format(self.getClassSuffix(), resp['value'], self.afOperation))
                exit(ERRNO_ASSERT_CAMERA_SETTING_WRONG_VALUE)
        applog_i("{:s}: Conditions met".format(self.getClassSuffix()))

class AutomationOp_PrintCameraDateTime(AutomationOp):
    def __init__(self, paramDict):
        return super().__init__(paramDict)
    def execute(self):
        resp = ccapi.getDateTime(self.retryInfo)
        infoStr = CCAPI.convertCameraDateTimeResp(resp)
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), infoStr))

class AutomationOp_SyncDateTime(AutomationOp):

    def __init__(self, paramDict):
        self.skewSecs = self.getDictValueAsScalar(paramDict, 'skewsecs', 0.7, float, fRequiredValue=False)
        return super().__init__(paramDict)
    def execute(self):
        # get camera's current date/time so we can later print the before/after camera time
        resp = ccapi.getDateTime(self.retryInfo)
        cameraCurrentDateTimeStr = CCAPI.convertCameraDateTimeResp(resp)
        # set the camera's date/time to this system's date time
        dateTimeStr = CCAPI.genCameraDateTimeStr(time.time() + self.skewSecs)
        ccapi.setDateTime(self.retryInfo, dateTimeStr, True if time.daylight else False)
        # print out what we've done
        cameraNewDateTimeStr = "{:s}{:s}".format(dateTimeStr, " DST" if time.daylight else "")
        infoStr = "Changed camera time from \"{:s}\" to \"{:s}\"".format(cameraCurrentDateTimeStr, cameraNewDateTimeStr)
        applog_i("{:s}: {:s}".format(self.getClassSuffix(), infoStr))

class AutomationOp_Group(AutomationOp):

    def __init__(self, paramDict):
        self.groupName = self.getDictValueAsStr(paramDict, 'groupname', default="")
        self.groupRepeatCount = self.getDictValueAsScalar(paramDict, 'grouprepeatcount', default=1)
        self.ops = []
        return super().__init__(paramDict)

    def appendOp(self, op):
        self.ops.append(op)

    def execute(self):
        for groupIteration in range(self.groupRepeatCount):
            if self.groupName != '.MainGroup': # don't print out banner for main group - it's an internal construct to contain the entire automation script
                applog_i("Performing group \"{:s}\", Iteration {:d}/{:d}".format(self.groupName, groupIteration+1, self.groupRepeatCount))
            for opIndex, op in enumerate(self.ops):
                repeatCount = op.repeatCount
                for opIteration in range(repeatCount):
                    op.execute()
                    if op.delayAfterSecs > 0:
                        delayWithConsoleMessage(op.delayAfterSecs, "'delayafter' OP param")

#
# list of all automation operation classes. Update this list whenever adding a new class. It is used to create
# instances based on the ops specified by the user on the command line or in the ops file
#
AutomationOpClasses = [
    AutomationOp_Group, AutomationOp_Delay, AutomationOp_PrintMovieQuality, AutomationOp_SetMovieQuality,
    AutomationOp_EnterMovieMode, AutomationOp_ExitMovieMode, AutomationOp_StartMovieRecord, AutomationOp_StopMovieRecord,
    AutomationOp_PrintCameraInfo, AutomationOp_PrintTemperatureStatus,
    AutomationOp_TakePhoto, AutomationOp_PrintBatteryInfo, AutomationOp_PrintShootingSettings, AutomationOp_GetInfoByUrl,
    AutomationOp_DisconnectWireless, AutomationOp_PrintAPI, AutomationOp_DownloadFileByUrl,
    AutomationOp_WaitForNewFilesOnCamera, AutomationOp_GetInfoOnNewFilesPolled, AutomationOp_RunExecutable,
    AutomationOp_DownloadNewFilesPolled, AutomationOp_PrintMessageToLog, AutomationOp_ExitApp,
    AutomationOp_PrintAperture, AutomationOp_SetAperture, AutomationOp_PrintShutterSpeed, AutomationOp_SetShutterSpeed,
    AutomationOp_PrintIso, AutomationOp_SetIso, AutomationOp_PrintExposureCompensation, AutomationOp_SetExposureCompensation,
    AutomationOp_PrintWhiteBalance, AutomationOp_SetWhiteBalance, AutomationOp_PrintShootingModeDial,
    AutomationOp_PrintDriveMode, AutomationOp_SetDriveMode, AutomationOp_PrintAfMethod, AutomationOp_SetAfMethod,
    AutomationOp_PrintMeteringMode, AutomationOp_SetMeteringMode, AutomationOp_PrintStillImageQuality, AutomationOp_SetStillImageQuality,
    AutomationOp_PrintLensInfo, AutomationOp_WaitForEnterKeyToContinue, AutomationOp_Beep, AutomationOp_AssertCameraSettings,
    AutomationOp_PrintMovieMode, AutomationOp_PrintAfOperation, AutomationOp_GetPendingEvents, AutomationOp_SyncDateTime,
    AutomationOp_PrintCameraDateTime
]


#
# Canon CCAPI access class
#
class CCAPI:

    class RetryInfo:
        def __init__(self, retryArgsDict):
            # timeouts/retries allowed
            self.timeoutSecs = retryArgsDict['timeout']
            self.downloadTimeoutSecs = retryArgsDict['downloadtimeout']
            self.transportErrorRetriesAllowed = retryArgsDict['transportretries']
            self.cmdErrorRetriesAllowed = retryArgsDict['cmdretries']
            self.retryDelaySecs = retryArgsDict['retrydelay']
            self.maxBusyTimeRetrySecs = retryArgsDict['maxbusyretrytime']
            self.fExitOnCmdErr = retryArgsDict['exitoncmderr']
        def cmdStarting(self):
            # vars to tracks errors that occurred during this request
            self.cmdTimeStart = secondsElapsed(None)
            self.transportErrorCount = 0
            self.cmdErrorCount = 0
        def shouldRetryBePerformed(self, fTransportError, request=None, resp=None):
            def _shouldRetryBePerformed(fTransportError, request, resp):
                if fTransportError:
                    self.transportErrorCount += 1
                    if self.transportErrorCount-1 < self.transportErrorRetriesAllowed:
                        return True
                    exit(ERRNO_CCAPI_TRANSPORT_ERROR)
                # not a transport error - must be a command error (ie, request delivered successfully but camera reported an error response
                status_code = request.status_code
                if status_code == 503 and (resp['message'] == 'Taken in preparation' or resp['message'] == 'Device busy'):
                    if secondsElapsed(self.cmdTimeStart) < self.maxBusyTimeRetrySecs:
                        return True
                    if self.fExitOnCmdErr:
                        applog_e("Exiting after exhausting 'maxbusyretrytime'({:d}) and 'exitoncmderr' is set".format(self.maxBusyTimeRetrySecs))
                        exit(ERRNO_CCAPI_CMD_ERROR)
                    return FALSE
                self.cmdErrorCount += 1
                if self.cmdErrorCount-1 < self.cmdErrorRetriesAllowed:
                    return True
                if self.fExitOnCmdErr:
                    applog_e("Exiting after exhausting 'cmdretries'({:d}) and 'exitoncmderr' is set".format(self.cmdErrorRetriesAllowed))
                    exit(ERRNO_CCAPI_CMD_ERROR)
                return False
            fPerformRetry = _shouldRetryBePerformed(fTransportError, request, resp)
            if fPerformRetry:
                delayWithConsoleMessage(self.retryDelaySecs, "'retrydelay'")
            return fPerformRetry


    def __init__(self):
        # communication settings
        self.ipAddress = ''
        self.port = DEFAULT_CCAPI_HTTP_PORT
        self.timeoutSecs = None
        self.transportErrorRetriesAllowed = None
        self.cmdErrorRetriesAllowed = None
        self.maxbusyretrytime = None
        # CCAPI URLs for various camera status/functions

    #
    # Converts a time value into a date/time string Canon uses for CCAPI /functions/datetime
    # @param timeEpoch Time value to convert
    # @return String containing date+time in Canon format
    #
    @staticmethod
    def genCameraDateTimeStr(timeEpoch):
        timeStruct = time.localtime(timeEpoch)
        timeStr = time.strftime("%a, %d %b %Y %H:%M:%S", timeStruct) # example: Sat, 29 Aug 2020 05:14:15
        timeZoneStr = time.strftime("%z", timeStruct) # example: -0700
        if timeStruct.tm_isdst:
            timeZoneStr = "{:05d}".format(int(timeZoneStr) - 100)
        return timeStr + " " + timeZoneStr

    #
    # Converts the camera's response from CCAPI /functions/datetime to a date/time string
    # @param timeEpoch Time value to convert
    # @return String containing date+time in Canon format
    #
    @staticmethod
    def convertCameraDateTimeResp(resp):
        if resp['success']:
            return "{:s}{:s}".format(resp['datetime'], " DST" if resp['dst'] else "")
        else:
            return "N/A"

    def genFullUrl(self, relativeUrl, verPrefix='ver100'):
        # example: http://192.168.1.142:8080/ccapi/ver100/devicestatus/temperature
        prefix = "/ccapi/" + verPrefix if verPrefix else ""
        return 'http://{:s}:{:d}{:s}{:s}'.format(self.ipAddress, self.port, prefix, relativeUrl)

    def _get(self, retryInfo, fullUrl, stream=False):
        timeoutSecs = retryInfo.timeoutSecs if not stream else retryInfo.downloadTimeoutSecs
        timeoutSecs = timeoutSecs if timeoutSecs > 0 else None
        retryInfo.cmdStarting()
        while True:
            applog_d_nc("GET {:s}".format(fullUrl))
            consoleWriteLine("GET {:s} ".format(re.findall(":[0-9]+(.*)", fullUrl)[0]))
            try:
                r = requests.get(fullUrl, stream=stream, timeout=timeoutSecs)
            except Exception as e:
                applog_e('** GET Failed ** "{:}"'.format(str(e)))
                retryInfo.shouldRetryBePerformed(True) # if this returns then it's assumed we're supported to retry otherwise exit() wouldn've been called
                continue
            consoleClearLine()
            if r.status_code != 200:
                # request failed
                resp = json.loads(r.text) if r.text else dict()
                applog_e('GET Request "{:s}" failed, status_code: {:d}, {}'.format(fullUrl, r.status_code, resp))
                if retryInfo.shouldRetryBePerformed(False, r, resp):
                    continue
            return r
        exit(ERRNO_HTTP_GET_FAILED)


    #
    # performs CCAPI "get" request, which obtains status/configuration information from the camera
    # @relativeUrl - Relative CCAPI URL describing what information is requested (ex: "devicestatus/temperature")
    # @return JSON object with parsed from JSON values returned by camera
    #
    def get(self, retryInfo, fullUrl):
        r = self._get(retryInfo, fullUrl, False)
        resp = json.loads(r.text) if r.text else dict()
        applog_d_nc("GET Result: status_code={:d}, data={}".format(r.status_code, resp))
        resp['status_code'] = r.status_code
        resp['success'] = (r.status_code == 200)
        return resp

    def getRelative(self, retryInfo, relativeUrl):
        return self.get(retryInfo, self.genFullUrl(relativeUrl))

    #
    # performs CCAPI "post" and "put" requests which performs an action or sets a configuration value
    # @return JSON object with parsed from JSON values returned by camera
    #
    def _postput(self, retryInfo, fullUrl, jsonInput, crudMethodStr):

        if crudMethodStr == 'POST':
            crudMethod = requests.post
        elif crudMethodStr == "PUT":
            crudMethod = requests.put
        else: raise AssertionError("_postput(): Invalid crudMethodStr \"{:s}\" passed".format(crudMethodStr))

        timeoutSecs = retryInfo.timeoutSecs if retryInfo.timeoutSecs > 0 else None
        retryInfo.cmdStarting()
        while True:
            applog_d_nc("{:s} {:s}, Data: {}".format(crudMethodStr, fullUrl, jsonInput))
            consoleWriteLine("{:s} {:s} ".format(crudMethodStr, re.findall(":[0-9]+(.*)", fullUrl)[0]))
            try:
                r = crudMethod(fullUrl, timeout=timeoutSecs, data=jsonInput)
            except Exception as e:
                applog_e('** {:s} Failed ** "{:}"'.format(crudMethodStr, str(e)))
                retryInfo.shouldRetryBePerformed(True) # if this returns then it's assumed we're supported to retry otherwise exit() wouldn've been called
                continue
            consoleClearLine()
            resp = json.loads(r.text) if r.text else dict()
            resp['status_code'] = r.status_code
            resp['success'] = (r.status_code == 200 or r.status_code == 202) # 202 means "Accepted" - occurs for network disconnect request
            if not resp['success']:
                # request failed
                applog_e('{:s} Request "{:s}" failed, status_code: {:d}, {}'.format(crudMethodStr, fullUrl, r.status_code, resp))
                if retryInfo.shouldRetryBePerformed(False, r, resp):
                    continue
                return resp
            applog_d_nc("{:s} Result: status_code={:d}, data={}".format(crudMethodStr, r.status_code, resp))
            return resp
        exit(ERRNO_HTTP_POST_FAILED)

    def post(self, retryInfo, fullUrl, jsonInput):
        return self._postput(retryInfo, fullUrl, jsonInput, 'POST')

    def put(self, retryInfo, fullUrl, jsonInput):
        return self._postput(retryInfo, fullUrl, jsonInput, 'PUT')

    def postRelative(self, retryInfo, relativeUrl, jsonInput):
        return self.post(retryInfo, self.genFullUrl(relativeUrl), jsonInput)

    def putRelative(self, retryInfo, relativeUrl, jsonInput):
        return self.put(retryInfo, self.genFullUrl(relativeUrl), jsonInput)

    def getApiInfo(self, retryInfo):
        return self.get(retryInfo, self.genFullUrl('/ccapi', verPrefix=None))

    def getCameraInfo(self, retryInfo):
        return self.getRelative(retryInfo, '/deviceinformation')

    def getDateTime(self, retryInfo):
        return self.getRelative(retryInfo, '/functions/datetime')

    def setDateTime(self, retryInfo, dateTimeStr, fDst):
        dict = { 'datetime' : dateTimeStr, 'dst' : fDst }
        return self.putRelative(retryInfo, '/functions/datetime', json.dumps(dict))

    def takeStillPhoto(self, retryInfo, fPerformAF=False):
        dict = { 'af' : fPerformAF }
        return self.postRelative(retryInfo, '/shooting/control/shutterbutton', json.dumps(dict))

    def getTemperatureStatus(self, retryInfo):
        return self.getRelative(retryInfo, '/devicestatus/temperature')

    def getMovieQuality(self, retryInfo):
        return self.getRelative(retryInfo, '/shooting/settings/moviequality')

    def setMovieQuality(self, retryInfo, movieQualityStr):
        dict = { 'value' : movieQualityStr }
        return self.putRelative(retryInfo, '/shooting/settings/moviequality', json.dumps(dict))

    def getMovieMode(self, retryInfo):
        return self.getRelative(retryInfo, '/shooting/control/moviemode')

    def setMovieMode(self, retryInfo, movieMode=True):
        dict = { 'action' : 'on' if movieMode==True else 'off' }
        return self.postRelative(retryInfo, '/shooting/control/moviemode', json.dumps(dict))

    def startStopMovieRecord(self, retryInfo, start=True):
        dict = { 'action' : 'start' if start==True else 'stop' }
        return self.postRelative(retryInfo, '/shooting/control/recbutton', json.dumps(dict))

    def getPolledUpdate(self, retryInfo):
        return self.getRelative(retryInfo, '/event/polling?continue=off')

    #
    # Waits for camera to indicate new file(s) are available on the camera (ie, a photo or video has
    # been taken since the last time the camera was polled).
    # @param maxWaitTimeSecs - Maximum time to wait for new file(s) before giving up. If this value is
    # zero then the camera will only be polled once
    # @return  An array of CCAPI URLs to the new file(s) or None if no new files were indicated
    #
    def pollForNewFilesOnCamera(self, retryInfo, maxWaitTimeSecs=5, pollIntervalSecs=0.5):
        # sample return value: 'http://192.168.1.142:8080/ccapi/ver100/contents/sd/100CANON/IMG_0327.JPG'
        timeStart = secondsElapsed(None)
        while True:
            updates = self.getPolledUpdate(retryInfo)
            if 'addedcontents' in updates:
                return updates['addedcontents']
            if maxWaitTimeSecs == 0 or secondsElapsed(timeStart) >= maxWaitTimeSecs:
                return None
            consoleWriteLine("Delaying {:.2f} on poll for new files".format(pollIntervalSecs))
            time.sleep(pollIntervalSecs)
            consoleClearLine()

    @staticmethod
    def getFilenameFromUrl(ccapiUrl):
        # get filename from URL by splitting the string into array by slashes and using the last element of the array
        # example URL: http://192.168.1.142:8080/ccapi/ver100/contents/sd/100CANON/IMG_0327.JPG
        urlComponents = ccapiUrl.split('/')
        return urlComponents[len(urlComponents)-1]

    def downloadFile(self, retryInfo, fullUrl, localFileFullPath):
        r = self._get(retryInfo, fullUrl, True)
        if r.status_code == 200:
            timeStart = secondsElapsed(None)
            consoleWriteLine("Downloading {:s}...".format(re.findall(":[0-9]+(.*)", fullUrl)[0]))
            try:
                with open(localFileFullPath, 'wb') as f:
                    r.raw.decode_content = True
                    shutil.copyfileobj(r.raw, f)
            except Exception as e:
                applog_e('** Download Failed ** "{:}"'.format(str(e)))
                exit(ERRNO_HTTP_DOWNLOAD_FAILED)
            timeElapsed = secondsElapsed(timeStart)
            consoleClearLine()
            return (False, timeElapsed)
        return (True, 0) # failed

    def getFileInfo(self, retryInfo, fullUrl):
        return self.get(retryInfo, fullUrl + '?kind=info')

    def getBatteryInfo(self, retryInfo):
        return self.getRelative(retryInfo, '/devicestatus/battery')

    def getShootingSettings(self, retryInfo):
        return self.getRelative(retryInfo, '/shooting/settings')

    def disconnectWirelss(self, retryInfo):
        dict = { 'action' : 'disconnect' }
        return self.postRelative(retryInfo, '/functions/wificonnection', json.dumps(dict))

    def getAperture(self, retryInfo):
        return self.getRelative(retryInfo, "/shooting/settings/av")

    def setAperture(self, retryInfo, apertureStr):
        dict = { 'value' : apertureStr }
        return self.putRelative(retryInfo, '/shooting/settings/av', json.dumps(dict))

    def getShutterSpeed(self, retryInfo):
        return self.getRelative(retryInfo, "/shooting/settings/tv")

    def setShutterSpeed(self, retryInfo, shutterSpeedStr):
        dict = { 'value' : shutterSpeedStr }
        return self.putRelative(retryInfo, '/shooting/settings/tv', json.dumps(dict))

    def getIso(self, retryInfo):
        return self.getRelative(retryInfo, "/shooting/settings/iso")

    def setIso(self, retryInfo, isoStr):
        dict = { 'value' : isoStr }
        return self.putRelative(retryInfo, '/shooting/settings/iso', json.dumps(dict))

    def getExposureCompensation(self, retryInfo):
        return self.getRelative(retryInfo, "/shooting/settings/exposure")

    def setExposureCompensation(self, retryInfo, exposureCompensationStr):
        dict = { 'value' : exposureCompensationStr }
        return self.putRelative(retryInfo, "/shooting/settings/exposure", json.dumps(dict))

    def getWhiteBalance(self, retryInfo):
        return self.getRelative(retryInfo, "/shooting/settings/wb")

    def setWhiteBalance(self, retryInfo, whiteBalanceStr):
        dict = { 'value' : whiteBalanceStr }
        return self.putRelative(retryInfo, '/shooting/settings/wb', json.dumps(dict))

    def getShootingModeDial(self, retryInfo):
        return self.getRelative(retryInfo, "/shooting/settings/shootingmodedial")

    def getDriveMode(self, retryInfo):
        return self.getRelative(retryInfo, "/shooting/settings/drive")

    def setDriveMode(self, retryInfo, driveModeStr):
        dict = { 'value' : driveModeStr }
        return self.putRelative(retryInfo, '/shooting/settings/drive', json.dumps(dict))

    def getAfOperation(self, retryInfo):
        return self.getRelative(retryInfo, "/shooting/settings/afoperation")

    def getAfMethod(self, retryInfo):
        return self.getRelative(retryInfo, "/shooting/settings/afmethod")

    def setAfMethod(self, retryInfo, afMethodStr):
        dict = { 'value' : afMethodStr }
        return self.putRelative(retryInfo, '/shooting/settings/afmethod', json.dumps(dict))

    def getMeteringMode(self, retryInfo):
        return self.getRelative(retryInfo, "/shooting/settings/metering")

    def setMeteringMode(self, retryInfo, meteringModeStr):
        dict = { 'value' : meteringModeStr }
        return self.putRelative(retryInfo, '/shooting/settings/metering', json.dumps(dict))

    def getStillImageQuality(self, retryInfo):
        return self.getRelative(retryInfo, "/shooting/settings/stillimagequality")

    def setStillImageQuality(self, retryInfo, rawQualityStr, jpegQualityStr):
        dict = { 'value' : { 'raw' : rawQualityStr, 'jpeg' : jpegQualityStr }}
        return self.putRelative(retryInfo, '/shooting/settings/stillimagequality', json.dumps(dict))

    def getLensInfo(self, retryInfo):
        return self.getRelative(retryInfo, "/devicestatus/lens")

#
# Converts a size value into human-readable form, including commas and a
# KB/MB/GB suffix as apporpriate
# @param sizeBytes Value to convert
# @return Value as a human-readable string
#
def getHumanReadableSize(sizeBytes):

    def truncate(number, digits):
        # https://stackoverflow.com/a/29257837
        return math.floor(number * 10 ** digits) / 10 ** digits

    if sizeBytes < 1024:
        return '{:,d} Bytes'.format(sizeBytes)
    if sizeBytes < 1024*1024:
        return '{:,.2f} KB'.format(truncate(sizeBytes/1024, 2))
    if sizeBytes < 1024*1024*1024:
        return '{:,.2f} MB'.format(sizeBytes/(1024*1024))
    return '{:,.2f} GB'.format(sizeBytes/(1024*1024*1024))

#
# Plays a system beep sound
#
def beep():
    sys.stdout.write('\a')

#
# Converts a string holding a time value in seconds or in hh:mm:ss/mm:ss
# timecode format into a scalar float value in seconds
# @param timeStr String holding time value
# @return float holding converted value in seconds
#
def convertTimeStrToSeconds(timeStr, fHandleConversionExeptions=True):
    listTimeSegments = timeStr.split(':') # if no colon is specified time is in seconds and this split will create a one-element list
    if len(listTimeSegments) > 3:
        applog_e("Time value \"{:s}\" has too many segments. The max number of segments is three, for hours:mins:secs".format(timeStr))
        exit(ERRNO_INVALID_TIME_VALUE_SPECIFIED)
    listTimeSegments.reverse()
    totalSeconds = 0.0
    try:
        for timeSegmentIndex in range(len(listTimeSegments)):
            totalSeconds += (60**timeSegmentIndex) * float(listTimeSegments[timeSegmentIndex])
    except ValueError as e:
        if not fHandleConversionExeptions:
            raise e
        applog_e("Unable to convert paramter value \"{:s}\" to a valid time value".format(timeStr))
        exit(ERRNO_INVALID_TIME_VALUE_SPECIFIED)
    return totalSeconds

#
# Delays execution for a specified amount of time, displaying
# a temporary console message during the delay
# @param delaySecs Number of seconds to delay (as float)
# @param delayReasonDesc Short description about the nature of the delay,
# which is displayed to the user

#
def delayWithConsoleMessage(delaySecs, delayReasonDesc=""):
    if delaySecs > 0:
        delayReasonStr = "(Reason: {:s})".format(delayReasonDesc) if delayReasonDesc != "" else ""
        secsRemaining = delaySecs
        while secsRemaining > 0:
            secsDelayThisIteration = min(secsRemaining, 1)
            consoleWriteLine("Delaying, {:.2f} seconds left {:}".format(secsRemaining, delayReasonStr))
            time.sleep(secsDelayThisIteration)
            secsRemaining -= secsDelayThisIteration
            consoleClearLine()
        applog_i("Delayed for {:.2f} seconds {:s}".format(delaySecs, delayReasonStr))

#
# Writes a message directly to console (stdout) without newline. This is used
# for single-line status/progress displays that update in-place
#
def consoleWriteLine(msg):
    sys.stdout.write(msg)
    sys.stdout.flush()

#
# Clears the current line of the console by printing a carriage return, a
# line of spaces, then another carriage return to return cursor to first
# column of line. This is used following a consoleWriteLine() sequence
#
def consoleClearLine(numCharsToClear=78):
    sys.stdout.write("\r" + " "*numCharsToClear + "\r")
    sys.stdout.flush()

# returns the number of seconds that have elapsed since
# the specified anchor time. if the anchor time is None
# then this routine returns the current time, which
# the caller can use for a subsequent call to get elapsed
# time. time values are floats
#
def secondsElapsed(timeAnchor):
    timeCurrent = time.time()
    if timeAnchor == None:
        return timeCurrent
    return timeCurrent - timeAnchor


#
# global vars
#
g  = GlobalVarsStruct()
ccapi = CCAPI()

def checkWritingNewFileExists(localFilenameWithPath, ifExistsOptionStr):

    #
    # generates a unique filename based on an original name by
    # adding a -new-%d suffix, repeating until it finds a suffix
    # number that's unique for the directory the file is in
    #
    def generateUniqueFilename(origFilenameWithPath):
        newSuffixCounter = 1
        while True:
            rootfilename = os.path.splitext(origFilenameWithPath)
            newFilename = "{:s}-new-{:d}{:s}".format(rootfilename[0], newSuffixCounter, rootfilename[1])
            if not os.path.exists(newFilename):
                return newFilename
            newSuffixCounter += 1

    #
    # prompts the user for a single-character selection (+enter),
    # only allowing characters specfied in validKeyList. check
    # is case-insensitive.
    #
    def promptWithSingleKeyResponse(promptStr, validKeyListStr):
        while True:
            key = input(promptStr).upper()
            if key and validKeyListStr.upper().find(key.upper()) != -1:
                return key

    if (os.path.exists(localFilenameWithPath)):
        if ifExistsOptionStr == 'prompt':
            applog_i("\"{:s}\" exists".format(localFilenameWithPath))
            keyResponse = promptWithSingleKeyResponse("(S)kip, (O)verwrite, (U)niquename, (E)xit [+enter]: ", 'soue')
        else:
            keyResponse = ''
        if ifExistsOptionStr == 'skip' or keyResponse == 'S':
            applog_i("Skipping \"{:s}\" - file exists".format(localFilenameWithPath))
            return None
        elif ifExistsOptionStr == 'overwrite' or keyResponse == 'O':
            applog_i("\"{:s}\" exists - will be overwritten".format(localFilenameWithPath))
            applog_d("{:s} - deleting existing file per user config".format(localFilenameWithPath))
            os.remove(localFilenameWithPath)
            return localFilenameWithPath
        elif ifExistsOptionStr == 'uniquename' or keyResponse == 'U':
            uniqueFilenameWithPath = generateUniqueFilename(localFilenameWithPath)
            applog_i("\"{:s}\" exists - will write to \"{:s}\"".format(localFilenameWithPath, uniqueFilenameWithPath))
            return uniqueFilenameWithPath
        elif ifExistsOptionStr == 'exit' or keyResponse == 'E':
            applog_i("\"{:s}\" exists - exiting per user config".format(localFilenameWithPath))
            exit(ERRNO_FILE_EXISTS_USER_SPECIFIED_EXIT)
    return localFilenameWithPath


#
# Returns instance of automation op class by string name
# @param name String holding name of class. A case-insensitive match is performed
# @return Class type or None if no class by name is found
#
def getAutomationOpClassByName(name):
    for classType in AutomationOpClasses:
        classNameSuffix = AutomationOp.getAutomationOpClassSuffix(classType.__qualname__)
        if classNameSuffix.lower() == name.lower():
            return classType
    return None


#
# Converts a list of strings into instances of automation ops, collected
# inside a main group
# @param lines List of strings, each containing an operation name and param=value pairs
# Examples of strings:
#   Delay secs=30
#   SaveFile filename="c:\My Documents\test.jpg"
# @return Main group instance, which has within it operations and possibly nested operation sub-groups
#
def processOpStrList(lines):

    def processOpListForGroup(groupInst, groupParamDict, groupNestingLevel, lineIndex, lines):

        #
        # remove params from the group dict that shouldn't be inherited by either child ops or child groups
        #
        groupParamDict.pop('groupname', None)
        groupParamDict.pop('grouprepeatcount', None)

        while lineIndex < len(lines):

            line = lines[lineIndex]

            #
            # converts a string of operation and param=value pairs into
            # a list. The operation and each param=value pairs are separated
            # by spaces. We allow for values that are within double-quotes,
            # which useful to support parameters that have spaces in them
            # like filenames
            #
            # sample string: Delay secs=5                                   Convers to Delay and secs=5
            # sample strnig: SaveFile Filename="c:\My Documents\test.txt"   Convers to SaveFile and Filename=c:\My Documents\test.txt
            #
            if re.findall("^\s*#", line):
                # commented-out line (has '#' as first non-whitespace character on line
                lineIndex += 1
                continue

            opAndParams = re.findall("(?:\".*?\"|\S)+", line)       # https://stackoverflow.com/a/53210803
            opAndParams = [re.sub('"', '', x) for x in opAndParams] # remove quotes

            if not opAndParams:
                # empty line
                lineIndex += 1
                continue

            applog_d("Translating op line: {:s}".format(line))

            opName = opAndParams[0]
            opNameLower = opName.lower()

            if opNameLower == "endgroup":
                # end of current group. step out of this recursed group
                if groupNestingLevel == 0:
                    applog_e("Error: 'EndGroup' found that doesn't match to a 'Group' (ie, extra 'EndGroup' specified)")
                    exit(ERRNO_UNMATCHED_END_GROUP_OP)
                return lineIndex+1

            # convert param=value pairs into dictionary
            try:
                paramDict = { x.split('=')[0].lower():x.split('=')[1] for idx, x in enumerate(opAndParams) if idx > 0 }
            except Exception as e:
                applog_e("Error translating parameter values for \"{:s}\" on line {:d}".format(line, lineIndex+1))
                exit(ERRNO_ERROR_PARSING_AUTOMATION_OP_PARAMS)
            paramDict = {**groupParamDict, **paramDict}     # merge the two dictionaries, with this op's params overriding the group's if present

            # match the operation to the list of operation classes supposed
            opClass = getAutomationOpClassByName(opName)
            if not opClass:
                applog_e("Unknown operation \"{:s}\" on line {:d}".format(opName, lineIndex+1))
                exit(ERRNO_UNKNOWN_AUTOMATION_OP)

            #
            # handle special case of 'RunExecutable', for which the arguments to the executable
            # are specified on the line below it. This is done beacuse those arguments may include
            # the same characters we use as delimters for our regular arguments, such as the equal
            # sign, quotes, etc... The format for the line below RunExecutable is:
            #
            #   RunExecutableArgs=xxx
            #
            # Where xxx are the literal arguments to pass to the executable. Those can include
            # double-quoted arguments for grouping. xxx can also be empty if there are no
            # arguments to pass to the executable
            #
            if opName.lower() == 'runexecutable':
                if lineIndex == len(lines)-1 or re.findall("^\s*RunExecutableArgs", lines[lineIndex+1], flags=re.IGNORECASE) == []:
                    applog_e("\"{:s}\" operation is missing the RunExecutableArgs= line below it on line {:d}".format(opName, lineIndex+1))
                    exit(ERRNO_RUNEXECUTABLE_OP_MISSING_ARGUMENT_LINE)
                argStr = re.findall("^\s*RunExecutableArgs\s*=\s*(.*)", lines[lineIndex+1], flags=re.IGNORECASE)[0]
                paramDict['args'] = argStr
                lineIndex += 1

            # create instance of operation using parameters specified
            opInst = opClass(paramDict.copy()) # use copy because op classes remove keys as values are used in order to detect superflous params

            unusedParamDictKeys = opInst.getUnusedParamDictKeys()
            if unusedParamDictKeys:
                applog_e("\"{:s}\" operation on line {:d} has the following unsupported parameters: {}".format(opName, lineIndex+1, unusedParamDictKeys))
                exit(ERRNO_UNKNOWN_OP_PARAM)

            # insert operation instance into current group
            groupInst.appendOp(opInst)

            if opName == "Group":
                # new group was just created. recurse to place subsequent ops into new gruop
                lineIndex = processOpListForGroup(opInst, paramDict, groupNestingLevel+1, lineIndex+1, lines)
            else:
                lineIndex += 1

        #
        # reached the end of the file
        #
        if groupNestingLevel > 0:
            # no more lines yet we're inside a nested group - user is missing one or more 'EndGroup' ops
            applog_e("Error: A 'Group' is missing a corresponding 'EndGroup'")
            exit(ERRNO_UNMATCHED_GROUP_OP)

        return lineIndex

    mainGroupParamDict = { 'groupname' : '.MainGroup' }
    mainGroup = AutomationOp_Group(mainGroupParamDict)
    processOpListForGroup(mainGroup, mainGroupParamDict, 0, 0, lines)

    return mainGroup

#
# Process a user-specified automation op file, creating instances of
# automation gruops and operations based on the content of the file
#
def processOpFile(opFilename):

    #
    # open and read contents of the op file
    #
    try:
        f = open(opFilename, "r")
    except Exception as e:
        applog_e("Unable to open \"{:s}\": {:s}".format(opFilename, str(e)))
        exit(ERRNO_CANT_OPEN_OP_FILE)
    try:
        lines = f.readlines()
    except Exception as e:
        applog_e("Unable to read \"{:s}\": {:s}".format(opFilename, str(e)))
        exit(ERRNO_CANT_READ_OP_FILE)

    lines = [x.rstrip('\n') for x in lines] # remove any trailing newlines

    return processOpStrList(lines)


#
# processCmdLine - Processes command line arguments
#
class ArgumentParserError(Exception): pass # from http://stackoverflow.com/questions/14728376/i-want-python-argparse-to-throw-an-exception-rather-than-usage
class ArgumentParserWithException(argparse.ArgumentParser):
    def error(self, message):
        raise ArgumentParserError(message)

def processCmdLine():

    def convertArgStrToBool(argName):
        argValue = g.args[argName]
        if argValue.lower() in ['1', 'true', 't', 'yes', 'y']:
            g.args[argName] = True
            return
        if argValue.lower() in ['0', 'false', 'f', 'no', 'n']:
            g.args[argName] = False
            return
        applog_e("Option \"{:s}\" requires a boolean value but \"{:s}\" specified".format(argName, argValue))
        exit(ERRNO_BAD_CMD_LINE_ARG_VALUE)

    parser = ArgumentParserWithException(fromfile_prefix_chars='!',\
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description='Canon CCAPI Automator (Horshack)',\
        epilog="Options can also be specified from a file. Use !<filename>. Each word in the file must be on its own line.\n\nYou "\
            "can abbreviate any argument name provided you use enough characters to\nuniquely distinguish it from other argument names.\n")
    parser.add_argument('--ipaddress', type=str.lower, help='IP address of camera.', default='', metavar="addr", required=True)
    parser.add_argument('--port', type=int, help='CCAPI Port configured on camera. Default is %(default)s', default=DEFAULT_CCAPI_HTTP_PORT, required=False)
    parser.add_argument('--opfile', type=str, help='File containing list of automation operations to perform. Example: --opfile \"c:\My Documents\ops.txt"', default=None, metavar="filename", required=False)
    parser.add_argument('--op', type=str, help='List of automation operations to perform, each separated with a "|". Example: --op \"Delay secs=5 | DisplayMovieQuality\"', default=None, metavar="oplist", required=False)
    parser.add_argument('--outputdir', type=str, help='Directory to store image/file(s) to.  Default is current directory. If path contains any spaces enclose it in double quotes. Example: --outputdir \"c:\My Documents\"', default=None, metavar="path", required=False)
    parser.add_argument('--ifexists', type=str.lower, choices=['uniquename', 'skip', 'overwrite', 'prompt', 'exit'], help='Action to take if file with same name already exists. Default is "%(default)s"', default='uniquename', required=False)
    parser.add_argument('--exitoncmderr', type=str, help='Exit if camera reports error for an op command. Default is "%(default)s"', default=str(DEFAULT_EXIT_ON_CMD_ERR), required=False)
    parser.add_argument('--timeout', type=float, help='CCAPI HTTP request timeout in seconds (0 = no timeout). Default is %(default)s seconds', default=DEFAULT_CCAPI_TIMEOUT_SECS, required=False)
    parser.add_argument('--downloadtimeout', type=int, help='CCAPI HTTP file download timeout in seconds (0 = no timeout). Default is %(default)s', default=DEFAULT_CCAPI_DOWNLOAD_TIMEOUT_SECS, required=False)
    parser.add_argument('--transportretries', type=int, help='CCAPI retries for transport errors (such as timeouts). Default is %(default)s', default=DEFAULT_CCAPI_TRANSPORT_ERROR_RETRIES, required=False)
    parser.add_argument('--cmdretries', type=int, help='CCAPI retries for cmds ending in error. Default is %(default)s', default=DEFAULT_CCAPI_CMD_ERROR_RETRIES, required=False)
    parser.add_argument('--retrydelay', type=float, help='CCAPI delay between retries for cmds ending in error. Default is %(default)s', default=DEFAULT_CCAPI_RETRY_DELAY_SECS, required=False)
    parser.add_argument('--maxbusyretrytime', type=int, help='How long to retry when camera reports busy. Default is %(default)s', default=DEFAULT_CCAPI_MAX_BUSY_RETRY_SECS, required=False)
    parser.add_argument('--logginglevel', type=str.lower, choices=['normal', 'verbose', 'debug' ], help='Sets how much information is saved to the result log. Default is "%(default)s"', default='normal', required=False)

    #
    # if there is a default arguments file present, add it to the argument list so that parse_args() will process it
    #
    defaultArgFilename = os.path.join(g.appDir, "{:s}-defaultopts".format(APP_NAME))
    if os.path.exists(defaultArgFilename):
        sys.argv.insert(1, "!" + defaultArgFilename) # insert as first arg (past script name), so that the options in the file can still be overriden by user-entered cmd line options

    if len(sys.argv) == 1:
        # print help if no parameters passed
        parser.print_help()
        exit(0)


    # perform the argparse
    try:
        args = vars(parser.parse_args())
    except ArgumentParserError as e:
        applog_e("Command line error: " + str(e))
        exit(ERRNO_BAD_CMD_LINE_ARG)

    # set our global var to the processed argument list and log them
    g.args = args

    #
    # do any processing required on command-line arguments
    #

    if not g.args['outputdir']:
        # no explicit output directory specified - use current directory
        g.args['outputdir'] = '.\\' if g.isWin32 else './'

    # process 'logginglevel' (at init we default to 'normal')
    if g.args['logginglevel'] == 'verbose':
        applog_set_loggingFlags(APPLOGF_LEVEL_INFORMATIONAL | APPLOGF_LEVEL_ERROR | APPLOGF_LEVEL_WARNING | APPLOGF_LEVEL_VERBOSE)
    elif g.args['logginglevel'] == 'debug':
        applog_set_loggingFlags(APPLOGF_LEVEL_INFORMATIONAL | APPLOGF_LEVEL_ERROR | APPLOGF_LEVEL_WARNING | APPLOGF_LEVEL_VERBOSE | APPLOGF_LEVEL_DEBUG)

    ccapi.ipAddress = g.args['ipaddress']
    ccapi.port = g.args['port']

    convertArgStrToBool('exitoncmderr')

    #
    # last step - process the operations specified by the user
    #
    if g.args['opfile'] and g.args['op']:
        applog_e("Both --opfile and --op specified on command line. Only one can be used")
        exit(ERRNO_BAD_CMD_LINE_ARG)
    if not g.args['opfile'] and not g.args['op']:
        applog_e("Either --opfile or --op must specified on command line")
        exit(ERRNO_BAD_CMD_LINE_ARG)
    if g.args['opfile']:
        g.mainAutomationGroup = processOpFile(g.args['opfile'])
    else:
        strList = g.args['op'].split('|') # multiple ops can be specified, separated by a | character
        g.mainAutomationGroup = processOpStrList(strList)

    # log the cmd line arguments
    applog_d("Orig cmd line: {:s}".format(str(sys.argv)))
    applog_d("Processed cmd line: {:s}".format(str(g.args)))


#
# sets app-level globals related to the platform we're running under and
# creates path to app directories, creating them if necessary
#
def establishAppEnvironment():

    g.appStartTime = time.time()
    g.isWin32 = (platform.system() == 'Windows')
    g.isOSX = (platform.system() == 'Darwin')

    #
    # determine the directory our script resides in, in case the
    # user is executing from a different working directory.
    #
    g.appDir = os.path.dirname(os.path.realpath(sys.argv[0]))

    #
    # setup our app's data directory, where we store our
    # logs and temporary files. By default we create it off
    # the directory containing the script file. If that directory
    # isn't writable then we use the user's home directory
    #
    appDataDir = "{:s}-appdata".format(APP_NAME)
    if os.access(g.appDir, os.W_OK):
        g.appDataDir = os.path.join(g.appDir, appDataDir)
    else:
        g.appDataDir = os.path.join(str(pathlib.Path.home()), appDataDir)
    # create the data directory if it doesn't already exist
    if not os.path.exists(g.appDataDir):
        os.makedirs(g.appDataDir)

#
# invoked near  exit, issues a final log message which is used by utils to signify a gracefully
# shutdown and then tells the applog module to shut itself down
#
def exit(errno):
    applog(">>>> {:s} session over (exit={:d}), logs at \"{:s}\"".format(APP_NAME, errno, g.appDataDir))
    applog_shutdown()
    sys.exit(errno)


#
# main routine
#
def main():

    #
    # init our environment and our logging mechanism
    #
    # Log filenames are canomate-log-last.txt (most recent session) and
    # canomate-log-lifetime.txt (all sessions)
    #
    establishAppEnvironment()
    errno = applog_init(APPLOGF_LEVEL_INFORMATIONAL | APPLOGF_LEVEL_ERROR, os.path.join(g.appDataDir, "{:s}-log-last.txt".format(APP_NAME)),\
        os.path.join(g.appDataDir, "{:s}-log-lifetime.txt".format(APP_NAME)))
    if errno:
        sys.exit(errno)

    #
    # print app banner and process the command line
    #
    print("") # console newline before our banner
    applog_i("{:s} v{:s} - Automation Utility for Canon cameras (uses Canon Control API via WiFi)".format(APP_NAME, APP_VERSION))
    applog_i("Copyright (c) Horshack, System Time: {:s}, Py: {:d}.{:d}.{:d}, OS: {:s}\n".format(getDateTimeStr(g.appStartTime),
        sys.version_info.major, sys.version_info.minor, sys.version_info.micro, platform.system()))

    #
    # process command line and execute user-supplied automation operations
    #
    try:
        processCmdLine()
        g.mainAutomationGroup.execute()
    except KeyboardInterrupt as e:
        try:
            applog_e("\n>>>> Terminated by ctrl-c user keypress")
            exit(ERRNO_CTRL_C_INTERRUPT)
        except KeyboardInterrupt as e:
            # for some reason we intermittently get a second SIGINT running on Linux in certain environs - ignore it
            pass
    except Exception as e:
        applog_e("Unhandled code exception: {:s}".format(str(e)))
        applog_e("Stack trace: {}".format(traceback.format_exc()))
        exit(ERRNO_CODE_EXCEPTION)

    exit(0)


#
# program entry point
#
if __name__ == "__main__":
    if sys.version_info.major < 3 and sys.version_info.minor < 5:
        print("Python version {:d}.{:d}.{:d} detected. Requires Python 3.5 or above".format(sys.version_info.major, sys.version_info.minor, sys.version_info.micro))
        sys.exit(ERRNO_BAD_PYTHON_VER)
    main()