o

#!/usr/bin/env python3
# Copyright (c) 2021, Oracle and/or its affiliates.  All rights reserved.
# This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license.

# ==============================================================================
# o - a wrapper for OCI CLI to simplify and edify
#
# Author: Kevin.Colwell@oracle.com
# ==============================================================================

import signal
import time
import textwrap
import subprocess
import re
import json
import getopt
import sys
import os
if (sys.platform != "win32"):
    import fcntl
import errno
import datetime
VERSION = "1.8"
UPDATED = "2023-06-12"

# ==============================================================================
# Global variables
# ==============================================================================
ocids_file_name = ''     # name of file where OCIDs are saved
ocid = {}                # dict of known ocids
ocids = []               # list of known ocids
orphans = []             # orphaned ocids (with no compartment) found in ocids file
command = []             # commands with required and optional params
show_help = False        # user asked for help
go = False               # execute the oci cli command
nogo = False             # don't execute due to error in ocid substitutions
quiet = 0                # quiet mode - 1: don't show command; 2: don't show headers
debug = False            # Debug output
region = ''              # region
region_code = ''         # 3-letter region code
ocids_in_output = False  # substitute names for ocids in output

# Universally useful fields as default output, if no "-o" on command line.
# This leverages the fact that fields not found in data are quietly ignored.
# We want to show the most interesting fields for the most common ocid types.
default_out_spec = 'name{:.30}#shape#cidr-block#prohibit-public{:5.5}' \
    '#lifec#size-in-gbs{:>11.11}#size-in-mbs{:>11.11}#size' \
    '#availability-domain{:.1}#fault-domain{:>1.1}' \
    '#resource-type' \
    '#created{:16.16}#compartment{:.20}#instance-id' \
    '#id{:>8.8}' \
    '#direction#destination' \
    '#ip-address#public-ip#private-ip' \
    '#vnic-id#desc{:.40}' \
    '#access-uri#storage-tier' \
    '#token'

matches = []                   # list of commands that match user input
all_options_for_this_cmd = ''  # for best_match
remaining_options = ''         # for best_match
options_out = []               # list of expanded options to be passed to oci cli
oci_command_line = ''          # the complete oci cli command in string form
column = {}                    # field format info: format:'{:}', offset:0, maxwidth:n
use_bold = True                # enable xterm bold highlighting
wrap = 120                     # wrap at column <wrap>
global_options = ''

# ==============================================================================


def usage(s=''):
    print("""
    o - a friendly helper for Oracle Cloud Infrastructure CLI

    usage: o [<o options>] <oci-command> [<oci-options>] [.]

      <o options>
         -o name#life#time:.16  Table output
                            Show fields that match "name", "life" or "time"
                            Truncate "time" at 16 characters
         -o name,life,time  CSV output
         -o name/life/time  Text output - one field per line
         -o /               Show all fields in text
         -o json            JSON format - unfiltered oci output
         -O                 Show OCIDs (not names) in output
         -i <infile>        Take JSON input from <infile>;  "o -i -" for stdin

      <oci-command>    The oci "service resource action" specification
      <oci-options>    Options and parameters for <oci-command>
                       These will be expanded into a more correct oci command

      . or go          If the line ends with "." (or "go") run the oci command
      !                If line ends with "!", force-run the command, even if
                       param substitution fails""")

    if s != "help":
        print("""
      o help           More "o" options and usage.""")
    else:
        print("""
    More output options:
      o -o shap.ocpus       Show matching sub-field, e.g. shape-config.ocpus
      o -o +route#dns-label Add fields default output
      o -o +shap.ocpus      Append fields to default output
      o -o +/               Keep default fields, change format to Text
      o -o created:10.10    Show first 10 characters of time-created
      o -o id:-10.10        Show last 10 characters of OCID

    More "o" options:
         -q            Suppress display of "oci" command line
         -qq           Also suppress headers and non-"data" results from output
                       Useful when piping results other commands

      o oci_commands   Build (or rebuild) the $HOME/.oci/oci_commands file

      o <tenancy-ocid> Set up $HOME/.oci/ocids file with compartment-ocids
                       from a tenancy.  Use this for first-time setup, or to add
                       additional tenancies.  Remember to set OCI_CLI_PROFILE
                       (or use --profile) for different tenancies.

      o ocids "name"   Show saved OCIDs for resources matching "name"

      o prune "name"   Remove OCIDs for resources matching "name" from
                       $HOME/.oci/ocids.  If "name" is a compartment, remove
                       all OCIDS under compartment "name".

      touch $HOME/.oci/.otmp    Activate "save last result" feature. With this
                                you can reformat output from the last command
                                without re-running the command with:
                                  o -o <fmt>

      o <oci-command> help      Show all parameters, including globals
      o <oci-command> --help .  Get help from oci""")
    print("""
    o Version {} - {}
    """.format(VERSION, UPDATED))
    if s not in ('', 'help'):
        print(bold(s) + '\n')
    exit(1)

# ==============================================================================


def interrupted(sig, frame):
    error_out("Interrupted")
    exit(1)

# ==============================================================================


def bold(s=''):
    return '\033[1m' + s + '\033[0m' if use_bold else s

# ==============================================================================


def error_out(msg):
    """Decode JSON ServiceError."""

    if msg.startswith('ServiceError:'):
        j = json.loads(msg[14:])
        print(bold('ServiceError(' + str(j['status']) + '): ' + str(j['code'])), file=sys.stderr)
        print(bold(j['message']), file=sys.stderr)
    else:
        print(bold(msg), file=sys.stderr)

# ==============================================================================


def read_oci_commands_file():
    """Read oci_commands for the list of all cli commands with parameters.
    First look for oci_commands where o is installed.
    If not found, look for ~/.oci/oci_commands
    If not found, create new file ~/.oci/oci_commands
    """

    global global_options
    pathname = os.path.dirname(sys.argv[0])
    if not pathname:
        pathname = '.'
    oci_commands_file = pathname + '/oci_commands'
    if not os.path.isfile(oci_commands_file):
        oci_commands_file = os.path.expanduser('~/.oci/oci_commands')

    command_list = []
    try:
        for line in open(oci_commands_file).read().splitlines():
            if re.match(r'oci ', line):
                command_list.append(parse_oci_command_options(line))
            elif re.match(r'global_options ', line):
                global_options = line[line.find(' ') + 1:]

    except FileNotFoundError:
        print(bold('No ' + oci_commands_file + ' file.'))
        print('Create it with:\n')
        print(bold('    o oci_commands'))
        print('\nIt takes about two minutes.')
        exit(1)
    except OSError as e:
        msg = "{0}".format(e)
        print(bold(msg))
        exit(1)
    if len(command_list) < 1000:
        print(bold('File ' + oci_commands_file + ' is incomplete.'))
        print('Re-create it with:')
        print(bold('    o oci_commands'))
        exit(1)
    return command_list

# ==============================================================================


def run_command(command):
    if os.system(command):
        if re.search('iam compartment get', command):
            return (1)
        print(bold("It looks like that didn't work.  So sorry."))
        exit(1)
    return 0

# ==============================================================================


def get_from_oci_config(key):
    """Look in config file for key = value"""

    s = oci_command_line
    if '--profile' in s:
        profile = s.split()[s.split().index('--profile') + 1]
    elif os.getenv('OCI_CLI_PROFILE'):
        profile = os.getenv('OCI_CLI_PROFILE')
    else:
        profile = 'DEFAULT'

    config_file = os.path.expanduser('~/.oci/config')
    if '--config' in oci_command_line:
        config_file = s.split()[s.split().index('--config') + 1]

    if not os.path.exists(config_file) and os.path.exists('/etc/oci/config'):
        config_file = '/etc/oci/config'

    try:
        value = None
        this_profile = ''
        with open(config_file, 'r') as f:
            for line in f:
                m = re.search(r'^\[(.+)\]', line)
                if m:
                    this_profile = m.group(1)

                if this_profile in ('DEFAULT', profile):
                    m = re.search(r'[^#]*' + key, line)
                    if m and not line.startswith('#'):
                        value = line.split('=')[1].strip()
                        if this_profile == profile:
                            return value
        return value

    except BaseException:
        return None

# ==============================================================================


def setup_ocids_file():
    """Create or update ocids_file for tenancy found on command line.
    If run with "o <tenancy-ocid>" run three o commands to populate ocids
        o <tenancy-ocid>
    Otherwise offer help and then exit.
    On completion exit; do not return from this function.
    """

    if not (len(sys.argv) in [2, 3] and re.search(
            'ocid1.tenancy.{50}', sys.argv[1])):
        # No ocid provide; offer help
        print(bold('No ' + ocids_file_name + ' found.\n'))
        print("""To setup "o" just run:
  \033[1mo <your-tenancy-ocid> \033[0m

and I'll do this for you:

  o iam compartment list -c <tenancy-ocid> -ciis true -all .
  o iam ad list -c <tenancy-ocid> .
  o iam region-subscription list .
        """)

        # To be extra helpful, get likely tenancy ocid from oci config file

        oci_config_file = os.path.expanduser('~/.oci/config')
        if not os.path.exists(oci_config_file) \
                and os.path.exists('/etc/oci/config'):
            oci_config_file = '/etc/oci/config'

        t = get_from_oci_config('tenancy')
        print('Try:\n', bold('  o ' + t))

    else:
        # <tenancy-ocid> provided... Let's roll!
        print("Tenancy ocid:", sys.argv[1])
        if len(sys.argv) == 3:
            print("Tenancy name:", sys.argv[2])
        print(bold('Setting up your ' + ocids_file_name + ' file:'))
        print(bold('\nGetting compartment names and ocids...'), end='')
        if run_command(sys.argv[0] + ' -o name iam compartment list -c ' + sys.argv[1] + ' -ciis true -all .'):
            print('\nThat didn\'t work.  Your account may need additional privileges.')
            exit(0)
        print(bold('\nGetting availability domains...'), end='')
        run_command(sys.argv[0] + ' -o name iam ad list -c ' + sys.argv[1] + ' .')
        print(bold('\nGetting regions...'), end='')
        run_command(sys.argv[0] + ' -o name iam region-subscription list .')
        print(bold('\nGetting tenancy name from root compartment...'), end='')
        if run_command(sys.argv[0] + ' -o name iam compartment get -c ' + sys.argv[1] + ' .'):
            # compartment get on tenancy failed; try saving a stub for tenancy
            print("\nCould not get root compartment.")
            tenancy_name = input(bold("Enter tenancy name: "))
            if not tenancy_name:
                tenancy_name = 'root'
            newids = {'root': {
                      'type': 'tenancy',
                      'alias': tenancy_name,
                      'id': sys.argv[1],
                      'name': tenancy_name,
                      },
                      }
            ocids_file('write', newids)
            exit(0)
        print('\n' + bold('All set. Have fun!\n'))

    exit(0)

# ==============================================================================


def ocids_file(action, new_ocids={}):
    """Take action on ocids_file_name:
    action=read - populate globals:
        ocid - dict of resources
        ocids - list of resources
    action=write - write ocids file
       If new_ocids = {}, write ocids to file (used by prune to remove entries)
       if new_ocids, merge new entries into file:
         - lock ocids file, then
         - read ocids file to get latest updates
         - merge new_ocids with ocids
         - rewrite file and unlock
    """
    global ocid, ocids, ocids_file_name

    if os.path.exists('.ocids'):
        ocids_file_name = '.ocids'
    elif os.path.exists('ocids'):
        ocids_file_name = 'ocids'
    else:
        ocids_file_name = os.path.expanduser('~/.oci/ocids')

    # Run setup if no ocids_file exists or
    # if this is an "o <tenancy>" setup command and new_ocids is empty
    if (not new_ocids
            and (not (re.search('iam compartment list .*-c ocid..tenancy', ' '.join(sys.argv)))
                 and (not os.path.exists(ocids_file_name)
                 or re.search('o [^c]*ocid..tenancy', ' '.join(sys.argv))))):
        setup_ocids_file()

    if action == 'read' and not os.path.exists(ocids_file_name):
        return

    tries = 1
    while tries <= 50:
        try:
            if os.path.exists(ocids_file_name):
                os.umask(0o077)
                f = open(ocids_file_name, 'r+')
                if new_ocids:
                    if (sys.platform != "win32"):
                        fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
                if new_ocids or action == 'read':
                    ocids = json.load(f)
                    # Convert list of ocids into dict of ocids.
                    # Use id as key, except for special types.
                    special_types = ['availabilitydomain', 'opcnextpage']
                    ocid = {i['id']: i for i in ocids if i['type'] not in special_types}
                    ocid.update({i['name']: i for i in ocids if i['type'] in special_types})
            else:
                os.umask(0o077)
                f = open(ocids_file_name, 'w+')

            if action == 'write':
                # merge
                ocid.update(new_ocids)
                # Don't save ocids for deleted resources
                ocid = {k: v for (k, v) in ocid.items()
                        if 'lifecycle-state' not in v
                        or v['lifecycle-state'] not in ('DELETED', 'TERMINATED')
                        and k not in orphans}
                # clean up temp values in ocid
                for k, v in ocid.items():
                    if 'score' in v:
                        del v['score']
                    if 'parents' in v:
                        del v['parents']

                # convert dict to list
                ocids = [ocid[k] for k in ocid.keys()]
                ocids.sort(key=lambda x:
                           (x['type'], x['alias'].lower(), x['id']))
                # write ocids list to ocid_file
                f.seek(0)
                print(json.dumps(ocids, indent=4), file=f)
                f.truncate()
            f.close()
            return

        except IOError as e:
            # ocids file locked for writes - take short nap and try to write again
            if e.errno != errno.EAGAIN:
                print(bold(ocids_file_name + ': not accessible'), file=sys.stderr)
            f.close()
            msec = (int.from_bytes(os.urandom(4)) % 200 + 5) * tries
            time.sleep(msec / 1000.)
            tries += 1
        except ValueError:
            # ocids file corrupt (write in progress?) - sleep and try to read again
            f.close()
            time.sleep(0.05 * tries)
            tries += 1

    if tries >= 50:
        print(bold(ocids_file_name + ':  ' + action + ' failed'), file=sys.stderr)

    return

# ==============================================================================


def get_new_ocids_from_returned_items(items):
    """Collect ocids found in results from oci cli.
    For consistency each entry must have: type, alias, id.
    In addition to those, we collect anything that might be used by omap.
    """

    # special case for region list - save the whole list in one entry
    if 'oci iam region list' in oci_command_line:
        return {'regionlist': {
                'type': 'regionlist',
                'alias': 'regionlist',
                'id': 'regionlist',
                'data': items}}

    if oci_command_line.startswith('oci search'):
        for i in items:
            if 'identifier' in i and 'id' not in i:
                i['id'] = i['identifier']

    collected = {}
    try:
        for item in [i for i in items if 'id' in i]:
            type = item['id'].split('.')[1] if '.' in item['id'] else ''
            # Special case: save AD-name:n because commands use name, not ocid
            ID = item['id']
            if 'availability' in item['id']:
                ID = item['name']
            if 'alias' in item and item['alias'] == 'opc-next-page':
                ID = item['alias']
                type = 'opcnextpage'
            if 'retention-rule' in oci_command_line:
                type = 'retentionrule'

            # Still need to validate with many OCID types:
            #    cpe, cross-connect, fast-connect, ip-sec, virtual-circuit
            if type == 'availabilitydomain':
                alias = item['name'][-1:]
            elif 'object-name' in item:
                alias = item['name']
                type = "par"
            elif (type == 'routetable'
                  and item['display-name'].startswith('Default Route Table for ')):
                alias = item['display-name'][len('Default Route Table for '):]
            elif (type == 'securitylist'
                  and item['display-name'].startswith('Default Security List for ')):
                alias = item['display-name'][len('Default Security List for '):]
            elif type == 'vnicattachment':
                if item['instance-id'] in ocid:
                    alias = ocid[item['instance-id']]['alias']
                else:
                    alias = item['instance-id'][-8:]
                alias = '-'.join([alias, str(item['nic-index']), 'vlan' + str(item['vlan-tag'])])
            elif type == 'providerservice':
                alias = str(item['provider-name']) + ' ' + str(item['provider-service-name'])
            elif 'instanceconsoleconnection' in type:
                alias = ocid[item['instance-id']]['alias']
            elif 'name' in item:
                alias = item['name']
            elif 'display-name' in item:
                alias = item['display-name']
            elif 'db-name' in type:
                alias = item['db-name']
            elif 'credential' in type:
                alias = item['description']
            else:
                # Unable to determine data type, don't add to collection
                # print("type:", type, json.dumps(item, indent=4))
                return ({})

            if not alias:
                alias = 'unnamed-' + type
            collected[ID] = {'type': type, 'alias': alias, 'id': item['id']}

            # Now add various non-empty fields depending on item type
            collected[ID].update({k: item[k] for k in item.keys() if (
                item[k] and (
                    k.endswith('-id')
                    or k.endswith('-tags')
                    or k.endswith('name')
                    or k.startswith('size')
                    or k in ['lifecycle-state',
                             'availability-domain',
                             'storage-tier',
                             'description',
                             'shape',
                             'cidr-block',
                             'dns-label',
                             'virtual-router-ip',
                             'virtual-router-mac',
                             'security-list-ids',
                             'route-rules',
                             'ip-addresses',
                             'private-ip',
                             'public-ip',
                             'is-primary',
                             'subnet-ids',
                             'access-type',
                             'created-by',
                             'time-expires',
                             'access-uri']))})
            if 'created-by' in item and '/' in item['created-by']:
                collected[ID]['created-by'] = item['created-by'].split('/')[1]
            if 'time-created' in item:
                collected[ID]['time-created'] = item['time-created'][:19]
            # Retain access-uri.
            if type == 'par' and ID in ocid and 'access-uri' in ocid[ID]:
                collected[ID]['access-uri'] = ocid[ID]['access-uri']
    except BaseException:
        print(bold('Unable to collect ocids of type ' + type
              + '. Please provide this data with your problem report:\n')
              + json.dumps(item, indent=4), file=sys.stderr)

    return collected

# ==============================================================================


def show_command(c, full=False, prefix='oci '):
    print(prefix + c['action'])
    if full:
        print('    requires\t' + re.sub(r' -', r'\n\t\t-', c['required']))
        print('    optional\t' + re.sub(r' -', r'\n\t\t-', c['optional']))
        if show_help:
            print('    global\t' + re.sub(r'\n', r'\n                ',
                  textwrap.fill(global_options, width=60, break_on_hyphens=False)))

# ==============================================================================


def parse_oci_command_options(line):
    """Separate oci command from --required and + --optional options."""
    c = {'action': '', 'required': '', 'optional': '', 'line': line}
    if ' + ' in line:
        [line, c['optional']] = line.split(' + ', 1)
    if ' --' in line:
        [c['action'], r] = line[4:].split(' --', 1)
        c['required'] = '--' + r
    else:
        c['action'] = line[4:]
    return (c)

# ==============================================================================


def args_match_command(c, params):
    """Compare argv with command "c" to see how well they match."""
    action = ' ' + c['action'] + ' '
    for a in params:

        # accept plurals in place of singulars
        if (len(a) > 3 and a[-1] == 's' and a not in ['address', 'access', 'waas']):
            a = a[:-1]
            if a[-2:] == 'ie':
                a = a[:-2]

        # Look for full-word match first
        if ' ' + a + ' ' in action:
            # Scratch out match
            action = re.sub(' ' + a + ' ', ' ', action, count=1)
            continue

        # look for word starting with arg
        if ' ' + a in action or '-' + a in action:
            # Scratch out this match
            action = re.sub(' ' + r'(\S+-)*' + a + r'(\S+)', ' ', action, count=1)
            continue

        # look for rlc -> really-long-command
        multi_re = ' ' + r'\w+\-'.join(list(a)) + r'\S+'
        if re.search(multi_re, action):
            # Scratch out this match
            action = re.sub(multi_re, ' ', action)
            continue
        return (False)
    return (True)

# ==============================================================================


def option_parameter(arg):
    """Look for match in remaining_options
        - Allow - in place of --
        - Replace partial opt with full option name
        --rlon ==> --really-long-option-name
    """
    a = arg.lstrip('-')
    if not a:
        return arg
    # special case for -c
    if a in ['c', 'cid']:
        a = 'compartment-id'

    m = re.search('(--' + a + ') ', remaining_options)
    if (m):
        return m.group(1)
    m = re.search('(--' + a + '[a-z-]*) ', remaining_options)
    if (m):
        return m.group(1)

    # build a regex a[0] + a[1] + -a[1] + -a[2]... for -rlon
    multi_re = '-+' + r'\w+\-'.join(list(a[:-1]))
    # special case if it ends in 'id'
    if len(a) > 2 and a[-2:] == 'id':
        multi_re += 'd\\b'
    else:
        multi_re += r'\w+\-' + a[-1] + r'[\w-]+'
    m = re.search(multi_re, remaining_options)
    if m:
        return m.group(0)

    # Unable to expand; just go with what user provided
    return arg

# ==============================================================================


def alias_match_score(spec, name):
    """Does user spec match this item name?
    score reflects quality of match.
    """
    if spec == name:
        return 40 + len(spec)
    elif spec.lower() == name.lower():
        return 25 + len(spec)
    elif re.match(spec + ' ', name):
        return 25 + int(200 * len(spec) / len(name)) / 10.0
    elif re.match(spec, name):
        return 20 + int(200 * len(spec) / len(name)) / 10.0
    elif re.match(spec, name, flags=re.I):
        return 10 + int(200 * len(spec) / len(name)) / 10.0
    elif re.search(spec, name, flags=re.I):
        return 5 + int(200 * len(spec) / len(name)) / 10.0
    return 0

# ==============================================================================


def get_item_parents(i):
    """Return heirarchy of compartments containing this item in list"""
    if 'parents' in i:
        return i['parents']
    if 'compartment-id' not in i:
        return [i['alias']]
    if i['compartment-id'] in ['', 'publisherCompartment']:
        return [i['alias']]
    if i['compartment-id'] in ocid:
        if i['type'] == 'availabilitydomain':
            return get_item_parents(ocid[i['compartment-id']]) + [i['name']]
        else:
            return get_item_parents(ocid[i['compartment-id']]) + [i['alias']]
    else:
        # Tag this as orphan for later removal?
        orphans.append(i['id'])
        return [i['alias']]

# ==============================================================================


def alias_match(spec, item):
    """Does user spec match this item alias - and parent(s)?
    Return True if match; score reflects quality of match
    """
    item['score'] = 0
    item['parents'] = get_item_parents(item)
    specs = list(filter(None, spec.split('/')))

    item_path = list.copy(item['parents'])
    for s in reversed(specs):
        # more specs than path parts?
        if not item_path:
            return False
        while item_path:
            i = item_path.pop()
            score = alias_match_score(s, i)
            if score > 0:
                if item['score'] == 0:
                    item['score'] = 2 * score
                else:
                    item['score'] += score
                break  # from while, to next "s"
            else:
                # not an alias match, look for ocid match
                if len(specs) == 1:
                    if re.search(spec, item['id'], flags=re.I):
                        item['score'] = int(1000 * len(spec) / len(item['id'])) / 10.0
                        return True
                    return False
                # This item heirarchy did not match specs.
                # Give up if it's the first part of spec or end of item_path
                if item['score'] == 0 or not item_path:
                    return False
                # Otherwise deduct 20% from score as a penalty
                item['score'] = int(item['score'] * .8)

    if item['score'] > 0:
        return True
    return False

# ==============================================================================


def type_match(option, item_type):
    """Compare option with item for type match, return boolean.
    If parameter --<option>-id doesn't match ocid1.<item_type>.oc1...
    add it to the list of exceptions.
    """

    if (option == item_type
            or (option.startswith('compartment') and item_type == 'tenancy')
            or (option.startswith('tenant') and item_type == 'tenancy')
            or (option == 'ig' and item_type == 'internetgateway')
            or (option == 'rt' and item_type == 'routetable')
            or (option == 'nsg' and item_type == 'networksecuritygroup')
            or (option == 'stack' and item_type == 'ormstack')
            or (option == 'sourcebootvolume' and item_type == 'bootvolume')
            or (option == 'image' and item_type == 'containerimage')
            or (option == 'repository' and item_type == 'containerrepo')
            or (option == 'dhcp' and item_type == 'dhcpoptions')
            or (option == 'application' and item_type == 'fnapp')
            or (option == 'function' and item_type == 'fnfunc')
            or (option == 'asset' and item_type in ('volume', 'bootvolume', 'volumegroup'))
            or (option == 'sddc' and item_type == 'vmwaresddc')
            or (option == 'avail' and len(item_type) == 1)):
        return True
    return False

# ==============================================================================


def ambiguous_value(option, value, matches):
    """Multiple matches with same score.  Ambiguous."""
    if (option):
        print(option, end=' ', file=sys.stderr)
    print(bold(value), 'is ambiguous. Try one of these:',
          file=sys.stderr)
    for m in matches:
        if m['score'] > 2:
            print('  (score=' + str(m['score']) + ')', '  ' + bold(('/'.join(m['parents'])
                  if 'parents' in m else m['alias'])), ' ({:8})'.format(m['id'][-8:]))
    global nogo
    nogo = True

# ==============================================================================


def datetime_parameter(dt):
    """construct datetime values relative to now or today +/- offset:
       now - current datetime
       today - start (00:00) of current day
       +/-<n>(wdhms) add/subtract <n> week/day/hour/minute/second."""

    # if time only (no date), prepend today's date
    if re.match(r'\d\d:\d\d', dt):
        return datetime.datetime.utcnow().strftime('%Y-%m-%dT') + dt + 'Z'

    dts = dt
    # return time delta from now|today|yesterday
    m = re.search(r'(?P<day>now|today|yesterday) *(?P<delta>.*)', dt)
    if m and m.group('day'):
        if m.group(1) == 'now':
            time = datetime.datetime.utcnow()
        elif m.group(1) in ('today', 'yesterday'):
            time = datetime.datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
            if m.group(1) == 'yesterday':
                time = time - datetime.timedelta(hours=24)
        dts = time.strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
    if m and m.group('delta'):
        deltas = {'s': 1, 'm': 60, 'h': 60*60, 'd': 60*60*24, 'w': 60*60*24*7}  # noqa: E226
        m2 = re.search(r'(?P<s>[@+-]{,1})(?P<v>[\d:Z]*)(?P<u>[smhdw]{0,1})', m.group('delta'))
        try:
            # optional time delta +/- <n> week/day/hour/minute/second
            value = int(m2.group('v')) * deltas[m2.group('u')]
            if m2.group('s') == '-':
                value = value * -1
            time = time + datetime.timedelta(seconds=value)
            dts = time.strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
        except (ValueError, KeyError):
            dts = dts[:11] + m.group('delta')
            if m.group('delta')[0] in '@+':
                dts = dts[:11] + m.group('delta')[1:]
            if dts[-1] != 'Z':
                dts = dts + 'Z'

    if dts.endswith('00:00:00Z'):
        return dts[:10]
    if dts.endswith(':00Z'):
        return dts[:16] + 'Z'
    return dts

# ==============================================================================


def get_region(r=''):
    """Determine the region in this order of precedence:
        r passed in - from command line
        OCI_CLI_REGION in env
        --profile on cmd line
        OCI_CLI_PROFILE in env
    Return key.lower,name  E.g.: "iad","us-ashburn-1"
    """

    if 'regionlist' not in ocid:
        return '', ''

    if not r:
        r = os.getenv('OCI_CLI_REGION')

    # If region not set, get it from config file
    if not r:
        r = get_from_oci_config('region')
    if not r:
        return None, None

    rmatch = [i for i in ocid['regionlist']['data'] if r.upper() == i['key'] or r in i['name']]
    if rmatch:
        return rmatch[0]['key'].lower(), rmatch[0]['name']
    return None, None

# ==============================================================================


def region_match(id):
    """Does ocid region match profile region?"""
    region_from_id = (id + '....').split('.')[3]
    if not region_code \
            or region_from_id == '' \
            or region_from_id == region_code \
            or region_from_id == region:
        return True
    return False

# ==============================================================================


def get_matching_ocid(option, value):
    """Look for ocid of correct type that matches value.
    If result is unique, or a strong match, return the OCID.
    If ambiguous, display possible matches and return None.
    """

    global nogo
    if value.startswith('ocid') and len(value) > 60:
        return value
    if '-' not in option:
        opt_type = option
    else:
        opt_type = re.sub('-', '', re.sub('-ids{0,1}$', '', option))
        # ocid type doesn't exactly match --resource-id type
        if opt_type == 'job':
            opt_type = 'ormjob'
        elif opt_type == 'node' and 'rover' in oci_command_line:
            opt_type = 'rovernode'
        elif opt_type == 'authtoken':
            opt_type = 'credential'
        elif opt_type == 'zonenameor':
            opt_type = 'dns-zone'
        elif opt_type == 'identityprovider':
            opt_type = 'saml2idp'
        elif opt_type == 'peer' and 'local' in oci_command_line:
            opt_type = 'localpeeringgateway'
        elif opt_type == 'peer' and 'remote' in oci_command_line:
            opt_type = 'remotepeeringconnection'
        elif opt_type == 'organization':
            opt_type = 'organizationsentity'
        elif opt_type == 'routedistribution':
            opt_type = 'drgroutedistribution'
        # Add new idtypes to the above when the --whatever-id does not match ocid1.whatevs...

    matches = [item for k, item in ocid.items() if
               type_match(opt_type, item['type'])
               and region_match(item['id'])
               and alias_match(value, item)]
    if not matches:
        matches = [item for k, item in ocid.items() if
                   type_match(opt_type, item['type'])
                   and alias_match(value, item)]
        if matches:
            print(bold('\n' + value + ' found in '
                  + matches[0]['id'].split('.')[3].upper() + ' region,'
                  + ' but you are working in ' + region_code.upper()))
    matches = sorted(matches, key=lambda i: i['score'], reverse=True)

    if option == 'bucket' and '/' not in value:
        return value

    if not matches:
        if option != 'bucket':
            print(opt_type, bold(value), 'not found.', file=sys.stderr)
            nogo = True
        return value

    # Is this match unique, or is it strong relative to other matches?
    if len(matches) == 1 or matches[0]['score'] > 5 + matches[1]['score']:
        if matches[0]['type'] in ('availabilitydomain', 'bucket'):
            if 'name' in matches[0]:
                return matches[0]['name']
            if 'display-name' in matches[0]:
                return matches[0]['display-name']
        return matches[0]['id']

    ambiguous_value(option, value, matches)
    return value

# ==============================================================================


def value_parameter(option, value):
    """Assist with command line parameters.
    Especially provide name -> ocid substitutions for --*id options.
    """

    # Get the parameter from help: --option [param_type]
    param_type = ''
    m = re.search(option + r' \[([\w -\|]+?)\]', all_options_for_this_cmd)
    if m:
        param_type = m.group(1)

    # If option is an OCID, see if value matches any saved ocids
    if option.endswith('-id') or option == "--availability-domain":
        match = get_matching_ocid(option, value)
        return match

    # Match "bucket-name" with bucket IDs
    if option == '--bucket-name':
        match = get_matching_ocid('bucket', value)
        return match

    # Provide opc-next-page if --page next
    if option in ('--page', '--start') and 'next' in value:
        for k, item in ocid.items():
            if item['alias'] == 'opc-next-page':
                return item['id']

    # Convert list of resource names into a JSON list [complex type]
    if option.endswith('-ids'):
        idlist = []
        for v in value.split(','):
            if not v:
                continue
            o = get_matching_ocid(option, v)
            idlist.append(v)
        return repr(json.dumps(idlist))

    # Assist with datetime parameters
    if param_type == 'datetime':
        return datetime_parameter(value)

    # Special help with log-search --search-query "search comp1 comp2 | ..."
    # Substitute "ocid1.compartment..." for one or more values.
    if option == '--search-query':
        m = re.search(r'search ([^|]+)', value)
        if m:
            for comp in m.group(1).split():
                c = comp
                if c.startswith('"'):
                    c = comp[1:-1]
                o = get_matching_ocid('compartment', c)
                if o != c:
                    value = re.sub(comp, '"' + o + '"', value)

    # Special help with structured-search --search-query "where compartmentId = ocid"
    if option == '--query-text':
        match = re.findall(r"(\bc\w*) *= *('*\w+'*)", value, flags=re.IGNORECASE)
        for (m, spec) in match:
            if m.lower() == 'compartmentid'[:len(m)]:
                if m.lower() != 'compartmentid':
                    value = re.sub(r'\b' + m + r'\b', 'compartmentId', value)
            if spec.startswith("'"):
                spec = spec[1:-1]
            o = get_matching_ocid('compartment', spec)
            if o != spec:
                value = re.sub(spec, "'" + o + "'", value)

    return value

# ==============================================================================


def shell_escape(value):
    """If param contains special characters, enclose in quotes or
    double-quotes, and escape inner double-quotes
    """

    if re.search(r'[ "\'?()<>{}]', value):
        if "'" not in value:
            value = "'" + value + "'"
        elif '"' not in value:
            value = '"' + value + '"'
        else:
            value = '"' + re.sub(r'"', r'\"', value) + '"'
    return value

# ==============================================================================


def get_fields_from(spec, keynames):
    """Return list of fields based on user specification.
    Field separator determines output formatting.
    """

    fields = []
    sep = '|'
    if ',' in spec:
        sep = ','   # csv out
    elif ' ' in spec:
        sep = ' '   # space separated columns
    elif '\t' in spec:
        sep = '\t'   # tab separated columns
    elif '#' in spec:
        sep = '#'   # aligned out columns
    elif '/' in spec:
        sep = '/'   # one field per line
    if not keynames:
        return fields, sep   # show all fields

    specs = spec.split(sep)

    if sep == r'#':
        sep = '|'
    if specs and not specs[-1]:
        specs.pop(-1)  # remove empty spec

    for sf in specs:
        s, fmt = sf, '{}'
        if s == '.':
            s = ''
        m = re.search('([^:{]+)(.*)', s)
        if m:
            s, fmt = m.group(1), m.group(2)
            if not fmt.startswith('{'):
                fmt = '{' + fmt + '}'
            if fmt.startswith('{:r') or fmt.startswith('{:-'):
                fmt = '{:>' + fmt[3:]
        if s in sorted(keynames):               # exact keyname match
            fields.append(s)
            column[s] = {'format': fmt, 'offset': 0, 'minwidth': len(s)}
            if debug: print('{:>30.30}'.format(s), '->', s)   # noqa: E701
        else:
            # Search for partial keyname matches
            sf = s.split('.')
            for k in sorted(keynames):
                if (len(user_out_spec) >= len(default_out_spec) and (
                        s == 'name' and k == 'quota-names'
                        or s == 'size' and 'resize' in k
                        or s == 'id' and 'bandwidth' in k
                        or s == 'id' and 'identity' in k)):
                    continue
                kf = k.split('.')
                # sub-key partial matches must partial-match: key.subkey
                if ((k not in fields)
                        and ((s.lower() in k.lower() and '.' not in k)
                             or ('.' in s and '.' in k
                                 and (sf[-1].lower() in kf[-1].lower())
                                 and (sf[-2].lower() in kf[-2].lower())))):
                    fields.append(k)
                    column[k] = {'format': fmt, 'offset': 0, 'minwidth': len(kf[-1])}
                    if debug: print('{:>30.30}'.format(s), '->', k)    # noqa: E701

    if len(fields) == 0:
        if user_out_spec != default_out_spec:
            print(bold("no matching output fields: " + spec), file=sys.stderr)
        return get_fields_from("/", keynames)
    return fields, sep

# ==============================================================================


def set_column_widths(fields, out_sep, item_list):
    global maxkeylen
    for k in fields:
        if ocids_in_output:
            maxwidth = max([len(str(f[k])) for f in item_list])
        else:
            maxwidth = 3
            for i in item_list:
                if k in i:
                    width = len(str(i[k]))
                    if k != 'id' and str(i[k]) in ocid:
                        width = len(ocid[i[k]]['alias'])
                    maxwidth = max([maxwidth, width])
                else:
                    width = len(get_field_from_item(k, i))
                    maxwidth = max([maxwidth, width])
        maxwidth = max([maxwidth, column[k]['minwidth']])
        column[k]['maxwidth'] = maxwidth

        m = re.search(r':[<^>]*\.(\d+)', column[k]['format'])
        if m:
            n = min(maxwidth, int(m.group(1)))
            s = column[k]['format'][0:column[k]['format'].index('.')]
            column[k]['format'] = s + str(n) + '.' + str(n) + '}'

        # if right-aligned and truncated, output right-most characters
        m = re.search(r'>.*\.(\d+)', column[k]['format'])
        if m:
            column[k]['offset'] = -1 * int(m.group(1))
        if column[k]['format'] == '{}' \
                and out_sep == '|' \
                and k is not fields[-1]:
            column[k]['format'] = '{:<' + str(maxwidth) + '.' + str(maxwidth) + '}'
    maxkeylen = max([len(str(k)) for k in fields])
    # keyfmt - with of longest field name, for raw text output
    column['keyfmt'] = '{:>' + str(maxkeylen) + '}'
    return

# ==============================================================================


def show_column_headers(out_fields, sep):
    if quiet > 1:
        return
    if sep == '|':
        for k in out_fields:
            # print(bold(column[k]['format'].format(k)), end=' ', file=sys.stderr)
            try:
                if k == 'id' and user_out_spec == default_out_spec:
                    print(bold(' reg'), end=' ', file=sys.stderr)
                print(bold(column[k]['format'].format(k.split(".")[-1])),
                      end=' ', file=sys.stderr)
            except ValueError:
                print(bold("bad format: " + k + column[k]['format']))
                column[k]['format'] = '{}'
        print('', file=sys.stderr)
    elif sep != '/':
        if sep == ',':
            print(bold(sep.join([repr(f) for f in out_fields])), file=sys.stderr)
        else:
            print(bold(sep.join(out_fields)), file=sys.stderr)

# ==============================================================================


def jdump_item(item, indent=2):
    """Remove json decorations, empty lines from json.dumps output"""

    return (re.sub(r'(?m)"\s*$|( *)"', r'\1',         # remove outer double-quotes
            re.sub(r'^\["', '"',                      # remove braces
            # noqa: E128 remove closing braces and trailing newline
            re.sub(r'(?m)[\[\]{},]+$', '',            # noqa: E128
            # noqa: E128 remove braces at beginning of lines, empty lines
            re.sub(r'(?m)^[\s{}[\],]+\n', '',         # noqa: E128
            # noqa: E128 change  key: [ value ] ->  key: value
            re.sub(r': \[\s+("[^"]*")\s+]', r': \1',  # noqa: E128
            json.dumps(item, indent=indent, ensure_ascii=False)))))))     # noqa: E128

# ==============================================================================


def get_field_from_item(field, item):
    """field contains "." - need to look for sub-fields"""
    value = ''
    if '.' not in field:
        if field in item:
            value = str(item[field])
    else:
        f = field.split('.')
        if f[-1] in item[f[0]]:
            value = jdump_item(item[f[0]][f[1]])
        else:
            if type(item[f[0]]) is list and len(item[f[0]]) > 0:
                subitem = item[f[0]]
            else:
                subitem = item[f[0]][f[1]] if f[1] in item[f[0]] else None
            if type(subitem) is dict and f[-1] in subitem:
                value = jdump_item(subitem[f[-1]])
            if type(subitem) is list and f[-1] in subitem[0]:
                s = ['    ' + v[f[-1]] for v in subitem if f[-1] in v]
                value = jdump_item(s)
    return(value.strip())

# ==============================================================================


def show_item(item, fields, sep):
    """Report result in user specified -o format."""

    out = []
    for field in fields:
        value = ''
        value = get_field_from_item(field, item)

        if field not in ('id', 'identifier') and value in ocid and not ocids_in_output:
            value = ocid[value]['alias']
            out.append(column[field]['format'].format(value[column[field]['offset']:]))
            continue

        if column[field]['format'] == '{}' and sep != '|':
            if field in item:
                out.append(jdump_item(item[field]))
            else:
                out.append(value)
        else:
            if field == 'id' and sep == '|' and user_out_spec == default_out_spec and 'ocid' in value:
                out.append(' ' + value.split('.')[3])
            out.append(column[field]['format'].format(re.sub(r'[{}\'[\]\n]', '', value)[
                       column[field]['offset']:]))

    # one field per line
    if sep == '/':
        indent = ''
        if quiet < 2:
            indent = '   ' + ' ' * maxkeylen
        for k, value in zip(fields, out):
            if quiet < 2:
                print(column['keyfmt'].format(k) + '  ', end='')
            value = re.sub(r'^ +', '', value)
            # Wrap long lines
            if k == 'access-uri':
                print(value)
            elif '\n' in value:
                value = re.sub(r'\n ', '\n' + indent, value)
                value = re.sub(r'$\n', '', value)
                value = re.sub(r'^  ', '', re.sub('\n  ', '\n', value))
                print(value)
            else:
                print(textwrap.fill(value,
                                    subsequent_indent=indent,
                                    break_on_hyphens=True,
                                    break_long_words=True,
                                    width=wrap - maxkeylen - 3))
        if len(fields) > 1:
            print('')

    # fields on one line, csv
    elif sep == ',':
        print(sep.join([repr(i) for i in out]))

    # fields on one line, space separated
    elif sep in (' ', '\t'):
        print((sep.join([str(item[f]) for f in fields])).rstrip())

    # sep is '|' or '#', table output
    else:
        print(' '.join(out))


# ==============================================================================
key_list = []


def get_key_list(items, prefix=''):
    global key_list
    if not prefix: key_list = []   # noqa: E701

    for i in items if isinstance(items, list) else [items]:
        for j in i.keys():
            key_list.append(prefix + j)
            if isinstance(i[j], dict) or \
               (isinstance(i[j], list) and i[j] and isinstance(i[j][0], dict)):
                get_key_list(i[j], prefix + j + '.')

    return sorted(list(set(key_list)))

# ==============================================================================


def output(jsonOut):
    """Send formatted output from oci command to stdout.

    jsonOut is the json returned by oci.
    Possible JSON 'data' returns:
       get:   {'data': {'compartment-id': 'ocid1.compartment.oc1..aaa...
       list:  {'data': [{'compartment-id': 'ocid1.compartment.oc1..aaid7...
       ns get:  { "data": "mytenancyname" }
       service-connector list:  {'data': { 'items': [ ]}}
    The useful payload usually is in "data", but could be a string, dict,
    list of dicts, etc.

    Format payload according to the "-o spec" and print to stdout.
    Show any additional non-"data" key-values, unless quiet.
    """

    if type(jsonOut) is dict and 'data' in jsonOut:
        # copy the returned 'data' into results list
        if type(jsonOut['data']) is list:
            results = jsonOut['data']
        elif type(jsonOut['data']) is dict:
            try:
                if 'items' in jsonOut['data'] and type(jsonOut['data']['items']) is list:
                    results = jsonOut['data']['items']
                # logging-search search-logs results:
                elif 'results' in jsonOut['data'] and type(jsonOut['data']['results']) is list\
                        and 'logContent' in jsonOut['data']['results'][0]['data']:
                    results = [i['data']['logContent']['data'] for i in jsonOut['data']['results']]
                else:
                    results = [jsonOut['data']]
            except (KeyError, IndexError):
                results = [jsonOut]
        else:
            results = [jsonOut]

    elif type(jsonOut) is list and type(jsonOut[0]) is dict:
        # Other JSON outputs:
        # --query extracts results from 'data' into various structures.
        # -gfcji: { "messages": [ { "key": "string", "value": "string" },
        #    or   {'agentConfig': {'isMonitoringDisabled': True}, 'definedTags...
        # I may be able to format the output, but don't try to save this data.
        results = jsonOut

    else:
        # It's JSON, but I don't understand its format.
        return 1

    # ==========================================================================
    # Above we filled results[] with data.
    # Below we format and display the data.
    # ==========================================================================

    if not user_out_spec:
        print(json.dumps(jsonOut, indent=4))
        return results

    # Get max width of each field - or format with user spec

    if user_out_spec == '/':
        out_fields, out_sep = get_fields_from(user_out_spec,
                                              list(set([k for i in results for k in i.keys()])))
    else:
        out_fields, out_sep = get_fields_from(user_out_spec, get_key_list(results))
    if out_fields:
        if debug: print("USER_SPEC:", user_out_spec)     # noqa: E701
        if debug: print("OUT_FIELDS:", out_fields)       # noqa: E701
        set_column_widths(out_fields, out_sep, results)
        if debug > 1: print("COLUMN:", column)             # noqa: E701
        # Then output header and requested fields from item_list
        show_column_headers(out_fields, out_sep)
        for item in results:
            show_item(item, out_fields, out_sep)

    # Any extra non-'data' key-values?  Send to stdout
    if type(jsonOut) is dict and jsonOut.keys() != ['data']:
        kmax = str(max([len(str(k)) for k in jsonOut]))
        kfmt = '{:>' + kmax + '.' + kmax + '}'
        nondata = '\n'
        for k in jsonOut:
            if k in ('opc-next-page', 'next-start-with'):
                # Save opc-next-page to ocids file
                results.append({"type": "",
                                "alias": "opc-next-page",
                                "name": "opc-next-page",
                                "id": jsonOut[k]})
                s = '--page' if k == 'opc-next-page' else '--start'
                nondata += 'For next page:  ' + s + ' next\n'
            elif not quiet and k != 'data' and jsonOut[k]:
                nondata += kfmt.format(k) + '  '
                if type(jsonOut[k]) is str:
                    nondata += jsonOut[k] + '\n'
                else:
                    nondata += jdump_item(jsonOut[k], None) + '\n'
        if not quiet and nondata > '\n':
            print(bold(nondata), file=sys.stdout)

    return results

# ==============================================================================


def omap(argv):
    ''' omap - map OCI resources collected in .ocids file '''

    # Sloppy.  I know.
    global ocid, ocids, wrap
    global compartments_only, instances, vcns, maxdepth, use_bold, long

    # ==========================================================================

    def omap_show_item_names(prefix, resourcetype, items):
        suffix = '' if len(items) == 1 else 's'
        itemstr = ', '.join([i['alias'] for i in items])
        itemstr = re.sub(r' \(Boot Volume\)', '', itemstr)
        itemstr = re.sub(r': ocid[^ ,]{73}', '...', itemstr)
        print(textwrap.fill(bold(str(len(items)) + ' ' + resourcetype + suffix + ': ') + itemstr,
              initial_indent=' ' * len(prefix),
              subsequent_indent='        ' + ' ' * len(prefix),
              break_on_hyphens=False,
              break_long_words=True,
              width=wrap))

    # ==========================================================================

    def omap_show_items(prefix, resourcetype, items):
        """Show each field in list, align columns based on max widths."""
        for i in items:
            i['alias'] = re.sub(r' \(Boot Volume\)', '', i['alias'])
            if 'score' in i:
                del i['score']
            if 'parents' in i:
                del i['parents']

        # get column widths for every field in every item
        column = {}
        for i in items:
            for k in i.keys():
                if k not in column:
                    if k == 'availability-domain':
                        column[k] = {'head': 'AD', 'maxwidth': 2, 'width': 2, 'format': '{:>2}'}
                    else:
                        maxwidth = max([len(str(f[k])) for f in items if k in f])
                        maxwidth = max(maxwidth, len(k))
                        if k.endswith('id'):
                            width = 8
                        else:
                            width = min(maxwidth, 30)
                        column[k] = {'head': k, 'maxwidth': maxwidth, 'width': width,
                                     'format': '{:' + str(width) + '.' + str(width) + '}'}

        # shorten long column headings
        for k in column.keys():
            column[k]['head'] = re.sub('default-', 'd-', column[k]['head'])
        for k in column.keys():
            if column[k]['head'].startswith('size-in-'):
                column[k]['head'] = column[k]['head'][8:]
                if column[k]['head'] == 'gbs':
                    column[k]['format'] = '{:>3.3}'
                else:
                    column[k]['format'] = '{:>8.8}'
        column['alias']['head'] = 'name'

        suffix = '' if len(items) == 1 else 's'
        print(prefix + bold(str(len(items)) + ' ' + resourcetype + suffix + ': '))
        header = []
        hide_these_columns = ['type', 'name', 'display-name']

        for k in column.keys():
            if k not in hide_these_columns:
                header.append(column[k]['format'].format(column[k]['head']))

        # show each item
        print(prefix + bold(' '.join(header)))
        for i in items:
            print(prefix, end='')
            for k in column.keys():
                if k in i and k not in hide_these_columns:
                    # right-truncate ocids
                    if long == 1 and k.endswith('id'):
                        print(column[k]['format'].format(i[k][-8:]), end=' ')
                    elif k == 'availability-domain':
                        print(column[k]['format'].format(str(i[k])[-1:]), end=' ')
                    else:
                        print(column[k]['format'].format(str(i[k])[:column[k]['maxwidth']]),
                              end=' ')
            print('')
        print('')
        return

    # ==========================================================================

    def omap_show_compartment(this, depth=0):
        """Show the resources in this compartment and children up to maxdepth.
        """

        indent = ''  # ' ' * (depth*4)

        # list everything in this compartment that is not DELETED
        stuff = [i for i in ocids
                 if ('compartment-id' in i and i['compartment-id'] == this['id']
                     and (('lifecycle-state' not in i)
                          or ('lifecycle-state' in i and i['lifecycle-state'] != 'DELETED')))]
        types = sorted(list(set([i['type'] for i in stuff])))

        subcompartments = [i for i in stuff if i['type'] == 'compartment']

        if long and subcompartments:
            print('\n' + indent + bold(this['type'].upper() + ' ' + this['alias']))
        else:
            # print(bold(indent + this['alias']), end='')
            if compartments_only:
                print(indent + this['alias'])
            else:
                print(bold(indent + this['alias']))

        if vcns:
            vcnlist = [i for i in stuff if i['type'] == 'vcn']
            for vcn in vcnlist:
                vcnstuff = [i for i in ocids if 'vcn-id' in i and i['vcn-id'] == vcn['id']]
                print('VCN:', vcn['alias'])
                for i in vcnstuff:
                    if i['type'] not in ['subnet', 'securitylist']:
                        print('    {:25}'.format(i['type'] + ':'), i['alias'])
                for subnet in [i for i in vcnstuff if i['type'] == 'subnet']:
                    print('\n    SUBNET', '{:18}'.format(subnet['alias']), subnet['cidr-block'])
                    # get vnics and vnicattachments
                    vnics = [i for i in ocids if i['type'] == 'vnic'
                             and 'subnet-id' in i and i['subnet-id'] == subnet['id']]
                    for vnic in vnics:
                        print('      vnic',
                              '{:18}'.format(vnic['alias']),
                              '{:18}'.format(vnic['private-ip'] if 'private-ip' in vnic else ''),
                              ('{:16}'.format(vnic['public-ip'] if 'public-ip' in vnic else '')),
                              end='')
                        vnicattachments = [i for i in stuff
                               if (i['type'] == 'vnicattachment'  # noqa: E128
                                   and i['vnic-id'] == vnic['id']
                                   and i['lifecycle-state'] == 'ATTACHED')]

                        # The vnicattachment name often is the same as the
                        # instance name.  But if either has been renamed,
                        # it's good to be able to connect the two.
                        for va in vnicattachments:
                            print('  inst:', ocid[va['instance-id']]['alias'], end='')
                        print('')
                    lbs = [i for i in ocids if i['type'] == 'loadbalancer'
                           and 'subnet-ids' in i and subnet['id'] in i['subnet-ids']]
                    for lb in lbs:
                        print('      ' + '{:21}'.format(lb['type']), '{:28}'.format(lb['alias']),
                              ', '.join([ip['ip-address'] for ip in lb['ip-addresses']]))
                    try:
                        print('      securitylists:',
                              ', '.join([ocid[sl]['alias'] for sl in subnet['security-list-ids']]))
                    except BaseException:
                        pass
                print('')
        else:
            for type in types:
                if ((not compartments_only and not instances)
                        or (compartments_only and type == 'compartment')
                        or (instances and type == 'instance')):
                    tstuff = [i for i in stuff if i['type'] == type]
                    if long:
                        omap_show_items(indent, type, tstuff)
                    elif not compartments_only:
                        omap_show_item_names(indent, type, tstuff)

        subcompartments = [i for i in stuff if i['type'] == 'compartment']
        if not compartments_only and stuff:
            print('')
        for c in subcompartments:
            c['alias'] = this['alias'] + '/' + c['alias']
            if (depth < maxdepth):
                omap_show_compartment(c, depth + 1)

    # ==========================================================================

    def omap_usage(s=''):
        print("""
    omap - Map the tenancy details in your .ocids (experimental)

    usage: o map [-c] [-d depth] [-ilvn] [-w <n>] [ <compartment> | help ]

               -c            compartments list
               -i            Instance centric view
               -v            VCN centric view
               -l            long listing - show more resource details
               -n            no highlighting (bold off)
               -d <depth>    show children <depth> levels down
               -w <n>        wrap long lines at <n> characters
               <compartment> starting point for map

    """)
        if s != '':
            print(bold(s) + '\n')
        exit(1)

    # ==========================================================================
    # start of omap()
    # ==========================================================================

    try:
        optvals, otherparams = getopt.getopt(argv, 'cd:ilnvw:')
    except getopt.error as err:
        omap_usage(str(err))

    opts = [o for o, v in optvals]
    if len(otherparams) == 0 or 'help' in otherparams:
        omap_usage('')
    if len(otherparams) > 1:
        omap_usage('too many arguments: ' + ' '.join(otherparams))

    compartments_only = '-c' in opts
    instances = '-i' in opts
    vcns = '-v' in opts
    maxdepth = 6
    if '-d' in opts:
        maxdepth = [int(v) for o, v in optvals if o == '-d'][0]
    use_bold = '-n' not in opts
    long = len([o for o in opts if '-l' == o])
    if '-w' in opts:
        wrap = int([int(v) for o, v in optvals if o == '-w'][0])
    start_param = otherparams[0]

    ocids_file('read')

    # Find starting point(s) - could be one or more matching compartments.
    map_start = value_parameter('--compartment-id', start_param)
    omap_show_compartment(ocid[map_start])
    exit(0)

# ==============================================================================


def prune(argv):
    ''' o prune - remove a branch from the .ocids stored values'''

    global ocid, ocids, go

    # ==========================================================================

    def prune_this(item):
        n = 0
        print("pruning",
              re.sub(r'\.[a-z0-9]*$', '...', item['id']) + item['id'][-8:],
              item['lifecycle-state'] if 'lifecycle-state' in item else '',
              '/'.join(item['parents']) if 'parents' in item else '')
        stuff = []
        if item['type'] == 'vcn':
            stuff = [i for i in ocids if ('vcn-id' in i and i['vcn-id'] == item['id'])]
        elif item['type'] == 'compartment':
            stuff = [i for i in ocids if ('compartment-id' in i
                                          and i['compartment-id'] == item['id'])]

        for i in stuff:
            n += prune_this(i)
        if item['id'] in ocid:
            del ocid[item['id']]
        elif item['type'] == 'availabilitydomain' and item['name'] in ocid:
            del ocid[item['name']]
        return n + 1

    # ==========================================================================
    # start of prune()
    # ==========================================================================

    if len(argv) == 0:
        usage('help')
        exit(0)
    if argv[-1] in ('go', '!', '.'):
        go = True
        argv.pop(-1)

    ocids_file('read')
    num_ocids = len(ocids)

    pruned = 0
    for arg in argv:
        # if arg matches one ocid, remove that entry
        # if multiple matches, look for vcn first, then compartment

        if arg in ['DELETED', 'TERMINATED', 'INACTIVE', 'DETACHED']:
            m = [item for k, item in ocid.items()
                 if 'lifecycle-state' in item and item['lifecycle-state'] == arg]
            for i in m:
                pruned += prune_this(i)
        else:
            m = sorted([item for k, item in ocid.items() if alias_match(
                arg, item)], key=lambda i: i['score'], reverse=True)
            if len(m) == 0:
                print(arg + ': no match')
            elif len(m) < 10:
                for i in m:
                    pruned += prune_this(i)
            else:
                vcn = [i for i in m if i['type'] == 'vcn']
                if vcn:
                    pruned += prune_this(vcn[0])
                else:
                    c = [i for i in m if i['type'] == 'compartment']
                    if c:
                        pruned += prune_this(c[0])
                    else:
                        print(arg + ': ambiguous - ', len(m), 'matches')

    item = 'item' if pruned == 1 else 'items'
    if not go:
        print(bold('Use "go" to prune ' + str(pruned) + '  ' + item + ' from ' + ocids_file_name))
    elif pruned > 0:
        ocids_file('write', {})
        print('Pruned', pruned, 'of', num_ocids, 'OCIDs. ', len(ocid), 'OCIDs remain in ',
              ocids_file_name)
    exit(0)

# ==============================================================================


def ocid_lookup(argv):
    '''Lookup resource names or ocids'''

    ocids_file('read')

    for arg in argv:
        m = sorted([item for k, item in ocid.items() if alias_match(arg, item)],
                   key=lambda i: i['score'], reverse=True)
        if len(m) == 0:
            print(arg + ': no match', file=sys.stderr)
        else:
            width = max(len(i['id']) for i in m)
            for item in m:
                s = ''
                if sys.argv[1] == 'ocids' and 'parents' in item:
                    s = '/'.join(item['parents'])
                print(('{:' + str(width) + '}').format(item['id']), s)
    exit(0)

# ==============================================================================


def get_oci_commands(argv):
    ''' o oci_commands - extract commands and parameters from oci --help '''

    def oci_command_out(cmd, eol=''):
        f.write(cmd + '\n')
        cmd = re.sub(r' [+-].*', '', cmd)
        print('\033[2K\r       ', cmd, end=eol, flush=True)

    def clean_up_param(s):
        # example in:  -c, --compartment-id TEXT
        #         out: --compartment-id [text]
        # print(s)
        param = re.sub(r' [A-Z[].*', '', s)
        type = s[len(param) + 1:]
        if type and not type.startswith('['):
            type = '[' + type.lower() + ']'
        longest_param = max(param.split(', '), key=len).strip()
        if longest_param != '--help':
            return longest_param + (' ' + type if type else '')

    oci_dir = os.path.expanduser('~/.oci')
    oci_commands_file = os.path.expanduser('~/.oci/oci_commands')

    # o oci_commands - no service name; get commands for all services
    if len(sys.argv) == 2:
        sp = subprocess.run(['oci', '--help'], stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE)
        out = sp.stdout.decode('utf-8')
        part1 = re.sub(r'.*Options:\n|Commands:\n.*', '', out, flags=re.S)
        globals = sorted(list(dict.fromkeys(re.findall(r'(--[A-Za-z-]*[a-z])[, \n]', part1))))
        if not os.path.exists(oci_dir):
            os.mkdir(oci_dir, mode=0o755)
        print("Creating", bold(oci_commands_file))
        f = open(oci_commands_file, 'w')
        f.write('global_options ' + ' '.join(globals) + '\n')
        f.close()

        part2 = re.sub(r'.*Commands:\n', '', out, flags=re.S)
        services = sorted(list(dict.fromkeys(re.findall(r'^ {2,4}([a-z-]+)', part2, flags=re.MULTILINE))))
        for n, service in enumerate(services, start=1):
            print('\033[2K\r({}/{}) Getting {} commands...'.format(
                  n, len(services), service) + 30 * ' ')
            subprocess.run([sys.argv[0], 'oci_commands', service])
        print('\033[2K\rDone.')
        exit(0)

    # o oci_commands <service>
    sp = subprocess.run(['oci'] + sys.argv[2:] + ['--help'],
                        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out = sp.stdout.decode('utf-8')

    if out.startswith('Usage'):
        # Click help instead of man page...
        # Recurse through Commands to get usage for each command individually.
        # Warn user that this is the slow method
        if sys.argv[2] == 'raw-request':
            print(bold("""
    The oci-cli on this system does not include preformatted help.
    This means that it'll take up to an hour to collect usage for all
    commands and save to {}
""".format(os.path.expanduser('~/.oci/oci_commands'))))

        if re.search(r'\nCommands:\n', out, flags=re.M):
            commands = re.sub(r'.*\nCommands:\n', '', out, flags=re.S)
            for line in commands.splitlines():
                if re.match(r'^  [a-z0-9-]+', line):
                    cmd = line.strip().split(' ')[0]
                    subprocess.run([sys.argv[0], 'oci_commands'] + sys.argv[2:] + [cmd])
            print('\033[2K\r', end='')
            exit(0)

        # Parse usage for individual command from click help
        m = re.search(r'Usage: (.*?) \[.*]', out, re.S)
        if m:
            cmd = re.sub(r'[ \n]+', ' ', m.group(1)).strip()
        param = ''
        required = []
        optional = []
        for ln in re.sub(r'.*\nOptions:\n', '', out, flags=re.S).splitlines():
            m = re.match(r'  (-.*?)(  |$)', ln)
            if m:
                if param:
                    optional.append(param)
                param = clean_up_param(m.group(1))
            if param and '[required]' in ln:
                required.append(param)
                param = ''
        if param:
            optional.append(param)

        if required:
            cmd += ' ' + ' '.join(sorted(required))
        if optional:
            cmd += ' + ' + ' '.join(sorted(optional))
        f = open(oci_commands_file, 'a')
        oci_command_out(cmd)
        f.close()
        exit(0)

    # Parse usage for multiple commands from man page help
    # Remove bold
    lines = re.sub(r'(\x08.|\x1B...)', '', out).splitlines()

    f = open(oci_commands_file, 'a')
    cmd = ''
    for n, line in enumerate(lines):
        # Look for start of new command
        # Single command man page has: ^USAGE
        # Multi-command man page has:  ^   Usage
        if re.match('^(USAGE|   Usage)', line):
            # if previous command in buffer, save it
            if cmd:
                oci_command_out(cmd)
            cmd = re.sub(' .OPTIONS.', '', lines[n + 1]).strip()
            if not cmd:
                cmd = re.sub(' .OPTIONS.', '', lines[n + 2]).strip()

        # look for parameter specification
        elif re.search(r'^ {6,7}-[a-z0-9\-,]+($|.* \[.+])', line):
            cmd = cmd + ' ' + clean_up_param(line.strip())

        elif re.search(r'optional parameters', line, re.I):
            cmd = cmd + ' +'

        # elif re.search('Accepted values are:', line):
        elif line.endswith('Accepted values are:'):
            s = re.sub(r', ', r'|', lines[n + 2].strip())
            cmd = cmd[:-5] + s + ']'

    oci_command_out(cmd, eol='\033[2K\r')
    f.close()
    exit(0)


# ==============================================================================
# MAIN PROGRAM STARTS HERE - choose map, prune, ocids, oci_commands or "o" main
# ==============================================================================
signal.signal(signal.SIGINT, interrupted)
if (sys.platform != "win32"):
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)

if len(sys.argv) > 1 and sys.argv[1] == 'map':
    omap(sys.argv[2:])

elif len(sys.argv) > 1 and sys.argv[1] == 'prune':
    prune(sys.argv[2:])

elif len(sys.argv) > 1 and sys.argv[1] == 'oci_commands':
    get_oci_commands(sys.argv[1:])

elif len(sys.argv) > 1 and sys.argv[1] in ('ocid', 'ocids'):
    ocid_lookup(sys.argv[2:])

# The above functions do not return, but exit on completion.
# If we got this far we're in o main.

command = read_oci_commands_file()
ocids_file('read')

# ==============================================================================
# Process 'o' options: -o 'output keys' ...  help go
# ==============================================================================
try:
    optvals, CLI_params = getopt.getopt(sys.argv[1:], 'Di:Oo:qw:')
except getopt.error as err:
    usage(str(err))

opts = [o for o, v in optvals]
quiet = sum(1 for o in opts if o == '-q')
debug = sum(1 for o in opts if o == '-D')
if '-w' in opts:
    wrap = int([int(v) for o, v in optvals if o == '-w'][0])
if '-O' in opts:
    ocids_in_output = True
if '-i' in opts:
    user_in_spec = [v for o, v in optvals if o == '-i'][0]
if '-o' in opts:
    user_out_spec = [v for o, v in optvals if o == '-o'][0]
    if user_out_spec.lower() in ['json', 'j']:
        user_out_spec = ''
    elif user_out_spec.lower() in ['text', 'raw']:
        user_out_spec = '/'
    elif user_out_spec[0] in '+/,#\t ':
        # if user spec starts with + or separator, append spec to defaults
        # if user spec is bare separator, show all fields
        # if spec contains a separator, use separator and user spec
        sep = '#'
        spec = user_out_spec
        if spec[0] == '+':
            spec = user_out_spec[1:]
        if spec[0] in '/,#\t ':
            sep = spec[0]
            spec = spec[1:]
        elif '/' in spec:
            sep = '/'
        elif ',' in spec:
            sep = ','
        if user_out_spec[0] != '+' and spec == '':
            user_out_spec = sep
        else:
            user_out_spec = re.sub(r'#', sep, default_out_spec) + sep + spec
else:
    user_out_spec = default_out_spec

# if -o specified but no command and no -i, get input from .oci/.otmp

if len(CLI_params) == 0 and '-o' in opts and 'user_in_spec' not in locals():
    user_in_spec = os.path.expanduser('~/.oci/.otmp')

if 'user_in_spec' in locals():
    try:
        if user_in_spec == '-':
            jsonIn = json.loads(sys.stdin.read())
        else:
            jsonIn = json.loads(open(user_in_spec).read())
    except:    # noqa: E722
        print(bold(user_in_spec) + ": trouble with input", file=sys.stderr)
        exit(1)

    output(jsonIn)
    exit(0)

if len(CLI_params) == 0:
    usage()

if CLI_params[-1] in ('go', '!', '.'):
    go = True
    if CLI_params[-1] == '!':
        go = 2
    CLI_params.pop(-1)
elif CLI_params[-1].endswith('!') or CLI_params[-1].endswith('.'):
    go = True
    if CLI_params[-1].endswith('!'):
        go = 2
    CLI_params[-1] = CLI_params[-1][:-1]
elif 'help' == CLI_params[-1]:
    if len(CLI_params) == 1:
        usage('help')
    show_help = True
    CLI_params.pop(-1)

# ==============================================================================
# Identify matching oci cli command(s)
# ==============================================================================
command_words = [i for i, w in enumerate(CLI_params) if w.startswith('-')]
ncw = command_words[0] if command_words else len(CLI_params)
options_in_args = len(command_words) > 0

# Prioritize core services above others.
priority_services = ('compute ', 'bv ', 'os ', 'network ', 'iam ', 'db ')

# Find matching commands.  Sort by priority, then by command length.
matches = sorted([c for c in command if args_match_command(c, CLI_params[:ncw])],
                 key=lambda c: (len(c['action']) if len(c['action']) < 25 else int(round(len(c['action']) / 10.) * 10.))
                 if c['action'].startswith(priority_services)
                 else (1.5 * len(c['action']) if len(c['action']) < 25 else 1.5 * int(round(len(c['action']) / 10.) * 10.)),
                 reverse=True)

# ==============================================================================
# Take action - either show help or 'go' run the command
# ==============================================================================
if len(matches) == 0:
    usage('no matching command found: ' + ' '.join(CLI_params)
          + '\n\nTry:  o .')

# If many matches, list matches and exit
elif not (len(matches) <= 16
          or (matches[-1]['action'].split(' ')[0] + ' ' in priority_services and len(matches) < 200 and ncw > 1)):
    print(bold('Possible commands:'))
    for c in matches:
        show_command(c, prefix='    ')
    print('use:', bold('o <command>'), 'for options')
    exit(0)

# Fewer than 16 matches - if options provided, try the shortest command
elif show_help:
    for c in matches:
        show_command(c, full=True)
    print('For oci help:', bold('o ' + c['action'] + ' --help .\n')
          + 'https://docs.oracle.com/en-us/iaas/tools/oci-cli/latest/oci_cli_docs/cmdref/'
          + c['action'].replace(' ', '/') + '.html')
    exit(0)

# Fewer than 16 matches. Didn't ask for help.
elif not go and quiet < 1:
    if len(matches) > 1:
        if options_in_args:
            print(bold(str(len(matches) - 1) + ' other matching commands not shown.'
                  + ' (Remove parameters to see other commands.)'), end='')
        else:
            print(bold('Possible matches:'))
            for i in range(len(matches)):
                if i != len(matches) - 1:
                    show_command(matches[i], prefix='    ')
        print(bold('\nBest match:'))
    show_command(matches[-1], full=True)

# ==============================================================================
# We've settled on matches[-1] as the best match.
# Continue interpreting additional parameters (even if no 'go')
# ==============================================================================
c = matches[-1]
oci_command_line = 'oci ' + c['action'] + ' '

if c['action'] in ('audit event list', 'logging-search search-logs') and \
        user_out_spec == default_out_spec:
    user_out_spec = ''

all_options_for_this_cmd = c['required'] + ' ' + c['optional'] + ' ' + global_options + ' '
remaining_options = all_options_for_this_cmd
while CLI_params and CLI_params[0][0] != '-':
    CLI_params.pop(0)

# Determine the target region for this command up front, so that we can filter
# ocids by region.  See if region or profile is on command line...
for i in reversed(range(len(CLI_params))):
    a = CLI_params[i]
    if len(a) == 0 or not a.startswith('-'):
        continue
    if option_parameter(a) in ('--region', '--profile'):
        options_out.append(option_parameter(a))
        if option_parameter(a) == '--region':
            region_code, region = get_region(CLI_params[i + 1])
            options_out.append(region)
        else:
            options_out.append(CLI_params[i + 1])
        del CLI_params[i:i + 2]

# Interpret remaining command line arguments based on command:
last_option = ''
for a in CLI_params:
    if len(a) == 0:
        continue
    if a.startswith('-'):
        options_out.append(option_parameter(a))
        last_option = options_out[-1]
        # don't match the same parameter more than once
        remaining_options = re.sub(last_option + ' '
                                   + r'(\[[A-Za-z0-9-_|\]]* )',
                                   '', remaining_options)
    else:
        options_out.append(value_parameter(last_option, a))

# if orphans found while processing parameters, remove them from ocids file
if orphans:
    ocids_file('write')

for o in options_out:
    if o.startswith('-') and o != '-':
        if (sys.platform != "win32"):
            oci_command_line += '\\\n   '
    oci_command_line += shell_escape(o) + ' '

# ==============================================================================
# Display the full oci cli command with expanded options
# ==============================================================================
if not go and len(options_out) == 0:
    exit(0)

if not quiet:
    print('\n' + oci_command_line + '\n', file=sys.stderr)

if nogo and go != 2:
    exit(1)
if not go:
    exit(0)

# ==============================================================================
# Execute the oci cli command (finally!)
# ==============================================================================

oci_command_list = ['oci'] + c['action'].split(' ') + options_out

try:
    # oci ... --file -  needs read(), not readline()
    if '--file' in options_out and '-' in options_out:
        cli = subprocess.run(oci_command_list, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        if cli.stdout != b'':
            sys.stdout.buffer.write(cli.stdout)
        if cli.stderr is not None and cli.stderr:
            error_out(cli.stderr.decode('utf-8'))
        exit(cli.returncode)

    cli = subprocess.Popen(oci_command_list, stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE, universal_newlines=True)
except subprocess.SubprocessError as err:
    print('ERROR:', err)
    exit(1)

if (sys.platform != "win32"):
    flags = fcntl.fcntl(cli.stdout, fcntl.F_GETFL)
    fcntl.fcntl(cli.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK)
    flags = fcntl.fcntl(cli.stderr, fcntl.F_GETFL)
    fcntl.fcntl(cli.stderr, fcntl.F_SETFL, flags | os.O_NONBLOCK)

# Read output from cli - show non-JSON on stdout
JSONstdout = ''
while True:
    if (sys.platform != "win32"):
        stderr = cli.stderr.readline()
        returncode = cli.poll()
        if stderr:
            print(bold(stderr), end='', flush=True)

    stdout = cli.stdout.readline()
    returncode = cli.poll()
    if stdout == '' and returncode is not None:
        break
    if stdout:
        if JSONstdout or stdout[0] in '{[':
            JSONstdout += stdout
        else:
            print(stdout, end='', flush=True)
    else:
        time.sleep(0.05)

stderr = cli.stderr.read()

# Interpret output as JSON
try:
    jsonOut = json.loads(JSONstdout)
except ValueError:
    # json.loads failed, show us what you got
    if JSONstdout != '':
        print(JSONstdout)
    if stderr is not None:
        error_out(stderr)
    exit(1)

tmpfile = os.path.expanduser('~/.oci/.otmp')
if os.path.isfile(tmpfile):
    try:
        os.umask(0o077)
        with open(tmpfile, 'w+') as f:
            f.write(JSONstdout)
    except IOError:
        pass

results = output(jsonOut)
if results == 1:
    print(JSONstdout)

# report stderr from oci cli, if any, AFTER results
if stderr != '':
    error_out(stderr)

if type(jsonOut) is dict and 'data' in jsonOut:
    newids = get_new_ocids_from_returned_items(results)
    if not newids.items() < ocid.items():
        ocids_file('write', newids)

exit(cli.returncode)
# ==============================================================================
# End of o main program
# ==============================================================================