shipit

#!/usr/bin/env python3

# Script to manage deployment/release of kubernetes services

# See description in services/README.md

import os
import sys

from collections import namedtuple

from typing import List, Callable, Optional, TypedDict, Union, Dict, Any, Set, NoReturn
from typing_extensions import NotRequired

if os.getenv("IN_DEV_CONTAINER") != "true":
  print("Must be run in dev container")
  sys.exit(1)

import argparse
import glob
import jsonschema
import os.path
import re
import subprocess
import yaml
import hashlib
import json
import rich

##################################
# Utility functions
##################################


def bad(str: str) -> NoReturn:
  print(str)
  sys.exit(1)


def do_nothing(str: str) -> None:
  pass


should_debug = False
should_dry_run = False
dry_run_target = None

# When running diffs, keep going through the errors but make sure to report something
error = None


def handle_error(msg: str) -> None:
  global error
  print(msg)
  if error is None:
    error = msg


def debug(str: str, str2: Any = None, str3: Any = None) -> None:
  if should_debug:
    if str2 != None:
      str = f"{str} {str2}"
    if str3 != None:
      str = f"{str} {str3}"
    rich.print(f"[bold red]DEBUG: {str}[/bold red]")


def readfile(filename: str) -> str:
  with open(filename, "r") as f:
    return f.read()


def writefile(filename: str, contents: str) -> None:
  with open(filename, "w") as f:
    f.write(contents)


def clean_filename(filename: str) -> str:
  abs = os.path.abspath(filename)
  rel = os.path.relpath(abs, os.getcwd())
  return rel


# Run an important command, keeping stdout/stderr going to the user
def run(name: str,
        args: Union[str, List[str]],
        on_error: Callable[[str], None] = bad,
        shell: bool = False,
        input: Optional[bytes] = None) -> None:
  try:
    if should_dry_run:
      color = "bold green"
    else:
      color = "bold magenta"
    rich.print(f"[{color}]Running `{name}` command[/{color}]")
    subprocess.run(args, shell=shell, input=input, check=True)
    print()
  except subprocess.CalledProcessError as e:
    msg = f"Error running {name} command:\n  {e}\n"
    on_error(msg)
    handle_error(msg)


# Runs a command that's part of running the script, captures stdout
def run_output(args: Union[str, List[str]],
               shell: bool = False,
               on_error: Callable[[str], None] = bad,
               input: Optional[str] = None,
               env: Any = None) -> str:
  try:
    if env != None:
      env = {**os.environ, **env}  # merge two dicts in python 3.8
    if should_dry_run:
      color = "bold green"
    else:
      color = "bold magenta"
    rich.print(f"[{color}]Running `{args}` command[/{color}]")

    return subprocess.check_output(args, shell=shell, env=env,
                                   input=input).decode("utf-8").strip()
  except subprocess.CalledProcessError as e:
    msg = f"Error running command:\n  {e}\n"
    on_error(msg)
    handle_error(msg)
    return ""


##################################
# definitions
##################################
REGION = "us-west1"
PROJECT = "balmy-ground-195100"
CLUSTER = readfile("current-cluster")
CLOUDSQL_INSTANCE_NAME = "balmy-ground-195100:us-west1:dark-west"
DEPLOY_LOCK_BUCKET = "gs://darklang-deploy-lock"

##################################
# Assertions
##################################


def assert_file_exists(dir: str, f: str) -> None:
  if not os.path.exists(os.path.join(dir, f)):
    bad(f"File {dir}/{f} does not exist")


def assert_dir_exists(service: str, d: str) -> None:
  if not os.path.exists(d):
    bad(f"Directory {d} does not exist (for service {service})")


def assert_string_in_file(filename: str, file_str: str, substr: str) -> None:
  if substr not in file_str:
    bad(f"String missing from {filename}: {substr}")


##################################
# Typing for shipit.yaml files
##################################

ShipitFileConfigMapSection = TypedDict('ShipitFileConfigMapSection', {
    "text-file": NotRequired[str],
    "env-file": NotRequired[str],
})

ShipitFileManuallyDeployedSection = TypedDict(
    'ShipitFileManuallyDeployedSection', {
        "configs": List[str],
        "configmaps": NotRequired[Dict[str, ShipitFileConfigMapSection]],
        "custom-diff": NotRequired[List[str]],
        "custom-apply": NotRequired[List[str]],
        "custom-post-apply": NotRequired[List[str]]
    })

ShipitFileReleaseSection = TypedDict(
    'ShipitFileReleaseSection', {
        "config-template": str,
        "versioned-configmaps": NotRequired[Dict[str, ShipitFileConfigMapSection]],
        "containers": List[str],
        "builtins": NotRequired[List[str]],
        "expected-args": NotRequired[List[str]]
    })

ShipitFileK8sSection = TypedDict(
    'ShipitFileK8sSection', {
        "namespace": str,
        "manually-deployed": ShipitFileManuallyDeployedSection,
        "release": NotRequired[ShipitFileReleaseSection]
    })


class ShipitFile(TypedDict):
  k8s: ShipitFileK8sSection


##################################
# Argument Types. For now we're trying to codify what we
# have. We may try to improve on this later
##################################


class ServicesArguments:
  services: List[str]


class ServiceArguments:
  service: str


class OutputManifestArguments(ServicesArguments):
  save_manifest: str


class ReleaseArgsArguments(ServicesArguments):
  manifest: str
  arg: Dict[str, str]


class ValidationArguments(ServicesArguments):
  pass


class ManualApplyArguments(ServiceArguments):
  pass


class ManualDiffArguments(ServicesArguments):
  pass


class ContainerBuildArguments(OutputManifestArguments):
  pass


class ContainerPullArguments(OutputManifestArguments):
  pass


class ContainerPushArguments(ServicesArguments):
  pass


class ContainerListArguments(ServicesArguments):
  pass


class ReleaseCurrentManifestArguments(OutputManifestArguments):
  pass


class ReleasePrepareArguments(ReleaseArgsArguments):
  pass


class ReleasePushArguments(ReleasePrepareArguments, ServicesArguments):
  pass


class ReleaseDiffArguments(ReleasePrepareArguments, ServicesArguments):
  pass


##################################
# Load services / config
##################################
def get_service_dirs(dirs: List[str]) -> List[str]:
  if dirs == []:
    dirs = [path for path in glob.glob(r'services/**') if "README.md" not in path]
  elif len(dirs) == 1 and dirs[0].endswith("shipit.yaml"):
    dirs = [os.path.dirname(dirs[0])]
  # Cleanup input
  dirs = [dir.rstrip("/").strip() for dir in dirs]
  dirs = sorted(dirs)

  return dirs


def get_service_config(dir: str) -> ShipitFile:
  try:
    file = open(dir + "/shipit.yaml")
  except:
    bad(f"Cannot find shipit.yaml in {dir}/")
  try:
    config: ShipitFile = yaml.load(file, Loader=yaml.Loader)
    validate_config(dir, config)
    return config
  except Exception as e:
    bad(f"Bad yaml in {dir}/shipit.yaml\n  {e}")


##################################
# Configmaps are present in both manually-deployed and automatically released config
##################################


def get_hashed_configmap_name(dir: str, name: str,
                              desc: ShipitFileConfigMapSection) -> str:
  cm_filename = desc.get("text-file") or desc["env-file"]
  cm_contents = readfile(os.path.join(dir, cm_filename))
  hash = hashlib.sha256(cm_contents.encode("utf-8")).hexdigest()[0:12]
  return f"{name}-{hash}"


ConfigMapDef = TypedDict("ConfigMapDef", {
    "hashed_name": str,
    "filespec": str,
    "namespace": str,
})


def collect_versioned_configmaps(
    dir: str, namespace: str,
    parent: ShipitFileReleaseSection) -> Dict[str, ConfigMapDef]:
  cms = {}
  for name, cm in parent.get("versioned-configmaps", {}).items():
    hashed_name = get_hashed_configmap_name(dir, name, cm)
    filespec = read_config_map_source(dir, cm)
    key = f"{namespace}/{hashed_name}"
    cmd: ConfigMapDef = {
        "hashed_name": hashed_name,
        "filespec": filespec,
        "namespace": namespace
    }
    cms[key] = cmd
  return cms


##################################
# Config commands
##################################


def get_all_config_file_arguments(dirs: List[str]) -> List[str]:
  files = []
  for dir in get_service_dirs(dirs):
    config = get_service_config(dir)
    for f in config['k8s']["manually-deployed"]["configs"]:
      files.append(os.path.join(dir, f))

  file_args = []
  for f in files:
    file_args.append("-f")
    file_args.append(f)
  return file_args


def read_config_map_source(dir: str, cm: ShipitFileConfigMapSection) -> str:
  text_file = cm.get("text-file")
  if text_file:
    filename = clean_filename(os.path.join(dir, text_file))
    return f"--from-file={filename}"
  else:
    env_file = cm["env-file"]
    filename = clean_filename(os.path.join(dir, env_file))
    return f"--from-env-file={filename}"


def manual_diff(cliargs: ManualDiffArguments) -> None:
  do_validation(cliargs.services)

  # Config files
  file_args = get_all_config_file_arguments(cliargs.services)
  if file_args == []:
    debug("No configs to diff, skipping")
  else:
    run("kubectl diff", ["kubectl", "diff"] + file_args, on_error=do_nothing)

  # Configmaps
  for dir in get_service_dirs(cliargs.services):
    config = get_service_config(dir)
    cms = config['k8s']["manually-deployed"].get("configmaps", {})
    namespace = config['k8s']['namespace']
    for name, cm in cms.items():
      filespec = read_config_map_source(dir, cm)
      run(f"kubectl diff configmap (in {namespace})",
          f"kubectl create configmap {name} {filespec} --namespace={namespace} --dry-run=client -o yaml | kubectl diff --filename=-",
          shell=True,
          on_error=do_nothing)

  # Custom diff
  for dir in get_service_dirs(cliargs.services):
    config = get_service_config(dir)
    manual = config['k8s'].get("manually-deployed")
    if manual:
      for c in manual.get("custom-diff", []):
        run("custom diff", c, shell=True, on_error=do_nothing)


def manual_apply(cliargs: ManualApplyArguments) -> None:
  dirs = [cliargs.service]
  do_validation(dirs)

  # Config files
  file_args = get_all_config_file_arguments(dirs)
  if file_args == []:
    debug("No configs to apply, skipping")
  else:
    if should_dry_run:
      run("kubectl apply",
          ["kubectl", "apply", f"--dry-run={dry_run_target}"] + file_args)
    else:
      run("kubectl apply", ["kubectl", "apply"] + file_args)

  # Configmaps
  for dir in get_service_dirs(dirs):
    config = get_service_config(dir)
    cms = config['k8s'].get("manually-deployed", {}).get("configmaps", {})
    namespace = config['k8s']['namespace']
    for name, cm in cms.items():
      filespec = read_config_map_source(dir, cm)
      if should_dry_run:
        run(f"kubectl apply configmap (in {namespace}) (dry-run)",
            f"kubectl create configmap {name} {filespec} --namespace={namespace} --dry-run=client -o yaml | kubectl apply --filename=- --dry-run={dry_run_target}",
            shell=True)
      else:
        run(f"kubectl apply configmap (in {namespace})",
            f"kubectl create configmap {name} {filespec} --namespace={namespace} --dry-run=client -o yaml | kubectl apply --filename=-",
            shell=True)

  # Custom apply
  for dir in get_service_dirs(dirs):
    config = get_service_config(dir)
    manual = config['k8s'].get("manually-deployed", {})
    for c in manual.get("custom-apply", []):
      if should_dry_run:
        print(f"skipping custom apply step: {c}")
      else:
        run("custom apply", c, shell=True)

  # Custom post-apply (used for restarting services)
  for dir in get_service_dirs(dirs):
    config = get_service_config(dir)
    manual = config['k8s'].get("manually-deployed", {})
    for c in manual.get("custom-post-apply", []):
      if should_dry_run:
        print(f"skipping post apply step: {c}")
      else:
        run("custom post apply", c, shell=True)


##################################
# Manifests
##################################
def manifest_validate(cliargs: ServicesArguments, shas: Dict[str, str]) -> None:
  containers = get_service_containers(cliargs)
  keys = set(shas.keys())
  for c in containers:
    if c not in keys:
      bad(f"Expected a sha for all containers, none found for {c}")
  for (c, sha) in shas.items():
    if sha is None:
      bad(f"Expected a sha for all containers, none found for {c}")
    if sha == "":
      bad(f"Empty sha found for {c}")
    if re.match("[a-f0-9]{9}", sha) is None:
      bad(f"Sha does not match expected format: {c}")


def manifest_save(cliargs: OutputManifestArguments, shas: Dict[str, str]) -> None:
  manifest_validate(cliargs, shas)
  with open(cliargs.save_manifest, "w") as f:
    json.dump(shas, f, indent=2, sort_keys=True)
  print(f"Saved manifest in {cliargs.save_manifest}")


def manifest_load(cliargs: ReleaseArgsArguments) -> Dict[str, str]:
  with open(cliargs.manifest, "r") as f:
    shas: Dict[str, str] = json.load(f)
  manifest_validate(cliargs, shas)
  return shas


##################################
# Container commands
##################################


def get_service_containers(cliargs: ServicesArguments) -> Set[str]:
  needed_containers = set()
  for dir in get_service_dirs(cliargs.services):
    config = get_service_config(dir)
    release = config['k8s'].get("release")
    if release:
      for c in release["containers"]:
        needed_containers.add(c)
  return needed_containers


def containers_list(cliargs: ContainerListArguments) -> None:
  do_validation(cliargs.services)
  status = run_output(["docker", "image", "ls"]).split("\n")
  service_containers = get_service_containers(cliargs)
  for line in status:
    for c in service_containers:
      if c in line:
        print(line)
        break


def containers_pull(cliargs: ContainerPullArguments) -> None:
  print("Pulling, if error you might need `gcloud auth configure-docker`")
  shas = {}
  do_validation(cliargs.services)
  for c in get_service_containers(cliargs):
    url = f"gcr.io/{PROJECT}/{c}"
    image = f"gcr.io/{PROJECT}/{c}:latest"
    run("docker pull", ["docker", "pull", image])
    # Now get the ID we just pulled
    id = run_output(["docker", "images", url, "-q"]).split("\n")[0]
    shas[c] = id
  manifest_save(cliargs, shas)


def containers_push(cliargs: ContainerPushArguments) -> None:
  print("Pushing, if error you might need `gcloud auth configure-docker`")
  do_validation(cliargs.services)

  ImageInfo = TypedDict('ImageInfo', {"image": str, "image_latest": str})

  def image_info(c: str) -> ImageInfo:
    image_id = run_output([f"docker images -q {c} | head -n 1"], shell=True)
    url = f"gcr.io/{PROJECT}/{c}"
    return {'image': f"{url}:{image_id}", 'image_latest': f"{url}:latest"}

  for c in get_service_containers(cliargs):
    info = image_info(c)
    run("docker tag id", ["docker", "tag", f"{c}:latest", info["image"]])
    run("docker tag latest", ["docker", "tag", f"{c}:latest", info["image_latest"]])

  for c in get_service_containers(cliargs):
    info = image_info(c)
    run("docker push tagged", ["docker", "push", info["image"]])
    run("docker push latest", ["docker", "push", info["image_latest"]])


def containers_build(cliargs: ContainerBuildArguments) -> None:
  do_validation(cliargs.services)

  def build(name: str, dockerfilename: str, commitSha: Optional[str]) -> None:
    print(f"building {name} docker image")
    dockerfile = readfile(dockerfilename)
    if commitSha:
      run("docker build", [
          "docker", "build", "--tag", f"{name}:latest", "--build-arg",
          f"GIT_COMMIT={commitSha}", "-"
      ],
          input=dockerfile.encode('utf-8'))
    else:
      run("docker build", ["docker", "build", "--tag", f"{name}:latest", "-"],
          input=dockerfile.encode('utf-8'))

  commitSha = run_output(["git", "rev-parse", "--short", "HEAD"])
  build("dark-base-service", "containers/base-service-Dockerfile", commitSha)
  build("dark-ocaml-service", "containers/ocaml-service-Dockerfile", None)
  build("dark-fsharp-service", "containers/fsharp-service-Dockerfile", None)

  # Each container name represents a container in containers/. If there's a prep.sh
  # file in the dir, run it first and then build the container in the directory it
  # echos. If there is no dockerfile, then do nothing (sometimes we use vendor
  # containers and so we just need to store config files).
  shas = {}
  for c in get_service_containers(cliargs):
    dir = os.path.join("containers", c)
    if os.path.isdir(dir) and os.path.exists(os.path.join(dir, "Dockerfile")):
      print(f"\nBuild container {c}")
      # CLEANUP: remove prep.sh files and this logic
      prep = os.path.join(dir, "prep.sh")
      if os.path.exists(prep):
        buildDir = run_output(prep)
      else:
        buildDir = os.getcwd()
      args = [
          "docker", "build", "--quiet", "--tag", f"{c}:latest", "--file",
          f"{dir}/Dockerfile", buildDir
      ]
      sha = run_output(args, env={"DOCKER_BUILDKIT": "1"})

      sha = sha[7:19]
      print(sha)
      shas[c] = sha
    else:
      print(f"\nNo dockerfile, skipping {c}")
  manifest_save(cliargs, shas)


##################################
# Releases
##################################
def get_expected_args(cliargs: ReleaseArgsArguments) -> Dict[str, str]:
  expected_args = {}
  for arg in cliargs.arg or []:
    items = arg.strip().split("=")
    k = items[0].strip()
    v = "=".join(items[1:]).strip()
    if (v.startswith("\"") and v.endswith("\"")) or \
       (v.startswith("'") and v.endswith("'")):
      v = v[1:-1]
    expected_args[k] = v
  return expected_args


def collect_release_templated_configs(cliargs: ServicesArguments) -> List[str]:
  files = []
  for dir in get_service_dirs(cliargs.services):
    config = get_service_config(dir)
    release = config['k8s'].get("release")
    if release:
      filename = os.path.join(dir, release["config-template"])
      target_filename = filename.replace(".template", "")
      files.append(target_filename)
  return files


def release_prepare(cliargs: ReleasePrepareArguments) -> None:
  do_validation(cliargs.services)
  builtins = {"CLOUDSQL_INSTANCE_NAME": CLOUDSQL_INSTANCE_NAME}
  expected_args = get_expected_args(cliargs)
  ids = manifest_load(cliargs)
  for dir in get_service_dirs(cliargs.services):
    config = get_service_config(dir)
    release = config['k8s'].get("release")
    if release:
      filename = os.path.join(dir, release["config-template"])

      # Fill in the blanks
      content = readfile(filename)
      for c in release["containers"]:
        v = ids[c]
        if v == None:
          bad(f"No value for `IMAGEID:{c}")
        if v == "":
          bad(f"Empty string for `IMAGEID:{c}")
        content = content.replace(f"{{IMAGEID:{c}}}", v)
      for name, desc in release.get("versioned-configmaps", {}).items():
        hashed_name = get_hashed_configmap_name(dir, name, desc)
        content = content.replace(f"{{VERSIONED-CONFIGMAP:{name}}}", hashed_name)
      for a in release.get("expected-args", []):
        value = expected_args.get(a)
        if value == None:
          bad(f"No value provided for `ARG:{a}. Pass it at the command line using `--arg {a}='SOME VALUE'`"
              )
        if value == "":
          bad(f"Empty string provided for `ARG:{a}. Pass it at the command line using `--arg {a}='SOME VALUE'`"
              )
        content = content.replace(f"{{ARG:{a}}}", expected_args[a])
      for v in release.get("builtins", []):
        b = builtins[v]
        if b == None:
          bad(f"No value for `BUILTIN:{v}. Probably an internal error")
        if b == "":
          bad(f"Empty string for `BUILTIN:{v}. Probably an internal error")
        content = content.replace(f"{{BUILTIN:{v}}}", b)

      # Write the non-template version
      target_filename = filename.replace(".template", "")
      rich.print(
          f"[italic yellow]Writing release file: {target_filename}[/italic yellow]")
      writefile(target_filename, content)


class K8sContainerSection(TypedDict):
  image: str
  name: str


class K8sTemplateSpecSection(TypedDict):
  containers: List[K8sContainerSection]


class K8sTemplateSection(TypedDict):
  spec: K8sTemplateSpecSection


class K8sSpecSection(TypedDict):
  template: K8sTemplateSection


class K8sDeploymentFile(TypedDict):
  spec: K8sSpecSection


# Read the current container shas from production
def release_current_manifest(cliargs: ReleaseCurrentManifestArguments) -> None:
  do_validation(cliargs.services)
  shas: Dict[str, str] = {}
  for dir in get_service_dirs(cliargs.services):
    config = get_service_config(dir)
    containers = set(config['k8s'].get("release", {}).get("containers", []))
    # Only check yamls with a containers section
    if len(containers) > 0:
      # get the deployment
      namespace = config['k8s']['namespace']
      service_name = os.path.basename(dir)
      k8s_config_str = run_output(
          f"kubectl get deployments/{service_name} -n {namespace} -o json",
          shell=True)
      k8s_config: K8sDeploymentFile = json.loads(k8s_config_str)
      for k8sC in k8s_config['spec']['template']['spec']['containers']:
        [name, version] = k8sC['image'].split(":", 1)
        name = name.split("/")[-1]
        # only use containers listed in config, as those are the only ones shipit manages
        if name in containers:
          if shas.get(name) == None:
            shas[name] = version
          else:
            # check we use the same version across all services
            if shas[name] != version:
              bad(f"{name} has multiple versions: {shas[name]} and {version}")
      for configC in containers:
        if shas.get(configC) == None:
          bad(f"{configC} not found in manifest")
  manifest_save(cliargs, shas)


def release_diff(cliargs: ReleaseDiffArguments) -> None:
  do_validation(cliargs.services)
  release_prepare(cliargs)

  # Configmaps
  all_configmaps = {}
  for dir in get_service_dirs(cliargs.services):
    config = get_service_config(dir)
    release = config['k8s'].get("release")
    namespace = config['k8s']['namespace']
    if release:
      maps = collect_versioned_configmaps(dir, namespace, release)
      all_configmaps.update(maps)
  for _, cm in all_configmaps.items():
    run(f"kubectl diff configmap (in {cm['namespace']})",
        f"kubectl create configmap {cm['hashed_name']} {cm['filespec']} --namespace={cm['namespace']} --dry-run=client -o yaml | kubectl diff --filename=-",
        shell=True,
        on_error=do_nothing)

  # diff it against production
  files = collect_release_templated_configs(cliargs)
  if files == []:
    debug("No release configs to diff, skipping")
  else:
    file_args = ["kubectl", "diff"]
    for f in files:
      file_args.append("-f")
      file_args.append(f)
    run("kubectl diff", file_args, on_error=do_nothing)


def release_push(cliargs: ReleasePushArguments) -> None:
  do_validation(cliargs.services)
  release_prepare(cliargs)

  # Do the config maps first, as they're used by the deploys
  all_configmaps = {}
  for dir in get_service_dirs(cliargs.services):
    config = get_service_config(dir)
    release = config['k8s'].get("release")
    namespace = config['k8s']["namespace"]
    if release:
      maps = collect_versioned_configmaps(dir, namespace, release)
      all_configmaps.update(maps)
  for _, cm in all_configmaps.items():
    if should_dry_run:
      run(f"kubectl apply configmap (in {cm['namespace']})",
          f"kubectl create configmap {cm['hashed_name']} {cm['filespec']} --namespace={cm['namespace']} --dry-run=client -o yaml | kubectl apply --filename=- --dry-run={dry_run_target}",
          shell=True)
    else:
      run(f"kubectl apply configmap (in {cm['namespace']})",
          f"kubectl create configmap {cm['hashed_name']} {cm['filespec']} --namespace={cm['namespace']} --dry-run=client -o yaml | kubectl apply --filename=-",
          shell=True)

  files = collect_release_templated_configs(cliargs)

  # diff it against production
  if files == []:
    debug("No release configs to apply, skipping")
  else:
    file_args = []
    for f in files:
      file_args.append("-f")
      file_args.append(f)
    if should_dry_run:
      run("kubectl apply --dry-run",
          ["kubectl", "apply", f"--dry-run={dry_run_target}"] + file_args)
    else:
      run("kubectl apply", ["kubectl", "apply"] + file_args)


##################################
# Validate config files
##################################

config_schema = """
type: object
properties:
  k8s:
    type: object
    properties:
      namespace:
        type: string
      manually-deployed:
        type: object
        properties:
          configs:
            type: array
            items:
              type: string
          configmaps:
            type: object
            additionalProperties:
              type: object
              propertyNames:
                pattern: "^[a-z][-a-z0-9]*$"
              properties:
                text-file:
                  type: string
                env-file:
                  type: string
              additionalProperties: false
          custom-diff:
            type: array
            items:
              type: string
          custom-apply:
            type: array
            items:
              type: string
          custom-post-apply:
            type: array
            items:
              type: string
        additionalProperties: false
        required: [configs]
      release:
        type: object
        properties:
          config-template:
            type: string
          containers:
            type: array
            items:
              type: string
          versioned-configmaps:
            additionalProperties:
              type: object
              propertyNames:
                pattern: "^[a-z][-a-z0-9]*$"
              properties:
                text-file:
                  type: string
                env-file:
                  type: string
              additionalProperties: false
          expected-args:
            type: array
            items:
              type: string
          builtins:
            type: array
            items:
              type: string
        required: [containers, config-template]
        additionalProperties: false
    required: [manually-deployed]
    additionalProperties: false
"""

K8sMetadata = TypedDict('K8sMetadata', {
    "namespace": NotRequired[str],
})

K8sFile = TypedDict('K8sFile', {
    "metadata": NotRequired[K8sMetadata],
})


def validate_config(dir: str, config: ShipitFile) -> None:

  def validate_k8s_namespace(filename: str, expected_namespace: str,
                             doc: Optional[K8sFile]) -> None:
    if doc is None:
      # Empty files are allowed
      return

    # Validate it's got the right namespace as well
    kind = doc.get("kind")
    if kind in [
        "CustomResourceDefinition", "ClusterRoleBinding", "Namespace", "ClusterRole",
        "Role", "RoleBinding", "MutatingWebhookConfiguration",
        "ValidatingWebhookConfiguration"
    ]:
      # No namespaces on these
      return

    metadata = doc.get("metadata")
    if metadata is None:
      bad(f"Missing metadata in {filename}, expecting \"\{ namespace: {expected_namespace}\"\}"
          )

    this_namespace = doc['metadata'].get('namespace')
    if this_namespace is None:
      bad(f"Unspecified namespace in {filename}, expecting \"{expected_namespace}\"")

    if this_namespace != namespace:
      bad(f"Config namespace in {filename} doesn't match shipit.yaml, expecting \"{expected_namespace}\" but got \"{this_namespace}\""
          )

  def validate_k8s_yaml(dir: str, f: str, expected_namespace: str) -> None:
    assert_file_exists(dir, f)
    filename = os.path.join(dir, f)
    # Yaml files can contain multiple documents
    docs = yaml.load_all(open(filename).read(), Loader=yaml.Loader)
    for i, doc in enumerate(docs):
      validate_k8s_namespace(f"{filename}[{i}]", expected_namespace, doc)

  def validate_configmaps(
      dir: str, configmaps: Optional[Dict[str, ShipitFileConfigMapSection]]) -> None:
    if configmaps is None:
      return
    for name, desc in configmaps.items():

      from_file: Optional[str] = desc.get("text-file") or desc.get("env-file")
      if from_file is not None:
        assert_file_exists(dir, from_file)
      else:
        bad(f"Need either `text-file` or `env-file` for {name}")

  k8s = config['k8s']

  namespace = k8s.get("namespace")
  if namespace is None:
    bad(f"missing namespace field in {dir}/shipit.yaml")
  else:
    if not re.match(r"^([-a-z0-9]+)$", namespace):
      bad(f"namespace \"{namespace}\" not in valid")

  manually_deployed = k8s["manually-deployed"]
  validate_configmaps(dir, manually_deployed.get("configmaps"))
  configs = manually_deployed["configs"]
  for f in configs:
    validate_k8s_yaml(dir, f, namespace)

  release = config['k8s'].get("release", None)
  if release:
    validate_configmaps(dir, release.get("versioned-configmaps"))
    assert_file_exists(dir, release['config-template'])
    template_filename = os.path.join(dir, release['config-template'])
    template_contents = open(template_filename).read()

    # Check they have the right namespace in them
    doc = yaml.load(template_contents, Loader=yaml.Loader)
    validate_k8s_namespace(template_filename, namespace, doc)

    # Check the containers are used in the template
    for c in release["containers"]:
      assert_dir_exists(dir, f"containers/{c}")
      assert_string_in_file(template_filename, template_contents, f"{{IMAGEID:{c}}}")

    # Check the vars are used in the template
    for var in release.get("builtins", []):
      assert_string_in_file(template_filename, template_contents,
                            f"{{BUILTIN:{var}}}")
    for var in release.get("expected-args", []):
      assert_string_in_file(template_filename, template_contents, f"{{ARG:{var}}}")
    for name in release.get("versioned-configmaps", {}).keys():
      assert_string_in_file(template_filename, template_contents,
                            f"{{VERSIONED-CONFIGMAP:{name}}}")

    # Check all template vars are defined
    for match in re.findall(r"\{([-A-Z0-9a-z:_]+)}", template_contents,
                            re.MULTILINE):
      builtin_match = re.match(r"^BUILTIN:([A-Z0-9_]+)$", match)
      expectedarg_match = re.match(r"^ARG:([A-Z0-9_]+)$", match)
      imageid_match = re.match(r"^IMAGEID:([-a-z0-9]+)$", match)
      versioned_configmap_match = re.match(r"^VERSIONED-CONFIGMAP:([-a-z0-9]+)$",
                                           match)
      if builtin_match:
        builtin = builtin_match.group(1)
        if builtin not in release["builtins"]:
          bad(f"builtin \"{builtin}\" not in `k8s.release.builtins` in {template_filename}"
              )

      elif expectedarg_match:
        expectedarg = expectedarg_match.group(1)
        if expectedarg not in release["expected-args"]:
          bad(f"expected arg \"{expectedarg}\" not in `k8s.release.expected-args` in {template_filename}"
              )

      elif imageid_match:
        id = imageid_match.group(1)
        if id not in release["containers"]:
          bad(f"imageid \"{id}\" not in `k8s.release.containers` in {template_filename}"
              )

      elif versioned_configmap_match:
        name = versioned_configmap_match.group(1)
        if name not in release.get("versioned-configmaps", {}):
          bad(f"configmap \"{name}\" not in `k8s.release.versioned-configmaps` in {template_filename}"
              )
      else:
        bad(f"Unexpected placeholder \"{{{match}}}\" in\n{template_filename}")


def do_validation(dirs: List[str]) -> None:
  for dir in get_service_dirs(dirs):
    debug(f"Validating config for {dir}")
    # We automatically validate when we load the file
    config: ShipitFile = get_service_config(dir)


def validate(cliargs: ServicesArguments) -> None:
  do_validation(cliargs.services)
  print("All shipit.yaml files successfully validated")


##################################
# Argument parser
##################################


def create_arg_parser() -> argparse.ArgumentParser:
  # The base parser has the commands shared by ALL subcommands
  base_parser = argparse.ArgumentParser(add_help=False)
  base_parser.add_argument('--debug',
                           action='store_true',
                           help="Print debug info about what's running")
  base_parser.set_defaults(debug=False)

  # Arguments for commands that allow dry-runs
  dryrun_parser = argparse.ArgumentParser(add_help=False)
  dryrun_parser.add_argument(
      '--dry-run',
      choices=['server', 'client'],
      action='store',
      help="do a kubernetes dry-run instead of actually doing the action")
  dryrun_parser.set_defaults(dry_run=False)

  # We want to be explicit about listing services for side-effecting commands
  services_parser = argparse.ArgumentParser(add_help=False)
  services_parser.add_argument(
      'services',
      action="store",
      nargs="*",
      help=
      "paths to the service definitions (directories within services/). Leave empty to run on all services"
  )

  output_manifest_parser = argparse.ArgumentParser(add_help=False)
  output_manifest_parser.add_argument('--save-manifest',
                                      action="store",
                                      required=True,
                                      help="path to store the release manifest")

  release_args_parser = argparse.ArgumentParser(add_help=False)
  release_args_parser.add_argument(
      '--manifest',
      action="store",
      required=True,
      help=
      "path to the release manifest, generated using `containers pull` or `containers build`"
  )
  release_args_parser.add_argument(
      '--arg',
      metavar="KEY=VALUE",
      action='append',
      help=
      "key/value pair that defines an `expected-arg` in the shipit.yamls. Provide once per key/value pair"
  )

  main_parser = argparse.ArgumentParser(
      description='Manage deployment of kubernetes services')
  main_subparsers = main_parser.add_subparsers()

  # Validate
  validate_parser = main_subparsers.add_parser(
      'validate',
      description="Validates shipit.yaml files",
      parents=[base_parser, services_parser])
  validate_parser.set_defaults(func=validate)

  # Manually applied things
  manual_parser = main_subparsers.add_parser('manual')
  manual_subparser = manual_parser.add_subparsers()

  manual_diff_parser = manual_subparser.add_parser(
      'diff',
      description=
      "Checks that the config files listed in the k8s.manually-deployed key of shipit.yaml are already properly deployed, using `kubectl diff`",
      parents=[base_parser, dryrun_parser, services_parser])
  manual_diff_parser.set_defaults(func=manual_diff)

  manual_apply_parser = manual_subparser.add_parser(
      'apply',
      description=
      "Apply the manually applied config for the config listed in the k8s.manually-deployed key of shipit.yaml, using `kubectl apply` and `kubectl create`",
      parents=[base_parser, dryrun_parser])
  manual_apply_parser.add_argument(
      'service',
      action="store",
      help=
      "path to the service definition. Required and only one service is supported")
  manual_apply_parser.set_defaults(func=manual_apply)

  # Containers
  containers_parser = main_subparsers.add_parser('containers')
  containers_subparser = containers_parser.add_subparsers()

  containers_build_parser = containers_subparser.add_parser(
      'build',
      description="Builds the container images needed by services; saves a manifest",
      parents=[base_parser, services_parser, output_manifest_parser])
  containers_build_parser.set_defaults(func=containers_build)

  containers_pull_parser = containers_subparser.add_parser(
      'pull',
      description="Pull the remote docker images used by services, saves a manifest",
      parents=[base_parser, services_parser, output_manifest_parser])
  containers_pull_parser.set_defaults(func=containers_pull)

  containers_push_parser = containers_subparser.add_parser(
      'push',
      description="Push local docker images used by services to gcr",
      parents=[base_parser, services_parser])
  containers_push_parser.set_defaults(func=containers_push)

  containers_list_parser = containers_subparser.add_parser(
      'list',
      description="List the docker images used by services",
      parents=[base_parser, services_parser])
  containers_list_parser.set_defaults(func=containers_list)

  # Releases
  release_parser = main_subparsers.add_parser('release')
  release_subparser = release_parser.add_subparsers()

  release_current_manifest_parser = release_subparser.add_parser(
      'current-manifest',
      description=
      "Reads the current manifest from production deployments; saves a manifest with --save-manifest",
      parents=[base_parser, services_parser, output_manifest_parser])
  release_current_manifest_parser.set_defaults(func=release_current_manifest)

  release_diff_parser = release_subparser.add_parser(
      'diff',
      description="Diffs the release against the production release",
      parents=[base_parser, services_parser, release_args_parser])
  release_diff_parser.set_defaults(func=release_diff)

  release_push_parser = release_subparser.add_parser(
      'push',
      description="Actually do the release",
      parents=[base_parser, dryrun_parser, services_parser, release_args_parser])
  release_push_parser.set_defaults(func=release_push)

  release_prepare_parser = release_subparser.add_parser(
      'prepare',
      description="Prepare the release, producing k8s yaml files",
      parents=[base_parser, services_parser, release_args_parser])
  release_prepare_parser.set_defaults(func=release_prepare)

  return main_parser


##################################
# Main
##################################


def main() -> None:
  global should_debug
  global should_dry_run
  global dry_run_target
  parser = create_arg_parser()  # type: argparse.ArgumentParser
  cliargs = parser.parse_args()
  should_debug = cliargs.debug
  dry_run_target = getattr(cliargs, "dry_run", False)
  should_dry_run = dry_run_target != False
  if dry_run_target not in [False, "client", "server"]:
    bad(f"Invalid dry_run target: {dry_run_target}")
  cliargs.func(cliargs)
  if error:
    sys.exit(1)


main()