AWS.ngs

ns(GlobalRes=Res, GlobalResDef=ResDef, global_test=test) {

	# TODO: alert or handle somehow the special case when anchor properties are changed,
	#       this might be a programming error.

	# TODO: track newly added resources and provide a method to list them or have a callback

	# TODO: Use JSON tags. Currently special symbols in tag name or value will cause problems.
	# http://docs.aws.amazon.com/cli/latest/userguide/shorthand-syntax.html

	# TODO: log describe AWS CLI commands too

	# Methods with added implementations
	global Str, init, find, latest, create, update, delete, users_ids, run, id, ids, (==), subset

	# --------------------------------------------------
	# Constants
	# --------------------------------------------------

	# From https://wiki.debian.org/Cloud/AmazonEC2Image/Jessie
	AMI_OWNER_DEBIAN = '379101102735'
	AMI_OWNER_AMAZON = '801119661308'
	INSTANCE_STATES = %[pending running shutting-down terminated stopping stopped]

	# --------------------------------------------------
	# Misc utilities
	# --------------------------------------------------

	F cli_tags(h:Hash) h / "Key=$X,Value=$Y"

	doc Convert Hash with AWS filters to command line arguments.
	doc Key-value pairs in which value is null or instance of EmptyBox are discarded.
	doc TODO: make it JSON when keys or values contain special characters such as = or ,
	F cli_filters(h:Hash) {
		unboxed = h.rejectv(NoData).mapv(only(Box, get)).reject(X == [])
		unboxed.mapv(only(Pfx, { "${A.val}*" })).mapv(only(Arr, X.join(","))).map("Name=$X,Values=$Y")
	}

	TEST AWS::cli_filters({}) == []
	TEST AWS::cli_filters({"x": "y", "z": null, "w": FullBox("ww")}) == ['Name=x,Values=y', 'Name=w,Values=ww']
	TEST AWS::cli_filters({"x": Pfx("aa"), "z": EmptyBox()}) == ['Name=x,Values=aa*']

	F cli_tags_filters(h:Hash) h / "Name=tag-key,Values=$X,Name=tag-value,Values=$Y"
	TEST AWS::cli_tags_filters({}) == []
	TEST AWS::cli_tags_filters({"x": "y"}) == ['Name=tag-key,Values=x,Name=tag-value,Values=y']

	_aws_cached_regions = null

	# --- regions ---
	_regions = cached({
		if r = ENV.Box('NGS_AWS_REGIONS').map(X.split(',')).get(null) {
			debug("AWS", "Regions: Using NGS_AWS_REGIONS environment variable with value $r")
			r
		} else {
			``aws ec2 describe-regions``.RegionName.sort()
		}
	})
	F regions() _regions()

	F regions(cb:Fun) {
		regs = regions()
		regs.pmap(F(r) {
			data = cb(r)
			data % { A.Region = r }
			data
		}).flatten()
	}

	# --- zones ---
	# TODO: support regions
	_zones = cached({ ``aws ec2 describe-availability-zones``.ZoneName.sort() })
	F zones() _zones()

	# --------------------------------------------------
	# Res
	# --------------------------------------------------

	type Res(GlobalRes)

	F run(r:Res, log_pfx:Str, cmd:Command) r.def.run("${r} ${log_pfx}", cmd)

	F update_tags(r:Res, props:Hash) {
		target_tags = r.def.opt_prop('Tags', props).get({})
		current_tags = r.props.get('Tags', {})

		if current_tags.sort() == target_tags.sort() {
			debug("AWS", "${r} Current tags: $current_tags are same as target tags, not updating")
			return
		}

		r.log('update tags', "Current tags: $current_tags. Target tags: $target_tags")
		diff = Diff(current_tags, target_tags)

		# Tags might be only in Anchor
		# so we should create them during first update
		if (tags = cli_tags(diff.add + diff.change)) {
			r.run('add tags', %(aws ec2 create-tags --resources ${r.id()} --tags $*tags))
		}

		# Tags might be only in Anchor
		# so we should not delete other tags
		if 'Tags' in props {
			if (tags = diff.remove / "Key=$X") {
				r.run('remove tags', %(aws ec2 delete-tags --resources ${r.id()} --tags $*tags))
			}
		}
	}

	# --------------------------------------------------
	# ResDef
	# --------------------------------------------------

	type ResDef(GlobalResDef)

	F init(rd:ResDef, **kw)           super(rd, {'regions': null, 'Tags': {}} + kw)
	F init(rd:ResDef, _ngs_name:Str)  super(rd, {'regions': null, 'Tags': {}, 'name': _ngs_name})
	F init(rd:ResDef, _ngs_tags:Hash) super(rd, {'regions': null, 'Tags': _ngs_tags})

	F run(rd:ResDef, log_pfx:Str, cmd:Command) {
		if rd.dry_run {
			rd.log(log_pfx, "would run: ${cmd.argv.join(' ')}")
			null
		} else {
			rd.log(log_pfx, "running: ${cmd.argv.join(' ')}")
			``$cmd``
		}
	}

	F _assert_anchor_has_only_known_keys(rd:ResDef, *known_keys) {
		rd.anchor.without('regions', null).without('Tags', {}).each(F(k, v) {
			k not in known_keys throws InvalidArgument("Anchor contains unknown key '$k'. Known keys are: $known_keys")
		})
	}

	F _assert_props_has_only_known_keys(rd:ResDef, props:Hash, *known_keys) {
		props.without('META').each(F(k, v) {
			k not in known_keys throws InvalidArgument("Props contains unknown key '$k'. Known keys are: $known_keys")
		})
	}

	# --------------------------------------------------
	# Vpc
	# --------------------------------------------------

	type Vpc(ResDef)
	type VpcRes(Res)

	F id(r:VpcRes) r.props.VpcId

	F find(rd:Vpc) {
		debug("AWS", "Find VPC $rd")
		rd._assert_anchor_has_only_known_keys('Tags', 'CidrBlock')
		filters = ['--filters'] +? (rd.anchor.Tags.cli_tags_filters() + cli_filters({'cidr': rd.anchor.get('CidrBlock')}))
		rd.resources = ``aws ec2 describe-vpcs $*filters``.map(VpcRes(rd, X))
		rd
	}

	F create(rd:Vpc, **props) {
		rd._assert_props_has_only_known_keys(props, 'Tags', 'CidrBlock')
		cidr = rd.req_prop('CidrBlock', props)
		result = rd.run('Create VPC', %(aws ec2 create-vpc --cidr-block $cidr))
		rd.resources = rd.resources or []
		if not(rd.dry_run) {
			filters = cli_filters({'vpc-id': result.Vpc.VpcId})
			rd.created(``aws ec2 describe-vpcs --filters $*filters``.map(VpcRes(rd, X)), props)
		}
	}

	F update(r:VpcRes, props:Hash) {
		r.def._assert_props_has_only_known_keys(props, 'Tags', 'CidrBlock')
		r.update_tags(props)
		# TODO: since we can't modify CidrBlock, we should at least assert that it matches
		if 'CidrBlock' in props {
			props.CidrBlock != r.props.CidrBlock throws InvalidArgument("CidrBlock can not be changed after VPC creation")
		}
	}

	F delete(r:VpcRes) {
		r.run('Delete VPC', %(aws ec2 delete-vpc --vpc-id ${r.props.VpcId}))
	}

	F Str(r:Vpc) {
		t = if r.resources is Null {
			''
		} else {
			" ${r.ids().join(',')}"
		}
		"<Aws::Vpc$t anchor=${r.anchor}>"
	}

	# --- Extract VPC ID ---
	F _eval_vpc_id(x) x
	F _eval_vpc_id(rd:Vpc) rd.ids().the_one()

	# --------------------------------------------------
	# Internet Gateway
	# --------------------------------------------------

	type Igw(ResDef)
	type IgwRes(Res)

	F id(r:IgwRes) r.props.InternetGatewayId

	F find(rd:Igw) {
		debug("AWS", "Find IGW $rd")
		rd._assert_anchor_has_only_known_keys('Tags', 'Attachments', 'InternetGatewayId')

		filters = Argv({
				'--internet-gateway-ids': rd.anchor.get('InternetGatewayId')
				'--filters': rd.anchor.Tags.cli_tags_filters() + rd.anchor.get('Attachments', []).map(F(att) cli_filters({'attachment.vpc-id': _eval_vpc_id(att.VpcId)})).flatten()
		})

		rd.resources = ``aws ec2 describe-internet-gateways $*filters``.map(IgwRes(rd, X))
		rd
	}

	F create(rd:Igw, **props) {
		rd.log('Create', 'IGW')
		rd._assert_anchor_has_only_known_keys('Tags', 'Attachments')
		result = rd.run('Create IGW', %(aws ec2 create-internet-gateway))
		rd.resources = rd.resources or []
		if not(rd.dry_run) {
			filters = cli_filters({'internet-gateway-id': result.InternetGateway.InternetGatewayId})
			rd.created(``aws ec2 describe-internet-gateways --filters $*filters``.map(IgwRes(rd, X)), props)
		}
	}

	F update(r:IgwRes, props:Hash) {
		debug("AWS", "${r} Update IGW")
		r.def._assert_anchor_has_only_known_keys('Tags', 'Attachments')
		r.update_tags(props)
		r.def.opt_prop('Attachments', props).each(F(target_attachments) {
			target_vpcs_ids = target_attachments.VpcId.map(_eval_vpc_id)
			current_vpcs_ids = r.props.Attachments.VpcId
			if current_vpcs_ids == target_vpcs_ids {
				debug("AWS", "${r} Current attachments: $current_vpcs_ids are same as target attachments, not updating")
				return
			}
			r.log('update_attachments/info', "Current attachments: $current_vpcs_ids. Target attachments: $target_vpcs_ids")
			diff = Diff(current_vpcs_ids, target_vpcs_ids)

			diff.add.each(F(vpc_id) {
				r.run('update_attachments/add', %(aws ec2 attach-internet-gateway --internet-gateway-id ${r.id()} --vpc-id $vpc_id))
			})

			# Attachments might be only in Anchor
			# so we should not delete other attachments
			if 'Attachments' in props {
				diff.remove.each(F(vpc_id) {
					r.run('update_attachments/remove', %(aws ec2 detach-internet-gateway --internet-gateway-id ${r.id()} --vpc-id $vpc_id))
				})
			}
		})

	}

	F delete(r:IgwRes) {
		r.run('Delete IGW', %(aws ec2 delete-internet-gateway --internet-gateway-id ${r.id()}))
	}

	# --------------------------------------------------
	# RouteTable (WIP)
	# --------------------------------------------------

	type RouteBox(FullBox)

	F _normalize(r:RouteBox) r.get().without('Origin').without('State').sort()

	F ==(a:RouteBox, b:RouteBox) _normalize(a) == _normalize(b)

	type RouteTable(ResDef)
	type RouteTableRes(Res)

	F id(r:RouteTableRes) r.props.RouteTableId

	F find(rd:RouteTable) {
		# TODO: route.gateway-id, route.instance-id
		debug("AWS", "Find RouteTable ${rd}")
		rd._assert_anchor_has_only_known_keys('Tags', 'VpcId')
		filters = Argv({
			'--filters': rd.anchor.Tags.cli_tags_filters() + cli_filters({'vpc-id': rd.anchor.Box('VpcId').map(_eval_vpc_id)})
		})
		rd.resources = ``aws ec2 describe-route-tables $*filters``.map(RouteTableRes(rd, X))
		rd
	}

	F update(r:RouteTableRes, props:Hash) {
		debug("AWS", "${r} Update RouteTable")
		# log("RouteTableRes entry $r")
		r.def.opt_prop('Routes', props) do F(target_routes) {
			target_routes .= normalize_presence_list()
			# log("target_routes $target_routes")
			# TODO: do not replace in-place, make a copy
			target_routes.map(get).each(F(r) {
				for prop in %[GatewayId InstanceId] {
					if prop in r {
						if r.(prop) is ResDef then r.(prop) = r.(prop).ids().the_one()
					}
				}
			})
			target_routes.each(F(r) r.val .= RouteBox())
			diff = Diff(r.props.Routes.map(RouteBox), target_routes, full=true) # apparently works without full. TODO: check why
			# log("RouteTableRes diff $diff")
			diff.add.map(get).each(F(route) {
				args = Argv({
					'--route-table-id': r.id()
					'--destination-cidr-block': route.DestinationCidrBlock
					'--gateway-id': route.get('GatewayId')
					'--instance-id': route.get('InstanceId')
					'--network-interface-id': route.get('NetworkInterfaceId')
					'--vpc-peering-connection-id': route.get('VpcPeeringConnectionId')
					'--nat-gateway-id': route.get('NatGatewayId')
				})
				r.run('Update route table/add route', %(aws ec2 create-route $*args))
			})
			diff.remove.map(get).each(F(route) {
				args = Argv({
					'--route-table-id': r.id()
					'--destination-cidr-block': route.DestinationCidrBlock
				})
				r.run('Update route table/remove route', %(aws ec2 delete-route $*args))
			})
		} # Routes
	}


	# --------------------------------------------------
	# SecGroup
	# --------------------------------------------------

	type SecGroup(ResDef)
	type SecGroupRes(Res)

	F init(rd:SecGroup, anchor:Arr) {
		warn("Using deprecated SecGroup anchor - array. Should use SecGroup(name, vpc) form.")
		super(rd, name=anchor[0], VpcId=anchor[1])
	}

	F init(rd:SecGroup, name:Str, VpcId) {
		guard VpcId is Str or VpcId is Vpc
		super(rd, name=name, VpcId=VpcId)
	}

	F id(r:SecGroupRes) r.props.GroupId

	F users_ids(r:SecGroup) r.resources / F(res) { {'GroupId': res.props.GroupId, 'UserId': res.props.OwnerId} }

	# TODO: refactor - start
	F vpc_id(rd:SecGroup) rd.anchor.get('VpcId')

	F vpc_id(rd:SecGroup) {
		guard rd.anchor.get('VpcId') is ResDef
		rd.anchor.VpcId.ids().the_one()
	}

	F vpc_id_args(rd:SecGroup) ['--vpc-id', vpc_id(rd)]

	F vpc_id_args(rd:SecGroup) {
		guard vpc_id(rd) is Null
		[]
	}

	F vpc_id_filter(rd:SecGroup) ['--filters', "Name=vpc-id,Values=${vpc_id(rd)}"]

	F vpc_id_filter(rd:SecGroup) {
		guard vpc_id(rd) is Null
		[]
	}
	# TODO: refactor - end

	F find(rd:SecGroup) {
		debug("AWS", "Find SG ${rd}")
		rd._assert_anchor_has_only_known_keys('VpcId', 'name', 'Tags')
		filters = ['--filters'] +? rd.anchor.Tags.cli_tags_filters()
		# XXX: check with default VPC / EC2 classic
		# Can not use --group-names as it will assume default VPC / EC2 classic
		resources = ``aws ec2 describe-security-groups $*{vpc_id_filter(rd)} $*filters``
		if 'name' in rd.anchor {
			resources .= filter(F(sg) sg.GroupName == rd.anchor.name)
		}
		rd.resources = resources.map(SecGroupRes(rd, X))
		rd
	}

	type IpPermBox(FullBox)

	# Cleans up AWS Security Group IpPermissions so it's possible to compare current and desired states of IpPermissions
	# TODO: cleanup FromPort -1 & ToPort -1
	F cleanup(x:IpPermBox) {
		ret = x.val
		ret .= filter(F(k, v) (v is not Arr) or v)
		ret .= without('IpRanges', [{"CidrIp": "0.0.0.0/0"}])
		if 'UserIdGroupPairs' in ret {
			# Hash is ordered.
			ret.UserIdGroupPairs .= map(sort)
		}
		ret
	}

	F ==(a:IpPermBox, b:IpPermBox) {
		a .= cleanup()
		b .= cleanup()
		a.len() != b.len() returns false
		not(Diff(a, b))
	}

	# TOOD: continue here
	F subset(smaller:IpPermBox, larger:IpPermBox) {
		a .= cleanup()
		b .= cleanup()
	}

	F create(rd:SecGroup, **props) {
		rd.log('Create', 'SG')
		rd.run('Create SG', %(aws ec2 create-security-group --group-name ${rd.anchor.name} --description ${rd.req_prop('Description', props)} $*{vpc_id_args(rd)}))
		if not(rd.dry_run) {
			# XXX use GroupId output of run()
			rd.find()
			rd.resources % update(X, props)
		}
	}

	F update(sg:SecGroupRes, props:Hash) {

		debug("AWS", "${sg} Updating SG")

		if 'IpPermissions' in props {
			# PrefixListIds - http://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_IpPermission.html
			# (Valid for AuthorizeSecurityGroupEgress, RevokeSecurityGroupEgress and DescribeSecurityGroups only)
			# sg.props.IpPermissions.each(F(perm) perm.del('PrefixListIds'))
			target_perms = normalize_presence_list(props.IpPermissions)
			target_perms.each(F(r) r.val .= IpPermBox())
			diff = Diff(sg.props.IpPermissions/IpPermBox, target_perms, full=true)

			if diff.add {
				sg.run('add SG ingress rules', %(aws ec2 authorize-security-group-ingress --group-id ${sg.props.GroupId} --ip-permissions $*{diff.add.val / encode_json}))
			}
			if diff.remove {
				sg.run('remove SG ingress rules', %(aws ec2 revoke-security-group-ingress --group-id ${sg.props.GroupId} --ip-permissions $*{diff.remove.val / encode_json}))
			}
		}

		if 'IpPermissionsEgress' in props {
			diff = Diff(sg.props.IpPermissionsEgress/IpPermBox, props.IpPermissionsEgress/IpPermBox, full=true)
			if diff.add {
				sg.run('add SG egress rules', %(aws ec2 authorize-security-group-egress --group-id ${sg.props.GroupId} --ip-permissions $*{diff.add.val / encode_json}))
			}
			if diff.remove {
				sg.run('remove SG egress rules', %(aws ec2 revoke-security-group-egress --group-id ${sg.props.GroupId} --ip-permissions $*{diff.remove.val / encode_json}))
			}
		}

		sg.update_tags(props)
	}

	F delete(r:SecGroupRes) {
		r.run('Delete security group', %(aws ec2 delete-security-group --group-id ${r.props.GroupId}))
	}

	# --------------------------------------------------
	# Subnet
	# --------------------------------------------------

	type Subnet(ResDef)
	type SubnetRes(Res)

	F id(r:SubnetRes) r.props.SubnetId

	F find(rd:Subnet) {
		debug("AWS", "Find Subnet ${rd}")
		rd._assert_anchor_has_only_known_keys('Tags', 'VpcId', 'CidrBlock', 'AvailabilityZone')
		filters = ['--filters'] +? ( rd.anchor.Tags.cli_tags_filters() + cli_filters({
			'vpc-id':             rd.anchor.Box('VpcId').map(_eval_vpc_id)
			'cidr':               rd.anchor.Box('CidrBlock')
			'availability-zone':  rd.anchor.Box('AvailabilityZone')
		}))
		rd.resources = ``aws ec2 describe-subnets $*filters``.map(SubnetRes(rd, X))
		rd
	}

	# TODO: Support multiple subnets so one could easily set up new VPC for RDS and ELB
	F create(rd:Subnet, **props) {
		rd._assert_props_has_only_known_keys(props, 'VpcId', 'CidrBlock', 'AvailabilityZone')
		vpcid = rd.req_prop('VpcId', props)._eval_vpc_id()
		cidr = rd.req_prop('CidrBlock', props)
		az = rd.opt_prop('AvailabilityZone', props).map({ ['--availability-zone', A] }).get([])
		result = rd.run('Create Subnet', %(aws ec2 create-subnet --vpc-id $vpcid --cidr-block $cidr $*az))
		rd.resources = rd.resources or []
		if not(rd.dry_run) {
			filters = cli_filters({'subnet-id': result.Subnet.SubnetId})
			rd.created(``aws ec2 describe-subnets --filters $*filters``.map(SubnetRes(rd, X)), props)
		}
	}

	F update(r:SubnetRes, props:Hash) {
		debug("AWS", "${r} Updating Subnet")
		r.def._assert_props_has_only_known_keys(props, 'Tags')
		r.update_tags(props)
	}

	F delete(r:SubnetRes) {
		r.run('Delete Subnet', %(aws ec2 delete-subnet --subnet-id ${r.props.SubnetId}))
	}

	# --------------------------------------------------
	# Image
	# --------------------------------------------------

	type Image(ResDef)
	type ImageRes(Res)

	# debian-jessie-amd64-hvm
	F find(rd:Image) {
		debug("AWS", "Find Image ${rd}")
		rd._assert_anchor_has_only_known_keys('OwnerId', 'Name', 'State', 'RootDeviceType', 'VolumeType')
		owners = ['--owners', rd.anchor.OwnerId] tor []
		filters = ['--filters'] +? cli_filters({
			'state':                            rd.anchor.get('State', 'available')
			'virtualization-type':              'hvm'
			'name':                             rd.anchor.Box('Name').filter({A is Str or A is Pfx})
			'root-device-type':                 rd.anchor.Box('RootDeviceType')
			'block-device-mapping.volume-type': rd.anchor.Box('VolumeType')
		})
		resources = ``aws ec2 describe-images $*owners $*filters``
		if 'Name' in rd.anchor {
			resources .= filter({A.Name ~ rd.anchor.Name})
		}
		rd.resources = resources / ImageRes(rd, X)
		debug("AWS", "found ${resources.len()} ${rd.typeof().name} resources")
		rd
	}

	# TODO: convert it to something more uniform,
	#       a property in Anchor for example, so one
	#       could just select the latest image with Anchor
	F latest(rd:Image) {
		if rd.resources is Null {
			rd.find()
		}
		assert(rd.resources.len() >= 1, "Can not find latest() image because there are no images")
		rd.resources = [rd.resources.sort({ A.props.CreationDate <= B.props.CreationDate })[-1]]
		rd
	}

	F id(r:ImageRes) r.props.ImageId


	# --------------------------------------------------
	# Instance
	# --------------------------------------------------

	# TODO: State property (now assuming "running")

	type Instance(ResDef)
	type InstanceRes(Res)

	F init(rd:Instance, **kwargs) {
		t = {} + kwargs
		t.State = kwargs.get('State', %[pending running])
		if t.State is NoData {
			t.State = INSTANCE_STATES - ['shutting-down', 'terminated']
		}
		if t.State == '*' {
			t.State = null
		}
		super(rd, **t)
	}

	F find(rd:Instance) {
		debug("AWS", "Find Instance ${rd}")
		filters = Argv({
			'--instance-ids': rd.anchor.get('InstanceId')
			'--filters': rd.anchor.Tags.cli_tags_filters() + cli_filters({
				'instance-state-name': rd.anchor.get('State')
				'vpc-id': rd.anchor.Box('VpcId').map(only(ResDef, ids)).map(the_one)
			})
		})

		debug("AWS::Instance", "Filters: $filters")

		regs = cond {
			rd.anchor.regions is Arr { rd.anchor.regions }
			rd.anchor.regions == '*' { regions() }
		}

		resources = if regs {
			instances = regs.pmap(F(r) {
				ins = ``aws ec2 describe-instances --region $r $*filters``
				ins % F(i) i.Region = r
				ins
			}).flatten()
			# .filter(F(i) try i.PublicIpAddress)
		} else {
			``aws ec2 describe-instances $*filters``
		}

		if regs {
			debug("AWS", "found ${resources.len()} ${rd.typeof().name} resources in ${resources.group({A.Region}).len()} regions (${resources.group({A.Placement.AvailabilityZone}).len()} zones)")
		} else {
			debug("AWS", "found ${resources.len()} ${rd.typeof().name} resources in ${resources.group({A.Placement.AvailabilityZone}).len()} zones")
		}

		rd.resources = resources / InstanceRes(rd, X)
		rd
	}

	F id(r:InstanceRes) r.props.InstanceId

	# TODO: Support multiple NICs?
	# TODO: ImageId should be able to be in props too, not just anchor
	F create(rd:Instance, **props) {
		rd.log('create', 'Instance')
		for prop_name in %[ImageId] {
			prop_name not in rd.anchor throws InvalidArgument("Anchor must contain $prop_name when creating an instance")
		}

		args = Argv({
			'--image-id':            rd._image_id()
			'--key-name':            rd.opt_prop('KeyName', props)
			'--security-group-ids':  _sgs(rd, props)
			'--subnet-id':           rd.opt_prop('SubnetId', props).map(only(ResDef, ids))
			'--instance-type':       rd.opt_prop('InstanceType', props)
			['--associate-public-ip-address', '--no-associate-public-ip-address']: rd.opt_prop('PublicIpAddress', props)
		})

		result = rd.run('Create Instance', %(aws ec2 run-instances $*args))
		if not(rd.dry_run) {
			# warn("INSTANCES: ${result.Instances / InstanceRes(rd, X)}")
			rd.created(result.Instances / InstanceRes(rd, X), props)
		}
	}

	# TODO: Support EC2 classic?
	F update(r:InstanceRes, props:Hash) {
		debug("AWS", "${r} Update Instance")
		r.update_tags(props)

		# TODO: make SecurityGroups work with Presence
		if 'SecurityGroups' in props {
			current_sgs = r.props.SecurityGroups.GroupId
			target_sgs = _sgs(r.def, props).get()
			diff = Diff(current_sgs, target_sgs)
			if diff {
				diff.add    % { debug('AWS', "${r} - Adding SG $A") }
				diff.remove % { debug('AWS', "${r} - Removing SG $A") }
				r.run('Set security groups', %(aws ec2 modify-instance-attribute --instance-id ${r.id()} --groups $*target_sgs))
			}
		}

		# TODO: make this call for all instances of ResDef at once, not one by one
		if 'State' in props {
			a = r.props.State.Name
			b = props.State

			econd {
				a == b { "nothing to do" }
				a == "pending" and b == "running" { "nothing to do"}
				a == "stopped" and b == "running" {
					r.run('Starting instance', %(aws ec2 start-instances --instance-ids ${r.id()}))
				}
				a == "stopping" and b == "stopped" { "nothing to do" }
				a == "running" and b == "stopped" {
					r.run('Stopping instance', %(aws ec2 stop-instances --instance-ids ${r.id()}))
				}
				true throw InvalidArgument("Don't know how to transition instances state from $a to $b").set('value', b)
			}
		}

		if 'SourceDestCheck' in props and props.SourceDestCheck != r.props.SourceDestCheck {
			args = Argv({
				['--source-dest-check', '--no-source-dest-check']: props.SourceDestCheck
			})
			r.run('Set SourceDestCheck', %(aws ec2 modify-instance-attribute --instance-id ${r.id()} $*args))
		}
	}

	F delete(r:InstanceRes) {
		r.run('Terminate instance', %(aws ec2 terminate-instances --instance-ids ${r.id()}))
	}

	# Needed for update_tags()
	F id(instance:InstanceRes) instance.props.InstanceId

	# TODO: Support EC2 classic?
	# TODO: Handle SGS in Anchor?
	F _sgs(rd:Instance, props:Hash) {
		rd.opt_prop('SecurityGroups', props).map(F(sgs) {
			debug("AWS::Instance", "_sgs(AWS::Instance, Hash): sgs $sgs")
			if sgs is ResDef return ids(sgs)
			debug("AWS::Instance", "_sgs(AWS::Instance, Hash): sgs was not ResDef")
			ret = sgs.map(F(sg) if sg is ResDef then sg.expect().ids() else [sg]).flatten()
			debug("AWS::Instance", "_sgs(AWS::Instance, Hash) -> $ret")
			ret
		})
	}

	F _image_id(rd:Instance) {
		guard rd.anchor.ImageId is Image
		assert(rd.anchor.ImageId.resources.len() == 1, "Expecting exacty one image described by Image resource definition")
		rd.anchor.ImageId.ids()[0]
	}

	F _image_id(rd:Instance) {
		guard rd.anchor.ImageId is Str
		rd.anchor.ImageId
	}

	# TODO: handle regions
	F wait_state(r:InstanceRes, state_name:Str) {
		retry(
			sleep = 5
			catch_exceptions = false
			logger = debug("AWS", X)
			title = "[waiting intstance ${r.id()} to be in state $state_name]"
			body = {
				# When we enter wait_state, the instance might already by in
				# the desired state.
				r.props.State.Name == state_name returns true
				debug("AWS", "Instance ${r.id()} state is ${r.props.State.Name}")
				r.props = ``aws ec2 describe-instances --instance-ids ${r.id()}``[0]
				r.props.State.Name == state_name returns true
			}
		)
		r
	}

	doc Get lines of console output
	F get_console_output(r:InstanceRes) {
		retry(
			sleep = 5
			title = "[waiting intstance ${r.id()} to have console output]"
			catch_exceptions = false
			logger = debug("AWS", X)
			title = "[waiting intstance ${r.id()} console output]"
			body = {
				try {
					``aws ec2 get-console-output --instance-id ${r.id()}``.Output.lines()
				} catch(e:KeyNotFound) {
					guard e.key == 'Output'
					false
				}
			}
		)
	}

	doc %RET - Array of Str. Each Str format "key-type base64key"
	F get_ssh_host_keys(r:InstanceRes) {
		# Observed bahviour, which might not always be true: when console output is ready, it already has the needed output
		r.get_console_output()["-----BEGIN SSH HOST KEY KEYS-----".."-----END SSH HOST KEY KEYS-----"].map({A.split(' ')[0..2].join(' ')})
	}

	F add_to_known_hosts(r:Instance, prop_name:Str) {
		# TODO: consider making it in parallel
		for res in r.resources {
			wait_state(res, 'running')
			add_to_known_hosts(res, res.props.(prop_name))
		}
	}

	doc WIP, don't use!
	F add_to_known_hosts(r:InstanceRes, ip_or_hostname) {
		f = ENV.get('NGS_KNOWN_HOSTS', ENV.HOME / '.ssh' / 'known_hosts')
		keys_from_console = r.get_ssh_host_keys()
		scanned_keys = $(ssh-keyscan -H $ip_or_hostname 2>/dev/null).lines()
		key_parts_to_compare = scanned_keys.map({A.split(' ')[1..3].join(' ')})
		echo("S $keys_from_console, $scanned_keys")
		if d = Diff(keys_from_console, key_parts_to_compare) {
			# throw SecurtyFail(
			throw Exception("Scanned SSH keys and SSH keys from console did not match").set('diff', d)
		}
	}

	# --------------------------------------------------
	# Elb
	# --------------------------------------------------

	type Elb(ResDef)
	type ElbRes(Res)

	F id(r:ElbRes) r.props.LoadBalancerName

	F find(rd:Elb) {
		debug("AWS", "Find ELB ${rd}")
		resources = ``aws elb describe-load-balancers``
		if rd.anchor.get('name') is Str {
			resources .= filter(F(desc) desc.LoadBalancerName == rd.anchor.name)
		}
		# TODO: make tags fetching lazy
		rd.resources = resources.pmap(F(props) {
			ret = props
			ret.Instances = props.Instances.InstanceId
			rd.log('find', "Fetching ELB tags for LB ${props.LoadBalancerName}")
			ret.Tags = ``aws elb describe-tags --load-balancer-names ${props.LoadBalancerName}``[0].Tags
			ElbRes(rd, ret)
		})
		rd.resources .= reject(F(r) {
			not(rd.anchor.Tags.subset(r.props.Tags))
		})
		rd
	}

	F create(rd:Elb, **props) {
		rd.log('create', "creating an ELB")
		rd.anchor.get('name') is not Str throws InvalidArgument("ELB name and must be a string, not ${rd.anchor.get('name')}")
		'ListenerDescriptions' not in props throws InvalidArgument("Elb must have 'ListenerDescriptions' property")

		('AvailabilityZones' not in props) and ('Subnets' not in props) throws InvalidArgument("Either 'AvailabilityZones' or 'Subnets' must be in Elb properties")
		'AvailabilityZones' in props throws NotImplemented("'AvailabilityZones' Elb property")
		argv = Argv({
			'--load-balancer-name': rd.anchor.name
			'--listeners': props.ListenerDescriptions.encode_json()
			'--subnets': rd.opt_prop('Subnets', props).map(only(ResDef, ids))
		})
		rd.run('create ELB', %(aws elb create-load-balancer $*argv))
		# TODO: Use specific find, with the name
		if not(rd.dry_run) {
			rd.find()
			rd.resources % update(X, props)
		}
	}

	F update(elb:ElbRes, props:Hash) {
		debug("AWS", "${elb} Update")
		elb.def.anchor.get('name') is not Str throws InvalidArgument("Elb is the ELB name and must be a string, not ${elb.def.anchor.get('name')}")

		# Tags
		if 'Tags' in props {
			diff = Diff(elb.props.get('Tags', {}), props.Tags)

			if (tags = cli_tags(diff.add + diff.change)) {
				elb.run('add ELB tags', %(aws elb add-tags --load-balancer-names ${elb.def.anchor.name} --tags $*tags))
			}

			if (tags = diff.remove) {
				elb.run('remove ELB tags', %(aws elb remove-tags --load-balancer-names ${elb.def.anchor.name} --tags $*tags))
			}

			# Health check. Note: only updates given properties and keeps the rest as they were.
			if 'HealthCheck' in props {
				diff = Diff(elb.props['HealthCheck'], props['HealthCheck'])
				assert(not(diff.add), "Can't handle additional health check properties ${diff.add}")
				if diff.change {
					elb.log('update', "ELB health check change: ${diff.change}")
					t = elb.props['HealthCheck'] + diff.change
					elb.run(
						'update ELB health check',
						%(aws elb configure-health-check --load-balancer-name ${elb.def.anchor.name} --health-check ${t.encode_json()})
					)
				}
			}
		}

		# Instances
		if 'Instances' in props {
			instances = props.Instances
			if instances is ResDef {
				instances .= ids()
			}
			diff = Diff(elb.props.Instances, instances)
			if diff.add {
				elb.run(
					'register instances with ELB',
					%(aws elb register-instances-with-load-balancer --load-balancer-name ${elb.def.anchor.name} --instances $*{diff.add})
				)
			}
			if diff.remove {
				elb.run(
					'deregister instances from ELB',
					%(aws elb deregister-instances-from-load-balancer --load-balancer-name ${elb.def.anchor.name} --instances $*{diff.remove})
				)
			}
		}

		# Security groups
		if 'SecurityGroups' in props {
			sgs = props.SecurityGroups
			if sgs is ResDef {
				if sgs.resources is Null {
					sgs.find()
				}
				sgs .= ids()
			}
			diff = Diff(elb.props.SecurityGroups, sgs)
			if diff.add or diff.remove {
				elb.run(
					'update ELB SGs',
					%(aws elb apply-security-groups-to-load-balancer --load-balancer-name ${elb.def.anchor.name} --security-groups $*{sgs})
				)
			}
		}

		# TODO: listeners
		if 'ListenerDescriptions' in props {
			diff = Diff(elb.props.ListenerDescriptions, props.ListenerDescriptions)
		}
	}

	F delete(elb:Elb) {
		elb.run('delete ELB', %(aws elb delete-load-balancer --load-balancer-name ${elb.anchor.name}))
	}

	# --------------------------------------------------
	# RecordSet
	# --------------------------------------------------

	# http://docs.aws.amazon.com/cli/latest/reference/route53/change-resource-record-sets.html

	# Anchor: [record_name, domain_name]. Most specific first to be unifrom with Subnet
	# Maybe support array or *-style wildcard for record_name?
	# Maybe support anchor hosted-zone-id in the future? Don't currently see how this could be useful.

	type RecordSet(ResDef)
	type RecordSetRes(Res)

	F validate(rd:RecordSet) {
		%[Name ZoneName].each(F(prop_name) {
			prop_name not in rd.anchor throws InvalidArgument("Anchor must contain $prop_name")
			not(rd.anchor[prop_name] ~ Sfx('.')) throws InvalidArgument("$prop_name (${rd.anchor[prop_name]}) must end with a dot")
		})
		"todo"
	}

	F find(rd:RecordSet) {
		debug("AWS", "Find RecordSet ${rd}")
		anchor = rd.anchor
		zones = ``aws route53 list-hosted-zones-by-name --dns-name ${anchor.ZoneName} --max-items 1``

		assert(zones.len() == 1 and zones[0].Name == anchor.ZoneName, "Zone ${anchor.ZoneName} not found when trying to create an RecordSet in it")
		zone_id = zones[0].Id
		rd.zone_id = zone_id

		type_args = if 'Type' in anchor {
			['--start-record-type', anchor.Type]
		} else {
			[]
		}

		# XXX: getting 100 rrs. Overkill / underkill.
		rrs = ``aws route53 list-resource-record-sets --hosted-zone-id $zone_id --start-record-name ${anchor.Name} $*type_args``

		rrs .= filter({A.Name == anchor.Name})

		if 'Type' in anchor {
			rrs .= filter({A.Type == anchor.Type})
		}

		if 'SetIdentifier' in anchor {
			rrs .= filter({A.SetIdentifier == anchor.SetIdentifier})
		}

		if 'Region' in anchor {
			rrs .= filter({A.Region == anchor.Region})
		}

		rd.resources = rrs / RecordSetRes(rd, X)
		rd
	}

	F _get_props(rd:RecordSet, props:Hash) {
		ret = {}
		ret.update(props)
		%[Name Type SetIdentifier].each(F(prop) {
			if prop in rd.anchor and prop not in ret {
				ret[prop] = rd.anchor[prop]
			}
		})
		ret
	}

	F create(rd:RecordSet, **props) {
		change_batch = {
			'Comment': props.get('Comment', "Added by AWS.ngs")
			'Changes': [{
				'Action': 'CREATE'
				'ResourceRecordSet': _get_props(rd, props)
			}]
		}
		rd.run('create RecordSet', %(aws route53 change-resource-record-sets --hosted-zone-id ${rd.zone_id} --change-batch ${change_batch.encode_json()}))
		if not(rd.dry_run) {
			# XXX use some id from run() above
			rd.find()
			rd.resources % update(X, props)
		}
	}

	F update(rrset:RecordSetRes, props:Hash) {
		props = _get_props(rrset.def, props)
		diff = Diff(rrset.props, props)
		# NOT DONE AND NOT SURE!
		if diff.add or diff.change or diff.remove {
			change_batch = {
				'Comment': props.get('Comment', "Updated by AWS.ngs")
				'Changes': [{
					'Action': 'UPSERT'
					'ResourceRecordSet': _get_props(rrset.def, props)
				}]
			}
			rrset.run('update RecordSet', %(aws route53 change-resource-record-sets --hosted-zone-id ${rrset.def.zone_id} --change-batch ${change_batch.encode_json()}))
		}
	}

	F delete(r:RecordSetRes) {
		change_batch = {
			'Comment': "Deleted by AWS.ngs"
			'Changes': [{
				'Action': 'DELETE'
				'ResourceRecordSet': r.props
			}]
		}
		r.run('Delete RecordSet', %(aws route53 change-resource-record-sets --hosted-zone-id ${r.def.zone_id} --change-batch ${change_batch.encode_json()}))

	}

	# --------------------------------------------------
	# Bucket
	# --------------------------------------------------

	# TODO

	util = ns {
		F world_open_port(port:Int, proto:Str='tcp') {
			{
				'IpProtocol': proto
				'FromPort': port
				'ToPort': port
				'IpRanges': [ { 'CidrIp': '0.0.0.0/0' } ]
			}
		}
		F world_open_port(port:Arr, proto:Str='tcp') port.map(world_open_port(X, proto))

		F world_open_ip_proto(proto) {
			{
				'IpProtocol': Str(proto)
				'IpRanges': [ { 'CidrIp': '0.0.0.0/0' } ]
			}

		}
		F world_open_ip_proto(proto:Arr) proto.map(world_open_ip_proto)
	}

	# ==================================================
	# Backwards compatibility with bunch of global variables
	# ==================================================

	# "global x = y" is not exported as it's syntactically
	# different from "x = y" (which is exported)
	global AWS_AMI_OWNER_DEBIAN = AMI_OWNER_DEBIAN
	global AWS_AMI_OWNER_AMAZON = AMI_OWNER_AMAZON
	global stdlib_aws_tags = cli_tags
	global aws_regions = regions
	global AwsRes = Res, AwsResDef = ResDef
	global AwsVpc = Vpc, AwsVpcRes = VpcRes
	global AwsSecGroup = SecGroup, AwsSecGroupRes = SecGroupRes
	global AwsSubnet = Subnet, AwsSubnetRes = SubnetRes
	global AwsImage = Image, AwsImageRes = ImageRes
	global AwsInstance = Instance, AwsInstanceRes = InstanceRes
	global AwsElb = Elb, AwsElbRes = ElbRes
	global AwsRecordSet = RecordSet, AwsRecordSetRes = RecordSetRes

	# ==================================================
	# Quick and dirty - set global variables
	# ==================================================

	F pollute(do_warn=true) {

		vars =
			_exports.filterk(/^AMI_OWNER/) +
			_exports.filterv(Type).without('Res').without('ResDef') +
			{
				'regions': regions
				'zones': zones
			}

		if do_warn {
			warn("Polluting global namespace with AWS-specific variables: ${vars.keys().join(', ')}")
		}

		vars.mapk(resolve_global_variable).each(set_global_variable)
	}
}