stdlib.ngs

STDLIB_EOL = "\n"
STDLIB_PATH_SEP = '/'
STDLIB_CHILD_TO_PARENT_PIPE_DIR = 1
STDLIB_PARENT_TO_CHILD_PIPE_DIR = 2
STDLIB_MAX_RAND_REAL = Real(RAND_MAX)
STDLIB_RAND_SAFETY_MAX = 100

# === Trivial init() helper ===

doc Trivial initialization helper for init(o, ...)
doc Sets object fields from the supplied parameters
doc %EX - # sets t.a and t.b
doc %EX - F init(t:MyType, a, b) init(args())
F init(h:Hash) {
	args = Arr(h)
	dst = args.shift()[1]
	args.each(F(field_name_value) dst.(field_name_value[0]) = field_name_value[1])
	dst
}

# === Exceptions ===

{
	doc Represents failed asserttion, thrown by assert(). Should be used to catch programming errors.
	doc %EXSYN - commands
	doc %EX - F box_area(x:Int, y:Int) {
	doc %EX - 	assert(x > 0)
	doc %EX - 	assert(y > 0)
	doc %EX - 	x * y
	doc %EX - }
	doc %EX - echo(box_area(5, 10))
	doc %EX - # Output: 50
	doc %EX - echo(box_area(-5, 10))
	doc %EX - # ... Exception of type AssertFail ...
	type AssertFail
	AssertFail.inherit(Exception)
}

doc IndexNotFound exception constructor
doc %RET - IndexNotFound type instance with the given message, container and key attributes
F init(e:IndexNotFound, message:Str, container, key) init(args())

doc Throws AssertFail with givens message if condition is false
doc %RET - Unspecified, do not count on this value
F assert(condition:Bool, msg:Str) not(condition) throws AssertFail(msg)

doc Throws AssertFail if condition is false
doc %RET - Unspecified, do not count on this value
F assert(condition:Bool) assert(condition, "Assertion failed")

F assert(condition, *args) {
	guard condition is not Bool
	super(Bool(condition), *args)
}

# === Namespaces =================================

F '::'(nti:NormalTypeInstance, k) nti.(k)
F '::'(h:Hash, k) h[k]

F '::='(nti:NormalTypeInstance, k, v) nti.(k) = v
F '::='(h:Hash, k, v) h[k] = v

# === Return =====================================

doc Implements calling of Return type instances
doc like the finish(i) call in the example below
doc %EX - F first(r:Range, predicate:Fun) {
doc %EX - 	finish = Return()
doc %EX - 	r.each(F(i) {
doc %EX - 		predicate(i) throws finish(i)
doc %EX - 	})
doc %EX - 	null
doc %EX - }
F call(r:Return, v=null) {
	r.val = v
	r
}

# === Misc functional ============================

{
	doc Parent type of all types that have each() method
	type Eachable

		doc Eachable which each() calls the callback with one argument
		type Eachable1
		Eachable1.inherit(Eachable)

		doc Eachable which each() calls the callback with two arguments
		type Eachable2
		Eachable2.inherit(Eachable)

	Int.inherit(Eachable1)
	Arr.inherit(Eachable1)
	Str.inherit(Eachable1)
	# TODO: Consider Eachable2 subtype for Hash (as callback has 2 arguments)
	Hash.inherit(Eachable2)
}


doc %RET - The given argument
F identity(x)  x

TEST identity(10) == 10


# TODO: Add more info about the newly created function - useful for debugging
#       See https://docs.python.org/2/library/functools.html#partial-objects
doc Returns partially-applied function
doc f - The base function
doc Xunderscore_unsupportedXbind_args - The arguments to partiall apply (bind the function to)
F partial(f:Fun, *bind_args)
	F(*args) f(*bind_args, *args)

TEST p = partial((-), 10); p(7) == 3


doc Same as partial() but the bound arguments are last ones
F partial_tail(f:Fun, *bind_args)
	F(*args) f(*args, *bind_args)

TEST p = partial_tail((-), 10); p(7) == -3

doc Sets the given attribute to the given value
doc %RET - The modified obj
doc %EX - s.len() != 1 throws InvalidArgument("ord() argument must be of length 1 exactly").set('given', s)
doc %EX - # Would else be written as
doc %EX - if s.len() != 1 {
doc %EX - 	e = InvalidArgument("ord() argument must be of length 1 exactly")
doc %EX - 	e.given = s
doc %EX - 	throw e
doc %EX - }
F set(obj, attr:Str, val) {
	obj.(attr) = val
	obj
}

doc Convert anything to a predicate.
doc anything - The value to compare to
doc %RET - A function that compares the only argument to the given value
F Pred(anything) F eq_pred(x) x == anything

doc Convert a function to predicate.
doc %RET - The given function, without any transformation
F Pred(f:Fun) f

doc Convert a type to a predicate.
doc t - A type
doc %RET - A function that checks whether the only argument is of type t
F Pred(t:Type) F is_pred(x) x is t

doc Map something to an Arr (array) of values using mapper.
doc something - Instance of any type that has each(something, callback) implemented
doc %RET - Arr
doc %EX - [1,2,3].map(X*4)  # [4,8,12]
F map(something:Eachable, mapper:Fun)
	collector
		something.each(collect + mapper)


doc Transform mapper to handle only items matching predicate. Non-matching items will be returned as is.
doc %EX - ["abc", 1, "def", 2].map(only(Int, X*2))  # ["abc", 2, "def", 4]
F only(predicate, mapper:Fun) {
	p = Pred(predicate)
	F wrapped_in_only(elt) {
		if p(elt) {
			mapper(elt)
		} else {
			elt
		}
	}
}

TEST ["abc", 1, "def", 2].map(only(Int, X*2)) == ["abc", 2, "def", 4]

doc Filter something to an Arr (array) of values using predicate.
doc something - Instance of any type that has each(something, callback) implemented.
doc predicate - Decision function to be called with each item as first argument.
doc predicate - When predicate(item) returns true, the item will appear in the resulting array.
doc %EX - (1...10).filter(F(num) num % 2 == 0)  # Gives [2,4,6,8,10]
doc %EX - (1...10).filter(X<5)  # Gives [1,2,3,4], predicate called as predicate(item, 5)
doc %RET - Arr
F filter(something:Eachable1, predicate) {
	p = Pred(predicate)
	collector
		something.each() do F(elt) {
			if p(elt)
				collect(elt)
		}
}

{
	type TheOneFail
	TheOneFail.inherit(InvalidArgument)
}

doc Find the only element that satisfies the predicate.
doc Throws TheOneFail exception if there are no elements that satisfy the predicate or if there is more than one element that satisfies the predicate.
doc %RET - The only element that satisfies the predicate.
F the_one(something:Eachable1, predicate) {
	p = Pred(predicate)
	ret = EmptyBox()
	something.each() do F(elt) {
		if p(elt) {
			ret throws TheOneFail("the_one() had more than one match")
			ret = FullBox(elt)
		}
	}
	not(ret) throws TheOneFail("the_one() had no matches")
	ret.val
}

TEST [1,2,11,3,4].the_one(X>10) == 11
TEST try [1,2,11,12,4].the_one(X>10) catch(e:TheOneFail) true
TEST try [1,2,3,4].the_one(X>10) catch(e:TheOneFail) true

doc Find the only element that satisfies the predicate and execute given code with the value
doc body - The code to execute when exactly one element that satisfies the predicate was found. Executed with the found value. It's value will be returned as result of the_one().
doc found_more - The code to execute when more than one element satisfies the predicate. It's value will be returned as result of the_one(). Defaults to function returning null.
doc found_none - The code to execute when none of the elements satisfy the predicate. It's value will be returned as result of the_one(). Defaults to function returning null.
doc %RET - Result of running on of the following: body, found_more, found_none
doc %EX - F name(dn:MethodDocNode) {
doc %EX - 	dn.children.the_one(MethodNameDocNode).text[0]
doc %EX - }
F the_one(something:Eachable1, predicate, body:Fun, found_more:Fun={null}, found_none:Fun={null}) {
	p = Pred(predicate)
	ret = EmptyBox()
	r = Return()
	something.each(F(elt) {
		if p(elt) {
			if ret {
				throw r(found_more())
			}
			ret = FullBox(elt)
		}
	})
	if ret {
		body(ret.get())
	} else {
		found_none()
	}
}

TEST ok = false; [1,2,11,3,4].the_one(X>10, F(the_value) { if(the_value == 11) ok = true });
TEST ok = false; [1,2,11,3,4].the_one(X>20, { does_not_matter }, found_none = { ok = true }); ok
TEST ok = false; [1,2,11,3,4].the_one(Int, { does_not_matter }, found_more = { ok = true }); ok

doc Extract the only element.
doc Throws TheOneFail exception if there are no elements or if there is more than one element.
doc %RET - The only element.
doc %EX - ``aws ec2 describe-security-groups --group-ids $sg_id``.the_one()  # {OwnerId=..., VpcId=..., GroupId=..., ...}
F the_one(something:Eachable1) something.the_one({true})

doc Filter nulls out.
doc DEPRECATED, USE something.without(null) INSTEAD.
doc something - Instance of any type that has each(something, callback) implemented
doc %RET - Arr (array) of original items without nulls
F filter(something:Eachable1) {
	error("USING DEPRECATED filter(something)")
	filter(something, X is not Null)
}

TEST [1,2,null,3].filter() == [1,2,3]

doc Filter something to an Arr (array) of values using predicate
doc predicate - Decision function to be called with each item as first argument.
doc predicate - When predicate(item) returns true, the item will not appear in the resulting array
doc something - Instance of any type that has each(something, callback) implemented
doc %EX - (1...10).reject(F(num) num % 2 == 0)  # Gives [1,3,5,7,9]
doc %EX - (1...10).reject(X<5)  # Gives [5,6,7,8,9,10], predicate called as predicate(item, 5)
doc %RET - Arr
F reject(something:Eachable, predicate) {
	p = Pred(predicate)
	something.filter(not + p)
}


doc EXPERIMENTAL! Do not use!
F take_while(something:Eachable1, predicate)
	collector {
		p = Pred(predicate)
		r = Return()
		something.each(F(elt) {
			if p(elt)
				collect(elt)
			else
				throw r()
		})
	}

TEST [1,2,3,1,2].take_while(X<3) == [1,2]

# Would look much better with an iterator of "something"
doc EXPERIMENTAL! Do not use!
F drop_while(something:Eachable1, predicate)
	collector {
		p = Pred(predicate)
		found = false
		something.each(F(elt) {
			if found { collect(elt); return null; }
			if not(p(elt)) {
				collect(elt)
				found = true
			}
		})
	}

TEST [1,2,3,1,2].drop_while(X<3) == [3,1,2]

doc Replace all occurrences of src with dst
doc something - Instance of any type that has each(something, callback) implemented
doc %RET - Arr
doc %EX - ["ssh", "IP", "w"].replace("IP", "10.0.0.100")  # ['ssh','10.0.0.100','w']
F replace(something:Eachable1, src, dst) map(something, only(X == src, {dst}))

TEST ["ssh", "IP", "w"].replace("IP", "10.0.0.100") == ['ssh','10.0.0.100','w']

doc Combine items to a single value using the supplied binary function
doc First f is applied to start and the first element of something
doc then on each step f is applied to previous result and next element
doc of something.
doc start - First argument of f, for the first call of f
doc something - Instance of any type that has each(something, callback) implemented
doc f - The combining function
doc %EX - F sum(something) something.reduce(0, (+))
F reduce(something:Eachable1, start, f:Fun) {
	ret = start
	something.each(F(elt) ret = f(ret, elt))
	ret
}

TEST [1,2,3].reduce(0, (+)) == 6

doc EXPERIMENTAL!
doc Map when there is more than one element. If there is exactly one element, it's left as is
doc mapper - Will be called with zero based index and successive elements from arr
doc %RET - Arr
F map_base_idx(base, n:Int, mapper:Fun) {
	n == 1 returns [base]
	n.map(mapper(base, X))
}

doc EXPERIMENTAL! Do not use!
doc Map val only when val is true in boolean context. Otherwise return the original val.
F map_true(val, mapper:Fun) only(identity, mapper)(val)

TEST [].map_true(["--filters"] + X) == []
TEST ["something"].map_true(["--filters"] + X) == ["--filters", "something"]

doc EXPERIMENTAL! Do not use!
doc Map val only when val is false in boolean context. Otherwise return the original val.
F map_false(val, mapper:Fun) only(not + identity, mapper)(val)

TEST [].map_false({ ["default"] }) == ["default"]
TEST ["something"].map_false({ ["default"] }) == ["something"]

doc Call cb with val
doc %RET - val
doc %EX - h.mapv(only(Pfx, { "${A.val}*" })).map("Name=$X,Values=$Y").tap(F(x) echo("DEBUG: $x")).map_true(['--filters'] + X)
F tap(val, cb:Fun) {
	cb(val)
	val
}

TEST ok=false; (7.tap({ if A == 7 ok = true}) == 7) and ok

doc Filter operator.
doc Same as calling x.filter(predicate)
doc %EX - [1,2,3,4] ? F(x) x > 2  # [3,4]
F ?(x, predicate:Fun) x.filter(predicate)

doc Map operator.
doc Same as calling x.map(mapper)
doc %EX - [1,2,3,4] / F(x) x * 10
F /(x, mapper:Fun) x.map(mapper)

doc Each operator.
doc Same as calling x.each(cb)
doc %EX - [1,2,3,4] % echo
F %(x, cb:Fun) x.each(cb)

doc Call operator.
doc Same as calling f(x)
doc %EX - [1,2,3,4] \ echo
F \(x, f:Fun) f(x)

# === ArrLike ====================================

{
	doc Parent type for user-defined types with array-like behaviour. Use in cases when you would like to inherit from built-in Arr. Inheriting from built-ins is not possible for now.
	doc arr_like_attr - name of the attribute that holds the underlying array, defaults to "items"
	doc items - default attribute for the underlying array
	doc %EX - type T(ArrLike)
	doc %EX - F Str(t:T) "<My array has ${len(t)} items totalling ${sum(t)}>"  # Override one of the Arr methods
	doc %EX - a = T()
	doc %EX - a.push(10)
	doc %EX - a.push(20)
	doc %EX - echo(a)  # <My array has 2 items totalling 30>
	doc %EX -
	doc %EX - # If you need init() you should have something like the following to allow ArrLike initialization:
	doc %EX - init(t:T) {
	doc %EX - 	super(t)
	doc %EX - 	...
	doc %EX - }
	type ArrLike
	ArrLike.inherit(Eachable1)
}

doc ArrLike constructor
doc attr - name of the attribute that holds the underlying array
F init(al:ArrLike, attr:Str='items') {
	al.arr_like_attr = attr
	al.(al.arr_like_attr) = []
}

doc Get length of the underlying array.
F len(al:ArrLike)               al.(al.arr_like_attr).len()

doc Set element in the underlying array.
F '[]'(al:ArrLike, idx:Int)     al.(al.arr_like_attr)[idx]

doc Get element from the underlying array.
F '[]='(al:ArrLike, idx:Int, x) al.(al.arr_like_attr)[idx] = x

doc Call cb for each element in the underlying array.
F each(al:ArrLike, cb:Fun) al.(al.arr_like_attr).each(cb)

doc Call Bool() on the underlying array.
F Bool(al:ArrLike) al.(al.arr_like_attr).Bool()

doc Push an element to the underlying array.
F push(al:ArrLike, val) al.(al.arr_like_attr).push(val)

# === HashLike ===================================

{
	type HashLike
	HashLike.inherit(Eachable2)
}

F in(name:Str, hl:HashLike)  name in hl.attrs().HashLike

F len(hl:HashLike)           hl.attrs().HashLike.len()
F '[]' (hl:HashLike, k)      hl.attrs().HashLike[k]
F '[]='(hl:HashLike, k, v)   hl.attrs().HashLike[k] = v

F .(hl:HashLike, k)          hl.attrs().HashLike[k]
F .=(hl:HashLike, k, v)      hl.attrs().HashLike[k] = v
F get(hl:HashLike, k, dflt)  hl.attrs().HashLike.get(k, dflt)
F get(hl:HashLike, k)        hl.get(k, null)

F each(hl:HashLike, cb:Fun)  hl.attrs().HashLike.each(cb)
F keys(hl:HashLike)          hl.attrs().HashLike.keys()
F values(hl:HashLike)        hl.attrs().HashLike.values()
F Bool(hl:HashLike)          hl.attrs().HashLike.Bool()

# TODO: mapk, mapv and friends
# TOOD: Hash() - copy or hash attribute?

F filter(hl:HashLike, cb:Fun) {
	ret = HashLike()
	ret.attrs().HashLike = hl.attrs().HashLike.filter(cb)
	ret
}

F init(hl:HashLike, h:Hash=null) {
	if hl.attrs() is not Hash {
		hl.attrs({})
	}
	hl.attrs().HashLike = h or {}
}

doc Get the underlying Hash with all the keys/values of the HashLike.
doc Note: the returned value is not a copy.
doc %RET - Hash
F Hash(hl:HashLike) hl.attrs().HashLike

# === Hook =======================================

{
	doc Hook is a simple pub-sub
	type Hook
	Hook.inherit(HashLike)
}

doc Hook constructor.
doc %RET - Hook
F init(hook:Hook) {
	super(hook)
	hook.attrs().idx = 0
}

doc Add unnamed handler.
doc The hook is automatically named "pushed-N" where N is sequential integer.
doc %RET - New hook name
F push(hook:Hook, handler:Closure) {
	name = "pushed-${hook.attrs().idx}"
	hook.attrs().idx += 1
	hook[name] = handler
	name
}

# TODO: consider optional? try/catch isolation
doc Runs all handlers passing all args.
doc args - Arguments to pass to handlers
doc %RET - Output of the combiner function applied to all handlers' results
F call(hook:Hook, *args) hook.attrs().HashLike.mapv(X(*args))

# === Real =======================================

# TODO: something more efficient
doc Convert a string to real (floating) number, inefficiently
doc %RET - Real
doc %EX - Real("1.1")  # 1.1
F Real(s:Str) {
	parts = s.split('.')
	parts.len() == 1 returns Real(Int(parts[0]))
	parts.len() != 2 throws InvalidArgument("Invalid argument ${s}")
	l = parts[1].len()
	frac = Real(Int(parts[1]))
	for(i;l) frac = frac / 10.0
	Real(Int(parts[0])) + frac
}

TEST Real('1.1') == 1.1

# === Fun helpers ================================

doc TODO: Defaut values
F StrParams(f:Fun) {
	guard f is Closure or f is NativeMethod
	f.params().map(F(p) {
		"${p.name}:${p.type.name}"
	}).join(', ') \ "($X)"
}

# === NativeMethod ===============================

doc String representation of native method.
doc %RET - "<Native method NAME>"
doc %EX - (%)[0].Str().echo()  # Outputs: <Native method %>
F Str(c:NativeMethod) "<Native method ${c.attrs().name}${StrParams(c)}>"

# === Closure ====================================

# TODO: show arguments:
#   "<Closure ${c.attrs().name tor "<anonymous>"}(HERE) at ${location}>"
doc String representation of a closure
doc %RET - Str
doc %EX - Real.constructors[-1].Str().echo()  # Outputs: <Closure Real at /usr/share/ngs/stdlib.ngs:350>
F Str(c:Closure) {
	info = c.ip().resolve_instruction_pointer()
	location = "${info.file}:${info.first_line}" tor "<unknown location>"
	"<Closure ${c.attrs().name tor "<anonymous>"}${StrParams(c)} at ${location}>"
}

# === Type =======================================

doc Type constructor
doc %EX - type MyType1
doc %EX - type MyType2(MyType1)
F Type(t:Str, doc, parent:Type) Type(t, doc, [parent])

doc Type constructor
doc %EX - type MyType1
doc %EX - type MyType2
doc %EX - type MyType3([MyType1, MyType2])
F Type(t:Str, doc, parents:Arr) {
	ret = Type(t, doc)
	parents % ret.inherit(X)
	ret
}

TEST type T1; type T2(T1); T2() is T1

doc Match a type. Same as "something is t".
doc %EX -   1 ~ Int  # true
doc %EX - "a" ~ Int  # false
F ~(something, t:Type) something is t

doc String representation of a type
doc %RET - "<Type NAME>"
doc %EX - Real.Str().echo()  # Outputs: <Type Real>
F Str(t:Type) "<Type ${t.name}>"

# === NormalType =================================

doc String representation of normal type instance i
doc Normal type is a user defined type. In addition some types defined by NGS are also normal types.
doc %EX - {
doc %EX -   type T
doc %EX -   # nti - Normal type instance
doc %EX -   nti = T()
doc %EX -   nti.a = 1
doc %EX -   nti.b = 2
doc %EX -   echo(Str(nti)) # <T a=1 b=2>
doc %EX - }
doc %RET - "<TYPE_NAME attr1=val1 attr2=val2 ...>"
F Str(i:NormalTypeInstance) {
	h = Hash(i)
	truncated = len(h) > 10
	if truncated {
		h .= limit(10)
	}
	# TODO limit key and value lengths
	h .= Strs()
	attrs = h.join(' ').map_true(' ' + X)

	"<${i.typeof().name}${attrs}${if truncated ' ...' else ''}>"
}

doc Set an attribute in an instance if it's not already set
doc %EX - mysomething.dflt(k, []).push(elt)
doc %RET - Attribute value, the already-existed or new.
F dflt(i:NormalTypeInstance, k, v) {
	if k not in i {
		i.(k) = v
	}
	i.(k)
}

TEST type T; t=T(); t.dflt("a", 1); t.a == 1
TEST type T; t=T(); t.a = 2; t.dflt("a", 1); t.a == 2

doc Set an attribute on instances if it's not already set
doc %RET - v
F dflt(arr:Arr, k, v) {
	guard arr.all({A is NormalTypeInstance or A is Hash})
	arr.each(X.dflt(k, v))
	v
}

doc Equaity test for normal type instances: must be of same type and have same attributes and their values
doc %RET - Bool
doc %EX - type T
doc %EX - t1 = T()
doc %EX - t1.a = 1
doc %EX - t2 = T()
doc %EX - t2.a = 1
doc %EX - echo(t1 == t2)  # Outputs: true
F ==(a:NormalTypeInstance, b:NormalTypeInstance) {
	typeof(a) != typeof(b) returns false
	Hash(a) == Hash(b)
}

# TODO: move to somewhere more appropriate, keep in mind - must be after dflt(NormalTypeInstance, ...)
F dflt(hl:HashLike, k, v) hl.attrs().HashLike.dflt(k, v)

# === Exceptions =================================

{
	# TODO: Investigate why type CException(Error) does not work at this point
	doc Base type for exceptions arising from errors returned by calling C functions
	type CException
	CException.inherit(Error)

	doc CException constructor. In addition to storing message attribute, adds errno and errno_name attributes.
	F init(e:CException, message:Str) {
		errno = get_c_errno()
		super(e, message)
		e.errno = errno
		e.errno_name = c_strerror(e.errno)
	}

	doc Exception representing a failure to kill() a process
	type KillFail
	KillFail.inherit(CException)
}

# === auto-load ==================================

doc Called when reading undefined global.
doc Implements autoloading.
doc Searches in $NGS_DIR/autoload/NAME.ngs
doc WARNING: May have security implications when looking up a name from untrusted source.
doc %EX - test("My web server runs") do { .... }  # $NGS_DIR/autoload/test.ngs is automatically loaded.
F global_not_found_handler(name:Str) {
	val = require("${NGS_DIR}/autoload/${name}.ngs")
	idx = resolve_global_variable(name)
	if not(is_global_variable_defined(idx)) {
		# echo("IDX $name $idx")
		set_global_variable(idx, val)
	}
}

# === Range ======================================

{ Range.inherit(Eachable1) }

doc Range constructor
F init(r:Range, start, end, step=1) init(args())

doc ExclusiveRange constructor
F ..(start, end)  ExclusiveRange(start, end)

doc InclusiveRange constructor
F ...(start, end) InclusiveRange(start, end)

doc Iterates over the elements of r, passing each in turn to cb.
doc cb - Function to be called with values from r
doc args - Additional arguments for calling cb
doc %RET - r
doc %EX - s=0
doc %EX - (1...10).each(F(i) { global s; s+=i })
doc %EX - echo(s)  # Outputs: 55
F each(r:Range, cb:Fun) {
	# TODO: support Real step
	econd {
		(r.end is Null) or (r.step == 0) {
			cmp = F(a,b) true
		}
		r.step > 0 {
			cmp = if r is ExclusiveRange (<) (<=)
		}
		r.step < 0 {
			cmp = if r is ExclusiveRange (>) (>=)
		}
	}
	for(i=r.start; cmp(i, r.end); i=i+r.step)
		cb(i)
	r
}

TEST (1..3).map(X*2) == [2, 4]
TEST (1...3).map(X*2) == [2, 4, 6]

doc Find first element in r that satisfies the predicate.
doc args - Additional arguments for calling the predicate.
doc The predicate is called with each one of the elements plus the args: predicate(i)
doc %RET - Either the element or null if element was not found.
doc %EX - (10..20).first(F(x) x % 3 == 0)  # 12 - first item divisible by 3
F first(r:Range, predicate:Fun) {
	finish = Return()
	r.each(F(i) {
		predicate(i) throws finish(i)
	})
	null
}

TEST (10..20).first(F(x) x % 3 == 0) == 12

doc TODO
F Str(r:ExclusiveRange) "<ExclusiveRange ${r.start}..${r.end} step ${r.step}>"

doc TODO
F Str(r:InclusiveRange) "<InclusiveRange ${r.start}...${r.end} step ${r.step}>"

# === Syntactic sugar ============================

doc Boolean negation for non-boolean values. Converts to boolean first.
doc %RET - Bool
F not(x) { guard (x is Bool) == false; x.Bool().not() }

doc Inequality operator
doc %EX - 1 !=2  # true, same as not(1==2)
F !=(a,b) not(a==b)

doc Non-sameness operator
doc %EX - h1 = {"a": 1}
doc %EX - h2 = {"a": 1}
doc %EX - h1 !== h2  # true, same as not(h1===h2)
doc %EX - h1 == h2   # true
F !==(a,b) not(a===b)

TEST 1 != 2
TEST (1 != 1) == false

TEST 1 !== 2
TEST (1 !== 1) == false


doc "not in" operator. Exactly same as "not(a in b)"
doc %EX - 10 not in [1,2]  # true
doc %EX - 1  not in [1,2]  # false
F 'not in'(a,b) not(a in b)

TEST (1 not in [1,2,3]) == false
TEST 10 not in [1,2,3]

doc "is not" operator. Exactly same as "not(a is b)".
doc %EX - 1    is not Null  # true
doc %EX - null is not Null  # false
F 'is not'(a,b) not(a is b)

TEST 1 is not Null
TEST (null is not Null) == false


doc Exactly same as "element in container". It's just more convenient in specific cases.
doc %EX - [1,2,3].has(2)  # true
doc %EX - [[1,2,3], [1,20,30], [100,200,300]].filter(X.has(1))  # [ [1,2,3], [1,20,30] ]
doc %RET - Bool
F has(container, element) element in container

TEST [1,2,3].has(1)
TEST [1,2,3].has(10).not()


doc Defines collector { ... collect(...) ... } behaviour for arrays
doc a - Initial array
doc body - The body after collector keyword and possible initial value, wrapped in a function
doc body - "collector THIS_CODE" or "collector/my_init THIS_CODE"
doc %RET - Constructed array
doc %EX - items = collector {
doc %EX -   collect(10)
doc %EX -   for(i;2) collect(i)
doc %EX -   collect(20)
doc %EX - }
doc %EX - echo(items)  # Outputs: [10,0,1,20]
doc %EX -
doc %EX - # Or start with few items:
doc %EX - items = collector/[100,200] {
doc %EX -   collect(10)
doc %EX -   for(i;2) collect(i)
doc %EX -   collect(20)
doc %EX - }
doc %EX - echo(items)  # Outputs: [100,200,10,0,1,20]
F collector(a:Arr, body:Fun) {
	body(F(elt) a.push(elt))
	a
}

doc Defines collector { ... collect(...) ... } behaviour for hashes
doc h - Initial hash
doc body - The body after collector keyword and initial value, wrapped in a function
doc body - "collector/{'my': 'hash'} THIS_CODE"
doc %RET - Constructed array
doc %EX - arr = [{"Name": "n1", "Value": "v1"},{"Name": "n2", "Value": "v2"}]
doc %EX - my_hash = collector/{}
doc %EX - 	arr.each(F(elt) collect(elt.Name, elt.Value))
doc %EX - echo(my_hash)  # Outputs: {n1=v1, n2=v2}
F collector(h:Hash, body:Fun) {
	body(F(k, v) h[k] = v)
	h
}

doc Defines collector { ... collect(...) ... } behaviour for integers (summarizes collected items).
doc n - Initial number
doc body - The body after collector keyword and initial value, wrapped in a function
doc body - "collector/100 THIS_CODE"
doc %RET - Constructed array
doc %EX - collector/0 { (1...10).each(collect) }  # 55
F collector(n:Int, body:Fun) {
	body(F(incr) n=n+incr)
	n
}

doc Comparison to null
doc %RET - false
F ==(a:Null, b) false

doc Comparison to null
doc %RET - false
F ==(a, b:Null) false

doc Comparison to null
doc %RET - true
F ==(a:Null, b:Null) true

doc Run cleanup after successful execution of body or exception in body
doc body - Main code to execute
doc cleanup - Cleanup code to execute
doc TODO: make sure cleanup() is not run twice in case of exception in cleanup() itself
doc %EX - finally(
doc %EX -   { while entry = c_readdir(d) { ... } },
doc %EX -   { ... c_closedir(d) ...}
doc %EX - )
doc %RET - Whatever body call returns
F finally(body:Fun, cleanup:Fun) {
	try {
		ret = body()
		cleanup()
		ret
	} catch(e) {
		cleanup()
		throw e
	}
}

TEST a=0; {1/1}.finally({a=10}) == 1 and a == 10
TEST a=0; (try finally() with {1/0} with {a=10} catch(e:DivisionByZero) "OK") == "OK" and a == 10

# --- Hash ---

doc Get hash key.
doc %EX - h = {"a": 1}
doc %EX - h.a  # 1, Same as h["a"]
F .(h:Hash, attr:Str) h[attr]

doc Set hash key.
doc %EX - h = {"a": 1}
doc %EX - h.a = 2  # 2, Same as h["a"] = 2
doc %RET - v
F .=(h:Hash, attr:Str, v) h[attr] = v

doc Get hash value by key or null if it does not exist
doc %EX - h = {"a": 1}
doc %EX - h.get("a")  # 1
doc %EX - h.get("b")  # null
F get(h:Hash, k) get(h, k, null)

doc Convert hash values to integers where possible
doc %RET - New Hash
doc %EX - %{k 7 kk "a"}["k"] is Str      # true
doc %EX - %{k 7 kk "a"}.n()              # {'k': 7, 'kk': 'a'}
doc %EX - %{k 7 kk "a"}.n()["k"] is Int  # true
F n(h:Hash) h.mapv({Int(A) tor A})

TEST %{k 7 kk "a"}.n() == {'k': 7, 'kk': 'a'}

# --- Arr ---
doc Return array made of given attribute of each element of given array
doc %EX - [{"x": 1}, {"x": 2}].x  # [1, 2]
doc %RET - Arr
F .(a:Arr, attr:Str) a.map(X.(attr))

TEST [{"x": 1}, {"x": 2}].x == [1, 2]

doc Return array made of given attribute of each element of given Eachable1 where present
doc %EX - [{"x": 1}, {"y": 2}].get("x")  # [1]
doc %EX - ``aws ec2 describe-instances``.Tags.get("role").uniq()
doc %EX - # Returns Arr of Str with roles. Does not crash if some machines do not have "role" tag.
doc %RET - Arr
F get(e:Eachable1, attr:Str)
	collector
		e.each(F(item) {
			try collect(item.(attr))
		})

doc Return array made of given attribute of each element of given Eachable1 where present or default value where the attribute is not present
doc Return array (? depends on map() ?) made of given attribute of each element of given Eachable1
doc %EX - [{"x": 1}, {"y": 2}].get("x", null)  # [1, null]
doc %RET - Arr
F get(e:Eachable1, attr:Str, dflt) e.map(X.get(attr, dflt))

doc Concatenate strings
doc %EX - "ab" + "cd"  # "abcd"
doc %RET - New Str
F +(s1:Str, s2:Str) "${s1}${s2}"

TEST "ab" + "cd" == "abcd"

# === Array ======================================


doc Checks whether element x is in array arr
doc x - Needle
doc arr - Haystack
doc %EX -  1 in [1,2,3]  # true
doc %EX - 10 in [1,2,3]  # false
doc %RET - Bool
F in(x, arr:Arr) {
	l = arr.len()
	for(i;l) arr[i] == x returns true
	false
}

# Tested by has() tests

doc Arrays equality test.
doc True if arrays are of the same length and all elements are equal (==)
doc %RET - Bool
F ==(a:Arr, b:Arr) {
	l = a.len()
	l != b.len() returns false
	for(i;l) a[i] != b[i] returns false
	true
}

TEST [1, 2] == [1, 2]
TEST [1, 3] != [1, 2]
TEST [1] != [1, 2]

doc Check whether there is any element in arr that satisfies the given predicate.
doc TODO: Make it work on anything with Eachable2 too.
doc e - The items to check
doc predicate - Test function
doc args - Additional arguments for calling the predicate.
doc %EX - [1,2,10].any(F(elt) elt > 5)   # true
doc %EX - [1,2,10].any(F(elt) elt > 15)  # false
doc %RET - Bool
F any(e:Eachable1, predicate) {
	p = Pred(predicate)
	r = Return()
	e.each(F(x) {
		p(x) throws r(true)
	})
	false
}

TEST [1,2,10].any(F(elt) elt > 5)
TEST [1,2,3].any(F(elt) elt > 5).not()

doc Check whether all elements in arr satisfy the given predicate.
doc e - The items to check
doc predicate - Test function
doc args - Additional arguments for calling the predicate.
doc %RET - Bool
doc %EX - [1,2,3].all(X<10)   # true
doc %EX - [1,2,10].all(X>5)   # false
F all(e:Eachable1, predicate) {
	p = Pred(predicate)
	r = Return()
	e.each(F(x) {
		not(p(x)) throws r(false)
	})
	true
}

TEST [1,2,3].all(X<10)
TEST [1,2,10].all(X>5).not()

doc Check that there is no element in e that satisfies the given predicate. Exactly same as not(any(arr, predicate)) .
doc %RET - Bool
doc %EX - [0,1,2].none(X>2)  # true
doc %EX - [0,1,2].none(X<2)  # false
F none(e:Eachable1, predicate:Fun) not(any(e, predicate))

TEST [0,1,2].none(X>2)
TEST [0,1,2].none(X<2).not()

TEST [1,2,3].map(X*4) == [4,8,12]

F subset(smaller:Arr, larger:Arr) {
	h = if larger.len() > 10 {
		larger.Hash({true})
	} else {
		larger
	}
	smaller.all(X in h)
}

TEST [1,2,3].subset([1,2,3,4])
TEST [1].subset([2]) == false

doc Iterates over the elements of arr, passing each in turn to cb along with args: cb(ITEM)
doc cb - Function to be called with values from arr
doc args - Additional arguments for calling cb
doc %RET - arr
doc %EX - s=0
doc %EX - [1,2,3].each(F(i) { global s; s+=i })
doc %EX - echo(s)  # Outputs: 6
F each(arr:Arr, cb:Fun) {
	l = arr.len()
	for(i;l) cb(arr[i])
	arr
}

# Tested by flatten() tests

# TODO: map_idx_val, each_idx_val for other types
doc Iterates over the elements of arr, passing each in turn to cb along with index and args: cb(INDEX, ITEM)
doc %RET - arr
doc %EX - [10,20,30].each_idx_val(F(idx, val) echo("Element #$idx equals to $val"))
F each_idx_val(arr:Arr, cb:Fun) {
	l = arr.len()
	for(i;l) cb(i, arr[i])
	arr
}

doc Map an Arr to an Arr (array) of values using mapper
doc mapper is called as mapper(INDEX, ITEM)
doc %RET - New Arr
doc %EX - echo("Array items: " + [10,20,30].map_idx_val(F(idx, val) "[$idx]=$val").join(", "))
doc %EX - # Outputs: Array items: [0]=10, [1]=20, [2]=30
F map_idx_val(arr:Arr, mapper:Fun) {
	l = arr.len()
	collector
		for(i;l) collect(mapper(i, arr[i]))
}

TEST r=[]; ["a", "b"].each_idx_val(F(idx, val) r.push([idx, val])); r == [[0, "a"], [1, "b"]]


# TODO: make it work with anything that has each() method
doc Process each N elements of an Array at a time.
doc Throws InvalidArgument if number of items in arr is not divisible by n.
doc cb is called as cb(eltI, ..., eltJ) where I is multiple of n and J is I+n-1
doc arr - Items to iterate in chunks of n
doc n - Number of items in chunk
doc cb - Function to be called with values from arr
doc args - Additional arguments for calling cb
doc %RET - arr
doc %EX - [1,2,3,4].each(2, F(a, b) echo("$a - $b"))  # Outputs: "1 - 2" and on the next line "3 - 4"
F each(arr:Arr, n:Int, cb:Fun) {
	arr.len() % n != 0 throws InvalidArgument("Can not iterate elements in groups of $n of array of size ${arr.len()}")
	chunks = arr.len() / n
	for(chunk_number; chunks) {
		cb(*arr[chunk_number*n..chunk_number*n+n])
	}
	arr
}

# Tested by map(arr:Arr, n:Int, cb:Fun)


doc Map each N elements of an Array at a time.
doc mapper is called as cb(eltI, ..., eltJ) where I is multiple of n and J is I+n-1
doc Throws InvalidArgument if number of items in arr is not divisible by n.
doc mapper is called as mapper(eltI, ..., eltJ) where I is multiple of n and J is I+n-1
doc arr - Items to iterate in chunks of n
doc n - Number of items in chunk
doc mapper - Function to be called with values from arr
doc args - Additional arguments for calling mapper
doc %RET - Arr
doc %EX - [1,2,3,4].map(2, F(a,b) "$a=$b").join("&")  # Outputs: 1=2&3=4
F map(arr:Arr, n:Int, mapper:Fun) collector arr.each(n, collect + mapper)

# Tested by ~(s:Str, r:RegExp)

doc Filter out all occurrences of specific value
doc arr - items to filter
doc without_elt - The value to filter out
doc %RET - Arr
doc %EX - [1,2,3,2].without(2)  # [1,3]
F without(arr:Arr, without_elt) arr.filter(X != without_elt)

TEST [1,2,3,2].without(2) == [1,3]

doc Filter out all values in a that are also in b
doc %RET - Arr
doc %EX - [1,2,3] - [5,6,1]  # [2,3]
F -(a:Arr, b:Arr) {
	h = if b.len() > 10 {
		b.Hash({true})
	} else {
		b
	}
	a.filter(X not in h)
}

TEST [1,2,3] - [5,6,1] == [2,3]
TEST [1,2,3] - [5,6,1,1,1,1,1,1,1,1,1,1,1,2] == [3]

doc Count number of items that satisfy the predicate.
doc TODO: Make it work on anything with each() method.
doc arr - Items to look at
doc predicate - Test function
doc args - Additional arguments for the predicate
doc %RET - Int
doc %EX - [1,2,3,11,12].count(X>10)  # 2
F count(arr:Arr, predicate) {
	p = Pred(predicate)
	collector/0
		arr.each(F(elt) {
			if p(elt)
				collect(1)
		})
}

TEST [1,2,3,11,12].count(X>10) == 2

doc Count all true values.
doc arr - Items to look at
doc %RET - Int
doc %EX - [0,1,2,null,false].count()  # 2 (Only 1 and 2 count as true values)
F count(arr:Arr) count(arr, identity)

TEST [0,1,2,null,false].count() == 2

doc Flatten one level.
doc e - Eachable with each element also Eachable
doc %RET - Arr
doc %EX - [[1], [2,3]].flatten()  # [1,2,3]
F flatten(e:Eachable) {
	guard e.all(Eachable)
	collector
		e.each(F(subarr) {
			subarr % collect
		})
}

TEST [[1], [2,3]].flatten() == [1,2,3]

doc Return unique values.
doc Warning: uses Hash so comparison is not using == but a built-in hash keys comparison.
doc %RET - Arr
doc %EX - [1,2,2,3,4,4].uniq()  # [1,2,3,4]
F uniq(arr:Arr) arr.Hash({true}).keys()

TEST [1,2,2,3,4,4].uniq() == [1,2,3,4]

doc Find first true value
doc %EX - [null, false, 10, 20].first()  # 10
F first(arr:Arr) first(arr, identity)

TEST [null, false, 10, 20].first() == 10

doc Find index of first value that satisfies the predicate.
doc TODO: Make it work on anything with each() method.
doc arr - Items to look at
doc predicate - Test function
doc args - Additional arguments for the predicate
doc %RET - Int or null
doc %EX - [1,2,11,3,4].index(X>10)  # 2
F index(arr:Arr, predicate) {
	p = Pred(predicate)
	l = arr.len()
	for(i;l)
		p(arr[i]) returns i
	null
}

doc Find all indexes of values that satisfy the predicate
doc TODO: Make it work on anything with each() method
doc arr - Items to look at
doc predicate - Test function
doc args - Additional arguments for the predicate
doc %RET - Arr of Int
doc %EX - [1,5,1,10].indexes(X>2)  # [1,3]
F indexes(arr:Arr, predicate) {
	p = Pred(predicate)
	l = arr.len()
	l.filter({ p(arr[A]) })
}

TEST [1,5,1,10].indexes(X>2) == [1,3]

doc Find first value that satisfies the predicate.
doc TODO: Make it work on anything with each() method
doc arr - Items to look at
doc predicate - Test function
doc args - Additional arguments for the predicate
doc %RET - Int or null
doc %EX - [1,2,11,3,4].first(X>10)  # 11
F first(arr:Arr, predicate:Fun) {
	i = arr.index(predicate)
	i is Null returns
	arr[i]
}

TEST [5,10,15].first(X>7) == 10
TEST [5,10,15].first(X>20) == null

doc Make new array which is a reversed given array
doc %RET - Arr
doc %EX - [1,2,3].reverse()  # [3,2,1]
F reverse(arr:Arr) {
	l = arr.len()
	l.map({ arr[l-A-1] })
}

TEST [1,2,3].reverse() == [3,2,1]

doc Calculate sum of the elements
doc %EX - [1,2,3].sum()  # 6
F sum(something) something.reduce(0, (+))

doc Prepend one element to the given array
doc %RET - Modified arr
doc %EX - x=[1,2]
doc %EX - x.unshift(3)
doc %EX - echo(x)  # Outputs: [3,1,2]
F unshift(arr:Arr, elt) {
	arr[0..0] = [elt]
	arr
}

TEST x=[1,2]; x.unshift(3); x == [3, 1, 2]

doc EXPERIMENTAL! Do not use!
doc a - Array of arrays to join
doc sep - Separator elements
doc %EX - [[1,2], [3,4]].join([10,20])  # [1,2,10,20,3,4]
F join(a:Arr, sep:Arr) {
	guard a.all(Arr)
	collector
		a.each_idx_val(F(idx, subarr) {
			subarr % collect
			if idx < a.len() - 1 {
				sep % collect
			}
		})
}

# --- Conversions ---

doc Make Arr from Hash. Each key/value pair becomes two-items array:
doc %RET - Arr of form [[k1, v1], [k2, v2], ...]
doc %EX - Arr({'x': 7, 'y': 8})  # [['x', 7], ['y', 8]]
F Arr(h:Hash)
	h.map(F(*pair) pair)

TEST Arr({'x': 7, 'y': 8}) == [['x', 7], ['y', 8]]

doc Make Arr from Arr. A no-op.
doc %RET - arr
F Arr(arr:Arr) arr

doc Make new array which aggregates elements from each of the args arrays
doc n-th element of the new array is an array that contains n-th element of each of the given arrays
doc If some of the arrays in args are shorter than others, they will be logically padded with null
doc %EX - zip([1,2,3], [4,5,6])  # [[1,4], [2,5], [3,6]]
doc %EX - zip([1,2,3], [4,5])    # [[1,4], [2,5], [3,null]]
doc %RET - Arr of form [[arg[0][0], arg[1][0], ... arg[i][0]], [arg[0][1], arg[1][1], ... arg[i][1]], ...]
F zip(*args) {
	arrays = args.map(Arr)
	arrays.max(len).map({ arrays.map(X.get(A, null)) })
}

TEST zip([1,2,3], [4,5,6]) == [[1,4], [2,5], [3,6]]
TEST zip([1,2,3], [4,5,6], [7,8,9,10]) == [[1,4,7], [2,5,8], [3,6,9], [null, null, 10]]

doc Convert range to an array
doc %EX - Arr(1..3)   # [1,2]
doc %EX - Arr(1...3)  # [1,2,3]
F Arr(r:Range) r / identity

TEST Arr(1..3) == [1,2]
TEST Arr(1...3) == [1,2,3]


# --- Access elements ---

doc Get array element by index from the end
doc idx - Negative index
doc %EX - [10,20,30][-1]  # 30
F '[]'(arr:Arr, idx:Int) {
	guard idx < 0
	i = arr.len() + idx
	i < 0 throws IndexNotFound("[](Arr, Int): Index $idx is illegal for array of size ${arr.len()}", arr, idx)
	arr[i]
}

doc Get array elements at specified indexes. Indexes specified by ExclusiveRange.
doc r - ExclusiveRange with negatve .end
doc %EX - [10,20,30,40][1..-1]  # [20,30]
doc %RET - Arr
F '[]'(arr:Arr, r:ExclusiveRange) {
	guard r.end is Int and r.end < 0
	r.end < 0 - arr.len() throws IndexNotFound("[](Arr, ExclusiveRange): Index ${r.end} (range end) is illegal for array of size ${arr.len()}", arr, idx)
	arr[r.start..arr.len() + r.end]
}

TEST [10,20,30,40][1..-1] == [20,30]


doc Get array elements at specified indexes.
doc arr - Array to pick items from
doc indexes - Indexes of items to pick
doc %EX - [10,20,30,40][[0,3]]  # [10, 40]
doc %RET - Arr
F '[]'(arr:Arr, indexes:Arr) indexes.map(arr[X])

TEST [10,20,30,40][[0,3]] == [10, 40]

doc Truncate an array if necessary so it would have maximum l elements.
doc a - Array to (possibly) truncate.
doc l - Maximum elements
doc %RET - Either a or new Arr of length l
doc %EX - [10,11,12].limit(2)   # [10,11]
doc %EX - [10,11,12].limit(10)  # [10,11,12]
F limit(a:Arr, l:Int) {
	a.len() <= l returns a
	a[0..l]
}

TEST [10,11,12].limit(10) == [10,11,12]
TEST [10,11,12].limit(2) == [10,11]

# --- Misc ---

ns {
	global min, max
	F most(arr:Arr, f:Fun) {
		guard arr
		ret = arr[0]
		arr.each(F(elt) if f(elt, ret) { ret = elt })
		ret
	}

	F most_by_cb(arr:Arr, f:Fun, cb:Fun) {
		guard arr
		ret = arr[0]
		ret_cb = cb(ret)
		arr.each(F(elt) {
			elt_cb = cb(elt)
			if f(elt_cb, ret_cb) {
				ret = elt
				ret_cb = elt_cb
			}
		})
		ret_cb
	}

	doc TODO
	F min(arr:Arr) most(arr, (<))
	doc TODO
	F max(arr:Arr) most(arr, (>))

	# EXPERIMENTAL!
	doc TODO
	F min(arr:Arr, cb:Fun) most_by_cb(arr, (<), cb)
	doc TODO
	F max(arr:Arr, cb:Fun) most_by_cb(arr, (>), cb)
}

doc Repeat all elements in arr n times
doc arr - Elements to repeat
doc n - Number of times to repeat the elements
doc %RET - Arr
doc %EX - [10,20] * 2  # [10,20,10,20]
F *(arr:Arr, n:Int)
	collector
		for(i;n)
			arr % collect

TEST [10,20] * 2 == [10,20,10,20]

doc Cartesian product
doc %RET - Arr of Arr[2]
doc %EX - [10,20] * [30,40]  # [[10, 30], [10, 40], [20, 30], [20, 40]]
F *(a:Arr, b:Arr)
	collector
		a.each(F(elt_a) {
			b.each(F(elt_b) {
				collect([elt_a, elt_b])
			})
		})

TEST [10,20] * [30,40] == [[10, 30], [10, 40], [20, 30], [20, 40]]

doc Echo non-string. Converts x to string first, using Str()
doc x - Anything but Str
doc %RET - null
doc %EX - echo(1)
F echo(x) {
	guard x is not Str
	echo(Str(x))
}

# Built-in join() can only handle Str elements
doc Join non-strings. Converts a elements to string first, then uses built-in join().
doc Warning - behaviour subject to change!
doc a - Array to join
doc s - Delimiter
doc %RET - Str
doc %EX - [1,2,3].join("::")  # The string 1::2::3
F join(a:Arr, s:Str) {
	guard a.any(X is not Str)
	a.map(Str).join(s)
	# TODO: maybe a.map({if A is Str then A else Str(A)}).join(s)
	# TODO: consider making Str(x) == x
}

doc Used for command line arguments such as ['--vpc-id', my_vpc_id()].nuke_null()
doc If my_vpc_id() is null the whole array is not needed
doc %EX - [1,2].nuke_null()  # [1,2]
doc %EX - [1,null].nuke_null()  # []
F nuke_null(a:Arr) {
	a.any(Null) returns []
	a
}

TEST [1,2].nuke_null() == [1,2]
TEST [1,null].nuke_null() == []

# Can be more efficent I guess
doc Merge sorted arrays.
doc lte - Less-then-or-equal function to use for comparison of items in a and b
doc %RET - Arr
doc %EX - merge_sorted([1,3,10], [0, 7], (<=))  # [0, 1, 3, 7, 10]
F merge_sorted(a:Arr, b:Arr, lte:Fun) {
	ai = Iter(a)
	bi = Iter(b)
	collector
		while ai or bi {
			i = econd {
				not(ai) bi
				not(bi) ai
				lte(ai.peek(), bi.peek()) ai
				true bi
			}
			collect(i.next())
		}
}

TEST merge_sorted([1,3,10], [0, 7], (<=)) == [0, 1, 3, 7, 10]

# Current implemetation: https://en.wikipedia.org/wiki/Merge_sort
# TODO: replacte with Timsort https://en.wikipedia.org/wiki/Timsort
doc Sort an array.
doc lte - Less-then-or-equal function to use for comparison of the items in a
doc %RET - Arr
doc %EX - sort([0,5,3,-1], (<=))  # [-1, 0, 3, 5]
F sort(a:Arr, lte:Fun=(<=)) {
	# XXX: Not copied!
	a.len() <= 1 returns a
	mid = a.len() / 2
	l = a[0..mid]
	r = a[mid..null]
	merge_sorted(sort(l, lte), sort(r, lte), lte)
}

TEST sort([0,5,3,-1], (<=)) == [-1, 0, 3, 5]

doc Sort an array based on attribute value
doc lte - Less-then-or-equal function to use for comparison of the items' attributes
doc %RET - Arr
doc %EX - [{'x': 1}, {'x': 5}, {'x': 3}].sort('x')  # [{'x': 1}, {'x': 3}, {'x': 5}]
F sort(a:Arr, attr:Str, lte:Fun=(<=)) a.sort(F(a, b) lte(a.(attr), b.(attr)))

TEST [{'x': 1}, {'x': 5}, {'x': 3}].sort('x') == [{'x': 1}, {'x': 3}, {'x': 5}]

doc TODO. Is it used even?
F split(a:Arr, delim) {
	ret = []
	cur = []
	a.each(F(elt) {
		if elt == delim {
			ret.push(cur)
			cur = []
		} else {
			cur.push(elt)
		}
	})
	ret.push(cur)
	ret
}


doc Compare arrays. Warning: Hash is used so internal Hash keys comparison is used, not ==
doc %EX - cmp([1,2], [2,3])  # {'add': [3], 'remove': [1]}
F cmp(a:Arr, b:Arr) {
	ah = Hash(a, {true})
	bh = Hash(b, {true})
	{
		'add': bh.filter(X not in ah).keys()
		'remove': ah.filter(X not in bh).keys()
	}
}

TEST cmp([1,2], [2,3]) == {'add': [3], 'remove': [1]}

doc Like cmp() but compare arrays without using Hash so that '==' is used for comparison
doc and not internal '===' for Hash keys. Much slower than cmp().
F compare(a:Arr, b:Arr) {
	{
		'add': b.filter(X not in a)
		'remove': a.filter(X not in b)
	}
}

doc Convert an array to NGS code that would produce the array when executed. Not fully functional yet.
doc %RET - Str
F code(a:Arr) "[" + a.map(code).join(', ') + "]"

# === Hash =======================================


# --- Comparisons ---

doc Compare two Hashes. Hashes must have same keys with same values in same order to return true.
doc %RET - Bool
F ==(a:Hash, b:Hash) Arr(a) == Arr(b)

doc Compare hashes
doc %EX - diff = cmp(current_tags, target_tags)
doc %EX - if (tags = stdlib_aws_tags(diff.add + diff.change)) {
doc %EX - 	r.run('update_tags/add', %(aws ec2 create-tags --resources ${r.id()} --tags $*tags))
doc %EX - }
doc %EX - if (tags = diff.remove / "Key=$X") {
doc %EX - 	r.run('update_tags/remove', %(aws ec2 delete-tags --resources ${r.id()} --tags $*tags))
doc %EX - }
F cmp(a:Hash, b:Hash) {
	{
		'add': b.filter(X not in a)
		'change': b.filter(X in a).filter(F(k, v) a[k] != v)
		'remove': a.filter(X not in b).keys()
	}
}

TEST cmp({"a": 1, "b":2}, {"b": 3, "d": 4}) == {"add":{"d":4}, "change":{"b":3}, "remove":["a"]}

F subset(smaller:Hash, larger:Hash) smaller.all(F(k, v) k in larger and larger[k] == v)

TEST subset({}, {"a": 1})
TEST subset({"a": 1}, {"a": 1})
TEST subset({"a": 1, "b":2}, {"b": 3, "d": 4}) == false
TEST subset({"a": 1}, {"a": 10}) == false

# --- Functional ---

doc Do nothing
doc %RET - null
F nop() null

doc Compose functions
doc %RET - Fun f(g(...))
doc %EX - F reject(something, predicate) {
doc %EX - 	something.filter(not + predicate)
doc %EX - }
F +(f:Fun, g:Fun) {
	F composed_function(*args) {
		f(g(*args))
	}
}

# Make it built in for speed?
doc Iterate a Hash.
doc h - Hash to iterate
doc cb - Function to call with successive keys and values
doc args - Additional arguments for calling cb
doc %RET - h
doc %EX - {"a": 1, "b": 2}.each(F(k, v) echo("$k=$v"))  # Outputs: "a=1" and on the next line "b=2"
F each(h:Hash, cb:Fun) {
	h.keys().each(F(k) cb(k, h[k]))
	h
}

doc Iterate a Hash.
doc h - Hash to iterate
doc cb - Function to call with successive indexes, keys and values
doc args - Additional arguments for calling cb
doc %RET - h
doc %EX - {"a": 1, "b": 2}.each_idx_key_val(F(idx, k, v) echo("[$idx] $k=$v"))
doc %EX - # Outputs: "[0] a=1" and on the next line "[1] b=2"
F each_idx_key_val(h:Hash, cb:Fun) {
	h.keys().each_idx_val(F(idx, k) cb(idx, k, h[k]))
	h
}

doc Map a Hash
doc h - Hash with source keys and values
doc mapper - Function to be called with keys and values from h
doc args - Additional arguments for calling mapper
doc %RET - Arr
doc %EX - {'a': 1, 'b': 2}.map(F(k, v) "${k}-$v")  # ['a-1', 'b-2']
F map(h:Hash, mapper:Fun)
	collector
		h.each(collect + mapper)

TEST {'a': 1, 'b': 2}.map(F(k, v) "${k}-$v") == ['a-1', 'b-2']

doc Map Hash keys. Build new Hash with same values as in h but keys mapped by mapper.
doc h - Source hash
doc mapper - Function to be called with keys
doc args - Additional arguments for calling mapper
doc %RET - Hash
doc %EX - mapk({"a": 1}, F(k) k+"z")  # {"az": 1}
F mapk(h:Hash, mapper:Fun)
	collector/{}
		h % { collect(mapper(A), B) }

TEST mapk({"a": 1}, X+"z") == {"az": 1}

doc Map Hash values. Build new Hash with same keys as in h but values mapped by mapper.
doc h - Source hash
doc mapper - Function to be called with values
doc args - Additional arguments for calling mapper
doc %RET - Hash
doc %EX - LEN = 3
doc %EX - lines_ = read("/usr/share/dict/words").lines()
doc %EX - long_lines = lines_.filter({A.len()>LEN})
doc %EX - prefix_to_lines = long_lines.group(F(line) line[0..LEN])  # {"pfx1": ["pfx1a", "pfx1b", ...], "pfx2": ["pfx2a", "pfx2b", ...], ...}
doc %EX - prefix_to_count = prefix_to_lines.mapv(len)  # {"pfx1": 30, "pfx2": 35, ...}
doc %EX - top = prefix_to_count.Arr().sort(F(a, b) b[1] <= a[1]).Hash()
doc %EX - top .= limit(10)
doc %EX - echo(top)  # Outputs: {con=1219, dis=1001, pro=808, pre=607, com=600, int=543, tra=498, ove=431, per=422, imp=421}
F mapv(h:Hash, mapper:Fun)
	collector/{}
		h % { collect(A, mapper(B)) }

TEST mapv({"a": 1}, X+1) == {"a": 2}


doc Map Hash keys and values. Build new Hash with keys and values mapped by mapper.
doc h - Source hash
doc mapper - Function to be called with keys and values
doc args - Additional arguments for calling mapper
doc %EX - mapkv({"a": 1}, {[A+"zz", B+10]})  # {"azz": 11}
F mapkv(h:Hash, mapper:Fun)
	collector/{}
		h % { collect(*mapper(A, B)) }

TEST mapkv({"a": 1}, {[A+"zz", B+10]}) == {"azz": 11}

doc Filter hash. Build new hash with kev-value pairs selected by predicate.
doc h - Source hash
doc predicate - Test function to be called with one key and one value at a time
doc args - Additional arguments for calling the predicate
doc %EX - {'a': 1, 'b': 2}.filter(F(k, v) k == 'a')  # {'a': 1}
F filter(h:Hash, predicate) {
	p = Pred(predicate)
	collector/{}
		h.each(F(k, v) {
			if p(k, v)
				collect(k, v)
		})
}

TEST {'a': 1, 'b': 2}.filter(F(k, v) k == 'a') == {'a': 1}

doc Filter hash by keys
F filterk(h:Hash, predicate) filter(h, Pred(predicate) + {A})

doc Filter hash by values
F filterv(h:Hash, predicate) filter(h, Pred(predicate) + {B})

doc Check whether all key-value pairs in Hash satisfy the given predicate
doc h - Hash to check
doc predicate - Test function to be called with one key and one value at a time
doc args - Additional arguments for calling the predicate.
doc %RET - Bool
doc %EX - {"x": 1, "y": 2}.all(F(k, v) v is Int)  # true
F all(h:Hash, predicate) {
	p = Pred(predicate)
	arr = h.keys()
	l = arr.len()
	for(i;l)
		not(p(arr[i], h[arr[i]])) returns false
	true
}

TEST {"x": 1, "y": 2}.all(F(k, v) v is Int)


doc Count number of key-value pairs in Hash that satisfy the predicate.
doc h - Hash to check
doc predicate - Test function to be called with one key and one value at a time
doc args - Additional arguments for calling the predicate.
doc %RET - Int
doc %EX - {'a': 1, 'b': 2, 'c': 11}.count(F(k, v) v>10)  # 1
F count(h:Hash, predicate) {
	p = Pred(predicate)
	collector/0
		h.each(F(k, v) {
			if p(k, v)
				collect(1)
		})
}

TEST {'a': 1, 'b': 2, 'c': 11}.count(F(k, v) v>10) == 1

doc Sort Hash by keys
doc %RET - Hash
doc %EX - {'b': 2, 'c': 11, 'a': 1}.sortk()  # {'a': 1, 'b': 2, 'c': 11}
F sortk(h:Hash, lte:Fun=(<=)) h.keys().sort(lte).Hash(h[X])

TEST {'b': 2, 'c': 11, 'a': 1}.sortk() == {'a': 1, 'b': 2, 'c': 11}
TEST {'a': 1, 'b': 2, 'c': 11}.sortk() == {'a': 1, 'b': 2, 'c': 11}

# --- Conversions ---

doc Create a Hash from Arr of Arr[2]
doc arr - Array of Arrays. Each one of the sub-arrays must have exactly two elements.
doc %RET - Hash
doc %EX - Hash([['a', 1], ['c', 3]])  # {'a': 1, 'c': 3}
F Hash(arr:Arr)
	collector/{}
		arr.each(F(pair) {
			collect(*pair)
		})

TEST Hash([['a', 1], ['c', 3]]) == {'a': 1, 'c': 3}

doc TODO. Not sure it's used anywhere. Maybe remove?
F Hash(arr:Arr, attr:Str)
	collector/{}
		arr.each(F(v) collect(v[attr], v))

TEST Hash([{'x': 1}, {'x': 2}], 'x') == {1: {'x': 1}, 2: {'x': 2}}

doc Create a Hash from keys in arr using cb for values calculation
doc arr - Keys of the hash to build
doc cb - Function to be called with one key at a time. Should calculate a value for the given key.
doc %RET - Hash
doc %EX - Hash([1,2], F(x) x*2)  # {1: 2, 2: 4}
F Hash(arr:Arr, cb:Fun)
	collector/{}
		arr.each(F(elt) collect(elt, cb(elt)))

TEST Hash([1,2], F(x) x*2) == {1: 2, 2: 4}


doc Create a Hash from keys in "keys" and corresponding values in "values"
doc keys - Keys for the new Hash
doc values - Values for the new Hash
doc %RET - Hash
doc %EX - Hash(["a", "b", "c"], [1,2,3])  # {"a": 1, "b": 2, "c": 3}
F Hash(keys:Arr, values:Arr)
	collector/{}
		keys.len().each(F(idx) collect(keys[idx], values[idx]))

TEST Hash(["a", "b", "c"], [1,2,3]) == {"a": 1, "b": 2, "c": 3}

doc Create Hash from Arr of something that has key and value attribute
doc arr - Arr with all the keys and values
doc key_attr - Name of the attribute holding the keys of newly created Hash
doc val_attr - Name of the attribute holding the values of newly created Hash
doc %EX - Hash([{"Name": "n1", "Value": "v1"},{"Name": "n2", "Value": "v2"}], "Name", "Value")  # {"n1": "v1", "n2": "v2"}
F Hash(arr:Arr, key_attr:Str, val_attr:Str)
	collector/{}
		arr.each(F(elt) collect(elt.(key_attr), elt.(val_attr)))

TEST Hash([{"Name": "n1", "Value": "v1"},{"Name": "n2", "Value": "v2"}], "Name", "Value") == {"n1": "v1", "n2": "v2"}


# --- Misc ---

doc Filter out specific key
doc e - Source
doc without_k - The key to filter out
doc %EX - {'a': 1, 'b': 2, 'c': 3}.without('a')  # {'b': 2, 'c': 3}
F without(e:Eachable2, without_k) e.filter(X != without_k)

TEST {'a': 1, 'b': 2, 'c': 3}.without('a') == {'b': 2, 'c': 3}

doc Filter out specific key-value pair
doc e - Source
doc without_k - The key to filter out
doc without_v - The value to filter out
doc %EX - {'a': 1, 'b': 2, 'c': 3}.without('a', 1).without('b', 22)  # {'b': 2, 'c': 3}
F without(e:Eachable2, without_k, without_v)
	e.reject(F(k, v) (k == without_k) and (v == without_v))

TEST {'a': 1, 'b': 2, 'c': 3}.without('a', 1).without('b', 22) == {'b': 2, 'c': 3}

doc Add Hashes. Builds new hash with key-value pairs from both a and b. If same key is present in both a and b, the value from b is used.
doc %RET - Hash
doc %EX - {'a': 1, 'b': 2, 'c': 3} + {'b': 20, 'd': 40}  # {'a': 1, 'b': 20, 'c': 3, 'd': 40}
F +(a:Hash, b:Hash) {
	{**a, **b}
}


TEST {'a': 1, 'b': 2, 'c': 3} + {'b': 20, 'd': 40} == {'a': 1, 'b': 20, 'c': 3, 'd': 40}

doc Build array of Str where each resulting string is of the form "KEY=VALUE"
doc h - Source hash
doc %RET - Arr of Str
doc %EX - {'a': 1, 'b': 2}.Strs()  # ['a=1', 'b=2']
F Strs(h:Hash) h / "$X=$Y"

TEST {'a': 1, 'b': 2}.Strs() == ['a=1', 'b=2']

doc Truncate a Hash if necessary so it would have maximum l key-value pairs.
doc h - Source hash
doc l - Maximum elements
doc %RET - Hash
doc %EX - {"a": 1, "b": 2}.limit(1)  # {"a": 1}
F limit(h:Hash, l:Int) h.keys().limit(l).Hash(h[X])

TEST {"a": 1, "b": 2}.limit(3) == {"a": 1, "b": 2}
TEST {"a": 1, "b": 2}.limit(1) == {"a": 1}

doc TODO.
doc Group items from a by key returned by cb
doc args - Additional arguments for calling cb
doc %RET - Hash
F group(a:Arr, cb:Fun) {
	ret = {}
	a.each(F(elt) {
		k = cb(elt)
		ret.dflt(k, []).push(elt)
	})
	ret
}

doc Convert a Hash to NGS code that would produce the given Hash when executed. Not fully functional yet.
doc %RET - Str
F code(h:Hash) {
	'{' + h.map(F(k, v) "${k.code()}: ${v.code()}").join(', ') + '}'
}

doc Set an key in a Hash if it's not already set
doc %EX - Myhash.dflt(k, []).push(elt)
doc %RET - Hash value, the already-existed or new.
F dflt(h:Hash, k, v) {
	if k not in h {
		h[k] = v
	}
	h[k]
}

# === Box ========================================

# Inspired by Option from Scala. http://www.scala-lang.org/api/2.12.x/scala/Option.html

{
	doc A box which might (FullBox) or might not (EmptyBox) contain a value
	type Box
		doc Represents presence of a value
		doc val - The value
		type FullBox(Box)
		doc Represents absence of a value
		type EmptyBox(Box)
}

doc Do not use directly!
doc Helper constructor that throws InvalidArgument when
doc FullBox is created with zero or more than one argument.
F init(b:FullBox, *args) throw InvalidArgument("FullBox must be initialized with exactly one value")

doc FullBox constructor. Saves val into .val
doc %EX - # Simplified code from the_one() method:
doc %EX - ret = EmptyBox()
doc %EX - something.each(F(elt) {
doc %EX - 	if predicate(elt) {
doc %EX - 		ret throws TheOneFail("the_one() had more than one match")
doc %EX - 		ret = FullBox(elt)
doc %EX - 	}
doc %EX - })
doc %EX - not(ret) throws TheOneFail("the_one() had no matches")
doc %EX - ret.val # Has the value
F init(b:FullBox, val) b.val = val

doc Always true
F Bool(fb:FullBox) true

doc Always false
F Bool(eb:EmptyBox) false

doc Zero for FullBox and one for EmptyBox
F to_exit_code(b:Box) b.Bool().to_exit_code()

doc Map FullBox value
doc mapper - mapper to be called with the FullBox value
doc %RET - FullBox with value returned by mapper
F map(fb:FullBox, mapper:Fun) FullBox(mapper(fb.val))

doc Do nothing
doc %RET - eb
F map(eb:EmptyBox, mapper:Fun) eb

doc Call cb with the value of the FullBox
doc %RET - fb
F each(fb:FullBox, cb:Fun) { cb(fb.val); fb }

doc Do nothing
doc %RET - eb
F each(eb:EmptyBox, cb:Fun) eb

doc Test FullBox value
doc predicate - Test function to be called with the value of the FullBox
doc %RET - Box. fb if predicate succeeds, EmptyBox if not.
F filter(fb:FullBox, predicate) { p = Pred(predicate); if p(fb.val) fb else EmptyBox() }

doc Do nothing
doc %RET - eb
F filter(eb:EmptyBox, predicate) eb

doc Test Box value
doc %EX - Box(5).any(Int)      # true
doc %EX - Box(5).any(Str)      # false
doc %EX - EmptyBox().any(Int)  # false
F any(b:Box, predicate) Bool(b.filter(predicate))

TEST Box(5).any(Int)
TEST EmptyBox().any(Str) == false
TEST EmptyBox().any(Int) == false

doc Test Box value
doc %EX - Box(5).none(Int)      # false
doc %EX - Box(5).none(Str)      # true
doc %EX - EmptyBox().none(Int)  # true
F none(b:Box, predicate) not(b.filter(predicate))

TEST Box(5).none(Int) == false
TEST Box(5).none(Str)
TEST EmptyBox().none(Int)

doc Get FullBox value
doc %RET - Any
F get(fb:FullBox, dflt=null) fb.val

doc Get EmptyBox value. Always throws InvalidArgument.
F get(eb:EmptyBox) throw InvalidArgument("Can not get value from EmptyBox")

doc Get EmptyBox value
doc %RET - dflt
F get(eb:EmptyBox, dflt) dflt

doc Convert FullBox to array
doc %RET - Arr of exactly one element, the value of FullBox
F Arr(fb:FullBox) [fb.val]

doc Convert EmptyBox to array
doc %RET - Empty array
F Arr(eb:EmptyBox) []

doc Convert anything to Box (always FullBox)
doc x - value to enclose in a box
doc %RET - FullBox with given value
F Box(x) FullBox(x)

doc Convert null to Box (always EmptyBox)
doc %RET - EmptyBox
F Box(n:Null) EmptyBox()

doc Convert array to a Box. Throws InvalidArgument if length of the array is not 0 or 1.
doc %RET - Box. FullBox if the array has one element, EmptyBox otherwise.
F Box(a:Arr) {
	l = a.len()
	econd {
		l == 0 EmptyBox()
		l == 1 FullBox(a[0])
		true   throw InvalidArgument("Box(a:Arr) only handles arrays of length of 0 or 1")
	}
}

doc Convert array value indexed by the given index
doc idx - key to look in hash
doc %RET - Box. FullBox if the array has the element indexed by idx, EmptyBox otherwise.
F Box(a:Arr, idx) {
	l = a.len()
	l <= idx returns EmptyBox()
	FullBox(a[idx])
}

doc Convert hash value indexed by the given key to a Box
doc k - key to look in hash
doc %RET - Box. FullBox if the hash has the element referenced by k, EmptyBox otherwise.
F Box(h:Hash, k) {
	k not in h returns EmptyBox()
	FullBox(h[k])
}

doc Do nothing
doc %RET - fb
F dflt(fb: FullBox, x) fb

doc Wrap x in a Box
doc %RET - FullBox with the value x
F dflt(eb: EmptyBox, x) FullBox(x)

# === Integer ====================================

# --- Functional ---

doc Iterate from zero up to but not including n
doc cb - Function to call with current number
doc args - Additional arguments for calling cb
doc %EX - 10.each(echo)  # Outputs numbers from 0 to 9 inclusive, one on each line
doc %RET - n
F each(n:Int, cb:Fun) {
	for(i;n) cb(i)
	n
}

TEST 3.map(identity) == [0, 1, 2]
TEST 3.map(X*3) == [0, 3, 6]

doc Call cb n times without arguments.
doc cb - Function to call
doc args - Arguments for calling cb
doc %EX - r=0; 5.times(F() r=r+2);  # r is now 10
F times(n:Int, cb:Fun) {
	for(i;n) cb()
	n
}

TEST r=0; 5.times(F() r=r+2); r==10

doc Convert base-10 string to Int.
doc Throws InvalidArgument if the number in s is not well-formatted.
doc %RET - Int
doc %EX - Int(" 100 ")  # 100
F Int(s:Str) Int(s, 10)

TEST Int(" 100 ") == 100
TEST try Int(" XX ") catch(e:InvalidArgument) true
TEST Int(" XX ", 36) == 1221

doc Convert Bool to int.
doc %RET - Int
doc %EX - true.Int()   # 1
doc %EX - false.Int()  # 0
F Int(b:Bool) if b 1 0

TEST true.Int() == 1
TEST false.Int() == 0

doc Convert an Int to NGS code that would produce the integer when executed. Not fully functional yet.
doc %RET - Str
F code(n:Int) Str(n)


# === Str ========================================

# https://docs.python.org/2/library/string.html
CHARS = ns {
	ascii_lowercase = 'abcdefghijklmnopqrstuvwxyz'
	ascii_uppercase = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
	ascii_letters = ascii_lowercase + ascii_uppercase
	digits = '0123456789'
	hexdigits_uppercase = '0123456789ABCDEF'
	hexdigits_lowercase = '0123456789abcdef'
	octdigits = '01234567'
	base64 = ascii_uppercase + ascii_lowercase + digits + '+/'
	base64url = ascii_uppercase + ascii_lowercase + digits + '-_'
}

# --- Constructors ---

# TODO: escaping?
doc TODO
F Str(s:Str) "'$s'"

doc Convert Null to string
doc %RET - the string "null"
F Str(n:Null) "null"
F Str(b:Bool) if b "true" "false"

doc Convert Arr to string
doc %RET - Str
F Str(a:Arr) "[${a.map(Str).join(',')}]"

doc Convert Hash to string
doc %RET - Str
F Str(h:Hash) "{" + h.map(F(k, v) "${k}=${v}").join(", ") + "}"

# --- Functional ---

doc Iterates over all string characters (currently bytes).
doc TODO: example.
doc cb - Function to be called with each character from s
doc args - Additional arguments for calling cb
doc %RET - s
F each(s:Str, cb:Fun) {
	s.len().each(F(idx) {
		cb(s[idx])
	})
	s
}

# --- Sugar ---

doc Find substring in a string.
doc haystack - The string to search in
doc needle - The string to find
doc %RET - Int or null
doc %EX - pos("abc", "cd")     # null
doc %EX - pos("abcdef", "cd")  # 2
F pos(haystack:Str, needle:Str) pos(haystack, needle, 0)

TEST pos("abc", "cd") == null
TEST pos("abcdef", "cd") == 2

TEST pos("a:b:c", ":") == 1
TEST pos("a:b:c", ":", 2) == 3

doc Determin if needle substring occurs at least once in haystack
doc haystack - The string to search in
doc needle - The string to find
doc %RET - Bool
doc %EX - "bc" in "abcd"
doc %EX - "x" not in "abcd"
F in(needle:Str, haystack:Str) pos(haystack, needle) is not Null

TEST "bc" in "abcd"
TEST "x" not in "abcd"

doc Get given character (currently byte) of the string
doc s - Original string
doc i - Index of the character to return
doc %EX - "abc"[0]  # "a"
F '[]'(s:Str, i:Int) s[i..i+1]

TEST ("abc"[0] == "a") and ("abc"[2] == "c")
TEST try "abc"[3] == "x" catch(e:InvalidArgument) true

doc Get given character (currently byte) of the string.
doc TODO: Document InvalidArgument exception.
doc s - Original string
doc i - Negative index of the character to return
doc %EX - "abc"[-1]  # "c"
F '[]'(s:Str, i:Int) {
	guard i < 0
	i < 0-s.len() throws InvalidArgument("Negative index into string is (abs) larger than the string")
	s[s.len()+i]
}

TEST ("abc"[-1] == "c") and ("abc"[-3] == "a")
TEST try "abc"[-4] == "x" catch(e:InvalidArgument) true

doc Get a substring. Indexes in s are specified by ExclusiveRange.
doc s - Original string
doc r - ExclusiveRange with negatve .end
doc %EX - "(Look ma, no parens)"[1..-1]  # "Look ma, no parens"
doc %RET - Str
F '[]'(s:Str, r:ExclusiveRange) {
	guard r.end is Int and r.end < 0
	r.end < 0 - s.len() throws InvalidArgument("Negative index (range end) into string is (abs) larger than the string")
	s[r.start..s.len() + r.end]
}

# --- Misc utilities ---

doc Split string by substring
doc %RET - Arr of Str
doc %EX - ":a:bc:d:".split(":")   # ["", "a", "bc", "d", ""]
doc %EX - ":a:bc:d:".split("bc")  # [":a:", ":d:"]
F split(s:Str, delim:Str) {
	l = delim.len()
	guard l > 0
	start = 0
	collector {
		while (p = pos(s, delim, start)) is not Null {
			collect(s[start..p])
			start = p + l
		}
		collect(s[start..null])
	}
}

TEST ":a:bc:d:".split(":") == ["", "a", "bc", "d", ""]
TEST ":a:bc:d:".split("bc") == [":a:", ":d:"]

doc Repeat string n times
doc %EX - "abc" * 3  # "abcabcabc"
F *(s:Str, n:Int) ([s]*n).join('')

# Tested with Str(Str,Int)

doc Convert anything to Str of a given width
F Str(x, target_width:Int) x.Str().Str(target_width)

doc Pad a string to given width with spaces
doc target_width - Positive pads on right, negative pads on left
doc %EX - Str("x", 3)  # 'x  '
doc %EX - Str("x",-3)  # '  x'
F Str(s:Str, target_width:Int) {
	l = s.len()
	pad_on_left = target_width < 0
	if pad_on_left {
		target_width = 0 - target_width
	}

	l >= target_width returns s

	padding = ' ' * (target_width - l)
	if pad_on_left {
		padding + s
	} else {
		s + padding
	}
}

# Not sure about this, might be confusing behaviour
doc Convert a number to a string and pad it
doc target_width - Positive pads on left, negative pads on right
doc %EX - Str(10,  4)  # '  10'
doc %EX - Str(10, -4)  # '10  '
F Str(n:Int, target_width:Int) n.Str().Str(0-target_width)

TEST Str(10,  4) == '  10'
TEST Str(10, -4) == '10  '
TEST Str("x", 3) == 'x  '
TEST Str("x",-3) == '  x'

doc Check whether a string starts with another string
doc %RET - Bool
doc %EX - "abcd".starts_with("ab")  # true
doc %EX - "ab".starts_with("abcd")  # false
F starts_with(haystack:Str, needle:Str) {
	needle.len() > haystack.len() returns false
	haystack[0..needle.len()] == needle
}

TEST "abcd".starts_with("ab")
TEST "ab".starts_with("abcd") == false

doc Check whether a string ends with another string
doc %RET - Bool
doc %EX - "abcd".ends_with("cd")  # true
doc %EX - "ab".ends_with("cdab")  # false
F ends_with(haystack:Str, needle:Str) {
	needle.len() > haystack.len() returns false
	l = haystack.len()
	haystack[l - needle.len()..l] == needle
}

TEST "abcd".ends_with("cd")
TEST "ab".ends_with("cdab") == false


doc Case-insensitive LessThan or Equal comparison for strings
doc %RET - Bool
F lte(a:Str, b:Str)
	c_strcasecmp(a, b) <= 0

doc Case sensitive LessThan or Equal comparison for strings
doc %RET - Bool
F '<='(a:Str, b:Str)
	c_strcmp(a, b) <= 0

doc Split s to strings using end-of-line separators.
doc UNIX and Windows line endings supported (Windows - not tested yet).
doc Warning: Max OS <= 9 line separation of CR (\r) is not supported
doc %EX - "xx\nyy".lines()  # %[xx yy]
F lines(s:Str) {
	# Note: Max OS <= 9 line separation of CR (\r) is not supported
	not(s) returns []
	s -= MaybeSfx('\n')
	s.split('\n').map(X - MaybeSfx('\r'))
}

doc Split s to strings using end-of-line separators and call cb for each one of the lines.
doc TODO: More efficient implementation, which would not have temporary array of all lines.
doc cb - Function to be called with each line
doc args - Additional arguments for calling cb
F lines(s:Str, cb:Fun) {
	lines(s).each(cb)
}

TEST "xx\nyy".lines() == %[xx yy]

doc Prepend each line in a with s
doc %EX - "a " + ["1", "2"]  # ["a 1", "a 2"]
F +(s:Str, a:Arr) {
	guard a.all(Str)
	a.map(s+X)
}
TEST "a " + ["1", "2"] == ["a 1", "a 2"]

doc Append s to each line in a
doc %EX - ["1", "2"] + " a"  # ["1 a", "2 a"]
F +(a:Arr, s:Str) {
	guard a.all(Str)
	a.map(X+s)
}
TEST ["1", "2"] + " a" == ["1 a", "2 a"]

doc Truncate a string if necessary so it would have maximum n characters (currently bytes).
doc s - The string to (possibly) truncate.
doc n - Maximum characters
doc marker - The truncation marker
doc %RET - Either s or new Str of length n
doc %EX - "abc".limit(5, "...")     # "abc"
doc %EX - "abcdef".limit(5, "...")  # "ab..."
doc %EX - "abcdef".limit(2)         # "ab"
F limit(s:Str, n:Int, marker:Str='') {
	s.len() <= n returns s
	if marker {
		lim = n - len(marker)
		s[0..lim] + marker
	} else {
		s[0..n]
	}
}

TEST "abc".limit(5, "...") == "abc"
TEST "abcdef".limit(5, "...") == "ab..."
TEST "abcdef".limit(2) == "ab"

doc Get character (currently byte) code. Throws InvalidArgument if s is not of length 1.
doc %EX - ord("A")  # 65 on my machine
F ord(s:Str) {
	s.len() != 1 throws InvalidArgument("ord() argument must be of length 1 exactly").set('given', s)
	ord(s, 0)
}

doc EXPERIMENTAL. For use with AWS Anchors.
F ~(a:Str, b:Str) a==b

doc Convert a Str to NGS code that would produce the string when executed. Not fully functional yet.
doc BUG: Does not do escaping.
doc %RET - Str
F code(s:Str) {
	# XXX TODO: escaping
	"'${s}'"
}


# === Logging and status reporting (WIP) =========

# TODO: timestamps

# TODO: allow control by facility and severity
#       and maybe source component or file.

# Not sure such flexibility is needed (NGS_ERR_FD)
stdlib_error_fd = ENV.NGS_ERR_FD.Int() tor 2

doc Log to standard output. Later log output will be treated specially by the shell. It will have suitable representation in the UI. Use log() when it's semantically a log.
F log(s:Str) {
	echo("[LOG] $s")
}

doc Debug to standard error.
doc "DEBUG" environment variable must be non-empty string to activate. Otherwise nothing is outputted.
doc TODO: Not output thread ID if there is only one thread.
doc TODO: Timestamps?
F debug(s:Str) {
	# TODO: Use RingBuffer when debug is off and
	#       show last N debug messages on uncaught exception
	if try ENV['DEBUG'] {
		echo(stdlib_error_fd, "[DEBUG ${c_getpid()} ${c_pthread_self().id()}] $s")
	}
}

doc Write error message to standard error.
F error(s:Str) {
	echo(stdlib_error_fd, "[ERROR] $s")
}

doc Write error message to standard error.
F warn(s:Str) {
	echo(stdlib_error_fd, "[WARNING] $s")
}

doc Send status to standard error. Later status output will be treated specially by the shell. It will have suitable representation in the UI. Use status() when it's semantically a status - a task that's being done.
F status(s:Str) {
	echo(stdlib_error_fd, "(status) $s")
}

doc Write messages in a to standard error and exit
F die(a:Arr, exit_code=1) {
	guard a.all(Str)
	a % echo(stdlib_error_fd, X)
	c_exit(exit_code)
}

doc Write message in s to standard error and exit
F die(s:Str, exit_code=1) {
	echo(stdlib_error_fd, s)
	c_exit(exit_code)
}

# === Path =======================================

{
	doc Represents file system path
	type Path
	doc Represents a socket file in a file system (S_IFSOCK)
	type SocketFile(Path)  # S_IFSOCK
	doc Represents a link in a file system (S_IFLNK)
	type Symlink(Path)     # S_IFLNK
	doc Represents a file in a file system (S_IFREG)
	type File(Path)        # S_IFREG
	doc Represents a block device in a file system (S_IFBLK)
	type BlockDevice(Path) # S_IFBLK
	doc Represents a directory in a file system (S_IFBLK)
	type Dir(Path)         # S_IFDIR
	doc Represents a character device in a file system (S_IFCHR)
	type CharDevice(Path)  # S_IFCHR
	doc Represents a fifo file in a file system (S_IFIFO)
	type FifoFile(Path)    # S_IFIFO

	doc Represents a failure in stat() system call.
	type StatFail(CException)

}

# TODO: reconsider naming
STDLIB_FILE_TYPE = {
	C_S_IFSOCK: SocketFile
	C_S_IFLNK:  Symlink
	C_S_IFREG:  File
	C_S_IFBLK:  BlockDevice
	C_S_IFDIR:  Dir
	C_S_IFCHR:  CharDevice
	C_S_IFIFO:  FifoFile
}

doc Issue stat() system call. Throws StatFail if the call fails.
doc %RET - Stat
doc %EX - stat("/tmp")  # <Stat st_dev=51714 st_ino=25 st_mode=17407 ...>
F stat(pathname:Str) {
	ret = c_stat(pathname)
	ret is Null throws StatFail("Failed to c_stat()").set('pathname', pathname)
	ret
}

doc Issue stat() system call. Throws StatFail if the call fails.
doc %RET - Stat
F stat(p:Path) super(p.path)

doc Issue lstat() system call. Throws StatFail if the call fails.
doc %RET - Stat
doc %EX - stat("/usr/bin/cal").st_mode   # 33261
doc %EX - lstat("/usr/bin/cal").st_mode  # 41471
F lstat(pathname:Str) {
	ret = c_lstat(pathname)
	ret is Null throws StatFail("Failed to c_lstat()").set('pathname', pathname)
	ret
}

doc Issue lstat() system call. Throws StatFail if the call fails.
doc %RET - Stat
F lstat(p:Path) super(p.path)

# TODO: better name
doc EXPERIMENTAL, do not use outside of stdlib!
doc Instanciates Path sub-type, depending on what p is
F specific(p:Path) {
	s = lstat(p)
	file_type = s.st_mode.band(C_S_IFMT)
	STDLIB_FILE_TYPE[file_type](p.path).set('stat', s)
}

doc Path constructor
doc s - path
doc subtype - Return Path sub-type, depending on lstat() call
doc %RET - Path or sub-type of Path
doc %EX - Path(".")  # <Path path=.>
doc %EX - Path(".", true)  # <Dir path=.>
doc %EX - ``find tmp/v8``.map(Path(X, true)).map(typeof).name.Stats()  # <Stats: {Dir=287, File=9220}>
F Path(s:Str, subtype=false) {
	ret = Path()
	ret.path = s
	if subtype {
		ret.specific()
	} else {
		ret
	}
}

doc Path constructor
doc s - path
F init(p:Path, s:Str) {
	p.path = s
}

doc Path constructor. Duplicates other Path.
doc other - Path to duplicate
F init(p:Path, other:Path) {
	p.path = other.path
}

doc Checks whether the path is accessible using access(2) and F_OK.
doc %RET - Bool
F Bool(p:Path) c_access(p.path, C_F_OK) == 0

TEST Path('/').Bool()
TEST Path('/no-such-file').Bool() == false
TEST Path('/', true) is Dir

# TODO: better exceptions (use correct type, not string, maybe use errno)
# TODO: support platforms with non ":" path delimiter?
doc Finds given binary and returns it's full path.
doc Throws ExecutableNotFound if the binary was not found.
doc Search strategy: "/" in the name of the binary means that given executable_name is a path so it's returned as-is (wrapped in Path).
doc If PATH environment variable exists, the given directories are searched.
doc If PATH is not set built-in value for PATH is used to search: /usr/local/bin:/usr/bin:/bin:/sbin:/usr/sbin
doc %RET - Path
doc %EX - find_in_path("ls")  # <Path path=/bin/ls>
F find_in_path(executable_name:Str) {
	debug("[find_in_path] got ${executable_name}")
	if "/" in executable_name {
		debug("[find_in_path] leaving unprocessed ${executable_name}")
		return Path(executable_name)
	}
	# TODO: rethink default PATH
	path = ENV.get('PATH', '/usr/local/bin:/usr/bin:/bin:/sbin:/usr/sbin').split(':')
	debug('[find_in_path] will search')
	ret = path.map(F(dir) Path(dir / executable_name)).first()
	debug("[find_in_path] ${executable_name} found at ${ret}")
	ret is Null throws ExecutableNotFound().set('path', path).set('name', executable_name)
	ret
}

doc String representation of a Path (or it's sub-type)
doc %RET - Str
doc %EX - find_in_path("ls").Str()  # The string: <Path path=/bin/ls>
F Str(p:Path) "<${p.typeof().name} path=${p.path}>"

doc Concatenate two path parts using STDLIB_PATH_SEP (currently "/") but should be platform-specific later.
doc %RET - Str
doc %EX - mydir="tmp"
doc %EX - mydir / "myfile"  # "tmp/myfile"
F /(a:Str, b:Str) "$a$STDLIB_PATH_SEP$b"

doc Concatenate two path parts using STDLIB_PATH_SEP (currently "/") but should be platform-specific later.
doc %RET - Str
doc %EX - mydir="tmp/"
doc %EX - mydir / "myfile"  # "tmp/myfile"
F /(a:Str, b:Str) {
	guard a and (a[-1] == STDLIB_PATH_SEP)
	"$a$b"
}

doc Concatenate two path parts using STDLIB_PATH_SEP (currently "/") but should be platform-specific later.
doc %RET - Str
doc %EX - mydir="./"
doc %EX - mydir / "myfile"  # "myfile"
F /(a:Str, b:Str) {
	guard (a == '.') or (a == './')
	b
}

doc Concatenate two Path-s using /(a:Str, b:Str)
doc %RET - Path
doc %EX - Path("tmp") / Path("v8")  # <Path path=tmp/v8>
F /(a:Path, b:Path) Path(a.path / b.path)

{ type DirFail(CException) }

doc List directory contents. Warning: "." and ".." are included.
doc Throws DirFail when directory can not be listed.
doc subtype - Select type of returned items: true for Path sub-type, false for Path type
doc %RET - Arr of Path or it's sub-type
doc %EX - dir("tmp/v8", true).filter(File)  # [ ..., <File path=tmp/v8/README.md fd=null>, <File path=tmp/v8/LICENSE.valgrind fd=null>, ... ]
doc %EX - dir("tmp/v8", true).filter(Dir)   # [ ..., <Dir path=tmp/v8/tools>, <Dir path=tmp/v8/infra>, ... ]
F dir(dirname:Str, subtype=false) collector dir(dirname, collect, subtype)

# TODO: recurse ?
# TODO: filter out "." and ".." ?
doc List directory contents and call cb with Path() of each found item. Warning: "." and ".." are included.
doc %RET - unspecified at this time, do not count on it
doc %EX - 's=Stats(); dir("tmp/v8", {s.push(A.typeof().name)}, true); s  # <Stats: {File=23, Dir=16}>'
F dir(dirname:Str, cb:Fun, subtype=false) {
	d = c_opendir(dirname)
	d is Null throws DirFail('Failed to list directory contents').set('dirname', dirname)
	finally()
		body => {
			while entry = c_readdir(d) {
				cb(Path(dirname / entry.d_name, subtype=subtype))
			}
		}
		cleanup => {
			r = c_closedir(d)
			r != 0 throws DirFail('Failed to close directory after listing').set('dirname', dirname)
		}
}

# XXX: might not be correct on all platforms
TEST dir("/").path.has("/var")
TEST dir("/").all({A.typeof().name == 'Path'})
TEST dir("/", subtype=true).all({ (A is Path) and (A.typeof().name != 'Path') })

doc Work in progress, do not use!
F glob(pattern:Str, start_dir='.') {


}

# === Commands and processes (WIP) ===============

# TODO: better exceptions (use correct type, not string, maybe use errno)
F dup2(oldfd:Int, newfd:Int) {
	while true {
		status = c_dup2(oldfd, newfd)
		status != -1 returns newfd
		get_c_errno() != C_EINTR throws "dup2() failed"
	}
}

# TODO: Use some thread-local copy of the environment?
#       This would allow simple handling of modifications
#       for exec'ed processes.

# TODO: detect and process errors, throw exception(s)
# TODO: better exceptions (use correct type, not string, maybe use errno)
doc Read all data from a file referenced by file descriptor without parsing it.
doc fd - File descriptor to read from
doc %RET - Str
doc %EX - read(0)  # All data from stdin
F read(fd:Int) {
	ret = []
	while true {
		debug("[read] will read from fd $fd")
		result = c_read(fd, 4096)
		debug("[read] read from fd $fd : <<${result}>>")
		result[0] == 0 returns ret.join('')
		if result[0] > 0 {
			ret.push(result[1])
		} else {
			e = get_c_errno()
			if e != C_EINTR {
				debug("[read] read from fd $fd failed, errno $e")
				throw "[read] failed to c_read()"
			}
		}
	}
}

doc Read all data from a file referenced by file descriptor 0 (stdin) without parsing it. Same as read(0)
F read() read(0)

doc Write data to a file referenced by file descriptor.
doc WARNING: Incomplete implementation.
doc TODO: handle errors, throw exceptions.
doc TODO: handle possible EINTR.
doc fd - File descriptor to write to
doc %RET - unspecified at this time, do not count on it
F store(s:Str, fd:Int) fd.c_write(s)

{ type TtyCheckFail(CException) }

doc Check whether given fd represents a TTY. Uses ISATTY(3).
doc Throws TtyCheckFail.
doc %RET - Bool
F isatty(fd:Int) {
	ret = c_isatty(fd)
	errno = get_c_errno()

	ret == 1 returns true
	ret != 0 throws TtyCheckFail("c_isatty($fd) - unknown response code $ret").set('fd', fd).set('isatty_returned', ret)

	errno in [C_EINVAL, C_ENOTTY] returns false
	throw TtyCheckFail("c_isatty() - failed").set('fd', fd)
}

# --- Pipe ---

# http://www.microhowto.info/howto/capture_the_output_of_a_child_process_in_c.html

{
	doc Wrapper around file descriptors returned by PIPE(2)
	type Pipe
	doc Represents failure of creating a Pipe when calling PIPE(2)
	type PipeCreateFail(CException)
}

# TODO: better exceptions (use correct type, not string, maybe use errno)
doc Pipe constructor. Create UNIX pipe using PIPE(2). Throws PipeCreateFail if pipe can not be created.
doc %EX - p = Pipe()
F init(pipe:Pipe) {
	result = c_pipe()
	# result: [result_code, read_end, write_end]
	result[0] != 0 throws PipeCreateFail("Failed to create Pipe")
	pipe.read_fd = result[1]
	pipe.write_fd = result[2]
	debug("[Pipe] created pipe with read_fd=${pipe.read_fd} and write_fd=${pipe.write_fd}")
}

doc Pipe constructor. Create UNIX pipe using PIPE(2). Throws PipeCreateFail if pipe can not be created.
doc direction - One of: STDLIB_CHILD_TO_PARENT_PIPE_DIR or STDLIB_PARENT_TO_CHILD_PIPE_DIR
doc %EX - p = Pipe(STDLIB_PARENT_TO_CHILD_PIPE_DIR)
F init(pipe:Pipe, direction:Int) {
	init(pipe)
	init(args())
}

F Str(p:Pipe) "<Pipe read_fd=${p.read_fd} write_fd=${p.write_fd}>"

# TODO: exception on fail, maybe close() wrapper for c_close()

doc Close writing end of the Pipe
F close_writing_end(p:Pipe) p.write_fd.c_close()

doc Close reading end of the Pipe
F close_reading_end(p:Pipe) p.read_fd.c_close()

doc Write to Pipe. TODO: document if it throws.
doc %RET - unspecified at this time, do not count on it
F store(s:Str, p:Pipe) p.write_fd.c_write(s)

doc Read from Pipe without parsing. TODO: document if it throws.
doc %RET - read data
doc %EX - data = read(myipe)
F read(p:Pipe) p.read_fd.read()

doc DUP2(2) writing file descriptor
F dup2_writing_end(p:Pipe, newfd:Int) dup2(p.write_fd, newfd)

doc DUP2(2) reading file descriptor
F dup2_reading_end(p:Pipe, newfd:Int) dup2(p.read_fd, newfd)

doc Get child process end file descriptor of Pipe. It's the writing end for Pipe(STDLIB_CHILD_TO_PARENT_PIPE_DIR).
F child_fd(p:Pipe) {
	guard p.direction == STDLIB_CHILD_TO_PARENT_PIPE_DIR
	p.write_fd
}

doc Get child process end file descriptor of Pipe. It's the reading end for Pipe(STDLIB_PARENT_TO_CHILD_PIPE_DIR).
F child_fd(p:Pipe) {
	guard p.direction == STDLIB_PARENT_TO_CHILD_PIPE_DIR
	p.read_fd
}

doc Get parent process end file descriptor of Pipe. It's the reading end for Pipe(STDLIB_CHILD_TO_PARENT_PIPE_DIR).
F parent_fd(p:Pipe) {
	guard p.direction == STDLIB_CHILD_TO_PARENT_PIPE_DIR
	p.read_fd
}

doc Get parent process end file descriptor of Pipe. It's the writing end for Pipe(STDLIB_PARENT_TO_CHILD_PIPE_DIR).
F parent_fd(p:Pipe) {
	guard p.direction == STDLIB_PARENT_TO_CHILD_PIPE_DIR
	p.write_fd
}

# --- File ---

{ type FileIOFail(CException) }

F init(f:File, path:Str) {
	init(args())
	f.fd = null
}

F init(f:File, fd:Int) init(args())

doc String representation of File
doc %EX - open(File("tmp/v8/LICENSE"), "r").Str().echo()  # Output: <File path=tmp/v8/LICENSE fd=4>
F Str(f:File) "<File path=${f.path} fd=${f.fd}>"

TEST File('xy').Str() == '<File path=xy fd=null>'

doc Open a file and set the "fd" field to the file descriptor. Uses OPEN(2).
doc Throws InvalidArgument if file is already open.
doc Throws FileIOFail if an underlying error occurs.
doc flags - Currently "r", "w" or "a" for read/write/append
doc %RET - f
F open(f:File, flags:Str) {
	f.fd is not Null throws InvalidArgument("open(f:File, flags:Str): file is already open").set('file', f)
	fd = c_open(f.path, flags)
	fd <= 0 throws FileIOFail("Failed to open the file ${f.path}").set('file', f)
	f.fd = fd
	f
}

# XXX: might not be correct on all platforms
TEST open(File("/etc/passwd"), "r").fd > 0

doc Get basename of the file. Uses BASENAME(1).
F basename(s:Str) $(basename $s).Str() - Sfx('\n')

# TODO: something more efficient than reading all of it. Some kind of streaming.
doc Iterate over lines of the file
doc f - Closed File
doc cb - function to call with successive lines from the file
F lines(f:File, cb:Fun) {
	finally()
		body => {
			f.open('r')
			f.fd.read().lines(cb)
		}
		cleanup => {
			f.close()
		}
}

# XXX: might not be correct on all platforms
TEST ok = false; File("/etc/passwd").lines({if ":" in A ok = true}); ok

F lines(f:File) collector f.lines(collect)

F lines(f:File, lines_:Arr) {
	finally()
		body => {
			f.open('w')
			(lines_ + STDLIB_EOL).each(store(X, f.fd))
		}
		cleanup => {
			f.close()
		}

}

doc Close a file and sets the "fd" field to null. Uses CLOSE(2).
doc Throws InvalidArgument if file is not open.
doc Throws FileIOFail if an underlying error occurs.
doc flags - Currently "r", "w" or "a" for read/write/append
doc %RET - f
F close(f:File) {
	f.fd is Null throws InvalidArgument("File is not open. close() expects an open file").set('file', f)
	ret = c_close(f.fd)
	ret < 0 throws FileIOFail("Failed to close the file $f").set('file', f)
	f.fd = null
}

doc read() and parse() the given file
F fetch(filename:Str, parse_hints:Hash={}) {
	data = read(filename)
	parse(data, {'filename': filename} + parse_hints)
}

doc Read standard input and parse() it, passing "source" equals "stdin" hint.
F fetch(parse_hints:Hash={}) read(0).parse({'source': 'stdin'} + parse_hints)

# --- Redir ---

doc TODO
F Str(r:Redir) "<Redirect ${r.fd} ${r.marker} ${r.datum}>"

# --- Process ---

{
	doc Running or finished process
	type Process
	
	doc Occurs when executable was not found in path or when executable was absolute or relative and was not found.
	type ExecutableNotFound(Exception)
}

doc Get process "stdout"
doc attr - "stdout"
doc %RET - Str
F '.'(p:Process, attr:Str) {
	guard attr == 'stdout'
	p.outputs[1]
}

doc Set process "stdout"
doc attr - "stdout"
doc %RET - Unspecified, do not count on this value
F '.='(p:Process, attr:Str, v) {
	guard attr == 'stdout'
	p.outputs[1] = v
}

doc Get process "stderr"
doc attr - "stderr"
doc %RET - Str
F '.'(p:Process, attr:Str) {
	guard attr == 'stderr'
	p.outputs[2]
}

doc Set process "stderr"
doc attr - "stderr"
doc %RET - Unspecified, do not count on this value
F '.='(p:Process, attr:Str, v) {
	guard attr == 'stderr'
	p.outputs[2] = v
}

doc Decide whether process finished normally: exit code must be 0.
doc p - any process
doc %RET - bool
F finished_ok(p:Process) p.exit_code == 0

doc Decide whether /bin/false process finished normally: exit code must be 1.
doc p - /bin/false process
doc %RET - bool
F finished_ok(p:Process) {
	guard p.executable.path in ['/bin/false', '/usr/bin/false']
	p.exit_code == 1
}

doc Decide whether specific process finished normally: exit code must be 0 or 1.
doc p - One of the processes: /usr/bin/test, /bin/fuser, /bin/ping
doc %EX - # Exit code 1 on the line below but no exception thanks to this finished_ok().
doc %EX - if $(test -f /) echo("/ is a file")  # Outputs nothing, / is a directory
doc %RET - bool
F finished_ok(p:Process) {
	guard p.executable.path in ['/usr/bin/test', '/bin/fuser', '/usr/bin/fuser', '/bin/ping', '/usr/bin/ping']
	p.exit_code in [0, 1]
}

F finished_ok(p:Process) {
	guard p.command.options.get('nofail')
	true
}

doc Process constructor
F init(p:Process, c:Command) {
	p.command = c
	p.executable = null
	p.pid = null
	p.exit_code = null
	p.exit_signal = null
	p.pipes = {}
	p.outputs = {1:null, 2:null}
	p.reading_threads = []
	p.writing_threads = []
	# p.lock = Lock()
}

doc Wait for process to finish and see whether exit code is 0
doc %RET - Bool
F Bool(p:Process) {
	p.wait()
	p.exit_code == 0
}

# TODO: better exceptions (use correct type, not string)
doc Wait for the process to finish. Waits for reading and writing threads to finish first.
doc %RET - p
F wait(p:Process) {

	p.pid is Null throws InvalidArgument("Can not wait() on Process without PID")

	p.exit_code is Int returns p

	debug("[wait] joining reading and writing threads")
	p.reading_threads.join()
	p.reading_threads = []
	p.writing_threads.join()
	p.writing_threads = []

	debug("[wait] will waitpid(${p.pid})")
	w = c_waitpid(p.pid)
	debug("[wait] waitpid(${p.pid}) -> ${w}")
	p.exit_code = C_WEXITSTATUS(w[1])
	p.exit_signal = C_WTERMSIG(w[1])
	not(finished_ok(p)) throws ProcessFail(p)
	p
}

doc Convert Process to exit code. Waites for the process to finish and uses its exit code.
F to_exit_code(p:Process) p.wait().exit_code

doc Wait for the process to finish and return its standard output.
doc %RET - Str
F Str(p:Process) p.wait().stdout

# --- $() ---

# TODO: better exceptions (use correct type, not string)
# TODO: c_waitpid() - handle signals
# TODO: split to methods for easy behaviour modification
# TODO: capture stderr
# TODO: provide additional communication channel on additional fd?
# TODO: handle built-in commands: do not execve,
# INFO: bash - redir.c
doc Execute an external command. Waits for the command to finish unless "&" is specified.
doc c - Command type instance, typically constructed by NGS when parsing the contents of $(...) .
doc %EX - if $(test -f /myfile) { ... }
doc %EX - # or
doc %EX - p = $(node main.js serve &)
doc %EX - YOUR_TESTS_OF_NODE_SERVER_HERE
doc %EX - p.kill()
doc %RET - Process
F '$()'(c:Command) {

	# Special case: $($cmd)
	if (c.argv.len() == 1) and (c.argv[0] is Command) {
		c = c.argv[0]
	}

	debug_pfx = '[stdlib $()]'
	debug("$debug_pfx Parsed command: ${c.argv}")

	process = Process(c)
	try {
		process.executable = find_in_path(c.argv[0])
	} catch(nf:ExecutableNotFound) {
		throw ProcessFail(process).set('cause', nf).set('message', 'Executable not found')
	}
	# child fd -> Pipe
	pipes = process.pipes
	if not(c.options.get('top_level')) {
		pipes[1] = Pipe(STDLIB_CHILD_TO_PARENT_PIPE_DIR)
	}

	c.redirects.each(F(r:Redir) {
		# WIP. Continue here: support functions and Pipe objects
	})

	pipes % { process.outputs[A] = null }

	pid = c_fork()

	pid == -1 throws "Failed to fork()"

	debug("$debug_pfx PID after fork: $pid")

	child_fds = pipes / child_fd(Y)
	parent_fds = pipes / parent_fd(Y)

	if pid == 0 {
		# === Child ===
		# TODO: -------------------- test - start --------------------

		for(i=3;i<100;i+=1) {
			if i not in child_fds {
				i.c_close()
			}
		}
		# TODO: -------------------- test - end --------------------
		parent_fds % c_close
		pipes % F(fd, pipe) {
			dup2(pipe.child_fd(), fd)
		}
		# Redirections (WIP) - start
		c.redirects.each(F(r:Redir) {

			F fd_or_dflt(fd:Int, dflt) fd
			F fd_or_dflt(fd:Null, dflt) dflt

			eswitch r.marker {
				'<'  { flags = 'r'; target_fd = fd_or_dflt(r.fd, 0); }
				'>'  { flags = 'w'; target_fd = fd_or_dflt(r.fd, 1); }
				'>>' { flags = 'a'; target_fd = fd_or_dflt(r.fd, 1); }
			}

			# TODO: Maybe support File or some other datum types later?
			if r.datum is Str {
				f = File(r.datum).open(flags)
				ok = c_dup2(f.fd, target_fd)
				# TODO: better exception
				ok < 0 throws "Failed to dup2(${f.fd}, ${target_fd}). Errno ${get_c_errno()}"
			}
		})
		# Redirections (WIP) - end
		result = c_execve(process.executable.path, c.argv, ENV.Strs())
		# TODO: propagate this error to parent, don't know how yet
		throw "$debug_pfx failed to execve()"
	}

	if pid > 0 {
		# === Parent ===
		process.pid = pid
		child_fds % c_close
		debug("$debug_pfx reading all output of the child process")
		# XXX: maybe make it chunk based with c_poll() and single thread
		pipes_to_read_from = pipes ? F(fd, pipe) { pipe.direction == STDLIB_CHILD_TO_PARENT_PIPE_DIR }
		# TODO: some error handling inside these threads and after joining them
		process.reading_threads = pipes_to_read_from.map(F(fd, pipe) {
			Thread({
				process.outputs[fd] = pipe.read()
				pipe.close_reading_end()
				debug("$debug_pfx read all output of the child process for descriptor $fd, closing reading/parent end")
			})
		})
		# echo("R ${c.reading_threads}")
		if not(c.options.get('&')) {
			process.wait()
		}
	}

	# TODO: pid < 0 => throw exception with errno

	process
}

TEST $(true).Bool()
TEST $(false).Bool() == false
TEST "$(/bin/echo -n abc)" == 'abc'

doc Enables "nofail" command prefix.
doc %EX - test
doc %EX - $(nofail ./NONE).exit_code.echo()  # Would cause exception without "nofail"
F '$()'(c:Command) {
	guard c.argv[0] == 'nofail'
	warn("Using deprecated 'nofail'")
	c.options.nofail = true
	c.argv.shift()
	$($c)
}


doc Get command standard output. Similar to bash.
F ``(c:Command) ('$()')(c).Str()

doc Get command standard output and parse() it.
doc %RET - structured data when the output can be parsed.
doc %EX - ``aws ec2 describe-instances`` is Arr  # true
doc %EX - ``aws ec2 describe-instances``.InstanceId.map(X[0..3]).join(",")  # i-3, i-9, i-8, i-a, i-7, ...
F ````(c:Command) {
	p = ('$()')(c)
	p.stdout.parse({'process': p})
}

doc Send signal to a process. Uses KILL(2). Throws KillFail on error.
doc sig - Signal to send. Defaults to SIGNALS.TERM
doc %RET - unspecified at this time, do not count on it
F kill(pid:Int, sig:Int=SIGNALS.TERM) {
	ret = c_kill(pid, sig)
	ret == -1 throws KillFail("Failed to kill pid $pid with signal $sig")
	ret != 0 throws Error("c_kill() did not return 0 or -1")
	ret
}

# XXX: Must join reader/writer threads!
doc Send signal to a Process. Uses KILL(2).
doc Throws InvalidArgument if Process does not have "pid".
doc Throws KillFail on error.
doc sig - Signal to send. Defaults to SIGNALS.TERM
doc %RET - unspecified at this time, do not count on it
F kill(p:Process, sig:Int=SIGNALS.TERM) {
	p.pid is Null throws InvalidArgument("Can't kill() a process that does not have a pid")
	kill(p.pid, sig)
}

doc Call parse(s, {})
F parse(s:Str) parse(s, {})

doc Attempt to parse JSON. Uses decode_json().
F parse(s:Str, hints:Hash) {
	ret = FullBox(decode_json(s)) tor EmptyBox()
	guard ret
	ret.val
}

# TODO: handle more than one occurrence of kvsep in each line
doc EXPERIMENTAL. KVS (key-value separator) hint for parse()
doc %EX - parse('a b\nc d', {'KVS': ' '}) == {'a': 'b', 'c': 'd'}
F parse(s:Str, hints:Hash) {
	guard 'KVS' in hints
	s.lines().map(split(X, hints.KVS)).Hash()
}

TEST parse('a b\nc d', {'KVS': ' '}) == {'a': 'b', 'c': 'd'}

doc EXPERIMENTAL. FS (field separator) hint for parse()
doc %EX - parse('a b\nc d', {'FS': ' '})  # [['a', 'b'], ['c', 'd']]
F parse(s:Str, hints:Hash) {
	guard 'FS' in hints
	s.lines().map(split(X, hints.FS))
}

TEST parse('a b\nc d', {'FS': ' '}) == [['a', 'b'], ['c', 'd']]

doc Handle fields_names hint - run parse() and make Arr[Hash] from Arr[Arr] using provided fields_names
doc %EX - backup_list = fetch('backup.list', {'FS': ' ', 'fields_names': %[env role]})
F parse(s:Str, hints:Hash) {
	guard 'fields_names' in hints
	parse(s, hints.without('fields_names')).map(F(row) {
		zip(hints.fields_names, row).Hash()
	})
}

# WIP
doc Not implemented yet
F unparse(data, hints:Hash) {
	1
}

doc Returns c, wihout any processing. Convenient way to pass ready-to-run Command as an argument
doc %EX - F run_when_needed(c:Command) { ... $(c) ... }
doc %EX - ...
doc %EX - run_when_needed(%(ls >/tmp/my_ls))
F '%()'(c:Command) c

doc Wait for the process and return lines of its stdout.
doc %RET - Arr of Str
doc %EX - $(seq 10 2 20).lines().map(Int)  # [10,12,14,16,18,20]
F lines(p:Process) {
	p.wait()
	p.stdout.lines()
}

# TODO: ability to handle running process & its streaming stdout
doc Iterate lines of Process' stdout, calling cb with successive lines.
doc Warning: current implementation waits for the process to finish and accumulates all its stdout.
F lines(p:Process, cb:Fun) {
	p.wait()
	p.stdout.lines(cb)
}

# === Parse aws output ===========================
# TODO: move to separate auto-loaded file maybe?

doc Convert "Tags" array in each element of AWS resources (typically returned by AWS CLI) into Hash.
doc Makes "Tags" much more usable.
F stdlib_aws_straighten_tags(a:Arr) a.each(stdlib_aws_straighten_tags)

doc Convert "Tags" array in the given AWS resource (typically returned by AWS CLI) into Hash.
doc %EX - t = {"Tags": [{"Key": "k", "Value": "v"},{"Key": "k2", "Value": "v2"}]}  # Not very convenient, is it?
doc %EX - t.stdlib_aws_straighten_tags()
doc %EX - t  # {Tags={k=v, k2=v2}}
F stdlib_aws_straighten_tags(h:Hash) {
	if 'Tags' in h {
		h.Tags = Hash(h.Tags.Key, h.Tags.Value)
	}
}

# For the case of { "SecurityGroups": [ "sg-XXXXXXXX" ] }
# Which is output of aws elb apply-security-groups-to-load-balancer --load-balancer-name SOMETHING -beame-servers-authz --security-groups sg-XXXXXXXX
doc A no-op
F stdlib_aws_straighten_tags(s:Str) null

TEST instances = [{"Tags": [{"Key": "k", "Value": "v"},{"Key": "k2", "Value": "v2"}]}]; stdlib_aws_straighten_tags(instances); instances.Tags == [{"k": "v", "k2": "v2"}]
TEST instances = ["something"]; stdlib_aws_straighten_tags(instances); instances == ["something"]

doc Parse the output of "aws" command. Extracts the array (see "s" below"). For "describe-instances", flattens the instance list.
doc s - Str containing JSON with a Hash at top level with exactly one Arr as value
doc hints - hints.process.command.argv[0] must be 'aws'
doc %RET - data structure, tyically an Arr at top level. If s is empty string - null.
F parse(s:Str, hints:Hash) {
	guard try hints['process'].command.argv[0] == 'aws'

	# aws ec2 create-tags/delete-tags
	s == '' returns null

	data = decode_json(s)
	guard data is Hash

	# Use the only top-level key that has array as it's value
	# Thanks goes to Elastic Beanstalk which complicates this,
	# all other APIs seem to return only one top-level key.
	# ( aws elasticbeanstalk describe-configuration-options )

	top_level_vals = data.values().filter(Arr)

	guard top_level_vals.len() == 1

	data = top_level_vals[0]

	# Anyone cares about Reservations? Certainly it's not the common case so...
	if data.all('ReservationId' in X) {
		data = data.Instances.flatten()
	}
	data.stdlib_aws_straighten_tags()
}

doc Parse the output of "find" command which does not use "-printf". Handles "-print0".
doc hints - hints.process.command.argv[0] must be 'find'
doc %RET - Arr of Str
F parse(s:Str, hints:Hash) {
	guard try hints['process'].command.argv[0] == 'find'
	argv = hints['process'].command.argv
	guard '-printf' not in argv
	zero_sep = '-print0' in argv
	if zero_sep then s.split(chr(0))[0..-1] else s.lines()
}

# === config =====================================

STDLIB_CONFIG = {}

doc Get configuration for the given key. Lookup order / first wins: (1) Environment variable NGS_$k (2) previously set config(k, v)
doc k - Configuration key
doc %RET - Any. If data found in environment variable, it will be parse()d or returned as Str if parse() fails.
doc %EX - conf = config("table_${t.name}")
F config(k:Str) {
	# TODO: think, organize
	#       what about structured data? JSON?
	# TODO: read from config files
	# TODO: read from command line switches
	debug("[config] getting key '$k'")
	v = try ENV["NGS_$k"]
	if v {
		debug("[config] Value config for key '$k' found in the environment variable 'NGS_${k}': '$v'")
		return parse(v) tor v
	}
	v = try STDLIB_CONFIG[k]
	if v is not Null {
		debug("Value config for key '$k' found in STDLIB_CONFIG: $v")
		return v
	}
	v
}

doc Set configuration. To be used with config(k:Str).
doc %EX - config('table_Instances', %[InstanceId tag_Name tag_env tag_role IPs InstanceType State KeyName SecurityGroups AZ RestOfTags])
F config(k:Str, v) {
	debug("Configuring ${k}=${v}")
	STDLIB_CONFIG[k] = v
}

# === Match, Pfx, Sfx ============================

{
	doc Successful or unsuccessful match result
	type Match

		doc Successful match result
		doc matches - Successful matches
		type MatchY(Match)

		doc Unsucessfull match result
		type MatchN(Match)

	doc Matching failure
	type MatchFail(Exception)

	type SubSeq
		doc Prefix subsequence of a sequence
		type Pfx(SubSeq)
			doc Prefix that might or might not be present for ~ matching to succeed
			type MaybePfx(Pfx)
			doc Prefix that must be present for ~ matching to succeed
			type MustPfx(Pfx)
		doc Suffix subsequence of a sequence
		type Sfx(SubSeq)
			doc Suffix that might or might not be present for ~ matching to succeed
			type MaybeSfx(Sfx)
			doc Suffix that must be present for ~ matching to succeed
			type MustSfx(Sfx)
}

doc Successful match constructor
F init(my:MatchY, matches:Arr) init(args())

doc %RET - true
F Bool(my:MatchY) true

doc %RET - false
F Bool(my:MatchN) false

doc Convert Match to exit code.
doc %RET - 0 for successful match (MatchY), 1 for unsuccessful match (MatchN).
F to_exit_code(m:Match) m.Bool().to_exit_code()

F init(mf:MatchFail, msg:Str, container, pattern) init(args())

doc SubSec construcor
F init(s:SubSeq, val) s.val = val

# The common case
doc Convenience method for creating MustPfx
doc %RET - MustPfx
F Pfx(val) MustPfx(val)

doc Convenience method for creating MustSfx
doc %RET - MustSfx
F Sfx(val) MustSfx(val)

F Str(s:SubSeq) "<${s.typeof().name} ${s.val}>"

doc Return string without the prefix. Throws MatchFail if pfx is MustPfx but s does not start with it.
doc s - original string
doc pfx - prefix to get rid of
doc %RET - s without pfx
doc %EX - "abcde" - Pfx("ab")  # "cde"
doc %EX - "abcde" - Pfx("xy")  # MatchFail exception
doc %EX - "abcde" - MaybePfx("xy")  # "abcde"
F -(s:Str, pfx:Pfx) {
	s.starts_with(pfx.val) returns s[pfx.val.len()..null]
	pfx is MustPfx throws MatchFail("Given string '$s' does not start with prefix '$pfx.val'", s, pfx)
	s
}

TEST "abc" - Pfx("a") == "bc"
TEST try "abc" - Pfx("wa") catch(mf:MatchFail) true
TEST "abc" - MaybePfx("wa") == "abc"


doc Return string without the suffix. Throws MatchFail if pfx is MustSfx but s does not end with it.
doc s - original string
doc sfx - suffix to get rid of
doc %RET - s without pfx
doc %EX - "abcde" - Sfx("de")  # "abc"
doc %EX - "abcde" - Sfx("xy")  # MatchFail exception
doc %EX - "abcde" - MaybeSfx("xy")  # "abcde"
F -(s:Str, sfx:Sfx) {
	s.ends_with(sfx.val) returns s[0 .. (s.len() - sfx.val.len())]
	sfx is MustSfx throws MatchFail("Given string '$s' does not end with suffix '$sfx.val'", s, sfx)
	s
}

TEST "abc" - Sfx("c") == "ab"
TEST try "abc" - Sfx("cd") catch(mf:MatchFail) true
TEST "abc" - MaybeSfx("cd") == "abc"

doc Check whether s starts with pfx.
doc %EX - "abcde" ~ Pfx("ab")  # <MatchY matches=['ab'] before= after=cde>
doc %EX - "abcde" ~ Pfx("xy")  # <MatchN >
doc %RET - Match
F ~(s:Str, pfx:Pfx) {
	not(s.starts_with(pfx.val)) returns MatchN()
	ret = MatchY([pfx.val])
	ret.before = ''
	ret.after = s[pfx.val.len()..null]
	ret
}

doc Check whether s ends with pfx.
doc %EX - "abcde" ~ Sfx("de")  # <MatchY matches=['de'] before=abc after=>
doc %EX - "abcde" ~ Pfx("xy")  # <MatchN >
doc %RET - Match
F ~(s:Str, sfx:Sfx) {
	not(s.ends_with(sfx.val)) returns MatchN()
	ret = MatchY([sfx.val])
	ret.before = s[0..(s.len() - sfx.val.len())]
	ret.after = ''
	ret
}

# TODO: make it a low priority method implementation
doc Convenience method to access matches in MatchY
doc %EX - ("abc" ~ /a(.)c/)[1]  # "b"
F '[]'(my:MatchY, idx:Int) my.matches[idx]

TEST ("abc" ~ /a(.)c/)[1] == "b"

F Pred(s:SubSeq) F subseq_pred(x) x ~ s


# === RegExp =====================================

# TODO: better exceptions
doc Find PCRE regular expression in s. Empty string without options returns MatchN. Throws Error if more than 20 captures are used or if there is an error during matching.
doc offset - search start offset
doc options - options to pass to underlying PCRE_EXEC(3).
doc %RET - Match
doc %EX - globals().keys().filter(/^C_PCRE/)  # lists possible options
doc %EX - "xabcy" ~ /a(.)c/  # <MatchY matches=['abc','b'] named={} positions=[[1,4],[2,3]] whole=abc before=x after=y>
doc %EX - m = ("xabcy" ~ /a(?P<mychar>.)c/)
doc %EX - echo(m.named.mychar)  # Outputs: b
F ~(s:Str, r:RegExp, offset:Int=0, options:Int=0) {
	not(options) and not(s) returns MatchN()
	result = c_pcre_exec(r, s, offset, options)
	if result is Int {
		result == C_PCRE_ERROR_NOMATCH returns MatchN()
		result == 0 throws Error("Too many captures. Max 20")
		result < 0 throws Error("RegExp matching error in c_pcre_exec").set('code', result)
		assert(false, "Unexpected integer result from c_pcre_exec")
	}
	assert(result is Arr, "c_pcre_exec must return either integer or an array")
	ret = MatchY(
		result.map(2, F(start_pos, end_pos) {
			if start_pos >= 0
				s[start_pos..end_pos]
			else
				null
		})
	)
	ret.named = r.names.mapv(F(capture_num) try ret.matches[capture_num])
	ret.positions = result.map(2, F(start_pos, end_pos) [start_pos, end_pos])
	ret.whole = ret.matches[0]
	ret.before = s[0..result[0]]
	ret.after = s[result[1]..null]
	ret
}

TEST m = "xabcy" ~ /a(.)c/; m.matches == ["abc", "b"] and m.before == 'x' and m.after == 'y' and m.whole == 'abc'
TEST m = "xAbcy" ~ /a(.)c/; m is MatchN

PCRE_NEWLINE_TO_OPTIONS = {
	         13: C_PCRE_NEWLINE_CR,
	         10: C_PCRE_NEWLINE_LF,
	13*256 + 10: C_PCRE_NEWLINE_CRLF,
	         -2: C_PCRE_NEWLINE_ANYCRLF,
	         -1: C_PCRE_NEWLINE_ANY,
}

doc Find all non-overlapping matches of regular expression in a string.
doc %RET - if collect_unmatched - Arr with each element being MatchY or Str, if not collect_unmatched - Arr of MatchY
doc %EX - ("x10ab20c30y" ~~ /[0-9]+/).whole.map(Int).sum()  # 60
doc %EX -
doc %EX - arr = (~~)("x10ab20c30y", /[0-9]+/, true)
doc %EX - arr .= map(F(elt) if elt is MatchY "[${elt.whole}]" else elt)
doc %EX - arr.join("")  # "x[10]ab[20]c[30]y"
F ~~(s:Str, r:RegExp, collect_unmatched:Bool=false) {
	collector {
		m = (~)(s, r, 0)
		if not(m) {
			if collect_unmatched collect(s)
			return null
		}
		start_offset = m.positions[0][0]
		next_offset = m.positions[0][1]
		if collect_unmatched and start_offset then collect(s[0..start_offset])
		collect(m)
		option_bits = r.options
		utf8 = option_bits.band(C_PCRE_UTF8).Bool()
		option_bits = option_bits.band([C_PCRE_NEWLINE_CR, C_PCRE_NEWLINE_LF, C_PCRE_NEWLINE_CRLF, C_PCRE_NEWLINE_ANY, C_PCRE_NEWLINE_ANYCRLF].reduce(0, bor))
		option_bits = option_bits or PCRE_NEWLINE_TO_OPTIONS.get(PCRE_NEWLINE, 0)
		crlf_is_newline = option_bits in [C_PCRE_NEWLINE_ANY, C_PCRE_NEWLINE_CRLF, C_PCRE_NEWLINE_ANYCRLF]
		while true {
			options = 0
			if start_offset == next_offset {
				next_offset == len(s) breaks
				options = bor(C_PCRE_NOTEMPTY_ATSTART, C_PCRE_ANCHORED)
			}
			m = (~)(s, r, next_offset, options)
			if m {
				if collect_unmatched {
					if next_offset != m.positions[0][0] {
						collect(s[next_offset..m.positions[0][0]])
					}
				}
				start_offset = m.positions[0][0]
				next_offset = m.positions[0][1]
				collect(m)
				continue
			}
			options == 0 breaks
			next_offset += 1
			if (crlf_is_newline) and (try s[next_offset-1..next_offset+1] == '\r\n') {
				next_offset += 1
				continue
			}
			if utf8 {
				while (next_offset < len(s)) and (s[next_offset].ord().band(0xC0) == 0x80) {
					next_offset += 1
				}
				continue
			}
		}
		if collect_unmatched and (next_offset < len(s)) {
			collect(s[next_offset..null])
		}
	}
}

# TODO: test ~~() \r\n skipping
# Crashes on Ubuntu 14.04 clang 3.5.0 (Travis CI)
TEST (try ENV.TRAVIS and ENV.CC == "clang") returns true; ("ab\r\ncd" ~~ /(*CRLF)|./m).whole.filter(len) == ['a', 'b', 'c', 'd']
# Crashes on Ubuntu 14.04 clang 3.5.0 (Travis CI)
TEST (try ENV.TRAVIS and ENV.CC == "clang") returns true; ("ab\r\ncd" ~~ /|./m).whole.filter(len) == ['a', 'b', '\r', 'c', 'd']
# UTF-8 of "я" is 209(D1), 143(8F)
# Skip for older library using try ... catch
TEST try ("aяb" ~~ /(*UTF)|\\x8F|b/).whole.filter(len) == ['b'] catch(e:RegExpCompileFail) 'not recognized' in e.message

# Last two are for repeated searches
STDLIB_REGEXP_FLAGS = {
	'i': C_PCRE_CASELESS
	'm': C_PCRE_MULTILINE
	's': C_PCRE_DOTALL
	'x': C_PCRE_EXTENDED

	'a': C_PCRE_ANCHORED
	'n': C_PCRE_NOTEMPTY_ATSTART
}

doc Regular expression constructor. Called by NGS when /myregex/ syntax is used.
doc %RET - RegExp
doc %EX - my_re = /blah/
F //(regexp:Str, flags:Str) {
	# Maybe add each(Str,Fun) ?
	# TOOD: bitwise or instead of sum()
	options = flags.map(F(flag) {
		STDLIB_REGEXP_FLAGS[flag] tor throw RegExpCompileFail("Unknown flag $flag").set('regexp', regexp).set('flags', flags)
	}).sum()

	c_pcre_compile(regexp, options)
}

TEST Bool("A" ~ /a/i)

doc Returns the string with one occurrence of regexp cut out of it. Throws InvalidArgument if s does not contain r.
doc %EX - "abc01def23" - /[0-9]+/  # "abcdef23"
doc %RET - Str
F -(s:Str, r:RegExp) {
	m = s ~ r
	not(m) throws InvalidArgument('String does not contain the RegExp').set('string', s).set('regexp', r)
	s[0..m.positions[0][0]] + s[m.positions[0][1]..null]
}

TEST "abc01def23" - /[0-9]+/ == "abcdef23"

doc Split string by regexp
doc %RET - Arr of Str
doc %EX "x10ab20c30y".split(/[0-9]+/).join(" :: ")  # "x :: ab :: c :: y"
F split(s:Str, r:RegExp) (~~)(s, r, true).filter(Str)

# Crashes on Ubuntu 14.04 clang 3.5.0 (Travis CI)
TEST (try ENV.TRAVIS and ENV.CC == "clang") returns true; "ab01de12f".split(/[0-9]+/) == ['ab', 'de', 'f']

doc Get string with all occurrences of r removed
doc %RET - Str
doc %EX - "x10ab20c30y".without(/[0-9]+/)  # "xabcy"
F without(s:Str, r:RegExp) s.split(r).join('')

TEST (try ENV.TRAVIS and ENV.CC == "clang") returns true; "ab01de12f".without(/[0-9]+/) == 'abdef'

# TODO: make it a low priority method implementation
doc Get substring of a string that corresponds to first match of given regular expression
doc %EX "x10ab20c30y"[/[0-9]+/]  # "10"
F '[]'(s:Str, r:RegExp) {
	not((m = s ~ r)) throws InvalidArgument("String does not contain the given RegExp").set('container', s).set('regexp', r)
	m.whole
}

TEST "ab01de12f"[/[0-9]+/] == '01'

doc Convert regular expression to a predicate.
doc %RET - A function that checks whether the only argument matches r
doc %EX - %[w1 w2].filter(/1/)  # %[w1]
F Pred(r:RegExp) F regexp_pred(x) x ~ r

TEST %[w1 w2].filter(/1/) == %[w1]

doc Replace all occurrences of r
doc mapper - Function that will be called with one matching string at a time that provides the replacement
doc %RET - Str
doc %EX - "x10ab20c30y".replace(/[0-9]+/, F(match_text) "[$match_text]")  # "x[10]ab[20]c[30]y"
F replace(s:Str, r:RegExp, mapper:Fun) {
	(~~)(s, r, true).map([ F(m:MatchY) mapper(m.whole), F(s:Str) s ]).join('')
}

TEST "x10ab20c30y".replace(/[0-9]+/, F(match_text) "[$match_text]") == "x[10]ab[20]c[30]y"

# === Fun ========================================

doc Call cb n times without any parameters and accumulate the results.
doc %EX - a = Box * 2; a[0] is Box and a[1] is Box and a[0] !== a[1]  # true
F *(cb:Fun, n:Int) n.map({ cb() })

TEST a = Box * 2; a[0] is Box and a[1] is Box and a[0] !== a[1]

# === Null =======================================

doc Convert null to NGS code that would produce null when executed.
doc %RET - Str
F code(n:Null) "null"


# === exit_hook - Print uncaught exception =======

doc Do nothing. Catch-all for exception_specific_message().
doc %RET - EmptyBox
F exception_specific_message(e:Exception) EmptyBox()

doc Format GlobalNotFound exception message.
doc %RET - FullBox with Str
F exception_specific_message(gnf:GlobalNotFound) FullBox("Global '${gnf.name}' not found")

doc Don't use, subject to change, including name
F SafeStr(val) "$val" tor "(Failed to inspect <${val.typeof().name tor '(unknown type)'}>)"

doc Inspect any value.
doc %RET - Arr with one element, Str
F inspect(val) [SafeStr(val)]

doc Inspect Real
doc %RET - Arr with one element, Str
F inspect(r:Real) ["Real: $r"]

doc Inspect Int
doc %RET - Arr with one element, Str
F inspect(i:Int) ["Int: $i"]

doc Inspect Str
doc %RET - Arr with one element, Str
F inspect(s:Str) ["String: $s"]

doc Inspect Arr
doc %RET - Arr of Str
F inspect(a:Arr) ["Array of size ${a.len()}"] + a.map_idx_val({"[${SafeStr(A)}] = ${SafeStr(B)}"})

doc Inspect Hash
doc %RET - Arr of Str
F inspect(h:Hash) ["Hash of size ${h.len()}"] + h.map({"[${SafeStr(A)}] = ${SafeStr(B)}"})

doc Inspect Process
doc %RET - Arr of Str
F inspect(p:Process) {
	collector {
		c = collect
		c("Process")
		c("command = ${p.command}")
		c("pid = ${p.pid}")
		c("exit_code = ${p.exit_code}")
		c("exit_signal = ${p.exit_signal}")
		%[stdout stderr].each(F(attr) {
			val = p.(attr)
			val_lines = if val is Null [] else val.lines()
			c("$attr (${val_lines.len()} lines):")
			("  " + val_lines) % c
		})
	}
}

doc Print exception to stderr. Uses inspect().
doc level - indentation level to use (two spaces for each level)
doc parent - parent exception if print_exception() was called for .cause exception
F print_exception(e:Exception, level:Int = 0, parent:Exception = null) {

	F err(message:Str) error("$pfx$message")

	pfx = "  " * level
	h = Hash(e)

	err("Exception of type ${e.typeof().name} occured")

	exception_specific_message(e).each(err)

	h.Box('message').map("Message: $X").each(err)

	info_keys = h.keys() - %[message backtrace cause]

	F print_info(k) {
		out = inspect(h[k]) tor ["Info: $k - (Failed to inspect)"]
		assert(out is Arr)
		assert(out.len() > 0)
		if out.len() == 1 {
			err("Info: $k = ${out[0]}")
		} else {
			err("Info: $k:")
			("  " + out) % err
		}

		# err("Info: $k is hash of size ${h[k].len()}")
	}

	info_keys % print_info

	if 'backtrace' in h {

		parent_frames = parent.backtrace.frames tor []

		h.backtrace.frames.each_idx_val(F(idx, frame) {

			# Don't print frames that overlap with parent exception
			frame.ip == (try parent_frames[idx].ip) returns

			l = resolve_instruction_pointer(frame.ip)
			closure = " [in ${frame.closure.attrs().name}]" tor ''
			try {
				doc = frame.closure.attrs().doc
				if doc is Str {
					closure = "$closure (${doc.limit(50, '...')})"
				}
			}
			err("Frame $idx: ${l.file}:${l.first_line}:${l.first_column} - ${l.last_line}:${l.last_column}$closure")
		})
	}

	if 'cause' in h {
		err("Cause:")
		print_exception(e.cause, level+1, e)
	}

}

exit_hook = Hook()

{
	# TODO: consider running this hook item last.
	#       If it has exception, it could prevent
	#       user-defined cleanups.
	exit_hook['print_exception'] = F(exit_info:Hash) {
		if exit_info.exception is not Null {
			error("--- Exception during execution of main code ---")
			print_exception(exit_info.exception)
		}
	}

	exit_hook['exception_to_exit_code'] = F(exit_info:Hash) {
		if exit_info.exception is not Null {
			try {
				exit_info.exit_code = exit_info.exception.to_exit_code()
			} catch(e:Exception) {
				error("--- Exception during converting exception to exit code ---")
				print_exception(e)
			}
		}
	}
}

# === Random =====================================
# This is _not_ crypto safe random!

srand(time())

doc Pick n random unique elements from something
doc %RET - Arr
F rand_uniq(something, n:Int) {
	# TODO: Fix the guard - it does not work with Range currently
	# guard n <= something.len()
	result = []
	safety_counter = 0
	while len(result) < n {
		r = rand(something)
		if r in result {
			safety_counter += 1
			safety_counter == STDLIB_RAND_SAFETY_MAX throws Error("Not making progress in rand_uniq(something, n:Int)")
		} else {
			result.push(r)
			safety_counter = 0
		}
	}
	result
}

doc Pick n elements from something. Uniqueness of picked elements is not guaranteed.
doc %RET - Any
F rand(something, n:Int) n.map({ rand(something) })

doc Pick random number from 0 up to but not including n
doc %RET - Int
F rand(n:Int) {
	n <= 0 throws InvalidArgument("rand(n:Int) - n must be 1 or greater")
	r = rand()
	(RAND_MAX+1) % n == 0 returns r % n
	scale = Real(n) / STDLIB_MAX_RAND_REAL
	Real(r) * scale \ floor \ Int
}

# TODO: Sequence super-type?

doc Pick one random element from array
doc %RET - Any
F rand(a:Arr) a[rand(a.len())]

doc Pick random character from a string
doc %RET - Str of length 1
F rand(a:Str) a[rand(a.len())]

F rand(a:Str, n:Int) super(a, n).join('')

doc Pick one random element from a range
doc %RET - Any
F rand(r:Range) {
	guard r.step == 1
	points = r.end - r.start
	if r is InclusiveRange points += 1
	rand(points) + r.start
}

# === retry facility =============================

{
	doc Represents failure on all retries
	type RetryFail(Error)
}

F body_missing_in_retry() throw InvalidArgument("'body' argument missing in retry()")

# TODO: Handle NoNext that could occur in sleep iterator
doc Retry. Executes given "body" "times" times. Throws RetryFail if all calls fail and fail_cb is not provided.
doc times - Limit of times to run "body"
doc sleep - Either sleep in seconds between tries or Iter that returns successive sleep times in seconds
doc logger - Function to use to log attempts, caught body exceptions and sleep times. Defaults to stdlib's debug.
doc success_predicate - CANDIDATE FOR REMOVAL. Run against body results to. Says whether the attempt succeeded. Defaults to Bool.
doc catch_exceptions - Catch exceptions that might occur in body and treat them as if body returned failure.
doc title - Prefix strings sent to logger with this string. Defaults to "<retry>".
doc progress_cb - Called before each attempt with (1) current attempt number and (2) limit of attempts ("times"). Defaults to do nothing.
doc success_cb - Called when body succeeds with the result that comes from body. Defaults to identity function.
doc fail_cb - If passed, called if all attempts fail. Defaults to null.
doc %RET - Any. Result of success_cb or result of fail_cb.
doc %EX - page = retry(times=10, sleep=ExpBackIter(), body={ `curl "http://flaky-site.com"` })
F retry(
	times:Int=60,
	sleep=1,
	logger:Fun=debug,
	success_predicate=Bool,
	catch_exceptions=true,
	title="<retry>",
	progress_cb:Fun={null},
	success_cb:Fun=identity,
	fail_cb=null,
	body:Fun=body_missing_in_retry
) {
	guard sleep is Int or sleep is Iter
	sleep_iter = if sleep is Int then ConstIter(sleep) else sleep
	result = null  # otherwise it will be local to try { ... }
	for(i;times) {
		n = i + 1
		progress_cb(n, times)
		logger("$title Attempt $n out of ${times}")
		try {
			result = body()
		} catch(e:Exception) {
			logger("$title Exception $e")
			not(catch_exceptions) throws e
		}
		if success_predicate(result) {
			return success_cb(result)
		}
		if(i < times - 1) {
			s = sleep_iter.next()
			logger("$title Sleeping ${s} seconds")
			$(sleep "$s")
		}
	}
	fail_cb is Null throws RetryFail()
	fail_cb()
}

TEST t = 0; retry(body={t+=1; false}, fail_cb={"OK"}, times=5, sleep=0) == "OK" and t == 5
TEST t = 0; retry(body={t+=1; "OK"}, fail_cb={"NOTOK"}, sleep=0) == "OK" and t == 1

# === Misc =======================================

doc Get process ID
F pid() c_getpid()

doc Get parent process ID
F ppid() c_getppid()

doc Encodes each character in a string as two uppercase hexadecimal digits
doc %RET - Str
F encode_hex(s:Str) {
	s.map(F(char) {
		n = ord(char)
		CHARS::hexdigits_uppercase[n / 16] + CHARS::hexdigits_uppercase[n % 16]
	}).join('')
}

TEST encode_hex('ab') == '6162'

STDLIB_INVALID_URI_CHAR_RE = /[^-a-zA-Z0-9_.!~*'()]/
#'

doc Encodes URI component, escaping with %XX hexadecimal codes.
doc %RET - Str
F encode_uri_component(s:Str) s.replace(STDLIB_INVALID_URI_CHAR_RE, F(char) '%' + encode_hex(char))

TEST encode_uri_component("ab+c%") == "ab%2Bc%25"


STDLIB_HTML_ESCAPE = {'&': '&amp;', '<': '&lt;', '>': '&gt;'}

doc Encodes HTML. Escapes &, < and > .
doc %RET - Str
F encode_html(s:Str) s.map(STDLIB_HTML_ESCAPE.get(X, X)).join('')

TEST "ab>+c%&'".encode_html() == "ab&gt;+c%&amp;'"


STDLIB_HTML_ATTR_ESCAPE = STDLIB_HTML_ESCAPE + {'"': '&quot;', "'": '&#39;'}

doc Encodes HTML attribute. Escapes &, <, >, " and ' .
doc %RET - Str
F encode_html_attr(s:Str) s.map(STDLIB_HTML_ATTR_ESCAPE.get(X, X)).join('')

TEST "ab>+c%&'".encode_html_attr() == "ab&gt;+c%&amp;&#39;"

doc Used by match EXPR { ... } construct.
doc %RET - a ~ b
F match(a,b) a ~ b

# === File =======================================

F store(s:Str, fname:Str) {
	f = File(fname)
	open(f, 'w')
	finally()
		body => {
			store(s, f.fd)
		}
		cleanup => {
			f.close()
		}
}


# === Networking =================================

### F read(s:Str) {
### 	# impl curl
### 	guard s ~ /^http[s]?:\/\//i
### 	guard "curl" found
### }

# === Caching ====================================

# TODO: *args and per-args combination wrapping
F cached(cb:Fun) {
	result = EmptyBox()
	F cached_function_wrapper() {
		if not(result) {
			result = FullBox(cb())
		}
		result.get()
	}
}

# === Misc tests =================================

# expr1 op= expr2
TEST a = 1; a+=1 ; a*=10; a == 20
TEST a=[{"x": 7}, {"x": 8}]; a .= x; a == [7, 8]
TEST a=[1,2]; a .= map(F(x) x*2); a[0]+=10; a == [12,4]

TEST o = {'a': 1, 'b': [null, false, true, 3.14], 'nothing': null}; decode_json(encode_json(o)) == o
TEST h={'a': 1, 'b': 2, 'c': 3}; h.update({'b': 20, 'd': 40}); h == {'a': 1, 'b': 20, 'c': 3, 'd': 40}

TEST F opt(a, b=10, *rest) [b, rest]; opt(1) == [10, []]
TEST F opt(a, b=10, *rest) [b, rest]; opt(1, 2) == [2, []]
TEST F opt(a, b=10, *rest) [b, rest]; opt(1, 2, 3) == [2, [3]]
TEST F opt(a, b=10, *rest) [b, rest]; opt(1, b=10) == [10, []]
TEST F opt(a, b=10, *rest, **kw) [b, rest, kw]; opt(1, b=20) == [20, [], {"b": 20}]
TEST F opt(a, b=10, *rest, **kw) [b, rest, kw]; opt(1, c=20) == [10, [], {"c": 20}]
TEST F opt(a, b=10, *rest) [b, rest]; try opt(1, c=20) == [10, []] catch(e:ImplNotFound) true
TEST F opt(a, b=10, *rest, **kw) [b, rest, kw]; opt(1, b=30, **{"b": 20}) == [20, [], {"b": 20}]
TEST F opt(a, b=10, *rest, **kw) [b, rest, kw]; opt(1, b=30, **{"b": 20}, b=40) == [40, [], {"b": 40}]
TEST F opt(a, b=10, *rest, **kw) [a, b, rest, kw]; opt(1, 100, 200, **{"b": 20}, b=40) == [1, 100, [200], {"b": 40}]
TEST F f(a) a; f(**{"a": 10}) == 10
TEST F f(a=7) a; f(**{"a": 10}) == 10

TEST F construct(x:Int) F() x; f1=construct(1); f2=construct(2); (f1==f1) and (f1!=f2)
TEST 1.1 < 1.11
TEST 1.1 + 1.1 == 2.2
TEST F uniq_attrs_func() "some body"; uniq_attrs_func[0].attrs({"a": 1}); uniq_attrs_func[0].attrs() == {"a": 1}
TEST attrs((+)[0])['name'] == '+'
# TODO: more introspection tests
TEST params((+)[0])[0]['name'] == 'a'
TEST params((+)[0])[0]['type'] === Real
TEST "config" in globals()
TEST args=['1', '+', '2']; $(expr $*args).Str()[0..1] == '3'
TEST time() > 1466662171
TEST a=[10,20,30]; f=a[X]; f(1) == 20
TEST a=[10,20,30]; f=X[1]; f(a) == 20

# === Arr tests ==================================
TEST [10,20,30,40][0..2] == [10,20]
TEST [10,20,30,40][1..2] == [20]
TEST (try [10,20,30,40][-1..2] catch(e:InvalidArgument) "OK") == "OK"
TEST [10,20,30,40][2..null] == [30, 40]
TEST (try [10,20,30,40][2..1] catch(e:InvalidArgument) "OK") == "OK"
TEST (try [10,20,30,40][2..10] catch(e:InvalidArgument) "OK") == "OK"
TEST [10,20,30,40][4..null] == []
TEST a=[1,2,3,4]; a[4..4]=["END"]; a[2..2]=["MIDDLE"]; a[0..0]=["START"]; a[0..1] = ["start"]; a == ["start", 1, 2, "MIDDLE", 3, 4, "END"]

# === Hash tests =================================
TEST h={"a": 1}; h.del("a"); h == {}
TEST h={}; h.del("a"); h == {}
TEST h={"a": 1, "b": 2}; h.del("a"); h == {"b": 2}

# === Str tests ==================================
TEST "abcd"[1..3] == "bc"
TEST s="abcd"; s[4..4]="END"; s[2..2]="MIDDLE"; s[0..0]="START"; s[0..1] = "s"; s == "sTARTabMIDDLEcdEND"

# === NormalTypeInstance tests ===================
TEST a = Path("a"); b = Path("a"); (a == b) and (a !== b)
TEST Path("a") != Path("b")

# === Exceptions tests ===========================
TEST h={}; try h["a"] catch(e:KeyNotFound) true
TEST a=[]; try a[0] catch(e:IndexNotFound) true
TEST a=[]; try a[0]=1 catch(e:IndexNotFound) true
TEST type T; try T().xyz catch(e:AttrNotFound) true
TEST try NO_SUCH_GLOBAL catch(e:GlobalNotFound) true
TEST try c_lseek(0, 0, "bad arg") catch(e:InvalidArgument) true
TEST try compile("xyz(", "something") catch(e:CompileFail) true
TEST try 1(2, 3) catch(e:DontKnowHowToCall) true
TEST type T; try echo(T(), T()) catch(e:ImplNotFound) true
TEST try F f() f(); try f() catch(e:StackDepthFail) true
TEST try [].pop() catch(e:EmptyArrayFail) true
TEST try [].shift() catch(e:EmptyArrayFail) true

# === Partial application tests ==================
TEST [{"val": 10, "flag": false}, {"val": 20, "flag": true}].filter(X.flag).val == [20]
TEST %[f1 f3].map({"f1": 1, "f2": 2, "f3": 3}.(X)) == [1, 3]
TEST [[1, 10], [2,20]].map(X[0]) == [1, 2]
TEST [1,2].map([10,20,30][X]) == [20, 30]
TEST [[1],[2,3]].filter(X.has(1)) == [[1]]
TEST {true}()
TEST f = F(x, y) x+y; f(X, 10)(20) == 30

# === Namespaces tests ===========================
TEST ns1 = ns { a = 1; _b = 2; a = 100 }; ns1 == {"a": 100}
TEST ns1 = ns { F f(x) x+1; }; ns1::f(1) == 2
TEST ns1 = ns { yes_one = 1; no_one = 11; no_two = 12; _exports.=reject(F(k, v) k ~ /^no/) }; ns1 == {"yes_one": 1}
TEST ns1 = ns { return 10 }; ns1 == 10
TEST ns1 = ns { _exports = 11 }; ns1 == 11
TEST global a=110; ns1 = ns(outer_a=a) { a=outer_a+1 }; ns1::a == a+1

# === Process tests ================================

TEST try $(cat NO_SUCH_FILE 2>/dev/null) catch(pf:ProcessFail) true
TEST $(nofail cat NO_SUCH_FILE 2>/dev/null); true

# === Redirections tests ===========================
# Warning: file name is not unique
TEST f='/tmp/ngs-redir-test'; $(echo -n abc >$f); $(echo -n def >>$f); txt=`cat $f`; $(rm $f); txt == 'abcdef'
TEST f='/tmp/ngs-redir-test'; $(nofail cat /NO-SUCH-FILE 2>$f); txt=`cat $f`; $(rm $f); (txt ~ /No such file or directory/) is MatchY