analysis.csym

#$cSYM/1.0(./analysis.ecs):-,-,-,-,0,2,3,4,8,9,10,11,12,13,14,15,16,17,18,19,21,22,23,24,25,26,27,29,30,31,33,34,35,37,38,39,41,42,43,45,46,47,49,50,51,55,56,57,58,59,60,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,236,237,238,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,278,279,280,281,282,283,284,285,287,288,289,290,291,292,293,294,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,321,322,323,324,325,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,344,345,346,346,347,348,349,350,351,352,356,357,358,359,360,361,362,363,364,365,367,368,369,370,371,372,373,375,376,377,378,379,380,381,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,460,461,462,463,464,465,466,467,468,469,473,474,475,476,477,479,480,481,482,483,484,485,486,487,488,489
package analysis

import analysis_impl as impl
import codec.json as json
import regex

# Aggregate Operators

struct operator
    var name = null
    var column = null
    var impl_func = null
    function as(str)
        name = str
        return this
    end
    function run(data)
        return impl_func(data, column)
    end
end

function __impl_make_operator(name, col, func)
    var op = new operator
    op.name = name
    op.column = col
    op.impl_func = func
    return move(op)
end

function avg(col)
    return __impl_make_operator("avg", col, impl.avg)
end

function average(col)
    return __impl_make_operator("avg", col, impl.avg)
end

function sum(col)
    return __impl_make_operator("sum", col, impl.sum)
end

function summery(col)
    return __impl_make_operator("sum", col, impl.sum)
end

function count(col)
    return __impl_make_operator("count", col, impl.count)
end

function count_distinct(col)
    return __impl_make_operator("count_distinct", col, impl.count_distinct)
end

# Dataframe

function __impl_align(text, width)
    var txt = " " + text
    var pad = width - text.size - 1
    foreach it in range(pad) do txt += " "
    return move(txt)
end

function __impl_show(header, data)
    var maxs = new array
    var width = header.size
    foreach it in header do maxs.push_back(it.size)
    foreach line in data
        if line.size != width
            system.out.println("analysis.dataframe.show: broken line founded!")
            return this
        end
        foreach i in range(width)
            var size = to_string(line.at(i)).size
            if size > maxs.at(i)
                maxs.at(i) = size
            end
        end
    end
    system.out.print("#")
    foreach idx in range(width) do system.out.print(__impl_align(header.at(idx), maxs.at(idx) + 2) + "#")
    system.out.println("")
    foreach line in data
        system.out.print("|")
        foreach idx in range(width) do system.out.print(__impl_align(to_string(line.at(idx)), maxs.at(idx) + 2) + "|")
        system.out.println("")
    end
end

class datarecord
    var host = null
    var data = null
    function construct(df, line)
        host = df
        data = line
    end
    function by_name(name)
        return data->at(host->header_map.at(name))
    end
    function by_index(idx)
        return data->at(idx)
    end
    function show()
        __impl_show(host->header, {*data})
        return this
    end
end

class dataframe
    var header_map = null
    var header = null
    var data = null
    function construct(_header)
        header_map = new hash_map
        foreach idx in range(_header.size) do header_map.insert(_header.at(idx), idx)
        header = _header
        data = new array
    end
    function filter(args, cond)
        if args.empty()
            return this
        end
        var cols = new array
        foreach it in args do cols.push_back(header_map.at(it))
        return __impl_make_dataframe(header, impl.filter(data, cols, cond))
    end
    function __groupby_impl(cols, iteration, child_data, idx)
        var result = (idx == 0 ? impl.groupby(child_data, cols.at(idx)) : impl.groupby_group(data, child_data, cols.at(idx)))
        if iteration > 0
            foreach it in result do it.second = __groupby_impl(cols, iteration - 1, it.second, idx + 1)
        end
        return move(result)
    end
    function groupby(...args)
        if args.empty()
            return this
        end
        var cols = new array
        foreach it in args do cols.push_back(header_map.at(it))
        return __impl_make_dataframe_groupby(header, &data, cols, __groupby_impl(cols, cols.size - 1, data, 0))
    end
    function select(...args)
        var cols = new array
        foreach it in args do cols.push_back(header_map.at(it))
        return __impl_make_dataframe(args, impl.select(data, cols))
    end
    function aggregate(...args)
        var result_header = new array, result_array = new array
        foreach op in args
            result_header.push_back(op.name)
            op.column = header_map.at(op.column)
            result_array.push_back(op.run(data))
        end
        return __impl_make_dataframe(result_header, {result_array})
    end
    function attach(...args)
        var result_header = header, result_array = new array
        foreach op in args
            result_header.push_back(op.name)
            op.column = header_map.at(op.column)
            result_array.push_back(op.run(data))
        end
        var new_data = data
        foreach line in new_data
            foreach it in result_array do line.push_back(it)
        end
        return __impl_make_dataframe(result_header, new_data)
    end
    function apply(col, func)
        var col_idx = header_map.at(col)
        foreach line in data do func(line.at(col_idx))
        return this
    end
    function enumerate(func)
        foreach i in range(data.size)
            func(i, new datarecord{&this, &data.at(i)})
        end
        return this
    end
    function foreach(func)
        foreach it in data
            func(new datarecord{&this, &it})
        end
        return this
    end
    function pick(func)
        var results = new array
        foreach line in data
            if func(new datarecord{&this, &line})
                results.push_back(line)
            end
        end
        return __impl_make_dataframe(header, move(results))
    end
    function by_index(idx)
        return new datarecord{&this, &data.at(idx)}
    end
    function append(...lines)
        foreach line in lines
            if line.size == header.size
                data.push_back(move(line))
            end
        end
        return this
    end
    function join_v(df, pkey, fill_val, drop_invalid)
        var pidx = null, pmap = null
        if pkey != null && header_map.exist(pkey) && df.header_map.exist(pkey)
            pidx = header_map.at(pkey)
            pmap = new hash_map
            foreach i in range(data.size) do pmap.insert(data.at(i).at(pidx), i)
        end
        var index = new array
        foreach key in header
            if df.header_map.exist(key)
                index.push_back(df.header_map.at(key))
            else
                index.push_back(null)
            end
        end
        foreach rec in df.data
            var line = new array
            foreach pos in index
                if pos != null
                    line.push_back(rec.at(pos))
                else
                    line.push_back(fill_val)
                end
            end
            if pmap != null
                if pmap.exist(line.at(pidx))
                    link target = data.at(pmap.at(line.at(pidx)))
                    foreach i in range(target.size)
                        if i != pidx && index.at(i) != null
                            swap(target.at(i), line.at(i))
                        end
                    end
                else if !drop_invalid
                    data.push_back(move(line))
                end
            else
                data.push_back(move(line))
            end
        end
        return this
    end
    function join(df)
        return join_v(df, null, null, false)
    end
    function find_broken_lines()
        var lines = new array
        foreach i in range(data.size)
            if data.at(i).size != header.size
                lines.push_back(i + 2)
            end
        end
        return move(lines)
    end
    function to_csv(file)
        var csv_data = data
        csv_data.push_front(header)
        impl.write_csv(csv_data, file)
        return this
    end
    function to_json()
        var j_arr = json.value.make_array()
        foreach line in data
            var j_obj = json.value.make_object()
            foreach i in range(header.size) do j_obj.set_member(header.at(i), json.value.make_string(to_string(line.at(i))))
            j_arr.arr_append(j_obj)
        end
        return move(j_arr)
    end
    function show()
        __impl_show(header, data)
        return this
    end
end

function __impl_make_dataframe(header, data)
    var df = new dataframe
    df.header_map = new hash_map
    foreach idx in range(header.size) do df.header_map.insert(header.at(idx), idx)
    df.header = header
    df.data = data
    return move(df)
end

function read_csv(path)
    var data = impl.read_csv(path)
    if data != null
        link header = data.front
        data.pop_front()
        return __impl_make_dataframe(header, move(data))
    end
end

function cartesian_product(dfx, dfy, pkey, fill_val, drop_invalid)
    var pidx = null, pmap = null
    if pkey != null && dfx.header_map.exist(pkey) && dfy.header_map.exist(pkey)
        pidx = dfx.header_map.at(pkey)
        pmap = new hash_map
        foreach i in range(dfx.data.size) do pmap.insert(dfx.data.at(i).at(pidx), i)
    end
    var keys = hash_set.intersect(dfx.header => hash_set, dfy.header => hash_set)
    var header = dfx.header
    var index = new array
    foreach key in dfx.header
        if dfy.header_map.exist(key)
            index.push_back(dfy.header_map.at(key))
        else
            index.push_back(null)
        end
    end
    foreach key in dfy.header
        if !keys.exist(key)
            header.push_back(key)
            index.push_back(dfy.header_map.at(key))
        end
    end
    var data = dfx.data
    # Align with padding
    foreach rec in data
        foreach i in range(header.size - rec.size)
            rec.push_back(fill_val)
        end
    end
    # Do product
    foreach rec in dfy.data
        var line = new array
        foreach pos in index
            if pos != null
                line.push_back(rec.at(pos))
            else
                line.push_back(fill_val)
            end
        end
        if pmap != null
            if pmap.exist(line.at(pidx))
                link target = data.at(pmap.at(line.at(pidx)))
                foreach i in range(target.size)
                    if i != pidx && index.at(i) != null
                        swap(target.at(i), line.at(i))
                    end
                end
            else if !drop_invalid
                data.push_back(line)
            end
        else
            data.push_back(line)
        end
    end
    return __impl_make_dataframe(header, data)
end

# Dataframe Groupby

function __impl_get_group(group, key, target)
    if typeid group != typeid hash_map
        throw runtime.exception("Group not exist!")
    end
    if group.exist(key)
        target.push_back(group.at(key))
    else
        foreach it in group do __get_group_impl(it.second, key, target)
    end
end

function __impl_unstack_group_impl(group, target)
    if typeid group == typeid hash_map
        foreach it in group do __impl_unstack_group_impl(it.second, target)
    else
        foreach it in group do target.push_back(it)
    end 
end

function __impl_unstack_group(group, data)
    var lines = new array
    __impl_unstack_group_impl(group, lines)
    var result = new array
    foreach idx in lines do result.push_back(data.at(idx))
    return move(result)
end

function __impl_aggregate_group(stack, data, group, target, ops)
    if typeid group == typeid hash_map
        target = new hash_map
        foreach it in group
            var child_data = null
            stack.push_back(it.first)
            __impl_aggregate_group(stack, data, it.second, child_data, ops)
            stack.pop_back()
            target.insert(it.first, child_data)
        end
    else
        target = stack
        var lines = new array
        foreach idx in group do lines.push_back(data.at(idx))
        foreach op in ops do target.push_back(op.run(lines))
        target = {target}
    end
end

function __impl_attach_group(data, group, target, ops)
    if typeid group == typeid hash_map
        target = new hash_map
        foreach it in group
            var child_data = null
            __impl_attach_group(data, it.second, child_data, ops)
            target.insert(it.first, child_data)
        end
    else
        target = new array
        var result = new array
        var lines = new array
        foreach idx in group do lines.push_back(data.at(idx))
        foreach op in ops do result.push_back(op.run(lines))
        foreach it in lines do target.push_back({it..., result...})
    end
end

class dataframe_groupby
    var header_map = null
    var header = null
    var groups = null
    var g_cols = null
    var data = null
    function get_group(key)
        var group_set = new array, lines = new array, result = new array
        __get_group_impl(groups, key, group_set)
        foreach it in group_set do __impl_unstack_group_impl(it, lines)
        foreach idx in lines do result.push_back(data->at(idx))
        return __impl_make_dataframe(header, result)
    end
    function unstack()
        return __impl_make_dataframe(header, __impl_unstack_group(groups, *data))
    end
    function aggregate(...args)
        var result_header = new array, result_map = null, result_array = new array
        foreach idx in g_cols do result_header.push_back(header.at(idx))
        foreach op in args
            result_header.push_back(op.name)
            op.column = header_map.at(op.column)
        end
        var stack = new array
        __impl_aggregate_group(stack, *data, groups, result_map, args)
        __impl_unstack_group_impl(result_map, result_array)
        return __impl_make_dataframe(result_header, result_array)
    end
    function attach(...args)
        var result_header = header, result_map = null, result_array = new array
        foreach op in args
            result_header.push_back(op.name)
            op.column = header_map.at(op.column)
        end
        __impl_attach_group(*data, groups, result_map, args)
        __impl_unstack_group_impl(result_map, result_array)
        return __impl_make_dataframe(result_header, result_array)
    end
end

function __impl_make_dataframe_groupby(header, data, cols, groups)
    var df = new dataframe_groupby
    df.header_map = new hash_map
    foreach idx in range(header.size) do df.header_map.insert(header.at(idx), idx)
    df.header = header
    df.groups = groups
    df.g_cols = cols
    df.data = data
    return move(df)
end

# Public Methods

function enumerate(data, func)
    foreach i in range(data.size)
        func(i, data.at(i))
    end
end

function match_regex(reg, str)
    var result = reg.match(str)
    if result.empty()
        return null
    end
    var data = new array
    foreach i in range(1, result.size())
        data.push_back(result.str(i))
    end
    return move(data)
end