initial

.gitignore

@@ -0,0 +1,15 @@
+
+.DS_Store
+
+*.sublime-project
+
+*.o
+
+.dub
+je
+
+*.tsv
+
+dub.selections.json
+
+in.jsonl

README.md

@@ -0,0 +1,108 @@
+[![Circle CI Docs](https://circleci.com/gh/tamediadigital/je.svg?style=shield&circle-token=:circle-ci-badge-token)](https://circleci.com/gh/tamediadigital/je)
+
+JE
+=====
+Fast JSON to TSV/CSV/JSON/User-defined-format Extractor.
+
+### Build je
+
+#### Requirements
+
+1. [dub](https://code.dlang.org/getting_started) package manager.
+2. D compiler. Options:
+  - [LDC](https://github.com/ldc-developers/ldc) (LLVM D Compiler) >= `1.1.0-beta2`
+  - [DMD](http://dlang.org/download.html) (DigitalMars D Compiler) >= `2.072.1`
+
+#### Build with the dub package manager
+
+To build project with LDC run following command from `je` project:
+```
+dub build --build=release-sse42 --compiler=ldmd2
+```
+or
+```
+dub build --build=release-native --compiler=ldmd2
+```
+
+To build project with DMD run
+```
+dub build --build=release
+```
+
+For more details run `je --help`.
+
+#### Usage
+
+After building, you can try je like this:
+```
+./je test.json --columns name:name,asdf:dependencies.asdf --input test.json
+```
+
+##### User defined output format
+```json
+$ cat in.jsonl 
+{"a":{"b":"\n"}, "d":2}
+{"a":{"b":0}, "d":1}
+{"a":{"b":2}}
+
+# query with non-positional style 
+$ ./je -c a.b,d -i in.jsonl --out=$'{"a":%s,"t":%s}\n'
+{"a":"\n","t":2}
+{"a":0,"t":1}
+{"a":2,"t":null}
+
+# query with positional style
+./je -c a.b,d -i in.jsonl --out=$'{"a":%2$s,"t":%1$s}\n'
+{"a":2,"t":"\n"}
+{"a":1,"t":0}
+{"a":null,"t":2}
+
+##### [Probabilistic Linear Counting](https://github.com/tamediadigital/lincount)
+
+```json
+$ cat in.jsonl
+{"a":{"b":0}, "d":1}
+{"a":{"b":0}, "d":2}
+{"a":{"b":"\n"}, "d":1}
+{"a":{"b":"\n"}, "d":2}
+
+$je -i in.jsonl --count=a.b --count=d --count="a.b&d"
+
+{"counter":1,"count":2}
+{"counter":2,"count":2}
+{"counter":3,"count":4}
+```
+
+```json
+$ cat in.jsonl
+{"a":{"b":10.0}, "d":null}
+{"a":{"b":10.0}}
+{"a":{"b":10.00}, "d":null}
+{"a":{"b":10.00}}
+
+$je -i in.jsonl --counta.b --count=d --count="a.b&d"
+{"_counter_1":2,"_counter_2":1,"_counter_3":2}
+
+// with named counters
+$je -i in.jsonl --count=a.b:a.b --count=d --count="expr:a.b&d"
+{"a.b":2,"_counter_2":1,"expr":2}
+```
+
+###### Unix Time-stamp partition
+```json
+$ cat in.jsonl
+{"a":{"ts":1485831253}, "param":0}
+{"a":{"ts":1485831254}, "param":1}
+{"a":{"ts":1485831255}, "param":1}
+{"a":{"ts":1485831400}, "param":0}
+{"a":{"ts":1485831401}, "param":1}
+
+$ ./je -i in.jsonl --count=param --count-algo=timestamp --count-algo-params=a.ts,60
+{"ts":1485831360,"_counter_1":2}
+{"ts":1485831240,"_counter_1":2}
+
+// with named counters
+$ ./je -i in.jsonl --count=param:param --count-algo=timestamp --count-algo-params=a.ts,60
+{"ts":1485831360,"param":2}
+{"ts":1485831240,"param":2}
+```

circle.yml

@@ -0,0 +1,25 @@
+machine:
+  environment:
+    DMD: 2.073.0
+    LDC: 1.3.0
+    DUB: 1.5.0
+    PATH: "${HOME}/dmd2/linux/bin64:${HOME}/ldc2-$LDC-linux-x86_64/bin:${PATH}"
+    LD_LIBRARY_PATH: "${HOME}/dmd2/linux/lib64:${HOME}/ldc2-$LDC-linux-x86_64/lib:${LD_LIBRARY_PATH}"
+checkout:
+  post:
+    - git submodule sync . && git submodule update --recursive --init || true
+    - git submodule sync . && git submodule update --recursive --init || true # duplication needed due to circleci bug, don't remove
+dependencies:
+  override:
+    - curl -fsSL --retry 3 "http://downloads.dlang.org/releases/2.x/$DMD/dmd.$DMD.linux.tar.xz" | tar -C ~ -Jxf -
+    - dmd --version
+    - curl -fsSL --retry 3 "https://github.com/ldc-developers/ldc/releases/download/v$LDC/ldc2-$LDC-linux-x86_64.tar.xz" | tar -C ~ -Jxf -
+    - ldmd2 --version
+    - curl -fsSL --retry 3 http://code.dlang.org/files/dub-${DUB}-linux-x86_64.tar.gz | tar -C ~/ldc2-$LDC-linux-x86_64/bin -zxf -
+    - dub --version
+test:
+  override:
+    - dub build --compiler=dmd
+    - dub build --compiler=ldmd2
+    - dub build --compiler=ldmd2 --build=release-sse42
+    - dub build --compiler=ldmd2 --build=release-native

dub.json

@@ -0,0 +1,23 @@
+{
+	"name": "je",
+	"authors": [
+		"Ilya Yaroshenko"
+	],
+	"dependencies": {
+		"asdf": "~>0.1.4",
+		"hll-d": "~>0.3.0"
+	},
+	"description": "Json Extract: a tool to extract data from json",
+	"copyright": "Copyright © 2016, Tamedia Digital",
+	"license": "proprietary",
+	"buildTypes": {
+		"release-sse42": {
+			"buildOptions": ["releaseMode", "optimize", "inline"],
+			"dflags-ldc": ["-mattr=+sse4.2"]
+		},
+		"release-native": {
+			"buildOptions": ["releaseMode", "optimize", "inline"],
+			"dflags-ldc": ["-mcpu=native"]
+		}
+	}
+}