First public release

.gitignore

@@ -0,0 +1,4 @@
+transporter
+!cmd/transporter
+!pkg/transporter
+

CONTRIBUTING.md

@@ -0,0 +1 @@
+# Contributing to Transporter

LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2014, Compose, Inc
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of [project] nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -0,0 +1,70 @@
+Transporter
+
+Build
+-----
+`go build -a ./cmd/...`
+
+
+Configure
+---------
+there is a sample config in test/config.yaml.  The config defines the endpoints, (either sources or sinks) that are available to the application.
+```yaml
+api:
+  interval: 1 # number of milliseconds between metrics pings
+  uri: "http://requestb.in/1a0zlf11"
+nodes:
+  localmongo:
+    type: mongo
+    uri: mongodb://localhost/boom
+  supernick:
+    type: elasticsearch
+    uri: http://10.0.0.1,10.0.0.2:9200/indexname
+  debug:
+    type: file
+    uri: stdout://
+  foofile:
+    type: file
+    uri: file:///tmp/foo
+  stdout:
+    type: file
+    uri: stdout://
+```
+
+There is also a sample 'application.js' in test/application.js.  The application is responsible for building transporter pipelines.
+Given the above config, this Transporter application.js will copy from a file (in /tmp/foo) to stdout.
+```js
+Source({name:"foofile"}).save({name:"stdout"})
+
+```
+
+This application.js will copy from the local mongo to a file on the local disk
+```js
+Source({name:"localmongo", namespace: "boom.foo"}).save({name:"tofile"})
+```
+
+Transformers can also configured in the application.js as follows
+```js
+var pipeline = Source({name:"mongodb-production", namespace: "compose.milestones2"})
+pipeline = pipeline.transform("transformers/transform1.js").transform("transformers/transform2.js")
+pipeline.save({name:"supernick", namespace: "something/posts2"});
+
+```
+Run
+---
+
+- list `transporter list --config ./test/config.yaml`
+- run `transporter run --config ./test/config.yaml ./test/application.js`
+- eval `transporter eval --config ./test/config.yaml 'Source({name:"localmongo", namespace: "boom.foo"}).save({name:"tofile"})' `
+- test `transporter test --config ./test/config.yaml test/application.js `
+
+Contributing to Transporter
+======================
+
+[![Circle CI](https://circleci.com/gh/compose/transporter/tree/master.png?style=badge)](https://circleci.com/gh/compose/transporter/tree/master)
+
+Want to help out with Transporter? Great! There are instructions to get you
+started [here](CONTRIBUTING.md).
+
+Licensing
+=========
+Transporter is licensed under the New BSD. See LICENSE for full license text.

circle.yml

@@ -0,0 +1,32 @@
+machine:
+  environment:
+    ME: "transporter"
+    GOBIN: "$CIRCLE_ARTIFACTS"
+
+dependencies:
+  override:
+    - test -d /home/ubuntu/.go_workspace/src/github.com/compose/ || mkdir -p /home/ubuntu/.go_workspace/src/github.com/compose/
+    - test -e /home/ubuntu/.go_workspace/src/github.com/compose/transporter || ln -s /home/ubuntu/transporter/ /home/ubuntu/.go_workspace/src/github.com/compose/
+    - cd /home/ubuntu/.go_workspace/src/github.com/compose/ && go get ./transporter/...
+    - test -d $HOME/boto || pip install -t $HOME/boto boto
+
+
+  cache_directories:
+  - "/home/ubuntu/boto"
+  - "/home/ubuntu/bin"
+
+test:
+  override:
+    # - $CIRCLE_ARTIFACTS/godep go test ./...
+    - go test ./... -tags=integration -v
+
+deployment:
+  default:
+    branch: [master, stage, dev, experimental]
+    commands:
+      - goxc -os="linux,darwin" -arch="amd64"
+      # - sh ./circle/build_deb.sh
+      # - tar zcvf $HOME/$ME-$CIRCLE_BRANCH.tgz $CIRCLE_ARTIFACTS/
+      # - cd $CIRCLE_ARTIFACTS && tar zcvf $HOME/$ME-$CIRCLE_BRANCH.tgz . ; cd $HOME/$ME
+      # - python ./circle/s3push.py "$HOME/$ME-$CIRCLE_BRANCH.tgz" "$ME-$CIRCLE_BRANCH.tgz"
+

cmd/transporter/command.go

@@ -0,0 +1,196 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+
+	"github.com/mitchellh/cli"
+)
+
+// a list of generators for all the subcommand types
+var subCommandFactory = map[string]cli.CommandFactory{
+	"list": func() (cli.Command, error) {
+		return &listCommand{}, nil
+	},
+	"test": func() (cli.Command, error) {
+		return &testCommand{}, nil
+	},
+	"run": func() (cli.Command, error) {
+		return &runCommand{}, nil
+	},
+	"eval": func() (cli.Command, error) {
+		return &evalCommand{}, nil
+	},
+}
+
+// listCommand loads the config, and lists the configured nodes
+type listCommand struct {
+	configFilename string
+}
+
+func (c *listCommand) Synopsis() string {
+	return "list all configured nodes"
+}
+
+func (c *listCommand) Help() string {
+	return `Usage: trasporter list --config [file]
+
+   list the nodes that have been configured in the configuration yaml`
+}
+
+func (c *listCommand) Run(args []string) int {
+	var configFilename string
+	cmdFlags := flag.NewFlagSet("list", flag.ContinueOnError)
+	cmdFlags.Usage = func() { c.Help() }
+	cmdFlags.StringVar(&configFilename, "config", "config.yaml", "config file")
+	cmdFlags.Parse(args)
+
+	config, err := LoadConfig(configFilename)
+	if err != nil {
+		fmt.Println(err)
+		return 1
+	}
+
+	for _, v := range config.Nodes {
+		fmt.Println(v)
+	}
+
+	return 0
+}
+
+// runCommand loads a js file, and compiles and runs a
+// javascript pipeline defined therein
+type runCommand struct {
+}
+
+func newRunCommand() (cli.Command, error) {
+	return &runCommand{}, nil
+}
+
+func (c *runCommand) Help() string {
+	return `Usage: transporter run [--config file] <filename>
+
+Run a transporter transporter application by sourcing a file containing the javascript application
+and compiling the transporter pipeline`
+}
+
+func (c *runCommand) Synopsis() string {
+	return "Run a transporter application loaded from a file"
+}
+
+func (c *runCommand) Run(args []string) int {
+	var configFilename string
+	cmdFlags := flag.NewFlagSet("run", flag.ContinueOnError)
+	cmdFlags.Usage = func() { c.Help() }
+	cmdFlags.StringVar(&configFilename, "config", "config.yaml", "config file")
+	cmdFlags.Parse(args)
+
+	config, err := LoadConfig(configFilename)
+	if err != nil {
+		fmt.Println(err)
+		return 1
+	}
+
+	builder, err := NewJavascriptBuilder(config, cmdFlags.Args()[0], "")
+	if err != nil {
+		fmt.Println(err)
+		return 1
+	}
+	if err = builder.Build(); err != nil {
+		fmt.Println(err)
+		return 1
+	}
+
+	if err = builder.Run(); err != nil {
+		fmt.Println(err)
+		return 1
+	}
+	return 0
+}
+
+// runCommand loads a js file, and compiles and runs a
+// javascript pipeline defined therein
+type testCommand struct {
+}
+
+func (c *testCommand) Help() string {
+	return `Usage: transporter test [--config file]  <filename>
+
+Compile a transporter application by sourcing an application file, but do not run it`
+}
+
+func (c *testCommand) Synopsis() string {
+	return "display the compiled nodes without starting a pipeline"
+}
+
+func (c *testCommand) Run(args []string) int {
+	var configFilename string
+	cmdFlags := flag.NewFlagSet("test", flag.ContinueOnError)
+	cmdFlags.Usage = func() { c.Help() }
+	cmdFlags.StringVar(&configFilename, "config", "config.yaml", "config file")
+	cmdFlags.Parse(args)
+
+	config, err := LoadConfig(configFilename)
+	if err != nil {
+		fmt.Println(err)
+		return 1
+	}
+
+	builder, err := NewJavascriptBuilder(config, cmdFlags.Args()[0], "")
+	if err != nil {
+		fmt.Println(err)
+		return 1
+	}
+	if err = builder.Build(); err != nil {
+		fmt.Println(err)
+		return 1
+	}
+	fmt.Println(builder)
+	return 0
+}
+
+// evalCommand compiles inline javascript into a transporter pipeline,
+// and runs it
+type evalCommand struct {
+}
+
+func (c *evalCommand) Help() string {
+	return `Usage: transporter eval [--config file]  <javascript>
+
+Compile a transporter application by evaluating the given javascript`
+}
+
+func (c *evalCommand) Synopsis() string {
+	return "Eval javascript to build and run a transporter application"
+}
+
+func (c *evalCommand) Run(args []string) int {
+	var configFilename string
+	cmdFlags := flag.NewFlagSet("run", flag.ContinueOnError)
+	cmdFlags.Usage = func() { c.Help() }
+	cmdFlags.StringVar(&configFilename, "config", "config.yaml", "config file")
+	cmdFlags.Parse(args)
+
+	config, err := LoadConfig(configFilename)
+	if err != nil {
+		fmt.Println(err)
+		return 1
+	}
+
+	builder, err := NewJavascriptBuilder(config, "", cmdFlags.Args()[0])
+	if err != nil {
+		fmt.Println(err)
+		return 1
+	}
+	if err = builder.Build(); err != nil {
+		fmt.Println(err)
+		return 1
+	}
+
+	if err = builder.Run(); err != nil {
+		fmt.Println(err)
+		return 1
+	}
+
+	return 0
+}

cmd/transporter/config.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"time"
+
+	"gopkg.in/yaml.v2"
+)
+
+// A Config stores meta information about the transporter.  This contains a
+// list of the the nodes that are available to a transporter (sources and sinks, not transformers)
+// as well as information about the api used to handle transporter events, and the interval
+// between metrics events.
+type Config struct {
+	API struct {
+		URI             string `json:"uri" yaml:"uri"`           // Uri to connect to
+		MetricsInterval string `json:"interval" yaml:"interval"` // how often to emit metrics, (in ms)
+		Key             string `json:"key" yaml:"key"`           // http basic auth password to send with each event
+		Pid             string `json:"pid" yaml:"pid"`           // http basic auth username to send with each event
+	} `json:"api" yaml:"api"`
+	Nodes map[string]struct {
+		Type string `json:"type" yaml:"type"`
+		URI  string `json:"uri" yaml:"uri"`
+	}
+}
+
+// LoadConfig loads a config yaml from a file on disk.
+// if the pid is not set in the yaml, pull it from the environment TRANSPORTER_PID.
+// if that env var isn't present, then generate a pid
+func LoadConfig(filename string) (config Config, err error) {
+	if filename == "" {
+		return
+	}
+
+	ba, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return
+	}
+
+	err = yaml.Unmarshal(ba, &config)
+
+	for k, v := range config.Nodes {
+		config.Nodes[k] = v
+	}
+
+	if len(config.API.Pid) < 1 {
+		config.API.Pid = os.Getenv("TRANSPORTER_PID")
+	}
+
+	if len(config.API.Pid) < 1 {
+		hostname, _ := os.Hostname()
+		config.API.Pid = fmt.Sprintf("%s@%d", hostname, time.Now().Unix())
+	}
+
+	return
+}