Initial commit

.gitignore

@@ -0,0 +1,26 @@
+.DS_Store
+*.out
+*.test
+*.xml
+*.swp
+.idea/
+*.iml
+*.cov
+*.html
+*.pyc
+.pyenv
+test.log
+*~
+.tmp
+.generated
+vendor/
+rocksdb/
+
+# Executables produced by Cherami repo
+cmd/standalone/standalone
+cmd/replicator/replicator
+cmd/tools/admin/cherami-admin
+cmd/tools/cli/cherami-cli
+cmd/tools/replicator/cherami-replicator-tool
+cmd/tools/store/cherami-store-tool
+cmd/tools/cassandra/cherami-cassandra-tool

LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2016 Uber Technologies, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+

Makefile

@@ -0,0 +1,104 @@
+.PHONY: bins clean setup test test-race thriftc
+
+SHELL = /bin/bash
+
+TEST_ENV=CHERAMI_ENVIRONMENT=test CHERAMI_CONFIG_DIR=`pwd`/config
+
+PROJECT_ROOT=github.com/uber/cherami-server
+export GO15VENDOREXPERIMENT=1
+NOVENDOR = $(shell GO15VENDOREXPERIMENT=1 glide novendor)
+
+export PATH := $(GOPATH)/bin:$(PATH)
+
+THRIFT_GENDIR=.generated
+
+THRIFT_DIR = idl/cherami
+THRIFT_INTERNAL_DIR = idl/cherami_internal
+
+THRIFT_SRCS = $(THRIFT_INTERNAL_DIR)/metadata.thrift \
+	      $(THRIFT_INTERNAL_DIR)/controller.thrift \
+	      $(THRIFT_INTERNAL_DIR)/admin.thrift \
+	      $(THRIFT_INTERNAL_DIR)/store.thrift \
+	      $(THRIFT_INTERNAL_DIR)/replicator.thrift \
+	      $(THRIFT_DIR)/cherami.thrift
+
+export CGO_CFLAGS=$(env CGO_FLAGS) -I$(CURDIR)/rocksdb/include
+export CGO_LDFLAGS=$(env CGO_LDFLAGS) -L$(CURDIR)/rocksdb/lib -lrocksdb
+
+export CHERAMI_STORE=$(shell dirname `mktemp -u store.test.XXX`)/cherami_store
+
+ifeq ($(shell uname -s),Linux)
+        export CGO_LDFLAGS=$(env CGO_LDFLAGS) -L$(CURDIR)/rocksdb/lib -lrocksdb -lsnappy -lgflags -lbz2 -lstdc++
+        export LD_LIBRARY_PATH=$(CURDIR)/rocksdb/lib:$(env LD_LIBRARY_PATH)
+endif
+
+# Automatically gather all srcs
+ALL_SRC := $(shell find . -name "*.go" | grep -v -e Godeps -e vendor \
+	-e ".*/\..*" \
+	-e ".*/_.*" \
+	-e ".*/mocks.*")
+
+# all directories with *_test.go files in them
+TEST_DIRS := $(sort $(dir $(filter %_test.go,$(ALL_SRC))))
+
+test: bins
+	@for dir in $(TEST_DIRS); do \
+		go test "$$dir" | tee -a test.log; \
+	done;
+
+test-race:
+	@for dir in $(TEST_DIRS); do \
+		go test -race "$$dir" | tee -a "$$dir"_test.log; \
+	done;	       
+
+THRIFT_GEN=$(GOPATH)/bin/thrift-gen
+
+define thriftrule
+THRIFT_GEN_SRC += $(THRIFT_GENDIR)/go/$1/tchan-$1.go
+
+$(THRIFT_GENDIR)/go/$1/tchan-$1.go:: $2 $(THRIFT_GEN)
+	@mkdir -p $(THRIFT_GENDIR)/go
+	$(ECHO_V)$(THRIFT_GEN) --generateThrift --packagePrefix $(PROJECT_ROOT)/$(THRIFT_GENDIR)/go/ --inputFile $2 --outputDir $(THRIFT_GENDIR)/go \
+		$(foreach template,$(THRIFT_TEMPLATES), --template $(template))
+endef
+
+$(foreach tsrc,$(THRIFT_SRCS),$(eval $(call \
+	thriftrule,$(basename $(notdir \
+	$(shell echo $(tsrc) | tr A-Z a-z))),$(tsrc))))
+
+
+checkrocksdb:
+	@if [ ! -d $(CURDIR)/rocksdb ]; then \
+		echo "please install rocksdb and copy the static libraries and include files within $(CURDIR)/rocksdb directory" >&2; \
+		echo "one can follow instructions here to build rocksdb: https://github.com/facebook/rocksdb/blob/master/INSTALL.md" >&2; \
+		echo "once the repo is cloned and libraries are built, please copy the liborcksdb* to $(CURDIR)/rocksdb/lib and rocksdb/include to $(CURDIR)/rocksdb/include" >&2; \
+		exit 1; \
+	fi
+
+checkcassandra:
+	@if ! which cqlsh | grep -q /; then \
+		echo "cqlsh not in PATH. please install cassandra and cqlsh" >&2; \
+		exit 1; \
+	fi
+
+thriftc: $(THRIFT_GEN_SRC)
+
+server_dep:
+	glide install
+
+bins: server_dep checkrocksdb checkcassandra thriftc
+	go build -i -o cmd/standalone/standalone cmd/standalone/main.go
+	go build -i -o cmd/replicator/replicator cmd/replicator/main.go
+	go build -i -o cmd/tools/cli/cherami-cli cmd/tools/cli/main.go
+	go build -i -o cmd/tools/admin/cherami-admin cmd/tools/admin/main.go
+	go build -i -o cmd/tools/replicator/cherami-replicator-tool cmd/tools/replicator/main.go
+	go build -i -o cmd/tools/cassandra/cherami-cassandra-tool cmd/tools/cassandra/main.go
+
+clean:
+	rm -f cmd/standalone/standalone
+	rm -f cmd/replicator/replicator
+	rm -f cmd/tools/cli/cherami-cli
+	rm -f cmd/tools/admin/cherami-admin
+	rm -f cmd/tools/replicator/cherami-replicator-tool
+	rm -f cmd/tools/cassandra/cherami-cassandra-tool
+	rm -rf .generated

README.md

@@ -0,0 +1,56 @@
+cherami-server
+==============
+
+Cherami is a distributed, scalable, durable, and highly available message queue system we developed at Uber Engineering to transport asynchronous tasks. 
+
+cherami-server is the actual implementation of the cherami architecture.
+
+Getting started
+---------------
+
+To get cherami-server:
+
+```
+git clone git@github.com:uber/cherami-server.git
+```
+
+Developing
+----------
+
+The cherami-server repository holds the implementation of cherami architecture. Before getting started, cherami-server has a bunch of dependencies as follows:
+
+* Make certain that `thrift` (OSX: `brew install thrift`) and `glide` are
+in your path (above).
+* `thrift-gen` is needed to autogenerate thrift APIs (`go get github.com/uber/tchannel-go/thrift/thrift-gen`)
+* `cassandra` which is our metadata store is required to be up and running
+* `rocksdb` is our data store and needs to be installed with the libraries being placed along with the cherami code in a folder called `rocksdb`:
+	* In OSX, we need rocksdb3.13 (which can be installed using brew)
+	* In Linux, download https://github.com/facebook/rocksdb and compile to get all the rocksdb libraries manually.
+
+After installing all the requisite dependencies, one can do the following to start cherami:
+
+* Setup the cherami keyspace for metadata (cassandra):
+```
+./scripts/cherami-setup-schema
+```
+
+* Build the cherami `standalone` and other binaries:
+```
+make bins
+```
+
+Once built, the service can be started as follows:
+```
+CHERAMI_ENVIRONMENT=laptop CHERAMI_CONFIG_DIR=`pwd`/config CHERAMI_STORE=/tmp/store ./cmd/standalone/standalone start all
+```
+
+One can use the CLI to verify if cherami is running properly:
+```
+./cmd/tools/cli/cherami-cli --hostport=<localIP>:4922 create destination /test/cherami
+```
+
+Documentation
+--------------
+
+Interested in learning more about Cherami? Read the blog post:
+[eng.uber.com.cherami](https://eng.uber.com/cherami/)

clients/controller/client.go

@@ -0,0 +1,36 @@
+// Copyright (c) 2016 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package controller
+
+import (
+	"github.com/uber/cherami-server/.generated/go/controller"
+	"github.com/uber/cherami-server/common"
+	"github.com/uber/tchannel-go"
+	"github.com/uber/tchannel-go/thrift"
+)
+
+// NewClient returns a new instance of cherami controller client
+func NewClient(ch *tchannel.Channel, hostAddr string) controller.TChanControllerClient {
+	tClient := thrift.NewClient(ch, common.ControllerServiceName, &thrift.ClientOptions{
+		HostPort: hostAddr,
+	})
+	return controller.NewTChanControllerClient(tClient)
+}