This repository has been archived by the owner on Feb 18, 2021. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 102
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Aravind Srinivasan
committed
Dec 28, 2016
0 parents
commit b49a72c
Showing
296 changed files
with
84,664 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
.DS_Store | ||
*.out | ||
*.test | ||
*.xml | ||
*.swp | ||
.idea/ | ||
*.iml | ||
*.cov | ||
*.html | ||
*.pyc | ||
.pyenv | ||
test.log | ||
*~ | ||
.tmp | ||
.generated | ||
vendor/ | ||
rocksdb/ | ||
|
||
# Executables produced by Cherami repo | ||
cmd/standalone/standalone | ||
cmd/replicator/replicator | ||
cmd/tools/admin/cherami-admin | ||
cmd/tools/cli/cherami-cli | ||
cmd/tools/replicator/cherami-replicator-tool | ||
cmd/tools/store/cherami-store-tool | ||
cmd/tools/cassandra/cherami-cassandra-tool |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
Copyright (c) 2016 Uber Technologies, Inc. | ||
|
||
Permission is hereby granted, free of charge, to any person obtaining a copy | ||
of this software and associated documentation files (the "Software"), to deal | ||
in the Software without restriction, including without limitation the rights | ||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
copies of the Software, and to permit persons to whom the Software is | ||
furnished to do so, subject to the following conditions: | ||
|
||
The above copyright notice and this permission notice shall be included in | ||
all copies or substantial portions of the Software. | ||
|
||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN | ||
THE SOFTWARE. | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,104 @@ | ||
.PHONY: bins clean setup test test-race thriftc | ||
|
||
SHELL = /bin/bash | ||
|
||
TEST_ENV=CHERAMI_ENVIRONMENT=test CHERAMI_CONFIG_DIR=`pwd`/config | ||
|
||
PROJECT_ROOT=github.com/uber/cherami-server | ||
export GO15VENDOREXPERIMENT=1 | ||
NOVENDOR = $(shell GO15VENDOREXPERIMENT=1 glide novendor) | ||
|
||
export PATH := $(GOPATH)/bin:$(PATH) | ||
|
||
THRIFT_GENDIR=.generated | ||
|
||
THRIFT_DIR = idl/cherami | ||
THRIFT_INTERNAL_DIR = idl/cherami_internal | ||
|
||
THRIFT_SRCS = $(THRIFT_INTERNAL_DIR)/metadata.thrift \ | ||
$(THRIFT_INTERNAL_DIR)/controller.thrift \ | ||
$(THRIFT_INTERNAL_DIR)/admin.thrift \ | ||
$(THRIFT_INTERNAL_DIR)/store.thrift \ | ||
$(THRIFT_INTERNAL_DIR)/replicator.thrift \ | ||
$(THRIFT_DIR)/cherami.thrift | ||
|
||
export CGO_CFLAGS=$(env CGO_FLAGS) -I$(CURDIR)/rocksdb/include | ||
export CGO_LDFLAGS=$(env CGO_LDFLAGS) -L$(CURDIR)/rocksdb/lib -lrocksdb | ||
|
||
export CHERAMI_STORE=$(shell dirname `mktemp -u store.test.XXX`)/cherami_store | ||
|
||
ifeq ($(shell uname -s),Linux) | ||
export CGO_LDFLAGS=$(env CGO_LDFLAGS) -L$(CURDIR)/rocksdb/lib -lrocksdb -lsnappy -lgflags -lbz2 -lstdc++ | ||
export LD_LIBRARY_PATH=$(CURDIR)/rocksdb/lib:$(env LD_LIBRARY_PATH) | ||
endif | ||
|
||
# Automatically gather all srcs | ||
ALL_SRC := $(shell find . -name "*.go" | grep -v -e Godeps -e vendor \ | ||
-e ".*/\..*" \ | ||
-e ".*/_.*" \ | ||
-e ".*/mocks.*") | ||
|
||
# all directories with *_test.go files in them | ||
TEST_DIRS := $(sort $(dir $(filter %_test.go,$(ALL_SRC)))) | ||
|
||
test: bins | ||
@for dir in $(TEST_DIRS); do \ | ||
go test "$$dir" | tee -a test.log; \ | ||
done; | ||
|
||
test-race: | ||
@for dir in $(TEST_DIRS); do \ | ||
go test -race "$$dir" | tee -a "$$dir"_test.log; \ | ||
done; | ||
|
||
THRIFT_GEN=$(GOPATH)/bin/thrift-gen | ||
|
||
define thriftrule | ||
THRIFT_GEN_SRC += $(THRIFT_GENDIR)/go/$1/tchan-$1.go | ||
|
||
$(THRIFT_GENDIR)/go/$1/tchan-$1.go:: $2 $(THRIFT_GEN) | ||
@mkdir -p $(THRIFT_GENDIR)/go | ||
$(ECHO_V)$(THRIFT_GEN) --generateThrift --packagePrefix $(PROJECT_ROOT)/$(THRIFT_GENDIR)/go/ --inputFile $2 --outputDir $(THRIFT_GENDIR)/go \ | ||
$(foreach template,$(THRIFT_TEMPLATES), --template $(template)) | ||
endef | ||
|
||
$(foreach tsrc,$(THRIFT_SRCS),$(eval $(call \ | ||
thriftrule,$(basename $(notdir \ | ||
$(shell echo $(tsrc) | tr A-Z a-z))),$(tsrc)))) | ||
|
||
|
||
checkrocksdb: | ||
@if [ ! -d $(CURDIR)/rocksdb ]; then \ | ||
echo "please install rocksdb and copy the static libraries and include files within $(CURDIR)/rocksdb directory" >&2; \ | ||
echo "one can follow instructions here to build rocksdb: https://github.com/facebook/rocksdb/blob/master/INSTALL.md" >&2; \ | ||
echo "once the repo is cloned and libraries are built, please copy the liborcksdb* to $(CURDIR)/rocksdb/lib and rocksdb/include to $(CURDIR)/rocksdb/include" >&2; \ | ||
exit 1; \ | ||
fi | ||
|
||
checkcassandra: | ||
@if ! which cqlsh | grep -q /; then \ | ||
echo "cqlsh not in PATH. please install cassandra and cqlsh" >&2; \ | ||
exit 1; \ | ||
fi | ||
|
||
thriftc: $(THRIFT_GEN_SRC) | ||
|
||
server_dep: | ||
glide install | ||
|
||
bins: server_dep checkrocksdb checkcassandra thriftc | ||
go build -i -o cmd/standalone/standalone cmd/standalone/main.go | ||
go build -i -o cmd/replicator/replicator cmd/replicator/main.go | ||
go build -i -o cmd/tools/cli/cherami-cli cmd/tools/cli/main.go | ||
go build -i -o cmd/tools/admin/cherami-admin cmd/tools/admin/main.go | ||
go build -i -o cmd/tools/replicator/cherami-replicator-tool cmd/tools/replicator/main.go | ||
go build -i -o cmd/tools/cassandra/cherami-cassandra-tool cmd/tools/cassandra/main.go | ||
|
||
clean: | ||
rm -f cmd/standalone/standalone | ||
rm -f cmd/replicator/replicator | ||
rm -f cmd/tools/cli/cherami-cli | ||
rm -f cmd/tools/admin/cherami-admin | ||
rm -f cmd/tools/replicator/cherami-replicator-tool | ||
rm -f cmd/tools/cassandra/cherami-cassandra-tool | ||
rm -rf .generated |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
cherami-server | ||
============== | ||
|
||
Cherami is a distributed, scalable, durable, and highly available message queue system we developed at Uber Engineering to transport asynchronous tasks. | ||
|
||
cherami-server is the actual implementation of the cherami architecture. | ||
|
||
Getting started | ||
--------------- | ||
|
||
To get cherami-server: | ||
|
||
``` | ||
git clone git@github.com:uber/cherami-server.git | ||
``` | ||
|
||
Developing | ||
---------- | ||
|
||
The cherami-server repository holds the implementation of cherami architecture. Before getting started, cherami-server has a bunch of dependencies as follows: | ||
|
||
* Make certain that `thrift` (OSX: `brew install thrift`) and `glide` are | ||
in your path (above). | ||
* `thrift-gen` is needed to autogenerate thrift APIs (`go get github.com/uber/tchannel-go/thrift/thrift-gen`) | ||
* `cassandra` which is our metadata store is required to be up and running | ||
* `rocksdb` is our data store and needs to be installed with the libraries being placed along with the cherami code in a folder called `rocksdb`: | ||
* In OSX, we need rocksdb3.13 (which can be installed using brew) | ||
* In Linux, download https://github.com/facebook/rocksdb and compile to get all the rocksdb libraries manually. | ||
|
||
After installing all the requisite dependencies, one can do the following to start cherami: | ||
|
||
* Setup the cherami keyspace for metadata (cassandra): | ||
``` | ||
./scripts/cherami-setup-schema | ||
``` | ||
|
||
* Build the cherami `standalone` and other binaries: | ||
``` | ||
make bins | ||
``` | ||
|
||
Once built, the service can be started as follows: | ||
``` | ||
CHERAMI_ENVIRONMENT=laptop CHERAMI_CONFIG_DIR=`pwd`/config CHERAMI_STORE=/tmp/store ./cmd/standalone/standalone start all | ||
``` | ||
|
||
One can use the CLI to verify if cherami is running properly: | ||
``` | ||
./cmd/tools/cli/cherami-cli --hostport=<localIP>:4922 create destination /test/cherami | ||
``` | ||
|
||
Documentation | ||
-------------- | ||
|
||
Interested in learning more about Cherami? Read the blog post: | ||
[eng.uber.com.cherami](https://eng.uber.com/cherami/) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
// Copyright (c) 2016 Uber Technologies, Inc. | ||
// | ||
// Permission is hereby granted, free of charge, to any person obtaining a copy | ||
// of this software and associated documentation files (the "Software"), to deal | ||
// in the Software without restriction, including without limitation the rights | ||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
// copies of the Software, and to permit persons to whom the Software is | ||
// furnished to do so, subject to the following conditions: | ||
// | ||
// The above copyright notice and this permission notice shall be included in | ||
// all copies or substantial portions of the Software. | ||
// | ||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN | ||
// THE SOFTWARE. | ||
|
||
package controller | ||
|
||
import ( | ||
"github.com/uber/cherami-server/.generated/go/controller" | ||
"github.com/uber/cherami-server/common" | ||
"github.com/uber/tchannel-go" | ||
"github.com/uber/tchannel-go/thrift" | ||
) | ||
|
||
// NewClient returns a new instance of cherami controller client | ||
func NewClient(ch *tchannel.Channel, hostAddr string) controller.TChanControllerClient { | ||
tClient := thrift.NewClient(ch, common.ControllerServiceName, &thrift.ClientOptions{ | ||
HostPort: hostAddr, | ||
}) | ||
return controller.NewTChanControllerClient(tClient) | ||
} |
Oops, something went wrong.